From 130582f6d75640ef5d8e2135c24d32f3e91e6a2a Mon Sep 17 00:00:00 2001 From: Farnam Date: Sun, 18 Oct 2026 16:10:17 +0330 Subject: [PATCH 01/10] Add structural code engine and gated native tools --- Cargo.lock | 163 ++++++ README.md | 6 + crates/tinytools-code/Cargo.toml | 32 ++ crates/tinytools-code/README.md | 52 ++ crates/tinytools-code/benches/structural.rs | 40 ++ crates/tinytools-code/src/analysis/README.md | 3 + crates/tinytools-code/src/analysis/extract.rs | 196 +++++++ crates/tinytools-code/src/analysis/fold.rs | 72 +++ crates/tinytools-code/src/analysis/mod.rs | 65 +++ crates/tinytools-code/src/analysis/types.rs | 106 ++++ crates/tinytools-code/src/catalog/README.md | 3 + crates/tinytools-code/src/catalog/builtins.rs | 29 ++ crates/tinytools-code/src/catalog/mod.rs | 169 +++++++ .../src/catalog/starter-manifests.json | 378 ++++++++++++++ .../catalog/templates/go.http-handler.v1.txt | 7 + .../templates/json.service-config.v1.txt | 5 + .../templates/python.resource-context.v1.txt | 9 + .../catalog/templates/rust.checked-add.v1.txt | 1 + .../templates/rust.fallible-function.v1.txt | 4 + .../templates/rust.public-field.v1.txt | 1 + .../templates/rust.tauri-command.v1.txt | 5 + .../templates/toml.service-config.v1.txt | 4 + .../templates/tsx.accessible-action.v1.txt | 4 + .../typescript.nullable-guard.v1.txt | 4 + .../typescript.typed-function.v1.txt | 3 + .../templates/typescript.vitest-suite.v1.txt | 6 + crates/tinytools-code/src/catalog/types.rs | 119 +++++ .../tinytools-code/src/catalog/validation.rs | 120 +++++ crates/tinytools-code/src/engine/README.md | 3 + crates/tinytools-code/src/engine/edit.rs | 41 ++ crates/tinytools-code/src/engine/mod.rs | 178 +++++++ crates/tinytools-code/src/engine/types.rs | 80 +++ crates/tinytools-code/src/error/mod.rs | 89 ++++ crates/tinytools-code/src/language/mod.rs | 71 +++ .../src/language/queries/go.scm | 4 + .../src/language/queries/javascript.scm | 4 + .../src/language/queries/json.scm | 1 + .../src/language/queries/python.scm | 3 + .../src/language/queries/rust.scm | 5 + .../src/language/queries/toml.scm | 1 + .../src/language/queries/typescript.scm | 6 + crates/tinytools-code/src/lib.rs | 33 ++ crates/tinytools-code/src/patch/README.md | 3 + crates/tinytools-code/src/patch/imports.rs | 126 +++++ .../tinytools-code/src/patch/indentation.rs | 176 +++++++ crates/tinytools-code/src/patch/mod.rs | 118 +++++ crates/tinytools-code/src/patch/types.rs | 46 ++ crates/tinytools-code/src/render/README.md | 3 + crates/tinytools-code/src/render/context.rs | 113 +++++ crates/tinytools-code/src/render/mod.rs | 95 ++++ .../tinytools-code/src/render/parameters.rs | 188 +++++++ crates/tinytools-code/src/render/slots.rs | 132 +++++ crates/tinytools-code/src/render/types.rs | 35 ++ crates/tinytools-code/tests/analysis_tests.rs | 346 +++++++++++++ crates/tinytools-code/tests/catalog_tests.rs | 153 ++++++ crates/tinytools-code/tests/error_tests.rs | 67 +++ .../tinytools-code/tests/fixtures/analysis.rs | 21 + crates/tinytools-code/tests/patch_tests.rs | 457 +++++++++++++++++ crates/tinytools-code/tests/render_tests.rs | 477 ++++++++++++++++++ crates/tinytools-std/Cargo.toml | 10 + crates/tinytools-std/README.md | 1 + crates/tinytools-std/src/code/README.md | 154 ++++++ crates/tinytools-std/src/code/arguments.rs | 149 ++++++ crates/tinytools-std/src/code/chunk_tests.rs | 255 ++++++++++ crates/tinytools-std/src/code/chunks.rs | 95 ++++ .../tinytools-std/src/code/contract_tests.rs | 175 +++++++ crates/tinytools-std/src/code/declaration.rs | 221 ++++++++ crates/tinytools-std/src/code/inspect.rs | 127 +++++ .../src/code/inspection_tests.rs | 171 +++++++ crates/tinytools-std/src/code/io.rs | 126 +++++ crates/tinytools-std/src/code/mod.rs | 85 ++++ crates/tinytools-std/src/code/mutate.rs | 248 +++++++++ .../tinytools-std/src/code/mutation_tests.rs | 287 +++++++++++ .../src/code/publication_tests.rs | 225 +++++++++ crates/tinytools-std/src/code/publish.rs | 133 +++++ crates/tinytools-std/src/code/response.rs | 57 +++ crates/tinytools-std/src/code/test_support.rs | 204 ++++++++ crates/tinytools-std/src/code/types.rs | 110 ++++ crates/tinytools-std/src/lib.rs | 7 + docs/plans/README.md | 2 + docs/plans/structural-code.md | 44 ++ docs/specs/README.md | 2 + docs/specs/structural-code.md | 53 ++ 83 files changed, 7622 insertions(+) create mode 100644 crates/tinytools-code/Cargo.toml create mode 100644 crates/tinytools-code/README.md create mode 100644 crates/tinytools-code/benches/structural.rs create mode 100644 crates/tinytools-code/src/analysis/README.md create mode 100644 crates/tinytools-code/src/analysis/extract.rs create mode 100644 crates/tinytools-code/src/analysis/fold.rs create mode 100644 crates/tinytools-code/src/analysis/mod.rs create mode 100644 crates/tinytools-code/src/analysis/types.rs create mode 100644 crates/tinytools-code/src/catalog/README.md create mode 100644 crates/tinytools-code/src/catalog/builtins.rs create mode 100644 crates/tinytools-code/src/catalog/mod.rs create mode 100644 crates/tinytools-code/src/catalog/starter-manifests.json create mode 100644 crates/tinytools-code/src/catalog/templates/go.http-handler.v1.txt create mode 100644 crates/tinytools-code/src/catalog/templates/json.service-config.v1.txt create mode 100644 crates/tinytools-code/src/catalog/templates/python.resource-context.v1.txt create mode 100644 crates/tinytools-code/src/catalog/templates/rust.checked-add.v1.txt create mode 100644 crates/tinytools-code/src/catalog/templates/rust.fallible-function.v1.txt create mode 100644 crates/tinytools-code/src/catalog/templates/rust.public-field.v1.txt create mode 100644 crates/tinytools-code/src/catalog/templates/rust.tauri-command.v1.txt create mode 100644 crates/tinytools-code/src/catalog/templates/toml.service-config.v1.txt create mode 100644 crates/tinytools-code/src/catalog/templates/tsx.accessible-action.v1.txt create mode 100644 crates/tinytools-code/src/catalog/templates/typescript.nullable-guard.v1.txt create mode 100644 crates/tinytools-code/src/catalog/templates/typescript.typed-function.v1.txt create mode 100644 crates/tinytools-code/src/catalog/templates/typescript.vitest-suite.v1.txt create mode 100644 crates/tinytools-code/src/catalog/types.rs create mode 100644 crates/tinytools-code/src/catalog/validation.rs create mode 100644 crates/tinytools-code/src/engine/README.md create mode 100644 crates/tinytools-code/src/engine/edit.rs create mode 100644 crates/tinytools-code/src/engine/mod.rs create mode 100644 crates/tinytools-code/src/engine/types.rs create mode 100644 crates/tinytools-code/src/error/mod.rs create mode 100644 crates/tinytools-code/src/language/mod.rs create mode 100644 crates/tinytools-code/src/language/queries/go.scm create mode 100644 crates/tinytools-code/src/language/queries/javascript.scm create mode 100644 crates/tinytools-code/src/language/queries/json.scm create mode 100644 crates/tinytools-code/src/language/queries/python.scm create mode 100644 crates/tinytools-code/src/language/queries/rust.scm create mode 100644 crates/tinytools-code/src/language/queries/toml.scm create mode 100644 crates/tinytools-code/src/language/queries/typescript.scm create mode 100644 crates/tinytools-code/src/lib.rs create mode 100644 crates/tinytools-code/src/patch/README.md create mode 100644 crates/tinytools-code/src/patch/imports.rs create mode 100644 crates/tinytools-code/src/patch/indentation.rs create mode 100644 crates/tinytools-code/src/patch/mod.rs create mode 100644 crates/tinytools-code/src/patch/types.rs create mode 100644 crates/tinytools-code/src/render/README.md create mode 100644 crates/tinytools-code/src/render/context.rs create mode 100644 crates/tinytools-code/src/render/mod.rs create mode 100644 crates/tinytools-code/src/render/parameters.rs create mode 100644 crates/tinytools-code/src/render/slots.rs create mode 100644 crates/tinytools-code/src/render/types.rs create mode 100644 crates/tinytools-code/tests/analysis_tests.rs create mode 100644 crates/tinytools-code/tests/catalog_tests.rs create mode 100644 crates/tinytools-code/tests/error_tests.rs create mode 100644 crates/tinytools-code/tests/fixtures/analysis.rs create mode 100644 crates/tinytools-code/tests/patch_tests.rs create mode 100644 crates/tinytools-code/tests/render_tests.rs create mode 100644 crates/tinytools-std/src/code/README.md create mode 100644 crates/tinytools-std/src/code/arguments.rs create mode 100644 crates/tinytools-std/src/code/chunk_tests.rs create mode 100644 crates/tinytools-std/src/code/chunks.rs create mode 100644 crates/tinytools-std/src/code/contract_tests.rs create mode 100644 crates/tinytools-std/src/code/declaration.rs create mode 100644 crates/tinytools-std/src/code/inspect.rs create mode 100644 crates/tinytools-std/src/code/inspection_tests.rs create mode 100644 crates/tinytools-std/src/code/io.rs create mode 100644 crates/tinytools-std/src/code/mod.rs create mode 100644 crates/tinytools-std/src/code/mutate.rs create mode 100644 crates/tinytools-std/src/code/mutation_tests.rs create mode 100644 crates/tinytools-std/src/code/publication_tests.rs create mode 100644 crates/tinytools-std/src/code/publish.rs create mode 100644 crates/tinytools-std/src/code/response.rs create mode 100644 crates/tinytools-std/src/code/test_support.rs create mode 100644 crates/tinytools-std/src/code/types.rs create mode 100644 docs/plans/structural-code.md create mode 100644 docs/specs/structural-code.md diff --git a/Cargo.lock b/Cargo.lock index aff432f3..c371b483 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,12 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + [[package]] name = "async-trait" version = "0.1.92" @@ -52,6 +58,19 @@ version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" +[[package]] +name = "blake3" +version = "1.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae" +dependencies = [ + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures", +] + [[package]] name = "block-buffer" version = "0.12.1" @@ -112,6 +131,12 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + [[package]] name = "cpufeatures" version = "0.3.1" @@ -152,6 +177,12 @@ dependencies = [ "syn", ] +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + [[package]] name = "errno" version = "0.3.14" @@ -286,6 +317,12 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + [[package]] name = "http" version = "1.5.0" @@ -498,6 +535,16 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", +] + [[package]] name = "ipnet" version = "2.12.2" @@ -960,6 +1007,7 @@ version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ + "indexmap", "itoa", "memchr", "serde", @@ -1034,6 +1082,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520" + [[package]] name = "subtle" version = "2.6.1" @@ -1136,6 +1190,23 @@ dependencies = [ "tracing", ] +[[package]] +name = "tinytools-code" +version = "0.5.0" +dependencies = [ + "blake3", + "serde", + "serde_json", + "tree-sitter", + "tree-sitter-go", + "tree-sitter-javascript", + "tree-sitter-json", + "tree-sitter-python", + "tree-sitter-rust", + "tree-sitter-toml-ng", + "tree-sitter-typescript", +] + [[package]] name = "tinytools-jev" version = "0.5.0" @@ -1162,10 +1233,12 @@ dependencies = [ "regex", "reqwest", "rustix", + "serde", "serde_json", "sha2", "tempfile", "tinytools", + "tinytools-code", "tokio", "tracing", "walkdir", @@ -1291,6 +1364,96 @@ dependencies = [ "once_cell", ] +[[package]] +name = "tree-sitter" +version = "0.25.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78f873475d258561b06f1c595d93308a7ed124d9977cb26b148c2084a4a3cc87" +dependencies = [ + "cc", + "regex", + "regex-syntax", + "serde_json", + "streaming-iterator", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-go" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8560a4d2f835cc0d4d2c2e03cbd0dde2f6114b43bc491164238d333e28b16ea" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-javascript" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68204f2abc0627a90bdf06e605f5c470aa26fdcb2081ea553a04bdad756693f5" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-json" +version = "0.24.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d727acca406c0020cffc6cf35516764f36c8e3dc4408e5ebe2cb35a947ec471" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-language" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "009994f150cc0cd50ff54917d5bc8bffe8cad10ca10d81c34da2ec421ae61782" + +[[package]] +name = "tree-sitter-python" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bf85fd39652e740bf60f46f4cda9492c3a9ad75880575bf14960f775cb74a1c" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-rust" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439e577dbe07423ec2582ac62c7531120dbfccfa6e5f92406f93dd271a120e45" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-toml-ng" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9adc2c898ae49730e857d75be403da3f92bb81d8e37a2f918a08dd10de5ebb1" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-typescript" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5f76ed8d947a75cc446d5fccd8b602ebf0cde64ccf2ffa434d873d7a575eff" +dependencies = [ + "cc", + "tree-sitter-language", +] + [[package]] name = "try-lock" version = "0.2.5" diff --git a/README.md b/README.md index 4bc3603c..c764237c 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,12 @@ second sibling: a `ToolRanker` backed by TypeSafe's Jev decision model through `tinyjevclient`, kept out of the vocabulary crate because it carries an HTTP transport. +[`tinytools-code`](crates/tinytools-code/README.md) is a pure sibling for bounded +syntax analysis, revision-checked edits and typed boilerplate rendering. The +optional `code` feature in [`tinytools-std`](crates/tinytools-std/README.md) exposes +the six filesystem-backed tools through a host's `FsGate`. Neither package adds +parser or runtime dependencies to the base vocabulary crate. + ## What is deliberately not here **No enforcement.** Nothing in this crate checks a `PermissionLevel`, applies a diff --git a/crates/tinytools-code/Cargo.toml b/crates/tinytools-code/Cargo.toml new file mode 100644 index 00000000..34bc0a57 --- /dev/null +++ b/crates/tinytools-code/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "tinytools-code" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +description = "Bounded syntax analysis, revision-checked patches and typed boilerplate rendering without I/O." +readme = "README.md" +publish = false + +[dependencies] +serde = { workspace = true } +serde_json = { workspace = true } +# Content revisions detect stale source and template snapshots without I/O. +blake3 = "1" +# Compiled, explicit grammars provide bounded structural parsing and queries. +tree-sitter = "0.25" +tree-sitter-rust = "0.24" +tree-sitter-typescript = "0.23" +tree-sitter-javascript = "0.25" +tree-sitter-python = "0.25" +tree-sitter-go = "0.25" +tree-sitter-json = "0.24" +tree-sitter-toml-ng = "0.7" + +[lints] +workspace = true + +[[bench]] +name = "structural" +harness = false diff --git a/crates/tinytools-code/README.md b/crates/tinytools-code/README.md new file mode 100644 index 00000000..91804886 --- /dev/null +++ b/crates/tinytools-code/README.md @@ -0,0 +1,52 @@ +# TinyTools Code + +Pure, bounded syntax analysis and boilerplate rendering. The engine accepts +caller-provided UTF-8 and an explicit Rust, TypeScript, TSX, JavaScript, Python, +Go, JSON or TOML grammar. It performs no filesystem or network access. + +`CodeEngine::analyze` returns lexical declarations/references, syntax diagnostics, +content revisions and a folded view. Cache outcomes disclose fresh, unchanged or +incremental parsing. Cache byte limits measure retained source, excluding native +tree allocations. A language hint can be obtained with `Language::from_path`. + +`CodeEngine::patch` requires an exact expected revision. Operations replace bodies +or whole declarations, insert imports, append members and wrap one selected node. +Original and resulting code must parse. Exact IDs identify a revision and source +range. Name selectors must be provably unique; truncated outlines only admit IDs. +No operation promises typechecking, semantic name resolution or behavior changes +equivalent to a compiler-assisted refactor. + +`ChunkCatalog::builtin` contains twelve patterns. Search and inspect are metadata +operations. Templates are separate and `builtin_source` only serves genuine +built-in entries. Custom template paths are portable relative paths, with actual +filesystem containment enforced by the native adapter. + +`render_chunk` expands typed `{{parameter}}` values and original standalone +`// @slot name`, `# @slot name` or `/* @slot name */` syntax comments once. It +refuses unknown, missing, duplicate or unused declarations. Supplied text is not +rescanned for directives. Parameter kinds are conservative ASCII identifiers, +uppercase-leading identifiers, syntax-checked type expressions, escaped strings, +JSON numbers and booleans. Required slots need nonempty code; optional slots can +be empty. A slot can require exactly one grammar node kind. Forbidden node kinds +apply to generated code, excluding artificial validation wrappers. + +File chunks validate as complete sources. Body/member chunks use explicit syntax +contexts. Rust members can be fields or impl items, Go members can be fields or +interface items, JavaScript/TypeScript members use classes, and Python members +use class suites. JSON and TOML support file chunks only. Dependency declarations +are metadata and never install packages. + +Default source/result limit: 1 MiB. Cache: 32 entries and 8 MiB of source. +Symbols/references: 2048 each. Diagnostics: 64. Manifests: 64 KiB; 32 parameters, +32 slots and 512 catalog entries. Configured limits have hard caps. All expansion +paths check limits before output growth, including indentation. + +The optional `tinytools-std` `code` feature provides FsGate-backed native tools. +Host permission, preview, result paging and atomic publication belong there. + +## Validation + +Run the repository's four contract commands and 90% per-file coverage gate. +Focused tests use `cargo test -p tinytools-code`. Run the fixed-fixture benchmark +with `cargo bench -p tinytools-code --bench structural`. It reports byte counts +and cold/cache/incremental timings, without token-savings or speed claims. diff --git a/crates/tinytools-code/benches/structural.rs b/crates/tinytools-code/benches/structural.rs new file mode 100644 index 00000000..3963a61b --- /dev/null +++ b/crates/tinytools-code/benches/structural.rs @@ -0,0 +1,40 @@ +//! Fixed-fixture cold, unchanged and incremental parse observations. + +use std::{hint::black_box, time::Instant}; +use tinytools_code::{CodeEngine, Language}; + +fn main() -> Result<(), tinytools_code::CodeError> { + let source = include_str!("../tests/fixtures/analysis.rs"); + let iterations = 100; + let start = Instant::now(); + for _ in 0..iterations { + black_box(CodeEngine::default().analyze("fixture", Language::Rust, source)?); + } + let cold = start.elapsed(); + let mut engine = CodeEngine::default(); + let report = engine.analyze("fixture", Language::Rust, source)?; + let start = Instant::now(); + for _ in 0..iterations { + black_box(engine.analyze("fixture", Language::Rust, source)?); + } + let cached = start.elapsed(); + let changed = source.replace("value * 2", "value * 3"); + let start = Instant::now(); + for index in 0..iterations { + black_box(engine.analyze( + "fixture", + Language::Rust, + if index % 2 == 0 { &changed } else { source }, + )?); + } + println!( + "iterations={iterations} source_bytes={} folded_bytes={} symbols={} cold_us={} cached_us={} incremental_us={}", + source.len(), + report.skeleton.len(), + report.symbols.len(), + cold.as_micros(), + cached.as_micros(), + start.elapsed().as_micros() + ); + Ok(()) +} diff --git a/crates/tinytools-code/src/analysis/README.md b/crates/tinytools-code/src/analysis/README.md new file mode 100644 index 00000000..9a10e3b0 --- /dev/null +++ b/crates/tinytools-code/src/analysis/README.md @@ -0,0 +1,3 @@ +# Lexical analysis + +Bundled query captures describe declarations, imports and calls. Names are lexical, not resolved compiler symbols. Results and diagnostics disclose truncation. Exact IDs bind a content revision to byte ranges; name-based selection refuses truncated outlines. Folded views preserve surrounding text and Python docstrings. diff --git a/crates/tinytools-code/src/analysis/extract.rs b/crates/tinytools-code/src/analysis/extract.rs new file mode 100644 index 00000000..f2b9f0af --- /dev/null +++ b/crates/tinytools-code/src/analysis/extract.rs @@ -0,0 +1,196 @@ +//! Bounded capture extraction without semantic name resolution. + +use super::{Diagnostic, Reference, SourceRange, Symbol}; +use tree_sitter::{Node, Query, QueryCursor, StreamingIterator, Tree}; + +pub(crate) fn node_text<'a>(node: Node<'_>, source: &'a str) -> &'a str { + source.get(node.byte_range()).unwrap_or_default() +} + +pub(crate) fn bounded_text(value: &str, max_bytes: usize) -> (&str, bool) { + let mut end = value.len().min(max_bytes); + while !value.is_char_boundary(end) { + end -= 1; + } + (&value[..end], end < value.len()) +} + +pub(crate) fn body_node(node: Node<'_>) -> Option> { + if let Some(body) = node.child_by_field_name("body") { + return Some(body); + } + if node.kind() == "type_alias_declaration" { + return node + .child_by_field_name("value") + .filter(|value| value.kind() == "object_type"); + } + if node.kind() == "type_spec" { + let value = node.child_by_field_name("type")?; + if value.kind() == "struct_type" { + return value.named_child(0); + } + if value.kind() == "interface_type" { + return Some(value); + } + } + if node.kind() == "variable_declarator" { + return node + .child_by_field_name("value") + .and_then(|value| value.child_by_field_name("body")); + } + None +} + +fn name_node(node: Node<'_>) -> Option> { + node.child_by_field_name("name") + .or_else(|| node.child_by_field_name("key")) + .or_else(|| { + (node.kind() == "impl_item") + .then(|| node.child_by_field_name("type")) + .flatten() + }) + .or_else(|| { + matches!( + node.kind(), + "table" | "table_array_element" | "pair" | "field_declaration" + ) + .then(|| node.named_child(0)) + .flatten() + }) +} + +fn name(node: Node<'_>, source: &str) -> Option { + let value = node_text(name_node(node)?, source).trim_matches(['\'', '"']); + (!value.is_empty() && value.len() <= 1024).then(|| value.to_owned()) +} + +fn qualified_name(node: Node<'_>, source: &str, own_name: &str) -> String { + let mut parts = vec![own_name.to_owned()]; + let mut length = own_name.len(); + let mut ancestor = node.parent(); + while let Some(parent) = ancestor { + if matches!( + parent.kind(), + "impl_item" + | "trait_item" + | "mod_item" + | "class_declaration" + | "class_definition" + | "interface_declaration" + | "function_definition" + | "function_item" + | "function_declaration" + ) && let Some(value) = name(parent, source) + { + if length + value.len() + 2 > 4096 { + break; + } + length += value.len() + 2; + parts.push(value); + } + ancestor = parent.parent(); + } + parts.reverse(); + parts.join("::") +} + +fn symbol(node: Node<'_>, source: &str, revision: &str) -> Option { + let name = name(node, source)?; + let body = body_node(node); + let end = body.map_or(node.end_byte(), |body| body.start_byte()); + let (signature, signature_truncated) = bounded_text( + source + .get(node.start_byte()..end) + .unwrap_or_default() + .trim_end(), + 4096, + ); + Some(Symbol { + id: format!("{revision}:{}:{}", node.start_byte(), node.end_byte()), + qualified_name: qualified_name(node, source, &name), + name, + kind: node.kind().to_owned(), + range: node.into(), + body: body.map(SourceRange::from), + signature: signature.to_owned(), + signature_truncated, + }) +} + +pub(super) fn captures( + tree: &Tree, + query: &Query, + source: &str, + revision: &str, + limit: usize, +) -> (Vec, Vec, bool, bool) { + let mut symbols = Vec::new(); + let mut references = Vec::new(); + let mut symbols_truncated = false; + let mut references_truncated = false; + let mut cursor = QueryCursor::new(); + cursor.set_match_limit(4096); + let mut matches = cursor.matches(query, tree.root_node(), source.as_bytes()); + while let Some(found) = matches.next() { + for capture in found.captures { + let kind = query.capture_names()[capture.index as usize]; + if kind == "symbol" { + if symbols.len() < limit { + if let Some(symbol) = symbol(capture.node, source, revision) { + symbols.push(symbol); + } else { + symbols_truncated = true; + } + } else { + symbols_truncated = true; + } + } else if references.len() < limit { + let (text, text_truncated) = bounded_text(node_text(capture.node, source), 1024); + references.push(Reference { + kind: kind.to_owned(), + text: text.to_owned(), + range: capture.node.into(), + text_truncated, + }); + } else { + references_truncated = true; + } + } + } + if cursor.did_exceed_match_limit() { + symbols_truncated = true; + references_truncated = true; + } + symbols.sort_by_key(|symbol| symbol.range.start_byte); + symbols.dedup_by(|a, b| a.id == b.id); + references.sort_by_key(|reference| reference.range.start_byte); + (symbols, references, symbols_truncated, references_truncated) +} + +pub(super) fn diagnostics(tree: &Tree, limit: usize) -> (Vec, bool) { + let mut diagnostics = Vec::new(); + let mut stack = vec![tree.root_node()]; + while let Some(node) = stack.pop() { + if node.is_error() || node.is_missing() { + if diagnostics.len() == limit { + return (diagnostics, true); + } + let kind = if node.is_missing() { + "missing" + } else { + "error" + }; + diagnostics.push(Diagnostic { + kind: kind.to_owned(), + message: format!("{kind} syntax node: {}", node.kind()), + range: node.into(), + }); + } + for index in (0..node.child_count()).rev() { + if let Some(child) = node.child(index) { + stack.push(child); + } + } + } + (diagnostics, false) +} diff --git a/crates/tinytools-code/src/analysis/fold.rs b/crates/tinytools-code/src/analysis/fold.rs new file mode 100644 index 00000000..61aa474b --- /dev/null +++ b/crates/tinytools-code/src/analysis/fold.rs @@ -0,0 +1,72 @@ +//! Readable folded bodies retain surrounding declarations and Python docstrings. + +use super::{Symbol, node_text}; +use crate::Language; +use tree_sitter::{Node, Tree}; + +pub(crate) fn is_docstring(node: Node<'_>, source: &str) -> bool { + if node.kind() != "expression_statement" || node.named_child_count() != 1 { + return false; + } + let Some(value) = node.named_child(0) else { + return false; + }; + if !matches!(value.kind(), "string" | "concatenated_string") { + return false; + } + let mut stack = vec![value]; + while let Some(part) = stack.pop() { + if part.kind() == "string_start" { + let prefix = node_text(part, source).to_ascii_lowercase(); + if prefix.contains('b') || prefix.contains('f') { + return false; + } + } + for index in 0..part.named_child_count() { + if let Some(child) = part.named_child(index) { + stack.push(child); + } + } + } + true +} + +fn replacement(tree: &Tree, source: &str, language: Language, start: usize, end: usize) -> String { + if language == Language::Python { + let node = tree.root_node().descendant_for_byte_range(start, end); + if let Some(first) = node + .and_then(|body| body.named_child(0)) + .filter(|first| is_docstring(*first, source)) + { + let line_start = source[..start].rfind('\n').map_or(0, |offset| offset + 1); + let indent = &source[line_start..start]; + let indent = if indent.chars().all(char::is_whitespace) { + indent + } else { + " " + }; + return format!("{}\n{indent}pass # body folded", node_text(first, source)); + } + return "pass # body folded".to_owned(); + } + "{ /* body folded */ }".to_owned() +} + +pub(super) fn fold(tree: &Tree, source: &str, language: Language, symbols: &[Symbol]) -> String { + let mut output = String::new(); + let mut copied = 0; + for body in symbols.iter().filter_map(|symbol| symbol.body.as_ref()) { + if body.start_byte < copied { + continue; + } + let folded = replacement(tree, source, language, body.start_byte, body.end_byte); + if folded.len() >= body.end_byte - body.start_byte { + continue; + } + output.push_str(&source[copied..body.start_byte]); + output.push_str(&folded); + copied = body.end_byte; + } + output.push_str(&source[copied..]); + output +} diff --git a/crates/tinytools-code/src/analysis/mod.rs b/crates/tinytools-code/src/analysis/mod.rs new file mode 100644 index 00000000..f4b01ed1 --- /dev/null +++ b/crates/tinytools-code/src/analysis/mod.rs @@ -0,0 +1,65 @@ +//! Lexical query captures, syntax diagnostics and source-preserving folded views. + +mod extract; +mod fold; +mod types; + +use crate::{CacheStatus, CodeError, CodeLimits, Language, error::enforce_limit, revision}; +pub(crate) use extract::{body_node, bounded_text, node_text}; +pub(crate) use fold::is_docstring; +use tree_sitter::{Query, Tree}; +pub use types::{Analysis, Diagnostic, Reference, SourceRange, Symbol}; + +/// Resolve an exact revision-addressed ID or a provably unique lexical name. +/// +/// # Errors +/// Refuses oversized selectors, missing symbols, ambiguous names and name-based +/// selection from truncated outlines. Exact IDs remain usable after truncation. +pub fn select_symbol<'a>(analysis: &'a Analysis, selector: &str) -> Result<&'a Symbol, CodeError> { + enforce_limit("symbol selector bytes", selector.len(), 4096)?; + if let Some(symbol) = analysis.symbols.iter().find(|symbol| symbol.id == selector) { + return Ok(symbol); + } + if analysis.symbols_truncated { + return Err(CodeError::AmbiguousSymbol(selector.to_owned())); + } + let mut matches = analysis + .symbols + .iter() + .filter(|symbol| symbol.name == selector || symbol.qualified_name == selector); + let first = matches + .next() + .ok_or_else(|| CodeError::SymbolNotFound(selector.to_owned()))?; + if matches.next().is_some() { + Err(CodeError::AmbiguousSymbol(selector.to_owned())) + } else { + Ok(first) + } +} + +pub(crate) fn analyze( + tree: &Tree, + query: &Query, + language: Language, + source: &str, + limits: CodeLimits, + cache_status: CacheStatus, +) -> Analysis { + let revision = revision(source); + let (symbols, references, symbols_truncated, references_truncated) = + extract::captures(tree, query, source, &revision, limits.max_records); + let (diagnostics, diagnostics_truncated) = extract::diagnostics(tree, limits.max_diagnostics); + let skeleton = fold::fold(tree, source, language, &symbols); + Analysis { + language, + revision, + symbols, + references, + diagnostics, + skeleton, + symbols_truncated, + references_truncated, + diagnostics_truncated, + cache_status, + } +} diff --git a/crates/tinytools-code/src/analysis/types.rs b/crates/tinytools-code/src/analysis/types.rs new file mode 100644 index 00000000..e4fcb56d --- /dev/null +++ b/crates/tinytools-code/src/analysis/types.rs @@ -0,0 +1,106 @@ +//! Serializable lexical analysis results and source coordinates. + +use crate::{CacheStatus, Language}; +use serde::{Deserialize, Serialize}; + +/// UTF-8 byte offsets and one-based line / byte-column coordinates. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SourceRange { + /// Inclusive byte start. + pub start_byte: usize, + /// Exclusive byte end. + pub end_byte: usize, + /// One-based start line. + pub start_line: usize, + /// One-based start byte column. + pub start_column: usize, + /// One-based end line. + pub end_line: usize, + /// One-based exclusive end byte column. + pub end_column: usize, +} + +impl From> for SourceRange { + fn from(node: tree_sitter::Node<'_>) -> Self { + let start = node.start_position(); + let end = node.end_position(); + Self { + start_byte: node.start_byte(), + end_byte: node.end_byte(), + start_line: start.row + 1, + start_column: start.column + 1, + end_line: end.row + 1, + end_column: end.column + 1, + } + } +} + +/// A lexical declaration captured by the language's bundled query. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Symbol { + /// Revision-addressed, exact-range identifier. + pub id: String, + /// Local declaration name. + pub name: String, + /// Lexical ancestor names joined with `::`; not semantic resolution. + pub qualified_name: String, + /// Grammar node kind. + pub kind: String, + /// Entire declaration range. + pub range: SourceRange, + /// Body/container range, when this declaration has one. + pub body: Option, + /// Declaration prefix, bounded to 4096 UTF-8 bytes. + pub signature: String, + /// Whether the signature exceeded its byte limit. + pub signature_truncated: bool, +} + +/// A lexical call or import occurrence, without resolved target identity. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Reference { + /// Query classification such as `call` or `import`. + pub kind: String, + /// Captured source text, at most 1024 bytes. + pub text: String, + /// Captured range. + pub range: SourceRange, + /// Whether the text was shortened. + pub text_truncated: bool, +} + +/// A missing or erroneous syntax node. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Diagnostic { + /// `error` or `missing`. + pub kind: String, + /// Syntax-only description. + pub message: String, + /// Node location. + pub range: SourceRange, +} + +/// Bounded analysis for one explicit source snapshot. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Analysis { + /// Grammar used. + pub language: Language, + /// BLAKE3 content revision. + pub revision: String, + /// Lexical declarations in source order. + pub symbols: Vec, + /// Lexical call/import captures. + pub references: Vec, + /// Syntax diagnostics. + pub diagnostics: Vec, + /// Source with captured declaration bodies folded. + pub skeleton: String, + /// Symbol count or query execution limit was reached. + pub symbols_truncated: bool, + /// Reference count or query execution limit was reached. + pub references_truncated: bool, + /// Diagnostic count limit was reached. + pub diagnostics_truncated: bool, + /// Tree reuse outcome for this request. + pub cache_status: CacheStatus, +} diff --git a/crates/tinytools-code/src/catalog/README.md b/crates/tinytools-code/src/catalog/README.md new file mode 100644 index 00000000..46bbfd20 --- /dev/null +++ b/crates/tinytools-code/src/catalog/README.md @@ -0,0 +1,3 @@ +# Chunk catalog + +Manifests contain typed metadata separately from template source. Paths must be portable and relative; filesystem containment belongs to the adapter. Catalog insertion validates bounded metadata and rejects duplicate IDs. Search is deterministic and metadata-only. Only built-in entries expose embedded source. diff --git a/crates/tinytools-code/src/catalog/builtins.rs b/crates/tinytools-code/src/catalog/builtins.rs new file mode 100644 index 00000000..676999d8 --- /dev/null +++ b/crates/tinytools-code/src/catalog/builtins.rs @@ -0,0 +1,29 @@ +//! Compile-time source for entries created by the built-in catalog. + +pub(super) fn source(id: &str) -> Option<&'static str> { + match id { + "rust.fallible-function.v1" => { + Some(include_str!("templates/rust.fallible-function.v1.txt")) + } + "rust.tauri-command.v1" => Some(include_str!("templates/rust.tauri-command.v1.txt")), + "typescript.typed-function.v1" => { + Some(include_str!("templates/typescript.typed-function.v1.txt")) + } + "tsx.accessible-action.v1" => Some(include_str!("templates/tsx.accessible-action.v1.txt")), + "python.resource-context.v1" => { + Some(include_str!("templates/python.resource-context.v1.txt")) + } + "go.http-handler.v1" => Some(include_str!("templates/go.http-handler.v1.txt")), + "typescript.vitest-suite.v1" => { + Some(include_str!("templates/typescript.vitest-suite.v1.txt")) + } + "json.service-config.v1" => Some(include_str!("templates/json.service-config.v1.txt")), + "toml.service-config.v1" => Some(include_str!("templates/toml.service-config.v1.txt")), + "rust.checked-add.v1" => Some(include_str!("templates/rust.checked-add.v1.txt")), + "typescript.nullable-guard.v1" => { + Some(include_str!("templates/typescript.nullable-guard.v1.txt")) + } + "rust.public-field.v1" => Some(include_str!("templates/rust.public-field.v1.txt")), + _ => None, + } +} diff --git a/crates/tinytools-code/src/catalog/mod.rs b/crates/tinytools-code/src/catalog/mod.rs new file mode 100644 index 00000000..35cd37df --- /dev/null +++ b/crates/tinytools-code/src/catalog/mod.rs @@ -0,0 +1,169 @@ +//! Metadata-first built-in and custom chunk discovery. + +mod builtins; +mod types; +mod validation; + +use crate::{ + CodeError, Language, + analysis::bounded_text, + error::{enforce_limit, invalid}, + revision, +}; +use std::collections::BTreeMap; +pub use types::{ + ChunkManifest, ChunkParameter, ChunkSlot, ChunkSummary, ChunkTarget, ParameterKind, +}; +pub(crate) use validation::identifier; + +#[derive(Debug, Clone)] +struct Entry { + manifest: ChunkManifest, + source: Option<&'static str>, +} + +/// A bounded catalog whose search results never expose template bodies. +#[derive(Debug, Clone, Default)] +pub struct ChunkCatalog { + entries: BTreeMap, +} + +impl ChunkManifest { + /// Check metadata, defaults, portable paths and bounded list sizes. + /// + /// # Errors + /// Refuses invalid paths, duplicate names, unknown kinds or exceeded bounds. + pub fn validate(&self) -> Result<(), CodeError> { + validation::validate(self) + } + + /// Revision of validated, normalized metadata; template content is separate. + /// + /// # Errors + /// Refuses invalid metadata or serialization failure. + pub fn revision(&self) -> Result { + self.validate()?; + let serialized = serde_json::to_string(self).map_err(|error| invalid(error.to_string()))?; + Ok(revision(&serialized)) + } +} + +impl ChunkCatalog { + /// Create an empty catalog for caller-supplied manifests. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Load the twelve shipped patterns without filesystem access. + /// + /// # Errors + /// Refuses inconsistent built-in metadata. + pub fn builtin() -> Result { + let manifests: Vec = + serde_json::from_str(include_str!("starter-manifests.json")) + .map_err(|error| invalid(error.to_string()))?; + let mut catalog = Self::new(); + for manifest in manifests { + let source = builtins::source(&manifest.id); + manifest.validate()?; + catalog + .entries + .insert(manifest.id.clone(), Entry { manifest, source }); + } + Ok(catalog) + } + + /// Insert validated custom metadata without granting access to built-in source. + /// + /// # Errors + /// Refuses invalid manifests, duplicate IDs or catalogs above 512 entries. + pub fn insert(&mut self, manifest: ChunkManifest) -> Result<(), CodeError> { + manifest.validate()?; + if self.entries.contains_key(&manifest.id) { + return Err(invalid("duplicate chunk ID")); + } + enforce_limit("catalog entries", self.entries.len() + 1, 512)?; + self.entries.insert( + manifest.id.clone(), + Entry { + manifest, + source: None, + }, + ); + Ok(()) + } + + /// Inspect one metadata record. + /// + /// # Errors + /// Refuses oversized or unknown IDs. + pub fn manifest(&self, id: &str) -> Result<&ChunkManifest, CodeError> { + enforce_limit("chunk ID bytes", id.len(), 128)?; + self.entries + .get(id) + .map(|entry| &entry.manifest) + .ok_or_else(|| CodeError::UnknownChunk(id.to_owned())) + } + + /// Alias for metadata inspection. + /// + /// # Errors + /// Refuses oversized or unknown IDs. + pub fn inspect(&self, id: &str) -> Result<&ChunkManifest, CodeError> { + self.manifest(id) + } + + /// Return source only for entries loaded by [`Self::builtin`]. + #[must_use] + pub fn builtin_source(&self, id: &str) -> Option<&'static str> { + self.entries.get(id).and_then(|entry| entry.source) + } + + /// Search bounded metadata terms with deterministic score/ID ordering. + /// + /// Query text is capped at 1024 bytes and 32 terms; at most 50 hits return. + #[must_use] + pub fn search( + &self, + query: &str, + language: Option, + limit: usize, + ) -> Vec { + let query = bounded_text(query, 1024).0.to_ascii_lowercase(); + let terms: Vec<_> = query.split_whitespace().take(32).collect(); + let mut hits: Vec<_> = self + .entries + .values() + .filter(|entry| language.is_none_or(|language| language == entry.manifest.language)) + .filter_map(|entry| { + let manifest = &entry.manifest; + let searchable = format!( + "{} {} {}", + manifest.id, + manifest.summary, + manifest.tags.join(" ") + ) + .to_ascii_lowercase(); + let score = terms + .iter() + .filter(|term| searchable.contains(**term)) + .count(); + (terms.is_empty() || score > 0).then_some((score, manifest)) + }) + .collect(); + hits.sort_by(|(score_a, a), (score_b, b)| { + score_b.cmp(score_a).then_with(|| a.id.cmp(&b.id)) + }); + hits.into_iter() + .take(limit.min(50)) + .map(|(_, manifest)| ChunkSummary { + id: manifest.id.clone(), + summary: manifest.summary.clone(), + language: manifest.language, + target: manifest.target, + tags: manifest.tags.clone(), + }) + .collect() + } +} diff --git a/crates/tinytools-code/src/catalog/starter-manifests.json b/crates/tinytools-code/src/catalog/starter-manifests.json new file mode 100644 index 00000000..39baa8cc --- /dev/null +++ b/crates/tinytools-code/src/catalog/starter-manifests.json @@ -0,0 +1,378 @@ +[ + { + "id": "rust.fallible-function.v1", + "summary": "Rust fallible function with an explicit result type", + "language": "rust", + "target": "file", + "template_path": "templates/rust.fallible-function.v1.txt", + "parameters": [ + { + "name": "name", + "kind": "identifier", + "required": false, + "default": "run" + }, + { + "name": "return_type", + "kind": "type_expr", + "required": false, + "default": "()" + }, + { + "name": "error_type", + "kind": "type_expr", + "required": false, + "default": "String" + } + ], + "slots": [ + { + "name": "body", + "required": false + } + ], + "dependencies": [], + "tags": [ + "rust", + "file" + ], + "forbidden_nodes": [] + }, + { + "id": "rust.tauri-command.v1", + "summary": "Tauri command returning a typed result", + "language": "rust", + "target": "file", + "template_path": "templates/rust.tauri-command.v1.txt", + "parameters": [ + { + "name": "name", + "kind": "identifier", + "required": false, + "default": "greet" + } + ], + "slots": [ + { + "name": "body", + "required": false + } + ], + "dependencies": [ + "tauri" + ], + "tags": [ + "rust", + "file" + ], + "forbidden_nodes": [] + }, + { + "id": "typescript.typed-function.v1", + "summary": "TypeScript function with explicit input and output types", + "language": "typescript", + "target": "file", + "template_path": "templates/typescript.typed-function.v1.txt", + "parameters": [ + { + "name": "name", + "kind": "identifier", + "required": false, + "default": "transform" + }, + { + "name": "input_type", + "kind": "type_expr", + "required": false, + "default": "string" + }, + { + "name": "return_type", + "kind": "type_expr", + "required": false, + "default": "string" + } + ], + "slots": [ + { + "name": "body", + "required": true + } + ], + "dependencies": [], + "tags": [ + "typescript", + "file" + ], + "forbidden_nodes": [] + }, + { + "id": "tsx.accessible-action.v1", + "summary": "React native button with an accessible text label", + "language": "tsx", + "target": "file", + "template_path": "templates/tsx.accessible-action.v1.txt", + "parameters": [ + { + "name": "name", + "kind": "pascal_case_ident", + "required": false, + "default": "Action" + }, + { + "name": "label", + "kind": "string", + "required": false, + "default": "Run action" + } + ], + "slots": [], + "dependencies": [ + "react" + ], + "tags": [ + "tsx", + "file" + ], + "forbidden_nodes": [] + }, + { + "id": "python.resource-context.v1", + "summary": "Python resource context with explicit setup and cleanup", + "language": "python", + "target": "file", + "template_path": "templates/python.resource-context.v1.txt", + "parameters": [ + { + "name": "name", + "kind": "identifier", + "required": false, + "default": "managed_resource" + } + ], + "slots": [ + { + "name": "setup", + "required": true + }, + { + "name": "cleanup", + "required": true + } + ], + "dependencies": [], + "tags": [ + "python", + "file" + ], + "forbidden_nodes": [] + }, + { + "id": "go.http-handler.v1", + "summary": "Go HTTP handler with caller-supplied response behavior", + "language": "go", + "target": "file", + "template_path": "templates/go.http-handler.v1.txt", + "parameters": [ + { + "name": "name", + "kind": "identifier", + "required": false, + "default": "Handle" + } + ], + "slots": [ + { + "name": "body", + "required": true + } + ], + "dependencies": [], + "tags": [ + "go", + "file" + ], + "forbidden_nodes": [] + }, + { + "id": "typescript.vitest-suite.v1", + "summary": "Vitest suite with required meaningful assertions", + "language": "typescript", + "target": "file", + "template_path": "templates/typescript.vitest-suite.v1.txt", + "parameters": [ + { + "name": "suite", + "kind": "string", + "required": false, + "default": "example" + } + ], + "slots": [ + { + "name": "assertions", + "required": true + } + ], + "dependencies": [ + "vitest" + ], + "tags": [ + "typescript", + "file" + ], + "forbidden_nodes": [] + }, + { + "id": "json.service-config.v1", + "summary": "JSON service configuration with typed values", + "language": "json", + "target": "file", + "template_path": "templates/json.service-config.v1.txt", + "parameters": [ + { + "name": "name", + "kind": "string", + "required": false, + "default": "service" + }, + { + "name": "port", + "kind": "number", + "required": false, + "default": 8080 + }, + { + "name": "enabled", + "kind": "boolean", + "required": false, + "default": true + } + ], + "slots": [], + "dependencies": [], + "tags": [ + "json", + "file" + ], + "forbidden_nodes": [] + }, + { + "id": "toml.service-config.v1", + "summary": "TOML service configuration with typed values", + "language": "toml", + "target": "file", + "template_path": "templates/toml.service-config.v1.txt", + "parameters": [ + { + "name": "name", + "kind": "string", + "required": false, + "default": "service" + }, + { + "name": "port", + "kind": "number", + "required": false, + "default": 8080 + }, + { + "name": "enabled", + "kind": "boolean", + "required": false, + "default": true + } + ], + "slots": [], + "dependencies": [], + "tags": [ + "toml", + "file" + ], + "forbidden_nodes": [] + }, + { + "id": "rust.checked-add.v1", + "summary": "Rust checked addition body with an explicit overflow error", + "language": "rust", + "target": "function_body", + "template_path": "templates/rust.checked-add.v1.txt", + "parameters": [ + { + "name": "left", + "kind": "identifier", + "required": false, + "default": "left" + }, + { + "name": "right", + "kind": "identifier", + "required": false, + "default": "right" + } + ], + "slots": [], + "dependencies": [], + "tags": [ + "rust", + "function_body" + ], + "forbidden_nodes": [] + }, + { + "id": "typescript.nullable-guard.v1", + "summary": "TypeScript nullish-value guard body", + "language": "typescript", + "target": "function_body", + "template_path": "templates/typescript.nullable-guard.v1.txt", + "parameters": [ + { + "name": "value", + "kind": "identifier", + "required": false, + "default": "value" + }, + { + "name": "message", + "kind": "string", + "required": false, + "default": "value is required" + } + ], + "slots": [], + "dependencies": [], + "tags": [ + "typescript", + "function_body" + ], + "forbidden_nodes": [] + }, + { + "id": "rust.public-field.v1", + "summary": "Rust public field for a struct", + "language": "rust", + "target": "member", + "template_path": "templates/rust.public-field.v1.txt", + "parameters": [ + { + "name": "name", + "kind": "identifier", + "required": false, + "default": "value" + }, + { + "name": "type", + "kind": "type_expr", + "required": false, + "default": "String" + } + ], + "slots": [], + "dependencies": [], + "tags": [ + "rust", + "member" + ], + "forbidden_nodes": [] + } +] diff --git a/crates/tinytools-code/src/catalog/templates/go.http-handler.v1.txt b/crates/tinytools-code/src/catalog/templates/go.http-handler.v1.txt new file mode 100644 index 00000000..7d79662c --- /dev/null +++ b/crates/tinytools-code/src/catalog/templates/go.http-handler.v1.txt @@ -0,0 +1,7 @@ +package main + +import "net/http" + +func {{name}}(w http.ResponseWriter, r *http.Request) { + // @slot body +} diff --git a/crates/tinytools-code/src/catalog/templates/json.service-config.v1.txt b/crates/tinytools-code/src/catalog/templates/json.service-config.v1.txt new file mode 100644 index 00000000..69498280 --- /dev/null +++ b/crates/tinytools-code/src/catalog/templates/json.service-config.v1.txt @@ -0,0 +1,5 @@ +{ + "name": {{name}}, + "port": {{port}}, + "enabled": {{enabled}} +} diff --git a/crates/tinytools-code/src/catalog/templates/python.resource-context.v1.txt b/crates/tinytools-code/src/catalog/templates/python.resource-context.v1.txt new file mode 100644 index 00000000..e426d7cf --- /dev/null +++ b/crates/tinytools-code/src/catalog/templates/python.resource-context.v1.txt @@ -0,0 +1,9 @@ +from contextlib import contextmanager + +@contextmanager +def {{name}}(): + # @slot setup + try: + yield resource + finally: + # @slot cleanup diff --git a/crates/tinytools-code/src/catalog/templates/rust.checked-add.v1.txt b/crates/tinytools-code/src/catalog/templates/rust.checked-add.v1.txt new file mode 100644 index 00000000..88203bbb --- /dev/null +++ b/crates/tinytools-code/src/catalog/templates/rust.checked-add.v1.txt @@ -0,0 +1 @@ +{{left}}.checked_add({{right}}).ok_or_else(|| "overflow".to_owned()) diff --git a/crates/tinytools-code/src/catalog/templates/rust.fallible-function.v1.txt b/crates/tinytools-code/src/catalog/templates/rust.fallible-function.v1.txt new file mode 100644 index 00000000..9b34c07d --- /dev/null +++ b/crates/tinytools-code/src/catalog/templates/rust.fallible-function.v1.txt @@ -0,0 +1,4 @@ +pub fn {{name}}() -> Result<{{return_type}}, {{error_type}}> { + // @slot body + Ok(()) +} diff --git a/crates/tinytools-code/src/catalog/templates/rust.public-field.v1.txt b/crates/tinytools-code/src/catalog/templates/rust.public-field.v1.txt new file mode 100644 index 00000000..7c2dd7fc --- /dev/null +++ b/crates/tinytools-code/src/catalog/templates/rust.public-field.v1.txt @@ -0,0 +1 @@ +pub {{name}}: {{type}}, diff --git a/crates/tinytools-code/src/catalog/templates/rust.tauri-command.v1.txt b/crates/tinytools-code/src/catalog/templates/rust.tauri-command.v1.txt new file mode 100644 index 00000000..559d8880 --- /dev/null +++ b/crates/tinytools-code/src/catalog/templates/rust.tauri-command.v1.txt @@ -0,0 +1,5 @@ +#[tauri::command] +pub fn {{name}}(input: String) -> Result { + // @slot body + Ok(input) +} diff --git a/crates/tinytools-code/src/catalog/templates/toml.service-config.v1.txt b/crates/tinytools-code/src/catalog/templates/toml.service-config.v1.txt new file mode 100644 index 00000000..0fc2f883 --- /dev/null +++ b/crates/tinytools-code/src/catalog/templates/toml.service-config.v1.txt @@ -0,0 +1,4 @@ +[service] +name = {{name}} +port = {{port}} +enabled = {{enabled}} diff --git a/crates/tinytools-code/src/catalog/templates/tsx.accessible-action.v1.txt b/crates/tinytools-code/src/catalog/templates/tsx.accessible-action.v1.txt new file mode 100644 index 00000000..92d7f85e --- /dev/null +++ b/crates/tinytools-code/src/catalog/templates/tsx.accessible-action.v1.txt @@ -0,0 +1,4 @@ +export function {{name}}() { + const label = {{label}}; + return ; +} diff --git a/crates/tinytools-code/src/catalog/templates/typescript.nullable-guard.v1.txt b/crates/tinytools-code/src/catalog/templates/typescript.nullable-guard.v1.txt new file mode 100644 index 00000000..2f6a597e --- /dev/null +++ b/crates/tinytools-code/src/catalog/templates/typescript.nullable-guard.v1.txt @@ -0,0 +1,4 @@ +if ({{value}} == null) { + throw new Error({{message}}); +} +return {{value}}; diff --git a/crates/tinytools-code/src/catalog/templates/typescript.typed-function.v1.txt b/crates/tinytools-code/src/catalog/templates/typescript.typed-function.v1.txt new file mode 100644 index 00000000..2b53bb24 --- /dev/null +++ b/crates/tinytools-code/src/catalog/templates/typescript.typed-function.v1.txt @@ -0,0 +1,3 @@ +export function {{name}}(value: {{input_type}}): {{return_type}} { + // @slot body +} diff --git a/crates/tinytools-code/src/catalog/templates/typescript.vitest-suite.v1.txt b/crates/tinytools-code/src/catalog/templates/typescript.vitest-suite.v1.txt new file mode 100644 index 00000000..fb2ffa3c --- /dev/null +++ b/crates/tinytools-code/src/catalog/templates/typescript.vitest-suite.v1.txt @@ -0,0 +1,6 @@ +import { describe, it, expect } from 'vitest'; +describe({{suite}}, () => { + it('handles the example', () => { + // @slot assertions + }); +}); diff --git a/crates/tinytools-code/src/catalog/types.rs b/crates/tinytools-code/src/catalog/types.rs new file mode 100644 index 00000000..8e776b31 --- /dev/null +++ b/crates/tinytools-code/src/catalog/types.rs @@ -0,0 +1,119 @@ +//! Portable, typed chunk metadata; dependency declarations are informational. + +use crate::Language; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +/// Where rendered source is intended to be inserted. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ChunkTarget { + /// A complete source file. + #[default] + File, + /// Statements replacing a function body. + FunctionBody, + /// A field, method or member inside a supported container. + Member, +} + +/// Validation and escaping rule for one parameter. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ParameterKind { + /// Conservative ASCII identifier. + Identifier, + /// ASCII identifier beginning with an uppercase letter. + PascalCaseIdent, + /// Exactly one grammar type expression. + TypeExpr, + /// Escaped string literal. + String, + /// JSON finite numeric value. + Number, + /// Boolean literal for the selected language. + Boolean, +} + +/// One named typed placeholder. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ChunkParameter { + /// Placeholder name in `{{name}}` syntax. + pub name: String, + /// Validation and escaping rule. + pub kind: ParameterKind, + /// Whether omission without a default is refused. + #[serde(default)] + pub required: bool, + /// Optional typed default. + #[serde(default)] + pub default: Option, + /// Human-readable purpose. + #[serde(default)] + pub description: String, +} + +/// One standalone comment slot for code supplied by the caller. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ChunkSlot { + /// Name following `@slot` in a standalone comment. + pub name: String, + /// Whether an omitted or empty binding is refused. + #[serde(default)] + pub required: bool, + /// Optional exact grammar node kind required for this slot's expansion. + #[serde(default)] + pub node_kind: Option, + /// Human-readable purpose. + #[serde(default)] + pub description: String, +} + +/// A versioned chunk's metadata, separate from template contents. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ChunkManifest { + /// Stable, versioned catalog ID. + pub id: String, + /// Short human-readable summary. + pub summary: String, + /// Grammar used for validation. + pub language: Language, + /// Insertion context. + #[serde(default)] + pub target: ChunkTarget, + /// Portable relative template path, interpreted by an adapter. + pub template_path: String, + /// Typed parameters. + #[serde(default)] + pub parameters: Vec, + /// Standalone code slots. + #[serde(default)] + pub slots: Vec, + /// Search metadata. + #[serde(default)] + pub tags: Vec, + /// Informational package requirements; never automatically installed. + #[serde(default)] + pub dependencies: Vec, + /// Grammar node kinds forbidden in generated source. + #[serde(default)] + pub forbidden_nodes: Vec, +} + +/// Metadata-only search result, excluding template bodies and defaults. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ChunkSummary { + /// Versioned catalog ID. + pub id: String, + /// Human-readable purpose. + pub summary: String, + /// Grammar supported by this pattern. + pub language: Language, + /// Intended insertion context. + pub target: ChunkTarget, + /// Search tags. + pub tags: Vec, +} diff --git a/crates/tinytools-code/src/catalog/validation.rs b/crates/tinytools-code/src/catalog/validation.rs new file mode 100644 index 00000000..4129140d --- /dev/null +++ b/crates/tinytools-code/src/catalog/validation.rs @@ -0,0 +1,120 @@ +//! Manifest shape, portable paths and typed metadata budgets. + +use super::{ChunkManifest, ParameterKind}; +use crate::{ + CodeError, + error::{enforce_limit, invalid}, +}; +use std::collections::BTreeSet; + +pub(crate) fn identifier(value: &str) -> bool { + let mut bytes = value.bytes(); + bytes + .next() + .is_some_and(|first| first.is_ascii_alphabetic() || first == b'_') + && bytes.all(|byte| byte.is_ascii_alphanumeric() || byte == b'_') +} + +fn text(value: &str, resource: &'static str, limit: usize) -> Result<(), CodeError> { + enforce_limit(resource, value.len(), limit)?; + if value.contains('\0') { + return Err(invalid("metadata cannot contain NUL")); + } + Ok(()) +} + +fn named(value: &str) -> Result<(), CodeError> { + text(value, "binding name bytes", 128)?; + if !identifier(value) { + return Err(invalid("binding names must be ASCII identifiers")); + } + Ok(()) +} + +fn template_path(value: &str) -> Result<(), CodeError> { + text(value, "template path bytes", 4096)?; + if value.contains(['\\', ':']) + || value.split('/').any(|part| { + part.is_empty() || part == "." || part == ".." || part.ends_with(['.', ' ']) + }) + { + return Err(invalid( + "template_path must be a portable, relative path without traversal or empty segments", + )); + } + Ok(()) +} + +pub(super) fn validate(manifest: &ChunkManifest) -> Result<(), CodeError> { + text(&manifest.id, "chunk ID bytes", 128)?; + if manifest.id.is_empty() + || !manifest + .id + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"._-".contains(&byte)) + { + return Err(invalid( + "chunk ID must contain only ASCII letters, digits, dots, dashes or underscores", + )); + } + text(&manifest.summary, "summary bytes", 1024)?; + if manifest.summary.trim().is_empty() { + return Err(invalid("chunk summary must not be empty")); + } + template_path(&manifest.template_path)?; + enforce_limit("parameters", manifest.parameters.len(), 32)?; + enforce_limit("slots", manifest.slots.len(), 32)?; + enforce_limit("tags", manifest.tags.len(), 32)?; + enforce_limit("dependencies", manifest.dependencies.len(), 64)?; + enforce_limit("forbidden node kinds", manifest.forbidden_nodes.len(), 32)?; + let mut names = BTreeSet::new(); + for parameter in &manifest.parameters { + named(¶meter.name)?; + if !names.insert(¶meter.name) { + return Err(invalid("duplicate parameter name")); + } + text(¶meter.description, "parameter description bytes", 1024)?; + if let Some(value) = ¶meter.default { + let valid = match parameter.kind { + ParameterKind::Identifier | ParameterKind::PascalCaseIdent => { + value.as_str().is_some_and(|value| { + identifier(value) + && (parameter.kind != ParameterKind::PascalCaseIdent + || value.as_bytes()[0].is_ascii_uppercase()) + }) + } + ParameterKind::TypeExpr | ParameterKind::String => value.is_string(), + ParameterKind::Number => value.is_number(), + ParameterKind::Boolean => value.is_boolean(), + }; + if !valid { + return Err(invalid( + "parameter default does not match its declared kind", + )); + } + text(&value.to_string(), "parameter default bytes", 4096)?; + } + } + names.clear(); + for slot in &manifest.slots { + named(&slot.name)?; + if !names.insert(&slot.name) { + return Err(invalid("duplicate slot name")); + } + text(&slot.description, "slot description bytes", 1024)?; + if let Some(kind) = &slot.node_kind { + named(kind)?; + } + } + for tag in &manifest.tags { + text(tag, "tag bytes", 128)?; + } + for dependency in &manifest.dependencies { + text(dependency, "dependency bytes", 256)?; + } + for kind in &manifest.forbidden_nodes { + named(kind)?; + } + let serialized = serde_json::to_string(manifest).map_err(|error| invalid(error.to_string()))?; + enforce_limit("manifest bytes", serialized.len(), 65_536) +} diff --git a/crates/tinytools-code/src/engine/README.md b/crates/tinytools-code/src/engine/README.md new file mode 100644 index 00000000..bcc61ecd --- /dev/null +++ b/crates/tinytools-code/src/engine/README.md @@ -0,0 +1,3 @@ +# Parser cache + +`CodeEngine` stores bounded source snapshots and trees by caller key and language. Unchanged input reuses a tree; changed input receives a minimal UTF-8-safe edit. LRU eviction measures source bytes, not native parser memory. Grammar/query data is compiled and no I/O occurs. diff --git a/crates/tinytools-code/src/engine/edit.rs b/crates/tinytools-code/src/engine/edit.rs new file mode 100644 index 00000000..a57bd052 --- /dev/null +++ b/crates/tinytools-code/src/engine/edit.rs @@ -0,0 +1,41 @@ +//! Minimal contiguous UTF-8 edits for incremental parser reuse. + +use tree_sitter::{InputEdit, Point}; + +fn point(source: &str, byte: usize) -> Point { + let prefix = &source[..byte]; + let row = prefix.bytes().filter(|value| *value == b'\n').count(); + let column = prefix.rfind('\n').map_or(byte, |offset| byte - offset - 1); + Point { row, column } +} + +pub(super) fn difference(old: &str, new: &str) -> InputEdit { + let mut start = old + .bytes() + .zip(new.bytes()) + .take_while(|(a, b)| a == b) + .count(); + while !old.is_char_boundary(start) || !new.is_char_boundary(start) { + start -= 1; + } + let suffix = old.as_bytes()[start..] + .iter() + .rev() + .zip(new.as_bytes()[start..].iter().rev()) + .take_while(|(a, b)| a == b) + .count(); + let mut old_end = old.len() - suffix; + let mut new_end = new.len() - suffix; + while !old.is_char_boundary(old_end) || !new.is_char_boundary(new_end) { + old_end += 1; + new_end += 1; + } + InputEdit { + start_byte: start, + old_end_byte: old_end, + new_end_byte: new_end, + start_position: point(old, start), + old_end_position: point(old, old_end), + new_end_position: point(new, new_end), + } +} diff --git a/crates/tinytools-code/src/engine/mod.rs b/crates/tinytools-code/src/engine/mod.rs new file mode 100644 index 00000000..2b0d38bf --- /dev/null +++ b/crates/tinytools-code/src/engine/mod.rs @@ -0,0 +1,178 @@ +//! Bounded LRU source snapshots and reusable incremental syntax trees. + +mod edit; +mod types; + +use crate::{Analysis, CodeError, Language, analysis, error::enforce_limit}; +use std::collections::{HashMap, VecDeque}; +use tree_sitter::{Parser, Query, Tree}; +pub use types::{CacheStats, CacheStatus, CodeLimits}; + +#[derive(Debug)] +struct CachedSource { + key: String, + language: Language, + source: String, + tree: Tree, +} + +/// A pure syntax engine with bounded, per-key retained source snapshots. +#[derive(Debug, Default)] +pub struct CodeEngine { + limits: CodeLimits, + cache: VecDeque, + queries: HashMap, + stats: CacheStats, +} + +/// Compute the content revision used for source, manifest and template guards. +#[must_use] +pub fn revision(source: &str) -> String { + blake3::hash(source.as_bytes()).to_hex().to_string() +} + +impl CodeEngine { + /// Create an engine with explicitly bounded resources. + /// + /// # Errors + /// Returns an error for zero required budgets or budgets above hard caps. + pub fn new(limits: CodeLimits) -> Result { + limits.validate()?; + Ok(Self { + limits, + ..Self::default() + }) + } + + /// Active resource budgets. + #[must_use] + pub fn limits(&self) -> CodeLimits { + self.limits + } + + /// Current usage and cumulative parser reuse counters. + #[must_use] + pub fn cache_stats(&self) -> CacheStats { + self.stats + } + + /// Drop retained source and trees, preserving cumulative counters. + pub fn clear_cache(&mut self) { + self.cache.clear(); + self.stats.entries = 0; + self.stats.source_bytes = 0; + } + + /// Analyze one caller-keyed snapshot with an explicit grammar. + /// + /// # Errors + /// Refuses oversized source/keys or unavailable compiled grammar/query data. + /// Invalid syntax is returned as diagnostics, not as an operation error. + pub fn analyze( + &mut self, + key: &str, + language: Language, + source: &str, + ) -> Result { + enforce_limit("cache key bytes", key.len(), 4096)?; + enforce_limit("source bytes", source.len(), self.limits.max_source_bytes)?; + let (tree, status) = self.cached_parse(key, language, source)?; + let query = match self.queries.entry(language) { + std::collections::hash_map::Entry::Occupied(entry) => entry.into_mut(), + std::collections::hash_map::Entry::Vacant(entry) => entry.insert( + Query::new(&language.grammar(), language.query()) + .map_err(|error| CodeError::Parser(error.to_string()))?, + ), + }; + Ok(analysis::analyze( + &tree, + query, + language, + source, + self.limits, + status, + )) + } + + /// Validate complete source without retaining it in the cache. + /// + /// # Errors + /// Refuses source beyond the configured budget or any missing/error node. + pub fn validate(&self, language: Language, source: &str) -> Result<(), CodeError> { + let tree = self.parse(language, source)?; + if tree.root_node().has_error() { + Err(CodeError::InvalidSyntax { language }) + } else { + Ok(()) + } + } + + pub(crate) fn parse(&self, language: Language, source: &str) -> Result { + enforce_limit("source bytes", source.len(), self.limits.max_source_bytes)?; + parse(language, source, None) + } + + fn cached_parse( + &mut self, + key: &str, + language: Language, + source: &str, + ) -> Result<(Tree, CacheStatus), CodeError> { + let position = self + .cache + .iter() + .position(|entry| entry.key == key && entry.language == language); + let previous = position.and_then(|index| self.cache.remove(index)); + let (tree, status) = if let Some(mut previous) = previous { + self.stats.source_bytes -= previous.source.len(); + if previous.source == source { + self.stats.hits += 1; + (previous.tree, CacheStatus::Cached) + } else { + previous + .tree + .edit(&edit::difference(&previous.source, source)); + self.stats.incremental_parses += 1; + ( + parse(language, source, Some(&previous.tree))?, + CacheStatus::Incremental, + ) + } + } else { + self.stats.cold_parses += 1; + (parse(language, source, None)?, CacheStatus::Fresh) + }; + if self.limits.max_cache_entries > 0 + && self.limits.max_cache_bytes > 0 + && source.len() <= self.limits.max_cache_bytes + { + while self.cache.len() >= self.limits.max_cache_entries + || self.stats.source_bytes + source.len() > self.limits.max_cache_bytes + { + if let Some(oldest) = self.cache.pop_front() { + self.stats.source_bytes -= oldest.source.len(); + self.stats.evictions += 1; + } + } + self.stats.source_bytes += source.len(); + self.cache.push_back(CachedSource { + key: key.to_owned(), + language, + source: source.to_owned(), + tree: tree.clone(), + }); + } + self.stats.entries = self.cache.len(); + Ok((tree, status)) + } +} + +fn parse(language: Language, source: &str, old: Option<&Tree>) -> Result { + let mut parser = Parser::new(); + parser + .set_language(&language.grammar()) + .map_err(|error| CodeError::Parser(error.to_string()))?; + parser + .parse(source, old) + .ok_or_else(|| CodeError::Parser("parsing returned no tree".to_owned())) +} diff --git a/crates/tinytools-code/src/engine/types.rs b/crates/tinytools-code/src/engine/types.rs new file mode 100644 index 00000000..e375bbee --- /dev/null +++ b/crates/tinytools-code/src/engine/types.rs @@ -0,0 +1,80 @@ +//! Parser and retained-source budgets, with observable cache outcomes. + +use crate::{ + CodeError, + error::{enforce_limit, invalid}, +}; +use serde::{Deserialize, Serialize}; + +/// Resource bounds applied before parsing or expanding source. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CodeLimits { + /// Maximum bytes in one source or rendered result. + pub max_source_bytes: usize, + /// Maximum cached parse trees; zero disables caching. + pub max_cache_entries: usize, + /// Maximum retained source bytes; excludes native tree allocations. + pub max_cache_bytes: usize, + /// Maximum symbols and references, each independently bounded. + pub max_records: usize, + /// Maximum syntax diagnostics. + pub max_diagnostics: usize, +} + +impl Default for CodeLimits { + fn default() -> Self { + Self { + max_source_bytes: 1_048_576, + max_cache_entries: 32, + max_cache_bytes: 8_388_608, + max_records: 2048, + max_diagnostics: 64, + } + } +} + +impl CodeLimits { + pub(crate) fn validate(self) -> Result<(), CodeError> { + enforce_limit("source budget", self.max_source_bytes, 16_777_216)?; + enforce_limit("cache entry budget", self.max_cache_entries, 1024)?; + enforce_limit("cache byte budget", self.max_cache_bytes, 134_217_728)?; + enforce_limit("record budget", self.max_records, 65_536)?; + enforce_limit("diagnostic budget", self.max_diagnostics, 4096)?; + if self.max_source_bytes == 0 || self.max_records == 0 || self.max_diagnostics == 0 { + return Err(invalid( + "source, record and diagnostic budgets must be positive", + )); + } + Ok(()) + } +} + +/// How this particular analysis obtained its parse tree. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum CacheStatus { + /// No reusable cached tree was available. + Fresh, + /// Source was identical to its cached snapshot. + Cached, + /// A prior tree was edited and incrementally reparsed. + Incremental, +} + +/// Cumulative cache counters plus current retained-source usage. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct CacheStats { + /// Currently retained entries. + pub entries: usize, + /// Currently retained UTF-8 source bytes. + pub source_bytes: usize, + /// Unchanged-source hits. + pub hits: u64, + /// Incremental reparses. + pub incremental_parses: u64, + /// Parses without a prior tree. + pub cold_parses: u64, + /// Entries evicted to meet budgets. + pub evictions: u64, +} diff --git a/crates/tinytools-code/src/error/mod.rs b/crates/tinytools-code/src/error/mod.rs new file mode 100644 index 00000000..6471ff09 --- /dev/null +++ b/crates/tinytools-code/src/error/mod.rs @@ -0,0 +1,89 @@ +//! Actionable refusals at syntax, revision and resource boundaries. + +use crate::Language; +use std::fmt; + +/// A structural operation could not safely produce its requested result. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum CodeError { + /// A configured or fixed resource budget was exceeded. + LimitExceeded { + /// Resource being limited. + resource: &'static str, + /// Maximum accepted value. + limit: usize, + /// Requested value. + actual: usize, + }, + /// A request or manifest violates its contract. + InvalidInput(String), + /// A compiled grammar or parser could not be initialized. + Parser(String), + /// Source has missing or erroneous syntax nodes. + InvalidSyntax { + /// Explicit grammar used for validation. + language: Language, + }, + /// The source differs from the caller's snapshot. + StaleRevision { + /// Caller-provided revision. + expected: String, + /// Revision of the supplied source. + actual: String, + }, + /// No symbol matched the selector. + SymbolNotFound(String), + /// More than one symbol can match, or the outline was truncated. + AmbiguousSymbol(String), + /// The selected grammar or node does not support this operation. + UnsupportedOperation(String), + /// No catalog entry has the requested ID. + UnknownChunk(String), +} + +impl fmt::Display for CodeError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::LimitExceeded { + resource, + limit, + actual, + } => write!(f, "{resource} exceeds limit {limit}: {actual}"), + Self::InvalidInput(message) => write!(f, "invalid input: {message}"), + Self::Parser(message) => write!(f, "parser initialization failed: {message}"), + Self::InvalidSyntax { language } => write!(f, "source does not parse as {language:?}"), + Self::StaleRevision { expected, actual } => { + write!(f, "stale revision: expected {expected}, current {actual}") + } + Self::SymbolNotFound(name) => write!(f, "symbol not found: {name}"), + Self::AmbiguousSymbol(name) => write!( + f, + "symbol selector is not provably unique: {name}; use an exact revision-addressed ID" + ), + Self::UnsupportedOperation(message) => write!(f, "unsupported operation: {message}"), + Self::UnknownChunk(id) => write!(f, "unknown chunk: {id}"), + } + } +} + +impl std::error::Error for CodeError {} + +pub(crate) fn enforce_limit( + resource: &'static str, + actual: usize, + limit: usize, +) -> Result<(), CodeError> { + if actual > limit { + Err(CodeError::LimitExceeded { + resource, + actual, + limit, + }) + } else { + Ok(()) + } +} + +pub(crate) fn invalid(message: impl Into) -> CodeError { + CodeError::InvalidInput(message.into()) +} diff --git a/crates/tinytools-code/src/language/mod.rs b/crates/tinytools-code/src/language/mod.rs new file mode 100644 index 00000000..528df4cd --- /dev/null +++ b/crates/tinytools-code/src/language/mod.rs @@ -0,0 +1,71 @@ +//! Explicit compiled grammars and fixed query packs; no language auto-detection. + +use serde::{Deserialize, Serialize}; + +/// A supported, explicitly selected grammar. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Language { + /// Rust source. + Rust, + /// TypeScript without JSX. + #[serde(rename = "typescript", alias = "ts")] + TypeScript, + /// TypeScript with JSX. + Tsx, + /// JavaScript, including JSX. + #[serde(rename = "javascript", alias = "js")] + JavaScript, + /// Python source. + Python, + /// Go source. + Go, + /// JSON data. + Json, + /// TOML data. + Toml, +} + +impl Language { + /// Infer a supported grammar from a conventional filename extension. + #[must_use] + pub fn from_path(path: &str) -> Option { + let extension = path.rsplit_once('.')?.1; + match extension.to_ascii_lowercase().as_str() { + "rs" => Some(Self::Rust), + "ts" | "mts" | "cts" => Some(Self::TypeScript), + "tsx" => Some(Self::Tsx), + "js" | "jsx" | "mjs" | "cjs" => Some(Self::JavaScript), + "py" | "pyi" => Some(Self::Python), + "go" => Some(Self::Go), + "json" => Some(Self::Json), + "toml" => Some(Self::Toml), + _ => None, + } + } + + pub(crate) fn grammar(self) -> tree_sitter::Language { + match self { + Self::Rust => tree_sitter_rust::LANGUAGE.into(), + Self::TypeScript => tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into(), + Self::Tsx => tree_sitter_typescript::LANGUAGE_TSX.into(), + Self::JavaScript => tree_sitter_javascript::LANGUAGE.into(), + Self::Python => tree_sitter_python::LANGUAGE.into(), + Self::Go => tree_sitter_go::LANGUAGE.into(), + Self::Json => tree_sitter_json::LANGUAGE.into(), + Self::Toml => tree_sitter_toml_ng::LANGUAGE.into(), + } + } + + pub(crate) fn query(self) -> &'static str { + match self { + Self::Rust => include_str!("queries/rust.scm"), + Self::TypeScript | Self::Tsx => include_str!("queries/typescript.scm"), + Self::JavaScript => include_str!("queries/javascript.scm"), + Self::Python => include_str!("queries/python.scm"), + Self::Go => include_str!("queries/go.scm"), + Self::Json => include_str!("queries/json.scm"), + Self::Toml => include_str!("queries/toml.scm"), + } + } +} diff --git a/crates/tinytools-code/src/language/queries/go.scm b/crates/tinytools-code/src/language/queries/go.scm new file mode 100644 index 00000000..ff09c662 --- /dev/null +++ b/crates/tinytools-code/src/language/queries/go.scm @@ -0,0 +1,4 @@ +[(function_declaration) (method_declaration) (type_spec) (var_spec) + (const_spec) (field_declaration)] @symbol +(import_declaration) @import +(call_expression) @call diff --git a/crates/tinytools-code/src/language/queries/javascript.scm b/crates/tinytools-code/src/language/queries/javascript.scm new file mode 100644 index 00000000..2c0a0330 --- /dev/null +++ b/crates/tinytools-code/src/language/queries/javascript.scm @@ -0,0 +1,4 @@ +[(function_declaration) (generator_function_declaration) (method_definition) + (class_declaration) (variable_declarator)] @symbol +(import_statement) @import +(call_expression) @call diff --git a/crates/tinytools-code/src/language/queries/json.scm b/crates/tinytools-code/src/language/queries/json.scm new file mode 100644 index 00000000..3cd3500a --- /dev/null +++ b/crates/tinytools-code/src/language/queries/json.scm @@ -0,0 +1 @@ +(pair) @symbol diff --git a/crates/tinytools-code/src/language/queries/python.scm b/crates/tinytools-code/src/language/queries/python.scm new file mode 100644 index 00000000..b26d9aac --- /dev/null +++ b/crates/tinytools-code/src/language/queries/python.scm @@ -0,0 +1,3 @@ +[(function_definition) (class_definition)] @symbol +[(import_statement) (import_from_statement) (future_import_statement)] @import +(call) @call diff --git a/crates/tinytools-code/src/language/queries/rust.scm b/crates/tinytools-code/src/language/queries/rust.scm new file mode 100644 index 00000000..ac6d8ab6 --- /dev/null +++ b/crates/tinytools-code/src/language/queries/rust.scm @@ -0,0 +1,5 @@ +[(function_item) (function_signature_item) (struct_item) (enum_item) + (trait_item) (impl_item) (mod_item) (type_item) (const_item) (static_item) + (field_declaration) (enum_variant)] @symbol +(use_declaration) @import +(call_expression) @call diff --git a/crates/tinytools-code/src/language/queries/toml.scm b/crates/tinytools-code/src/language/queries/toml.scm new file mode 100644 index 00000000..ad71cfa4 --- /dev/null +++ b/crates/tinytools-code/src/language/queries/toml.scm @@ -0,0 +1 @@ +[(table) (table_array_element) (pair)] @symbol diff --git a/crates/tinytools-code/src/language/queries/typescript.scm b/crates/tinytools-code/src/language/queries/typescript.scm new file mode 100644 index 00000000..51074287 --- /dev/null +++ b/crates/tinytools-code/src/language/queries/typescript.scm @@ -0,0 +1,6 @@ +[(function_declaration) (generator_function_declaration) (method_definition) + (class_declaration) (interface_declaration) (type_alias_declaration) + (enum_declaration) (variable_declarator) (public_field_definition) + (method_signature) (property_signature)] @symbol +(import_statement) @import +(call_expression) @call diff --git a/crates/tinytools-code/src/lib.rs b/crates/tinytools-code/src/lib.rs new file mode 100644 index 00000000..9914f9b8 --- /dev/null +++ b/crates/tinytools-code/src/lib.rs @@ -0,0 +1,33 @@ +//! Pure, bounded structural code tools. +//! +//! [`CodeEngine`] parses explicitly selected grammars, produces lexical outlines +//! and applies revision-checked edits. [`ChunkCatalog`] and [`render_chunk`] +//! provide typed, one-pass boilerplate. Filesystem access and host permissions +//! belong to adapters: this crate performs no I/O or semantic typechecking. +//! +//! ``` +//! use tinytools_code::{CodeEngine, Language}; +//! let mut engine = CodeEngine::default(); +//! let report = engine.analyze("example.rs", Language::Rust, "fn greet() {}")?; +//! assert_eq!(report.symbols[0].name, "greet"); +//! # Ok::<(), tinytools_code::CodeError>(()) +//! ``` + +mod analysis; +mod catalog; +mod engine; +mod error; +mod language; +mod patch; +mod render; + +pub use analysis::{Analysis, Diagnostic, Reference, SourceRange, Symbol, select_symbol}; +pub use catalog::{ + ChunkCatalog, ChunkManifest, ChunkParameter, ChunkSlot, ChunkSummary, ChunkTarget, + ParameterKind, +}; +pub use engine::{CacheStats, CacheStatus, CodeEngine, CodeLimits, revision}; +pub use error::CodeError; +pub use language::Language; +pub use patch::{PatchOperation, PatchRequest, PatchResult}; +pub use render::{ChunkBindings, RenderedChunk, render_chunk}; diff --git a/crates/tinytools-code/src/patch/README.md b/crates/tinytools-code/src/patch/README.md new file mode 100644 index 00000000..95f5a080 --- /dev/null +++ b/crates/tinytools-code/src/patch/README.md @@ -0,0 +1,3 @@ +# Targeted edits + +Expected revisions are mandatory. Every operation validates original and resulting syntax and bounds output before construction. Body and member edits preserve adjacent source. Import placement honors module prologues and attached docs. Syntax acceptance is not a guarantee of typing or semantic equivalence. diff --git a/crates/tinytools-code/src/patch/imports.rs b/crates/tinytools-code/src/patch/imports.rs new file mode 100644 index 00000000..bcebd5c2 --- /dev/null +++ b/crates/tinytools-code/src/patch/imports.rs @@ -0,0 +1,126 @@ +//! Import insertion preserves module prologues and attached outer documentation. + +use crate::{ + CodeEngine, CodeError, Language, + analysis::{is_docstring, node_text}, + error::invalid, +}; +use tree_sitter::{Node, Tree}; + +fn import_kind(kind: &str) -> bool { + matches!( + kind, + "use_declaration" + | "import_statement" + | "import_from_statement" + | "future_import_statement" + | "import_declaration" + ) +} + +fn candidate(engine: &CodeEngine, language: Language, code: &str) -> Result { + let candidate = if language == Language::Go { + format!("package chunk\n{code}") + } else { + code.to_owned() + }; + let tree = engine.parse(language, &candidate)?; + if tree.root_node().has_error() { + return Err(CodeError::InvalidSyntax { language }); + } + let nodes: Vec<_> = (0..tree.root_node().named_child_count()) + .filter_map(|index| tree.root_node().named_child(index)) + .filter(|node| node.kind() != "package_clause") + .collect(); + if nodes.len() != 1 || !import_kind(nodes[0].kind()) { + return Err(invalid( + "insert_import requires exactly one complete import declaration", + )); + } + Ok(tree) +} + +fn attached_doc(language: Language, text: &str) -> bool { + match language { + Language::Rust => { + (text.starts_with("///") && !text.starts_with("////")) + || (text.starts_with("/**") && !text.starts_with("/***")) + } + Language::JavaScript | Language::TypeScript | Language::Tsx => text.starts_with("/**"), + _ => false, + } +} + +fn directive(node: Node<'_>) -> bool { + node.kind() == "expression_statement" + && node + .named_child(0) + .is_some_and(|value| value.kind() == "string") +} + +pub(super) fn insert( + engine: &CodeEngine, + tree: &Tree, + language: Language, + source: &str, + code: &str, +) -> Result<(usize, usize, String), CodeError> { + if matches!(language, Language::Json | Language::Toml) { + return Err(CodeError::UnsupportedOperation( + "data formats do not have imports".to_owned(), + )); + } + let candidate = candidate(engine, language, code)?; + let future = candidate + .root_node() + .named_child(0) + .is_some_and(|node| node.kind() == "future_import_statement"); + let mut offset = 0; + let mut python_statement_seen = false; + for index in 0..tree.root_node().named_child_count() { + let Some(node) = tree.root_node().named_child(index) else { + continue; + }; + let text = node_text(node, source); + if import_kind(node.kind()) && text.trim() == code.trim() { + return Ok((0, 0, String::new())); + } + if node.kind().contains("comment") + || node.kind() == "hash_bang_line" + || node.kind() == "shebang" + { + if attached_doc(language, text) { + break; + } + offset = node.end_byte(); + continue; + } + if language == Language::Python && !python_statement_seen && is_docstring(node, source) { + python_statement_seen = true; + offset = node.end_byte(); + continue; + } + python_statement_seen = true; + if import_kind(node.kind()) { + if future && node.kind() != "future_import_statement" { + break; + } + offset = node.end_byte(); + } else if matches!(node.kind(), "package_clause" | "inner_attribute_item") + || (matches!( + language, + Language::JavaScript | Language::TypeScript | Language::Tsx + ) && directive(node)) + { + offset = node.end_byte(); + } else { + break; + } + } + let prefix = if offset > 0 && !source[..offset].ends_with('\n') { + "\n" + } else { + "" + }; + Ok((offset, offset, format!("{prefix}{}\n", code.trim()))) +} diff --git a/crates/tinytools-code/src/patch/indentation.rs b/crates/tinytools-code/src/patch/indentation.rs new file mode 100644 index 00000000..9778c694 --- /dev/null +++ b/crates/tinytools-code/src/patch/indentation.rs @@ -0,0 +1,176 @@ +//! Bounded indentation and container edits preserve bytes outside the edit. + +use crate::{ + CodeError, Language, + error::{enforce_limit, invalid}, +}; +use tree_sitter::Node; + +pub(super) fn line_indent(source: &str, offset: usize) -> &str { + let start = source[..offset] + .rfind('\n') + .map_or(0, |position| position + 1); + let prefix = &source[start..offset]; + let end = prefix + .find(|value: char| value != ' ' && value != '\t') + .unwrap_or(prefix.len()); + &prefix[..end] +} + +pub(super) fn indented(code: &str, indent: &str, limit: usize) -> Result { + let mut output = String::new(); + for (index, line) in code.trim().lines().enumerate() { + let added = line + .len() + .saturating_add(if line.is_empty() { 0 } else { indent.len() }) + .saturating_add(usize::from(index > 0)); + enforce_limit("indented bytes", output.len().saturating_add(added), limit)?; + if index > 0 { + output.push('\n'); + } + if !line.is_empty() { + output.push_str(indent); + } + output.push_str(line); + } + Ok(output) +} + +pub(super) fn braces(node: Node<'_>) -> Option<(usize, usize)> { + let mut open = None; + let mut close = None; + for index in 0..node.child_count() { + if let Some(child) = node.child(index) { + if child.kind() == "{" { + open = Some(child.end_byte()); + } + if child.kind() == "}" { + close = Some(child.start_byte()); + } + } + } + open.zip(close) +} + +pub(super) fn replace_body( + body: Node<'_>, + source: &str, + language: Language, + code: &str, + limit: usize, +) -> Result<(usize, usize, String), CodeError> { + let outer = line_indent(source, body.start_byte()); + if language == Language::Python { + let line_start = source[..body.start_byte()] + .rfind('\n') + .map_or(0, |position| position + 1); + let inline = !source[line_start..body.start_byte()].trim().is_empty(); + let indent = if inline { + format!("{outer} ") + } else { + outer.to_owned() + }; + let replacement = indented( + if code.trim().is_empty() { "pass" } else { code }, + &indent, + limit, + )?; + return if inline { + Ok(( + body.start_byte(), + body.end_byte(), + format!("\n{replacement}"), + )) + } else { + Ok((line_start, body.end_byte(), replacement)) + }; + } + let (start, end) = braces(body).ok_or_else(|| { + CodeError::UnsupportedOperation("body has no brace-delimited block".to_owned()) + })?; + let child_indent = format!("{outer} "); + let replacement = indented(code, &child_indent, limit)?; + Ok((start, end, format!("\n{replacement}\n{outer}"))) +} + +fn last_member(body: Node<'_>) -> Option> { + (0..body.named_child_count()) + .rev() + .filter_map(|index| body.named_child(index)) + .find(|node| !node.kind().contains("comment")) +} + +pub(super) fn append_member( + body: Node<'_>, + source: &str, + language: Language, + code: &str, + limit: usize, +) -> Result<(usize, usize, String), CodeError> { + if code.trim().is_empty() { + return Err(invalid("member source must not be empty")); + } + if language == Language::Python { + let original = &source[body.byte_range()]; + let outer = line_indent(source, body.start_byte()); + let line_start = source[..body.start_byte()] + .rfind('\n') + .map_or(0, |position| position + 1); + let inline = !source[line_start..body.start_byte()].trim().is_empty(); + if inline { + let indent = format!("{outer} "); + let original = indented(original, &indent, limit)?; + let added = indented(code, &indent, limit)?; + return Ok(( + body.start_byte(), + body.end_byte(), + format!("\n{original}\n{added}"), + )); + } + let added = indented(code, outer, limit)?; + return Ok((body.end_byte(), body.end_byte(), format!("\n{added}"))); + } + let (_, close) = braces(body).ok_or_else(|| { + CodeError::UnsupportedOperation("selected node has no member container".to_owned()) + })?; + let outer = line_indent(source, body.start_byte()); + let indent = format!("{outer} "); + let added = indented(code, &indent, limit)?; + let line_start = source[..close] + .rfind('\n') + .map_or(0, |position| position + 1); + let inline_close = !source[line_start..close].trim().is_empty(); + let insertion = if inline_close { close } else { line_start }; + let comma = language == Language::Rust + && matches!(body.kind(), "field_declaration_list" | "enum_variant_list"); + let added_comma = if comma && !code.trim_end().ends_with(',') { + "," + } else { + "" + }; + // The untouched closing line already contains its indentation when the + // insertion begins at that line's start. + let closing_indent = if inline_close { outer } else { "" }; + if comma + && let Some(last) = last_member(body) + && !source[last.end_byte()..close].trim_start().starts_with(',') + { + let suffix = &source[last.end_byte()..insertion]; + let separator = if suffix.ends_with('\n') { "" } else { "\n" }; + return Ok(( + last.end_byte(), + insertion, + format!(",{suffix}{separator}{added}{added_comma}\n{closing_indent}"), + )); + } + let separator = if insertion > 0 && source.as_bytes()[insertion - 1] == b'\n' { + "" + } else { + "\n" + }; + Ok(( + insertion, + insertion, + format!("{separator}{added}{added_comma}\n{closing_indent}"), + )) +} diff --git a/crates/tinytools-code/src/patch/mod.rs b/crates/tinytools-code/src/patch/mod.rs new file mode 100644 index 00000000..fab61312 --- /dev/null +++ b/crates/tinytools-code/src/patch/mod.rs @@ -0,0 +1,118 @@ +//! Revision-checked targeted syntax edits; no filesystem publication occurs here. + +mod imports; +mod indentation; +mod types; + +use crate::{ + CodeEngine, CodeError, Language, + analysis::body_node, + error::{enforce_limit, invalid}, + revision, select_symbol, +}; +pub use types::{PatchOperation, PatchRequest, PatchResult}; + +impl CodeEngine { + /// Prepare a targeted edit against an exact source revision. + /// + /// # Errors + /// Refuses stale revisions, invalid original/result syntax, ambiguous or + /// missing symbols, unsupported containers and exceeded resource budgets. + pub fn patch( + &mut self, + key: &str, + language: Language, + source: &str, + request: &PatchRequest, + ) -> Result { + let limit = self.limits().max_source_bytes; + enforce_limit("source bytes", source.len(), limit)?; + enforce_limit("patch bytes", request.code.len(), limit)?; + enforce_limit("revision bytes", request.expected_revision.len(), 128)?; + let before_revision = revision(source); + if request.expected_revision != before_revision { + return Err(CodeError::StaleRevision { + expected: request.expected_revision.clone(), + actual: before_revision, + }); + } + let tree = self.parse(language, source)?; + if tree.root_node().has_error() { + return Err(CodeError::InvalidSyntax { language }); + } + let (start, end, replacement) = if request.operation == PatchOperation::InsertImport { + imports::insert(self, &tree, language, source, &request.code)? + } else { + let report = self.analyze(key, language, source)?; + let selector = request + .selector + .as_deref() + .ok_or_else(|| invalid("this operation requires a symbol selector"))?; + let symbol = select_symbol(&report, selector)?; + let node = tree + .root_node() + .descendant_for_byte_range(symbol.range.start_byte, symbol.range.end_byte) + .ok_or_else(|| CodeError::SymbolNotFound(selector.to_owned()))?; + match request.operation { + PatchOperation::ReplaceSymbol => ( + symbol.range.start_byte, + symbol.range.end_byte, + request.code.clone(), + ), + PatchOperation::Wrap => { + let (prefix, suffix) = request + .code + .split_once("{{node}}") + .ok_or_else(|| invalid("wrap requires exactly one {{node}} marker"))?; + if suffix.contains("{{node}}") { + return Err(invalid("wrap requires exactly one {{node}} marker")); + } + let original = &source[symbol.range.start_byte..symbol.range.end_byte]; + enforce_limit( + "wrapped bytes", + prefix + .len() + .saturating_add(original.len()) + .saturating_add(suffix.len()), + limit, + )?; + ( + symbol.range.start_byte, + symbol.range.end_byte, + format!("{prefix}{original}{suffix}"), + ) + } + PatchOperation::ReplaceBody | PatchOperation::AppendMember => { + let body = body_node(node).ok_or_else(|| { + CodeError::UnsupportedOperation("selected symbol has no body".to_owned()) + })?; + if request.operation == PatchOperation::ReplaceBody { + indentation::replace_body(body, source, language, &request.code, limit)? + } else { + indentation::append_member(body, source, language, &request.code, limit)? + } + } + PatchOperation::InsertImport => { + return Err(invalid("import selection is not a symbol operation")); + } + } + }; + let result_bytes = source + .len() + .saturating_sub(end - start) + .saturating_add(replacement.len()); + enforce_limit("patched bytes", result_bytes, limit)?; + let mut result = String::with_capacity(result_bytes); + result.push_str(&source[..start]); + result.push_str(&replacement); + result.push_str(&source[end..]); + self.validate(language, &result)?; + let after_revision = revision(&result); + Ok(PatchResult { + changed: before_revision != after_revision, + before_revision, + after_revision, + source: result, + }) + } +} diff --git a/crates/tinytools-code/src/patch/types.rs b/crates/tinytools-code/src/patch/types.rs new file mode 100644 index 00000000..f49ce54e --- /dev/null +++ b/crates/tinytools-code/src/patch/types.rs @@ -0,0 +1,46 @@ +//! Revision-bound edit request and result contracts. + +use serde::{Deserialize, Serialize}; + +/// A targeted syntactic operation. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PatchOperation { + /// Replace a declaration body, preserving its delimiters and signature. + ReplaceBody, + /// Replace the selected declaration in full. + ReplaceSymbol, + /// Insert exactly one import into the module prologue. + InsertImport, + /// Append a member to a supported declaration container. + AppendMember, + /// Expand exactly one `{{node}}` marker around the selected declaration. + Wrap, +} + +/// Every patch requires the caller's exact source revision. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PatchRequest { + /// Expected content revision. + pub expected_revision: String, + /// Exact ID, qualified name or unambiguous name; unused for imports. + pub selector: Option, + /// Syntactic edit operation. + pub operation: PatchOperation, + /// Replacement source or one-pass wrapping template. + pub code: String, +} + +/// A pure patch result; an adapter decides whether and how to persist it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PatchResult { + /// Original source revision. + pub before_revision: String, + /// Resulting source revision. + pub after_revision: String, + /// Entire resulting source. + pub source: String, + /// Whether any source byte changed. + pub changed: bool, +} diff --git a/crates/tinytools-code/src/render/README.md b/crates/tinytools-code/src/render/README.md new file mode 100644 index 00000000..779a1ebd --- /dev/null +++ b/crates/tinytools-code/src/render/README.md @@ -0,0 +1,3 @@ +# Typed rendering + +Parameters expand only original markers. Code slots must be original standalone grammar comments. Inserted strings are not rescanned. Type expressions must occupy exactly one syntax node in an explicit context. Generated-node constraints exclude artificial wrapper nodes. Every append, including indentation, is bounded. diff --git a/crates/tinytools-code/src/render/context.rs b/crates/tinytools-code/src/render/context.rs new file mode 100644 index 00000000..519ed6b5 --- /dev/null +++ b/crates/tinytools-code/src/render/context.rs @@ -0,0 +1,113 @@ +//! Synthetic parse contexts expose only generated source to constraints. + +use crate::{ChunkTarget, CodeEngine, CodeError, Language, error::enforce_limit}; +use tree_sitter::Tree; + +pub(super) struct Context { + pub tree: Tree, + pub prefix: usize, + pub indentation: usize, +} + +impl Context { + pub(super) fn offset(&self, source: &str, byte: usize) -> usize { + self.prefix + + byte + + self.indentation * source[..byte].bytes().filter(|byte| *byte == b'\n').count() + } +} + +fn wrapped( + engine: &CodeEngine, + language: Language, + source: &str, + prefix: &str, + suffix: &str, + indentation: usize, +) -> Result { + let newlines = source.bytes().filter(|byte| *byte == b'\n').count(); + let context_bytes = prefix + .len() + .saturating_add(source.len()) + .saturating_add(indentation.saturating_mul(newlines)) + .saturating_add(suffix.len()); + enforce_limit( + "parse context bytes", + context_bytes, + engine.limits().max_source_bytes, + )?; + let source = if indentation == 0 { + source.to_owned() + } else { + source.replace('\n', &format!("\n{}", " ".repeat(indentation))) + }; + let tree = engine.parse(language, &format!("{prefix}{source}{suffix}"))?; + if tree.root_node().has_error() { + return Err(CodeError::InvalidSyntax { language }); + } + Ok(Context { + tree, + prefix: prefix.len(), + indentation, + }) +} + +pub(super) fn parse( + engine: &CodeEngine, + language: Language, + target: ChunkTarget, + source: &str, +) -> Result { + match (language, target) { + (_, ChunkTarget::File) => wrapped(engine, language, source, "", "", 0), + (Language::Rust, ChunkTarget::FunctionBody) => { + wrapped(engine, language, source, "fn __chunk() {\n", "\n}", 0) + } + (Language::Rust, ChunkTarget::Member) => { + wrapped(engine, language, source, "struct __Chunk {\n", "\n}", 0) + .or_else(|_| wrapped(engine, language, source, "impl __Chunk {\n", "\n}", 0)) + } + (Language::Python, ChunkTarget::FunctionBody) => { + wrapped(engine, language, source, "def __chunk():\n ", "\n", 4) + } + (Language::Python, ChunkTarget::Member) => { + wrapped(engine, language, source, "class __Chunk:\n ", "\n", 4) + } + (Language::Go, ChunkTarget::FunctionBody) => wrapped( + engine, + language, + source, + "package chunk\nfunc __chunk() {\n", + "\n}\n", + 0, + ), + (Language::Go, ChunkTarget::Member) => wrapped( + engine, + language, + source, + "package chunk\ntype __Chunk struct {\n", + "\n}\n", + 0, + ) + .or_else(|_| { + wrapped( + engine, + language, + source, + "package chunk\ntype __Chunk interface {\n", + "\n}\n", + 0, + ) + }), + ( + Language::JavaScript | Language::TypeScript | Language::Tsx, + ChunkTarget::FunctionBody, + ) => wrapped(engine, language, source, "function __chunk() {\n", "\n}", 0), + (Language::JavaScript | Language::TypeScript | Language::Tsx, ChunkTarget::Member) => { + wrapped(engine, language, source, "class __Chunk {\n", "\n}", 0) + } + _ => Err(CodeError::UnsupportedOperation( + "data formats only support file chunks".to_owned(), + )), + } +} diff --git a/crates/tinytools-code/src/render/mod.rs b/crates/tinytools-code/src/render/mod.rs new file mode 100644 index 00000000..47a834a1 --- /dev/null +++ b/crates/tinytools-code/src/render/mod.rs @@ -0,0 +1,95 @@ +//! One-pass typed rendering followed by syntax and generated-node validation. + +mod context; +mod parameters; +mod slots; +mod types; + +use crate::{ + ChunkManifest, CodeEngine, CodeError, + error::{enforce_limit, invalid}, + revision, +}; +pub use types::{ChunkBindings, RenderedChunk}; + +/// Render typed parameters and original standalone slots, then validate syntax. +/// +/// # Errors +/// Refuses invalid metadata, unknown/missing/unused bindings, unsafe type +/// expressions, malformed slots, generated-node violations and resource excess. +pub fn render_chunk( + engine: &mut CodeEngine, + manifest: &ChunkManifest, + template: &str, + bindings: &ChunkBindings, +) -> Result { + manifest.validate()?; + enforce_limit( + "template bytes", + template.len(), + engine.limits().max_source_bytes, + )?; + let expanded = parameters::expand(engine, manifest, template, bindings)?; + let (source, ranges) = slots::expand(engine, manifest, &expanded, bindings)?; + let parsed = context::parse(engine, manifest.language, manifest.target, &source)?; + for range in ranges { + let Some(kind) = manifest + .slots + .iter() + .find(|slot| slot.name == range.name) + .and_then(|slot| slot.node_kind.as_ref()) + else { + continue; + }; + let snippet = &source[range.start..range.end]; + let start = range.start + snippet.len() - snippet.trim_start().len(); + let end = range.end - snippet.len() + snippet.trim_end().len(); + if start == end { + continue; + } + let start = parsed.offset(&source, start); + let end = parsed.offset(&source, end); + let exact = parsed + .tree + .root_node() + .descendant_for_byte_range(start, end); + if !exact.is_some_and(|node| { + node.start_byte() == start && node.end_byte() == end && node.kind() == kind + }) { + return Err(invalid(format!( + "slot {} must produce exactly one {kind} node", + range.name + ))); + } + } + let generated_start = parsed.offset(&source, 0); + let generated_end = parsed.offset(&source, source.len()); + let mut stack = vec![parsed.tree.root_node()]; + while let Some(node) = stack.pop() { + if node.start_byte() >= generated_start + && node.end_byte() <= generated_end + && manifest + .forbidden_nodes + .iter() + .any(|kind| kind == node.kind()) + { + return Err(invalid(format!( + "generated source contains forbidden node kind: {}", + node.kind() + ))); + } + for index in 0..node.named_child_count() { + if let Some(child) = node.named_child(index) { + stack.push(child); + } + } + } + Ok(RenderedChunk { + id: manifest.id.clone(), + language: manifest.language, + target: manifest.target, + revision: revision(&source), + source, + dependencies: manifest.dependencies.clone(), + }) +} diff --git a/crates/tinytools-code/src/render/parameters.rs b/crates/tinytools-code/src/render/parameters.rs new file mode 100644 index 00000000..db2f1e08 --- /dev/null +++ b/crates/tinytools-code/src/render/parameters.rs @@ -0,0 +1,188 @@ +//! Typed literal escaping and exact type-expression boundaries. + +use crate::{ + ChunkBindings, ChunkManifest, CodeEngine, CodeError, Language, ParameterKind, + catalog::identifier, + error::{enforce_limit, invalid}, +}; +use serde_json::Value; +use std::collections::{BTreeMap, BTreeSet}; + +pub(super) struct Expanded { + pub source: String, + pub protected: Vec<(usize, usize)>, +} + +pub(super) fn append(output: &mut String, text: &str, limit: usize) -> Result<(), CodeError> { + enforce_limit( + "rendered bytes", + output.len().saturating_add(text.len()), + limit, + )?; + output.push_str(text); + Ok(()) +} + +fn type_expression( + engine: &CodeEngine, + language: Language, + value: &str, +) -> Result { + let value = value.trim(); + if value.is_empty() { + return Err(invalid("type expression is empty")); + } + let (prefix, suffix) = match language { + Language::Rust | Language::TypeScript | Language::Tsx => ("type __Chunk = ", ";"), + Language::Python => ("value: ", "\n"), + Language::Go => ("package chunk\ntype __Chunk ", "\n"), + _ => { + return Err(CodeError::UnsupportedOperation( + "this grammar has no typed parameter context".to_owned(), + )); + } + }; + let source = format!("{prefix}{value}{suffix}"); + let tree = engine.parse(language, &source)?; + if tree.root_node().has_error() { + return Err(CodeError::InvalidSyntax { language }); + } + let exact = tree + .root_node() + .descendant_for_byte_range(prefix.len(), prefix.len() + value.len()); + if !exact.is_some_and(|node| { + node.start_byte() == prefix.len() && node.end_byte() == prefix.len() + value.len() + }) { + return Err(invalid( + "type expression must occupy exactly one syntax node", + )); + } + Ok(value.to_owned()) +} + +fn value( + engine: &CodeEngine, + language: Language, + kind: ParameterKind, + value: &Value, +) -> Result { + match kind { + ParameterKind::Number => value + .as_number() + .map(ToString::to_string) + .ok_or_else(|| invalid("number parameter needs a JSON number")), + ParameterKind::Boolean => value + .as_bool() + .map(|value| { + if language == Language::Python { + if value { "True" } else { "False" } + } else if value { + "true" + } else { + "false" + } + .to_owned() + }) + .ok_or_else(|| invalid("boolean parameter needs a JSON boolean")), + _ => { + let value = value + .as_str() + .ok_or_else(|| invalid("text parameter needs a JSON string"))?; + enforce_limit("parameter bytes", value.len(), 4096)?; + match kind { + ParameterKind::Identifier | ParameterKind::PascalCaseIdent => { + if !identifier(value) + || (kind == ParameterKind::PascalCaseIdent + && !value.as_bytes()[0].is_ascii_uppercase()) + { + return Err(invalid("identifier parameter violates its declared kind")); + } + Ok(value.to_owned()) + } + ParameterKind::TypeExpr => type_expression(engine, language, value), + ParameterKind::String => { + if language == Language::Rust { + Ok(format!("\"{}\"", value.escape_default())) + } else { + serde_json::to_string(value).map_err(|error| invalid(error.to_string())) + } + } + _ => Err(invalid("text value has a non-text parameter kind")), + } + } + } +} + +pub(super) fn expand( + engine: &CodeEngine, + manifest: &ChunkManifest, + template: &str, + bindings: &ChunkBindings, +) -> Result { + enforce_limit("parameter bindings", bindings.parameters.len(), 32)?; + enforce_limit("slot bindings", bindings.slots.len(), 32)?; + for name in bindings.parameters.keys().chain(bindings.slots.keys()) { + enforce_limit("binding name bytes", name.len(), 128)?; + } + for name in bindings.parameters.keys() { + if !manifest + .parameters + .iter() + .any(|parameter| parameter.name == *name) + { + return Err(invalid(format!("unknown parameter: {name}"))); + } + } + for name in bindings.slots.keys() { + if !manifest.slots.iter().any(|slot| slot.name == *name) { + return Err(invalid(format!("unknown slot: {name}"))); + } + } + let mut values = BTreeMap::new(); + for parameter in &manifest.parameters { + let supplied = bindings + .parameters + .get(¶meter.name) + .or(parameter.default.as_ref()); + if let Some(supplied) = supplied { + values.insert( + parameter.name.as_str(), + value(engine, manifest.language, parameter.kind, supplied)?, + ); + } else if parameter.required { + return Err(invalid(format!("missing parameter: {}", parameter.name))); + } + } + let limit = engine.limits().max_source_bytes; + let mut source = String::new(); + let mut protected = Vec::new(); + let mut used = BTreeSet::new(); + let mut remaining = template; + while let Some(start) = remaining.find("{{") { + append(&mut source, &remaining[..start], limit)?; + let marker = &remaining[start + 2..]; + let end = marker + .find("}}") + .ok_or_else(|| invalid("unterminated parameter marker"))?; + let name = &marker[..end]; + enforce_limit("parameter marker bytes", name.len(), 128)?; + let value = values + .get(name) + .ok_or_else(|| invalid(format!("missing or undeclared parameter: {name}")))?; + let begin = source.len(); + append(&mut source, value, limit)?; + enforce_limit("parameter occurrences", protected.len() + 1, 16_384)?; + protected.push((begin, source.len())); + used.insert(name); + remaining = &marker[end + 2..]; + } + append(&mut source, remaining, limit)?; + if manifest + .parameters + .iter() + .any(|parameter| !used.contains(parameter.name.as_str())) + { + return Err(invalid("declared parameter is unused by the template")); + } + Ok(Expanded { source, protected }) +} diff --git a/crates/tinytools-code/src/render/slots.rs b/crates/tinytools-code/src/render/slots.rs new file mode 100644 index 00000000..a0d5dfe2 --- /dev/null +++ b/crates/tinytools-code/src/render/slots.rs @@ -0,0 +1,132 @@ +//! Only original, standalone syntax comments can introduce code slots. + +use super::parameters::{Expanded, append}; +use crate::{ + ChunkBindings, ChunkManifest, CodeEngine, CodeError, + analysis::node_text, + error::{enforce_limit, invalid}, +}; +use std::collections::BTreeSet; + +pub(super) struct SlotRange { + pub name: String, + pub start: usize, + pub end: usize, +} + +fn marker(text: &str) -> Option<&str> { + let text = text.trim(); + let text = text + .strip_prefix("//") + .or_else(|| text.strip_prefix('#')) + .or_else(|| { + text.strip_prefix("/*") + .and_then(|text| text.strip_suffix("*/")) + })? + .trim(); + let name = text.strip_prefix("@slot ")?.trim(); + (!name.is_empty() && !name.chars().any(char::is_whitespace)).then_some(name) +} + +fn indent_binding(value: &str, indentation: &str, limit: usize) -> Result { + let mut output = String::new(); + for (index, line) in value.trim_end().lines().enumerate() { + let added = line + .len() + .saturating_add(if line.is_empty() { + 0 + } else { + indentation.len() + }) + .saturating_add(usize::from(index > 0)); + enforce_limit( + "indented slot bytes", + output.len().saturating_add(added), + limit, + )?; + if index > 0 { + output.push('\n'); + } + if !line.is_empty() { + output.push_str(indentation); + } + output.push_str(line); + } + Ok(output) +} + +pub(super) fn expand( + engine: &CodeEngine, + manifest: &ChunkManifest, + expanded: &Expanded, + bindings: &ChunkBindings, +) -> Result<(String, Vec), CodeError> { + let template = &expanded.source; + let tree = engine.parse(manifest.language, template)?; + let mut stack = vec![tree.root_node()]; + let mut slots = Vec::new(); + while let Some(node) = stack.pop() { + if node.kind().contains("comment") + && !expanded + .protected + .iter() + .any(|(start, end)| *start < node.end_byte() && *end > node.start_byte()) + && let Some(name) = marker(node_text(node, template)) + { + let start = template[..node.start_byte()] + .rfind('\n') + .map_or(0, |offset| offset + 1); + let end = template[node.end_byte()..] + .find('\n') + .map_or(template.len(), |offset| node.end_byte() + offset); + let indent = &template[start..node.start_byte()]; + if indent.trim().is_empty() && template[node.end_byte()..end].trim().is_empty() { + enforce_limit("slot name bytes", name.len(), 128)?; + enforce_limit("slot occurrences", slots.len() + 1, 32)?; + slots.push((start, node.end_byte(), name.to_owned(), indent.to_owned())); + } + } + for index in 0..node.named_child_count() { + if let Some(child) = node.named_child(index) { + stack.push(child); + } + } + } + slots.sort_by_key(|(start, _, _, _)| *start); + let mut used = BTreeSet::new(); + let mut output = String::new(); + let mut ranges = Vec::new(); + let mut copied = 0; + let limit = engine.limits().max_source_bytes; + for (start, end, name, indent) in slots { + if !used.insert(name.clone()) { + return Err(invalid("slot appears more than once")); + } + let slot = manifest + .slots + .iter() + .find(|slot| slot.name == name) + .ok_or_else(|| invalid(format!("undeclared slot: {name}")))?; + let value = bindings.slots.get(&name).map_or("", String::as_str); + if slot.required && value.trim().is_empty() { + return Err(invalid(format!("missing required slot: {name}"))); + } + enforce_limit("slot bytes", value.len(), limit)?; + append(&mut output, &template[copied..start], limit)?; + let begin = output.len(); + append(&mut output, &indent_binding(value, &indent, limit)?, limit)?; + ranges.push(SlotRange { + name, + start: begin, + end: output.len(), + }); + copied = end; + } + append(&mut output, &template[copied..], limit)?; + if manifest.slots.iter().any(|slot| !used.contains(&slot.name)) { + return Err(invalid( + "declared slot is not an original standalone comment", + )); + } + Ok((output, ranges)) +} diff --git a/crates/tinytools-code/src/render/types.rs b/crates/tinytools-code/src/render/types.rs new file mode 100644 index 00000000..58cf937b --- /dev/null +++ b/crates/tinytools-code/src/render/types.rs @@ -0,0 +1,35 @@ +//! Caller bindings and rendered source receipts. + +use crate::{ChunkTarget, Language}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::collections::BTreeMap; + +/// Caller-provided values; inserted text is never expanded recursively. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ChunkBindings { + /// Typed parameter values by declared name. + #[serde(default)] + pub parameters: BTreeMap, + /// Code supplied for declared slots. + #[serde(default)] + pub slots: BTreeMap, +} + +/// A validated rendering, without filesystem effects or dependency installation. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RenderedChunk { + /// Pattern ID. + pub id: String, + /// Grammar used. + pub language: Language, + /// Insertion context. + pub target: ChunkTarget, + /// Rendered code. + pub source: String, + /// Content revision of the rendered code. + pub revision: String, + /// Informational dependencies carried through from metadata. + pub dependencies: Vec, +} diff --git a/crates/tinytools-code/tests/analysis_tests.rs b/crates/tinytools-code/tests/analysis_tests.rs new file mode 100644 index 00000000..71e5fdb0 --- /dev/null +++ b/crates/tinytools-code/tests/analysis_tests.rs @@ -0,0 +1,346 @@ +//! Grammar, lexical selection, cache and diagnostic boundary regressions. + +use tinytools_code::{ + Analysis, CacheStatus, CodeEngine, CodeError, CodeLimits, Language, select_symbol, +}; + +#[test] +fn all_explicit_grammars_produce_lexical_outlines() -> Result<(), CodeError> { + for (language, source, name) in [ + ( + Language::Rust, + "use std::fmt; fn greet() { other(); }", + "greet", + ), + ( + Language::TypeScript, + "import { x } from 'x'; interface Item { value: string; } function greet(): void { x(); }", + "greet", + ), + ( + Language::Tsx, + "export function Action() { return ; }", + "Action", + ), + ( + Language::JavaScript, + "import x from 'x'; function greet() { x(); }", + "greet", + ), + ( + Language::Python, + "import os\ndef greet():\n print('hi')\n", + "greet", + ), + ( + Language::Go, + "package main\nimport \"fmt\"\nfunc greet() { fmt.Println(1) }", + "greet", + ), + (Language::Json, "{\"name\": \"value\"}", "name"), + (Language::Toml, "[service]\nname = 'value'\n", "name"), + ] { + let report = CodeEngine::default().analyze("file", language, source)?; + assert!( + report.diagnostics.is_empty(), + "{language:?}: {:?}", + report.diagnostics + ); + assert!( + report.symbols.iter().any(|symbol| symbol.name == name), + "{language:?}: {:?}", + report.symbols + ); + if matches!( + language, + Language::Rust + | Language::TypeScript + | Language::JavaScript + | Language::Python + | Language::Go + ) { + assert!( + report + .references + .iter() + .any(|reference| reference.kind == "call") + ); + assert!( + report + .references + .iter() + .any(|reference| reference.kind == "import") + ); + } + } + Ok(()) +} + +#[test] +fn symbol_selection_requires_unique_names_or_revision_ids() -> Result<(), CodeError> { + let source = "mod first { fn same() {} } mod second { fn same() {} }"; + let report = CodeEngine::default().analyze("file", Language::Rust, source)?; + assert!(matches!( + select_symbol(&report, "same"), + Err(CodeError::AmbiguousSymbol(_)) + )); + assert_eq!(select_symbol(&report, "first::same")?.name, "same"); + assert!(matches!( + select_symbol(&report, "missing"), + Err(CodeError::SymbolNotFound(_)) + )); + let symbol = select_symbol(&report, "second::same")?; + assert_eq!(select_symbol(&report, &symbol.id)?, symbol); + assert!(symbol.range.start_line > 0 && symbol.range.start_column > 0); + assert!(symbol.range.end_byte > symbol.range.start_byte); + Ok(()) +} + +#[test] +fn unicode_incremental_edits_match_fresh_parsing() -> Result<(), CodeError> { + let mut engine = CodeEngine::default(); + for (index, source) in [ + "fn café() { let value = \"é\"; }", + "fn café() {\n let value = \"ê\";\n}", + "fn tea() {}", + "", + "fn tea() {}", + ] + .iter() + .enumerate() + { + let mut report = engine.analyze("file", Language::Rust, source)?; + assert_eq!( + report.cache_status, + if index == 0 { + CacheStatus::Fresh + } else { + CacheStatus::Incremental + } + ); + report.cache_status = CacheStatus::Fresh; + assert_eq!( + report, + CodeEngine::default().analyze("file", Language::Rust, source)? + ); + } + assert_eq!( + engine + .analyze("file", Language::Rust, "fn tea() {}")? + .cache_status, + CacheStatus::Cached + ); + let stats = engine.cache_stats(); + assert_eq!( + (stats.entries, stats.hits, stats.incremental_parses), + (1, 1, 4) + ); + engine.clear_cache(); + assert_eq!( + ( + engine.cache_stats().entries, + engine.cache_stats().source_bytes + ), + (0, 0) + ); + Ok(()) +} + +#[test] +fn lru_and_zero_budgets_enforce_retained_source_bounds() -> Result<(), CodeError> { + let mut engine = CodeEngine::new(CodeLimits { + max_cache_entries: 2, + max_cache_bytes: 30, + ..CodeLimits::default() + })?; + for key in ["a", "b", "a", "c"] { + engine.analyze(key, Language::Rust, "fn run() {}")?; + } + assert_eq!(engine.cache_stats().evictions, 1); + assert_eq!( + engine + .analyze("b", Language::Rust, "fn run() {}")? + .cache_status, + CacheStatus::Fresh + ); + assert!(engine.cache_stats().source_bytes <= 30); + for limits in [ + CodeLimits { + max_cache_entries: 0, + ..CodeLimits::default() + }, + CodeLimits { + max_cache_bytes: 0, + ..CodeLimits::default() + }, + CodeLimits { + max_cache_bytes: 1, + ..CodeLimits::default() + }, + ] { + let mut engine = CodeEngine::new(limits)?; + for source in ["", "fn run() {}"] { + engine.analyze("file", Language::Rust, source)?; + if !source.is_empty() || limits.max_cache_bytes == 0 || limits.max_cache_entries == 0 { + assert_eq!( + engine.analyze("file", Language::Rust, source)?.cache_status, + CacheStatus::Fresh + ); + } + } + } + Ok(()) +} + +#[test] +fn record_caps_never_imply_name_uniqueness() -> Result<(), CodeError> { + let mut engine = CodeEngine::new(CodeLimits { + max_records: 1, + ..CodeLimits::default() + })?; + let report = engine.analyze( + "file", + Language::Rust, + "fn same() { one(); two(); } mod nested { fn same() {} }", + )?; + assert!(report.symbols_truncated && report.references_truncated); + assert_eq!((report.symbols.len(), report.references.len()), (1, 1)); + assert!(matches!( + select_symbol(&report, "same"), + Err(CodeError::AmbiguousSymbol(_)) + )); + assert_eq!(select_symbol(&report, &report.symbols[0].id)?.name, "same"); + Ok(()) +} + +#[test] +fn diagnostic_caps_and_invalid_source_are_explicit() -> Result<(), CodeError> { + let source = "fn run() { let = ; let = ; let = ; }"; + let mut engine = CodeEngine::new(CodeLimits { + max_diagnostics: 1, + ..CodeLimits::default() + })?; + let report = engine.analyze("file", Language::Rust, source)?; + assert_eq!(report.diagnostics.len(), 1); + assert!(report.diagnostics_truncated); + assert!(matches!( + engine.validate(Language::Rust, source), + Err(CodeError::InvalidSyntax { .. }) + )); + Ok(()) +} + +#[test] +fn engine_configuration_and_request_budgets_are_checked() -> Result<(), CodeError> { + for limits in [ + CodeLimits { + max_source_bytes: 0, + ..CodeLimits::default() + }, + CodeLimits { + max_records: 0, + ..CodeLimits::default() + }, + CodeLimits { + max_diagnostics: 0, + ..CodeLimits::default() + }, + CodeLimits { + max_source_bytes: 16_777_217, + ..CodeLimits::default() + }, + CodeLimits { + max_cache_entries: 1025, + ..CodeLimits::default() + }, + CodeLimits { + max_cache_bytes: 134_217_729, + ..CodeLimits::default() + }, + CodeLimits { + max_records: 65_537, + ..CodeLimits::default() + }, + CodeLimits { + max_diagnostics: 4097, + ..CodeLimits::default() + }, + ] { + assert!(CodeEngine::new(limits).is_err()); + } + let mut engine = CodeEngine::new(CodeLimits { + max_source_bytes: 8, + ..CodeLimits::default() + })?; + assert!(matches!( + engine.analyze("file", Language::Rust, "fn run() {}"), + Err(CodeError::LimitExceeded { .. }) + )); + assert!(matches!( + engine.analyze(&"x".repeat(4097), Language::Rust, ""), + Err(CodeError::LimitExceeded { .. }) + )); + Ok(()) +} + +#[test] +fn folded_views_preserve_docs_and_surrounding_source() -> Result<(), CodeError> { + let source = + "// license\n/// Docs\nfn run() {\n let first = 1;\n let second = 2;\n let third = 3;\n}\n"; + let report = CodeEngine::default().analyze("rust", Language::Rust, source)?; + assert!( + report + .skeleton + .starts_with("// license\n/// Docs\nfn run()") + ); + assert!(report.skeleton.contains("body folded")); + assert!(report.skeleton.len() < source.len()); + for literal in ["\"doc\"", "r\"doc\"", "\"\"\"long docs\"\"\""] { + let source = format!( + "def run():\n {literal}\n value = 123456\n other = 98765\n return value + other\n" + ); + let report = CodeEngine::default().analyze("python", Language::Python, &source)?; + assert!(report.skeleton.contains(literal)); + assert!(!report.skeleton.contains("98765")); + } + Ok(()) +} + +#[test] +fn language_hints_and_analysis_wire_form_round_trip() -> Result<(), Box> { + for (path, language) in [ + ("test.RS", Language::Rust), + ("a.mts", Language::TypeScript), + ("a.cts", Language::TypeScript), + ("a.tsx", Language::Tsx), + ("a.jsx", Language::JavaScript), + ("a.cjs", Language::JavaScript), + ("a.mjs", Language::JavaScript), + ("a.pyi", Language::Python), + ("a.go", Language::Go), + ("a.json", Language::Json), + ("a.toml", Language::Toml), + ] { + assert_eq!(Language::from_path(path), Some(language)); + } + assert_eq!(Language::from_path("file"), None); + assert_eq!(Language::from_path("file.unknown"), None); + assert_eq!( + serde_json::from_str::("\"ts\"")?, + Language::TypeScript + ); + assert_eq!( + serde_json::from_str::("\"js\"")?, + Language::JavaScript + ); + let report = CodeEngine::default().analyze( + "fixture", + Language::Rust, + include_str!("fixtures/analysis.rs"), + )?; + assert_eq!( + report, + serde_json::from_str::(&serde_json::to_string(&report)?)? + ); + Ok(()) +} diff --git a/crates/tinytools-code/tests/catalog_tests.rs b/crates/tinytools-code/tests/catalog_tests.rs new file mode 100644 index 00000000..d538a84c --- /dev/null +++ b/crates/tinytools-code/tests/catalog_tests.rs @@ -0,0 +1,153 @@ +//! Metadata discovery, portable custom manifests and catalog budget regressions. + +use serde_json::json; +use tinytools_code::{ChunkCatalog, ChunkManifest, CodeError, Language}; + +fn manifest() -> Result { + serde_json::from_value( + json!({"id":"custom.example.v1","summary":"An example","language":"rust","template_path":"templates/example.rs"}), + ) +} + +#[test] +fn builtin_catalog_is_metadata_only_and_deterministic() -> Result<(), Box> { + let catalog = ChunkCatalog::builtin()?; + let all = catalog.search("", None, 100); + assert_eq!(all.len(), 12); + assert_eq!(all, catalog.search("", None, 100)); + assert_eq!(catalog.search("", None, 0).len(), 0); + assert_eq!(catalog.search("nonsenseunmatched", None, 3).len(), 0); + assert!( + catalog + .search("fallible function", Some(Language::Rust), 2) + .iter() + .all(|entry| entry.language == Language::Rust) + ); + for summary in all { + let metadata = catalog.inspect(&summary.id)?; + assert_eq!(metadata, catalog.manifest(&summary.id)?); + assert!(catalog.builtin_source(&summary.id).is_some()); + assert_eq!(metadata.revision()?, metadata.revision()?); + let wire = serde_json::to_value(&summary)?; + assert!(wire.get("source").is_none()); + assert!(wire.get("template").is_none()); + } + Ok(()) +} + +#[test] +fn custom_entries_cannot_claim_embedded_source_or_overwrite_ids() +-> Result<(), Box> { + let mut catalog = ChunkCatalog::new(); + let mut custom = manifest()?; + custom.id = "rust.fallible-function.v1".to_owned(); + catalog.insert(custom.clone())?; + assert_eq!(catalog.builtin_source(&custom.id), None); + assert!(catalog.insert(custom).is_err()); + assert!(matches!( + catalog.inspect("missing"), + Err(CodeError::UnknownChunk(_)) + )); + assert!(matches!( + catalog.inspect(&"x".repeat(129)), + Err(CodeError::LimitExceeded { .. }) + )); + assert_eq!(catalog.builtin_source("missing"), None); + Ok(()) +} + +#[test] +fn unsafe_paths_unknown_fields_and_wrong_defaults_are_refused() +-> Result<(), Box> { + for path in [ + "", + "/absolute", + "../parent", + "a/../b", + "a/./b", + "a//b", + "C:/drive", + "a\\b", + "a/space ", + "a/dot.", + "a\0b", + ] { + let mut custom = manifest()?; + custom.template_path = path.to_owned(); + assert!(custom.validate().is_err(), "accepted {path:?}"); + } + let mut value = serde_json::to_value(manifest()?)?; + value["shell"] = json!("execute something"); + assert!(serde_json::from_value::(value).is_err()); + for (kind, default) in [ + ("identifier", json!("not valid")), + ("pascal_case_ident", json!("lower")), + ("type_expr", json!(5)), + ("string", json!(true)), + ("number", json!("5")), + ("boolean", json!(1)), + ] { + let mut value = serde_json::to_value(manifest()?)?; + value["parameters"] = json!([{"name":"value","kind":kind,"default":default}]); + assert!( + serde_json::from_value::(value)? + .validate() + .is_err() + ); + } + Ok(()) +} + +#[test] +fn manifest_lists_and_names_are_bounded_and_unique() -> Result<(), Box> { + for update in [ + json!({"id":""}), + json!({"id":"bad/id"}), + json!({"summary":" "}), + json!({"parameters":[{"name":"x","kind":"string"},{"name":"x","kind":"string"}]}), + json!({"slots":[{"name":"x"},{"name":"x"}]}), + json!({"slots":[{"name":"bad name"}]}), + json!({"slots":[{"name":"x","node_kind":"bad-kind"}]}), + json!({"forbidden_nodes":["invalid kind"]}), + json!({"tags":vec!["tag";33]}), + json!({"dependencies":vec!["dep";65]}), + json!({"parameters":(0..33).map(|index| json!({"name":format!("x{index}"),"kind":"string"})).collect::>()}), + json!({"slots":(0..33).map(|index| json!({"name":format!("x{index}")})).collect::>()}), + json!({"forbidden_nodes":vec!["call_expression";33]}), + json!({"summary":"x".repeat(1025)}), + ] { + let mut value = serde_json::to_value(manifest()?)?; + if let Some(update) = update.as_object() { + for (key, item) in update { + value[key] = item.clone(); + } + } + assert!( + serde_json::from_value::(value)? + .validate() + .is_err() + ); + } + Ok(()) +} + +#[test] +fn catalog_capacity_and_metadata_revisions_are_stable() -> Result<(), Box> { + let mut catalog = ChunkCatalog::default(); + for index in 0..512 { + let mut custom = manifest()?; + custom.id = format!("custom.{index}.v1"); + catalog.insert(custom)?; + } + assert!(matches!( + catalog.insert(manifest()?), + Err(CodeError::LimitExceeded { .. }) + )); + assert_eq!(catalog.search(&"é".repeat(1025), None, 50).len(), 0); + assert_eq!(catalog.search("", None, 999).len(), 50); + let mut custom = manifest()?; + let before = custom.revision()?; + custom.summary.push_str(" changed"); + assert_ne!(before, custom.revision()?); + Ok(()) +} diff --git a/crates/tinytools-code/tests/error_tests.rs b/crates/tinytools-code/tests/error_tests.rs new file mode 100644 index 00000000..54fc8e7d --- /dev/null +++ b/crates/tinytools-code/tests/error_tests.rs @@ -0,0 +1,67 @@ +//! Public error messages identify the failed operation and corrective boundary. + +use tinytools_code::{ + ChunkCatalog, CodeEngine, CodeError, CodeLimits, Language, PatchOperation, PatchRequest, + revision, select_symbol, +}; + +#[test] +fn public_refusals_produce_actionable_messages() -> Result<(), CodeError> { + let mut engine = CodeEngine::default(); + let report = engine.analyze("file", Language::Rust, "fn run() {}")?; + let errors = [ + CodeEngine::new(CodeLimits { + max_records: 0, + ..CodeLimits::default() + }) + .err(), + engine.validate(Language::Rust, "fn {").err(), + select_symbol(&report, "missing").err(), + select_symbol(&report, &"x".repeat(4097)).err(), + ChunkCatalog::new().inspect("missing").err(), + engine + .patch( + "file", + Language::Rust, + "fn run() {}", + &PatchRequest { + expected_revision: revision("other"), + selector: Some("run".to_owned()), + operation: PatchOperation::ReplaceBody, + code: String::new(), + }, + ) + .err(), + ]; + for error in errors { + assert!(error.is_some_and(|error| error.to_string().len() > 10)); + } + for error in [ + CodeError::Parser("compiled grammar mismatch".to_owned()), + CodeError::AmbiguousSymbol("name".to_owned()), + CodeError::UnsupportedOperation("data imports".to_owned()), + ] { + assert!(error.to_string().contains(':')); + assert!(std::error::Error::source(&error).is_none()); + } + Ok(()) +} + +#[test] +fn missing_tokens_are_reported_with_locations() -> Result<(), CodeError> { + let report = + CodeEngine::default().analyze("file", Language::Rust, "fn run() { let value = 1 }")?; + assert!( + report + .diagnostics + .iter() + .any(|diagnostic| diagnostic.kind == "missing") + ); + assert!( + report + .diagnostics + .iter() + .all(|diagnostic| diagnostic.range.start_line == 1) + ); + Ok(()) +} diff --git a/crates/tinytools-code/tests/fixtures/analysis.rs b/crates/tinytools-code/tests/fixtures/analysis.rs new file mode 100644 index 00000000..2ef4c807 --- /dev/null +++ b/crates/tinytools-code/tests/fixtures/analysis.rs @@ -0,0 +1,21 @@ +//! Shared lexical-analysis and benchmark fixture. + +/// A named value. +pub struct Record { + pub value: i32, +} + +impl Record { + pub fn increment(&mut self) { + self.value += 1; + self.value += 2; + } +} + +pub mod nested { + pub fn calculate(value: i32) -> i32 { + let doubled = value * 2; + let adjusted = doubled + 1; + adjusted + } +} diff --git a/crates/tinytools-code/tests/patch_tests.rs b/crates/tinytools-code/tests/patch_tests.rs new file mode 100644 index 00000000..6e426996 --- /dev/null +++ b/crates/tinytools-code/tests/patch_tests.rs @@ -0,0 +1,457 @@ +//! Targeted edits preserve neighboring source and reject stale or unsafe inputs. + +use tinytools_code::{ + CodeEngine, CodeError, CodeLimits, Language, PatchOperation, PatchRequest, PatchResult, + revision, +}; + +fn request( + source: &str, + operation: PatchOperation, + selector: Option<&str>, + code: &str, +) -> PatchRequest { + PatchRequest { + expected_revision: revision(source), + operation, + selector: selector.map(str::to_owned), + code: code.to_owned(), + } +} + +#[test] +fn body_edits_are_revision_checked_and_preserve_siblings() -> Result<(), CodeError> { + let source = "// license\nfn first() { old(); }\n// untouched\nfn second() {}\n"; + let mut engine = CodeEngine::default(); + let edit = request(source, PatchOperation::ReplaceBody, Some("first"), "new();"); + let result = engine.patch("file", Language::Rust, source, &edit)?; + assert!(result.changed); + assert!(result.source.starts_with("// license\nfn first() {")); + assert!(result.source.ends_with("\n// untouched\nfn second() {}\n")); + assert!(result.source.contains("new();") && !result.source.contains("old();")); + assert_eq!(result.after_revision, revision(&result.source)); + assert!(matches!( + engine.patch("file", Language::Rust, &result.source, &edit), + Err(CodeError::StaleRevision { .. }) + )); + Ok(()) +} + +#[test] +fn replacement_and_one_pass_wrapping_touch_only_the_selected_range() -> Result<(), CodeError> { + let source = "fn old() {}\nfn keep() {}"; + let mut engine = CodeEngine::default(); + let result = engine.patch( + "file", + Language::Rust, + source, + &request( + source, + PatchOperation::ReplaceSymbol, + Some("old"), + "pub fn newer() {}", + ), + )?; + assert_eq!(result.source, "pub fn newer() {}\nfn keep() {}"); + let wrapped = engine.patch( + "file", + Language::Rust, + source, + &request( + source, + PatchOperation::Wrap, + Some("old"), + "#[allow(dead_code)]\n{{node}}", + ), + )?; + assert_eq!( + wrapped.source, + "#[allow(dead_code)]\nfn old() {}\nfn keep() {}" + ); + for code in ["no marker", "{{node}} {{node}}"] { + assert!( + engine + .patch( + "file", + Language::Rust, + source, + &request(source, PatchOperation::Wrap, Some("old"), code) + ) + .is_err() + ); + } + Ok(()) +} + +#[test] +fn invalid_ambiguous_or_missing_targets_are_refused() { + let mut engine = CodeEngine::default(); + for (source, selector, code) in [ + ("fn old() {}", Some("old"), "fn {"), + ("fn old() {}", Some("missing"), "fn x() {}"), + ("fn old() {}", None, "fn x() {}"), + ("fn old( {", Some("old"), "fn x() {}"), + ( + "mod a { fn old() {} } mod b { fn old() {} }", + Some("old"), + "fn x() {}", + ), + ] { + assert!( + engine + .patch( + "file", + Language::Rust, + source, + &request(source, PatchOperation::ReplaceSymbol, selector, code) + ) + .is_err() + ); + } + let source = "const VALUE: i32 = 1;"; + assert!(matches!( + engine.patch( + "file", + Language::Rust, + source, + &request(source, PatchOperation::ReplaceBody, Some("VALUE"), "2") + ), + Err(CodeError::UnsupportedOperation(_)) + )); + let source = "const value = () => 1;"; + assert!(matches!( + engine.patch( + "file", + Language::JavaScript, + source, + &request(source, PatchOperation::ReplaceBody, Some("value"), "2") + ), + Err(CodeError::UnsupportedOperation(_)) + )); +} + +#[test] +fn truncated_outline_requires_exact_ids_for_mutations() -> Result<(), CodeError> { + let source = "fn same() {} mod nested { fn same() {} }"; + let mut engine = CodeEngine::new(CodeLimits { + max_records: 1, + ..CodeLimits::default() + })?; + assert!(matches!( + engine.patch( + "file", + Language::Rust, + source, + &request(source, PatchOperation::ReplaceBody, Some("same"), "call();") + ), + Err(CodeError::AmbiguousSymbol(_)) + )); + let report = engine.analyze("file", Language::Rust, source)?; + let result = engine.patch( + "file", + Language::Rust, + source, + &request( + source, + PatchOperation::ReplaceBody, + Some(&report.symbols[0].id), + "call();", + ), + )?; + assert!(result.source.ends_with("mod nested { fn same() {} }")); + Ok(()) +} + +#[test] +fn imports_preserve_prologues_and_attached_documentation() -> Result<(), CodeError> { + let mut engine = CodeEngine::default(); + for (language, source, import, preserved) in [ + ( + Language::Rust, + "//! Module docs\n/// Operation docs\npub fn operation() {}", + "use std::fmt;", + "/// Operation docs\npub fn operation() {}", + ), + ( + Language::Rust, + "#![no_std]\n#[cfg(test)]\npub fn operation() {}", + "use core::fmt;", + "#[cfg(test)]\npub fn operation() {}", + ), + ( + Language::TypeScript, + "'use strict';\n/** Operation docs */\nfunction operation() {}", + "import x from 'x';", + "/** Operation docs */\nfunction operation() {}", + ), + ( + Language::JavaScript, + "#!/usr/bin/env node\n'use strict';\nfunction run() {}", + "import x from 'x';", + "function run() {}", + ), + ( + Language::Go, + "package main\nfunc run() {}", + "import \"fmt\"", + "func run() {}", + ), + ] { + let result = engine.patch( + "file", + language, + source, + &request(source, PatchOperation::InsertImport, None, import), + )?; + assert!(result.source.contains(preserved)); + assert!(result.source.find(import) < result.source.find(preserved)); + if source.starts_with("#![no_std]") { + assert!(result.source.starts_with("#![no_std]\nuse core::fmt;")); + } + let duplicate = engine.patch( + "file", + language, + &result.source, + &request(&result.source, PatchOperation::InsertImport, None, import), + )?; + assert!(!duplicate.changed); + } + Ok(()) +} + +#[test] +fn python_future_imports_follow_only_the_first_real_docstring() -> Result<(), CodeError> { + let mut engine = CodeEngine::default(); + for literal in ["\"doc\"", "r\"doc\"", "\"\"\"doc\"\"\""] { + let source = format!("#!/usr/bin/python\n{literal}\n\"extra\"\nimport os\nvalue = 1\n"); + let result = engine.patch( + "file", + Language::Python, + &source, + &request( + &source, + PatchOperation::InsertImport, + None, + "from __future__ import annotations", + ), + )?; + assert!(result.source.find(literal) < result.source.find("from __future__")); + assert!(result.source.find("from __future__") < result.source.find("\"extra\"")); + } + for source in [ + "b\"bytes\"\nvalue = 1\n", + "f\"string\"\nvalue = 1\n", + "from __future__ import annotations\nimport os\nvalue = 1\n", + ] { + let result = engine.patch( + "file", + Language::Python, + source, + &request( + source, + PatchOperation::InsertImport, + None, + "from __future__ import division", + ), + )?; + assert!( + result.source.find("from __future__ import division") < result.source.find("value") + ); + if source.starts_with(['b', 'f']) && !source.starts_with("from") { + assert!(result.source.starts_with("from __future__ import division")); + } + } + Ok(()) +} + +#[test] +fn imports_refuse_non_imports_and_data_grammars() { + let mut engine = CodeEngine::default(); + for code in ["fn run() {}", "use a; use b;", "use ;"] { + let source = "fn old() {}"; + assert!( + engine + .patch( + "file", + Language::Rust, + source, + &request(source, PatchOperation::InsertImport, None, code) + ) + .is_err() + ); + } + for (language, source) in [(Language::Json, "{}"), (Language::Toml, "value=1")] { + assert!(matches!( + engine.patch( + "file", + language, + source, + &request(source, PatchOperation::InsertImport, None, "import x") + ), + Err(CodeError::UnsupportedOperation(_)) + )); + } +} + +#[test] +fn member_edits_handle_fields_methods_and_container_syntax() -> Result<(), CodeError> { + let mut engine = CodeEngine::default(); + for (language, source, name, member) in [ + ( + Language::Rust, + "struct Thing { pub(crate) first: i32 // attached\n}", + "Thing", + "pub second: i32", + ), + ( + Language::Rust, + "enum Kind { First // attached\n}", + "Kind", + "Second", + ), + ( + Language::Rust, + "impl Thing { fn first(&self) {} }", + "Thing", + "fn second(&self) {}", + ), + ( + Language::TypeScript, + "type Thing = { first: string; };", + "Thing", + "second: number;", + ), + ( + Language::TypeScript, + "class Thing {}", + "Thing", + "second(): void {}", + ), + ( + Language::Python, + "class Thing: pass\n", + "Thing", + "def second(self):\n return 2", + ), + ( + Language::Python, + "class Thing:\n first = 1\n", + "Thing", + "second = 2", + ), + ( + Language::Go, + "package main\ntype Thing struct { First int }\n", + "Thing", + "Second string", + ), + ] { + let result = engine.patch( + "file", + language, + source, + &request(source, PatchOperation::AppendMember, Some(name), member), + )?; + assert!(result.source.contains("second") || result.source.contains("Second")); + if source.contains("// attached") { + assert!(result.source.contains(", // attached")); + } + } + Ok(()) +} + +#[test] +fn python_body_replacement_supports_tabs_inline_suites_and_empty_bodies() -> Result<(), CodeError> { + let mut engine = CodeEngine::default(); + for source in [ + "def run(): return 1\n", + "def run():\n\treturn 1\n", + "def run():\n return 1\n", + ] { + for code in ["value = 2\nreturn value", ""] { + let result = engine.patch( + "file", + Language::Python, + source, + &request(source, PatchOperation::ReplaceBody, Some("run"), code), + )?; + assert!(!result.source.contains("return 1")); + } + } + Ok(()) +} + +#[test] +fn comma_terminated_members_preserve_nested_closing_indentation() -> Result<(), CodeError> { + let source = "mod container {\n struct Thing {\n first: i32 // attached\n }\n}\n"; + let result = CodeEngine::default().patch( + "file", + Language::Rust, + source, + &request( + source, + PatchOperation::AppendMember, + Some("Thing"), + "pub second: i32,", + ), + )?; + assert!(result.source.contains("first: i32, // attached")); + assert!(result.source.contains(" pub second: i32,\n }\n}")); + Ok(()) +} + +#[test] +fn member_insertion_preserves_an_unindented_existing_closing_line() -> Result<(), CodeError> { + let source = "mod container {\n struct Thing {\n first: i32,\n}\n}\n"; + let result = CodeEngine::default().patch( + "file", + Language::Rust, + source, + &request( + source, + PatchOperation::AppendMember, + Some("Thing"), + "pub second: i32,", + ), + )?; + assert!(result.source.ends_with(" pub second: i32,\n}\n}\n")); + Ok(()) +} + +#[test] +fn patch_wire_form_and_expansion_budgets_are_enforced() -> Result<(), Box> { + let source = "fn run() {}"; + let edit = request(source, PatchOperation::ReplaceBody, Some("run"), "call();"); + assert_eq!( + edit, + serde_json::from_str::(&serde_json::to_string(&edit)?)? + ); + assert!(serde_json::from_str::("{\"expected_revision\":\"x\",\"operation\":\"replace_body\",\"selector\":null,\"code\":\"\",\"extra\":true}").is_err()); + let result = CodeEngine::default().patch("file", Language::Rust, source, &edit)?; + assert_eq!( + result, + serde_json::from_str::(&serde_json::to_string(&result)?)? + ); + let mut engine = CodeEngine::new(CodeLimits { + max_source_bytes: 1024, + ..CodeLimits::default() + })?; + let source = format!("fn run() {{\n{}let old = 1;\n}}", " ".repeat(256)); + let huge = request( + &source, + PatchOperation::ReplaceBody, + Some("run"), + &"let next = 2;\n".repeat(100), + ); + assert!(matches!( + engine.patch("file", Language::Rust, &source, &huge), + Err(CodeError::LimitExceeded { .. }) + )); + let mut stale = request("fn run() {}", PatchOperation::ReplaceBody, Some("run"), ""); + stale.expected_revision = "x".repeat(129); + assert!(matches!( + engine.patch("file", Language::Rust, "fn run() {}", &stale), + Err(CodeError::LimitExceeded { .. }) + )); + Ok(()) +} diff --git a/crates/tinytools-code/tests/render_tests.rs b/crates/tinytools-code/tests/render_tests.rs new file mode 100644 index 00000000..beb67561 --- /dev/null +++ b/crates/tinytools-code/tests/render_tests.rs @@ -0,0 +1,477 @@ +//! One-pass rendering, typed placeholders, syntax contexts and slot constraints. + +use serde_json::json; +use tinytools_code::{ + ChunkBindings, ChunkCatalog, ChunkManifest, ChunkTarget, CodeEngine, CodeError, CodeLimits, + Language, RenderedChunk, render_chunk, +}; + +fn manifest(language: Language, target: ChunkTarget) -> ChunkManifest { + ChunkManifest { + id: "custom.example.v1".to_owned(), + summary: "An example".to_owned(), + language, + target, + template_path: "template.txt".to_owned(), + parameters: Vec::new(), + slots: Vec::new(), + tags: Vec::new(), + dependencies: Vec::new(), + forbidden_nodes: Vec::new(), + } +} + +#[test] +fn every_builtin_pattern_renders_with_its_declared_contract() +-> Result<(), Box> { + let catalog = ChunkCatalog::builtin()?; + let mut engine = CodeEngine::default(); + for summary in catalog.search("", None, 50) { + let metadata = catalog.inspect(&summary.id)?; + let Some(template) = catalog.builtin_source(&summary.id) else { + return Err("missing built-in template".into()); + }; + let mut bindings = ChunkBindings::default(); + for slot in &metadata.slots { + if slot.required { + let source = match (metadata.language, slot.name.as_str()) { + (Language::Python, "setup") => "resource = object()", + (Language::Python, _) => "del resource", + (Language::Go, _) => "w.WriteHeader(http.StatusOK)", + (_, "assertions") => "expect(1 + 1).toBe(2);", + _ => "return value;", + }; + bindings.slots.insert(slot.name.clone(), source.to_owned()); + } + } + let result = render_chunk(&mut engine, metadata, template, &bindings)?; + assert!(!result.source.contains("@slot")); + assert_eq!(result.id, metadata.id); + assert_eq!(result.dependencies, metadata.dependencies); + assert_eq!( + result, + serde_json::from_str::(&serde_json::to_string(&result)?)? + ); + } + Ok(()) +} + +#[test] +fn inserted_string_markers_and_quotes_are_never_rescanned() -> Result<(), Box> +{ + let mut metadata = manifest(Language::Rust, ChunkTarget::File); + metadata.parameters = + serde_json::from_value(json!([{"name":"text","kind":"string","required":true}]))?; + let mut bindings = ChunkBindings::default(); + bindings + .parameters + .insert("text".to_owned(), json!("{{evil}}\"\n// @slot evil")); + let result = render_chunk( + &mut CodeEngine::default(), + &metadata, + "const TEXT: &str = {{text}};", + &bindings, + )?; + assert!(result.source.contains("{{evil}}")); + assert!(result.source.contains("\\\"\\n")); + Ok(()) +} + +#[test] +fn rust_strings_escape_control_characters_with_rust_literal_syntax() +-> Result<(), Box> { + let mut metadata = manifest(Language::Rust, ChunkTarget::File); + metadata.parameters = + serde_json::from_value(json!([{"name":"text","kind":"string","required":true}]))?; + let mut bindings = ChunkBindings::default(); + bindings + .parameters + .insert("text".to_owned(), json!("\0\u{0008}\u{000c}é\"\\\n")); + let result = render_chunk( + &mut CodeEngine::default(), + &metadata, + "const TEXT: &str = {{text}};", + &bindings, + )?; + assert!(result.source.contains("\\u{0}")); + assert!(result.source.contains("\\u{8}")); + assert!(result.source.contains("\\u{c}")); + assert!(result.source.contains("\\\"\\\\\\n")); + Ok(()) +} + +#[test] +fn identifiers_and_type_expressions_reject_statement_injection() +-> Result<(), Box> { + let catalog = ChunkCatalog::builtin()?; + let metadata = catalog.inspect("rust.public-field.v1")?; + let Some(template) = catalog.builtin_source(&metadata.id) else { + return Err("missing template".into()); + }; + for (name, value) in [ + ("name", "field; injected"), + ("type", "i32; fn injected() {}"), + ("type", ""), + ("type", "Vec<"), + ] { + let mut bindings = ChunkBindings::default(); + bindings.parameters.insert(name.to_owned(), json!(value)); + assert!(render_chunk(&mut CodeEngine::default(), metadata, template, &bindings).is_err()); + } + let metadata = catalog.inspect("tsx.accessible-action.v1")?; + let Some(template) = catalog.builtin_source(&metadata.id) else { + return Err("missing template".into()); + }; + let mut bindings = ChunkBindings::default(); + bindings + .parameters + .insert("name".to_owned(), json!("lower")); + assert!(render_chunk(&mut CodeEngine::default(), metadata, template, &bindings).is_err()); + Ok(()) +} + +#[test] +fn slots_must_be_original_standalone_comments_and_are_one_pass() +-> Result<(), Box> { + let mut metadata = manifest(Language::Rust, ChunkTarget::File); + metadata.slots = serde_json::from_value(json!([{"name":"body","required":true}]))?; + let mut bindings = ChunkBindings::default(); + bindings.slots.insert( + "body".to_owned(), + "let text = \"// @slot injected\";".to_owned(), + ); + let result = render_chunk( + &mut CodeEngine::default(), + &metadata, + "fn run() {\n // @slot body\n}", + &bindings, + )?; + assert!(result.source.contains("// @slot injected")); + for template in [ + "fn run() { let text = \"// @slot body\"; }", + "fn run() {} // @slot body", + "fn run() {\n// @slot body\n// @slot body\n}", + "fn run() {\n// @slot unknown\n}", + ] { + assert!(render_chunk(&mut CodeEngine::default(), &metadata, template, &bindings).is_err()); + } + let result = render_chunk( + &mut CodeEngine::default(), + &metadata, + "fn run() {\n /* @slot body */\n}", + &bindings, + )?; + assert!(result.source.contains("let text")); + Ok(()) +} + +#[test] +fn required_unknown_and_unused_bindings_are_refused() -> Result<(), Box> { + let mut metadata = manifest(Language::Rust, ChunkTarget::File); + metadata.parameters = + serde_json::from_value(json!([{"name":"name","kind":"identifier","required":true}]))?; + let mut bindings = ChunkBindings::default(); + assert!( + render_chunk( + &mut CodeEngine::default(), + &metadata, + "fn {{name}}() {}", + &bindings + ) + .is_err() + ); + bindings + .parameters + .insert("unknown".to_owned(), json!("run")); + assert!( + render_chunk( + &mut CodeEngine::default(), + &metadata, + "fn {{name}}() {}", + &bindings + ) + .is_err() + ); + bindings.parameters.clear(); + bindings.parameters.insert("name".to_owned(), json!("run")); + for template in ["fn run() {}", "fn {{name() {}", "fn {{missing}}() {}"] { + assert!(render_chunk(&mut CodeEngine::default(), &metadata, template, &bindings).is_err()); + } + metadata.parameters.clear(); + bindings.parameters.clear(); + metadata.slots = serde_json::from_value(json!([{"name":"body","required":true}]))?; + assert!( + render_chunk( + &mut CodeEngine::default(), + &metadata, + "fn run() {\n// @slot body\n}", + &bindings + ) + .is_err() + ); + bindings + .slots + .insert("unknown".to_owned(), "let x=1;".to_owned()); + assert!( + render_chunk( + &mut CodeEngine::default(), + &metadata, + "fn run() {\n// @slot body\n}", + &bindings + ) + .is_err() + ); + Ok(()) +} + +#[test] +fn exact_slot_nodes_and_generated_node_restrictions_are_enforced() +-> Result<(), Box> { + let mut metadata = manifest(Language::Rust, ChunkTarget::File); + metadata.slots = serde_json::from_value( + json!([{"name":"body","required":true,"node_kind":"let_declaration"}]), + )?; + let mut bindings = ChunkBindings::default(); + bindings + .slots + .insert("body".to_owned(), "let value = 1;".to_owned()); + let template = "fn run() {\n // @slot body\n}"; + render_chunk(&mut CodeEngine::default(), &metadata, template, &bindings)?; + bindings + .slots + .insert("body".to_owned(), "call();".to_owned()); + assert!(render_chunk(&mut CodeEngine::default(), &metadata, template, &bindings).is_err()); + bindings.slots.insert( + "body".to_owned(), + "let value = 1; let other = 2;".to_owned(), + ); + assert!(render_chunk(&mut CodeEngine::default(), &metadata, template, &bindings).is_err()); + metadata.slots[0].node_kind = None; + metadata.forbidden_nodes.push("call_expression".to_owned()); + bindings + .slots + .insert("body".to_owned(), "call();".to_owned()); + assert!(render_chunk(&mut CodeEngine::default(), &metadata, template, &bindings).is_err()); + Ok(()) +} + +#[test] +fn member_contexts_support_methods_fields_and_interfaces() -> Result<(), CodeError> { + for (language, source) in [ + (Language::Rust, "pub value: i32,"), + (Language::Rust, "fn added(&self) {}"), + (Language::Go, "Value string"), + (Language::Go, "Run() error"), + (Language::TypeScript, "run(): void {}"), + (Language::JavaScript, "run() {}"), + (Language::Python, "def run(self):\n return 1"), + ] { + let metadata = manifest(language, ChunkTarget::Member); + render_chunk( + &mut CodeEngine::default(), + &metadata, + source, + &ChunkBindings::default(), + )?; + } + for language in [Language::Json, Language::Toml] { + assert!(matches!( + render_chunk( + &mut CodeEngine::default(), + &manifest(language, ChunkTarget::Member), + "", + &ChunkBindings::default() + ), + Err(CodeError::UnsupportedOperation(_)) + )); + } + Ok(()) +} + +#[test] +fn synthetic_wrapper_nodes_are_excluded_from_generated_restrictions() -> Result<(), CodeError> { + let mut metadata = manifest(Language::Rust, ChunkTarget::FunctionBody); + metadata.forbidden_nodes = vec!["function_item".to_owned()]; + render_chunk( + &mut CodeEngine::default(), + &metadata, + "let value = 1;", + &ChunkBindings::default(), + )?; + assert!( + render_chunk( + &mut CodeEngine::default(), + &metadata, + "fn nested() {}", + &ChunkBindings::default() + ) + .is_err() + ); + Ok(()) +} + +#[test] +fn python_slots_keep_multiline_node_coordinates_in_body_context() +-> Result<(), Box> { + let mut metadata = manifest(Language::Python, ChunkTarget::FunctionBody); + metadata.slots = serde_json::from_value( + json!([{"name":"body","required":true,"node_kind":"if_statement"}]), + )?; + let mut bindings = ChunkBindings::default(); + bindings + .slots + .insert("body".to_owned(), "if True:\n return 1".to_owned()); + render_chunk( + &mut CodeEngine::default(), + &metadata, + "# @slot body", + &bindings, + )?; + for (language, source) in [ + (Language::Go, "return"), + (Language::JavaScript, "return 1;"), + (Language::TypeScript, "return 1;"), + ] { + render_chunk( + &mut CodeEngine::default(), + &manifest(language, ChunkTarget::FunctionBody), + source, + &ChunkBindings::default(), + )?; + } + Ok(()) +} + +#[test] +fn typed_values_cover_boolean_number_and_language_type_contexts() +-> Result<(), Box> { + for (language, template, value) in [ + (Language::Rust, "type Alias = {{type}};", "Vec"), + ( + Language::TypeScript, + "type Alias = {{type}};", + "Array", + ), + (Language::Python, "value: {{type}}", "list[str]"), + ( + Language::Go, + "package main\ntype Alias {{type}}\n", + "map[string]int", + ), + ] { + let mut metadata = manifest(language, ChunkTarget::File); + metadata.parameters = + serde_json::from_value(json!([{"name":"type","kind":"type_expr","required":true}]))?; + let mut bindings = ChunkBindings::default(); + bindings.parameters.insert("type".to_owned(), json!(value)); + render_chunk(&mut CodeEngine::default(), &metadata, template, &bindings)?; + } + let mut metadata = manifest(Language::Python, ChunkTarget::File); + metadata.parameters = serde_json::from_value( + json!([{"name":"flag","kind":"boolean"},{"name":"count","kind":"number"}]), + )?; + let mut bindings = ChunkBindings::default(); + bindings.parameters.insert("flag".to_owned(), json!(false)); + bindings.parameters.insert("count".to_owned(), json!(1.5)); + let rendered = render_chunk( + &mut CodeEngine::default(), + &metadata, + "flag = {{flag}}\ncount = {{count}}", + &bindings, + )?; + assert!(rendered.source.contains("False")); + for (name, wrong) in [("flag", json!("true")), ("count", json!("1.5"))] { + let mut invalid = bindings.clone(); + invalid.parameters.insert(name.to_owned(), wrong); + assert!( + render_chunk( + &mut CodeEngine::default(), + &metadata, + "flag = {{flag}}\ncount = {{count}}", + &invalid + ) + .is_err() + ); + } + Ok(()) +} + +#[test] +fn unsupported_type_contexts_invalid_code_and_binding_wire_errors_are_refused() +-> Result<(), Box> { + for language in [Language::JavaScript, Language::Json, Language::Toml] { + let mut metadata = manifest(language, ChunkTarget::File); + metadata.parameters = + serde_json::from_value(json!([{"name":"type","kind":"type_expr","default":"string"}]))?; + assert!(matches!( + render_chunk( + &mut CodeEngine::default(), + &metadata, + "{{type}}", + &ChunkBindings::default() + ), + Err(CodeError::UnsupportedOperation(_)) + )); + } + assert!( + render_chunk( + &mut CodeEngine::default(), + &manifest(Language::Rust, ChunkTarget::File), + "fn {", + &ChunkBindings::default() + ) + .is_err() + ); + let bindings = ChunkBindings::default(); + assert_eq!( + bindings, + serde_json::from_str::(&serde_json::to_string(&bindings)?)? + ); + assert!(serde_json::from_str::("{\"shell\":true}").is_err()); + Ok(()) +} + +#[test] +fn rendered_expansion_and_binding_names_are_bounded() -> Result<(), Box> { + let mut metadata = manifest(Language::Rust, ChunkTarget::File); + metadata.slots = serde_json::from_value(json!([{"name":"body","required":true}]))?; + let template = format!("fn run() {{\n{}// @slot body\n}}", " ".repeat(256)); + let mut bindings = ChunkBindings::default(); + bindings + .slots + .insert("body".to_owned(), "let next = 2;\n".repeat(16)); + let mut engine = CodeEngine::new(CodeLimits { + max_source_bytes: 1024, + ..CodeLimits::default() + })?; + assert!(matches!( + render_chunk(&mut engine, &metadata, &template, &bindings), + Err(CodeError::LimitExceeded { .. }) + )); + bindings.slots.insert( + "body".to_owned(), + "let next = 2;\n\nlet last = 3;".to_owned(), + ); + assert!( + render_chunk(&mut engine, &metadata, &template, &bindings)? + .source + .len() + <= 1024 + ); + bindings + .parameters + .insert("x".repeat(129), json!("anything")); + assert!(matches!( + render_chunk(&mut engine, &metadata, &template, &bindings), + Err(CodeError::LimitExceeded { .. }) + )); + bindings.parameters.clear(); + bindings.slots = (0..33) + .map(|index| (format!("slot{index}"), String::new())) + .collect(); + assert!(matches!( + render_chunk(&mut engine, &metadata, &template, &bindings), + Err(CodeError::LimitExceeded { .. }) + )); + Ok(()) +} diff --git a/crates/tinytools-std/Cargo.toml b/crates/tinytools-std/Cargo.toml index b4240bc6..16784cac 100644 --- a/crates/tinytools-std/Cargo.toml +++ b/crates/tinytools-std/Cargo.toml @@ -10,6 +10,10 @@ documentation = "https://docs.rs/tinytools-std" readme = "README.md" publish = false +[features] +# Structural inspection and revision-checked edits are opt-in native tools. +code = ["dep:tinytools-code", "dep:serde", "dep:tempfile"] + [dependencies] anyhow = { workspace = true } async-trait = { workspace = true } @@ -36,9 +40,15 @@ reqwest = { version = "0.12", default-features = false, features = ["rustls-tls" # Safe effective-credential executable checks for Unix PATH candidates. rustix = { version = "1", default-features = false, features = ["fs", "process"] } serde_json = { workspace = true } +# The optional code tools decode strict, typed argument packets. +serde = { workspace = true, optional = true } # `curl` reports the SHA-256 of what it downloaded. sha2 = "0.11" tinytools = { path = "../tinytools", version = "0.5.0" } +# Pure syntax analysis and typed chunk rendering; no host policy lives there. +tinytools-code = { path = "../tinytools-code", version = "0.5.0", optional = true } +# Same-directory staging and atomic publication for structural edits. +tempfile = { version = "3", optional = true } # `spawn_blocking` keeps synchronous system DNS resolution off async executor threads; # `fs`/`process`/`time` serve the filesystem tools. tokio = { version = "1", default-features = false, features = ["rt", "sync", "fs", "process", "time", "io-util", "macros"] } diff --git a/crates/tinytools-std/README.md b/crates/tinytools-std/README.md index 48216d6f..21565f85 100644 --- a/crates/tinytools-std/README.md +++ b/crates/tinytools-std/README.md @@ -10,5 +10,6 @@ Host-independent building blocks for agent tools, extracted from OpenHuman. | `network` | The network tools: `http_request`, `web_fetch`, `curl`, `pushover`. Every tool takes an `Arc`: the tool does the I/O, the host's gate (autonomy, action budget, approval, privacy mode, proxy) decides whether it may and how. Two smaller seams keep host behavior out: `PaymentHook` answers a `402 Payment Required` for `http_request`, and `HtmlExtractor` converts pages to Markdown for `web_fetch`. `WebFetchTool::new_async` accepts a fallible `AsyncHtmlExtractor` for transforms supplied by a remote service or loadable module; detection and extraction are awaited with the configured request timeout applied to each operation, and failures propagate without a local retry. Both constructors extract before applying the output cap, and skip extraction for raw or explicitly non-HTML responses. Names, descriptions and schemas are pinned by `src/network/fixtures/`. | | `sanitize` | Lexical sanitization helpers for untrusted tool, skill and capability metadata, including control and instruction-fence removal plus UTF-8-safe byte truncation. | | `detect_tools` | `find_on_path` and the read-only `detect_tools` tool. | +| `code` (optional feature) | Six deferred structural tools: `ast_outline`, `ast_symbol`, `ast_patch`, `chunk_search`, `chunk_inspect`, and `chunk_apply`. `code_tools` and `code_tools_with_options` take an `Arc` and return a fallible tool list. Reads and receipts are bounded; only literal `dry_run: true` previews edits. Actual writes require content revisions and publish atomically after receipt preflight and final path/revision checks. See the [arguments, receipts, limits and publication contract](src/code/README.md). | No enforcement of host policy lives here; the crate only supplies mechanisms. The `filesystem` tools' name, description and JSON Schema are pinned by the fixtures in `src/filesystem/fixtures/`. diff --git a/crates/tinytools-std/src/code/README.md b/crates/tinytools-std/src/code/README.md new file mode 100644 index 00000000..742f9c10 --- /dev/null +++ b/crates/tinytools-std/src/code/README.md @@ -0,0 +1,154 @@ +# Structural code tools + +Enable the `code` Cargo feature and call `tinytools_std::code_tools(gate)` to +obtain six `Box` values. `code_tools_with_options` accepts +`CodeToolOptions` when a host needs tighter resource limits. Construction is +fallible because options and the bundled catalog are validated. + +The tools use the pure `tinytools-code` engine for parsing and rendering. This +module owns native filesystem mechanics; the supplied `FsGate` owns policy. +Neither module executes generated code, installs declared dependencies, resolves +semantic types, or treats a syntax check as a successful build or test run. + +## Availability and authorization + +All six tools are deferred, belong to family `code`, and carry tags `code`, +`structural`, and `pack:code`. Every static minimum permission is `ReadOnly` so +hosts can retain preview access. `ast_patch` and `chunk_apply` require `Write` +unless the invocation contains the literal JSON boolean `dry_run: true`. +Missing, false, null, string and numeric flags never lower the declaration. +Nonboolean flags fail strict decoding before any filesystem mutation. + +Write calls consult the original gate's autonomy and action budget. The host's +`write_needs_approval` controls external-effect routing. Previews neither prompt +as writes nor consume actions. A trusted `ToolRunContext` workspace descriptor +scopes path validation per call without changing the original gate or moving +its accounting into a separate policy object. + +## Arguments + +Unknown fields are rejected. `language` can explicitly select `rust`, +`typescript`, `tsx`, `javascript`, `python`, `go`, `json`, or `toml`; otherwise +inspection and patch tools infer it from a supported filename extension. + +| Tool | Required arguments | Optional arguments | +| --- | --- | --- | +| `ast_outline` | `path` | `language`, `offset`, `limit`, `start_line`, `line_count` | +| `ast_symbol` | `path`, `selector` | `language`, `start_line`, `line_count` | +| `ast_patch` | `path`, `expected_revision`, `operation`, `code` | `language`, `selector`, `dry_run`, `start_line`, `line_count` | +| `chunk_search` | None | `query`, `language`, `offset`, `limit` | +| `chunk_inspect` | Exactly one of `id` or `manifest_path` | `offset`, `limit` | +| `chunk_apply` | `path`, `chunk_revision`, exactly one of `id` or `manifest_path`, plus the destination guard below | `bindings`, `selector`, `dry_run`, `start_line`, `line_count` | + +`selector` accepts a revision-bound symbol ID, an unambiguous qualified name, +or an unambiguous local name. A truncated symbol collection only permits IDs. +The patch operations are `replace_body`, `replace_symbol`, `insert_import`, +`append_member`, and `wrap`. All except `insert_import` need a selector; a wrap +uses exactly one `{{node}}` marker. Source revisions are 64 lowercase hexadecimal +digits returned by the engine, not Git commit hashes. + +For `chunk_apply`, edit an existing file with `expected_revision`, or create a +new file with `create_new: true` and no `expected_revision`. Creation refuses any +existing target and requires its parent directory to exist. A complete-file +chunk replaces the complete file and accepts no selector. Function-body and +member chunks require an existing file and a selector, and use the same pure +patch engine as `ast_patch`. `bindings` has two optional objects: `parameters` +for typed values and `slots` for supplied code strings. + +`chunk_search` searches the twelve bundled patterns. To use a custom pattern, +inspect its `manifest_path` and pass that same path to `chunk_apply`. The +manifest's template path must be portable and relative to the manifest's +directory. Both files pass the gate, and the resolved template must remain +inside that directory, including after symlink resolution. A returned +`chunk_revision` binds the exact manifest and template contents. Reformatting a +custom manifest changes this content identity even when its normalized +`manifest_revision` remains the same. + +## Receipts and paging + +Successful results contain a JSON block. List pages use this shape: + +```json +{"items": [], "offset": 0, "total": 0, "next_offset": null} +``` + +`offset` is zero-based. `limit` is positive and bounded by `max_page_items`. +Outline symbols, references and diagnostics each have their own page object. +Their separate truncation flags disclose parser collection limits; a page's +`total` counts retained records, not records beyond those limits. + +Source and folded-skeleton pages use this shape: + +```json +{"text": "fn example() {}\n", "start_line": 1, "total_lines": 1, + "next_start_line": null, "truncated": false} +``` + +`start_line` is one-based, relative to the returned source selection: the file +for mutation receipts, the folded source for outlines, or the declaration for +`ast_symbol`. Symbol ranges independently preserve absolute source coordinates. +`line_count` must be positive and no greater than `max_page_lines`. A single +oversized line can exceed the output budget even on a one-line page; that call +returns a bounded error instead of silently dropping content. + +Mutation receipts include `path`, `language`, `before_revision` (null for +creation), `revision`, `changed`, `written`, `dry_run`, and a source page. Chunk +receipts also include `chunk_id`, `chunk_revision`, `rendered_revision`, and +informational `dependencies`. `written: true` means publication succeeded. +An unchanged result reports `written: false` and consumes no action. + +For example, preview a body edit after reading the current revision: + +```json +{"path":"src/lib.rs","expected_revision":"<64 lowercase hex digits>", + "selector":"calculate","operation":"replace_body","code":"42", + "dry_run":true} +``` + +## Resource bounds + +| `CodeToolOptions` field | Default | +| --- | --- | +| `limits` | `CodeLimits::default()`; 1 MiB source, 32 cached entries, 8 MiB retained source | +| `max_argument_bytes` | 1 MiB | +| `max_manifest_bytes` | 64 KiB | +| `max_template_bytes` | 256 KiB | +| `max_output_bytes` | 64 KiB | +| `max_page_items` | 32 | +| `max_page_lines` | 80 | + +Argument validation counts serialized UTF-8 and JSON escape bytes with a writer +that stops at the limit, without allocating a second complete argument buffer. +Native inputs must be regular UTF-8 files. Reads take at most the configured +bound plus one byte so a file growing during a read cannot cause an unbounded +allocation. The output bound measures the actual pretty-printed model-visible +JSON. Errors are truncated only at UTF-8 boundaries. The complete success +receipt is checked before any staging file or action-budget consumption. + +## Publication and concurrency + +The tools share a mutation mutex and participate in `file_state`'s path lock +when that coordinator is enabled. Successful publications record the current +file-state agent's write. Exact source revisions remain required regardless of +whether the optional coordinator is enabled. + +Edits stage a complete file in the destination's validated directory. Existing +permission bits are retained; new files use the temporary-file library's +private creation permissions. Before publication, the adapter reauthorizes the +original destination path and rechecks target, manifest and template revisions. +The original gate's write authority is checked after action reservation and +again after the final asynchronous path and revision checks; the reserved +action is charged only once. +Creation uses no-clobber publication, and replacement uses an atomic rename. +Failed checks and cancellation drop the staged file, preserving the target. +The final atomic publication is a short synchronous operation: cancellation +cannot expose a partially written destination. + +These checks coordinate cooperating tools and detect changes observed before +publication. They do not claim an operating-system compare-and-swap against an +unrelated writer that changes a file between the last revision check and rename, +or power-loss durability of a directory entry. Filesystem confinement remains +the host gate's responsibility. + +See the accepted [specification](../../../../docs/specs/structural-code.md) and +[implementation plan](../../../../docs/plans/structural-code.md). diff --git a/crates/tinytools-std/src/code/arguments.rs b/crates/tinytools-std/src/code/arguments.rs new file mode 100644 index 00000000..45081651 --- /dev/null +++ b/crates/tinytools-std/src/code/arguments.rs @@ -0,0 +1,149 @@ +//! Strict argument packets; malformed preview flags never become writes. + +use anyhow::{Context, ensure}; +use serde::{Deserialize, de::DeserializeOwned}; +use serde_json::Value; +use tinytools_code::{ChunkBindings, Language, PatchOperation}; + +/// Stop serialization at the bound instead of copying an oversized value. +pub(super) fn check_size(args: &Value, maximum: usize) -> anyhow::Result<()> { + serde_json::to_writer(ArgumentBudget { remaining: maximum }, args) + .context("structural tool arguments exceed the byte budget") +} + +struct ArgumentBudget { + remaining: usize, +} + +impl std::io::Write for ArgumentBudget { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.remaining = self + .remaining + .checked_sub(bytes.len()) + .ok_or_else(|| std::io::Error::other("argument byte budget exhausted"))?; + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +pub(super) fn decode(args: Value) -> anyhow::Result { + serde_json::from_value(args).context("invalid structural tool arguments") +} + +pub(super) fn language(path: &str, explicit: Option) -> anyhow::Result { + explicit + .or_else(|| Language::from_path(path)) + .context("unknown filename extension; supply an explicit supported language") +} + +pub(super) fn literal_preview(args: &Value) -> bool { + args.get("dry_run") == Some(&Value::Bool(true)) +} + +pub(super) fn page_size(requested: Option, maximum: usize) -> anyhow::Result { + let size = requested.unwrap_or(maximum); + ensure!( + size > 0 && size <= maximum, + "page size must be between 1 and {maximum}" + ); + Ok(size) +} + +pub(super) fn require_revision(value: &str) -> anyhow::Result<()> { + ensure!( + value.len() == 64 + && value + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)), + "expected revision must contain 64 lowercase hexadecimal digits" + ); + Ok(()) +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Outline { + pub(super) path: String, + pub(super) language: Option, + #[serde(default)] + pub(super) offset: usize, + pub(super) limit: Option, + #[serde(default = "first_line")] + pub(super) start_line: usize, + pub(super) line_count: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Symbol { + pub(super) path: String, + pub(super) language: Option, + pub(super) selector: String, + #[serde(default = "first_line")] + pub(super) start_line: usize, + pub(super) line_count: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Patch { + pub(super) path: String, + pub(super) language: Option, + pub(super) expected_revision: String, + pub(super) selector: Option, + pub(super) operation: PatchOperation, + pub(super) code: String, + #[serde(default)] + pub(super) dry_run: bool, + #[serde(default = "first_line")] + pub(super) start_line: usize, + pub(super) line_count: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Search { + #[serde(default)] + pub(super) query: String, + pub(super) language: Option, + #[serde(default)] + pub(super) offset: usize, + pub(super) limit: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Inspect { + pub(super) id: Option, + pub(super) manifest_path: Option, + #[serde(default)] + pub(super) offset: usize, + pub(super) limit: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Apply { + pub(super) path: String, + pub(super) id: Option, + pub(super) manifest_path: Option, + pub(super) chunk_revision: String, + #[serde(default)] + pub(super) bindings: ChunkBindings, + pub(super) selector: Option, + pub(super) expected_revision: Option, + #[serde(default)] + pub(super) create_new: bool, + #[serde(default)] + pub(super) dry_run: bool, + #[serde(default = "first_line")] + pub(super) start_line: usize, + pub(super) line_count: Option, +} + +const fn first_line() -> usize { + 1 +} diff --git a/crates/tinytools-std/src/code/chunk_tests.rs b/crates/tinytools-std/src/code/chunk_tests.rs new file mode 100644 index 00000000..1451b914 --- /dev/null +++ b/crates/tinytools-std/src/code/chunk_tests.rs @@ -0,0 +1,255 @@ +//! Custom catalog provenance, native bounds and destination regressions. + +#![allow(clippy::unwrap_used)] + +use super::test_support::*; +use super::*; +use serde_json::json; +use std::sync::atomic::Ordering; +use tinytools::{ToolCallOptions, WorkspaceDescriptor}; + +#[tokio::test] +async fn a_custom_chunk_revision_binds_raw_metadata_and_template_content() { + let dir = tempfile::tempdir().unwrap(); + custom_manifest(dir.path(), "file", "fn created() {}\n"); + let tools = code_tools(TestGate::new(dir.path())).unwrap(); + let entry = tool(&tools, "chunk_inspect"); + let original = success(entry, json!({"manifest_path":"manifest.json"})).await; + let path = dir.path().join("manifest.json"); + std::fs::write( + &path, + format!("{}\n", std::fs::read_to_string(&path).unwrap()), + ) + .unwrap(); + let reformatted = success(entry, json!({"manifest_path":"manifest.json"})).await; + assert_ne!(original["chunk_revision"], reformatted["chunk_revision"]); + assert_eq!( + original["manifest_revision"], + reformatted["manifest_revision"] + ); + std::fs::write(dir.path().join("template.rs.txt"), "fn different() {}\n").unwrap(); + let edited = success(entry, json!({"manifest_path":"manifest.json"})).await; + assert_ne!(reformatted["chunk_revision"], edited["chunk_revision"]); + assert_eq!( + reformatted["manifest_revision"], + edited["manifest_revision"] + ); + assert!( + failure( + tool(&tools, "chunk_apply"), + json!({"manifest_path":"manifest.json", "path":"new.rs", "create_new":true, + "chunk_revision":original["chunk_revision"]}) + ) + .await + .contains("chunk revision") + ); + assert!(!dir.path().join("new.rs").exists()); +} + +#[tokio::test] +async fn malformed_manifests_and_nonportable_templates_fail_before_any_effect() { + let dir = tempfile::tempdir().unwrap(); + let gate = TestGate::new(dir.path()); + let tools = code_tools(gate.clone()).unwrap(); + for path in [ + "../outside.txt", + "/absolute.txt", + "dir\\template.txt", + "C:template.txt", + "a//b", + "", + ] { + std::fs::write(dir.path().join("manifest.json"), serde_json::to_vec(&json!({ + "id":"custom.pattern.v1", "summary":"Pattern", "language":"rust", "template_path":path + })).unwrap()).unwrap(); + assert!( + !failure( + tool(&tools, "chunk_inspect"), + json!({"manifest_path":"manifest.json"}) + ) + .await + .is_empty() + ); + } + std::fs::write(dir.path().join("manifest.json"), "{not json}").unwrap(); + assert!( + failure( + tool(&tools, "chunk_inspect"), + json!({"manifest_path":"manifest.json"}) + ) + .await + .contains("manifest") + ); + assert!( + failure( + tool(&tools, "chunk_inspect"), + json!({"id":"rust.fallible-function.v1","manifest_path":"manifest.json"}) + ) + .await + .contains("exactly one") + ); + assert_eq!(gate.state.used.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn manifest_and_template_reads_each_enforce_their_own_byte_budget() { + let dir = tempfile::tempdir().unwrap(); + custom_manifest(dir.path(), "file", "fn created() {}\n"); + let gate = TestGate::new(dir.path()); + let manifest_limited = code_tools_with_options( + gate.clone(), + CodeToolOptions { + max_manifest_bytes: 8, + ..Default::default() + }, + ) + .unwrap(); + assert!( + failure( + tool(&manifest_limited, "chunk_inspect"), + json!({"manifest_path":"manifest.json"}) + ) + .await + .contains("read budget") + ); + let template_limited = code_tools_with_options( + gate, + CodeToolOptions { + max_template_bytes: 8, + ..Default::default() + }, + ) + .unwrap(); + assert!( + failure( + tool(&template_limited, "chunk_inspect"), + json!({"manifest_path":"manifest.json"}) + ) + .await + .contains("read budget") + ); +} + +#[cfg(unix)] +#[tokio::test] +async fn a_template_symlink_cannot_escape_its_manifest_directory_inside_the_workspace() { + let dir = tempfile::tempdir().unwrap(); + let patterns = dir.path().join("patterns"); + std::fs::create_dir(&patterns).unwrap(); + custom_manifest(&patterns, "file", "fn original() {}\n"); + std::fs::remove_file(patterns.join("template.rs.txt")).unwrap(); + std::fs::write(dir.path().join("outside.rs"), "fn outside() {}\n").unwrap(); + std::os::unix::fs::symlink( + dir.path().join("outside.rs"), + patterns.join("template.rs.txt"), + ) + .unwrap(); + let tools = code_tools(TestGate::new(dir.path())).unwrap(); + assert!( + failure( + tool(&tools, "chunk_inspect"), + json!({"manifest_path":"patterns/manifest.json"}) + ) + .await + .contains("manifest directory") + ); +} + +#[tokio::test] +async fn missing_parent_directories_are_not_created_by_a_failed_chunk_publication() { + let dir = tempfile::tempdir().unwrap(); + let tools = code_tools(TestGate::new(dir.path())).unwrap(); + let inspected = success( + tool(&tools, "chunk_inspect"), + json!({"id":"rust.fallible-function.v1"}), + ) + .await; + assert!( + failure( + tool(&tools, "chunk_apply"), + json!({"id":"rust.fallible-function.v1", "path":"missing/new.rs", "create_new":true, + "chunk_revision":inspected["chunk_revision"]}) + ) + .await + .contains("parent directory") + ); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 0); +} + +#[tokio::test] +async fn an_unchanged_patch_has_a_receipt_but_does_not_consume_an_action() { + let dir = tempfile::tempdir().unwrap(); + let source = "fn value() -> i32 { 1 }\n"; + std::fs::write(dir.path().join("source.rs"), source).unwrap(); + let gate = TestGate::new(dir.path()); + let tools = code_tools(gate.clone()).unwrap(); + let mut args = patch_args(source); + args["operation"] = json!("replace_symbol"); + args["code"] = json!(source.trim_end()); + let receipt = success(tool(&tools, "ast_patch"), args).await; + assert_eq!(receipt["changed"], false); + assert_eq!(receipt["written"], false); + assert_eq!(gate.state.used.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn chunk_receipt_overflow_never_creates_a_destination() { + let dir = tempfile::tempdir().unwrap(); + let gate = TestGate::new(dir.path()); + let tools = code_tools(gate.clone()).unwrap(); + let inspected = success( + tool(&tools, "chunk_inspect"), + json!({"id":"rust.fallible-function.v1"}), + ) + .await; + let bounded = code_tools_with_options( + gate.clone(), + CodeToolOptions { + max_output_bytes: 160, + ..Default::default() + }, + ) + .unwrap(); + assert!( + failure( + tool(&bounded, "chunk_apply"), + json!({"id":"rust.fallible-function.v1", "path":"new.rs", "create_new":true, + "chunk_revision":inspected["chunk_revision"]}) + ) + .await + .contains("output budget") + ); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 0); + assert_eq!(gate.state.used.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn isolated_workspace_writes_land_in_the_child_and_charge_the_original_gate() { + let parent = tempfile::tempdir().unwrap(); + let child = tempfile::tempdir().unwrap(); + let source = "fn value() -> i32 { 1 }\n"; + std::fs::write(parent.path().join("source.rs"), source).unwrap(); + std::fs::write(child.path().join("source.rs"), source).unwrap(); + let gate = TestGate::new(parent.path()); + let tools = code_tools(gate.clone()).unwrap(); + let context = Workspace(WorkspaceDescriptor::new(child.path().to_path_buf())); + let result = tool(&tools, "ast_patch") + .execute_with_context( + patch_args(source), + ToolCallOptions::default(), + Some(&context), + ) + .await + .unwrap(); + assert!(!result.is_error, "{}", result.output()); + assert_eq!( + std::fs::read_to_string(parent.path().join("source.rs")).unwrap(), + source + ); + assert!( + std::fs::read_to_string(child.path().join("source.rs")) + .unwrap() + .contains('2') + ); + assert_eq!(gate.state.used.load(Ordering::SeqCst), 1); +} diff --git a/crates/tinytools-std/src/code/chunks.rs b/crates/tinytools-std/src/code/chunks.rs new file mode 100644 index 00000000..0acccfa1 --- /dev/null +++ b/crates/tinytools-std/src/code/chunks.rs @@ -0,0 +1,95 @@ +//! Content-bound chunk snapshots, including gated custom templates. + +use std::path::Path; +use std::sync::Arc; + +use anyhow::{Context, ensure}; +use tinytools_code::{ChunkManifest, revision}; + +use super::{io, types::CodeTool}; +use crate::filesystem::FsGate; + +pub(super) struct LoadedChunk { + pub(super) manifest: ChunkManifest, + pub(super) template: String, + pub(super) revision: String, + pub(super) guards: Vec, +} + +impl CodeTool { + pub(super) async fn load_chunk( + &self, + gate: &Arc, + id: Option<&str>, + manifest_path: Option<&str>, + ) -> anyhow::Result { + ensure!( + id.is_some() != manifest_path.is_some(), + "provide exactly one of id or manifest_path" + ); + if let Some(id) = id { + let manifest = self.shared.catalog.manifest(id)?.clone(); + let template = self + .shared + .catalog + .builtin_source(id) + .context("bundled chunk has no template")? + .to_owned(); + let encoded = serde_json::to_string(&manifest)?; + return Ok(LoadedChunk { + manifest, + revision: chunk_revision(&encoded, &template), + template, + guards: Vec::new(), + }); + } + let original = manifest_path.context("missing custom manifest path")?; + let options = self.shared.options; + let snapshot = io::read(gate, original, options.max_manifest_bytes).await?; + let manifest: ChunkManifest = + serde_json::from_str(&snapshot.source).context("invalid custom chunk manifest")?; + manifest.validate()?; + let parent = snapshot + .guard + .resolved + .parent() + .context("manifest has no parent directory")?; + let template_path = relative_template(parent, &manifest.template_path)?; + let template = io::read( + gate, + template_path + .to_str() + .context("template path must be UTF-8")?, + options.max_template_bytes, + ) + .await?; + ensure!( + template.guard.resolved.starts_with(parent), + "custom template must remain within the manifest directory" + ); + Ok(LoadedChunk { + manifest, + revision: chunk_revision(&snapshot.source, &template.source), + template: template.source, + guards: vec![snapshot.guard, template.guard], + }) + } +} + +fn relative_template(parent: &Path, value: &str) -> anyhow::Result { + ensure!( + !value.is_empty() + && !value.contains(['\\', ':']) + && !value + .split('/') + .any(|part| part.is_empty() || part == "." || part == ".."), + "custom template path must be portable and relative to the manifest" + ); + Ok(parent.join(value)) +} + +fn chunk_revision(manifest: &str, template: &str) -> String { + // Hash each bounded component before combining so no delimiter ambiguity + // or accidental cross-component concatenation can preserve the identity. + revision(&format!("{}:{}", revision(manifest), revision(template))) +} diff --git a/crates/tinytools-std/src/code/contract_tests.rs b/crates/tinytools-std/src/code/contract_tests.rs new file mode 100644 index 00000000..4720549f --- /dev/null +++ b/crates/tinytools-std/src/code/contract_tests.rs @@ -0,0 +1,175 @@ +//! Permission, exposure, schema and strict argument-boundary regressions. + +#![allow(clippy::unwrap_used)] + +use super::test_support::*; +use super::*; +use serde_json::{Value, json}; +use tinytools::{PermissionLevel, ToolExposure}; + +#[test] +fn deferred_tools_preserve_static_preview_access_and_classify_every_write_argument() { + let dir = tempfile::tempdir().unwrap(); + let tools = code_tools(TestGate::new(dir.path())).unwrap(); + assert_eq!( + tools.iter().map(|tool| tool.name()).collect::>(), + [ + "ast_outline", + "ast_symbol", + "ast_patch", + "chunk_search", + "chunk_inspect", + "chunk_apply" + ] + ); + for entry in &tools { + assert_eq!(entry.permission_level(), PermissionLevel::ReadOnly); + assert_eq!(entry.exposure(), ToolExposure::Deferred); + assert_eq!(entry.family(), Some("code")); + assert_eq!(entry.tags(), ["code", "structural", "pack:code"]); + assert!(entry.policy().classified); + let writes = ["ast_patch", "chunk_apply"].contains(&entry.name()); + assert_eq!(entry.policy().side_effects.writes_files, writes); + assert_eq!(entry.external_effect(), writes); + assert_eq!(entry.parameters_schema()["additionalProperties"], false); + for value in [ + Value::Null, + json!(false), + json!("true"), + json!(1), + json!({}), + ] { + let args = json!({"dry_run":value}); + assert_eq!( + entry.permission_level_with_args(&args), + if writes { + PermissionLevel::Write + } else { + PermissionLevel::ReadOnly + } + ); + assert_eq!(entry.external_effect_with_args(&args), writes); + assert_eq!(entry.is_concurrency_safe(&args), !writes); + } + assert_eq!( + entry.permission_level_with_args(&json!({"dry_run":true})), + PermissionLevel::ReadOnly + ); + assert!(!entry.external_effect_with_args(&json!({"dry_run":true}))); + assert!(entry.is_concurrency_safe(&json!({"dry_run":true}))); + } +} + +#[test] +fn rejects_invalid_host_bounds_before_constructing_tools() { + let dir = tempfile::tempdir().unwrap(); + let invalid = [ + CodeToolOptions { + max_argument_bytes: 0, + ..Default::default() + }, + CodeToolOptions { + max_manifest_bytes: 1_048_577, + ..Default::default() + }, + CodeToolOptions { + max_template_bytes: 0, + ..Default::default() + }, + CodeToolOptions { + max_output_bytes: 16_777_217, + ..Default::default() + }, + CodeToolOptions { + max_page_items: 0, + ..Default::default() + }, + CodeToolOptions { + max_page_lines: 4097, + ..Default::default() + }, + CodeToolOptions { + limits: tinytools_code::CodeLimits { + max_source_bytes: 0, + ..Default::default() + }, + ..Default::default() + }, + ]; + for options in invalid { + assert!(code_tools_with_options(TestGate::new(dir.path()), options).is_err()); + } +} + +#[tokio::test] +async fn malformed_or_oversized_arguments_fail_without_consuming_a_write_budget() { + let dir = tempfile::tempdir().unwrap(); + let gate = TestGate::new(dir.path()); + let tools = code_tools_with_options( + gate.clone(), + CodeToolOptions { + max_argument_bytes: 512, + max_output_bytes: 31, + ..Default::default() + }, + ) + .unwrap(); + for args in [ + json!(null), + json!({"path":"x","dry_run":"true"}), + json!({"unknown":"分析".repeat(300)}), + ] { + let result = tool(&tools, "ast_patch").execute(args).await.unwrap(); + assert!(result.is_error); + assert!(result.output().len() <= 31); + } + assert_eq!(gate.state.used.load(std::sync::atomic::Ordering::SeqCst), 0); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 0); +} + +#[test] +fn output_pages_and_error_truncation_preserve_utf8_and_report_continuations() { + let error = response::bounded_error("分析abc", 4); + assert_eq!(error.output(), "分"); + let records = response::records(&[1, 2, 3], usize::MAX, 2); + assert_eq!(records["items"], json!([])); + assert_eq!(records["next_offset"], Value::Null); + let page = response::source_page("one\ntwo\nthree", 2, 1).unwrap(); + assert_eq!(page["text"], "two\n"); + assert_eq!(page["next_start_line"], 3); + assert_eq!(page["truncated"], true); + assert!(response::source_page("", 0, 1).is_err()); +} + +#[tokio::test] +async fn argument_budget_counts_serialized_utf8_and_escape_bytes_at_the_exact_boundary() { + let dir = tempfile::tempdir().unwrap(); + // The compact object contributes twelve bytes around each query value. + for (query, bytes) in [("a", 13), ("\n", 14), ("分", 15), ("\"\\", 16), ("\0", 18)] { + let args = json!({"query":query}); + for (maximum, accepted) in [(bytes, true), (bytes - 1, false)] { + let gate = TestGate::new(dir.path()); + let tools = code_tools_with_options( + gate.clone(), + CodeToolOptions { + max_argument_bytes: maximum, + ..Default::default() + }, + ) + .unwrap(); + let result = tool(&tools, "chunk_search") + .execute(args.clone()) + .await + .unwrap(); + assert_eq!( + !result.is_error, accepted, + "query {query:?}, limit {maximum}" + ); + if !accepted { + assert!(result.output().contains("arguments exceed the byte budget")); + } + assert_eq!(gate.state.used.load(std::sync::atomic::Ordering::SeqCst), 0); + } + } + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 0); +} diff --git a/crates/tinytools-std/src/code/declaration.rs b/crates/tinytools-std/src/code/declaration.rs new file mode 100644 index 00000000..d18928f3 --- /dev/null +++ b/crates/tinytools-std/src/code/declaration.rs @@ -0,0 +1,221 @@ +//! Tool metadata, strict dispatch and the host's per-call workspace scope. + +use std::sync::Arc; + +use async_trait::async_trait; +use serde_json::{Value, json}; +use tinytools::{ + PermissionLevel, Tool, ToolAccess, ToolCallOptions, ToolExposure, ToolPolicy, ToolResult, + ToolRunContext, ToolRuntime, ToolSideEffects, WorkspaceAccess, +}; + +use super::{ + arguments, response, + types::{CodeTool, Kind}, +}; +use crate::filesystem::FsGate; + +#[async_trait] +impl Tool for CodeTool { + fn name(&self) -> &str { + self.kind.name() + } + + fn description(&self) -> &str { + match self.kind { + Kind::Outline => { + "Inspect a bounded lexical outline and folded source with content revisions and paging. This does not resolve semantic types." + } + Kind::Symbol => { + "Read one unambiguous lexical symbol by revision-bound ID, qualified name or local name; source is paginated." + } + Kind::Patch => { + "Apply one syntax-checked structural edit to an exact expected revision. Set dry_run to the literal boolean true for a read-only preview." + } + Kind::Search => { + "Search bundled typed code patterns by metadata. Dependencies are informational and are never installed." + } + Kind::Inspect => { + "Inspect a bundled pattern or gated custom manifest. Returns the content-bound chunk_revision required by chunk_apply." + } + Kind::Apply => { + "Render a typed pattern into a revision-checked file or selected container. Requires chunk_revision; use create_new for no-clobber creation, or expected_revision for edits. Literal dry_run true previews without writing." + } + } + } + + fn parameters_schema(&self) -> Value { + schema(self.kind) + } + + fn permission_level(&self) -> PermissionLevel { + PermissionLevel::ReadOnly + } + + fn permission_level_with_args(&self, args: &Value) -> PermissionLevel { + if self.kind.mutates() && !arguments::literal_preview(args) { + PermissionLevel::Write + } else { + PermissionLevel::ReadOnly + } + } + + fn exposure(&self) -> ToolExposure { + ToolExposure::Deferred + } + + fn family(&self) -> Option<&str> { + Some("code") + } + + fn tags(&self) -> Vec { + ["code", "structural", "pack:code"] + .map(str::to_owned) + .to_vec() + } + + fn is_concurrency_safe(&self, args: &Value) -> bool { + !self.kind.mutates() || arguments::literal_preview(args) + } + + fn external_effect(&self) -> bool { + self.kind.mutates() && self.gate.write_needs_approval() + } + + fn external_effect_with_args(&self, args: &Value) -> bool { + self.external_effect() && !arguments::literal_preview(args) + } + + fn policy(&self) -> ToolPolicy { + let writes = self.kind.mutates(); + ToolPolicy::classified() + .with_side_effects(ToolSideEffects { + read_only: !writes, + writes_files: writes, + ..ToolSideEffects::default() + }) + .with_runtime(ToolRuntime { + idempotent: !writes, + cancelable: true, + max_result_bytes: Some(self.shared.options.max_output_bytes), + ..ToolRuntime::default() + }) + .with_access(ToolAccess { + workspace: WorkspaceAccess::Scoped, + trusted_roots: vec![self.gate.action_dir().display().to_string()], + background_safe: !writes, + ..ToolAccess::default() + }) + } + + async fn execute(&self, args: Value) -> anyhow::Result { + self.execute_with_context(args, ToolCallOptions::default(), None) + .await + } + + async fn execute_with_context( + &self, + args: Value, + _options: ToolCallOptions, + context: Option<&dyn ToolRunContext>, + ) -> anyhow::Result { + let gate = context.and_then(ToolRunContext::workspace).map_or_else( + || Arc::clone(&self.gate), + |workspace| self.gate.scoped_to_workspace(&workspace.root), + ); + Ok(match self.dispatch(args, &gate).await { + Ok(result) => result, + Err(error) => { + response::bounded_error(&format!("{error:#}"), self.shared.options.max_output_bytes) + } + }) + } +} + +impl CodeTool { + async fn dispatch(&self, args: Value, gate: &Arc) -> anyhow::Result { + arguments::check_size(&args, self.shared.options.max_argument_bytes)?; + match self.kind { + Kind::Outline => self.outline(arguments::decode(args)?, gate).await, + Kind::Symbol => self.symbol(arguments::decode(args)?, gate).await, + Kind::Patch => self.patch(arguments::decode(args)?, gate).await, + Kind::Search => self.search(&arguments::decode(args)?), + Kind::Inspect => self.inspect(arguments::decode(args)?, gate).await, + Kind::Apply => self.apply(arguments::decode(args)?, gate).await, + } + } +} + +fn schema(kind: Kind) -> Value { + let mut properties = serde_json::Map::new(); + let required = match kind { + Kind::Outline | Kind::Symbol | Kind::Patch => { + properties.insert("path".into(), json!({"type":"string"})); + properties.insert("language".into(), json!({"type":"string","enum":["rust","typescript","tsx","javascript","python","go","json","toml"]})); + if kind == Kind::Outline { + add_record_page(&mut properties); + vec!["path"] + } else { + properties.insert("selector".into(), json!({"type":"string"})); + if kind == Kind::Symbol { + vec!["path", "selector"] + } else { + properties.insert("expected_revision".into(), revision_schema()); + properties.insert("operation".into(), json!({"type":"string","enum":["replace_body","replace_symbol","insert_import","append_member","wrap"]})); + properties.insert("code".into(), json!({"type":"string"})); + properties.insert("dry_run".into(), json!({"type":"boolean","default":false})); + vec!["path", "expected_revision", "operation", "code"] + } + } + } + Kind::Search => { + properties.insert("query".into(), json!({"type":"string","default":""})); + properties.insert("language".into(), json!({"type":"string","enum":["rust","typescript","tsx","javascript","python","go","json","toml"]})); + add_record_page(&mut properties); + vec![] + } + Kind::Inspect | Kind::Apply => { + properties.insert("id".into(), json!({"type":"string","description":"Bundled chunk ID; mutually exclusive with manifest_path"})); + properties.insert("manifest_path".into(), json!({"type":"string","description":"Custom manifest file permitted by FsGate; mutually exclusive with id"})); + if kind == Kind::Inspect { + add_record_page(&mut properties); + vec![] + } else { + properties.insert("path".into(), json!({"type":"string"})); + properties.insert("chunk_revision".into(), revision_schema()); + properties.insert("expected_revision".into(), revision_schema()); + properties.insert("selector".into(), json!({"type":"string"})); + properties.insert( + "create_new".into(), + json!({"type":"boolean","default":false}), + ); + properties.insert("dry_run".into(), json!({"type":"boolean","default":false})); + properties.insert("bindings".into(), json!({"type":"object","properties":{"parameters":{"type":"object"},"slots":{"type":"object","additionalProperties":{"type":"string"}}},"additionalProperties":false})); + vec!["path", "chunk_revision"] + } + } + }; + if matches!( + kind, + Kind::Outline | Kind::Symbol | Kind::Patch | Kind::Apply + ) { + properties.insert( + "start_line".into(), + json!({"type":"integer","minimum":1,"default":1}), + ); + properties.insert("line_count".into(), json!({"type":"integer","minimum":1})); + } + json!({"type":"object","properties":properties,"required":required,"additionalProperties":false}) +} + +fn add_record_page(properties: &mut serde_json::Map) { + properties.insert( + "offset".into(), + json!({"type":"integer","minimum":0,"default":0}), + ); + properties.insert("limit".into(), json!({"type":"integer","minimum":1})); +} + +fn revision_schema() -> Value { + json!({"type":"string","pattern":"^[0-9a-f]{64}$"}) +} diff --git a/crates/tinytools-std/src/code/inspect.rs b/crates/tinytools-std/src/code/inspect.rs new file mode 100644 index 00000000..e327fefd --- /dev/null +++ b/crates/tinytools-std/src/code/inspect.rs @@ -0,0 +1,127 @@ +//! Bounded structural and catalog inspection with explicit continuation fields. + +use std::sync::Arc; + +use anyhow::{Context, ensure}; +use serde_json::json; +use tinytools::ToolResult; +use tinytools_code::select_symbol; + +use super::{arguments, io, response, types::CodeTool}; +use crate::filesystem::FsGate; + +impl CodeTool { + pub(super) async fn outline( + &self, + args: arguments::Outline, + gate: &Arc, + ) -> anyhow::Result { + let options = self.shared.options; + let limit = arguments::page_size(args.limit, options.max_page_items)?; + let lines = arguments::page_size(args.line_count, options.max_page_lines)?; + let language = arguments::language(&args.path, args.language)?; + let snapshot = io::read(gate, &args.path, options.limits.max_source_bytes).await?; + let report = self.shared.engine.lock().analyze( + &snapshot.guard.resolved.to_string_lossy(), + language, + &snapshot.source, + )?; + response::bounded_json( + json!({ + "path": args.path, + "language": language, + "revision": report.revision, + "symbols": response::records(&report.symbols, args.offset, limit), + "references": response::records(&report.references, args.offset, limit), + "diagnostics": response::records(&report.diagnostics, args.offset, limit), + "symbols_truncated": report.symbols_truncated, + "references_truncated": report.references_truncated, + "diagnostics_truncated": report.diagnostics_truncated, + "skeleton": response::source_page(&report.skeleton, args.start_line, lines)?, + "cache_status": report.cache_status + }), + options.max_output_bytes, + ) + } + + pub(super) async fn symbol( + &self, + args: arguments::Symbol, + gate: &Arc, + ) -> anyhow::Result { + let options = self.shared.options; + let lines = arguments::page_size(args.line_count, options.max_page_lines)?; + let language = arguments::language(&args.path, args.language)?; + let snapshot = io::read(gate, &args.path, options.limits.max_source_bytes).await?; + let report = self.shared.engine.lock().analyze( + &snapshot.guard.resolved.to_string_lossy(), + language, + &snapshot.source, + )?; + let symbol = select_symbol(&report, &args.selector)?; + let source = snapshot + .source + .get(symbol.range.start_byte..symbol.range.end_byte) + .context("symbol range is outside its source snapshot")?; + response::bounded_json( + json!({ + "path": args.path, + "language": language, + "revision": report.revision, + "symbol": symbol, + "source": response::source_page(source, args.start_line, lines)? + }), + options.max_output_bytes, + ) + } + + pub(super) fn search(&self, args: &arguments::Search) -> anyhow::Result { + let options = self.shared.options; + let limit = arguments::page_size(args.limit, options.max_page_items)?; + ensure!( + args.query.len() <= 1024, + "chunk search query exceeds 1024 bytes" + ); + // The built-in catalog contains twelve entries; fifty is the pure + // search cap, so the adapter can page every bundled match exactly. + let matches = self.shared.catalog.search(&args.query, args.language, 50); + response::bounded_json( + json!({ + "query": args.query, + "chunks": response::records(&matches, args.offset, limit) + }), + options.max_output_bytes, + ) + } + + pub(super) async fn inspect( + &self, + args: arguments::Inspect, + gate: &Arc, + ) -> anyhow::Result { + let options = self.shared.options; + let limit = arguments::page_size(args.limit, options.max_page_items)?; + let chunk = self + .load_chunk(gate, args.id.as_deref(), args.manifest_path.as_deref()) + .await?; + let manifest = &chunk.manifest; + response::bounded_json( + json!({ + "id": manifest.id, + "summary": manifest.summary, + "language": manifest.language, + "target": manifest.target, + "chunk_revision": chunk.revision, + "manifest_revision": manifest.revision()?, + "template_revision": tinytools_code::revision(&chunk.template), + "template_path": manifest.template_path, + "parameters": response::records(&manifest.parameters, args.offset, limit), + "slots": response::records(&manifest.slots, args.offset, limit), + "tags": response::records(&manifest.tags, args.offset, limit), + "dependencies": response::records(&manifest.dependencies, args.offset, limit), + "forbidden_nodes": response::records(&manifest.forbidden_nodes, args.offset, limit) + }), + options.max_output_bytes, + ) + } +} diff --git a/crates/tinytools-std/src/code/inspection_tests.rs b/crates/tinytools-std/src/code/inspection_tests.rs new file mode 100644 index 00000000..51386e25 --- /dev/null +++ b/crates/tinytools-std/src/code/inspection_tests.rs @@ -0,0 +1,171 @@ +//! Real filesystem inspection, source paging and catalog-boundary regressions. + +#![allow(clippy::unwrap_used)] + +use super::test_support::*; +use super::*; +use serde_json::json; +use std::sync::atomic::Ordering; +use tinytools::{ToolCallOptions, WorkspaceDescriptor}; + +#[tokio::test] +async fn outline_and_symbol_pages_identify_the_exact_source_snapshot() { + let dir = tempfile::tempdir().unwrap(); + let source = "fn first() -> i32 {\n 1\n}\nfn second() {}\n"; + std::fs::write(dir.path().join("source.rs"), source).unwrap(); + let tools = code_tools(TestGate::new(dir.path())).unwrap(); + let outline = success( + tool(&tools, "ast_outline"), + json!({"path":"source.rs","limit":1,"line_count":1}), + ) + .await; + assert_eq!(outline["revision"], tinytools_code::revision(source)); + assert_eq!(outline["symbols"]["items"][0]["name"], "first"); + assert_eq!(outline["symbols"]["next_offset"], 1); + assert_eq!(outline["cache_status"], "fresh"); + let second = success( + tool(&tools, "ast_outline"), + json!({"path":"source.rs","offset":1,"limit":1}), + ) + .await; + assert_eq!(second["symbols"]["items"][0]["name"], "second"); + assert_eq!(second["cache_status"], "cached"); + let symbol = success(tool(&tools, "ast_symbol"), json!({"path":"source.rs","selector":outline["symbols"]["items"][0]["id"],"start_line":2,"line_count":1})).await; + assert_eq!(symbol["source"]["text"], " 1\n"); + assert_eq!(symbol["source"]["next_start_line"], 3); + assert_eq!(symbol["symbol"]["name"], "first"); + std::fs::write(dir.path().join("source.rs"), source.replace('1', "2")).unwrap(); + let changed = success(tool(&tools, "ast_outline"), json!({"path":"source.rs"})).await; + assert_ne!(changed["revision"], outline["revision"]); + assert_eq!(changed["cache_status"], "incremental"); +} + +#[tokio::test] +async fn denied_missing_binary_large_and_non_file_inputs_are_model_visible_errors() { + let dir = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + std::fs::write(outside.path().join("outside.rs"), "fn outside() {}").unwrap(); + std::fs::write(dir.path().join("binary.rs"), [0xff, 0xfe]).unwrap(); + std::fs::write(dir.path().join("large.rs"), " ".repeat(64)).unwrap(); + std::fs::create_dir(dir.path().join("folder.rs")).unwrap(); + let gate = TestGate::new(dir.path()); + let tools = code_tools_with_options( + gate.clone(), + CodeToolOptions { + limits: tinytools_code::CodeLimits { + max_source_bytes: 32, + ..Default::default() + }, + ..Default::default() + }, + ) + .unwrap(); + for path in [ + "missing.rs".to_owned(), + "binary.rs".into(), + "large.rs".into(), + "folder.rs".into(), + outside.path().join("outside.rs").display().to_string(), + "../outside.rs".into(), + ] { + assert!( + !failure(tool(&tools, "ast_outline"), json!({"path":path})) + .await + .is_empty() + ); + } + assert!( + failure(tool(&tools, "ast_outline"), json!({"path":"unknown"})) + .await + .contains("explicit") + ); + assert!( + failure( + tool(&tools, "ast_outline"), + json!({"path":"source.rs","limit":0}) + ) + .await + .contains("page size") + ); + assert!( + failure( + tool(&tools, "ast_symbol"), + json!({"path":"source.rs","selector":"x","line_count":81}) + ) + .await + .contains("page size") + ); + gate.state.denied.store(true, Ordering::SeqCst); + assert!( + failure(tool(&tools, "ast_outline"), json!({"path":"binary.rs"})) + .await + .contains("policy-blocked") + ); + assert_eq!(gate.state.used.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn the_trusted_run_workspace_scopes_reads_without_mutating_the_parent_gate() { + let parent = tempfile::tempdir().unwrap(); + let child = tempfile::tempdir().unwrap(); + std::fs::write(parent.path().join("source.rs"), "fn parent() {}").unwrap(); + std::fs::write(child.path().join("source.rs"), "fn child() {}").unwrap(); + let gate = TestGate::new(parent.path()); + let tools = code_tools(gate.clone()).unwrap(); + let context = Workspace(WorkspaceDescriptor::new(child.path().to_path_buf())); + let result = tool(&tools, "ast_outline") + .execute_with_context( + json!({"path":"source.rs"}), + ToolCallOptions::default(), + Some(&context), + ) + .await + .unwrap(); + assert!(!result.is_error, "{}", result.output()); + assert!(result.output().contains("child")); + let ordinary = success(tool(&tools, "ast_outline"), json!({"path":"source.rs"})).await; + assert_eq!(ordinary["symbols"]["items"][0]["name"], "parent"); + assert_eq!(gate.action_dir(), parent.path()); +} + +#[tokio::test] +async fn bundled_catalog_search_and_inspection_are_paged_metadata_only() { + let dir = tempfile::tempdir().unwrap(); + let gate = TestGate::new(dir.path()); + gate.state.denied.store(true, Ordering::SeqCst); + let tools = code_tools(gate.clone()).unwrap(); + let first = success(tool(&tools, "chunk_search"), json!({"limit":1})).await; + assert_eq!(first["chunks"]["total"], 12); + assert_eq!(first["chunks"]["next_offset"], 1); + let inspected = success( + tool(&tools, "chunk_inspect"), + json!({"id":"rust.fallible-function.v1","limit":1}), + ) + .await; + assert_eq!(inspected["parameters"]["items"][0]["name"], "name"); + assert_eq!(inspected["parameters"]["next_offset"], 1); + assert_eq!(inspected["chunk_revision"].as_str().unwrap().len(), 64); + assert!(inspected.get("source").is_none()); + assert!( + failure(tool(&tools, "chunk_inspect"), json!({})) + .await + .contains("exactly one") + ); + assert!( + failure( + tool(&tools, "chunk_inspect"), + json!({"id":"no-such-pattern"}) + ) + .await + .contains("no-such-pattern") + ); + assert!( + failure( + tool(&tools, "chunk_search"), + json!({"query":"x".repeat(1025)}) + ) + .await + .contains("1024") + ); + assert_eq!(gate.state.used.load(Ordering::SeqCst), 0); +} diff --git a/crates/tinytools-std/src/code/io.rs b/crates/tinytools-std/src/code/io.rs new file mode 100644 index 00000000..cec3e0b8 --- /dev/null +++ b/crates/tinytools-std/src/code/io.rs @@ -0,0 +1,126 @@ +//! Bounded regular-file reads and reauthorization of retained source guards. + +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use anyhow::{Context, anyhow, ensure}; +use tinytools_code::revision; +use tokio::io::AsyncReadExt; + +use crate::filesystem::FsGate; + +#[derive(Debug, Clone)] +pub(super) struct ReadGuard { + pub(super) original: String, + pub(super) resolved: PathBuf, + pub(super) revision: String, + pub(super) maximum: usize, +} + +pub(super) struct Snapshot { + pub(super) guard: ReadGuard, + pub(super) source: String, +} + +pub(super) async fn read( + gate: &Arc, + original: &str, + maximum: usize, +) -> anyhow::Result { + let resolved = gate + .validate_path(original) + .await + .map_err(|message| anyhow!(message))?; + let source = read_bounded(&resolved, maximum).await?; + Ok(Snapshot { + guard: ReadGuard { + original: original.into(), + resolved, + revision: revision(&source), + maximum, + }, + source, + }) +} + +pub(super) async fn read_bounded(path: &Path, maximum: usize) -> anyhow::Result { + let metadata = tokio::fs::metadata(path) + .await + .context("could not inspect structural input")?; + ensure!( + metadata.is_file(), + "structural input must be a regular file" + ); + let mut options = tokio::fs::OpenOptions::new(); + options.read(true); + // The gate already resolved the path. Refuse a later symlink swap, and + // never block on a FIFO substituted between the metadata and open calls. + #[cfg(unix)] + options.custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK); + let file = options + .open(path) + .await + .context("could not open structural input")?; + let metadata = file + .metadata() + .await + .context("could not inspect structural input")?; + ensure!( + metadata.is_file(), + "structural input must be a regular file" + ); + let bound = u64::try_from(maximum)?.saturating_add(1); + let mut bytes = Vec::new(); + file.take(bound) + .read_to_end(&mut bytes) + .await + .context("could not read structural input")?; + ensure!( + bytes.len() <= maximum, + "structural input exceeds the {maximum}-byte read budget" + ); + String::from_utf8(bytes).context("structural input must be valid UTF-8") +} + +pub(super) async fn verify(gate: &Arc, guard: &ReadGuard) -> anyhow::Result<()> { + let current = read(gate, &guard.original, guard.maximum).await?; + ensure!( + current.guard.resolved == guard.resolved, + "structural input path changed during the call; inspect it again" + ); + ensure!( + current.guard.revision == guard.revision, + "structural input revision changed during the call; inspect it again" + ); + Ok(()) +} + +pub(super) async fn resolve_target( + gate: &Arc, + original: &str, +) -> anyhow::Result { + let path = gate + .validate_parent_path(original) + .await + .map_err(|message| anyhow!(message))?; + reject_symlink(&path).await?; + Ok(path) +} + +pub(super) async fn reject_symlink(path: &Path) -> anyhow::Result<()> { + match tokio::fs::symlink_metadata(path).await { + Ok(metadata) => { + ensure!( + !metadata.file_type().is_symlink(), + "refusing to replace a symlink" + ); + ensure!( + metadata.is_file(), + "structural destination must be a regular file" + ); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error).context("could not inspect structural destination"), + } + Ok(()) +} diff --git a/crates/tinytools-std/src/code/mod.rs b/crates/tinytools-std/src/code/mod.rs new file mode 100644 index 00000000..f877aaf5 --- /dev/null +++ b/crates/tinytools-std/src/code/mod.rs @@ -0,0 +1,85 @@ +//! Optional structural tools behind the host's filesystem gate. +//! +//! [`code_tools`] builds six deferred tools sharing a bounded parser cache and +//! write lock. Only the literal JSON boolean `dry_run: true` previews edits; +//! real writes require exact content revisions and publish atomically. See the +//! module README for the argument and receipt contracts. + +mod arguments; +mod chunks; +mod declaration; +mod inspect; +mod io; +mod mutate; +mod publish; +mod response; +mod types; + +use std::sync::Arc; + +use parking_lot::Mutex; +use tinytools::Tool; +use tinytools_code::{ChunkCatalog, CodeEngine}; + +use crate::filesystem::FsGate; + +pub use types::CodeToolOptions; +use types::{CodeTool, Kind, Shared}; + +/// Build the six deferred structural tools with their default resource bounds. +/// +/// # Errors +/// +/// Returns an error if the bundled catalog or parser configuration is invalid. +pub fn code_tools(gate: Arc) -> anyhow::Result>> { + code_tools_with_options(gate, CodeToolOptions::default()) +} + +/// Build the structural tools with host-selected parser and I/O budgets. +/// +/// All returned tools share the same parser cache and mutation lock. They do +/// not install dependencies, execute generated code, or grant filesystem access. +/// +/// # Errors +/// +/// Returns an error for invalid resource bounds or an invalid bundled catalog. +pub fn code_tools_with_options( + gate: Arc, + options: CodeToolOptions, +) -> anyhow::Result>> { + options.validate()?; + let shared = Arc::new(Shared { + engine: Mutex::new(CodeEngine::new(options.limits)?), + catalog: ChunkCatalog::builtin()?, + write_lock: tokio::sync::Mutex::new(()), + options, + }); + Ok(Kind::ALL + .into_iter() + .map(move |kind| { + Box::new(CodeTool { + kind, + gate: Arc::clone(&gate), + shared: Arc::clone(&shared), + }) as Box + }) + .collect()) +} + +#[cfg(test)] +#[path = "chunk_tests.rs"] +mod chunk_tests; +#[cfg(test)] +#[path = "contract_tests.rs"] +mod contract_tests; +#[cfg(test)] +#[path = "inspection_tests.rs"] +mod inspection_tests; +#[cfg(test)] +#[path = "mutation_tests.rs"] +mod mutation_tests; +#[cfg(test)] +#[path = "publication_tests.rs"] +mod publication_tests; +#[cfg(test)] +mod test_support; diff --git a/crates/tinytools-std/src/code/mutate.rs b/crates/tinytools-std/src/code/mutate.rs new file mode 100644 index 00000000..3d91f6a6 --- /dev/null +++ b/crates/tinytools-std/src/code/mutate.rs @@ -0,0 +1,248 @@ +//! Revision-checked edits and chunk rendering with receipt preflight. + +use std::sync::Arc; + +use anyhow::{Context, ensure}; +use serde_json::{Value, json}; +use tinytools::ToolResult; +use tinytools_code::{ChunkTarget, PatchOperation, PatchRequest, render_chunk, revision}; + +use super::{ + arguments, io, + publish::{self, Destination}, + response, + types::CodeTool, +}; +use crate::{file_state, filesystem::FsGate}; + +struct Change { + destination: Destination, + source: String, + changed: bool, + language: tinytools_code::Language, +} + +impl CodeTool { + pub(super) async fn patch( + &self, + args: arguments::Patch, + gate: &Arc, + ) -> anyhow::Result { + arguments::require_revision(&args.expected_revision)?; + self.permit_write(args.dry_run)?; + let language = arguments::language(&args.path, args.language)?; + let resolved = io::resolve_target(gate, &args.path).await?; + let _group_lock = self.shared.write_lock.lock().await; + let _path_lock = file_state::acquire_path_lock(&resolved).await; + let snapshot = io::read( + gate, + &args.path, + self.shared.options.limits.max_source_bytes, + ) + .await?; + ensure!( + snapshot.guard.resolved == resolved, + "structural destination resolves inconsistently" + ); + let result = self.shared.engine.lock().patch( + &resolved.to_string_lossy(), + language, + &snapshot.source, + &PatchRequest { + expected_revision: args.expected_revision, + selector: args.selector, + operation: args.operation, + code: args.code, + }, + )?; + let change = Change { + destination: Destination { + original: args.path, + resolved, + before: Some(snapshot.guard), + }, + source: result.source, + changed: result.changed, + language, + }; + let receipt = self.receipt( + &change, + args.dry_run, + args.start_line, + args.line_count, + None, + )?; + if change.changed && !args.dry_run { + self.publish(gate, &change.destination, &[], &change.source) + .await?; + } + Ok(receipt) + } + + pub(super) async fn apply( + &self, + args: arguments::Apply, + gate: &Arc, + ) -> anyhow::Result { + arguments::require_revision(&args.chunk_revision)?; + ensure!( + args.create_new != args.expected_revision.is_some(), + "provide expected_revision for edits, or create_new: true without expected_revision" + ); + if let Some(expected) = &args.expected_revision { + arguments::require_revision(expected)?; + } + self.permit_write(args.dry_run)?; + let chunk = self + .load_chunk(gate, args.id.as_deref(), args.manifest_path.as_deref()) + .await?; + ensure!( + chunk.revision == args.chunk_revision, + "chunk revision mismatch; inspect the current manifest and template" + ); + ensure!( + !args.create_new || chunk.manifest.target == ChunkTarget::File, + "create_new requires a complete-file chunk" + ); + let rendered = render_chunk( + &mut self.shared.engine.lock(), + &chunk.manifest, + &chunk.template, + &args.bindings, + )?; + let resolved = io::resolve_target(gate, &args.path).await?; + let _group_lock = self.shared.write_lock.lock().await; + let _path_lock = file_state::acquire_path_lock(&resolved).await; + let snapshot = self.destination_snapshot(gate, &args, &resolved).await?; + let change = self.chunk_change(&args, resolved, snapshot, &rendered)?; + let extra = json!({ + "chunk_id": rendered.id, + "chunk_revision": chunk.revision, + "rendered_revision": rendered.revision, + "dependencies": rendered.dependencies + }); + let receipt = self.receipt( + &change, + args.dry_run, + args.start_line, + args.line_count, + Some(extra), + )?; + if change.changed && !args.dry_run { + self.publish(gate, &change.destination, &chunk.guards, &change.source) + .await?; + } + Ok(receipt) + } + + async fn destination_snapshot( + &self, + gate: &Arc, + args: &arguments::Apply, + resolved: &std::path::Path, + ) -> anyhow::Result> { + if args.create_new { + publish::require_absent(resolved).await?; + return Ok(None); + } + let snapshot = io::read( + gate, + &args.path, + self.shared.options.limits.max_source_bytes, + ) + .await?; + ensure!( + snapshot.guard.resolved == resolved, + "structural destination resolves inconsistently" + ); + ensure!( + Some(&snapshot.guard.revision) == args.expected_revision.as_ref(), + "source revision mismatch; inspect the current destination" + ); + Ok(Some(snapshot)) + } + + fn chunk_change( + &self, + args: &arguments::Apply, + resolved: std::path::PathBuf, + snapshot: Option, + rendered: &tinytools_code::RenderedChunk, + ) -> anyhow::Result { + let (source, changed) = if rendered.target == ChunkTarget::File { + ensure!( + args.selector.is_none(), + "a complete-file chunk does not accept a selector" + ); + if let Some(before) = &snapshot { + self.shared + .engine + .lock() + .validate(rendered.language, &before.source)?; + } + ( + rendered.source.clone(), + snapshot + .as_ref() + .is_none_or(|before| before.source != rendered.source), + ) + } else { + let before = snapshot + .as_ref() + .context("container chunks require an existing destination")?; + let operation = if rendered.target == ChunkTarget::FunctionBody { + PatchOperation::ReplaceBody + } else { + PatchOperation::AppendMember + }; + let patched = self.shared.engine.lock().patch( + &resolved.to_string_lossy(), + rendered.language, + &before.source, + &PatchRequest { + expected_revision: before.guard.revision.clone(), + selector: args.selector.clone(), + operation, + code: rendered.source.clone(), + }, + )?; + (patched.source, patched.changed) + }; + Ok(Change { + destination: Destination { + original: args.path.clone(), + resolved, + before: snapshot.map(|before| before.guard), + }, + source, + changed, + language: rendered.language, + }) + } + + fn receipt( + &self, + change: &Change, + dry_run: bool, + start_line: usize, + line_count: Option, + extra: Option, + ) -> anyhow::Result { + let options = self.shared.options; + let lines = arguments::page_size(line_count, options.max_page_lines)?; + let mut value = json!({ + "path": change.destination.original, + "language": change.language, + "before_revision": change.destination.before.as_ref().map(|before| &before.revision), + "revision": revision(&change.source), + "changed": change.changed, + "written": change.changed && !dry_run, + "dry_run": dry_run, + "source": response::source_page(&change.source, start_line, lines)? + }); + if let (Some(object), Some(Value::Object(extra))) = (value.as_object_mut(), extra) { + object.extend(extra); + } + response::bounded_json(value, options.max_output_bytes) + } +} diff --git a/crates/tinytools-std/src/code/mutation_tests.rs b/crates/tinytools-std/src/code/mutation_tests.rs new file mode 100644 index 00000000..7a446773 --- /dev/null +++ b/crates/tinytools-std/src/code/mutation_tests.rs @@ -0,0 +1,287 @@ +//! Preview, revision, typed-chunk and actual filesystem mutation regressions. + +#![allow(clippy::unwrap_used)] + +use super::test_support::*; +use super::*; +use serde_json::{Value, json}; +use std::sync::atomic::Ordering; +use tinytools_code::revision; + +const SOURCE: &str = "fn value() -> i32 { 1 }\n"; + +#[tokio::test] +async fn literal_previews_work_in_read_only_mode_and_do_not_consume_actions() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("source.rs"); + std::fs::write(&path, SOURCE).unwrap(); + let gate = TestGate::new(dir.path()); + gate.state.read_only.store(true, Ordering::SeqCst); + let tools = code_tools(gate.clone()).unwrap(); + let mut args = patch_args(SOURCE); + args["dry_run"] = json!(true); + let preview = success(tool(&tools, "ast_patch"), args.clone()).await; + assert_eq!(preview["written"], false); + assert_eq!(preview["dry_run"], true); + assert_eq!(preview["changed"], true); + assert_eq!(preview["before_revision"], revision(SOURCE)); + assert_ne!(preview["revision"], revision(SOURCE)); + assert_eq!(std::fs::read_to_string(&path).unwrap(), SOURCE); + args["dry_run"] = json!(false); + assert!( + failure(tool(&tools, "ast_patch"), args.clone()) + .await + .contains("policy-blocked") + ); + for invalid in [json!("true"), Value::Null, json!(1)] { + args["dry_run"] = invalid; + assert!( + failure(tool(&tools, "ast_patch"), args.clone()) + .await + .contains("arguments") + ); + } + assert_eq!(gate.state.used.load(Ordering::SeqCst), 0); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 1); +} + +#[tokio::test] +async fn a_real_patch_requires_the_current_revision_and_records_the_actual_write() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("source.rs"); + std::fs::write(&path, SOURCE).unwrap(); + let gate = TestGate::new(dir.path()); + let tools = code_tools(gate.clone()).unwrap(); + crate::file_state::init_global(true); + let resolved = path.canonicalize().unwrap(); + crate::file_state::record_read( + "code-parent", + resolved.clone(), + std::time::SystemTime::now(), + false, + std::time::Instant::now(), + ); + let receipt = crate::file_state::with_file_state_agent_id( + "code-writer".into(), + success(tool(&tools, "ast_patch"), patch_args(SOURCE)), + ) + .await; + let changed = std::fs::read_to_string(&path).unwrap(); + assert!(changed.contains('2')); + assert_eq!(receipt["written"], true); + assert_eq!(receipt["dry_run"], false); + assert_eq!(receipt["revision"], revision(&changed)); + assert_eq!(gate.state.used.load(Ordering::SeqCst), 1); + assert!(crate::file_state::check_stale_read("code-parent", &resolved).is_some()); + assert!( + failure(tool(&tools, "ast_patch"), patch_args(SOURCE)) + .await + .contains("revision") + ); + assert_eq!(std::fs::read_to_string(&path).unwrap(), changed); + assert_eq!(gate.state.used.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn concurrent_edits_from_one_snapshot_cannot_overwrite_each_other() { + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("source.rs"), SOURCE).unwrap(); + let gate = TestGate::new(dir.path()); + let tools = code_tools(gate.clone()).unwrap(); + let entry = tool(&tools, "ast_patch"); + let mut second = patch_args(SOURCE); + second["code"] = json!("3"); + let (first, second) = tokio::join!(entry.execute(patch_args(SOURCE)), entry.execute(second)); + assert_ne!(first.unwrap().is_error, second.unwrap().is_error); + assert_eq!(gate.state.used.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn chunks_require_inspected_content_and_create_without_clobbering() { + let dir = tempfile::tempdir().unwrap(); + let gate = TestGate::new(dir.path()); + let tools = code_tools(gate.clone()).unwrap(); + let inspected = success( + tool(&tools, "chunk_inspect"), + json!({"id":"rust.fallible-function.v1"}), + ) + .await; + let mut args = json!({"id":"rust.fallible-function.v1", "path":"new.rs", "create_new":true, + "chunk_revision":inspected["chunk_revision"], "bindings":{"parameters":{"name":"created"}}, "dry_run":true}); + let preview = success(tool(&tools, "chunk_apply"), args.clone()).await; + assert_eq!(preview["before_revision"], Value::Null); + assert_eq!(preview["written"], false); + assert!( + preview["source"]["text"] + .as_str() + .unwrap() + .contains("fn created") + ); + assert!(!dir.path().join("new.rs").exists()); + args["dry_run"] = json!(false); + let receipt = success(tool(&tools, "chunk_apply"), args.clone()).await; + assert_eq!(receipt["written"], true); + let original = std::fs::read_to_string(dir.path().join("new.rs")).unwrap(); + assert_eq!(receipt["revision"], revision(&original)); + assert!( + failure(tool(&tools, "chunk_apply"), args.clone()) + .await + .contains("existing") + ); + args["path"] = json!("another.rs"); + args["chunk_revision"] = json!("a".repeat(64)); + assert!( + failure(tool(&tools, "chunk_apply"), args) + .await + .contains("chunk revision") + ); + assert_eq!( + std::fs::read_to_string(dir.path().join("new.rs")).unwrap(), + original + ); + assert_eq!(gate.state.used.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn chunks_edit_current_files_bodies_and_members_through_the_same_revision_guard() { + let dir = tempfile::tempdir().unwrap(); + let tools = code_tools(TestGate::new(dir.path())).unwrap(); + std::fs::write(dir.path().join("source.rs"), SOURCE).unwrap(); + let file = success( + tool(&tools, "chunk_inspect"), + json!({"id":"rust.fallible-function.v1"}), + ) + .await; + let receipt = success( + tool(&tools, "chunk_apply"), + json!({"id":"rust.fallible-function.v1", "path":"source.rs", + "chunk_revision":file["chunk_revision"], "expected_revision":revision(SOURCE)}), + ) + .await; + assert_eq!(receipt["written"], true); + let body_source = "fn sum(left: u32, right: u32) -> Result { Ok(0) }"; + std::fs::write(dir.path().join("source.rs"), body_source).unwrap(); + let body = success( + tool(&tools, "chunk_inspect"), + json!({"id":"rust.checked-add.v1"}), + ) + .await; + let receipt = success( + tool(&tools, "chunk_apply"), + json!({"id":"rust.checked-add.v1", "path":"source.rs", "selector":"sum", + "chunk_revision":body["chunk_revision"], "expected_revision":revision(body_source)}), + ) + .await; + assert!( + receipt["source"]["text"] + .as_str() + .unwrap() + .contains("checked_add") + ); + custom_manifest(dir.path(), "member", "fn added(&self) {}\n"); + let member_source = "struct Thing;\nimpl Thing {}\n"; + std::fs::write(dir.path().join("source.rs"), member_source).unwrap(); + let outline = success(tool(&tools, "ast_outline"), json!({"path":"source.rs"})).await; + let selected = outline["symbols"]["items"] + .as_array() + .unwrap() + .iter() + .find(|item| item["kind"] == "impl_item") + .unwrap()["id"] + .clone(); + let member = success( + tool(&tools, "chunk_inspect"), + json!({"manifest_path":"manifest.json"}), + ) + .await; + let receipt = success( + tool(&tools, "chunk_apply"), + json!({"manifest_path":"manifest.json", "path":"source.rs", "selector":selected, + "chunk_revision":member["chunk_revision"], "expected_revision":revision(member_source)}), + ) + .await; + assert!( + receipt["source"]["text"] + .as_str() + .unwrap() + .contains("fn added") + ); +} + +#[tokio::test] +async fn inconsistent_chunk_destinations_and_invalid_bindings_never_write() { + let dir = tempfile::tempdir().unwrap(); + let gate = TestGate::new(dir.path()); + let tools = code_tools(gate.clone()).unwrap(); + let inspected = success( + tool(&tools, "chunk_inspect"), + json!({"id":"rust.fallible-function.v1"}), + ) + .await; + let base = json!({"id":"rust.fallible-function.v1", "path":"source.rs", "chunk_revision":inspected["chunk_revision"]}); + assert!( + failure(tool(&tools, "chunk_apply"), base.clone()) + .await + .contains("expected_revision") + ); + let mut args = base; + args["create_new"] = json!(true); + args["expected_revision"] = json!(revision(SOURCE)); + assert!( + failure(tool(&tools, "chunk_apply"), args.clone()) + .await + .contains("without expected_revision") + ); + args.as_object_mut().unwrap().remove("expected_revision"); + args["bindings"] = json!({"parameters":{"name":"not an identifier"}}); + assert!( + !failure(tool(&tools, "chunk_apply"), args.clone()) + .await + .is_empty() + ); + args["bindings"] = json!({}); + args["selector"] = json!("unnecessary"); + assert!( + failure(tool(&tools, "chunk_apply"), args) + .await + .contains("selector") + ); + let member = success( + tool(&tools, "chunk_inspect"), + json!({"id":"rust.public-field.v1"}), + ) + .await; + assert!( + failure( + tool(&tools, "chunk_apply"), + json!({"id":"rust.public-field.v1", "path":"new.rs", "create_new":true, + "chunk_revision":member["chunk_revision"]}) + ) + .await + .contains("complete-file") + ); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 0); + assert_eq!(gate.state.used.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn oversized_receipts_are_refused_before_any_write_or_budget_consumption() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("source.rs"); + std::fs::write(&path, SOURCE).unwrap(); + let gate = TestGate::new(dir.path()); + let tools = code_tools_with_options( + gate.clone(), + CodeToolOptions { + max_output_bytes: 160, + ..Default::default() + }, + ) + .unwrap(); + let error = failure(tool(&tools, "ast_patch"), patch_args(SOURCE)).await; + assert!(error.contains("output budget")); + assert!(error.len() <= 160); + assert_eq!(std::fs::read_to_string(&path).unwrap(), SOURCE); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 1); + assert_eq!(gate.state.used.load(Ordering::SeqCst), 0); +} diff --git a/crates/tinytools-std/src/code/publication_tests.rs b/crates/tinytools-std/src/code/publication_tests.rs new file mode 100644 index 00000000..841035be --- /dev/null +++ b/crates/tinytools-std/src/code/publication_tests.rs @@ -0,0 +1,225 @@ +//! Revision races, final policy decisions, atomicity and staging cleanup. + +#![allow(clippy::unwrap_used)] + +use super::test_support::*; +use super::*; +use serde_json::json; +use std::sync::atomic::Ordering; + +const SOURCE: &str = "fn value() -> i32 { 1 }\n"; + +#[tokio::test] +async fn an_external_write_after_admission_is_preserved_and_staging_is_removed() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("source.rs"); + std::fs::write(&path, SOURCE).unwrap(); + let gate = TestGate::new(dir.path()); + *gate.state.action.lock() = Some(Action::Write( + path.clone(), + "fn value() -> i32 { 99 }".into(), + )); + let tools = code_tools(gate.clone()).unwrap(); + assert!( + failure(tool(&tools, "ast_patch"), patch_args(SOURCE)) + .await + .contains("revision changed") + ); + assert!(std::fs::read_to_string(path).unwrap().contains("99")); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 1); +} + +#[tokio::test] +async fn original_path_reauthorization_and_late_read_only_policy_prevent_publication() { + for revoke_at in ["path", "action", "publication"] { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("source.rs"); + std::fs::write(&path, SOURCE).unwrap(); + let gate = TestGate::new(dir.path()); + if revoke_at == "action" { + *gate.state.action.lock() = Some(Action::Revoke); + } else if revoke_at == "publication" { + gate.state.revoke_at_recheck.store(true, Ordering::SeqCst); + } else { + gate.state.deny_recheck.store(true, Ordering::SeqCst); + } + let tools = code_tools(gate).unwrap(); + assert!( + failure(tool(&tools, "ast_patch"), patch_args(SOURCE)) + .await + .contains("policy-blocked") + ); + assert_eq!(std::fs::read_to_string(path).unwrap(), SOURCE); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 1); + } +} + +#[tokio::test] +async fn the_final_available_action_can_publish_after_reserving_its_budget() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("source.rs"); + std::fs::write(&path, SOURCE).unwrap(); + let gate = Arc::new(TestGate { + root: dir.path().to_path_buf(), + state: Arc::new(State::default()), + max_actions: 1, + }); + let tools = code_tools(gate.clone()).unwrap(); + let receipt = success(tool(&tools, "ast_patch"), patch_args(SOURCE)).await; + assert_eq!(receipt["written"], true); + assert!(std::fs::read_to_string(path).unwrap().contains('2')); + assert_eq!(gate.state.used.load(Ordering::SeqCst), 1); + assert!(gate.is_rate_limited()); +} + +#[tokio::test] +async fn depleted_and_racing_action_budgets_refuse_before_staging() { + for predepleted in [false, true] { + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("source.rs"), SOURCE).unwrap(); + let gate = TestGate::new(dir.path()); + if predepleted { + gate.state.used.store(100, Ordering::SeqCst); + } else { + gate.state.refuse_record.store(true, Ordering::SeqCst); + } + let tools = code_tools(gate).unwrap(); + assert!( + failure(tool(&tools, "ast_patch"), patch_args(SOURCE)) + .await + .contains("budget") + ); + assert_eq!( + std::fs::read_to_string(dir.path().join("source.rs")).unwrap(), + SOURCE + ); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 1); + } +} + +#[tokio::test] +async fn deleted_destinations_and_new_file_collisions_do_not_get_recreated_or_clobbered() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("source.rs"); + std::fs::write(&path, SOURCE).unwrap(); + let gate = TestGate::new(dir.path()); + *gate.state.action.lock() = Some(Action::Remove(path.clone())); + let tools = code_tools(gate.clone()).unwrap(); + assert!( + failure(tool(&tools, "ast_patch"), patch_args(SOURCE)) + .await + .contains("permissions") + ); + assert!(!path.exists()); + let inspected = success( + tool(&tools, "chunk_inspect"), + json!({"id":"rust.fallible-function.v1"}), + ) + .await; + *gate.state.action.lock() = Some(Action::Write(path.clone(), "external writer".into())); + assert!( + failure( + tool(&tools, "chunk_apply"), + json!({"id":"rust.fallible-function.v1","path":"source.rs", "create_new":true, + "chunk_revision":inspected["chunk_revision"]}) + ) + .await + .contains("existing") + ); + assert_eq!(std::fs::read_to_string(path).unwrap(), "external writer"); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 1); +} + +#[tokio::test] +async fn a_changed_custom_manifest_or_template_invalidates_publication() { + for manifest_changed in [false, true] { + let dir = tempfile::tempdir().unwrap(); + custom_manifest(dir.path(), "file", "fn created() {}\n"); + let gate = TestGate::new(dir.path()); + let tools = code_tools(gate.clone()).unwrap(); + let inspected = success( + tool(&tools, "chunk_inspect"), + json!({"manifest_path":"manifest.json"}), + ) + .await; + let path = dir.path().join(if manifest_changed { + "manifest.json" + } else { + "template.rs.txt" + }); + let changed = if manifest_changed { + std::fs::read_to_string(&path) + .unwrap() + .replace("Custom pattern", "Changed pattern") + } else { + "fn changed() {}\n".into() + }; + *gate.state.action.lock() = Some(Action::Write(path, changed)); + assert!( + failure( + tool(&tools, "chunk_apply"), + json!({"manifest_path":"manifest.json", "path":"new.rs", "create_new":true, + "chunk_revision":inspected["chunk_revision"]}) + ) + .await + .contains("revision changed") + ); + assert!(!dir.path().join("new.rs").exists()); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 2); + } +} + +#[tokio::test] +async fn cancellation_after_staging_keeps_the_original_and_removes_the_temporary_file() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("source.rs"); + std::fs::write(&path, SOURCE).unwrap(); + let entered = Arc::new(tokio::sync::Notify::new()); + let released = Arc::new(tokio::sync::Notify::new()); + let gate = Arc::new(TestGate { + root: dir.path().to_path_buf(), + max_actions: 100, + state: Arc::new(State { + pause_recheck: Some((entered.clone(), released)), + ..Default::default() + }), + }); + let tools = code_tools(gate).unwrap(); + let mut task = + tokio::spawn(async move { tool(&tools, "ast_patch").execute(patch_args(SOURCE)).await }); + let completed = tokio::select! { + () = entered.notified() => None, + result = &mut task => Some(result), + }; + assert!( + completed.is_none(), + "call did not reach its staging pause: {completed:?}" + ); + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + assert_eq!(std::fs::read_to_string(path).unwrap(), SOURCE); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 1); +} + +#[cfg(unix)] +#[tokio::test] +async fn replacement_preserves_permissions_and_symlink_destinations_are_refused() { + use std::os::unix::fs::{PermissionsExt, symlink}; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("source.rs"); + std::fs::write(&path, SOURCE).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o751)).unwrap(); + let tools = code_tools(TestGate::new(dir.path())).unwrap(); + success(tool(&tools, "ast_patch"), patch_args(SOURCE)).await; + assert_eq!( + std::fs::metadata(&path).unwrap().permissions().mode() & 0o777, + 0o751 + ); + std::fs::rename(&path, dir.path().join("real.rs")).unwrap(); + symlink(dir.path().join("real.rs"), &path).unwrap(); + assert!( + failure(tool(&tools, "ast_patch"), patch_args(SOURCE)) + .await + .contains("symlink") + ); +} diff --git a/crates/tinytools-std/src/code/publish.rs b/crates/tinytools-std/src/code/publish.rs new file mode 100644 index 00000000..cd4629ae --- /dev/null +++ b/crates/tinytools-std/src/code/publish.rs @@ -0,0 +1,133 @@ +//! Same-directory staging and atomic publication after final source checks. + +use std::io::Write; +use std::path::PathBuf; +use std::sync::Arc; + +use anyhow::{Context, ensure}; +use tempfile::NamedTempFile; + +use super::{io, types::CodeTool}; +use crate::{file_state, filesystem::FsGate}; + +pub(super) struct Destination { + pub(super) original: String, + pub(super) resolved: PathBuf, + pub(super) before: Option, +} + +impl CodeTool { + pub(super) fn permit_write(&self, preview: bool) -> anyhow::Result<()> { + if !preview { + self.require_write_authority()?; + ensure!( + !self.gate.is_rate_limited(), + "structural action budget exhausted" + ); + } + Ok(()) + } + + fn require_write_authority(&self) -> anyhow::Result<()> { + ensure!( + self.gate.can_act() && !self.gate.is_read_only(), + "[policy-blocked] structural writes are read-only under the host policy" + ); + Ok(()) + } + + pub(super) async fn publish( + &self, + gate: &Arc, + destination: &Destination, + guards: &[io::ReadGuard], + source: &str, + ) -> anyhow::Result<()> { + self.permit_write(false)?; + ensure!( + self.gate.record_action(), + "structural action budget exhausted" + ); + self.require_write_authority()?; + let staged = stage(destination, source).await?; + // Custom metadata and templates can change while parsing or staging. + // Revalidate their original paths as well as their exact content. + for guard in guards { + io::verify(gate, guard).await?; + } + let resolved = io::resolve_target(gate, &destination.original).await?; + ensure!( + resolved == destination.resolved, + "structural destination path changed during the call" + ); + if let Some(before) = &destination.before { + io::verify(gate, before).await?; + } else { + require_absent(&resolved).await?; + } + // Reauthorization can yield while the host revokes write access. + // The action is already reserved: recheck authority, not its budget. + self.require_write_authority()?; + if destination.before.is_some() { + staged + .persist(&resolved) + .map_err(|error| error.error) + .context("atomic structural replacement failed")?; + } else { + staged + .persist_noclobber(&resolved) + .map_err(|error| error.error) + .context("atomic structural creation failed")?; + } + if let Some(agent) = file_state::current_file_state_agent_id() { + file_state::record_write(&agent, resolved); + } + Ok(()) + } +} + +pub(super) async fn require_absent(path: &std::path::Path) -> anyhow::Result<()> { + match tokio::fs::symlink_metadata(path).await { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error).context("could not inspect creation destination"), + Ok(_) => anyhow::bail!("create_new refuses an existing destination"), + } +} + +async fn stage(destination: &Destination, source: &str) -> anyhow::Result { + let parent = destination + .resolved + .parent() + .context("destination has no parent")? + .to_path_buf(); + let permissions = if destination.before.is_some() { + Some( + tokio::fs::metadata(&destination.resolved) + .await + .context("could not inspect destination permissions")? + .permissions(), + ) + } else { + None + }; + let source = source.to_owned(); + tokio::task::spawn_blocking(move || { + let mut file = tempfile::Builder::new() + .prefix(".tinytools-code-") + .tempfile_in(parent) + .context("could not stage structural output; parent directory must exist")?; + file.write_all(source.as_bytes()) + .context("could not stage complete structural output")?; + if let Some(permissions) = permissions { + file.as_file() + .set_permissions(permissions) + .context("could not preserve file permissions")?; + } + file.as_file() + .sync_all() + .context("could not sync staged structural output")?; + Ok::<_, anyhow::Error>(file) + }) + .await + .context("structural staging task failed")? +} diff --git a/crates/tinytools-std/src/code/response.rs b/crates/tinytools-std/src/code/response.rs new file mode 100644 index 00000000..10d1b52a --- /dev/null +++ b/crates/tinytools-std/src/code/response.rs @@ -0,0 +1,57 @@ +//! JSON receipt preflight and byte-safe model-visible paging. + +use anyhow::ensure; +use serde::Serialize; +use serde_json::{Value, json}; +use tinytools::ToolResult; + +pub(super) fn bounded_json(value: Value, maximum: usize) -> anyhow::Result { + let result = ToolResult::json(value); + ensure!( + result.output().len() <= maximum, + "structural result exceeds the {maximum}-byte output budget; reduce the requested page" + ); + Ok(result) +} + +pub(super) fn bounded_error(message: &str, maximum: usize) -> ToolResult { + let mut end = message.len().min(maximum); + while !message.is_char_boundary(end) { + end -= 1; + } + ToolResult::error(&message[..end]) +} + +pub(super) fn records(items: &[T], offset: usize, limit: usize) -> Value { + let start = offset.min(items.len()); + let end = start.saturating_add(limit).min(items.len()); + json!({ + "items": &items[start..end], + "offset": start, + "total": items.len(), + "next_offset": (end < items.len()).then_some(end) + }) +} + +pub(super) fn source_page( + source: &str, + start_line: usize, + line_count: usize, +) -> anyhow::Result { + ensure!(start_line > 0, "start_line must be at least 1"); + let total = source.lines().count().max(1); + let start = start_line.saturating_sub(1).min(total); + let end = start.saturating_add(line_count).min(total); + let text: String = source + .split_inclusive('\n') + .skip(start) + .take(end - start) + .collect(); + Ok(json!({ + "text": text, + "start_line": start + 1, + "total_lines": total, + "next_start_line": (end < total).then_some(end + 1), + "truncated": start > 0 || end < total + })) +} diff --git a/crates/tinytools-std/src/code/test_support.rs b/crates/tinytools-std/src/code/test_support.rs new file mode 100644 index 00000000..b8ccca59 --- /dev/null +++ b/crates/tinytools-std/src/code/test_support.rs @@ -0,0 +1,204 @@ +//! A deterministic host gate and public-tool fixtures for native regressions. + +#![allow(clippy::unwrap_used)] + +use std::path::{Component, Path, PathBuf}; +use std::sync::{ + Arc, + atomic::{AtomicBool, AtomicUsize, Ordering}, +}; + +use async_trait::async_trait; +use parking_lot::Mutex; +use serde_json::{Value, json}; +use tinytools::{Tool, ToolRunContext, WorkspaceDescriptor}; + +use super::*; + +#[derive(Debug)] +pub(super) enum Action { + Write(PathBuf, String), + Revoke, + Remove(PathBuf), +} + +#[derive(Debug, Default)] +pub(super) struct State { + pub(super) read_only: AtomicBool, + pub(super) denied: AtomicBool, + pub(super) used: AtomicUsize, + pub(super) refuse_record: AtomicBool, + pub(super) action: Mutex>, + pub(super) parent_calls: AtomicUsize, + pub(super) deny_recheck: AtomicBool, + pub(super) revoke_at_recheck: AtomicBool, + pub(super) pause_recheck: Option<(Arc, Arc)>, +} + +#[derive(Debug)] +pub(super) struct TestGate { + pub(super) root: PathBuf, + pub(super) state: Arc, + pub(super) max_actions: usize, +} + +impl TestGate { + pub(super) fn new(root: &Path) -> Arc { + Arc::new(Self { + root: root.to_path_buf(), + state: Arc::new(State::default()), + max_actions: 100, + }) + } + + fn candidate(&self, path: &str) -> Result { + if self.state.denied.load(Ordering::SeqCst) || !self.is_path_string_allowed(path) { + return Err("[policy-blocked] test host denied path".into()); + } + let path = Path::new(path); + Ok(if path.is_absolute() { + path.to_path_buf() + } else { + self.root.join(path) + }) + } + + fn inside(&self, path: PathBuf) -> Result { + let root = self + .root + .canonicalize() + .map_err(|error| error.to_string())?; + if !path.starts_with(root) { + return Err("[policy-blocked] outside test workspace".into()); + } + Ok(path) + } +} + +#[async_trait] +impl FsGate for TestGate { + fn can_act(&self) -> bool { + !self.state.read_only.load(Ordering::SeqCst) + } + fn is_read_only(&self) -> bool { + !self.can_act() + } + fn is_rate_limited(&self) -> bool { + self.state.used.load(Ordering::SeqCst) >= self.max_actions + } + fn record_action(&self) -> bool { + if self.state.refuse_record.load(Ordering::SeqCst) { + return false; + } + let permitted = self.state.used.fetch_add(1, Ordering::SeqCst) < self.max_actions; + if let Some(action) = self.state.action.lock().take() { + match action { + Action::Write(path, source) => std::fs::write(path, source).unwrap(), + Action::Revoke => self.state.read_only.store(true, Ordering::SeqCst), + Action::Remove(path) => std::fs::remove_file(path).unwrap(), + } + } + permitted + } + fn write_needs_approval(&self) -> bool { + true + } + fn action_dir(&self) -> &Path { + &self.root + } + fn is_path_string_allowed(&self, path: &str) -> bool { + !path.contains('\0') + && !Path::new(path) + .components() + .any(|part| part == Component::ParentDir) + } + async fn validate_path(&self, path: &str) -> Result { + let full = self.candidate(path)?; + self.inside( + tokio::fs::canonicalize(full) + .await + .map_err(|error| error.to_string())?, + ) + } + async fn validate_parent_path(&self, path: &str) -> Result { + let calls = self.state.parent_calls.fetch_add(1, Ordering::SeqCst); + if calls > 0 { + if self.state.deny_recheck.load(Ordering::SeqCst) { + return Err("[policy-blocked] original path revoked".into()); + } + if let Some((entered, released)) = &self.state.pause_recheck { + entered.notify_one(); + released.notified().await; + } + if self.state.revoke_at_recheck.load(Ordering::SeqCst) { + self.state.read_only.store(true, Ordering::SeqCst); + } + } + let full = self.candidate(path)?; + let mut existing = full.parent().ok_or("missing parent")?.to_path_buf(); + while !existing.exists() { + existing = existing.parent().ok_or("missing ancestor")?.to_path_buf(); + } + let canonical = tokio::fs::canonicalize(&existing) + .await + .map_err(|error| error.to_string())?; + self.inside( + canonical.join( + full.strip_prefix(existing) + .map_err(|error| error.to_string())?, + ), + ) + } + fn scoped_to_workspace(&self, root: &Path) -> Arc { + Arc::new(Self { + root: root.to_path_buf(), + state: Arc::clone(&self.state), + max_actions: self.max_actions, + }) + } +} + +pub(super) struct Workspace(pub(super) WorkspaceDescriptor); +impl ToolRunContext for Workspace { + fn workspace(&self) -> Option<&WorkspaceDescriptor> { + Some(&self.0) + } +} + +pub(super) fn tool<'a>(tools: &'a [Box], name: &str) -> &'a dyn Tool { + tools + .iter() + .find(|tool| tool.name() == name) + .unwrap() + .as_ref() +} + +pub(super) async fn success(tool: &dyn Tool, args: Value) -> Value { + let result = tool.execute(args).await.unwrap(); + assert!(!result.is_error, "{}: {}", tool.name(), result.output()); + serde_json::from_str(&result.output()).unwrap() +} + +pub(super) async fn failure(tool: &dyn Tool, args: Value) -> String { + let result = tool.execute(args).await.unwrap(); + assert!(result.is_error, "unexpected success: {}", result.output()); + result.output() +} + +pub(super) fn patch_args(source: &str) -> Value { + json!({"path":"source.rs","expected_revision":tinytools_code::revision(source), + "selector":"value","operation":"replace_body","code":"2"}) +} + +pub(super) fn custom_manifest(root: &Path, target: &str, template: &str) { + std::fs::write(root.join("template.rs.txt"), template).unwrap(); + std::fs::write( + root.join("manifest.json"), + serde_json::to_vec(&json!({ + "id":"custom.pattern.v1", "summary":"Custom pattern", "language":"rust", + "target":target, "template_path":"template.rs.txt", "parameters":[], "slots":[] + })) + .unwrap(), + ) + .unwrap(); +} diff --git a/crates/tinytools-std/src/code/types.rs b/crates/tinytools-std/src/code/types.rs new file mode 100644 index 00000000..d73cff66 --- /dev/null +++ b/crates/tinytools-std/src/code/types.rs @@ -0,0 +1,110 @@ +//! Host-configured budgets and the shared native adapter state. + +use std::sync::Arc; + +use anyhow::ensure; +use parking_lot::Mutex; +use tinytools_code::{ChunkCatalog, CodeEngine, CodeLimits}; + +use crate::filesystem::FsGate; + +/// Resource limits for the optional structural tools. +/// +/// Source and parser-cache limits live in [`CodeLimits`]. The remaining limits +/// cover native argument decoding, metadata, templates and model-visible output. +#[derive(Debug, Clone, Copy)] +pub struct CodeToolOptions { + /// Pure parser, source, cache and record bounds. + pub limits: CodeLimits, + /// Maximum serialized argument bytes accepted by one invocation. + pub max_argument_bytes: usize, + /// Maximum bytes read from a custom manifest. + pub max_manifest_bytes: usize, + /// Maximum bytes read from a custom template. + pub max_template_bytes: usize, + /// Maximum UTF-8 bytes in the model-visible JSON or error text. + pub max_output_bytes: usize, + /// Maximum records returned per list page; also the default page size. + pub max_page_items: usize, + /// Maximum source lines returned per page; also the default page size. + pub max_page_lines: usize, +} + +impl Default for CodeToolOptions { + fn default() -> Self { + Self { + limits: CodeLimits::default(), + max_argument_bytes: 1_048_576, + max_manifest_bytes: 65_536, + max_template_bytes: 262_144, + max_output_bytes: 65_536, + max_page_items: 32, + max_page_lines: 80, + } + } +} + +impl CodeToolOptions { + pub(super) fn validate(self) -> anyhow::Result<()> { + for (name, value, maximum) in [ + ("argument", self.max_argument_bytes, 16_777_216), + ("manifest", self.max_manifest_bytes, 1_048_576), + ("template", self.max_template_bytes, 16_777_216), + ("output", self.max_output_bytes, 16_777_216), + ("page item", self.max_page_items, 1024), + ("page line", self.max_page_lines, 4096), + ] { + ensure!(value > 0 && value <= maximum, "invalid {name} budget"); + } + Ok(()) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum Kind { + Outline, + Symbol, + Patch, + Search, + Inspect, + Apply, +} + +impl Kind { + pub(super) const ALL: [Self; 6] = [ + Self::Outline, + Self::Symbol, + Self::Patch, + Self::Search, + Self::Inspect, + Self::Apply, + ]; + + pub(super) fn name(self) -> &'static str { + match self { + Self::Outline => "ast_outline", + Self::Symbol => "ast_symbol", + Self::Patch => "ast_patch", + Self::Search => "chunk_search", + Self::Inspect => "chunk_inspect", + Self::Apply => "chunk_apply", + } + } + + pub(super) fn mutates(self) -> bool { + matches!(self, Self::Patch | Self::Apply) + } +} + +pub(super) struct Shared { + pub(super) engine: Mutex, + pub(super) catalog: ChunkCatalog, + pub(super) write_lock: tokio::sync::Mutex<()>, + pub(super) options: CodeToolOptions, +} + +pub(super) struct CodeTool { + pub(super) kind: Kind, + pub(super) gate: Arc, + pub(super) shared: Arc, +} diff --git a/crates/tinytools-std/src/lib.rs b/crates/tinytools-std/src/lib.rs index af17de9f..a5d1d28b 100644 --- a/crates/tinytools-std/src/lib.rs +++ b/crates/tinytools-std/src/lib.rs @@ -15,6 +15,8 @@ //! - [`detect_tools`] — `PATH` probing and the read-only `detect_tools` tool. //! - [`sanitize`] — lexical sanitization and truncation for untrusted metadata. //! - [`timeout`] — tool timeout bounds, process-group kill and child reaping. +//! - `code` (optional feature) — gated structural inspection and revision-safe +//! edits, with typed boilerplate and bounded JSON receipts. //! //! # Example //! @@ -33,6 +35,11 @@ //! assert_eq!(tool.permission_level(), PermissionLevel::ReadOnly); //! ``` +#[cfg(feature = "code")] +pub mod code; +#[cfg(feature = "code")] +pub use code::{CodeToolOptions, code_tools, code_tools_with_options}; + pub mod detect_tools; pub mod file_state; pub mod filesystem; diff --git a/docs/plans/README.md b/docs/plans/README.md index 797e80a7..b11907fb 100644 --- a/docs/plans/README.md +++ b/docs/plans/README.md @@ -22,3 +22,5 @@ into the plan. See [`example-retry-policy.md`](example-retry-policy.md) for a test-first sample. Completed plan: [collapsed tools and standard helpers](collapsed-tools-and-standard-helpers.md). + +Active: [structural code and boilerplate](structural-code.md). diff --git a/docs/plans/structural-code.md b/docs/plans/structural-code.md new file mode 100644 index 00000000..d5ff0409 --- /dev/null +++ b/docs/plans/structural-code.md @@ -0,0 +1,44 @@ +# Structural code implementation + +Spec: [Structural code and boilerplate](../specs/structural-code.md). + +1. Add public typed contracts, explicit grammars and query fixtures under + `crates/tinytools-code`; test parser/cache and error boundaries first. +2. Add lexical analysis and folded bodies, then revision-checked edits with + regressions for ambiguity, imports, comments and UTF-8 boundaries. +3. Add typed manifests, twelve patterns and one-pass rendering; test injection, + generated-node and expansion-budget failures. +4. Add the optional `crates/tinytools-std/src/code/` adapter with host gates, + bounded I/O, previews and revision-guarded atomic publication tests. +5. Update public/module docs and benchmark. Checkpoint focused validated source. +6. Run formatting, strict all-target/all-feature Clippy, build and tests; then + default-feature tests, strict rustdoc and the unchanged per-file coverage gate. +7. Publish owner source, then move consumer pointers in separate commits. + +## Completion + +- [x] Pure implementation and regression tests +- [x] Native adapter and regression tests +- [x] Documentation and benchmark +- [x] Focused source checkpoint +- [ ] Four contract gates, default tests and strict rustdoc +- [ ] 90% per-file source coverage +- [ ] Owner publication and separate consumer pointer updates + +## Focused validation checkpoint + +On 2026-10-18, `cargo test -p tinytools-code --all-targets --locked --offline -j1` +passed all 41 integration tests and ran the fixed-fixture benchmark. +`cargo clippy -p tinytools-code --all-targets --locked --offline -j1 -- -D warnings`, +scoped formatting and `git diff --check` also passed. Regressions cover Rust +comma/comment/closing-indent preservation, inner attributes before imports, +control-character string escaping, Python future-import placement, and Go +member contexts. + +The native adapter passed 32 focused tests with `cargo test -p tinytools-std +--features code --lib code:: --locked --offline`, strict all-target/all-feature +Clippy, scoped formatting and diff checks. Its regressions include exact JSON +byte budgets, late authority revocation, the last available action, stale +revisions, custom template containment, cancellation and atomic publication. +This is a 73-test source checkpoint. The complete workspace gates, default +tests, strict rustdoc, declared MSRV and per-file coverage remain pending. diff --git a/docs/specs/README.md b/docs/specs/README.md index e26eb135..c2ab3a8c 100644 --- a/docs/specs/README.md +++ b/docs/specs/README.md @@ -23,3 +23,5 @@ the contract; production code still belongs under `src/`. See [`example-retry-policy.md`](example-retry-policy.md) for a complete sample. Implemented host-facing helper contracts: [collapsed tools and standard helpers](collapsed-tools-and-standard-helpers.md), [tool rules](tool-rules.md). + +Accepted: [structural code and boilerplate](structural-code.md). diff --git a/docs/specs/structural-code.md b/docs/specs/structural-code.md new file mode 100644 index 00000000..90f806b6 --- /dev/null +++ b/docs/specs/structural-code.md @@ -0,0 +1,53 @@ +# Structural code and boilerplate + +Status: Accepted. Owner: TinyTools. + +## Problem and boundary + +Agents need bounded structural inspection and revision-checked edits without +shipping whole files or rewriting unrelated declarations. A pure engine owns +syntax and rendering; optional native tools own filesystem mechanics behind the +host's `FsGate`. The dependency-light `tinytools` vocabulary remains unchanged. + +## Behavior + +`tinytools-code` accepts explicit Rust, TypeScript, TSX, JavaScript, Python, Go, +JSON, and TOML languages. It returns lexical symbols/references, syntax +diagnostics and folded source. A bounded cache reuses unchanged trees and applies +UTF-8-safe incremental edits. Cache byte accounting measures retained source, +not Tree-sitter's native allocations. Results disclose record truncation. + +Patches require an exact source revision and an unambiguous symbol selector. +They replace bodies or declarations, insert imports, append members, or wrap one +node. Original and final source must parse. Source outside the selected edit is +preserved. A truncated outline only permits revision-addressed IDs. These are +syntax guarantees, not typechecking or semantic reference resolution. + +A typed chunk catalog offers twelve starter patterns. Search and inspect return +metadata. Rendering expands typed parameters and standalone comment slots in +one pass; inserted text is never interpreted as another template directive. +Manifests may constrain generated node kinds. Custom manifests use portable +relative template paths. Dependency declarations are informational and never +install anything. Template syntax, expansions and output are bounded. + +`tinytools-std`'s optional `code` feature exports deferred `ast_outline`, +`ast_symbol`, `ast_patch`, `chunk_search`, `chunk_inspect`, and `chunk_apply` tools. +Every filesystem read/write passes the host gate, including custom manifests and +templates. Tools bound source, manifest, template and response bytes and expose +paging for inspection. Receipts are JSON. Mutations require caller-provided +expected revisions. Only the literal JSON boolean `dry_run: true` grants a +read-only preview; static minimum permission is ReadOnly. Preview has no external +effect. Responses are budgeted before publication; files are staged, revision +guards rechecked, and atomically published with no-clobber creation or +permission-preserving replacement. Cancellation cannot expose a partial file. + +## Acceptance + +All grammars, cache paths, ambiguity, revision failures, import prologues, +container edits, slot/type restrictions, budget failures and native filesystem +races have deterministic regressions. The four workspace gates, default feature +tests, strict rustdoc and unchanged 90% per-file coverage gate must pass. Engine +benchmarks report source/folded bytes and timings without speed or token claims. +Owner source is committed before consumers move gitlinks in separate commits. + +No blocking open questions. See the [implementation plan](../plans/structural-code.md). From 206f1637d295908604c8b8e0c75662446a24a41a Mon Sep 17 00:00:00 2001 From: Farnam Date: Sun, 18 Oct 2026 16:19:15 +0330 Subject: [PATCH 02/10] Align structural inspection defaults and strict test assertions --- crates/tinytools-std/src/code/README.md | 9 +- crates/tinytools-std/src/code/arguments.rs | 6 + crates/tinytools-std/src/code/chunk_tests.rs | 4 +- crates/tinytools-std/src/code/declaration.rs | 12 +- crates/tinytools-std/src/code/inspect.rs | 39 +++--- .../src/code/inspection_tests.rs | 119 +++++++++++++++++- .../tinytools-std/src/code/mutation_tests.rs | 4 +- docs/plans/structural-code.md | 18 ++- docs/specs/structural-code.md | 7 ++ 9 files changed, 189 insertions(+), 29 deletions(-) diff --git a/crates/tinytools-std/src/code/README.md b/crates/tinytools-std/src/code/README.md index 742f9c10..d213edb1 100644 --- a/crates/tinytools-std/src/code/README.md +++ b/crates/tinytools-std/src/code/README.md @@ -33,7 +33,7 @@ inspection and patch tools infer it from a supported filename extension. | Tool | Required arguments | Optional arguments | | --- | --- | --- | -| `ast_outline` | `path` | `language`, `offset`, `limit`, `start_line`, `line_count` | +| `ast_outline` | `path` | `language`, `level`, `offset`, `limit`, `start_line`, `line_count` | | `ast_symbol` | `path`, `selector` | `language`, `start_line`, `line_count` | | `ast_patch` | `path`, `expected_revision`, `operation`, `code` | `language`, `selector`, `dry_run`, `start_line`, `line_count` | | `chunk_search` | None | `query`, `language`, `offset`, `limit` | @@ -73,10 +73,17 @@ Successful results contain a JSON block. List pages use this shape: ``` `offset` is zero-based. `limit` is positive and bounded by `max_page_items`. +`chunk_search` defaults to five results, or the configured `max_page_items` if +that is smaller; an explicit `limit` can select any positive size up to that +bound. Other record lists default to `max_page_items`. Outline symbols, references and diagnostics each have their own page object. Their separate truncation flags disclose parser collection limits; a page's `total` counts retained records, not records beyond those limits. +`ast_outline` accepts `level: 0` for metadata only, with no `skeleton` field. +The default `level: 1` includes a folded source page alongside that metadata. +Other levels and noninteger values are rejected before reading the file. + Source and folded-skeleton pages use this shape: ```json diff --git a/crates/tinytools-std/src/code/arguments.rs b/crates/tinytools-std/src/code/arguments.rs index 45081651..7a0fae31 100644 --- a/crates/tinytools-std/src/code/arguments.rs +++ b/crates/tinytools-std/src/code/arguments.rs @@ -68,6 +68,8 @@ pub(super) fn require_revision(value: &str) -> anyhow::Result<()> { pub(super) struct Outline { pub(super) path: String, pub(super) language: Option, + #[serde(default = "outline_level")] + pub(super) level: u8, #[serde(default)] pub(super) offset: usize, pub(super) limit: Option, @@ -147,3 +149,7 @@ pub(super) struct Apply { const fn first_line() -> usize { 1 } + +const fn outline_level() -> u8 { + 1 +} diff --git a/crates/tinytools-std/src/code/chunk_tests.rs b/crates/tinytools-std/src/code/chunk_tests.rs index 1451b914..1e52c64a 100644 --- a/crates/tinytools-std/src/code/chunk_tests.rs +++ b/crates/tinytools-std/src/code/chunk_tests.rs @@ -63,12 +63,12 @@ async fn malformed_manifests_and_nonportable_templates_fail_before_any_effect() "id":"custom.pattern.v1", "summary":"Pattern", "language":"rust", "template_path":path })).unwrap()).unwrap(); assert!( - !failure( + failure( tool(&tools, "chunk_inspect"), json!({"manifest_path":"manifest.json"}) ) .await - .is_empty() + .contains("template_path must be a portable, relative path") ); } std::fs::write(dir.path().join("manifest.json"), "{not json}").unwrap(); diff --git a/crates/tinytools-std/src/code/declaration.rs b/crates/tinytools-std/src/code/declaration.rs index d18928f3..dd5bd898 100644 --- a/crates/tinytools-std/src/code/declaration.rs +++ b/crates/tinytools-std/src/code/declaration.rs @@ -45,7 +45,7 @@ impl Tool for CodeTool { } fn parameters_schema(&self) -> Value { - schema(self.kind) + schema(self.kind, self.shared.options.max_page_items) } fn permission_level(&self) -> PermissionLevel { @@ -146,13 +146,17 @@ impl CodeTool { } } -fn schema(kind: Kind) -> Value { +fn schema(kind: Kind, max_page_items: usize) -> Value { let mut properties = serde_json::Map::new(); let required = match kind { Kind::Outline | Kind::Symbol | Kind::Patch => { properties.insert("path".into(), json!({"type":"string"})); properties.insert("language".into(), json!({"type":"string","enum":["rust","typescript","tsx","javascript","python","go","json","toml"]})); if kind == Kind::Outline { + properties.insert( + "level".into(), + json!({"type":"integer","enum":[0,1],"default":1,"description":"0 returns metadata only; 1 also returns the folded source page"}), + ); add_record_page(&mut properties); vec!["path"] } else { @@ -172,6 +176,10 @@ fn schema(kind: Kind) -> Value { properties.insert("query".into(), json!({"type":"string","default":""})); properties.insert("language".into(), json!({"type":"string","enum":["rust","typescript","tsx","javascript","python","go","json","toml"]})); add_record_page(&mut properties); + properties.insert( + "limit".into(), + json!({"type":"integer","minimum":1,"maximum":max_page_items,"default":max_page_items.min(5)}), + ); vec![] } Kind::Inspect | Kind::Apply => { diff --git a/crates/tinytools-std/src/code/inspect.rs b/crates/tinytools-std/src/code/inspect.rs index e327fefd..e12670f3 100644 --- a/crates/tinytools-std/src/code/inspect.rs +++ b/crates/tinytools-std/src/code/inspect.rs @@ -17,6 +17,8 @@ impl CodeTool { gate: &Arc, ) -> anyhow::Result { let options = self.shared.options; + ensure!(args.level <= 1, "outline level must be 0 or 1"); + ensure!(args.start_line > 0, "start_line must be at least 1"); let limit = arguments::page_size(args.limit, options.max_page_items)?; let lines = arguments::page_size(args.line_count, options.max_page_lines)?; let language = arguments::language(&args.path, args.language)?; @@ -26,22 +28,22 @@ impl CodeTool { language, &snapshot.source, )?; - response::bounded_json( - json!({ - "path": args.path, - "language": language, - "revision": report.revision, - "symbols": response::records(&report.symbols, args.offset, limit), - "references": response::records(&report.references, args.offset, limit), - "diagnostics": response::records(&report.diagnostics, args.offset, limit), - "symbols_truncated": report.symbols_truncated, - "references_truncated": report.references_truncated, - "diagnostics_truncated": report.diagnostics_truncated, - "skeleton": response::source_page(&report.skeleton, args.start_line, lines)?, - "cache_status": report.cache_status - }), - options.max_output_bytes, - ) + let mut result = json!({ + "path": args.path, + "language": language, + "revision": report.revision, + "symbols": response::records(&report.symbols, args.offset, limit), + "references": response::records(&report.references, args.offset, limit), + "diagnostics": response::records(&report.diagnostics, args.offset, limit), + "symbols_truncated": report.symbols_truncated, + "references_truncated": report.references_truncated, + "diagnostics_truncated": report.diagnostics_truncated, + "cache_status": report.cache_status + }); + if args.level == 1 { + result["skeleton"] = response::source_page(&report.skeleton, args.start_line, lines)?; + } + response::bounded_json(result, options.max_output_bytes) } pub(super) async fn symbol( @@ -77,7 +79,10 @@ impl CodeTool { pub(super) fn search(&self, args: &arguments::Search) -> anyhow::Result { let options = self.shared.options; - let limit = arguments::page_size(args.limit, options.max_page_items)?; + let limit = arguments::page_size( + Some(args.limit.unwrap_or(options.max_page_items.min(5))), + options.max_page_items, + )?; ensure!( args.query.len() <= 1024, "chunk search query exceeds 1024 bytes" diff --git a/crates/tinytools-std/src/code/inspection_tests.rs b/crates/tinytools-std/src/code/inspection_tests.rs index 51386e25..397ef056 100644 --- a/crates/tinytools-std/src/code/inspection_tests.rs +++ b/crates/tinytools-std/src/code/inspection_tests.rs @@ -40,6 +40,118 @@ async fn outline_and_symbol_pages_identify_the_exact_source_snapshot() { assert_eq!(changed["cache_status"], "incremental"); } +#[tokio::test] +async fn outline_levels_preserve_metadata_and_default_to_paged_folded_source() { + let dir = tempfile::tempdir().unwrap(); + let source = "fn first() -> i32 {\n 1\n}\nfn second() {}\n"; + std::fs::write(dir.path().join("source.rs"), source).unwrap(); + let tools = code_tools(TestGate::new(dir.path())).unwrap(); + let entry = tool(&tools, "ast_outline"); + let metadata = success(entry, json!({"path":"source.rs","level":0,"limit":1})).await; + let default = success(entry, json!({"path":"source.rs","limit":1,"line_count":1})).await; + let explicit = success( + entry, + json!({"path":"source.rs","level":1,"limit":1,"line_count":1}), + ) + .await; + assert!(metadata.get("skeleton").is_none()); + assert_eq!(metadata["revision"], tinytools_code::revision(source)); + assert_eq!(metadata["symbols"], default["symbols"]); + assert_eq!(metadata["symbols"]["next_offset"], 1); + assert_eq!(default["skeleton"], explicit["skeleton"]); + assert_eq!(default["skeleton"]["start_line"], 1); + assert_eq!(default["skeleton"]["next_start_line"], 2); + assert_eq!( + entry.parameters_schema()["properties"]["level"]["default"], + 1 + ); + assert_eq!( + entry.parameters_schema()["properties"]["level"]["enum"], + json!([0, 1]) + ); +} + +#[tokio::test] +async fn outline_rejects_unsupported_or_malformed_levels_before_reading_a_path() { + let dir = tempfile::tempdir().unwrap(); + let gate = TestGate::new(dir.path()); + gate.state.denied.store(true, Ordering::SeqCst); + let tools = code_tools(gate.clone()).unwrap(); + for level in [ + json!(2), + json!(255), + json!(-1), + json!(256), + json!(1.5), + json!("1"), + json!(true), + json!(null), + ] { + let error = failure( + tool(&tools, "ast_outline"), + json!({"path":"missing.rs","level":level}), + ) + .await; + assert!( + error.contains("outline level") || error.contains("invalid structural tool arguments") + ); + assert!(!error.contains("policy-blocked")); + } + assert!( + failure( + tool(&tools, "ast_outline"), + json!({"path":"missing.rs","level":0,"start_line":0}) + ) + .await + .contains("start_line") + ); + assert_eq!(gate.state.used.load(Ordering::SeqCst), 0); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 0); +} + +#[tokio::test] +async fn chunk_search_defaults_to_five_bounded_results_with_explicit_continuation() { + let dir = tempfile::tempdir().unwrap(); + for maximum in [2, 32] { + let tools = code_tools_with_options( + TestGate::new(dir.path()), + CodeToolOptions { + max_page_items: maximum, + ..Default::default() + }, + ) + .unwrap(); + let entry = tool(&tools, "chunk_search"); + let first = success(entry, json!({})).await; + let default_limit = maximum.min(5); + assert_eq!( + first["chunks"]["items"].as_array().unwrap().len(), + default_limit + ); + assert_eq!(first["chunks"]["total"], 12); + assert_eq!(first["chunks"]["next_offset"], default_limit); + assert_eq!( + entry.parameters_schema()["properties"]["limit"]["default"], + default_limit + ); + let next = success(entry, json!({"offset":10,"limit":maximum})).await; + assert_eq!(next["chunks"]["items"].as_array().unwrap().len(), 2); + assert_eq!(next["chunks"]["next_offset"], json!(null)); + let explicit = success(entry, json!({"limit":maximum})).await; + assert_eq!( + explicit["chunks"]["items"].as_array().unwrap().len(), + maximum.min(12) + ); + for invalid in [0, maximum + 1] { + assert!( + failure(entry, json!({"limit":invalid})) + .await + .contains("page size") + ); + } + } +} + #[tokio::test] async fn denied_missing_binary_large_and_non_file_inputs_are_model_visible_errors() { let dir = tempfile::tempdir().unwrap(); @@ -68,10 +180,9 @@ async fn denied_missing_binary_large_and_non_file_inputs_are_model_visible_error outside.path().join("outside.rs").display().to_string(), "../outside.rs".into(), ] { - assert!( - !failure(tool(&tools, "ast_outline"), json!({"path":path})) - .await - .is_empty() + assert_ne!( + failure(tool(&tools, "ast_outline"), json!({"path":path})).await, + "" ); } assert!( diff --git a/crates/tinytools-std/src/code/mutation_tests.rs b/crates/tinytools-std/src/code/mutation_tests.rs index 7a446773..82182d32 100644 --- a/crates/tinytools-std/src/code/mutation_tests.rs +++ b/crates/tinytools-std/src/code/mutation_tests.rs @@ -235,9 +235,9 @@ async fn inconsistent_chunk_destinations_and_invalid_bindings_never_write() { args.as_object_mut().unwrap().remove("expected_revision"); args["bindings"] = json!({"parameters":{"name":"not an identifier"}}); assert!( - !failure(tool(&tools, "chunk_apply"), args.clone()) + failure(tool(&tools, "chunk_apply"), args.clone()) .await - .is_empty() + .contains("identifier parameter violates its declared kind") ); args["bindings"] = json!({}); args["selector"] = json!("unnecessary"); diff --git a/docs/plans/structural-code.md b/docs/plans/structural-code.md index d5ff0409..383e6eca 100644 --- a/docs/plans/structural-code.md +++ b/docs/plans/structural-code.md @@ -9,7 +9,8 @@ Spec: [Structural code and boilerplate](../specs/structural-code.md). 3. Add typed manifests, twelve patterns and one-pass rendering; test injection, generated-node and expansion-budget failures. 4. Add the optional `crates/tinytools-std/src/code/` adapter with host gates, - bounded I/O, previews and revision-guarded atomic publication tests. + bounded I/O, previews and revision-guarded atomic publication tests. Pin + outline levels 0/1 and the five-result search default with API regressions. 5. Update public/module docs and benchmark. Checkpoint focused validated source. 6. Run formatting, strict all-target/all-feature Clippy, build and tests; then default-feature tests, strict rustdoc and the unchanged per-file coverage gate. @@ -42,3 +43,18 @@ byte budgets, late authority revocation, the last available action, stale revisions, custom template containment, cancellation and atomic publication. This is a 73-test source checkpoint. The complete workspace gates, default tests, strict rustdoc, declared MSRV and per-file coverage remain pending. + +## Compatibility follow-up + +After the initial source checkpoint, align the deferred tool contracts with the +consumer roadmap: default `ast_outline` to level 1, omit `skeleton` at level 0, +and reject other levels before file reads. Default `chunk_search` to +`min(5, max_page_items)` while retaining explicit bounded page limits. Update +the schemas and native tool documentation alongside focused regressions before +running the complete owner gates on the resulting source. + +The follow-up passed all 35 native tests on Rust 1.96.1 and workspace formatting +on Rust 1.99.0. It also replaces three assertions flagged by the initial +upstream Rust 1.99 Clippy run. The initial published source passed CI's strict +documentation, supply-chain checks and Rust 1.88 build; complete workspace +tests and per-file coverage still need to pass on the final source. diff --git a/docs/specs/structural-code.md b/docs/specs/structural-code.md index 90f806b6..1d8fdad1 100644 --- a/docs/specs/structural-code.md +++ b/docs/specs/structural-code.md @@ -41,6 +41,13 @@ effect. Responses are budgeted before publication; files are staged, revision guards rechecked, and atomically published with no-clobber creation or permission-preserving replacement. Cancellation cannot expose a partial file. +`ast_outline` accepts detail `level` 0 or 1 and defaults to 1. Level 0 returns +structural records and diagnostics without the `skeleton` field; level 1 includes +the paged folded source. Invalid levels fail before reading the source file. +`chunk_search` defaults to at most five results, capped by the host's configured +`max_page_items`; an explicit positive `limit` may request a larger page within +that host limit. Both tools preserve the existing byte and paging limits. + ## Acceptance All grammars, cache paths, ambiguity, revision failures, import prologues, From daec6c7a563336dd83d2e65ae38b2b169179aada Mon Sep 17 00:00:00 2001 From: Farnam Date: Sun, 18 Oct 2026 16:32:10 +0330 Subject: [PATCH 03/10] Validate grammar constraints and cover UTF-8 edit boundaries --- crates/tinytools-code/src/catalog/README.md | 2 +- crates/tinytools-code/src/catalog/types.rs | 4 +- .../tinytools-code/src/catalog/validation.rs | 21 +++++++-- crates/tinytools-code/tests/analysis_tests.rs | 19 ++++++++ crates/tinytools-code/tests/catalog_tests.rs | 43 +++++++++++++++++++ docs/plans/structural-code.md | 21 +++++++++ docs/specs/structural-code.md | 6 ++- 7 files changed, 108 insertions(+), 8 deletions(-) diff --git a/crates/tinytools-code/src/catalog/README.md b/crates/tinytools-code/src/catalog/README.md index 46bbfd20..45bf8268 100644 --- a/crates/tinytools-code/src/catalog/README.md +++ b/crates/tinytools-code/src/catalog/README.md @@ -1,3 +1,3 @@ # Chunk catalog -Manifests contain typed metadata separately from template source. Paths must be portable and relative; filesystem containment belongs to the adapter. Catalog insertion validates bounded metadata and rejects duplicate IDs. Search is deterministic and metadata-only. Only built-in entries expose embedded source. +Manifests contain typed metadata separately from template source. Paths must be portable and relative; filesystem containment belongs to the adapter. Catalog insertion validates bounded metadata and rejects duplicate IDs. Slot and forbidden-node constraints must name visible, named nodes from the selected grammar; unknown names are errors. Search is deterministic and metadata-only. Only built-in entries expose embedded source. diff --git a/crates/tinytools-code/src/catalog/types.rs b/crates/tinytools-code/src/catalog/types.rs index 8e776b31..59914b11 100644 --- a/crates/tinytools-code/src/catalog/types.rs +++ b/crates/tinytools-code/src/catalog/types.rs @@ -63,7 +63,7 @@ pub struct ChunkSlot { /// Whether an omitted or empty binding is refused. #[serde(default)] pub required: bool, - /// Optional exact grammar node kind required for this slot's expansion. + /// Optional exact visible, named grammar node kind for this slot's expansion. #[serde(default)] pub node_kind: Option, /// Human-readable purpose. @@ -98,7 +98,7 @@ pub struct ChunkManifest { /// Informational package requirements; never automatically installed. #[serde(default)] pub dependencies: Vec, - /// Grammar node kinds forbidden in generated source. + /// Visible, named grammar node kinds forbidden in generated source. #[serde(default)] pub forbidden_nodes: Vec, } diff --git a/crates/tinytools-code/src/catalog/validation.rs b/crates/tinytools-code/src/catalog/validation.rs index 4129140d..73d084c2 100644 --- a/crates/tinytools-code/src/catalog/validation.rs +++ b/crates/tinytools-code/src/catalog/validation.rs @@ -2,7 +2,7 @@ use super::{ChunkManifest, ParameterKind}; use crate::{ - CodeError, + CodeError, Language, error::{enforce_limit, invalid}, }; use std::collections::BTreeSet; @@ -31,6 +31,21 @@ fn named(value: &str) -> Result<(), CodeError> { Ok(()) } +fn node_kind(language: Language, value: &str) -> Result<(), CodeError> { + named(value)?; + let grammar = language.grammar(); + let id = grammar.id_for_node_kind(value, true); + if grammar.node_kind_for_id(id) != Some(value) + || !grammar.node_kind_is_named(id) + || !grammar.node_kind_is_visible(id) + { + return Err(invalid(format!( + "constraint {value} is not a visible, named node kind in {language:?}" + ))); + } + Ok(()) +} + fn template_path(value: &str) -> Result<(), CodeError> { text(value, "template path bytes", 4096)?; if value.contains(['\\', ':']) @@ -103,7 +118,7 @@ pub(super) fn validate(manifest: &ChunkManifest) -> Result<(), CodeError> { } text(&slot.description, "slot description bytes", 1024)?; if let Some(kind) = &slot.node_kind { - named(kind)?; + node_kind(manifest.language, kind)?; } } for tag in &manifest.tags { @@ -113,7 +128,7 @@ pub(super) fn validate(manifest: &ChunkManifest) -> Result<(), CodeError> { text(dependency, "dependency bytes", 256)?; } for kind in &manifest.forbidden_nodes { - named(kind)?; + node_kind(manifest.language, kind)?; } let serialized = serde_json::to_string(manifest).map_err(|error| invalid(error.to_string()))?; enforce_limit("manifest bytes", serialized.len(), 65_536) diff --git a/crates/tinytools-code/tests/analysis_tests.rs b/crates/tinytools-code/tests/analysis_tests.rs index 71e5fdb0..c812fc0d 100644 --- a/crates/tinytools-code/tests/analysis_tests.rs +++ b/crates/tinytools-code/tests/analysis_tests.rs @@ -146,6 +146,25 @@ fn unicode_incremental_edits_match_fresh_parsing() -> Result<(), CodeError> { Ok(()) } +#[test] +fn incremental_edits_expand_partial_utf8_prefixes_and_suffixes() -> Result<(), CodeError> { + for (before, after) in [("café", "cafê"), ("é", "Ω")] { + let old = format!("fn {before}() -> i32 {{\n 1\n}}\n"); + let new = format!("fn {after}() -> i32 {{\n 1\n}}\n"); + let mut engine = CodeEngine::default(); + engine.analyze("unicode.rs", Language::Rust, &old)?; + let mut incremental = engine.analyze("unicode.rs", Language::Rust, &new)?; + assert_eq!(incremental.cache_status, CacheStatus::Incremental); + assert_eq!(select_symbol(&incremental, after)?.name, after); + incremental.cache_status = CacheStatus::Fresh; + assert_eq!( + incremental, + CodeEngine::default().analyze("unicode.rs", Language::Rust, &new)? + ); + } + Ok(()) +} + #[test] fn lru_and_zero_budgets_enforce_retained_source_bounds() -> Result<(), CodeError> { let mut engine = CodeEngine::new(CodeLimits { diff --git a/crates/tinytools-code/tests/catalog_tests.rs b/crates/tinytools-code/tests/catalog_tests.rs index d538a84c..406b8042 100644 --- a/crates/tinytools-code/tests/catalog_tests.rs +++ b/crates/tinytools-code/tests/catalog_tests.rs @@ -151,3 +151,46 @@ fn catalog_capacity_and_metadata_revisions_are_stable() -> Result<(), Box Result<(), Box> { + for (language, kind) in [ + (Language::Rust, "function_item"), + (Language::TypeScript, "interface_declaration"), + (Language::Tsx, "jsx_element"), + (Language::JavaScript, "function_declaration"), + (Language::Python, "function_definition"), + (Language::Go, "function_declaration"), + (Language::Json, "pair"), + (Language::Toml, "table"), + ] { + let mut value = serde_json::to_value(manifest()?)?; + value["language"] = serde_json::to_value(language)?; + value["forbidden_nodes"] = json!([kind]); + value["slots"] = json!([{"name":"body","node_kind":kind}]); + serde_json::from_value::(value.clone())?.validate()?; + + value["forbidden_nodes"] = json!(["not_a_grammar_node"]); + assert!(matches!( + serde_json::from_value::(value.clone())?.validate(), + Err(CodeError::InvalidInput(_)) + )); + value["forbidden_nodes"] = json!([kind]); + value["slots"][0]["node_kind"] = json!("not_a_grammar_node"); + assert!(matches!( + serde_json::from_value::(value)?.validate(), + Err(CodeError::InvalidInput(_)) + )); + } + + let mut custom = manifest()?; + for kind in ["function_definition", "unsafe"] { + custom.forbidden_nodes = vec![kind.to_owned()]; + assert!( + custom.validate().is_err(), + "accepted Rust constraint {kind}" + ); + } + Ok(()) +} diff --git a/docs/plans/structural-code.md b/docs/plans/structural-code.md index 383e6eca..6f32bc7a 100644 --- a/docs/plans/structural-code.md +++ b/docs/plans/structural-code.md @@ -58,3 +58,24 @@ on Rust 1.99.0. It also replaces three assertions flagged by the initial upstream Rust 1.99 Clippy run. The initial published source passed CI's strict documentation, supply-chain checks and Rust 1.88 build; complete workspace tests and per-file coverage still need to pass on the final source. + +## Grammar constraint validation + +A focused regression reproduced that identifier-shaped unknown node kinds were +accepted as manifest constraints. Validate slot and forbidden-node kinds against +the selected grammar's visible named nodes, preserving existing source matching +and rejecting ineffective constraints before rendering. Test valid kinds across +all eight grammars, misspellings, another grammar's kind and anonymous keywords; +then run the final owner gates and unchanged per-file coverage requirement. + +The second upstream CI run passed formatting, strict Clippy, the complete +workspace build, both feature test configurations, documentation, MSRV and +supply-chain checks. Its only coverage failure was `engine/edit.rs` at 87.5%. +Add a public-API regression comparing incremental and fresh parsing when a +replacement character shares a partial UTF-8 prefix or suffix with its previous +value. Keep the implementation and the 90% threshold unchanged. + +This follow-up passed all 43 pure integration tests and the benchmark on Rust +1.96.1, plus workspace formatting and strict all-target/all-feature Clippy on +Rust 1.99.0. The final upstream run must confirm the full workspace tests and +per-file coverage on the published source. diff --git a/docs/specs/structural-code.md b/docs/specs/structural-code.md index 1d8fdad1..bd93a163 100644 --- a/docs/specs/structural-code.md +++ b/docs/specs/structural-code.md @@ -26,8 +26,10 @@ syntax guarantees, not typechecking or semantic reference resolution. A typed chunk catalog offers twelve starter patterns. Search and inspect return metadata. Rendering expands typed parameters and standalone comment slots in one pass; inserted text is never interpreted as another template directive. -Manifests may constrain generated node kinds. Custom manifests use portable -relative template paths. Dependency declarations are informational and never +Manifests may constrain generated node kinds. Constraint names must identify +visible, named nodes in the selected grammar; unknown or inapplicable names are +rejected before rendering. Custom manifests use portable relative template +paths. Dependency declarations are informational and never install anything. Template syntax, expansions and output are bounded. `tinytools-std`'s optional `code` feature exports deferred `ast_outline`, From 72ff2d5f5207b6d924e0b65d13b87a90b4ddfa8d Mon Sep 17 00:00:00 2001 From: Farnam Date: Sun, 18 Oct 2026 22:55:29 +0330 Subject: [PATCH 04/10] ci: audit TinyTools upstream synchronization for OpenHuman PR 14 --- .../openhuman-tinytools-sync-once.yml | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 .github/workflows/openhuman-tinytools-sync-once.yml diff --git a/.github/workflows/openhuman-tinytools-sync-once.yml b/.github/workflows/openhuman-tinytools-sync-once.yml new file mode 100644 index 00000000..538f16b4 --- /dev/null +++ b/.github/workflows/openhuman-tinytools-sync-once.yml @@ -0,0 +1,50 @@ +name: OpenHuman TinyTools upstream sync (one-shot) +on: + push: + branches: [feat/structural-code] + paths: [.github/workflows/openhuman-tinytools-sync-once.yml] +permissions: + contents: write +jobs: + integrate: + runs-on: ubuntu-latest + timeout-minutes: 35 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: false + persist-credentials: true + - name: Fetch and merge canonical TinyTools + shell: bash + run: | + set -euo pipefail + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + branch="feat/structural-code" + current="$(git ls-remote origin "refs/heads/$branch" | cut -f1)" + test "$current" = "$GITHUB_SHA" || { echo "::error::Branch changed; refusing stale merge"; exit 3; } + git remote add canonical https://github.com/tinyhumansai/tinytools.git + git fetch --no-tags canonical +refs/heads/main:refs/remotes/canonical/main + incoming="$(git rev-parse canonical/main)" + echo "Canonical TinyTools head: $incoming" + echo "Merge base: $(git merge-base HEAD "$incoming")" + mkdir -p merge-audit + echo "$incoming" > merge-audit/upstream-sha.txt + if git merge --no-commit --no-ff "$incoming"; then + git rm -- .github/workflows/openhuman-tinytools-sync-once.yml + git diff --cached --check + git commit -m "merge: onboard canonical TinyTools main ($incoming)" + git push origin "HEAD:refs/heads/$branch" + else + git diff --name-only --diff-filter=U | tee merge-audit/conflicts.txt + git diff --cc > merge-audit/combined.diff + git ls-files -u > merge-audit/stages.txt + echo "::error::TinyTools conflicts require resolution" + exit 2 + fi + - if: failure() + uses: actions/upload-artifact@v4 + with: + name: openhuman-tinytools-merge-audit + path: merge-audit/ From 2f36f7b6d5409bf2f77b84347c78453ef99b5abd Mon Sep 17 00:00:00 2001 From: Farnam Date: Sun, 18 Oct 2026 22:58:04 +0330 Subject: [PATCH 05/10] ci: preserve workflow files during upstream source merge --- .github/workflows/openhuman-tinytools-sync-once.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/openhuman-tinytools-sync-once.yml b/.github/workflows/openhuman-tinytools-sync-once.yml index 538f16b4..b512d2e4 100644 --- a/.github/workflows/openhuman-tinytools-sync-once.yml +++ b/.github/workflows/openhuman-tinytools-sync-once.yml @@ -32,7 +32,10 @@ jobs: mkdir -p merge-audit echo "$incoming" > merge-audit/upstream-sha.txt if git merge --no-commit --no-ff "$incoming"; then - git rm -- .github/workflows/openhuman-tinytools-sync-once.yml + # GitHub Actions' token lacks workflows permission; keep the branch's + # workflow files unchanged and carry upstream CI YAML over through + # the user-authorized GitHub connector after this content merge. + git restore --source=HEAD --staged --worktree -- .github/workflows git diff --cached --check git commit -m "merge: onboard canonical TinyTools main ($incoming)" git push origin "HEAD:refs/heads/$branch" From 8edde80d20a8595daa98f5f4eb46902451f51233 Mon Sep 17 00:00:00 2001 From: Farnam Date: Sun, 18 Oct 2026 23:01:57 +0330 Subject: [PATCH 06/10] ci: restore canonical TinyTools workflow_dispatch after upstream merge --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a94287e7..7dedc467 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,6 +3,7 @@ name: CI on: push: pull_request: + workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }} From 0d349716deafc4d91a595bf85400e8445cd1a6e8 Mon Sep 17 00:00:00 2001 From: Farnam Date: Sun, 18 Oct 2026 23:02:00 +0330 Subject: [PATCH 07/10] chore: remove one-shot TinyTools upstream integration workflow --- .../openhuman-tinytools-sync-once.yml | 53 ------------------- 1 file changed, 53 deletions(-) delete mode 100644 .github/workflows/openhuman-tinytools-sync-once.yml diff --git a/.github/workflows/openhuman-tinytools-sync-once.yml b/.github/workflows/openhuman-tinytools-sync-once.yml deleted file mode 100644 index b512d2e4..00000000 --- a/.github/workflows/openhuman-tinytools-sync-once.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: OpenHuman TinyTools upstream sync (one-shot) -on: - push: - branches: [feat/structural-code] - paths: [.github/workflows/openhuman-tinytools-sync-once.yml] -permissions: - contents: write -jobs: - integrate: - runs-on: ubuntu-latest - timeout-minutes: 35 - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: false - persist-credentials: true - - name: Fetch and merge canonical TinyTools - shell: bash - run: | - set -euo pipefail - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - branch="feat/structural-code" - current="$(git ls-remote origin "refs/heads/$branch" | cut -f1)" - test "$current" = "$GITHUB_SHA" || { echo "::error::Branch changed; refusing stale merge"; exit 3; } - git remote add canonical https://github.com/tinyhumansai/tinytools.git - git fetch --no-tags canonical +refs/heads/main:refs/remotes/canonical/main - incoming="$(git rev-parse canonical/main)" - echo "Canonical TinyTools head: $incoming" - echo "Merge base: $(git merge-base HEAD "$incoming")" - mkdir -p merge-audit - echo "$incoming" > merge-audit/upstream-sha.txt - if git merge --no-commit --no-ff "$incoming"; then - # GitHub Actions' token lacks workflows permission; keep the branch's - # workflow files unchanged and carry upstream CI YAML over through - # the user-authorized GitHub connector after this content merge. - git restore --source=HEAD --staged --worktree -- .github/workflows - git diff --cached --check - git commit -m "merge: onboard canonical TinyTools main ($incoming)" - git push origin "HEAD:refs/heads/$branch" - else - git diff --name-only --diff-filter=U | tee merge-audit/conflicts.txt - git diff --cc > merge-audit/combined.diff - git ls-files -u > merge-audit/stages.txt - echo "::error::TinyTools conflicts require resolution" - exit 2 - fi - - if: failure() - uses: actions/upload-artifact@v4 - with: - name: openhuman-tinytools-merge-audit - path: merge-audit/ From 61945ffaa447f463228cb64775dc71140340bed3 Mon Sep 17 00:00:00 2001 From: Farnam Date: Fri, 23 Oct 2026 15:36:56 +0330 Subject: [PATCH 08/10] Make DNS and child environment checks portable on Windows --- crates/tinytools-std/src/timeout/mod_tests.rs | 9 ++++++++- crates/tinytools-std/src/url_guard/mod.rs | 3 +++ crates/tinytools-std/src/url_guard/mod_tests.rs | 5 +++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/crates/tinytools-std/src/timeout/mod_tests.rs b/crates/tinytools-std/src/timeout/mod_tests.rs index 83996460..e72030fa 100644 --- a/crates/tinytools-std/src/timeout/mod_tests.rs +++ b/crates/tinytools-std/src/timeout/mod_tests.rs @@ -338,7 +338,14 @@ async fn command_environment_replaces_the_child_environment() { ]); let out = env .scope(async { - let mut cmd = sh("echo ${ONLY_THIS:-missing}-${HOME:-nohome}"); + // Git Bash synthesizes HOME when absent; USERPROFILE is inherited + // from Windows and therefore tests removal without shell defaults. + let script = if cfg!(windows) { + "echo ${ONLY_THIS:-missing}-${USERPROFILE:-nohome}" + } else { + "echo ${ONLY_THIS:-missing}-${HOME:-nohome}" + }; + let mut cmd = sh(script); output_unbounded(&mut cmd).await.unwrap() }) .await; diff --git a/crates/tinytools-std/src/url_guard/mod.rs b/crates/tinytools-std/src/url_guard/mod.rs index 18b5ecd7..9cbec9a0 100644 --- a/crates/tinytools-std/src/url_guard/mod.rs +++ b/crates/tinytools-std/src/url_guard/mod.rs @@ -230,6 +230,9 @@ pub(crate) fn connect_host(url: &str) -> anyhow::Result { } async fn resolve_host_ips(host: String, port: u16) -> anyhow::Result> { + if host.trim().is_empty() { + anyhow::bail!("DNS host must not be empty"); + } let log_host = host.clone(); tokio::task::spawn_blocking(move || { (host.as_str(), port) diff --git a/crates/tinytools-std/src/url_guard/mod_tests.rs b/crates/tinytools-std/src/url_guard/mod_tests.rs index ecf14fce..7d0e7936 100644 --- a/crates/tinytools-std/src/url_guard/mod_tests.rs +++ b/crates/tinytools-std/src/url_guard/mod_tests.rs @@ -46,6 +46,11 @@ async fn system_dns_resolves_numeric_loopback_without_external_network() -> anyh let resolved = super::resolve_host_ips("127.0.0.1".to_string(), 80).await?; assert_eq!(resolved, vec!["127.0.0.1".parse::()?]); assert!(super::resolve_host_ips(String::new(), 80).await.is_err()); + assert!( + super::resolve_host_ips(" \t".to_string(), 80) + .await + .is_err() + ); Ok(()) } From 8ac07cc1a2a5005d052adc4a6485ed96725f4e12 Mon Sep 17 00:00:00 2001 From: Farnam Date: Fri, 23 Oct 2026 15:38:52 +0330 Subject: [PATCH 09/10] Keep non-Linux process cleanup lint clean --- .../tinytools-std/src/filesystem/workspace_state/mod_tests.rs | 3 +++ crates/tinytools-std/src/timeout/group_exit.rs | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/crates/tinytools-std/src/filesystem/workspace_state/mod_tests.rs b/crates/tinytools-std/src/filesystem/workspace_state/mod_tests.rs index a815bf74..2143e854 100644 --- a/crates/tinytools-std/src/filesystem/workspace_state/mod_tests.rs +++ b/crates/tinytools-std/src/filesystem/workspace_state/mod_tests.rs @@ -62,6 +62,7 @@ async fn include_tree_false_omits_directory_tree() { /// where `file_write` puts things. The hook exits non-zero so git falls /// back to a normal scan and `status` still succeeds; the point of the test /// is whether the command ran at all, not what it returned. +#[cfg(unix)] fn plant_fsmonitor_hook(dir: &TempDir) -> std::path::PathBuf { let hook = dir.path().join("hook.sh"); let marker = dir.path().join("COMMAND_RAN"); @@ -90,6 +91,7 @@ fn plant_fsmonitor_hook(dir: &TempDir) -> std::path::PathBuf { marker } +#[cfg(unix)] fn git_init(dir: &TempDir) { // `.status().unwrap()` only unwraps the *spawn*: a non-zero exit would // pass silently here and surface later as a confusing assertion about @@ -104,6 +106,7 @@ fn git_init(dir: &TempDir) { } /// Set a repository config key with `git config`, asserting it took. +#[cfg(unix)] fn set_config(dir: &TempDir, key: &str, value: &str) { let ok = std::process::Command::new("git") .args(["config", key, value]) diff --git a/crates/tinytools-std/src/timeout/group_exit.rs b/crates/tinytools-std/src/timeout/group_exit.rs index a30cc7e1..52734350 100644 --- a/crates/tinytools-std/src/timeout/group_exit.rs +++ b/crates/tinytools-std/src/timeout/group_exit.rs @@ -19,8 +19,8 @@ pub(super) async fn wait(pid: u32) -> std::io::Result<()> { /// Other platforms retain their existing direct-child acknowledgement. #[cfg(not(target_os = "linux"))] -pub(super) async fn wait(_: u32) -> std::io::Result<()> { - Ok(()) +pub(super) fn wait(_: u32) -> std::future::Ready> { + std::future::ready(Ok(())) } /// Inspect kernel process states without reaping the reserved group leader. From a49d5f9680119c415afaaa56c0147c70929ae942 Mon Sep 17 00:00:00 2001 From: Farnam Date: Fri, 23 Oct 2026 15:43:21 +0330 Subject: [PATCH 10/10] Cover resolver errors without external DNS --- crates/tinytools-std/src/url_guard/mod_tests.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crates/tinytools-std/src/url_guard/mod_tests.rs b/crates/tinytools-std/src/url_guard/mod_tests.rs index 7d0e7936..5b26c1a1 100644 --- a/crates/tinytools-std/src/url_guard/mod_tests.rs +++ b/crates/tinytools-std/src/url_guard/mod_tests.rs @@ -51,6 +51,11 @@ async fn system_dns_resolves_numeric_loopback_without_external_network() -> anyh .await .is_err() ); + // An embedded NUL is rejected by the OS resolver without network access. + let error = super::resolve_host_ips("invalid\0host".to_string(), 80) + .await + .rejection()?; + assert!(error.contains("DNS resolution failed")); Ok(()) }