diff --git a/.github/workflows/parser-benchmarks.yml b/.github/workflows/parser-benchmarks.yml new file mode 100644 index 0000000..ce55171 --- /dev/null +++ b/.github/workflows/parser-benchmarks.yml @@ -0,0 +1,80 @@ +run-name: "Parser benchmarks (${{ github.event_name == 'schedule' && 'scheduled' || format('@{0}', github.actor) }})" + +# Deterministic benchmarks for libgraphql-parser, run under Valgrind's +# Callgrind (via iai-callgrind). Callgrind executes the benchmarks on +# a simulated CPU and counts exact instructions + simulated cache +# hits/misses, so results are reproducible even on noisy shared CI +# runners — unlike wall-clock benchmarks, which are meaningless there. +# +# Runs on demand (workflow_dispatch) and on a monthly schedule to +# check how libgraphql-parser compares against the latest releases of +# other GraphQL parsers on crates.io. The markdown report lands in +# the workflow run's summary page; raw iai-callgrind output is +# uploaded as an artifact. + +name: "Parser benchmarks" + +on: + workflow_dispatch: + inputs: + use_latest_competitor_parsers: + description: "Benchmark against the latest crates.io releases of graphql-parser and apollo-parser (instead of the versions pinned in Cargo.lock)" + type: "boolean" + default: true + schedule: + # 06:23 UTC on the 2nd of every month. Odd minute chosen to avoid + # the on-the-hour thundering herd on GitHub's shared runners. + - cron: "23 6 2 * *" + +env: + CARGO_TERM_COLOR: "always" + +# The job only reads the repo and uploads a workflow artifact; it +# never needs write access to repository contents, issues, or PRs. +permissions: + contents: "read" + +jobs: + callgrind-benchmarks: + runs-on: "ubuntu-latest" + timeout-minutes: 90 + steps: + - uses: "actions/checkout@v4" + + - name: "Install valgrind" + run: | + sudo apt-get update + sudo apt-get install --yes valgrind + valgrind --version + + - name: "Install iai-callgrind-runner (version-matched to Cargo.lock)" + run: | + version="$( + cargo metadata --format-version 1 \ + | jq -r '.packages[] | select(.name == "iai-callgrind") | .version' + )" + echo "Installing iai-callgrind-runner ${version}" + cargo install iai-callgrind-runner --version "${version}" --locked + + - name: "Fetch Shopify Admin schema fixture" + run: "${GITHUB_WORKSPACE}/crates/libgraphql-parser/scripts/fetch-shopify-admin-graphql-schema-fixture.sh" + + - name: "Update competitor parsers to latest crates.io releases" + if: "github.event_name == 'schedule' || inputs.use_latest_competitor_parsers" + run: "${GITHUB_WORKSPACE}/crates/libgraphql-parser/scripts/ci/parser-benchmarks.use-latest-competitor-parsers.sh" + + - name: "Run Callgrind benchmarks" + run: "${GITHUB_WORKSPACE}/crates/libgraphql-parser/scripts/run-callgrind-benchmarks.sh" + + - name: "Publish report to workflow summary" + run: "cat ${GITHUB_WORKSPACE}/target/iai/REPORT.md >> ${GITHUB_STEP_SUMMARY}" + + - name: "Upload raw iai-callgrind results" + uses: "actions/upload-artifact@v4" + with: + if-no-files-found: "error" + name: "callgrind-benchmark-results" + path: | + target/iai/**/summary.json + target/iai/REPORT.md + retention-days: 90 diff --git a/Cargo.lock b/Cargo.lock index 983fe9c..dbedb7c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -46,6 +46,15 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bincode" version = "2.0.1" @@ -246,6 +255,27 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "syn", +] + [[package]] name = "either" version = "1.15.0" @@ -365,6 +395,42 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" +[[package]] +name = "iai-callgrind" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b1e4910d3a9137442723dfb772c32dc10674c4181ca078d2fd227cd5dce9db0" +dependencies = [ + "bincode 1.3.3", + "derive_more", + "iai-callgrind-macros", + "iai-callgrind-runner", +] + +[[package]] +name = "iai-callgrind-macros" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d03775318d3f9f01b39ac6612b01464006dc397a654a89dd57df2fd34fb68c3" +dependencies = [ + "derive_more", + "proc-macro-error2", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn", +] + +[[package]] +name = "iai-callgrind-runner" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b74c9743c00c3bca4aaffc69c87cae56837796cd362438daf354a3f785788c68" +dependencies = [ + "serde", +] + [[package]] name = "id-arena" version = "2.3.0" @@ -454,7 +520,7 @@ dependencies = [ name = "libgraphql-core" version = "0.0.8" dependencies = [ - "bincode", + "bincode 2.0.1", "graphql-parser", "indexmap", "inherent", @@ -468,7 +534,7 @@ dependencies = [ name = "libgraphql-core-v1" version = "0.0.1" dependencies = [ - "bincode", + "bincode 2.0.1", "indexmap", "inherent", "libgraphql-parser", @@ -480,7 +546,7 @@ dependencies = [ name = "libgraphql-macros" version = "0.0.10" dependencies = [ - "bincode", + "bincode 2.0.1", "libgraphql", "libgraphql-core", "libgraphql-parser", @@ -497,6 +563,7 @@ dependencies = [ "apollo-parser", "criterion", "graphql-parser", + "iai-callgrind", "inherent", "memchr", "proptest", @@ -591,6 +658,28 @@ dependencies = [ "syn", ] +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "proc-macro2" version = "1.0.101" @@ -751,6 +840,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rustix" version = "1.1.3" diff --git a/Cargo.toml b/Cargo.toml index 6dcdb9a..6594935 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,11 +13,12 @@ apollo-parser = "0.8" bincode = { version = "2.0.1", features = ["serde"] } criterion = { version = "0.5", features = ["html_reports"] } graphql-parser = "0.4.0" -proptest = "1.6" +iai-callgrind = "0.16.1" indexmap = { version = "2.10.0", features = ["serde"] } inherent = "1.0.12" memchr = "2.7" proc-macro2 = "1.0.101" +proptest = "1.6" quote = "1.0.40" rayon = "1.10" serde = { version = "1.0.226", features = ["derive"] } diff --git a/crates/libgraphql-core-v1/src/schema/schema_builder.rs b/crates/libgraphql-core-v1/src/schema/schema_builder.rs index 5ba4dc5..c9fdf79 100644 --- a/crates/libgraphql-core-v1/src/schema/schema_builder.rs +++ b/crates/libgraphql-core-v1/src/schema/schema_builder.rs @@ -688,16 +688,14 @@ impl SchemaBuilder { )); } }, - GraphQLType::Enum(enum_t) => { - if enum_t.values().is_empty() { - self.errors.push(SchemaBuildError::new( - SchemaBuildErrorKind::EnumWithNoValues { - type_name: enum_t.name().to_string(), - }, - enum_t.span(), - vec![], - )); - } + GraphQLType::Enum(enum_t) if enum_t.values().is_empty() => { + self.errors.push(SchemaBuildError::new( + SchemaBuildErrorKind::EnumWithNoValues { + type_name: enum_t.name().to_string(), + }, + enum_t.span(), + vec![], + )); }, _ => {}, } diff --git a/crates/libgraphql-parser/Cargo.toml b/crates/libgraphql-parser/Cargo.toml index 52d2eec..cc5ad66 100644 --- a/crates/libgraphql-parser/Cargo.toml +++ b/crates/libgraphql-parser/Cargo.toml @@ -21,6 +21,15 @@ apollo-parser.workspace = true criterion.workspace = true proptest.workspace = true +# Valgrind (and therefore iai-callgrind) only supports Linux. The +# callgrind benchmark target compiles to a no-op on other platforms. +[target.'cfg(target_os = "linux")'.dev-dependencies] +iai-callgrind.workspace = true + [[bench]] name = "parse_benchmarks" harness = false + +[[bench]] +name = "callgrind_benchmarks" +harness = false diff --git a/crates/libgraphql-parser/README.md b/crates/libgraphql-parser/README.md index f3b500c..e187561 100644 --- a/crates/libgraphql-parser/README.md +++ b/crates/libgraphql-parser/README.md @@ -276,6 +276,36 @@ script which will also aggregate results in a table like below: | github (~1.2 MB) | 2.06 ms | ~568 MiB/s | | shopify_admin (~3.1 MB) | 3.68 ms | ~840 MiB/s | +### Deterministic (Callgrind) Benchmarks + +Wall-clock benchmarks are meaningful only on quiet, dedicated +hardware, so a second benchmark suite runs the same fixtures and +comparison parsers under [Valgrind's Callgrind](https://valgrind.org/docs/manual/cl-manual.html) +via [iai-callgrind](https://github.com/iai-callgrind/iai-callgrind). +Callgrind executes each benchmark on a simulated CPU and counts the +exact number of instructions executed plus simulated cache +hits/misses (from which an "estimated cycles" metric is derived). +Repeated runs of the same binary produce identical counts, making +this suite suitable for noisy shared CI runners and for tracking +small regressions that wall-clock noise would swallow. + +Run it locally (Linux only — Valgrind does not support macOS on +Apple Silicon) with: + +```sh +cargo install iai-callgrind-runner --version +./crates/libgraphql-parser/scripts/run-callgrind-benchmarks.sh +``` + +The same suite runs in CI via the "Parser benchmarks" GitHub Actions +workflow (`.github/workflows/parser-benchmarks.yml`), on demand and +on a monthly schedule. By default the CI run first updates +`graphql-parser` and `apollo-parser` to their latest crates.io +releases, so each report reflects how `libgraphql-parser` compares +against the current state of the ecosystem. The markdown report is +published to the workflow run's summary page and the raw +iai-callgrind data is uploaded as an artifact. + ## Core Types | Type | Description | diff --git a/crates/libgraphql-parser/benches/callgrind_benchmarks.rs b/crates/libgraphql-parser/benches/callgrind_benchmarks.rs new file mode 100644 index 0000000..44c4390 --- /dev/null +++ b/crates/libgraphql-parser/benches/callgrind_benchmarks.rs @@ -0,0 +1,287 @@ +//! Deterministic, virtualized-CPU benchmarks for libgraphql-parser. +//! +//! These benchmarks run under Valgrind's Callgrind (via +//! `iai-callgrind`), which executes the benchmark on a simulated CPU +//! and counts the exact number of instructions executed along with +//! simulated cache hits/misses. Because the measurements come from a +//! virtualized CPU rather than wall-clock time, they are fully +//! deterministic: repeated runs of the same binary produce identical +//! counts regardless of machine load, CPU frequency scaling, or +//! shared-runner noise. This makes them suitable for CI environments +//! where wall-clock benchmarks (see `parse_benchmarks.rs`) would be +//! hopelessly noisy. +//! +//! The trade-off is that instruction counts and "estimated cycles" +//! (derived from the cache simulation) are a proxy for real-world +//! latency, not a replacement: they do not model superscalar +//! execution, branch prediction, or memory-level parallelism. Use +//! these benchmarks for regression tracking and cross-parser +//! comparison trends; use the Criterion benchmarks on quiet bare +//! metal for headline wall-clock numbers. +//! +//! Valgrind only supports Linux (notably: not macOS on Apple +//! Silicon), so this benchmark target compiles to a stub that prints +//! an explanation on other platforms. +//! +//! Usage (requires `valgrind` and a matching-version +//! `iai-callgrind-runner` on `$PATH`): +//! +//! ```sh +//! cargo install iai-callgrind-runner --version 0.16.1 +//! cargo bench -p libgraphql-parser --bench callgrind_benchmarks +//! ``` +//! +//! Or use the wrapper script which also formats results as markdown: +//! +//! ```sh +//! ./crates/libgraphql-parser/scripts/run-callgrind-benchmarks.sh +//! ``` + +#[cfg(target_os = "linux")] +mod fixtures; + +#[cfg(target_os = "linux")] +mod callgrind_benches { + use crate::fixtures; + use iai_callgrind::Callgrind; + use iai_callgrind::LibraryBenchmarkConfig; + use iai_callgrind::library_benchmark; + use iai_callgrind::library_benchmark_group; + use iai_callgrind::main; + use libgraphql_parser::GraphQLParser; + use libgraphql_parser::GraphQLParserConfig; + use libgraphql_parser::token::StrGraphQLTokenSource; + use std::hint::black_box; + + // All fixtures are known-valid GraphQL for every parser below, + // so every benchmark measures a successful full parse. If a + // fixture were rejected by one parser, that parser's counts + // would reflect a cheap early error return rather than + // comparable work — keep this invariant in mind when adding + // fixtures. + // + // The compared parsers also differ in output fidelity: + // libgraphql-parser's default config and apollo-parser both + // retain lossless syntax/trivia information, while + // graphql-parser and libgraphql-parser's lean mode produce a + // semantic AST only. graphql-parser is additionally benchmarked + // in both its owned-`String` mode (matching the Criterion suite + // in parse_benchmarks.rs) and its zero-copy `&str` mode. + + /// Resolves a schema-document fixture name to its GraphQL source + /// text. Runs as an iai-callgrind `setup` function, so none of + /// the work done here (file I/O included) is attributed to the + /// measured benchmark. + fn schema_source(name: &str) -> String { + match name { + "small" => fixtures::SMALL_SCHEMA.to_string(), + "medium" => fixtures::MEDIUM_SCHEMA.to_string(), + "large" => fixtures::LARGE_SCHEMA.to_string(), + "starwars" => fixtures::STARWARS_SCHEMA.to_string(), + "github" => fixtures::GITHUB_SCHEMA.to_string(), + "shopify_admin" => fixtures::load_shopify_admin_schema(), + _ => panic!("Unknown schema fixture: {name}"), + } + } + + /// Resolves an executable-document fixture name to its GraphQL + /// source text. Runs as an iai-callgrind `setup` function (see + /// `schema_source()`). + fn executable_source(name: &str) -> String { + match name { + "simple" => fixtures::SIMPLE_QUERY.to_string(), + "complex" => fixtures::COMPLEX_QUERY.to_string(), + "nested_10" => fixtures::operations::deeply_nested_query(10), + "nested_30" => fixtures::operations::deeply_nested_query(30), + "many_ops_50" => fixtures::operations::many_operations(50), + _ => panic!("Unknown executable fixture: {name}"), + } + } + + // ─── Schema Document Parsing ───────────────────────────── + + #[library_benchmark(setup = schema_source)] + #[bench::small("small")] + #[bench::medium("medium")] + #[bench::large("large")] + #[bench::starwars("starwars")] + #[bench::github("github")] + #[bench::shopify_admin("shopify_admin")] + fn schema_libgraphql(schema: String) { + let parser = GraphQLParser::new(&schema); + black_box(parser.parse_schema_document()); + } + + #[library_benchmark(setup = schema_source)] + #[bench::small("small")] + #[bench::medium("medium")] + #[bench::large("large")] + #[bench::starwars("starwars")] + #[bench::github("github")] + #[bench::shopify_admin("shopify_admin")] + fn schema_libgraphql_lean(schema: String) { + let parser = GraphQLParser::with_config(&schema, GraphQLParserConfig::lean()); + black_box(parser.parse_schema_document()); + } + + #[library_benchmark(setup = schema_source)] + #[bench::small("small")] + #[bench::medium("medium")] + #[bench::large("large")] + #[bench::starwars("starwars")] + #[bench::github("github")] + #[bench::shopify_admin("shopify_admin")] + fn schema_graphql_parser(schema: String) { + let _ = black_box(graphql_parser::schema::parse_schema::(&schema)); + } + + #[library_benchmark(setup = schema_source)] + #[bench::small("small")] + #[bench::medium("medium")] + #[bench::large("large")] + #[bench::starwars("starwars")] + #[bench::github("github")] + #[bench::shopify_admin("shopify_admin")] + fn schema_graphql_parser_borrowed(schema: String) { + let _ = black_box(graphql_parser::schema::parse_schema::<&str>(&schema)); + } + + #[library_benchmark(setup = schema_source)] + #[bench::small("small")] + #[bench::medium("medium")] + #[bench::large("large")] + #[bench::starwars("starwars")] + #[bench::github("github")] + #[bench::shopify_admin("shopify_admin")] + fn schema_apollo_parser(schema: String) { + let parser = apollo_parser::Parser::new(&schema); + black_box(parser.parse()); + } + + // ─── Executable Document Parsing ───────────────────────── + + #[library_benchmark(setup = executable_source)] + #[bench::simple("simple")] + #[bench::complex("complex")] + #[bench::nested_10("nested_10")] + #[bench::nested_30("nested_30")] + #[bench::many_ops_50("many_ops_50")] + fn executable_libgraphql(query: String) { + let parser = GraphQLParser::new(&query); + black_box(parser.parse_executable_document()); + } + + #[library_benchmark(setup = executable_source)] + #[bench::simple("simple")] + #[bench::complex("complex")] + #[bench::nested_10("nested_10")] + #[bench::nested_30("nested_30")] + #[bench::many_ops_50("many_ops_50")] + fn executable_libgraphql_lean(query: String) { + let parser = GraphQLParser::with_config(&query, GraphQLParserConfig::lean()); + black_box(parser.parse_executable_document()); + } + + #[library_benchmark(setup = executable_source)] + #[bench::simple("simple")] + #[bench::complex("complex")] + #[bench::nested_10("nested_10")] + #[bench::nested_30("nested_30")] + #[bench::many_ops_50("many_ops_50")] + fn executable_graphql_parser(query: String) { + let _ = black_box(graphql_parser::query::parse_query::(&query)); + } + + #[library_benchmark(setup = executable_source)] + #[bench::simple("simple")] + #[bench::complex("complex")] + #[bench::nested_10("nested_10")] + #[bench::nested_30("nested_30")] + #[bench::many_ops_50("many_ops_50")] + fn executable_graphql_parser_borrowed(query: String) { + let _ = black_box(graphql_parser::query::parse_query::<&str>(&query)); + } + + #[library_benchmark(setup = executable_source)] + #[bench::simple("simple")] + #[bench::complex("complex")] + #[bench::nested_10("nested_10")] + #[bench::nested_30("nested_30")] + #[bench::many_ops_50("many_ops_50")] + fn executable_apollo_parser(query: String) { + let parser = apollo_parser::Parser::new(&query); + black_box(parser.parse()); + } + + // ─── Lexer (Tokenization Only) ─────────────────────────── + + #[library_benchmark(setup = schema_source)] + #[bench::small("small")] + #[bench::medium("medium")] + #[bench::large("large")] + #[bench::starwars("starwars")] + #[bench::github("github")] + #[bench::shopify_admin("shopify_admin")] + fn lexer_libgraphql(schema: String) { + let source = StrGraphQLTokenSource::new(&schema); + for token in source { + black_box(token); + } + } + + // ─── Groups & Entrypoint ───────────────────────────────── + + library_benchmark_group!( + name = schema_parse; + benchmarks = + schema_libgraphql, + schema_libgraphql_lean, + schema_graphql_parser, + schema_graphql_parser_borrowed, + schema_apollo_parser, + ); + + library_benchmark_group!( + name = executable_parse; + benchmarks = + executable_libgraphql, + executable_libgraphql_lean, + executable_graphql_parser, + executable_graphql_parser_borrowed, + executable_apollo_parser, + ); + + library_benchmark_group!( + name = lexer; + benchmarks = lexer_libgraphql, + ); + + main!( + // `--cache-sim=yes` enables Callgrind's cache simulation, + // which adds L1/LL hit/miss counts and a derived "estimated + // cycles" metric on top of raw instruction counts. This is + // explicit (rather than relying on defaults) so results stay + // comparable across iai-callgrind versions. + config = LibraryBenchmarkConfig::default() + .tool(Callgrind::with_args(["--cache-sim=yes"])); + library_benchmark_groups = schema_parse, executable_parse, lexer + ); + + pub fn run() { + main(); + } +} + +#[cfg(target_os = "linux")] +fn main() { + callgrind_benches::run(); +} + +#[cfg(not(target_os = "linux"))] +fn main() { + eprintln!( + "The callgrind_benchmarks bench target requires Valgrind, which only \ + supports Linux. Use the Criterion wall-clock benchmarks instead:\n\n \ + cargo bench -p libgraphql-parser --bench parse_benchmarks\n", + ); +} diff --git a/crates/libgraphql-parser/scripts/ci/parser-benchmarks.use-latest-competitor-parsers.sh b/crates/libgraphql-parser/scripts/ci/parser-benchmarks.use-latest-competitor-parsers.sh new file mode 100755 index 0000000..7e90157 --- /dev/null +++ b/crates/libgraphql-parser/scripts/ci/parser-benchmarks.use-latest-competitor-parsers.sh @@ -0,0 +1,99 @@ +#!/bin/bash + +# Updates the comparison ("competitor") GraphQL parsers used by the +# libgraphql-parser benchmarks to their latest stable releases on +# crates.io, so that periodic CI benchmark runs always compare +# against what users would get from a fresh `cargo add` today. +# +# This rewrites the version requirements in the workspace Cargo.toml +# to exact-pinned latest versions (e.g. `apollo-parser = "=0.9.2"`) +# and runs `cargo update` for each crate. The changes are meant for +# transient CI use only and are NOT intended to be committed. +# +# Note: a new competitor release with breaking API changes can cause +# the benchmark build to fail. That failure is a useful signal (the +# benchmark code needs updating), not something to paper over. +# +# Note: graphql-parser is also a regular (non-benchmark) dependency +# of other workspace crates (e.g. libgraphql-core), so bumping the +# shared workspace version affects any workspace build performed +# after this script runs — another reason these changes must stay +# confined to a transient CI checkout. +# +# Usage: +# ./crates/libgraphql-parser/scripts/ci/parser-benchmarks.use-latest-competitor-parsers.sh + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)" +source "${REPO_ROOT}/scripts/_include.sh" + +WORKSPACE_CARGO_TOML="${REPO_ROOT}/Cargo.toml" + +# Crate names below are interpolated into grep/sed regex patterns +# unescaped. That is safe for these names (`-` is not special in the +# positions used), but escape any future addition whose name contains +# regex metacharacters. +COMPETITOR_CRATES=("graphql-parser" "apollo-parser") + +# ─── Prerequisites ──────────────────────────────────────── + +assert_installed cargo || exit 1 +assert_installed curl || exit 1 +assert_installed jq || exit 1 + +# ─── Helpers ────────────────────────────────────────────── + +# Print the latest non-yanked, non-prerelease version of a crate by +# querying the crates.io sparse index (https://index.crates.io). +# Index lines are ordered by publish date, not semver, so a backport +# release published after a newer version would appear last; sort by +# version before picking the highest. +latest_stable_crate_version() { + local crate_name="$1" + local shard="${crate_name:0:2}/${crate_name:2:2}" + curl --fail --silent --show-error "https://index.crates.io/${shard}/${crate_name}" \ + | jq -r ' + select(.yanked | not) + | .vers + | select(test("-") | not) + ' \ + | sort --version-sort \ + | tail -1 +} + +# ─── Update Each Competitor Crate ──────────────────────── + +for crate in "${COMPETITOR_CRATES[@]}"; do + latest="$(latest_stable_crate_version "$crate")" + if [ -z "$latest" ]; then + echo "${UNICODE_RED_X} Could not determine the latest version of ${crate}" >&2 + exit 1 + fi + + echo "Pinning ${crate} to latest stable release: ${latest}" + + if ! grep -qE "^${crate} = \"[^\"]*\"$" "${WORKSPACE_CARGO_TOML}"; then + { + echo "${UNICODE_RED_X} Could not find a '${crate} = \"...\"' entry in:" + echo " ${WORKSPACE_CARGO_TOML}" + } >&2 + exit 1 + fi + + sed -i -E \ + "s|^${crate} = \"[^\"]*\"$|${crate} = \"=${latest}\"|" \ + "${WORKSPACE_CARGO_TOML}" + + cargo update --package "$crate" --precise "$latest" +done + +echo "" +echo "${UNICODE_GREEN_CHECK} Competitor parser versions now in the dependency graph:" +cargo metadata --format-version 1 \ + | jq -r ' + .packages[] + | select(.name == "graphql-parser" or .name == "apollo-parser") + | " \(.name) \(.version)" + ' diff --git a/crates/libgraphql-parser/scripts/run-callgrind-benchmarks.sh b/crates/libgraphql-parser/scripts/run-callgrind-benchmarks.sh new file mode 100755 index 0000000..7e0a5bc --- /dev/null +++ b/crates/libgraphql-parser/scripts/run-callgrind-benchmarks.sh @@ -0,0 +1,331 @@ +#!/bin/bash + +# Runs the deterministic Callgrind-based benchmarks for the +# libgraphql-parser crate (see benches/callgrind_benchmarks.rs) and +# formats the results as markdown tables. +# +# Unlike the wall-clock Criterion benchmarks (run-benchmarks.sh), +# these run on Valgrind's simulated CPU and measure exact instruction +# counts plus simulated cache behavior. Results are deterministic and +# machine-independent, which makes them suitable for noisy CI runners +# and for tracking regressions over time. +# +# Requirements: +# - Linux (Valgrind does not support macOS on Apple Silicon) +# - valgrind on $PATH +# - iai-callgrind-runner on $PATH, same version as the +# iai-callgrind crate in Cargo.lock: +# cargo install iai-callgrind-runner --version +# - The Shopify Admin schema fixture (fetched via +# scripts/fetch-shopify-admin-graphql-schema-fixture.sh) +# +# Usage: +# ./crates/libgraphql-parser/scripts/run-callgrind-benchmarks.sh +# ./crates/libgraphql-parser/scripts/run-callgrind-benchmarks.sh --format-only +# +# Arguments: +# --format-only Skip running the benchmarks and only (re)format the +# report from an existing target/iai directory. +# +# The markdown report is written to target/iai/REPORT.md and echoed +# to stdout. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" +source "${REPO_ROOT}/scripts/_include.sh" + +IAI_DIR="${REPO_ROOT}/target/iai" +REPORT_FILE="${IAI_DIR}/REPORT.md" +SUMMARY_ROOT="${IAI_DIR}/libgraphql-parser/callgrind_benchmarks/callgrind_benches" +SHOPIFY_FIXTURE="${REPO_ROOT}/crates/libgraphql-parser/benches/fixtures" +SHOPIFY_FIXTURE+="/third-party/shopify-admin-schema/schema.graphql" + +# ─── Parse Arguments ────────────────────────────────────── + +FORMAT_ONLY=false +for arg in "$@"; do + case "$arg" in + --format-only) FORMAT_ONLY=true ;; + *) echo "Unknown argument: $arg" >&2; exit 1 ;; + esac +done + +# ─── Prerequisites ──────────────────────────────────────── + +assert_installed jq || exit 1 +assert_installed cargo || exit 1 + +if ! $FORMAT_ONLY; then + if [[ "$OSTYPE" != "linux"* ]]; then + { + echo "${UNICODE_RED_X} Valgrind (and therefore this benchmark suite) only runs on" + echo " Linux. Use run-benchmarks.sh (Criterion, wall-clock) instead." + } >&2 + exit 1 + fi + + assert_installed valgrind || exit 1 + + IAI_CALLGRIND_VERSION=$( + cargo metadata --format-version 1 2>/dev/null \ + | jq -r '.packages[] | select(.name == "iai-callgrind") | .version' + ) + + if ! is_installed iai-callgrind-runner; then + { + echo "${UNICODE_RED_X} iai-callgrind-runner is not installed (or not on \$PATH)." + echo " Install the version matching Cargo.lock with:" + echo "" + echo " cargo install iai-callgrind-runner --version ${IAI_CALLGRIND_VERSION}" + } >&2 + exit 1 + fi + + if [ ! -f "${SHOPIFY_FIXTURE}" ]; then + { + echo "${UNICODE_RED_X} The Shopify Admin schema benchmark fixture is missing. It is" + echo " not checked in to the repository and must be fetched first:" + echo "" + echo " ${REPO_ROOT}/crates/libgraphql-parser/scripts/fetch-shopify-admin-graphql-schema-fixture.sh" + } >&2 + exit 1 + fi +fi + +# ─── Run Benchmarks ────────────────────────────────────── + +if ! $FORMAT_ONLY; then + echo "" + echo "════════════════════════════════════════════════════════" + echo " libgraphql-parser Callgrind benchmarks (deterministic)" + echo "════════════════════════════════════════════════════════" + echo "" + + cargo bench --package libgraphql-parser --bench callgrind_benchmarks -- \ + --save-summary=json +fi + +if [ ! -d "${SUMMARY_ROOT}" ]; then + { + echo "${UNICODE_RED_X} No benchmark summaries found under:" + echo " ${SUMMARY_ROOT}" + echo " Run this script without --format-only first." + } >&2 + exit 1 +fi + +# ─── Helpers ────────────────────────────────────────────── + +# Read a single integer Callgrind metric (e.g. Ir, EstimatedCycles) +# from an iai-callgrind summary.json. Prints an empty string if the +# summary file is missing. +# +# The metric value is `.Left.Int` on a first-ever run, but when a +# previous run's data exists under target/iai, iai-callgrind diffs +# against it and the shape becomes `.Both[0].Int` (current run +# first). Handle both so re-runs without clearing target/iai still +# format correctly. +read_metric() { + local group="$1" + local bench_fn="$2" + local fixture="$3" + local metric="$4" + local json="${SUMMARY_ROOT}/${group}/${bench_fn}.${fixture}/summary.json" + if [ -f "$json" ]; then + jq -r \ + --arg metric "$metric" \ + '.profiles[0].summaries.total.summary.Callgrind[$metric].metrics + | (.Left // .Both[0]) + | .Int // empty' \ + "$json" + else + echo "" + fi +} + +# Format an integer with thousands separators (e.g. 12345678 -> +# 12,345,678). Prints N/A for an empty input. +format_int() { + local val="$1" + if [ -z "$val" ]; then + echo "N/A" + return + fi + # Insert separators right-to-left in groups of three. Avoids + # printf "%'d", which requires locale support that minimal CI + # containers often lack. + echo "$val" | rev | sed 's/\([0-9]\{3\}\)/\1,/g' | rev | sed 's/^,//' +} + +# Return the 0-based index of the minimum value among the arguments. +# Empty arguments are ignored but still occupy an index, so a missing +# metric for one parser can never shift the winner onto the wrong +# column. +find_min_idx() { + local min="" idx=-1 i=0 val + for val in "$@"; do + if [ -n "$val" ] && { [ -z "$min" ] || [ "$val" -lt "$min" ]; }; then + min="$val" + idx=$i + fi + i=$((i + 1)) + done + echo "$idx" +} + +# Emit one markdown table comparing all parsers on a single metric +# for every fixture in a benchmark group. The libgraphql lean-mode +# column is informational and excluded from best-value bolding since +# lean mode does strictly less work than the other parsers' default +# configurations (see the fidelity caveat in the report header). +emit_comparison_table() { + local group="$1" + local metric="$2" + local lg_fn="$3" + local lg_lean_fn="$4" + local gp_fn="$5" + local gp_borrowed_fn="$6" + local ap_fn="$7" + shift 7 + local fixtures=("$@") + + echo -n "| Input | \`libgraphql-parser\` | \`libgraphql-parser\` (lean)" + echo -n " | \`graphql-parser\` | \`graphql-parser\` (zero-copy)" + echo " | \`apollo-parser\` |" + echo -n "|-------|---------------------|----------------------------" + echo -n "|------------------|------------------------------" + echo "|-----------------|" + + local fixture + for fixture in "${fixtures[@]}"; do + local lg lg_lean gp gp_borrowed ap + lg=$(read_metric "$group" "$lg_fn" "$fixture" "$metric") + lg_lean=$(read_metric "$group" "$lg_lean_fn" "$fixture" "$metric") + gp=$(read_metric "$group" "$gp_fn" "$fixture" "$metric") + gp_borrowed=$(read_metric "$group" "$gp_borrowed_fn" "$fixture" "$metric") + ap=$(read_metric "$group" "$ap_fn" "$fixture" "$metric") + + local min_idx + min_idx=$(find_min_idx "$lg" "$gp" "$gp_borrowed" "$ap") + + local cells=() + local vals=("$lg" "$gp" "$gp_borrowed" "$ap") + local j + for j in 0 1 2 3; do + local formatted + formatted=$(format_int "${vals[$j]}") + if [ "$j" -eq "$min_idx" ]; then + cells+=("**${formatted}**") + else + cells+=("${formatted}") + fi + done + + echo -n "| ${fixture} | ${cells[0]} | $(format_int "$lg_lean")" + echo " | ${cells[1]} | ${cells[2]} | ${cells[3]} |" + done +} + +# ─── Detect Environment Metadata ───────────────────────── + +BENCH_DATE="$(date +%Y-%m-%d)" +RUSTC_VERSION="$(rustc --version)" +VALGRIND_VERSION="$(valgrind --version 2>/dev/null || echo "valgrind (version unknown)")" + +GRAPHQL_PARSER_VERSION=$( + cargo metadata --format-version 1 2>/dev/null \ + | jq -r '.packages[] | select(.name == "graphql-parser") | .version' +) +APOLLO_PARSER_VERSION=$( + cargo metadata --format-version 1 2>/dev/null \ + | jq -r '.packages[] | select(.name == "apollo-parser") | .version' +) + +SCHEMA_FIXTURES=("small" "medium" "large" "starwars" "github" "shopify_admin") +EXEC_FIXTURES=("simple" "complex" "nested_10" "nested_30" "many_ops_50") + +# ─── Generate Report ───────────────────────────────────── + +mkdir -p "${IAI_DIR}" + +{ + echo "# libgraphql-parser Callgrind Benchmark Report" + echo "" + echo "> **Measured:** ${BENCH_DATE} under ${VALGRIND_VERSION} (Callgrind," + echo "> \`--cache-sim=yes\`), ${RUSTC_VERSION}, \`bench\` profile." + echo "> Comparison parsers: \`graphql-parser\` ${GRAPHQL_PARSER_VERSION}," + echo "> \`apollo-parser\` ${APOLLO_PARSER_VERSION}." + echo ">" + echo "> All numbers are deterministic simulated-CPU measurements:" + echo "> **Instructions** is the exact count of CPU instructions executed;" + echo "> **Estimated Cycles** is derived from Callgrind's cache simulation" + echo "> (\`L1 hits + 5 * LL hits + 35 * RAM hits\`). Lower is better. These are" + echo "> a machine-independent proxy for relative performance, not" + echo "> wall-clock time." + echo ">" + echo "> **Fidelity caveat:** the compared parsers do different amounts of" + echo "> work. \`libgraphql-parser\` (default) and \`apollo-parser\` both retain" + echo "> lossless syntax/trivia information; \`graphql-parser\` and" + echo "> \`libgraphql-parser\` (lean) produce a semantic AST only. The" + echo "> \`graphql-parser\` column uses its owned-\`String\` mode (matching the" + echo "> Criterion suite) while (zero-copy) uses its borrowed \`&str\` mode." + echo "> The fairest single-axis comparisons are lean vs. \`graphql-parser\`" + echo "> and default vs. \`apollo-parser\`." + echo "" + + echo "## Schema Document Parsing" + echo "" + echo "### Instructions" + echo "" + emit_comparison_table "schema_parse" "Ir" \ + "schema_libgraphql" "schema_libgraphql_lean" \ + "schema_graphql_parser" "schema_graphql_parser_borrowed" \ + "schema_apollo_parser" \ + "${SCHEMA_FIXTURES[@]}" + echo "" + echo "### Estimated Cycles" + echo "" + emit_comparison_table "schema_parse" "EstimatedCycles" \ + "schema_libgraphql" "schema_libgraphql_lean" \ + "schema_graphql_parser" "schema_graphql_parser_borrowed" \ + "schema_apollo_parser" \ + "${SCHEMA_FIXTURES[@]}" + echo "" + + echo "## Executable Document Parsing" + echo "" + echo "### Instructions" + echo "" + emit_comparison_table "executable_parse" "Ir" \ + "executable_libgraphql" "executable_libgraphql_lean" \ + "executable_graphql_parser" "executable_graphql_parser_borrowed" \ + "executable_apollo_parser" \ + "${EXEC_FIXTURES[@]}" + echo "" + echo "### Estimated Cycles" + echo "" + emit_comparison_table "executable_parse" "EstimatedCycles" \ + "executable_libgraphql" "executable_libgraphql_lean" \ + "executable_graphql_parser" "executable_graphql_parser_borrowed" \ + "executable_apollo_parser" \ + "${EXEC_FIXTURES[@]}" + echo "" + + echo "## Lexer (Tokenization Only)" + echo "" + echo "| Input | Instructions | Estimated Cycles |" + echo "|-------|--------------|------------------|" + for fixture in "${SCHEMA_FIXTURES[@]}"; do + ir=$(read_metric "lexer" "lexer_libgraphql" "$fixture" "Ir") + cycles=$(read_metric "lexer" "lexer_libgraphql" "$fixture" "EstimatedCycles") + echo "| ${fixture} | $(format_int "$ir") | $(format_int "$cycles") |" + done + echo "" +} > "${REPORT_FILE}" + +cat "${REPORT_FILE}" + +echo "" >&2 +echo "${UNICODE_GREEN_CHECK} Report written to ${REPORT_FILE}" >&2