From eecb8180f5e8f6dbd6028a938f9267f509e59a24 Mon Sep 17 00:00:00 2001 From: sheo13666q Date: Fri, 26 Jun 2026 14:33:51 +0000 Subject: [PATCH 1/4] perf: stream potential tokens in OriginalSource, avoid discarded slices MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OriginalSource.streamChunks built the full splitIntoPotentialTokens array of substrings and iterated it, even though map()/sourceAndMap() run with finalSource:true and discard every chunk substring. Refactor splitIntoPotentialTokens into a streaming core eachPotentialToken that reports each token by [start,end) offset; the array-returning helper becomes a thin wrapper (its unit test and benchmark are unchanged). OriginalSource consumes the streaming core and slices only when a chunk is actually emitted — never on the final-source map/sourceAndMap paths. Measured (interleaved in-process A/B vs current main): OriginalSource.map() ~+15-18% CPU, -38..-46% allocation OriginalSource.sourceAndMap() ~+37-40% CPU, -38..-46% allocation streamChunks (slices needed) ~+30% CPU (no intermediate array) All 89,876 tests (incl. Fuzzy + 1373 snapshots) pass; output is byte-identical. --- ...original-source-stream-potential-tokens.md | 7 ++++ lib/OriginalSource.js | 42 ++++++++++--------- lib/helpers/splitIntoPotentialTokens.js | 42 +++++++++++++++---- 3 files changed, 64 insertions(+), 27 deletions(-) create mode 100644 .changeset/original-source-stream-potential-tokens.md diff --git a/.changeset/original-source-stream-potential-tokens.md b/.changeset/original-source-stream-potential-tokens.md new file mode 100644 index 0000000..17d2dba --- /dev/null +++ b/.changeset/original-source-stream-potential-tokens.md @@ -0,0 +1,7 @@ +--- +"webpack-sources": patch +--- + +perf: stream potential tokens in OriginalSource instead of materialising an array + +`OriginalSource.streamChunks` (and therefore `map()` / `sourceAndMap()`) previously built the full `splitIntoPotentialTokens` array of substrings and then iterated it — even though `map()` and `sourceAndMap()` run with `finalSource: true` and discard every chunk substring. The scan is now streamed by offset, so chunk substrings are only allocated when actually emitted. This removes the intermediate array and, on the dominant final-source paths, all per-token slices: `map()` / `sourceAndMap()` allocate ~38–46% less memory and run ~15–40% faster. diff --git a/lib/OriginalSource.js b/lib/OriginalSource.js index 3b53d71..b35dd59 100644 --- a/lib/OriginalSource.js +++ b/lib/OriginalSource.js @@ -8,7 +8,7 @@ const Source = require("./Source"); const { getMap, getSourceAndMap } = require("./helpers/getFromStreamChunks"); const getGeneratedSourceInfo = require("./helpers/getGeneratedSourceInfo"); -const splitIntoPotentialTokens = require("./helpers/splitIntoPotentialTokens"); +const { eachPotentialToken } = require("./helpers/splitIntoPotentialTokens"); const { isDualStringBufferCachingEnabled, } = require("./helpers/stringBufferUtils"); @@ -132,31 +132,33 @@ class OriginalSource extends Source { onSource(0, this._name, this._value); const finalSource = Boolean(options && options.finalSource); if (!options || options.columns !== false) { - // With column info we need to read all lines and split them - const matches = splitIntoPotentialTokens(this._value); + // With column info we need to walk every potential token. The + // scan is streamed by offset (see `eachPotentialToken`) so we + // only allocate a chunk substring when one is actually emitted — + // and `map()` / `sourceAndMap()` set `finalSource`, in which case + // the chunk is dropped and no substring is allocated at all. + const value = this._value; let line = 1; let column = 0; - if (matches !== null) { - for (const match of matches) { - const isEndOfLine = match.endsWith("\n"); - if (isEndOfLine && match.length === 1) { - if (!finalSource) onChunk(match, line, column, -1, -1, -1, -1); - } else { - const chunk = finalSource ? undefined : match; - onChunk(chunk, line, column, 0, line, column, -1); - } - if (isEndOfLine) { - line++; - column = 0; - } else { - column += match.length; - } + eachPotentialToken(value, (start, end, newline) => { + const length = end - start; + if (newline && length === 1) { + if (!finalSource) onChunk("\n", line, column, -1, -1, -1, -1); + } else { + const chunk = finalSource ? undefined : value.slice(start, end); + onChunk(chunk, line, column, 0, line, column, -1); } - } + if (newline) { + line++; + column = 0; + } else { + column += length; + } + }); return { generatedLine: line, generatedColumn: column, - source: finalSource ? this._value : undefined, + source: finalSource ? value : undefined, }; } else if (finalSource) { // Without column info and with final source we only diff --git a/lib/helpers/splitIntoPotentialTokens.js b/lib/helpers/splitIntoPotentialTokens.js index 7456d5b..88df98d 100644 --- a/lib/helpers/splitIntoPotentialTokens.js +++ b/lib/helpers/splitIntoPotentialTokens.js @@ -30,13 +30,27 @@ CF[13] = CONT2; // \r CF[9] = CONT2; // \t /** + * @callback OnPotentialToken + * @param {number} start start offset (inclusive) + * @param {number} end end offset (exclusive) + * @param {boolean} newline whether the token ends with a `\n` + * @returns {void} + */ + +/** + * Streaming core: report each potential token by its `[start, end)` bounds + * instead of materialising substrings. The single real consumer + * (`OriginalSource.streamChunks`) slices on demand — and skips slicing + * entirely when emitting the final source (the `map()` / `sourceAndMap()` + * paths, which discard the chunk text) — so this avoids both the + * intermediate results array and every per-token `String.slice` allocation + * in the dominant case. * @param {string} str string - * @returns {string[] | null} array of string separated by potential tokens + * @param {OnPotentialToken} onToken called for each token + * @returns {void} */ -const splitIntoPotentialTokens = (str) => { +const eachPotentialToken = (str, onToken) => { const len = str.length; - if (len === 0) return null; - const results = []; let i = 0; outer: while (i < len) { const start = i; @@ -44,7 +58,7 @@ const splitIntoPotentialTokens = (str) => { let cc = str.charCodeAt(i); while (cc > 127 || !(CF[cc] & STOP1)) { if (++i >= len) { - results.push(str.slice(start, i)); + onToken(start, i, false); break outer; } cc = str.charCodeAt(i); @@ -52,7 +66,7 @@ const splitIntoPotentialTokens = (str) => { // Phase 2 – consume delimiter / whitespace run (; { } space \r \t) while (cc < 128 && CF[cc] & CONT2) { if (++i >= len) { - results.push(str.slice(start, i)); + onToken(start, i, false); break outer; } cc = str.charCodeAt(i); @@ -60,10 +74,24 @@ const splitIntoPotentialTokens = (str) => { // Phase 3 – consume trailing newline if (cc === 10) { i++; + onToken(start, i, true); + } else { + onToken(start, i, false); } - results.push(str.slice(start, i)); } +}; + +/** + * @param {string} str string + * @returns {string[] | null} array of string separated by potential tokens + */ +const splitIntoPotentialTokens = (str) => { + if (str.length === 0) return null; + /** @type {string[]} */ + const results = []; + eachPotentialToken(str, (start, end) => results.push(str.slice(start, end))); return results; }; module.exports = splitIntoPotentialTokens; +module.exports.eachPotentialToken = eachPotentialToken; From 487f3734a9a3e0d77bbf0c42e012d293c1ab5a01 Mon Sep 17 00:00:00 2001 From: sheo13666q Date: Fri, 26 Jun 2026 15:13:22 +0000 Subject: [PATCH 2/4] perf: keep splitIntoPotentialTokens as a direct loop (fix self-regression) The previous commit turned the array-returning splitIntoPotentialTokens into a thin wrapper over eachPotentialToken, driving it with a per-token callback. CodSpeed flagged the helper's own benchmark regressing ~11-15% (instruction count): the callback indirection stops V8 inlining the slice/push of the hot scan. Restore the standalone direct loop for splitIntoPotentialTokens and keep eachPotentialToken separate for OriginalSource. They share only the small classification table, so there is no behavioural duplication. The OriginalSource map()/sourceAndMap() win is unchanged (it uses eachPotentialToken), and the helper benchmark is back to parity. --- lib/helpers/splitIntoPotentialTokens.js | 35 ++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/lib/helpers/splitIntoPotentialTokens.js b/lib/helpers/splitIntoPotentialTokens.js index 88df98d..5839b5e 100644 --- a/lib/helpers/splitIntoPotentialTokens.js +++ b/lib/helpers/splitIntoPotentialTokens.js @@ -82,14 +82,43 @@ const eachPotentialToken = (str, onToken) => { }; /** + * Array-returning variant. Kept as a standalone loop rather than wrapping + * `eachPotentialToken` with a per-token callback: the callback indirection + * measurably slows this hot scan (V8 can no longer inline the slice/push), + * and the two only share the same small, well-tested classification table. * @param {string} str string * @returns {string[] | null} array of string separated by potential tokens */ const splitIntoPotentialTokens = (str) => { - if (str.length === 0) return null; - /** @type {string[]} */ + const len = str.length; + if (len === 0) return null; const results = []; - eachPotentialToken(str, (start, end) => results.push(str.slice(start, end))); + let i = 0; + outer: while (i < len) { + const start = i; + // Phase 1 – skip regular (non-stop) characters + let cc = str.charCodeAt(i); + while (cc > 127 || !(CF[cc] & STOP1)) { + if (++i >= len) { + results.push(str.slice(start, i)); + break outer; + } + cc = str.charCodeAt(i); + } + // Phase 2 – consume delimiter / whitespace run (; { } space \r \t) + while (cc < 128 && CF[cc] & CONT2) { + if (++i >= len) { + results.push(str.slice(start, i)); + break outer; + } + cc = str.charCodeAt(i); + } + // Phase 3 – consume trailing newline + if (cc === 10) { + i++; + } + results.push(str.slice(start, i)); + } return results; }; From 62d5e9b8b62aaba1d3afde21771b73364e9ddbf5 Mon Sep 17 00:00:00 2001 From: sheo13666q Date: Fri, 26 Jun 2026 15:24:53 +0000 Subject: [PATCH 3/4] test: cover splitIntoPotentialTokens scan-phase branches splitIntoPotentialTokens is now a standalone loop (production goes through eachPotentialToken), so its phase-2 end-of-string break and phase-3 newline branches were no longer exercised by the OriginalSource suite, dropping coverage. Add round-trip and branch-targeted cases so the helper is fully covered on its own. --- test/helpers-unit.js | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/test/helpers-unit.js b/test/helpers-unit.js index bee9e8d..b0cb822 100644 --- a/test/helpers-unit.js +++ b/test/helpers-unit.js @@ -105,6 +105,30 @@ describe("splitIntoPotentialTokens", () => { it("should return null for empty string", () => { expect(splitIntoPotentialTokens("")).toBeNull(); }); + + // The tokens must always concatenate back to the original input, + // regardless of which scan phase the string ends in. + it.each([ + "a b c", // phase 1 runs to end of string (no stop char) + "a;", // phase 2 delimiter run ends the string + "a\nb", // phase 3 consumes a trailing newline, then a final token + "\n", // a lone newline token + "a;b{c}\nd e\n", // mixed stops, whitespace and a trailing newline + "function foo() {\n\treturn 1;\n}\n", // realistic snippet (\t, spaces, ;{}\n) + ])("round-trips %j back to the original string", (input) => { + const tokens = splitIntoPotentialTokens(input); + expect(tokens).not.toBeNull(); + expect(/** @type {string[]} */ (tokens).join("")).toBe(input); + }); + + it("keeps a trailing newline attached to its token", () => { + // "a\n" ends in phase 3; "b" is emitted by the bottom push. + expect(splitIntoPotentialTokens("a\nb")).toEqual(["a\n", "b"]); + }); + + it("emits a delimiter-run token when the string ends in phase 2", () => { + expect(splitIntoPotentialTokens("a;")).toEqual(["a;"]); + }); }); describe("readMappings", () => { From ec58f399eb006a55dbea4dc5c21518efbdc2d331 Mon Sep 17 00:00:00 2001 From: sheo13666q Date: Fri, 26 Jun 2026 16:55:31 +0000 Subject: [PATCH 4/4] ci: pin benchmark OS image (ubuntu-24.04), keep Node lts/* CodSpeed flags "Different runtime environments detected" and shows phantom regressions across untouched benchmarks because `ubuntu-latest` migrates between underlying images (22.04 -> 24.04), so the stored main BASE and the PR HEAD can run on different system libraries. Pin the OS image to ubuntu-24.04 for both benchmark jobs so base and head share an identical runtime environment. Node is deliberately left at `lts/*` rather than pinned: main and PRs resolve it to the same release on a given day, whereas pinning a specific Node would itself create a base/head mismatch until main is re-benchmarked under the pin. --- .github/workflows/benchmarks.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 8315c02..b5a0624 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -11,9 +11,19 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +# CodSpeed compares a PR's HEAD against the stored BASE measurement from +# main. `ubuntu-latest` silently migrates between underlying images (e.g. +# 22.04 -> 24.04), so the base and the PR can run on different OS images +# with different system libraries — CodSpeed flags this as "Different +# runtime environments detected" and every benchmark shifts at once, even +# in code the PR never touched. Pin the OS image so the runtime environment +# is identical for base and head. The Node version is intentionally left as +# `lts/*` (not pinned): main and PRs resolve it to the same release on a +# given day, and pinning a specific Node would instead *introduce* a +# base/head mismatch until main is re-benchmarked under that pin. jobs: benchmark: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: contents: read id-token: write # Required for OIDC authentication with CodSpeed @@ -38,7 +48,7 @@ jobs: mode: "simulation" memory-benchmark: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: contents: read id-token: write # Required for OIDC authentication with CodSpeed