From 8c6b103b7e413384f3461ddd75be5fe0bef2d7c6 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 21 Oct 2026 11:55:35 +0200 Subject: [PATCH 01/21] feat(core): add per-benchmark process isolation Benchmarks of one process share allocator, JIT and thread state, so what one benchmark leaves behind (raised mmap threshold, fragmented heap, optimized code, retained caches) changes the measurement of every benchmark after it. Adding or changing one benchmark can then shift unrelated results. Add `Isolation` for the integrations. It is on by default and turned off with `CODSPEED_PROCESS_ISOLATION=false`. `Isolation.run` re-executes the current command with `CODSPEED_ISOLATED_BENCHMARK` set to one benchmark URI, so each benchmark gets a fresh process. The child only runs that benchmark, which overrides user filters. - `CODSPEED_PROCESS_ISOLATION`: unset, `1` or `true` is on, `0` or `false` is off, anything else is an error. - `CODSPEED_ISOLATED_BENCHMARK`: set by the parent on each child. - `CODSPEED_ISOLATION_COMMAND`: JSON command to re-run, for runners that measure in worker processes whose argv is not the benchmark command. The child is started with `process.execPath` and `process.execArgv`, so the V8 flags set by the runner survive. A failing child ends the parent with the same exit code (128 + N when signal N killed it). Children run sequentially with `spawnSync`, which keeps the output in order. Re-exec is used instead of fork because fork only copies the calling thread. --- packages/core/src/index.ts | 1 + packages/core/src/isolation.ts | 182 +++++++++++++++++++++++++++++++++ 2 files changed, 183 insertions(+) create mode 100644 packages/core/src/isolation.ts diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 63ac394f..514abe22 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -51,6 +51,7 @@ export type { SetupInstrumentsResponse, } from "./generated/openapi"; export { getV8Flags, tryIntrospect } from "./introspection"; +export * from "./isolation"; export { getUnsupportedNodeVersionWarning, SUPPORTED_NODE_MAJORS, diff --git a/packages/core/src/isolation.ts b/packages/core/src/isolation.ts new file mode 100644 index 00000000..0a8e013a --- /dev/null +++ b/packages/core/src/isolation.ts @@ -0,0 +1,182 @@ +/** + * Process isolation: every benchmark runs in a fresh copy of the benchmark + * process, so the allocator state, JIT state, and handles that other + * benchmarks or setup code leave behind can't change its measurement. + */ +import { spawnSync, type SpawnSyncReturns } from "child_process"; +import { constants } from "os"; + +/** Isolation is on unless this is `0` or `false`. */ +export const PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION"; +/** Set on each child process: the URI of the only benchmark it runs. */ +export const ISOLATED_BENCHMARK_ENV = "CODSPEED_ISOLATED_BENCHMARK"; +/** + * JSON array `[executable, ...args]` of the command that runs the benchmarks + * again. Integrations whose benchmarks run in a worker process (vitest) set it + * from the main process, because re-executing the worker alone can't work. + */ +export const ISOLATION_COMMAND_ENV = "CODSPEED_ISOLATION_COMMAND"; + +/** Exit code for a child that ended with neither an exit code nor a signal. */ +const GENERIC_FAILURE_EXIT_CODE = 1; +/** Shells report a child killed by signal N as exit code 128 + N. */ +const SIGNAL_EXIT_CODE_OFFSET = 128; + +/** How an integration runs each benchmark. */ +export type IsolationMode = + /** All benchmarks share this process and its state. */ + | { kind: "SharedProcess" } + /** Every benchmark runs in its own child process. */ + | { kind: "ProcessPerBenchmark" } + /** A child spawned in `ProcessPerBenchmark` mode, restricted to `uri`. */ + | { kind: "IsolatedBenchmark"; uri: string }; + +export class Isolation { + private static instance: Isolation | undefined; + private static readonly sharedInstance = new Isolation({ + kind: "SharedProcess", + }); + + private constructor(readonly mode: IsolationMode) {} + + /** + * The isolation of runs where no instrument measures anything (the hooks are + * not active), so there is nothing to isolate. + */ + static shared(): Isolation { + return Isolation.sharedInstance; + } + + /** The isolation of this process, read once from the environment. */ + static current(): Isolation { + Isolation.instance ??= new Isolation(modeFromEnv()); + return Isolation.instance; + } + + /** The only benchmark this process may run. It overrides user filters. */ + onlyBenchmark(): string | undefined { + switch (this.mode.kind) { + case "IsolatedBenchmark": + return this.mode.uri; + case "SharedProcess": + case "ProcessPerBenchmark": + return undefined; + } + } + + /** Whether this process must run the benchmark `uri`. */ + shouldRun(uri: string): boolean { + const only = this.onlyBenchmark(); + return only === undefined || only === uri; + } + + /** Whether this process is a child that runs one benchmark for a parent. */ + isChild(): boolean { + return this.onlyBenchmark() !== undefined; + } + + /** + * Runs the benchmark `uri`. `bench` must contain the whole measurement, + * from `startBenchmark` to `setExecutedBenchmark`. + */ + async run(uri: string, bench: () => Promise | void): Promise { + switch (this.mode.kind) { + case "SharedProcess": + case "IsolatedBenchmark": + await bench(); + return; + case "ProcessPerBenchmark": + runInChildProcess(uri); + return; + } + } + + /** Synchronous variant of {@link Isolation.run}. */ + runSync(uri: string, bench: () => void): void { + switch (this.mode.kind) { + case "SharedProcess": + case "IsolatedBenchmark": + bench(); + return; + case "ProcessPerBenchmark": + runInChildProcess(uri); + return; + } + } +} + +function modeFromEnv(): IsolationMode { + const uri = process.env[ISOLATED_BENCHMARK_ENV]; + if (uri !== undefined) { + return { kind: "IsolatedBenchmark", uri }; + } + + const value = process.env[PROCESS_ISOLATION_ENV]; + switch (value) { + case undefined: + case "1": + case "true": + return { kind: "ProcessPerBenchmark" }; + case "0": + case "false": + return { kind: "SharedProcess" }; + default: + throw new Error( + `${PROCESS_ISOLATION_ENV} must be \`true\` or \`false\`, got \`${value}\``, + ); + } +} + +/** + * Records the command of this process as the one that children run. Call it + * from the main process of integrations that measure in worker processes. It + * overwrites a command inherited from an outer process, which is not ours. + */ +export function recordIsolationCommand(): void { + process.env[ISOLATION_COMMAND_ENV] = JSON.stringify(currentCommand()); +} + +function currentCommand(): string[] { + // execArgv keeps the V8 flags (e.g. --perf-prof) that the runner set. + return [process.execPath, ...process.execArgv, ...process.argv.slice(1)]; +} + +function isolationCommand(): string[] { + const recorded = process.env[ISOLATION_COMMAND_ENV]; + if (recorded === undefined) { + return currentCommand(); + } + return JSON.parse(recorded) as string[]; +} + +/** + * Runs the benchmark command again, limited to `uri`, and exits with the + * child's code if it fails. Re-exec, not fork(): Node can't fork, and a fork + * would only copy the calling thread. + */ +function runInChildProcess(uri: string): void { + const [executable, ...args] = isolationCommand(); + // spawnSync blocks the event loop, so stdout written before this call is + // already flushed and output stays in order. + const result = spawnSync(executable, args, { + stdio: "inherit", + env: { ...process.env, [ISOLATED_BENCHMARK_ENV]: uri }, + }); + if (result.error !== undefined) { + throw new Error( + `[CodSpeed] failed to start the process for ${uri}: ${result.error.message}`, + ); + } + if (result.status === 0) { + return; + } + process.exit(exitCode(result)); +} + +/** Exit code of a failed child, in the shell convention. */ +function exitCode(result: SpawnSyncReturns): number { + if (result.signal !== null) { + return SIGNAL_EXIT_CODE_OFFSET + constants.signals[result.signal]; + } + return result.status ?? GENERIC_FAILURE_EXIT_CODE; +} From 7b9f42adabe9c9027f526f4afb6a3bd8c1a98269 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 21 Oct 2026 11:55:36 +0200 Subject: [PATCH 02/21] feat(benchmark.js): run each benchmark in its own process Route every instrumented benchmark through `Isolation.run`. A child process only runs its own benchmark and skips the banner, the summary line and the completion listeners that the parent already handles. A run call that owns no benchmark of the child does not touch the instruments. --- packages/benchmark.js-plugin/src/index.ts | 130 +++++++++++++--------- 1 file changed, 80 insertions(+), 50 deletions(-) diff --git a/packages/benchmark.js-plugin/src/index.ts b/packages/benchmark.js-plugin/src/index.ts index fa69a35a..747b5fc2 100644 --- a/packages/benchmark.js-plugin/src/index.ts +++ b/packages/benchmark.js-plugin/src/index.ts @@ -1,6 +1,7 @@ import { getCallingFile, InstrumentHooks, + Isolation, mongoMeasurement, optimizeFunction, optimizeFunctionSync, @@ -168,63 +169,92 @@ async function runBenchmarks({ baseUri, benchmarkCompletedListeners, }: RunBenchmarksOptions): Promise { - console.log(`[CodSpeed] running with @codspeed/benchmark.js v${__VERSION__}`); + const isolation = Isolation.current(); + const isChild = isolation.isChild(); + const selected = benches + .map((bench, i) => ({ + bench, + uri: bench.uri ?? `${baseUri}::unknown_${i}`, + })) + .filter(({ uri }) => isolation.shouldRun(uri)); + // A child only measures one benchmark. A run call that owns none of them + // must not touch the instruments, the call that owns it does. + if (isChild && selected.length === 0) { + return; + } + + // The parent already printed the banner and the summary, children stay quiet. + if (!isChild) { + console.log( + `[CodSpeed] running with @codspeed/benchmark.js v${__VERSION__}`, + ); + } setupCore(); - for (let i = 0; i < benches.length; i++) { - const bench = benches[i]; - const uri = bench.uri ?? `${baseUri}::unknown_${i}`; - const isAsync = bench.options.async || bench.options.defer; - let benchPayload; - if (bench.options.defer) { - benchPayload = () => { - return new Promise((resolve, reject) => { - (bench.fn as CallableFunction)({ resolve, reject }); - }); - }; - } else if (bench.options.async) { - benchPayload = async () => { - await (bench.fn as CallableFunction)(); - }; - } else { - benchPayload = bench.fn as CallableFunction; + for (const { bench, uri } of selected) { + await isolation.run(uri, () => measureBenchmark(bench, uri)); + // The parent reports for every benchmark, the child ran only one of them. + if (!isChild) { + benchmarkCompletedListeners.forEach((listener) => listener()); } + } + teardownCore(); + if (!isChild) { + console.log(`[CodSpeed] Done running ${benches.length} benches.`); + } +} - if (typeof bench.options.setup === "function") { - await bench.options.setup(); - } +async function measureBenchmark( + bench: BenchmarkWithOptions, + uri: string, +): Promise { + const isAsync = bench.options.async || bench.options.defer; + let benchPayload; + if (bench.options.defer) { + benchPayload = () => { + const { promise, resolve, reject } = Promise.withResolvers(); + (bench.fn as CallableFunction)({ resolve, reject }); + return promise; + }; + } else if (bench.options.async) { + benchPayload = async () => { + await (bench.fn as CallableFunction)(); + }; + } else { + benchPayload = bench.fn as CallableFunction; + } - if (isAsync) { - await optimizeFunction(benchPayload); - await mongoMeasurement.start(uri); - global.gc?.(); - await wrapWithRootFrame(async () => { - InstrumentHooks.startBenchmark(); - await benchPayload(); - InstrumentHooks.stopBenchmark(); - InstrumentHooks.setExecutedBenchmark(process.pid, uri); - })(); - await mongoMeasurement.stop(uri); - } else { - optimizeFunctionSync(benchPayload); - await mongoMeasurement.start(uri); - wrapWithRootFrameSync(() => { - InstrumentHooks.startBenchmark(); - benchPayload(); - InstrumentHooks.stopBenchmark(); - InstrumentHooks.setExecutedBenchmark(process.pid, uri); - })(); - await mongoMeasurement.stop(uri); - } + if (typeof bench.options.setup === "function") { + await bench.options.setup(); + } - if (typeof bench.options.teardown === "function") { - await bench.options.teardown(); - } + if (isAsync) { + await optimizeFunction(benchPayload); + await mongoMeasurement.start(uri); + global.gc?.(); + await wrapWithRootFrame(async () => { + InstrumentHooks.startBenchmark(); + await benchPayload(); + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(process.pid, uri); + })(); + await mongoMeasurement.stop(uri); + } else { + optimizeFunctionSync(benchPayload); + await mongoMeasurement.start(uri); + wrapWithRootFrameSync(() => { + InstrumentHooks.startBenchmark(); + benchPayload(); + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(process.pid, uri); + })(); + await mongoMeasurement.stop(uri); + } - console.log(` ✔ Measured ${uri}`); - benchmarkCompletedListeners.forEach((listener) => listener()); + if (typeof bench.options.teardown === "function") { + await bench.options.teardown(); } - teardownCore(); - console.log(`[CodSpeed] Done running ${benches.length} benches.`); + + console.log(` ✔ Measured ${uri}`); } /** From cf218bdae141937d9da01abb2e91be097b304025 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 21 Oct 2026 11:55:37 +0200 Subject: [PATCH 03/21] feat(tinybench): run each benchmark in its own process The analysis runner measures every task through `Isolation.run` (async and sync). Walltime runs and runs without active instrument hooks are not isolated. A child only measures its own task and stays quiet about the banner and the summary line. --- packages/tinybench-plugin/src/analysis.ts | 7 ++- packages/tinybench-plugin/src/shared.ts | 53 ++++++++++++++++++----- 2 files changed, 49 insertions(+), 11 deletions(-) diff --git a/packages/tinybench-plugin/src/analysis.ts b/packages/tinybench-plugin/src/analysis.ts index 65bd6183..750925e5 100644 --- a/packages/tinybench-plugin/src/analysis.ts +++ b/packages/tinybench-plugin/src/analysis.ts @@ -1,6 +1,7 @@ import { getCodspeedRunnerMode, InstrumentHooks, + Isolation, mongoMeasurement, optimizeFunction, optimizeFunctionSync, @@ -14,7 +15,11 @@ export function setupCodspeedAnalysisBench( bench: Bench, rootCallingFile: string, ): void { - const runner = new AnalysisBenchRunner(bench, rootCallingFile); + // Without instrumentation nothing is measured, so there is nothing to isolate. + const isolation = InstrumentHooks.isInstrumented() + ? Isolation.current() + : Isolation.shared(); + const runner = new AnalysisBenchRunner(bench, rootCallingFile, isolation); runner.setupBenchMethods(); } diff --git a/packages/tinybench-plugin/src/shared.ts b/packages/tinybench-plugin/src/shared.ts index 0fb811c5..14828bac 100644 --- a/packages/tinybench-plugin/src/shared.ts +++ b/packages/tinybench-plugin/src/shared.ts @@ -1,4 +1,4 @@ -import { setupCore, teardownCore } from "@codspeed/core"; +import { Isolation, setupCore, teardownCore } from "@codspeed/core"; import { Bench, Task } from "tinybench"; import { CapturedTaskData, getTaskData } from "./taskData"; import { getTaskUri } from "./uri"; @@ -8,10 +8,19 @@ declare const __VERSION__: string; export abstract class BaseBenchRunner { protected bench: Bench; protected rootCallingFile: string; - - constructor(bench: Bench, rootCallingFile: string) { + /** Runners that measure pass the isolation of this process; walltime shares it. */ + private readonly isolation: Isolation; + private readonly isChild: boolean; + + constructor( + bench: Bench, + rootCallingFile: string, + isolation: Isolation = Isolation.shared(), + ) { this.bench = bench; this.rootCallingFile = rootCallingFile; + this.isolation = isolation; + this.isChild = isolation.isChild(); } private setupBenchRun(): void { @@ -20,6 +29,10 @@ export abstract class BaseBenchRunner { } private logStart(): void { + // The parent already printed the banner, children stay quiet. + if (this.isChild) { + return; + } console.log( `[CodSpeed] running with @codspeed/tinybench v${__VERSION__} (${this.getModeName()})`, ); @@ -45,7 +58,12 @@ export abstract class BaseBenchRunner { protected finalizeBenchRun(): Task[] { teardownCore(); - console.log(`[CodSpeed] Done running ${this.bench.tasks.length} benches.`); + // The parent already printed the summary line, children stay quiet. + if (!this.isChild) { + console.log( + `[CodSpeed] Done running ${this.bench.tasks.length} benches.`, + ); + } return this.bench.tasks; } @@ -57,25 +75,40 @@ export abstract class BaseBenchRunner { public setupBenchMethods(): void { this.bench.run = async () => { + const tasks = this.selectTasks(); + // A child that owns none of this bench's tasks must not touch the + // instruments, the bench that owns its task does. + if (this.isChild && tasks.length === 0) { + return this.bench.tasks; + } this.setupBenchRun(); - for (const task of this.bench.tasks) { - const uri = this.getTaskUri(task); - await this.runTaskAsync(task, uri); + for (const { task, uri } of tasks) { + await this.isolation.run(uri, () => this.runTaskAsync(task, uri)); } return this.finalizeAsyncRun(); }; this.bench.runSync = () => { + const tasks = this.selectTasks(); + if (this.isChild && tasks.length === 0) { + return this.bench.tasks; + } this.setupBenchRun(); - for (const task of this.bench.tasks) { - const uri = this.getTaskUri(task); - this.runTaskSync(task, uri); + for (const { task, uri } of tasks) { + this.isolation.runSync(uri, () => this.runTaskSync(task, uri)); } return this.finalizeSyncRun(); }; } + + /** The tasks this process measures: all of them, or the isolated one. */ + private selectTasks(): { task: Task; uri: string }[] { + return this.bench.tasks + .map((task) => ({ task, uri: this.getTaskUri(task) })) + .filter(({ uri }) => this.isolation.shouldRun(uri)); + } } From b14741529de5fbeef4fc36a5a7d72a8fe1e628c4 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 21 Oct 2026 11:55:37 +0200 Subject: [PATCH 04/21] feat(vitest): run each benchmark in its own process Both the Vitest 5 provider and the legacy analysis runner measure through `Isolation.run`. An isolated child still reports every benchmark to Vitest, but only measures its own. Vitest measures in forked workers, and the argv of a worker is not the command that starts the benchmarks. The plugin records the command of the main Vitest process in `CODSPEED_ISOLATION_COMMAND` from its `config()` hook, before the workers are forked, and each worker re-runs that command once per benchmark. This means one full Vitest run per benchmark. --- packages/vitest-plugin/src/index.ts | 5 + packages/vitest-plugin/src/legacy/analysis.ts | 99 ++++++++++++------- packages/vitest-plugin/src/v5/provider.ts | 16 ++- 3 files changed, 84 insertions(+), 36 deletions(-) diff --git a/packages/vitest-plugin/src/index.ts b/packages/vitest-plugin/src/index.ts index cb3af1ae..ae81cf3b 100644 --- a/packages/vitest-plugin/src/index.ts +++ b/packages/vitest-plugin/src/index.ts @@ -3,6 +3,7 @@ import { getV8Flags, InstrumentHooks, mongoMeasurement, + recordIsolationCommand, SetupInstrumentsRequestBody, SetupInstrumentsResponse, } from "@codspeed/core"; @@ -38,6 +39,10 @@ export default function codspeedPlugin(): Plugin { }, enforce: "post", config(userConfig): ViteUserConfig { + // Vitest measures in forked workers, whose argv is not the command that + // starts the benchmarks. Record the command of this main process so the + // workers can run it again for each benchmark. Workers inherit the env. + recordIsolationCommand(); return { test: { pool: "forks", diff --git a/packages/vitest-plugin/src/legacy/analysis.ts b/packages/vitest-plugin/src/legacy/analysis.ts index f71f0c0a..7dc9f7d3 100644 --- a/packages/vitest-plugin/src/legacy/analysis.ts +++ b/packages/vitest-plugin/src/legacy/analysis.ts @@ -1,5 +1,6 @@ import { InstrumentHooks, + Isolation, logDebug, mongoMeasurement, optimizeFunction, @@ -41,44 +42,50 @@ async function runAnalysisBench( suite: RunnerTestSuite, currentSuiteName: string, tinybenchModule: Tinybench, + isolation: Isolation, ) { const uri = `${currentSuiteName}::${benchmark.name}`; - const fn = getBenchFn(benchmark); - - // Constructing a Bench applies tinybench's no-op defaults for the setup and - // teardown hooks and gives them the Task they expect. The bench itself is - // never run: this runner drives the benchmark function directly. - const bench = new tinybenchModule.Bench(getBenchOptions(benchmark)); - const task = new tinybenchModule.Task(bench, benchmark.name, fn); - - await bench.setup(task, "warmup"); - await optimizeFunction(async () => { + if (!isolation.shouldRun(uri)) return; + + await isolation.run(uri, async () => { + const fn = getBenchFn(benchmark); + + // Constructing a Bench applies tinybench's no-op defaults for the setup and + // teardown hooks and gives them the Task they expect. The bench itself is + // never run: this runner drives the benchmark function directly. + const bench = new tinybenchModule.Bench(getBenchOptions(benchmark)); + const task = new tinybenchModule.Task(bench, benchmark.name, fn); + + await bench.setup(task, "warmup"); + await optimizeFunction(async () => { + await callSuiteHook(suite, benchmark, "beforeEach"); + await fn(); + await callSuiteHook(suite, benchmark, "afterEach"); + }); + await bench.teardown(task, "warmup"); + + await bench.setup(task, "run"); await callSuiteHook(suite, benchmark, "beforeEach"); - await fn(); + await mongoMeasurement.start(uri); + global.gc?.(); + await wrapWithRootFrame(async () => { + InstrumentHooks.startBenchmark(); + await fn(); + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(process.pid, uri); + })(); + await mongoMeasurement.stop(uri); await callSuiteHook(suite, benchmark, "afterEach"); + await bench.teardown(task, "run"); + + logCodSpeed(`${uri} done`); }); - await bench.teardown(task, "warmup"); - - await bench.setup(task, "run"); - await callSuiteHook(suite, benchmark, "beforeEach"); - await mongoMeasurement.start(uri); - global.gc?.(); - await wrapWithRootFrame(async () => { - InstrumentHooks.startBenchmark(); - await fn(); - InstrumentHooks.stopBenchmark(); - InstrumentHooks.setExecutedBenchmark(process.pid, uri); - })(); - await mongoMeasurement.stop(uri); - await callSuiteHook(suite, benchmark, "afterEach"); - await bench.teardown(task, "run"); - - logCodSpeed(`${uri} done`); } async function runAnalysisBenchmarkSuite( suite: RunnerTestSuite, tinybenchModule: Tinybench, + isolation: Isolation, parentSuiteName?: string, ) { const currentSuiteName = parentSuiteName @@ -91,9 +98,20 @@ async function runAnalysisBenchmarkSuite( if (task.mode !== "run") continue; if (task.type === "suite") { - await runAnalysisBenchmarkSuite(task, tinybenchModule, currentSuiteName); + await runAnalysisBenchmarkSuite( + task, + tinybenchModule, + isolation, + currentSuiteName, + ); } else if (isVitestTaskBenchmark(task)) { - await runAnalysisBench(task, suite, currentSuiteName, tinybenchModule); + await runAnalysisBench( + task, + suite, + currentSuiteName, + tinybenchModule, + isolation, + ); } } @@ -107,9 +125,24 @@ export class AnalysisRunner extends NodeBenchmarkRunner { patchRootSuiteWithFullFilePath(suite); - logCodSpeed(`running suite ${suite.name}`); - await runAnalysisBenchmarkSuite(suite, await this.importTinybench()); - logCodSpeed(`running suite ${suite.name} done`); + // Without instrumentation nothing is measured, so there is nothing to isolate. + const isolation = InstrumentHooks.isInstrumented() + ? Isolation.current() + : Isolation.shared(); + // The parent already printed the suite lines, children stay quiet. + const isChild = isolation.isChild(); + + if (!isChild) { + logCodSpeed(`running suite ${suite.name}`); + } + await runAnalysisBenchmarkSuite( + suite, + await this.importTinybench(), + isolation, + ); + if (!isChild) { + logCodSpeed(`running suite ${suite.name} done`); + } teardownCore(); } diff --git a/packages/vitest-plugin/src/v5/provider.ts b/packages/vitest-plugin/src/v5/provider.ts index ef38fe0e..c54d2017 100644 --- a/packages/vitest-plugin/src/v5/provider.ts +++ b/packages/vitest-plugin/src/v5/provider.ts @@ -2,6 +2,7 @@ import { getGitDir, getInstrumentMode, InstrumentHooks, + Isolation, optimizeFunction, optimizeFunctionSync, setupCore, @@ -155,12 +156,21 @@ async function runAnalysis( registrations: BenchRegistrationInput[], getUri: (name: string) => string, ): Promise { - const label = InstrumentHooks.isInstrumented() ? "Measured" : "Checked"; + const instrumented = InstrumentHooks.isInstrumented(); + const label = instrumented ? "Measured" : "Checked"; + // Without instrumentation nothing is measured, so there is nothing to isolate. + const isolation = instrumented ? Isolation.current() : Isolation.shared(); const results: BenchResult[] = []; for (const { name, fn, fnOpts } of registrations) { const uri = getUri(name); - await runAnalysisTask(fn, fnOpts, uri); - console.log(`[CodSpeed] ${label} ${uri}`); + // An isolated child still reports every benchmark to Vitest, but it only + // measures its own. + if (isolation.shouldRun(uri)) { + await isolation.run(uri, async () => { + await runAnalysisTask(fn, fnOpts, uri); + console.log(`[CodSpeed] ${label} ${uri}`); + }); + } results.push(analysisResult(name)); } return results; From d4c8e425662a3d17faec7a23c6ea885159d89e00 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 21 Oct 2026 11:55:45 +0200 Subject: [PATCH 05/21] test: cover process isolation end to end Run fixtures in simulation mode with the instrument hooks stubbed (they only exist under the CodSpeed runner) and check, for core, benchmark.js, tinybench and Vitest 5, that: - every benchmark runs in a fresh process by default; - `CODSPEED_PROCESS_ISOLATION=false` runs them all in one process; - a failing benchmark fails the run; - core forwards the child exit code (128 + N for signals) and the execArgv. Tests that drive a plugin in-process against a mocked core now set `CODSPEED_PROCESS_ISOLATION=false`, because isolation would start the test runner again for every benchmark. --- .../tests/fixtures/isolation.cjs | 37 ++++ .../tests/index.integ.test.ts | 4 + .../tests/isolation.integ.test.ts | 107 ++++++++++++ packages/core/tests/fixtures/isolation.cjs | 27 +++ packages/core/tests/isolation.integ.test.ts | 162 ++++++++++++++++++ .../tests/fixtures/isolation.mjs | 46 +++++ .../tests/index.integ.test.ts | 3 + .../tests/isolation.integ.test.ts | 122 +++++++++++++ .../src/__tests__/instrumented.test.ts | 6 + .../src/__tests__/isolation.test.ts | 113 ++++++++++++ .../tests/fixtures/isolation/pids.bench.ts | 19 ++ .../tests/fixtures/isolation/stub.mjs | 16 ++ .../tests/fixtures/isolation/vitest.config.ts | 12 ++ 13 files changed, 674 insertions(+) create mode 100644 packages/benchmark.js-plugin/tests/fixtures/isolation.cjs create mode 100644 packages/benchmark.js-plugin/tests/isolation.integ.test.ts create mode 100644 packages/core/tests/fixtures/isolation.cjs create mode 100644 packages/core/tests/isolation.integ.test.ts create mode 100644 packages/tinybench-plugin/tests/fixtures/isolation.mjs create mode 100644 packages/tinybench-plugin/tests/isolation.integ.test.ts create mode 100644 packages/vitest-plugin/src/__tests__/isolation.test.ts create mode 100644 packages/vitest-plugin/tests/fixtures/isolation/pids.bench.ts create mode 100644 packages/vitest-plugin/tests/fixtures/isolation/stub.mjs create mode 100644 packages/vitest-plugin/tests/fixtures/isolation/vitest.config.ts diff --git a/packages/benchmark.js-plugin/tests/fixtures/isolation.cjs b/packages/benchmark.js-plugin/tests/fixtures/isolation.cjs new file mode 100644 index 00000000..1ab23728 --- /dev/null +++ b/packages/benchmark.js-plugin/tests/fixtures/isolation.cjs @@ -0,0 +1,37 @@ +// Run by `isolation.integ.test.ts`. Prints the PID that measures each benchmark. +const { InstrumentHooks } = require("@codspeed/core"); +const Benchmark = require("benchmark"); +const { withCodSpeed } = require("../../dist/index.cjs.js"); + +// There is no instrument hook runtime outside the CodSpeed runner, so pretend +// the hooks are active. The child processes run this file again, so they get +// the same override. +InstrumentHooks.isInstrumented = () => true; + +// The plugin reports every measured benchmark here, with the PID of the +// process that measured it. +const setExecutedBenchmark = InstrumentHooks.setExecutedBenchmark; +InstrumentHooks.setExecutedBenchmark = (pid, uri) => { + setExecutedBenchmark(pid, uri); + console.log(`executed ${pid} ${uri}`); +}; + +const suite = withCodSpeed(new Benchmark.Suite("isolation")); + +suite + .add("first", () => {}) + .add("second", () => {}) + .add("third", () => {}) + .add("async", { + defer: true, + fn: (deferred) => deferred.resolve(), + }); + +if (process.env.ISOLATION_FIXTURE_FAIL !== undefined) { + suite.add("failing", () => { + throw new Error("benchmark failed"); + }); +} + +// An unhandled rejection ends the process with exit code 1. +suite.run(); diff --git a/packages/benchmark.js-plugin/tests/index.integ.test.ts b/packages/benchmark.js-plugin/tests/index.integ.test.ts index f3e21034..3d2256f5 100644 --- a/packages/benchmark.js-plugin/tests/index.integ.test.ts +++ b/packages/benchmark.js-plugin/tests/index.integ.test.ts @@ -11,6 +11,7 @@ jest.mock("@codspeed/core", () => { const actual = jest.requireActual("@codspeed/core"); mockCore.getGitDir = actual.getGitDir; mockCore.getCallingFile = actual.getCallingFile; + mockCore.Isolation = actual.Isolation; mockCore.wrapWithRootFrame = actual.wrapWithRootFrame; mockCore.wrapWithRootFrameSync = actual.wrapWithRootFrameSync; return mockCore; @@ -19,6 +20,9 @@ jest.mock("@codspeed/core", () => { beforeEach(() => { mockReset(mockCore); jest.clearAllMocks(); + // These tests drive the plugin in this process against a mocked core, so + // there is no benchmark process to spawn. + process.env.CODSPEED_PROCESS_ISOLATION = "false"; }); const benchOptions: Benchmark.Options = { diff --git a/packages/benchmark.js-plugin/tests/isolation.integ.test.ts b/packages/benchmark.js-plugin/tests/isolation.integ.test.ts new file mode 100644 index 00000000..0c59e91d --- /dev/null +++ b/packages/benchmark.js-plugin/tests/isolation.integ.test.ts @@ -0,0 +1,107 @@ +import { getV8Flags } from "@codspeed/core"; +import { spawnSync } from "child_process"; +import { join } from "path"; + +const FIXTURE = join(__dirname, "fixtures", "isolation.cjs"); +const PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION"; +/** `first`, `second`, `third` and the deferred `async` benchmark. */ +const BENCHMARK_COUNT = 4; +/** Fails a run whose benchmark process got stuck instead of blocking the suite. */ +const RUN_TIMEOUT_MS = 60_000; +const EXECUTED_LINE = /^executed (\d+) (.+)$/; + +interface FixtureRun { + status: number | null; + stderr: string; + /** PID that measured each benchmark. */ + pidsByUri: Map; + /** PID of the process the test started. */ + parentPid: number; +} + +interface FixtureOptions { + /** Value of CODSPEED_PROCESS_ISOLATION, unset when undefined. */ + isolation?: string; + failing?: boolean; +} + +function runFixture({ isolation, failing }: FixtureOptions = {}): FixtureRun { + const env: NodeJS.ProcessEnv = { + ...process.env, + CODSPEED_ENV: "true", + CODSPEED_RUNNER_MODE: "simulation", + }; + delete env[PROCESS_ISOLATION_ENV]; + if (isolation !== undefined) { + env[PROCESS_ISOLATION_ENV] = isolation; + } + if (failing) { + env.ISOLATION_FIXTURE_FAIL = "1"; + } + + // The runner starts Node with these flags, and children must keep them. + const result = spawnSync(process.execPath, [...getV8Flags(), FIXTURE], { + env, + encoding: "utf8", + timeout: RUN_TIMEOUT_MS, + }); + + const pidsByUri = new Map(); + for (const line of result.stdout.split("\n")) { + const match = EXECUTED_LINE.exec(line); + if (match) { + pidsByUri.set(match[2], Number(match[1])); + } + } + return { + status: result.status, + stderr: result.stderr, + pidsByUri, + parentPid: result.pid, + }; +} + +describe("process isolation", () => { + it.each([undefined, "1", "true"])( + "measures every benchmark in its own process (isolation=%s)", + (isolation) => { + const run = runFixture({ isolation }); + + expect(run.status).toBe(0); + expect(run.pidsByUri.size).toBe(BENCHMARK_COUNT); + const pids = new Set(run.pidsByUri.values()); + expect(pids.size).toBe(BENCHMARK_COUNT); + // The parent only spawns children, it never measures. + expect(pids).not.toContain(run.parentPid); + }, + ); + + it.each(["0", "false"])( + "measures all benchmarks in one process (isolation=%s)", + (isolation) => { + const run = runFixture({ isolation }); + + expect(run.status).toBe(0); + expect(run.pidsByUri.size).toBe(BENCHMARK_COUNT); + expect(new Set(run.pidsByUri.values())).toEqual(new Set([run.parentPid])); + }, + ); + + it.each(["true", "false"])( + "fails the run when a benchmark throws (isolation=%s)", + (isolation) => { + const run = runFixture({ isolation, failing: true }); + + expect(run.status).not.toBe(0); + expect(run.stderr).toContain("benchmark failed"); + }, + ); + + it("rejects values other than true and false", () => { + const run = runFixture({ isolation: "maybe" }); + + expect(run.status).not.toBe(0); + expect(run.stderr).toContain(`${PROCESS_ISOLATION_ENV} must be`); + expect(run.pidsByUri.size).toBe(0); + }); +}); diff --git a/packages/core/tests/fixtures/isolation.cjs b/packages/core/tests/fixtures/isolation.cjs new file mode 100644 index 00000000..46205d3b --- /dev/null +++ b/packages/core/tests/fixtures/isolation.cjs @@ -0,0 +1,27 @@ +// Run by `isolation.integ.test.ts`. Runs two benchmarks through `Isolation`. +const { Isolation } = require("../../dist/index.cjs.js"); + +const isolation = Isolation.current(); + +function bench(uri) { + console.log(`ran ${process.pid} ${uri} ${JSON.stringify(process.execArgv)}`); + switch (process.env.ISOLATION_FIXTURE_ACTION) { + case "exit": + process.exit(3); + case "kill": + process.kill(process.pid, "SIGKILL"); + } +} + +async function main() { + for (const uri of ["a", "b"]) { + if (!isolation.shouldRun(uri)) continue; + if (process.env.ISOLATION_FIXTURE_SYNC !== undefined) { + isolation.runSync(uri, () => bench(uri)); + } else { + await isolation.run(uri, async () => bench(uri)); + } + } +} + +main(); diff --git a/packages/core/tests/isolation.integ.test.ts b/packages/core/tests/isolation.integ.test.ts new file mode 100644 index 00000000..8649d8c5 --- /dev/null +++ b/packages/core/tests/isolation.integ.test.ts @@ -0,0 +1,162 @@ +import { spawnSync } from "child_process"; +import { constants } from "os"; +import { join } from "path"; +import { + ISOLATED_BENCHMARK_ENV, + Isolation, + PROCESS_ISOLATION_ENV, +} from "../src/isolation"; + +const FIXTURE = join(__dirname, "fixtures", "isolation.cjs"); +/** Fails a run whose benchmark process got stuck instead of blocking the suite. */ +const RUN_TIMEOUT_MS = 60_000; +const SIGNAL_EXIT_CODE_OFFSET = 128; +const CHILD_EXIT_CODE = 3; +const RAN_LINE = /^ran (\d+) (\S+) (.+)$/; +/** Passed to Node to check that children keep the flags of their parent. */ +const NODE_FLAG = "--no-warnings"; + +/** `Isolation.current()` reads the environment once, so load a fresh copy. */ +function currentIsolation(env: Record): Isolation { + const saved = Object.fromEntries( + Object.keys(env).map((key) => [key, process.env[key]]), + ); + setEnv(env); + try { + let isolation: Isolation | undefined; + jest.isolateModules(() => { + // eslint-disable-next-line @typescript-eslint/no-require-imports + isolation = require("../src/isolation").Isolation.current(); + }); + return isolation as unknown as Isolation; + } finally { + setEnv(saved); + } +} + +function setEnv(env: Record): void { + for (const [key, value] of Object.entries(env)) { + if (value === undefined) { + delete process.env[key]; + } else { + process.env[key] = value; + } + } +} + +describe("Isolation.current", () => { + it.each([undefined, "1", "true"])( + "runs one process per benchmark (isolation=%s)", + (value) => { + const isolation = currentIsolation({ + [PROCESS_ISOLATION_ENV]: value, + [ISOLATED_BENCHMARK_ENV]: undefined, + }); + + expect(isolation.mode).toEqual({ kind: "ProcessPerBenchmark" }); + expect(isolation.onlyBenchmark()).toBeUndefined(); + expect(isolation.isChild()).toBe(false); + expect(isolation.shouldRun("any")).toBe(true); + }, + ); + + it.each(["0", "false"])("shares one process (isolation=%s)", (value) => { + const isolation = currentIsolation({ + [PROCESS_ISOLATION_ENV]: value, + [ISOLATED_BENCHMARK_ENV]: undefined, + }); + + expect(isolation.mode).toEqual({ kind: "SharedProcess" }); + expect(isolation.isChild()).toBe(false); + expect(isolation.shouldRun("any")).toBe(true); + }); + + it("restricts a child to its benchmark, whatever the isolation setting", () => { + const isolation = currentIsolation({ + [PROCESS_ISOLATION_ENV]: "false", + [ISOLATED_BENCHMARK_ENV]: "file::bench", + }); + + expect(isolation.mode).toEqual({ + kind: "IsolatedBenchmark", + uri: "file::bench", + }); + expect(isolation.onlyBenchmark()).toBe("file::bench"); + expect(isolation.isChild()).toBe(true); + expect(isolation.shouldRun("file::bench")).toBe(true); + // Exact match only: a prefix is another benchmark. + expect(isolation.shouldRun("file::bench 2")).toBe(false); + expect(isolation.shouldRun("file")).toBe(false); + }); + + it("rejects any other value", () => { + expect(() => + currentIsolation({ + [PROCESS_ISOLATION_ENV]: "maybe", + [ISOLATED_BENCHMARK_ENV]: undefined, + }), + ).toThrow(`${PROCESS_ISOLATION_ENV} must be \`true\` or \`false\``); + }); + + it("never spawns when no instrument measures", () => { + expect(Isolation.shared().mode).toEqual({ kind: "SharedProcess" }); + }); +}); + +interface FixtureOptions { + action?: "exit" | "kill"; + sync?: boolean; +} + +function runFixture({ action, sync }: FixtureOptions = {}) { + const env: NodeJS.ProcessEnv = { + ...process.env, + ISOLATION_FIXTURE_ACTION: action, + }; + delete env[PROCESS_ISOLATION_ENV]; + delete env[ISOLATED_BENCHMARK_ENV]; + if (sync) { + env.ISOLATION_FIXTURE_SYNC = "1"; + } + const result = spawnSync(process.execPath, [NODE_FLAG, FIXTURE], { + env, + encoding: "utf8", + timeout: RUN_TIMEOUT_MS, + }); + const ran = result.stdout + .split("\n") + .map((line) => RAN_LINE.exec(line)) + .filter((match): match is RegExpExecArray => match !== null); + return { status: result.status, ran }; +} + +describe.each([ + { name: "async", sync: false }, + { name: "sync", sync: true }, +])("Isolation child process ($name run)", ({ sync }) => { + it("runs each benchmark in a fresh process with the flags of its parent", () => { + const { status, ran } = runFixture({ sync }); + + expect(status).toBe(0); + expect(ran.map((match) => match[2])).toEqual(["a", "b"]); + expect(new Set(ran.map((match) => match[1])).size).toBe(2); + for (const match of ran) { + expect(JSON.parse(match[3])).toEqual([NODE_FLAG]); + } + }); + + it("exits with the exit code of a failing child", () => { + const { status, ran } = runFixture({ sync, action: "exit" }); + + expect(status).toBe(CHILD_EXIT_CODE); + // The parent stops at the first failure. + expect(ran.map((match) => match[2])).toEqual(["a"]); + }); + + it("exits with 128 + N when a signal N killed the child", () => { + const { status, ran } = runFixture({ sync, action: "kill" }); + + expect(status).toBe(SIGNAL_EXIT_CODE_OFFSET + constants.signals.SIGKILL); + expect(ran.map((match) => match[2])).toEqual(["a"]); + }); +}); diff --git a/packages/tinybench-plugin/tests/fixtures/isolation.mjs b/packages/tinybench-plugin/tests/fixtures/isolation.mjs new file mode 100644 index 00000000..ec2424da --- /dev/null +++ b/packages/tinybench-plugin/tests/fixtures/isolation.mjs @@ -0,0 +1,46 @@ +// Run by `isolation.integ.test.ts`. Prints the PID that measures each benchmark. +import { InstrumentHooks } from "@codspeed/core"; +import { Bench } from "tinybench"; +import { withCodSpeed } from "../../dist/index.es5.js"; + +// There is no instrument hook runtime outside the CodSpeed runner, so pretend +// the hooks are active. The child processes run this file again, so they get +// the same override. +InstrumentHooks.isInstrumented = () => true; + +// The plugin reports every measured benchmark here, with the PID of the +// process that measured it. +const setExecutedBenchmark = InstrumentHooks.setExecutedBenchmark; +InstrumentHooks.setExecutedBenchmark = (pid, uri) => { + setExecutedBenchmark(pid, uri); + console.log(`executed ${pid} ${uri}`); +}; + +// tinybench calls each function once when it is added, so a failing benchmark +// may only throw once the measurement started. +let measuring = false; +const startBenchmark = InstrumentHooks.startBenchmark; +InstrumentHooks.startBenchmark = () => { + measuring = true; + startBenchmark(); +}; + +const bench = withCodSpeed(new Bench({ time: 1, warmupTime: 0 })); + +for (const name of ["first", "second", "third"]) { + bench.add(name, () => {}); +} + +if (process.env.ISOLATION_FIXTURE_FAIL !== undefined) { + bench.add("failing", () => { + if (measuring) { + throw new Error("benchmark failed"); + } + }); +} + +if (process.env.ISOLATION_FIXTURE_SYNC !== undefined) { + bench.runSync(); +} else { + await bench.run(); +} diff --git a/packages/tinybench-plugin/tests/index.integ.test.ts b/packages/tinybench-plugin/tests/index.integ.test.ts index 5e71eeb2..6de91c6e 100644 --- a/packages/tinybench-plugin/tests/index.integ.test.ts +++ b/packages/tinybench-plugin/tests/index.integ.test.ts @@ -41,6 +41,9 @@ vi.mock("@codspeed/core", async (importOriginal) => { beforeEach(() => { process.env.CODSPEED_ENV = "true"; process.env.CODSPEED_RUNNER_MODE = "instrumentation"; + // These tests drive the plugin in this process against a mocked core, so + // there is no benchmark process to spawn. + process.env.CODSPEED_PROCESS_ISOLATION = "false"; vi.clearAllMocks(); }); diff --git a/packages/tinybench-plugin/tests/isolation.integ.test.ts b/packages/tinybench-plugin/tests/isolation.integ.test.ts new file mode 100644 index 00000000..27970175 --- /dev/null +++ b/packages/tinybench-plugin/tests/isolation.integ.test.ts @@ -0,0 +1,122 @@ +import { getV8Flags } from "@codspeed/core"; +import { spawnSync } from "child_process"; +import { fileURLToPath } from "url"; +import { describe, expect, it } from "vitest"; + +const FIXTURE = fileURLToPath( + new URL("./fixtures/isolation.mjs", import.meta.url), +); +const PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION"; +const BENCHMARK_COUNT = 3; +/** Fails a run whose benchmark process got stuck instead of blocking the suite. */ +const RUN_TIMEOUT_MS = 60_000; +const EXECUTED_LINE = /^executed (\d+) (.+)$/; + +interface FixtureRun { + status: number | null; + stderr: string; + /** PID that measured each benchmark, in the order they were measured. */ + pidsByUri: Map; + /** PID of the process the test started. */ + parentPid: number; +} + +interface FixtureOptions { + /** Value of CODSPEED_PROCESS_ISOLATION, unset when undefined. */ + isolation?: string; + failing?: boolean; + sync?: boolean; +} + +function runFixture({ + isolation, + failing, + sync, +}: FixtureOptions = {}): FixtureRun { + const env: NodeJS.ProcessEnv = { + ...process.env, + CODSPEED_ENV: "true", + CODSPEED_RUNNER_MODE: "simulation", + }; + delete env[PROCESS_ISOLATION_ENV]; + if (isolation !== undefined) { + env[PROCESS_ISOLATION_ENV] = isolation; + } + if (failing) { + env.ISOLATION_FIXTURE_FAIL = "1"; + } + if (sync) { + env.ISOLATION_FIXTURE_SYNC = "1"; + } + + // The runner starts Node with these flags, and children must keep them. + const result = spawnSync(process.execPath, [...getV8Flags(), FIXTURE], { + env, + encoding: "utf8", + timeout: RUN_TIMEOUT_MS, + }); + + const pidsByUri = new Map(); + for (const line of result.stdout.split("\n")) { + const match = EXECUTED_LINE.exec(line); + if (match) { + pidsByUri.set(match[2], Number(match[1])); + } + } + return { + status: result.status, + stderr: result.stderr, + pidsByUri, + parentPid: result.pid, + }; +} + +describe.each([ + { name: "async", sync: false }, + { name: "sync", sync: true }, +])("process isolation ($name run)", ({ sync }) => { + it.each([undefined, "1", "true"])( + "measures every benchmark in its own process (isolation=%s)", + (isolation) => { + const run = runFixture({ isolation, sync }); + + expect(run.status).toBe(0); + expect(run.pidsByUri.size).toBe(BENCHMARK_COUNT); + const pids = new Set(run.pidsByUri.values()); + expect(pids.size).toBe(BENCHMARK_COUNT); + // The parent only spawns children, it never measures. + expect(pids).not.toContain(run.parentPid); + }, + ); + + it.each(["0", "false"])( + "measures all benchmarks in one process (isolation=%s)", + (isolation) => { + const run = runFixture({ isolation, sync }); + + expect(run.status).toBe(0); + expect(run.pidsByUri.size).toBe(BENCHMARK_COUNT); + expect(new Set(run.pidsByUri.values())).toEqual(new Set([run.parentPid])); + }, + ); + + it.each(["true", "false"])( + "fails the run when a benchmark throws (isolation=%s)", + (isolation) => { + const run = runFixture({ isolation, failing: true, sync }); + + expect(run.status).not.toBe(0); + expect(run.stderr).toContain("benchmark failed"); + }, + ); +}); + +describe("process isolation setting", () => { + it("rejects values other than true and false", () => { + const run = runFixture({ isolation: "maybe" }); + + expect(run.status).not.toBe(0); + expect(run.stderr).toContain(`${PROCESS_ISOLATION_ENV} must be`); + expect(run.pidsByUri.size).toBe(0); + }); +}); diff --git a/packages/vitest-plugin/src/__tests__/instrumented.test.ts b/packages/vitest-plugin/src/__tests__/instrumented.test.ts index ec1a1fd9..2e7f54c6 100644 --- a/packages/vitest-plugin/src/__tests__/instrumented.test.ts +++ b/packages/vitest-plugin/src/__tests__/instrumented.test.ts @@ -6,6 +6,8 @@ import { getBenchFn, getBenchOptions } from "../legacy/compat"; const coreMocks = vi.hoisted(() => { return { InstrumentHooks: { + // A plain function: the mocks above are reset between tests. + isInstrumented: () => true, startBenchmark: vi.fn(), stopBenchmark: vi.fn(), setExecutedBenchmark: vi.fn(), @@ -21,6 +23,10 @@ const coreMocks = vi.hoisted(() => { global.eval = vi.fn(); +// The runner is driven in this process against a mocked core, so there is no +// benchmark process to spawn. +process.env.CODSPEED_PROCESS_ISOLATION = "false"; + vi.mock("@codspeed/core", async (importOriginal) => { const mod = await importOriginal(); return { ...mod, ...coreMocks }; diff --git a/packages/vitest-plugin/src/__tests__/isolation.test.ts b/packages/vitest-plugin/src/__tests__/isolation.test.ts new file mode 100644 index 00000000..6683ccf1 --- /dev/null +++ b/packages/vitest-plugin/src/__tests__/isolation.test.ts @@ -0,0 +1,113 @@ +import { spawnSync } from "child_process"; +import { mkdtempSync, readFileSync, rmSync } from "fs"; +import { createRequire } from "module"; +import { tmpdir } from "os"; +import { dirname, join } from "path"; +import { fileURLToPath } from "url"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +const FIXTURE_DIR = fileURLToPath( + new URL("../../tests/fixtures/isolation", import.meta.url), +); +const PRELOAD = join(FIXTURE_DIR, "stub.mjs"); +const VITEST_BIN = join( + dirname(createRequire(import.meta.url).resolve("vitest/package.json")), + "vitest.mjs", +); +const PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION"; +const BENCHMARK_COUNT = 3; +/** Fails a run whose benchmark process got stuck instead of blocking the suite. */ +const RUN_TIMEOUT_MS = 120_000; +const EXECUTED_LINE = /^executed (\d+) (.+)$/; + +let logDir: string; +let logFile: string; + +beforeEach(() => { + logDir = mkdtempSync(join(tmpdir(), "codspeed-isolation-")); + logFile = join(logDir, "executed.log"); +}); + +afterEach(() => { + rmSync(logDir, { recursive: true, force: true }); +}); + +interface FixtureOptions { + /** Value of CODSPEED_PROCESS_ISOLATION, unset when undefined. */ + isolation?: string; + failing?: boolean; +} + +/** Runs `vitest bench` on the fixture project and returns what it measured. */ +function runFixture({ isolation, failing }: FixtureOptions = {}) { + const env: NodeJS.ProcessEnv = { + ...process.env, + CODSPEED_ENV: "true", + CODSPEED_RUNNER_MODE: "simulation", + ISOLATION_LOG_FILE: logFile, + NODE_OPTIONS: [process.env.NODE_OPTIONS, `--import ${PRELOAD}`] + .filter(Boolean) + .join(" "), + }; + delete env[PROCESS_ISOLATION_ENV]; + if (isolation !== undefined) { + env[PROCESS_ISOLATION_ENV] = isolation; + } + if (failing) { + env.ISOLATION_FIXTURE_FAIL = "1"; + } + + const result = spawnSync(process.execPath, [VITEST_BIN, "bench", "--run"], { + cwd: FIXTURE_DIR, + env, + encoding: "utf8", + timeout: RUN_TIMEOUT_MS, + }); + + const pidsByUri = new Map(); + for (const line of readFileSync(logFile, "utf8").split("\n")) { + const match = EXECUTED_LINE.exec(line); + if (match) { + pidsByUri.set(match[2], Number(match[1])); + } + } + return { status: result.status, pidsByUri, parentPid: result.pid }; +} + +describe("process isolation", () => { + it.each([undefined, "1", "true"])( + "measures every benchmark in its own process (isolation=%s)", + (isolation) => { + const run = runFixture({ isolation }); + + expect(run.status).toBe(0); + expect(run.pidsByUri.size).toBe(BENCHMARK_COUNT); + const pids = new Set(run.pidsByUri.values()); + expect(pids.size).toBe(BENCHMARK_COUNT); + expect(pids).not.toContain(run.parentPid); + }, + RUN_TIMEOUT_MS, + ); + + it.each(["0", "false"])( + "measures all benchmarks in one process (isolation=%s)", + (isolation) => { + const run = runFixture({ isolation }); + + expect(run.status).toBe(0); + expect(run.pidsByUri.size).toBe(BENCHMARK_COUNT); + expect(new Set(run.pidsByUri.values()).size).toBe(1); + }, + RUN_TIMEOUT_MS, + ); + + it.each(["true", "false"])( + "fails the run when a benchmark throws (isolation=%s)", + (isolation) => { + const run = runFixture({ isolation, failing: true }); + + expect(run.status).not.toBe(0); + }, + RUN_TIMEOUT_MS, + ); +}); diff --git a/packages/vitest-plugin/tests/fixtures/isolation/pids.bench.ts b/packages/vitest-plugin/tests/fixtures/isolation/pids.bench.ts new file mode 100644 index 00000000..c803dcbc --- /dev/null +++ b/packages/vitest-plugin/tests/fixtures/isolation/pids.bench.ts @@ -0,0 +1,19 @@ +import { describe, test } from "vitest"; + +describe("isolation", () => { + test("benchmarks", async ({ bench }) => { + const benchmarks = [ + bench("first", () => {}), + bench("second", () => {}), + bench("third", () => {}), + ]; + if (process.env.ISOLATION_FIXTURE_FAIL !== undefined) { + benchmarks.push( + bench("failing", () => { + throw new Error("benchmark failed"); + }), + ); + } + await bench.compare(...benchmarks); + }); +}); diff --git a/packages/vitest-plugin/tests/fixtures/isolation/stub.mjs b/packages/vitest-plugin/tests/fixtures/isolation/stub.mjs new file mode 100644 index 00000000..bdfd51b6 --- /dev/null +++ b/packages/vitest-plugin/tests/fixtures/isolation/stub.mjs @@ -0,0 +1,16 @@ +// Preloaded with `node --import` into the Vitest process, its workers, and the +// Vitest runs that process isolation starts. +import { InstrumentHooks } from "@codspeed/core"; +import { appendFileSync } from "fs"; + +// There is no instrument hook runtime outside the CodSpeed runner, so pretend +// the hooks are active. +InstrumentHooks.isInstrumented = () => true; + +// Vitest swallows what workers print, so the PID that measures each benchmark +// goes to a file. +const setExecutedBenchmark = InstrumentHooks.setExecutedBenchmark; +InstrumentHooks.setExecutedBenchmark = (pid, uri) => { + setExecutedBenchmark(pid, uri); + appendFileSync(process.env.ISOLATION_LOG_FILE, `executed ${pid} ${uri}\n`); +}; diff --git a/packages/vitest-plugin/tests/fixtures/isolation/vitest.config.ts b/packages/vitest-plugin/tests/fixtures/isolation/vitest.config.ts new file mode 100644 index 00000000..df313ac4 --- /dev/null +++ b/packages/vitest-plugin/tests/fixtures/isolation/vitest.config.ts @@ -0,0 +1,12 @@ +import { defineConfig } from "vitest/config"; +import codspeedPlugin from "../../../dist/index.mjs"; + +export default defineConfig({ + plugins: [codspeedPlugin()], + define: { + __VERSION__: JSON.stringify("1.0.0"), + }, + test: { + include: ["**/*.bench.ts"], + }, +}); From 4bd0611e6514c9040f81fd455673e77c51969218 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 21 Oct 2026 11:55:45 +0200 Subject: [PATCH 06/21] docs: describe process isolation Document the environment contract, the parent and child roles, and the Vitest and test caveats. --- CLAUDE.md | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index 00a76a53..ab1d6183 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -115,4 +115,26 @@ so exercise all three of these when developing or reviewing a plugin change Running these locally outside the CodSpeed runner is expected to log `instrument-hooks: failed to write environment.json` and skip actual measurement writes — the point is to verify the plugin's control flow and output per mode, -not to produce real measurements. \ No newline at end of file +not to produce real measurements. + +## Process isolation + +In instrumentation/simulation/memory runs with active instrument hooks, every +benchmark runs in its own process (`packages/core/src/isolation.ts`), so the +allocator and JIT state of one benchmark can't change the measurement of the +next. Walltime runs and runs outside CodSpeed are not affected. + +- `CODSPEED_PROCESS_ISOLATION`: unset, `1` or `true` turns it on (default), + `0` or `false` runs all benchmarks in one process, any other value is an error. +- The parent process re-runs itself (`process.execPath`, `process.execArgv` and + `process.argv.slice(1)`, so the V8 flags survive) once per benchmark with + `CODSPEED_ISOLATED_BENCHMARK=`. The child runs only that benchmark, whatever + the user filters are, and the parent exits with the child's exit code + (`128 + N` for signal `N`). +- The child prints no banner or summary lines. They come from the parent. +- Vitest measures in forked workers, so the plugin records the command of the + main Vitest process (`CODSPEED_ISOLATION_COMMAND`) and each worker runs that + command, i.e. one Vitest run per benchmark. +- Tests that drive a plugin in-process against a mocked core must set + `CODSPEED_PROCESS_ISOLATION=false`, or the plugin would start the test runner + again for every benchmark. \ No newline at end of file From 3105892b946713e34ba35ec3ed6120ab7eabfdad Mon Sep 17 00:00:00 2001 From: not-matthias Date: Tue, 27 Oct 2026 19:24:35 +0100 Subject: [PATCH 07/21] perf(vitest): restrict isolated children to their benchmark file Avoid transforming and importing unrelated benchmark files in each child. Replace discovery globs before Vitest resolves its configuration, preserving fresh processes and profiling flags. --- docs/introspection.md | 10 ++++++++++ packages/vitest-plugin/src/index.ts | 21 +++++++++++++++++++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/docs/introspection.md b/docs/introspection.md index 9e3fe464..c892dff4 100644 --- a/docs/introspection.md +++ b/docs/introspection.md @@ -13,3 +13,13 @@ ```bash CI=1 CODSPEED_DEBUG=true pnpm --filter with-typescript-esm bench-tinybench ``` + +## Vitest process isolation + +Isolated Vitest children restrict benchmark discovery to the file named in +`CODSPEED_ISOLATED_BENCHMARK`. Other benchmark files are not transformed or imported +in that child. The parent still discovers the full suite, and each child still +loads the Vitest configuration and setup files. + +Walltime forks retain fresh processes and the profiling flags. This avoids +repeated unrelated file imports; it does not remove Vitest's per-process startup. diff --git a/packages/vitest-plugin/src/index.ts b/packages/vitest-plugin/src/index.ts index ae81cf3b..f57c9338 100644 --- a/packages/vitest-plugin/src/index.ts +++ b/packages/vitest-plugin/src/index.ts @@ -1,7 +1,9 @@ import { + getGitDir, getInstrumentMode, getV8Flags, InstrumentHooks, + ISOLATED_BENCHMARK_ENV, mongoMeasurement, recordIsolationCommand, SetupInstrumentsRequestBody, @@ -38,11 +40,26 @@ export default function codspeedPlugin(): Plugin { return true; }, enforce: "post", - config(userConfig): ViteUserConfig { + config(userConfig: ViteUserConfig): ViteUserConfig { // Vitest measures in forked workers, whose argv is not the command that // starts the benchmarks. Record the command of this main process so the // workers can run it again for each benchmark. Workers inherit the env. recordIsolationCommand(); + const uri = process.env[ISOLATED_BENCHMARK_ENV]; + if (uri !== undefined) { + const gitDir = getGitDir(userConfig.root ?? process.cwd()); + if (gitDir === undefined) { + throw new Error("Could not find a git repository"); + } + // Replace rather than merge include globs: isolated children must not + // transform and import unrelated benchmark files. + const file = join(gitDir, uri.split("::")[0]) + .replace(/\\/g, "/") + .replace(/([*?{}()[\]!+@])/g, "\\$1"); + userConfig.test ??= {}; + userConfig.test.benchmark ??= {}; + userConfig.test.benchmark.include = [file]; + } return { test: { pool: "forks", @@ -50,7 +67,7 @@ export default function codspeedPlugin(): Plugin { ...resolveVitestBackend().getBenchmarkTestConfig( getV8Flags(), resolveFile, - (userConfig as ViteUserConfig).test, + userConfig.test, ), }, }; From 8f1c9a21372f72f30f1e9354274ac86f5941c254 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Tue, 27 Oct 2026 19:25:55 +0100 Subject: [PATCH 08/21] docs: describe file filtering for instrumented isolation --- docs/introspection.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/introspection.md b/docs/introspection.md index c892dff4..24258ec3 100644 --- a/docs/introspection.md +++ b/docs/introspection.md @@ -21,5 +21,5 @@ Isolated Vitest children restrict benchmark discovery to the file named in in that child. The parent still discovers the full suite, and each child still loads the Vitest configuration and setup files. -Walltime forks retain fresh processes and the profiling flags. This avoids +Isolated children retain fresh processes and the profiling flags. This avoids repeated unrelated file imports; it does not remove Vitest's per-process startup. From 8816d3e797561ab96fcd89350b895a2a6dd29f0e Mon Sep 17 00:00:00 2001 From: not-matthias Date: Tue, 27 Oct 2026 19:39:47 +0100 Subject: [PATCH 09/21] test: trim redundant process isolation coverage Keep environment alias validation in core instead of repeating it through every plugin. Retain real process, opt-out, failure, signal, and flag propagation checks while dropping getter-only assertions and redundant documentation. --- CLAUDE.md | 24 +------ .../tests/isolation.integ.test.ts | 65 ++++++------------- packages/core/src/isolation.ts | 7 -- packages/core/tests/isolation.integ.test.ts | 65 +++++-------------- .../tests/isolation.integ.test.ts | 65 +++++++------------ .../src/__tests__/isolation.test.ts | 53 +++++---------- 6 files changed, 77 insertions(+), 202 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index ab1d6183..00a76a53 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -115,26 +115,4 @@ so exercise all three of these when developing or reviewing a plugin change Running these locally outside the CodSpeed runner is expected to log `instrument-hooks: failed to write environment.json` and skip actual measurement writes — the point is to verify the plugin's control flow and output per mode, -not to produce real measurements. - -## Process isolation - -In instrumentation/simulation/memory runs with active instrument hooks, every -benchmark runs in its own process (`packages/core/src/isolation.ts`), so the -allocator and JIT state of one benchmark can't change the measurement of the -next. Walltime runs and runs outside CodSpeed are not affected. - -- `CODSPEED_PROCESS_ISOLATION`: unset, `1` or `true` turns it on (default), - `0` or `false` runs all benchmarks in one process, any other value is an error. -- The parent process re-runs itself (`process.execPath`, `process.execArgv` and - `process.argv.slice(1)`, so the V8 flags survive) once per benchmark with - `CODSPEED_ISOLATED_BENCHMARK=`. The child runs only that benchmark, whatever - the user filters are, and the parent exits with the child's exit code - (`128 + N` for signal `N`). -- The child prints no banner or summary lines. They come from the parent. -- Vitest measures in forked workers, so the plugin records the command of the - main Vitest process (`CODSPEED_ISOLATION_COMMAND`) and each worker runs that - command, i.e. one Vitest run per benchmark. -- Tests that drive a plugin in-process against a mocked core must set - `CODSPEED_PROCESS_ISOLATION=false`, or the plugin would start the test runner - again for every benchmark. \ No newline at end of file +not to produce real measurements. \ No newline at end of file diff --git a/packages/benchmark.js-plugin/tests/isolation.integ.test.ts b/packages/benchmark.js-plugin/tests/isolation.integ.test.ts index 0c59e91d..a741c4f4 100644 --- a/packages/benchmark.js-plugin/tests/isolation.integ.test.ts +++ b/packages/benchmark.js-plugin/tests/isolation.integ.test.ts @@ -10,22 +10,14 @@ const BENCHMARK_COUNT = 4; const RUN_TIMEOUT_MS = 60_000; const EXECUTED_LINE = /^executed (\d+) (.+)$/; -interface FixtureRun { - status: number | null; - stderr: string; - /** PID that measured each benchmark. */ - pidsByUri: Map; - /** PID of the process the test started. */ - parentPid: number; -} - interface FixtureOptions { /** Value of CODSPEED_PROCESS_ISOLATION, unset when undefined. */ isolation?: string; failing?: boolean; } -function runFixture({ isolation, failing }: FixtureOptions = {}): FixtureRun { +/** Runs the fixture and returns the PID that measured each benchmark. */ +function runFixture({ isolation, failing }: FixtureOptions = {}) { const env: NodeJS.ProcessEnv = { ...process.env, CODSPEED_ENV: "true", @@ -62,46 +54,29 @@ function runFixture({ isolation, failing }: FixtureOptions = {}): FixtureRun { } describe("process isolation", () => { - it.each([undefined, "1", "true"])( - "measures every benchmark in its own process (isolation=%s)", - (isolation) => { - const run = runFixture({ isolation }); - - expect(run.status).toBe(0); - expect(run.pidsByUri.size).toBe(BENCHMARK_COUNT); - const pids = new Set(run.pidsByUri.values()); - expect(pids.size).toBe(BENCHMARK_COUNT); - // The parent only spawns children, it never measures. - expect(pids).not.toContain(run.parentPid); - }, - ); + it("measures every benchmark in its own process", () => { + const run = runFixture(); - it.each(["0", "false"])( - "measures all benchmarks in one process (isolation=%s)", - (isolation) => { - const run = runFixture({ isolation }); - - expect(run.status).toBe(0); - expect(run.pidsByUri.size).toBe(BENCHMARK_COUNT); - expect(new Set(run.pidsByUri.values())).toEqual(new Set([run.parentPid])); - }, - ); + expect(run.status).toBe(0); + expect(run.pidsByUri.size).toBe(BENCHMARK_COUNT); + const pids = new Set(run.pidsByUri.values()); + expect(pids.size).toBe(BENCHMARK_COUNT); + // The parent only spawns children, it never measures. + expect(pids).not.toContain(run.parentPid); + }); - it.each(["true", "false"])( - "fails the run when a benchmark throws (isolation=%s)", - (isolation) => { - const run = runFixture({ isolation, failing: true }); + it("measures all benchmarks in one process when disabled", () => { + const run = runFixture({ isolation: "false" }); - expect(run.status).not.toBe(0); - expect(run.stderr).toContain("benchmark failed"); - }, - ); + expect(run.status).toBe(0); + expect(run.pidsByUri.size).toBe(BENCHMARK_COUNT); + expect(new Set(run.pidsByUri.values())).toEqual(new Set([run.parentPid])); + }); - it("rejects values other than true and false", () => { - const run = runFixture({ isolation: "maybe" }); + it("fails the run when a benchmark throws", () => { + const run = runFixture({ isolation: "true", failing: true }); expect(run.status).not.toBe(0); - expect(run.stderr).toContain(`${PROCESS_ISOLATION_ENV} must be`); - expect(run.pidsByUri.size).toBe(0); + expect(run.stderr).toContain("benchmark failed"); }); }); diff --git a/packages/core/src/isolation.ts b/packages/core/src/isolation.ts index 0a8e013a..a39667db 100644 --- a/packages/core/src/isolation.ts +++ b/packages/core/src/isolation.ts @@ -22,7 +22,6 @@ const GENERIC_FAILURE_EXIT_CODE = 1; /** Shells report a child killed by signal N as exit code 128 + N. */ const SIGNAL_EXIT_CODE_OFFSET = 128; -/** How an integration runs each benchmark. */ export type IsolationMode = /** All benchmarks share this process and its state. */ | { kind: "SharedProcess" } @@ -47,7 +46,6 @@ export class Isolation { return Isolation.sharedInstance; } - /** The isolation of this process, read once from the environment. */ static current(): Isolation { Isolation.instance ??= new Isolation(modeFromEnv()); return Isolation.instance; @@ -64,13 +62,11 @@ export class Isolation { } } - /** Whether this process must run the benchmark `uri`. */ shouldRun(uri: string): boolean { const only = this.onlyBenchmark(); return only === undefined || only === uri; } - /** Whether this process is a child that runs one benchmark for a parent. */ isChild(): boolean { return this.onlyBenchmark() !== undefined; } @@ -156,8 +152,6 @@ function isolationCommand(): string[] { */ function runInChildProcess(uri: string): void { const [executable, ...args] = isolationCommand(); - // spawnSync blocks the event loop, so stdout written before this call is - // already flushed and output stays in order. const result = spawnSync(executable, args, { stdio: "inherit", env: { ...process.env, [ISOLATED_BENCHMARK_ENV]: uri }, @@ -173,7 +167,6 @@ function runInChildProcess(uri: string): void { process.exit(exitCode(result)); } -/** Exit code of a failed child, in the shell convention. */ function exitCode(result: SpawnSyncReturns): number { if (result.signal !== null) { return SIGNAL_EXIT_CODE_OFFSET + constants.signals[result.signal]; diff --git a/packages/core/tests/isolation.integ.test.ts b/packages/core/tests/isolation.integ.test.ts index 8649d8c5..76f9d821 100644 --- a/packages/core/tests/isolation.integ.test.ts +++ b/packages/core/tests/isolation.integ.test.ts @@ -18,57 +18,34 @@ const NODE_FLAG = "--no-warnings"; /** `Isolation.current()` reads the environment once, so load a fresh copy. */ function currentIsolation(env: Record): Isolation { - const saved = Object.fromEntries( - Object.keys(env).map((key) => [key, process.env[key]]), - ); - setEnv(env); + const saved = process.env; + process.env = { ...saved, ...env }; try { - let isolation: Isolation | undefined; + let isolation: Isolation; jest.isolateModules(() => { // eslint-disable-next-line @typescript-eslint/no-require-imports isolation = require("../src/isolation").Isolation.current(); }); - return isolation as unknown as Isolation; + return isolation!; } finally { - setEnv(saved); - } -} - -function setEnv(env: Record): void { - for (const [key, value] of Object.entries(env)) { - if (value === undefined) { - delete process.env[key]; - } else { - process.env[key] = value; - } + process.env = saved; } } describe("Isolation.current", () => { - it.each([undefined, "1", "true"])( - "runs one process per benchmark (isolation=%s)", - (value) => { - const isolation = currentIsolation({ + it.each([ + [undefined, "ProcessPerBenchmark"], + ["1", "ProcessPerBenchmark"], + ["true", "ProcessPerBenchmark"], + ["0", "SharedProcess"], + ["false", "SharedProcess"], + ])("selects %s as %s", (value, kind) => { + expect( + currentIsolation({ [PROCESS_ISOLATION_ENV]: value, [ISOLATED_BENCHMARK_ENV]: undefined, - }); - - expect(isolation.mode).toEqual({ kind: "ProcessPerBenchmark" }); - expect(isolation.onlyBenchmark()).toBeUndefined(); - expect(isolation.isChild()).toBe(false); - expect(isolation.shouldRun("any")).toBe(true); - }, - ); - - it.each(["0", "false"])("shares one process (isolation=%s)", (value) => { - const isolation = currentIsolation({ - [PROCESS_ISOLATION_ENV]: value, - [ISOLATED_BENCHMARK_ENV]: undefined, - }); - - expect(isolation.mode).toEqual({ kind: "SharedProcess" }); - expect(isolation.isChild()).toBe(false); - expect(isolation.shouldRun("any")).toBe(true); + }).mode.kind, + ).toBe(kind); }); it("restricts a child to its benchmark, whatever the isolation setting", () => { @@ -77,12 +54,6 @@ describe("Isolation.current", () => { [ISOLATED_BENCHMARK_ENV]: "file::bench", }); - expect(isolation.mode).toEqual({ - kind: "IsolatedBenchmark", - uri: "file::bench", - }); - expect(isolation.onlyBenchmark()).toBe("file::bench"); - expect(isolation.isChild()).toBe(true); expect(isolation.shouldRun("file::bench")).toBe(true); // Exact match only: a prefix is another benchmark. expect(isolation.shouldRun("file::bench 2")).toBe(false); @@ -97,10 +68,6 @@ describe("Isolation.current", () => { }), ).toThrow(`${PROCESS_ISOLATION_ENV} must be \`true\` or \`false\``); }); - - it("never spawns when no instrument measures", () => { - expect(Isolation.shared().mode).toEqual({ kind: "SharedProcess" }); - }); }); interface FixtureOptions { diff --git a/packages/tinybench-plugin/tests/isolation.integ.test.ts b/packages/tinybench-plugin/tests/isolation.integ.test.ts index 27970175..7f6d0059 100644 --- a/packages/tinybench-plugin/tests/isolation.integ.test.ts +++ b/packages/tinybench-plugin/tests/isolation.integ.test.ts @@ -22,14 +22,14 @@ interface FixtureRun { } interface FixtureOptions { - /** Value of CODSPEED_PROCESS_ISOLATION, unset when undefined. */ - isolation?: string; + /** Opt out of isolation: every benchmark shares the process we start. */ + shared?: boolean; failing?: boolean; sync?: boolean; } function runFixture({ - isolation, + shared, failing, sync, }: FixtureOptions = {}): FixtureRun { @@ -39,8 +39,8 @@ function runFixture({ CODSPEED_RUNNER_MODE: "simulation", }; delete env[PROCESS_ISOLATION_ENV]; - if (isolation !== undefined) { - env[PROCESS_ISOLATION_ENV] = isolation; + if (shared) { + env[PROCESS_ISOLATION_ENV] = "false"; } if (failing) { env.ISOLATION_FIXTURE_FAIL = "1"; @@ -75,48 +75,29 @@ describe.each([ { name: "async", sync: false }, { name: "sync", sync: true }, ])("process isolation ($name run)", ({ sync }) => { - it.each([undefined, "1", "true"])( - "measures every benchmark in its own process (isolation=%s)", - (isolation) => { - const run = runFixture({ isolation, sync }); + it("measures every benchmark in its own process", () => { + const run = runFixture({ sync }); - expect(run.status).toBe(0); - expect(run.pidsByUri.size).toBe(BENCHMARK_COUNT); - const pids = new Set(run.pidsByUri.values()); - expect(pids.size).toBe(BENCHMARK_COUNT); - // The parent only spawns children, it never measures. - expect(pids).not.toContain(run.parentPid); - }, - ); - - it.each(["0", "false"])( - "measures all benchmarks in one process (isolation=%s)", - (isolation) => { - const run = runFixture({ isolation, sync }); - - expect(run.status).toBe(0); - expect(run.pidsByUri.size).toBe(BENCHMARK_COUNT); - expect(new Set(run.pidsByUri.values())).toEqual(new Set([run.parentPid])); - }, - ); + expect(run.status).toBe(0); + expect(run.pidsByUri.size).toBe(BENCHMARK_COUNT); + const pids = new Set(run.pidsByUri.values()); + expect(pids.size).toBe(BENCHMARK_COUNT); + // The parent only spawns children, it never measures. + expect(pids).not.toContain(run.parentPid); + }); - it.each(["true", "false"])( - "fails the run when a benchmark throws (isolation=%s)", - (isolation) => { - const run = runFixture({ isolation, failing: true, sync }); + it("measures every benchmark in the parent process when disabled", () => { + const run = runFixture({ shared: true, sync }); - expect(run.status).not.toBe(0); - expect(run.stderr).toContain("benchmark failed"); - }, - ); -}); + expect(run.status).toBe(0); + expect(run.pidsByUri.size).toBe(BENCHMARK_COUNT); + expect(new Set(run.pidsByUri.values())).toEqual(new Set([run.parentPid])); + }); -describe("process isolation setting", () => { - it("rejects values other than true and false", () => { - const run = runFixture({ isolation: "maybe" }); + it("fails the run when a benchmark throws", () => { + const run = runFixture({ failing: true, sync }); expect(run.status).not.toBe(0); - expect(run.stderr).toContain(`${PROCESS_ISOLATION_ENV} must be`); - expect(run.pidsByUri.size).toBe(0); + expect(run.stderr).toContain("benchmark failed"); }); }); diff --git a/packages/vitest-plugin/src/__tests__/isolation.test.ts b/packages/vitest-plugin/src/__tests__/isolation.test.ts index 6683ccf1..30275341 100644 --- a/packages/vitest-plugin/src/__tests__/isolation.test.ts +++ b/packages/vitest-plugin/src/__tests__/isolation.test.ts @@ -16,7 +16,6 @@ const VITEST_BIN = join( ); const PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION"; const BENCHMARK_COUNT = 3; -/** Fails a run whose benchmark process got stuck instead of blocking the suite. */ const RUN_TIMEOUT_MS = 120_000; const EXECUTED_LINE = /^executed (\d+) (.+)$/; @@ -32,14 +31,10 @@ afterEach(() => { rmSync(logDir, { recursive: true, force: true }); }); -interface FixtureOptions { - /** Value of CODSPEED_PROCESS_ISOLATION, unset when undefined. */ - isolation?: string; - failing?: boolean; -} - -/** Runs `vitest bench` on the fixture project and returns what it measured. */ -function runFixture({ isolation, failing }: FixtureOptions = {}) { +function runFixture({ + isolation, + failing, +}: { isolation?: string; failing?: boolean } = {}) { const env: NodeJS.ProcessEnv = { ...process.env, CODSPEED_ENV: "true", @@ -49,10 +44,7 @@ function runFixture({ isolation, failing }: FixtureOptions = {}) { .filter(Boolean) .join(" "), }; - delete env[PROCESS_ISOLATION_ENV]; - if (isolation !== undefined) { - env[PROCESS_ISOLATION_ENV] = isolation; - } + env[PROCESS_ISOLATION_ENV] = isolation; if (failing) { env.ISOLATION_FIXTURE_FAIL = "1"; } @@ -75,38 +67,27 @@ function runFixture({ isolation, failing }: FixtureOptions = {}) { } describe("process isolation", () => { - it.each([undefined, "1", "true"])( - "measures every benchmark in its own process (isolation=%s)", - (isolation) => { + it.each([ + { isolation: undefined, processes: BENCHMARK_COUNT }, + { isolation: "false", processes: 1 }, + ])( + "measures benchmarks in $processes processes (isolation=$isolation)", + ({ isolation, processes }) => { const run = runFixture({ isolation }); - expect(run.status).toBe(0); expect(run.pidsByUri.size).toBe(BENCHMARK_COUNT); const pids = new Set(run.pidsByUri.values()); - expect(pids.size).toBe(BENCHMARK_COUNT); + expect(pids.size).toBe(processes); expect(pids).not.toContain(run.parentPid); }, RUN_TIMEOUT_MS, ); - it.each(["0", "false"])( - "measures all benchmarks in one process (isolation=%s)", - (isolation) => { - const run = runFixture({ isolation }); - - expect(run.status).toBe(0); - expect(run.pidsByUri.size).toBe(BENCHMARK_COUNT); - expect(new Set(run.pidsByUri.values()).size).toBe(1); - }, - RUN_TIMEOUT_MS, - ); - - it.each(["true", "false"])( - "fails the run when a benchmark throws (isolation=%s)", - (isolation) => { - const run = runFixture({ isolation, failing: true }); - - expect(run.status).not.toBe(0); + it( + "fails the run when an isolated benchmark throws", + () => { + const run = runFixture({ failing: true }); + expect(run.status).toBe(1); }, RUN_TIMEOUT_MS, ); From 35e1d71b4f5cfe72cda35348d48dcd1e97e0f29d Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 28 Oct 2026 20:51:40 +0100 Subject: [PATCH 10/21] fix(vitest-plugin): leave suite hooks to isolated children The legacy analysis runner ran beforeAll/afterAll in the process that only starts children, so a child setting up the same resources (e.g. a fixed port) failed while the parent still held them. --- packages/vitest-plugin/src/legacy/analysis.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/vitest-plugin/src/legacy/analysis.ts b/packages/vitest-plugin/src/legacy/analysis.ts index 7dc9f7d3..fea4cc6f 100644 --- a/packages/vitest-plugin/src/legacy/analysis.ts +++ b/packages/vitest-plugin/src/legacy/analysis.ts @@ -92,7 +92,13 @@ async function runAnalysisBenchmarkSuite( ? parentSuiteName + "::" + suite.name : suite.name; - await callSuiteHook(suite, suite, "beforeAll"); + // A process that only starts children leaves the suite hooks to them: + // running them here too would hold their resources (e.g. a port) while a + // child sets them up again. + const runsSuiteHooks = isolation.mode.kind !== "ProcessPerBenchmark"; + if (runsSuiteHooks) { + await callSuiteHook(suite, suite, "beforeAll"); + } for (const task of suite.tasks) { if (task.mode !== "run") continue; @@ -115,7 +121,9 @@ async function runAnalysisBenchmarkSuite( } } - await callSuiteHook(suite, suite, "afterAll"); + if (runsSuiteHooks) { + await callSuiteHook(suite, suite, "afterAll"); + } } export class AnalysisRunner extends NodeBenchmarkRunner { From 1d6e63fc14cb5f5955ce6b734fc92ff4fc5c5f08 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 28 Oct 2026 20:51:41 +0100 Subject: [PATCH 11/21] fix(benchmark.js-plugin): call complete listeners in isolated children A child skipped the complete listeners of runs that don't own its benchmark, so a benchmark started from such a listener was never measured. --- packages/benchmark.js-plugin/src/index.ts | 33 ++++++++++--------- .../tests/fixtures/isolation.cjs | 13 ++++++++ .../tests/isolation.integ.test.ts | 4 +-- 3 files changed, 32 insertions(+), 18 deletions(-) diff --git a/packages/benchmark.js-plugin/src/index.ts b/packages/benchmark.js-plugin/src/index.ts index 747b5fc2..49107fea 100644 --- a/packages/benchmark.js-plugin/src/index.ts +++ b/packages/benchmark.js-plugin/src/index.ts @@ -171,17 +171,13 @@ async function runBenchmarks({ }: RunBenchmarksOptions): Promise { const isolation = Isolation.current(); const isChild = isolation.isChild(); - const selected = benches - .map((bench, i) => ({ - bench, - uri: bench.uri ?? `${baseUri}::unknown_${i}`, - })) - .filter(({ uri }) => isolation.shouldRun(uri)); + const all = benches.map((bench, i) => ({ + bench, + uri: bench.uri ?? `${baseUri}::unknown_${i}`, + })); // A child only measures one benchmark. A run call that owns none of them // must not touch the instruments, the call that owns it does. - if (isChild && selected.length === 0) { - return; - } + const measures = all.some(({ uri }) => isolation.shouldRun(uri)); // The parent already printed the banner and the summary, children stay quiet. if (!isChild) { @@ -189,15 +185,20 @@ async function runBenchmarks({ `[CodSpeed] running with @codspeed/benchmark.js v${__VERSION__}`, ); } - setupCore(); - for (const { bench, uri } of selected) { - await isolation.run(uri, () => measureBenchmark(bench, uri)); - // The parent reports for every benchmark, the child ran only one of them. - if (!isChild) { - benchmarkCompletedListeners.forEach((listener) => listener()); + if (measures) { + setupCore(); + } + for (const { bench, uri } of all) { + if (isolation.shouldRun(uri)) { + await isolation.run(uri, () => measureBenchmark(bench, uri)); } + // Also in children, for benchmarks they skip: a `complete` listener may + // start the run that owns the child's benchmark. + benchmarkCompletedListeners.forEach((listener) => listener()); + } + if (measures) { + teardownCore(); } - teardownCore(); if (!isChild) { console.log(`[CodSpeed] Done running ${benches.length} benches.`); } diff --git a/packages/benchmark.js-plugin/tests/fixtures/isolation.cjs b/packages/benchmark.js-plugin/tests/fixtures/isolation.cjs index 1ab23728..b1d51e57 100644 --- a/packages/benchmark.js-plugin/tests/fixtures/isolation.cjs +++ b/packages/benchmark.js-plugin/tests/fixtures/isolation.cjs @@ -33,5 +33,18 @@ if (process.env.ISOLATION_FIXTURE_FAIL !== undefined) { }); } +// Started from a `complete` listener of the first suite: a child measuring +// `chained` only reaches it through that listener. +const chained = withCodSpeed(new Benchmark.Suite("chained")); +chained.add("chained", () => {}); +let completed = 0; +suite.on("complete", () => { + completed++; + if (completed === suite.length) { + // After the first run tears down, so the two runs don't overlap. + setImmediate(() => chained.run()); + } +}); + // An unhandled rejection ends the process with exit code 1. suite.run(); diff --git a/packages/benchmark.js-plugin/tests/isolation.integ.test.ts b/packages/benchmark.js-plugin/tests/isolation.integ.test.ts index a741c4f4..c3d1a9b1 100644 --- a/packages/benchmark.js-plugin/tests/isolation.integ.test.ts +++ b/packages/benchmark.js-plugin/tests/isolation.integ.test.ts @@ -4,8 +4,8 @@ import { join } from "path"; const FIXTURE = join(__dirname, "fixtures", "isolation.cjs"); const PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION"; -/** `first`, `second`, `third` and the deferred `async` benchmark. */ -const BENCHMARK_COUNT = 4; +/** `first`, `second`, `third`, the deferred `async`, and `chained`. */ +const BENCHMARK_COUNT = 5; /** Fails a run whose benchmark process got stuck instead of blocking the suite. */ const RUN_TIMEOUT_MS = 60_000; const EXECUTED_LINE = /^executed (\d+) (.+)$/; From bb5d79f99b39ce7155d1eeecbbdc6c22e2e8517a Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 28 Oct 2026 21:25:15 +0100 Subject: [PATCH 12/21] fix(core): keep the event loop free while an isolated child runs Isolation.run waited for the child with spawnSync. In a Vitest worker that blocked the worker's RPC with its main process for the whole child run, and under the simulation instrument the calls timed out (Timeout calling "onTaskUpdate"), failing the run. --- packages/core/src/isolation.ts | 59 ++++++++++++++++++++++++++-------- 1 file changed, 45 insertions(+), 14 deletions(-) diff --git a/packages/core/src/isolation.ts b/packages/core/src/isolation.ts index a39667db..4ed58f15 100644 --- a/packages/core/src/isolation.ts +++ b/packages/core/src/isolation.ts @@ -3,7 +3,7 @@ * process, so the allocator state, JIT state, and handles that other * benchmarks or setup code leave behind can't change its measurement. */ -import { spawnSync, type SpawnSyncReturns } from "child_process"; +import { spawn, spawnSync } from "child_process"; import { constants } from "os"; /** Isolation is on unless this is `0` or `false`. */ @@ -82,7 +82,7 @@ export class Isolation { await bench(); return; case "ProcessPerBenchmark": - runInChildProcess(uri); + await runInChildProcessAsync(uri); return; } } @@ -154,22 +154,53 @@ function runInChildProcess(uri: string): void { const [executable, ...args] = isolationCommand(); const result = spawnSync(executable, args, { stdio: "inherit", - env: { ...process.env, [ISOLATED_BENCHMARK_ENV]: uri }, + env: childEnv(uri), }); if (result.error !== undefined) { - throw new Error( - `[CodSpeed] failed to start the process for ${uri}: ${result.error.message}`, - ); - } - if (result.status === 0) { - return; + throw startError(uri, result.error); } - process.exit(exitCode(result)); + exitOnFailure(result.status, result.signal); +} + +/** + * {@link runInChildProcess} without blocking the event loop while the child + * runs: a Vitest worker must keep answering its main process, whose calls + * time out otherwise. + */ +async function runInChildProcessAsync(uri: string): Promise { + const [executable, ...args] = isolationCommand(); + const child = spawn(executable, args, { + stdio: "inherit", + env: childEnv(uri), + }); + const { promise, resolve, reject } = + Promise.withResolvers<[number | null, NodeJS.Signals | null]>(); + child.once("error", (error) => reject(startError(uri, error))); + child.once("close", (code, signal) => resolve([code, signal])); + const [status, signal] = await promise; + exitOnFailure(status, signal); +} + +function childEnv(uri: string): NodeJS.ProcessEnv { + return { ...process.env, [ISOLATED_BENCHMARK_ENV]: uri }; } -function exitCode(result: SpawnSyncReturns): number { - if (result.signal !== null) { - return SIGNAL_EXIT_CODE_OFFSET + constants.signals[result.signal]; +function startError(uri: string, error: Error): Error { + return new Error( + `[CodSpeed] failed to start the process for ${uri}: ${error.message}`, + ); +} + +/** Exits with a failed child's code, in the shell convention. */ +function exitOnFailure( + status: number | null, + signal: NodeJS.Signals | null, +): void { + if (status === 0) { + return; + } + if (signal !== null) { + process.exit(SIGNAL_EXIT_CODE_OFFSET + constants.signals[signal]); } - return result.status ?? GENERIC_FAILURE_EXIT_CODE; + process.exit(status ?? GENERIC_FAILURE_EXIT_CODE); } From 860d65c0a033639174bd67b0d779e1fc83a381bc Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 28 Oct 2026 21:28:33 +0100 Subject: [PATCH 13/21] fix(vitest-plugin): escape backslashes in the isolated benchmark glob Separators are already normalized to slashes; escaping every glob metacharacter, backslash included, makes the escaping complete. --- packages/vitest-plugin/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/vitest-plugin/src/index.ts b/packages/vitest-plugin/src/index.ts index f57c9338..72f48223 100644 --- a/packages/vitest-plugin/src/index.ts +++ b/packages/vitest-plugin/src/index.ts @@ -55,7 +55,7 @@ export default function codspeedPlugin(): Plugin { // transform and import unrelated benchmark files. const file = join(gitDir, uri.split("::")[0]) .replace(/\\/g, "/") - .replace(/([*?{}()[\]!+@])/g, "\\$1"); + .replace(/[\\*?{}()[\]!+@]/g, "\\$&"); userConfig.test ??= {}; userConfig.test.benchmark ??= {}; userConfig.test.benchmark.include = [file]; From 8cf269aefb7a318141490ff7b88207f8e7cfe781 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 23 Oct 2026 12:23:20 +0200 Subject: [PATCH 14/21] fix(vitest-plugin): prefix walltime URIs of top-level benches with the file path The walltime runner only registered URI prefixes for nested suites, so a bench declared outside a `describe` block reported `::` to the runner while its results used `::`. The profile and results URIs then no longer matched and the benchmark had no flamegraph. Register the root file suite too, and add a top-level bench to the Vitest 3 and 4 examples. Refs COD-3984 --- .../with-vitest-v3/src/fibonacci.bench.ts | 4 + .../with-vitest-v4/src/fibonacci.bench.ts | 4 + .../src/__tests__/walltime.test.ts | 114 ++++++++++++++++++ packages/vitest-plugin/src/legacy/walltime.ts | 2 +- 4 files changed, 123 insertions(+), 1 deletion(-) create mode 100644 packages/vitest-plugin/src/__tests__/walltime.test.ts diff --git a/examples/with-vitest-v3/src/fibonacci.bench.ts b/examples/with-vitest-v3/src/fibonacci.bench.ts index 227a6e67..1349a1ad 100644 --- a/examples/with-vitest-v3/src/fibonacci.bench.ts +++ b/examples/with-vitest-v3/src/fibonacci.bench.ts @@ -18,3 +18,7 @@ describe("fibonacci", () => { iterativeFibonacci(20); }); }); + +bench("top-level recursive fibo 15", () => { + recursiveFibonacci(15); +}); diff --git a/examples/with-vitest-v4/src/fibonacci.bench.ts b/examples/with-vitest-v4/src/fibonacci.bench.ts index 227a6e67..1349a1ad 100644 --- a/examples/with-vitest-v4/src/fibonacci.bench.ts +++ b/examples/with-vitest-v4/src/fibonacci.bench.ts @@ -18,3 +18,7 @@ describe("fibonacci", () => { iterativeFibonacci(20); }); }); + +bench("top-level recursive fibo 15", () => { + recursiveFibonacci(15); +}); diff --git a/packages/vitest-plugin/src/__tests__/walltime.test.ts b/packages/vitest-plugin/src/__tests__/walltime.test.ts new file mode 100644 index 00000000..da60f241 --- /dev/null +++ b/packages/vitest-plugin/src/__tests__/walltime.test.ts @@ -0,0 +1,114 @@ +import { fromPartial } from "@total-typescript/shoehorn"; +import { describe, expect, it, vi, type RunnerTestSuite } from "vitest"; +import { WalltimeRunner } from "../legacy/walltime"; + +const coreMocks = vi.hoisted(() => { + return { + InstrumentHooks: { + startBenchmark: vi.fn(), + stopBenchmark: vi.fn(), + setExecutedBenchmark: vi.fn(), + addMarker: vi.fn(), + currentTimestamp: vi.fn(() => 0n), + }, + setupCore: vi.fn(), + }; +}); + +vi.mock("@codspeed/core", async (importOriginal) => { + const mod = await importOriginal(); + return { ...mod, ...coreMocks }; +}); + +console.warn = vi.fn(); + +// Fully mocked: it loads the Vitest 3/4 runner, absent from the Vitest 5 devDependency. +vi.mock("../legacy/compat", () => { + interface SuiteLike { + id: string; + type: string; + name: string; + tasks: SuiteLike[]; + } + + // Mirrors Vitest's benchmark suite execution: sub-suites run first, then + // the suite's own benches run after a `suite-prepare` event for the suite. + class NodeBenchmarkRunner { + async importTinybench() { + return import("tinybench"); + } + + async runSuite(suite: SuiteLike): Promise { + await this.runBenchmarkSuite(suite); + } + + private async runBenchmarkSuite(suite: SuiteLike): Promise { + for (const task of suite.tasks) { + if (task.type === "suite") { + await this.runBenchmarkSuite(task); + } + } + + const benches = suite.tasks.filter((task) => task.type === "test"); + if (benches.length === 0) { + return; + } + + await (this as unknown as WalltimeRunner).onTaskUpdate( + [], + [[suite.id, "suite-prepare", undefined]], + ); + const { Bench } = await this.importTinybench(); + for (const task of benches) { + const bench = new Bench({ + iterations: 1, + time: 0, + warmupIterations: 0, + warmupTime: 0, + }); + bench.add(task.name, () => {}); + await bench.run(); + } + } + } + + return { + NodeBenchmarkRunner, + getBenchOptions: vi.fn(), + }; +}); + +const FILE_URI = "packages/vitest-plugin/src/__tests__/walltime.test.ts"; + +describe("WalltimeRunner", () => { + it("prefixes top-level and nested bench URIs with the file path", async () => { + const runner = new WalltimeRunner(fromPartial({})); + const suite = fromPartial({ + id: "file", + type: "suite", + file: { filepath: __filename }, + name: "walltime.test.ts", + tasks: [ + { + id: "nested", + type: "suite", + name: "nested suite", + tasks: [ + { type: "test", name: "nested bench", meta: { benchmark: true } }, + ], + }, + { type: "test", name: "top-level bench", meta: { benchmark: true } }, + ], + }); + + await runner.runSuite(suite); + + const uris = coreMocks.InstrumentHooks.setExecutedBenchmark.mock.calls.map( + ([, uri]) => uri, + ); + expect(uris).toEqual([ + `${FILE_URI}::nested suite::nested bench`, + `${FILE_URI}::top-level bench`, + ]); + }); +}); diff --git a/packages/vitest-plugin/src/legacy/walltime.ts b/packages/vitest-plugin/src/legacy/walltime.ts index 09271df8..8a4f0d7e 100644 --- a/packages/vitest-plugin/src/legacy/walltime.ts +++ b/packages/vitest-plugin/src/legacy/walltime.ts @@ -51,10 +51,10 @@ export class WalltimeRunner extends NodeBenchmarkRunner { private populateBenchmarkUris(suite: RunnerTestSuite, parentPath = ""): void { const currentPath = parentPath !== "" ? `${parentPath}::${suite.name}` : suite.name; + this.suiteUris.set(suite.id, currentPath); for (const task of suite.tasks) { if (task.type === "suite") { - this.suiteUris.set(task.id, `${currentPath}::${task.name}`); this.populateBenchmarkUris(task, currentPath); } } From 416347fe204a9a495a272f756b3b6f66f1a7b752 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 23 Oct 2026 14:30:22 +0200 Subject: [PATCH 15/21] feat(vitest-plugin): add walltime forks and GC between rounds Walltime rounds vary when a major GC lands in some of them, and a single process can carry JIT and allocator state that a fresh one would not. Two opt-in settings, modeled on JMH, target this: - CODSPEED_WALLTIME_GC_BETWEEN_ROUNDS=true adds --expose-gc and runs a full GC in tinybench's beforeEach hook, outside the timed call (Vitest 3, 4 and 5). - CODSPEED_WALLTIME_FORKS= runs each benchmark in N fresh processes and reports all their rounds as one result (Vitest 3 and 4). Forks re-run the Vitest command restricted to one URI, like process isolation, and hand their samples to the parent through a file. The legacy runner now patches tinybench's Task prototype once, so new runner instances no longer stack root-frame wrappers. --- CLAUDE.md | 18 +- packages/core/src/introspection.ts | 4 + packages/core/src/isolation.ts | 72 +++++- packages/core/src/walltime/index.ts | 1 + packages/core/src/walltime/options.ts | 54 ++++ packages/core/tests/fixtures/forks.cjs | 21 ++ packages/core/tests/isolation.integ.test.ts | 83 +++++++ .../core/tests/walltimeOptions.integ.test.ts | 86 +++++++ .../src/__tests__/v5-walltime.test.ts | 104 ++++++++ .../src/__tests__/walltime.test.ts | 195 +++++++++++++-- packages/vitest-plugin/src/instrument.ts | 24 ++ packages/vitest-plugin/src/legacy/walltime.ts | 231 +++++++++++++++--- packages/vitest-plugin/src/v5/provider.ts | 22 +- 13 files changed, 861 insertions(+), 54 deletions(-) create mode 100644 packages/core/src/walltime/options.ts create mode 100644 packages/core/tests/fixtures/forks.cjs create mode 100644 packages/core/tests/walltimeOptions.integ.test.ts create mode 100644 packages/vitest-plugin/src/__tests__/v5-walltime.test.ts diff --git a/CLAUDE.md b/CLAUDE.md index 00a76a53..70c10e3f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -115,4 +115,20 @@ so exercise all three of these when developing or reviewing a plugin change Running these locally outside the CodSpeed runner is expected to log `instrument-hooks: failed to write environment.json` and skip actual measurement writes — the point is to verify the plugin's control flow and output per mode, -not to produce real measurements. \ No newline at end of file +not to produce real measurements. + +## Walltime forks and GC between rounds + +Both are opt-in and only apply to walltime runs. + +- `CODSPEED_WALLTIME_FORKS=` runs each benchmark in `N` fresh processes, one + after the other, and reports all their rounds together as one result. Unset + runs the benchmark in the benchmark process. Vitest 3/4 only. Like isolation, + each child re-runs the Vitest command with `CODSPEED_ISOLATED_BENCHMARK=`; + it writes its round samples to the file named by `CODSPEED_FORK_SAMPLES_FILE` + instead of reporting, and the parent merges them. +- `CODSPEED_WALLTIME_GC_BETWEEN_ROUNDS=true` runs a full GC before every round + (warmup and measured). It adds `--expose-gc` to the V8 flags and collects in + the tinybench `beforeEach` hook, outside the timed call. Vitest 3/4/5. Meant + for benchmarks with long rounds: a GC costs milliseconds, so time-based + benchmarks of microsecond functions take far longer to finish. diff --git a/packages/core/src/introspection.ts b/packages/core/src/introspection.ts index f4ac29ef..f632b7f0 100644 --- a/packages/core/src/introspection.ts +++ b/packages/core/src/introspection.ts @@ -2,6 +2,7 @@ import { writeFileSync } from "fs"; import path from "path"; import { getInstrumentMode } from "./runnerMode"; +import { isGcBetweenRoundsEnabled } from "./walltime/options"; const CUSTOM_INTROSPECTION_EXIT_CODE = 0; @@ -49,6 +50,9 @@ export const getV8Flags = () => { case "walltime": { // Emit the V8 jitdump flags.push("--perf-prof"); + if (isGcBetweenRoundsEnabled()) { + flags.push("--expose-gc"); + } const v8LogDir = process.env.CODSPEED_V8_LOG; if (v8LogDir) { diff --git a/packages/core/src/isolation.ts b/packages/core/src/isolation.ts index 4ed58f15..4e563e31 100644 --- a/packages/core/src/isolation.ts +++ b/packages/core/src/isolation.ts @@ -4,7 +4,15 @@ * benchmarks or setup code leave behind can't change its measurement. */ import { spawn, spawnSync } from "child_process"; -import { constants } from "os"; +import { + existsSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "fs"; +import { constants, tmpdir } from "os"; +import { join } from "path"; /** Isolation is on unless this is `0` or `false`. */ export const PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION"; @@ -16,6 +24,8 @@ export const ISOLATED_BENCHMARK_ENV = "CODSPEED_ISOLATED_BENCHMARK"; * from the main process, because re-executing the worker alone can't work. */ export const ISOLATION_COMMAND_ENV = "CODSPEED_ISOLATION_COMMAND"; +/** Set on each fork of {@link runForks}: where the fork writes its samples. */ +export const FORK_SAMPLES_FILE_ENV = "CODSPEED_FORK_SAMPLES_FILE"; /** Exit code for a child that ended with neither an exit code nor a signal. */ const GENERIC_FAILURE_EXIT_CODE = 1; @@ -145,16 +155,65 @@ function isolationCommand(): string[] { return JSON.parse(recorded) as string[]; } +/** + * Runs the benchmark `uri` in `forks` fresh processes, one after the other, + * and returns the round samples of all of them, in milliseconds. Each fork + * measures the benchmark like an isolated child and writes its samples to the + * file named by {@link FORK_SAMPLES_FILE_ENV} instead of reporting results. + */ +export function runForks(uri: string, forks: number): number[] { + const dir = mkdtempSync(join(tmpdir(), "codspeed-forks-")); + try { + let samples: number[] = []; + for (let fork = 0; fork < forks; fork++) { + const file = join(dir, `${fork}.json`); + runInChildProcess(uri, { [FORK_SAMPLES_FILE_ENV]: file }); + if (!existsSync(file)) { + throw new Error( + `[CodSpeed] fork ${fork + 1}/${forks} of ${uri} did not run the benchmark`, + ); + } + const forkSamples = ( + JSON.parse(readFileSync(file, "utf-8")) as ForkSamples + ).samples; + // Not push(...): spreading a time-based run's samples overflows the stack. + samples = samples.concat(forkSamples); + } + return samples; + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +/** Whether this process is a fork started by {@link runForks}. */ +export function isFork(): boolean { + return process.env[FORK_SAMPLES_FILE_ENV] !== undefined; +} + +/** Hands the round samples (milliseconds) of this fork to its parent. */ +export function writeForkSamples(samples: number[]): void { + const file = process.env[FORK_SAMPLES_FILE_ENV]; + if (file === undefined) { + throw new Error(`[CodSpeed] ${FORK_SAMPLES_FILE_ENV} is not set`); + } + const data: ForkSamples = { samples }; + writeFileSync(file, JSON.stringify(data)); +} + +interface ForkSamples { + samples: number[]; +} + /** * Runs the benchmark command again, limited to `uri`, and exits with the * child's code if it fails. Re-exec, not fork(): Node can't fork, and a fork * would only copy the calling thread. */ -function runInChildProcess(uri: string): void { +function runInChildProcess(uri: string, env: NodeJS.ProcessEnv = {}): void { const [executable, ...args] = isolationCommand(); const result = spawnSync(executable, args, { stdio: "inherit", - env: childEnv(uri), + env: childEnv(uri, env), }); if (result.error !== undefined) { throw startError(uri, result.error); @@ -181,8 +240,11 @@ async function runInChildProcessAsync(uri: string): Promise { exitOnFailure(status, signal); } -function childEnv(uri: string): NodeJS.ProcessEnv { - return { ...process.env, [ISOLATED_BENCHMARK_ENV]: uri }; +function childEnv( + uri: string, + env: NodeJS.ProcessEnv = {}, +): NodeJS.ProcessEnv { + return { ...process.env, ...env, [ISOLATED_BENCHMARK_ENV]: uri }; } function startError(uri: string, error: Error): Error { diff --git a/packages/core/src/walltime/index.ts b/packages/core/src/walltime/index.ts index b2d86059..b2a8b0f4 100644 --- a/packages/core/src/walltime/index.ts +++ b/packages/core/src/walltime/index.ts @@ -60,5 +60,6 @@ export function writeWalltimeResults( } export * from "./interfaces"; +export * from "./options"; export * from "./quantiles"; export * from "./utils"; diff --git a/packages/core/src/walltime/options.ts b/packages/core/src/walltime/options.ts new file mode 100644 index 00000000..3ee359ff --- /dev/null +++ b/packages/core/src/walltime/options.ts @@ -0,0 +1,54 @@ +/** + * Number of fresh processes that each walltime benchmark runs in, whose rounds + * are reported together. Unset: the benchmark runs in the benchmark process. + */ +export const WALLTIME_FORKS_ENV = "CODSPEED_WALLTIME_FORKS"; +/** When `true`, a full GC runs before every walltime round, outside its timing. */ +export const WALLTIME_GC_BETWEEN_ROUNDS_ENV = + "CODSPEED_WALLTIME_GC_BETWEEN_ROUNDS"; + +/** The number of forks per walltime benchmark, or `undefined` for none. */ +export function getWalltimeForks(): number | undefined { + const value = process.env[WALLTIME_FORKS_ENV]; + if (value === undefined || value === "") { + return undefined; + } + const forks = Number(value); + if (!Number.isInteger(forks) || forks < 1) { + throw new Error( + `${WALLTIME_FORKS_ENV} must be a positive integer, got \`${value}\``, + ); + } + return forks; +} + +export function isGcBetweenRoundsEnabled(): boolean { + const value = process.env[WALLTIME_GC_BETWEEN_ROUNDS_ENV]; + switch (value) { + case undefined: + case "": + case "0": + case "false": + return false; + case "1": + case "true": + return true; + default: + throw new Error( + `${WALLTIME_GC_BETWEEN_ROUNDS_ENV} must be \`true\` or \`false\`, got \`${value}\``, + ); + } +} + +/** + * Runs a full GC. Only valid when GC between rounds is enabled, which adds + * `--expose-gc` to the V8 flags. + */ +export function collectGarbage(): void { + if (global.gc === undefined) { + throw new Error( + `[CodSpeed] ${WALLTIME_GC_BETWEEN_ROUNDS_ENV} needs the --expose-gc V8 flag`, + ); + } + global.gc(); +} diff --git a/packages/core/tests/fixtures/forks.cjs b/packages/core/tests/fixtures/forks.cjs new file mode 100644 index 00000000..c5601c65 --- /dev/null +++ b/packages/core/tests/fixtures/forks.cjs @@ -0,0 +1,21 @@ +// Run by `isolation.integ.test.ts`. Runs one benchmark through `runForks`. +const { + isFork, + runForks, + writeForkSamples, +} = require("../../dist/index.cjs.js"); + +if (isFork()) { + console.log(`fork ${process.pid}`); + switch (process.env.FORKS_FIXTURE_ACTION) { + case "exit": + process.exit(3); + case "no-samples": + process.exit(0); + } + const count = Number(process.env.FORKS_FIXTURE_SAMPLES); + writeForkSamples(Array.from({ length: count }, (_, i) => i)); +} else { + const samples = runForks("a", Number(process.env.FORKS_FIXTURE_COUNT)); + console.log(`samples ${samples.length}`); +} diff --git a/packages/core/tests/isolation.integ.test.ts b/packages/core/tests/isolation.integ.test.ts index 76f9d821..78650ad9 100644 --- a/packages/core/tests/isolation.integ.test.ts +++ b/packages/core/tests/isolation.integ.test.ts @@ -2,6 +2,7 @@ import { spawnSync } from "child_process"; import { constants } from "os"; import { join } from "path"; import { + FORK_SAMPLES_FILE_ENV, ISOLATED_BENCHMARK_ENV, Isolation, PROCESS_ISOLATION_ENV, @@ -127,3 +128,85 @@ describe.each([ expect(ran.map((match) => match[2])).toEqual(["a"]); }); }); + +const FORKS_FIXTURE = join(__dirname, "fixtures", "forks.cjs"); +const FORK_LINE = /^fork (\d+)$/; +const SAMPLES_LINE = /^samples (\d+)$/; +const SAMPLES_PER_FORK = 2; +/** More samples than fit in the arguments of one call. */ +const MANY_SAMPLES_PER_FORK = 300_000; + +function runForksFixture( + count: number, + action?: "exit" | "no-samples", + samplesPerFork = SAMPLES_PER_FORK, +) { + const env: NodeJS.ProcessEnv = { + ...process.env, + FORKS_FIXTURE_COUNT: String(count), + FORKS_FIXTURE_ACTION: action, + FORKS_FIXTURE_SAMPLES: String(samplesPerFork), + }; + delete env[PROCESS_ISOLATION_ENV]; + delete env[ISOLATED_BENCHMARK_ENV]; + delete env[FORK_SAMPLES_FILE_ENV]; + const result = spawnSync(process.execPath, [FORKS_FIXTURE], { + env, + encoding: "utf8", + timeout: RUN_TIMEOUT_MS, + }); + const lines = result.stdout.split("\n"); + const pids = lines + .map((line) => FORK_LINE.exec(line)) + .filter((match): match is RegExpExecArray => match !== null) + .map((match) => match[1]); + const samplesMatch = lines + .map((line) => SAMPLES_LINE.exec(line)) + .find((match) => match !== null); + return { + status: result.status, + stderr: result.stderr, + pids, + sampleCount: samplesMatch ? Number(samplesMatch[1]) : undefined, + }; +} + +describe("runForks", () => { + it("runs each fork in a fresh process and merges their samples", () => { + const forks = 3; + const { status, pids, sampleCount } = runForksFixture(forks); + + expect(status).toBe(0); + expect(new Set(pids).size).toBe(forks); + expect(sampleCount).toBe(forks * SAMPLES_PER_FORK); + }); + + it("merges forks with many samples each", () => { + const forks = 2; + const { status, sampleCount } = runForksFixture( + forks, + undefined, + MANY_SAMPLES_PER_FORK, + ); + + expect(status).toBe(0); + expect(sampleCount).toBe(forks * MANY_SAMPLES_PER_FORK); + }); + + it("fails when a fork wrote no samples", () => { + const { status, stderr, sampleCount } = runForksFixture(2, "no-samples"); + + expect(status).not.toBe(0); + expect(stderr).toContain("did not run the benchmark"); + expect(sampleCount).toBeUndefined(); + }); + + it("exits with the exit code of a failing fork", () => { + const { status, pids, sampleCount } = runForksFixture(2, "exit"); + + expect(status).toBe(CHILD_EXIT_CODE); + // The parent stops at the first failure. + expect(pids).toHaveLength(1); + expect(sampleCount).toBeUndefined(); + }); +}); diff --git a/packages/core/tests/walltimeOptions.integ.test.ts b/packages/core/tests/walltimeOptions.integ.test.ts new file mode 100644 index 00000000..dc11e288 --- /dev/null +++ b/packages/core/tests/walltimeOptions.integ.test.ts @@ -0,0 +1,86 @@ +import { getV8Flags } from "../src/introspection"; +import { + WALLTIME_FORKS_ENV, + WALLTIME_GC_BETWEEN_ROUNDS_ENV, + getWalltimeForks, + isGcBetweenRoundsEnabled, +} from "../src/walltime/options"; + +const EXPOSE_GC_FLAG = "--expose-gc"; +const ENV_KEYS = [ + WALLTIME_FORKS_ENV, + WALLTIME_GC_BETWEEN_ROUNDS_ENV, + "CODSPEED_ENV", + "CODSPEED_RUNNER_MODE", +]; + +const savedEnv = Object.fromEntries( + ENV_KEYS.map((key) => [key, process.env[key]]), +); + +function setEnv(key: string, value: string | undefined): void { + if (value === undefined) { + delete process.env[key]; + } else { + process.env[key] = value; + } +} + +afterEach(() => { + for (const [key, value] of Object.entries(savedEnv)) { + setEnv(key, value); + } +}); + +describe("getWalltimeForks", () => { + it.each([undefined, ""])("returns undefined when unset (%p)", (value) => { + setEnv(WALLTIME_FORKS_ENV, value); + expect(getWalltimeForks()).toBeUndefined(); + }); + + it("returns the number of forks", () => { + setEnv(WALLTIME_FORKS_ENV, "3"); + expect(getWalltimeForks()).toBe(3); + }); + + it.each(["0", "-1", "1.5", "abc"])("rejects %p", (value) => { + setEnv(WALLTIME_FORKS_ENV, value); + expect(() => getWalltimeForks()).toThrow(WALLTIME_FORKS_ENV); + }); +}); + +describe("isGcBetweenRoundsEnabled", () => { + it.each([undefined, "0", "false"])("is disabled for %p", (value) => { + setEnv(WALLTIME_GC_BETWEEN_ROUNDS_ENV, value); + expect(isGcBetweenRoundsEnabled()).toBe(false); + }); + + it.each(["1", "true"])("is enabled for %p", (value) => { + setEnv(WALLTIME_GC_BETWEEN_ROUNDS_ENV, value); + expect(isGcBetweenRoundsEnabled()).toBe(true); + }); + + it("rejects any other value", () => { + setEnv(WALLTIME_GC_BETWEEN_ROUNDS_ENV, "maybe"); + expect(() => isGcBetweenRoundsEnabled()).toThrow( + WALLTIME_GC_BETWEEN_ROUNDS_ENV, + ); + }); +}); + +describe("walltime V8 flags", () => { + beforeEach(() => { + setEnv("CODSPEED_ENV", "1"); + setEnv("CODSPEED_RUNNER_MODE", "walltime"); + }); + + it("exposes the GC when GC between rounds is enabled", () => { + setEnv(WALLTIME_GC_BETWEEN_ROUNDS_ENV, "true"); + expect(getV8Flags()).toContain(EXPOSE_GC_FLAG); + }); + + it("does not expose the GC otherwise", () => { + setEnv(WALLTIME_GC_BETWEEN_ROUNDS_ENV, undefined); + expect(getV8Flags()).not.toContain(EXPOSE_GC_FLAG); + }); +}); diff --git a/packages/vitest-plugin/src/__tests__/v5-walltime.test.ts b/packages/vitest-plugin/src/__tests__/v5-walltime.test.ts new file mode 100644 index 00000000..a6838a57 --- /dev/null +++ b/packages/vitest-plugin/src/__tests__/v5-walltime.test.ts @@ -0,0 +1,104 @@ +import type * as Core from "@codspeed/core"; +import { fromPartial } from "@total-typescript/shoehorn"; +import type { + BenchmarkGroup, + BenchRegistrationInput, + BenchRunOptions, +} from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import type * as Instrument from "../instrument"; +import provider from "../v5/provider"; + +const coreMocks = vi.hoisted(() => ({ + InstrumentHooks: { + startBenchmark: vi.fn(), + stopBenchmark: vi.fn(), + setExecutedBenchmark: vi.fn(), + addMarker: vi.fn(), + currentTimestamp: vi.fn(() => 0n), + }, + getInstrumentMode: vi.fn(() => "walltime"), + getGitDir: vi.fn(() => "/repo"), + setupCore: vi.fn(), + teardownCore: vi.fn(), + collectGarbage: vi.fn(), +})); + +vi.mock("@codspeed/core", async (importOriginal) => { + const mod = await importOriginal(); + return { ...mod, ...coreMocks }; +}); + +vi.mock("../instrument", async (importOriginal) => { + const mod = await importOriginal(); + return { ...mod, writeAndLogWalltimeResults: vi.fn() }; +}); + +const ITERATIONS = 3; + +function run(registrations: BenchRegistrationInput[]) { + const options: BenchRunOptions = { + iterations: ITERATIONS, + time: 0, + warmup: false, + }; + return provider.run( + fromPartial({ + test: fromPartial({ + file: { filepath: "/repo/a.bench.ts" }, + fullTestName: "suite > test", + context: {}, + }), + registrations, + options, + }), + ); +} + +describe("v5 walltime provider", () => { + afterEach(() => { + vi.unstubAllEnvs(); + vi.clearAllMocks(); + }); + + it("collects garbage after the user's beforeEach, before every round", async () => { + vi.stubEnv("CODSPEED_WALLTIME_GC_BETWEEN_ROUNDS", "true"); + const calls: string[] = []; + coreMocks.collectGarbage.mockImplementation(() => calls.push("gc")); + + await run([ + fromPartial({ + name: "bench", + fn: () => { + calls.push("fn"); + }, + fnOpts: { beforeEach: () => calls.push("beforeEach") }, + }), + ]); + + expect(calls.length).toBeGreaterThanOrEqual(ITERATIONS * 3); + for (let i = 0; i < calls.length; i += 3) { + expect(calls.slice(i, i + 3)).toEqual(["beforeEach", "gc", "fn"]); + } + }); + + it("does not collect garbage when disabled", async () => { + await run([ + fromPartial({ name: "bench", fn: () => {} }), + ]); + + expect(coreMocks.collectGarbage).not.toHaveBeenCalled(); + }); + + it("rejects forks", async () => { + vi.stubEnv("CODSPEED_WALLTIME_FORKS", "2"); + + await expect( + run([ + fromPartial({ name: "bench", fn: () => {} }), + ]), + ).rejects.toThrow( + "[CodSpeed] CODSPEED_WALLTIME_FORKS is not supported with Vitest 5 yet", + ); + }); +}); diff --git a/packages/vitest-plugin/src/__tests__/walltime.test.ts b/packages/vitest-plugin/src/__tests__/walltime.test.ts index da60f241..91975e88 100644 --- a/packages/vitest-plugin/src/__tests__/walltime.test.ts +++ b/packages/vitest-plugin/src/__tests__/walltime.test.ts @@ -1,5 +1,13 @@ import { fromPartial } from "@total-typescript/shoehorn"; -import { describe, expect, it, vi, type RunnerTestSuite } from "vitest"; +import { + afterEach, + beforeEach, + describe, + expect, + it, + vi, + type RunnerTestSuite, +} from "vitest"; import { WalltimeRunner } from "../legacy/walltime"; const coreMocks = vi.hoisted(() => { @@ -12,9 +20,26 @@ const coreMocks = vi.hoisted(() => { currentTimestamp: vi.fn(() => 0n), }, setupCore: vi.fn(), + writeWalltimeResults: vi.fn(), + collectGarbage: vi.fn(), + isGcBetweenRoundsEnabled: vi.fn(() => false), + getWalltimeForks: vi.fn((): number | undefined => undefined), + isFork: vi.fn(() => false), + runForks: vi.fn((): number[] => []), + writeForkSamples: vi.fn(), }; }); +const BENCH_OPTIONS = { + iterations: 3, + time: 0, + warmupIterations: 2, + warmupTime: 0, +}; + +/** Calls of the bench functions and of the mocked GC, in order. */ +const events = vi.hoisted((): string[] => []); + vi.mock("@codspeed/core", async (importOriginal) => { const mod = await importOriginal(); return { ...mod, ...coreMocks }; @@ -28,11 +53,25 @@ vi.mock("../legacy/compat", () => { id: string; type: string; name: string; + mode?: string; + result?: unknown; tasks: SuiteLike[]; } + // Like Vitest, keeps each bench's function next to its task. + const benchFns = new WeakMap void>(); + const getBenchFn = (task: SuiteLike) => { + if (!benchFns.has(task)) { + benchFns.set(task, () => { + events.push(`fn:${task.name}`); + }); + } + return benchFns.get(task)!; + }; + // Mirrors Vitest's benchmark suite execution: sub-suites run first, then - // the suite's own benches run after a `suite-prepare` event for the suite. + // the suite's own benches run after a `suite-prepare` event for the suite, + // each as a tinybench `Task` that is warmed up then run. class NodeBenchmarkRunner { async importTinybench() { return import("tinybench"); @@ -58,29 +97,87 @@ vi.mock("../legacy/compat", () => { [], [[suite.id, "suite-prepare", undefined]], ); - const { Bench } = await this.importTinybench(); + const { Bench, Task } = await this.importTinybench(); for (const task of benches) { - const bench = new Bench({ - iterations: 1, - time: 0, - warmupIterations: 0, - warmupTime: 0, - }); - bench.add(task.name, () => {}); - await bench.run(); + if (task.mode !== undefined && task.mode !== "run") { + continue; + } + const tinybenchTask = new Task( + new Bench({ + iterations: 3, + time: 0, + warmupIterations: 2, + warmupTime: 0, + }), + task.name, + getBenchFn(task), + ); + await tinybenchTask.warmup(); + await tinybenchTask.run(); + task.result = { state: "pass", benchmark: tinybenchTask.result }; } } } return { NodeBenchmarkRunner, - getBenchOptions: vi.fn(), + getBenchFn, + getBenchOptions: vi.fn(() => ({ + iterations: 3, + time: 0, + warmupIterations: 2, + warmupTime: 0, + })), }; }); const FILE_URI = "packages/vitest-plugin/src/__tests__/walltime.test.ts"; +const NESTED_URI = `${FILE_URI}::nested suite::nested bench`; +const TOP_LEVEL_URI = `${FILE_URI}::top-level bench`; + +function createSuite(): RunnerTestSuite { + return fromPartial({ + id: "file", + type: "suite", + file: { filepath: __filename }, + name: "walltime.test.ts", + tasks: [ + { + id: "nested", + type: "suite", + name: "nested suite", + tasks: [ + { type: "test", name: "nested bench", meta: { benchmark: true } }, + ], + }, + { type: "test", name: "top-level bench", meta: { benchmark: true } }, + ], + }); +} + +function writtenBenchmarks() { + return coreMocks.writeWalltimeResults.mock.calls.flatMap( + ([benchmarks]) => benchmarks, + ); +} + describe("WalltimeRunner", () => { + beforeEach(() => { + vi.clearAllMocks(); + events.length = 0; + coreMocks.collectGarbage.mockImplementation(() => { + events.push("gc"); + }); + }); + + afterEach(() => { + coreMocks.isGcBetweenRoundsEnabled.mockReturnValue(false); + coreMocks.getWalltimeForks.mockReturnValue(undefined); + coreMocks.isFork.mockReturnValue(false); + delete process.env.CODSPEED_ISOLATED_BENCHMARK; + }); + it("prefixes top-level and nested bench URIs with the file path", async () => { const runner = new WalltimeRunner(fromPartial({})); const suite = fromPartial({ @@ -106,9 +203,77 @@ describe("WalltimeRunner", () => { const uris = coreMocks.InstrumentHooks.setExecutedBenchmark.mock.calls.map( ([, uri]) => uri, ); - expect(uris).toEqual([ - `${FILE_URI}::nested suite::nested bench`, - `${FILE_URI}::top-level bench`, + expect(uris).toEqual([NESTED_URI, TOP_LEVEL_URI]); + }); + + it("collects garbage before every warmup and measured round", async () => { + coreMocks.isGcBetweenRoundsEnabled.mockReturnValue(true); + const runner = new WalltimeRunner(fromPartial({})); + + await runner.runSuite(createSuite()); + + // Every call of the bench function (including tinybench's async probe, + // which also runs `beforeEach` first) is preceded by exactly one GC. + const rounds = BENCH_OPTIONS.warmupIterations + BENCH_OPTIONS.iterations; + for (const name of ["nested bench", "top-level bench"]) { + const benchEvents = events.filter( + (event) => event === "gc" || event === `fn:${name}`, + ); + const fnCalls = benchEvents.filter((event) => event !== "gc").length; + expect(fnCalls).toBeGreaterThanOrEqual(rounds); + } + expect(events.length % 2).toBe(0); + for (let i = 0; i < events.length; i += 2) { + expect(events[i]).toBe("gc"); + expect(events[i + 1]).toMatch(/^fn:/); + } + }); + + it("merges the samples of every fork into one result", async () => { + const forkSamples = [3, 1, 2, 5, 4]; + coreMocks.getWalltimeForks.mockReturnValue(3); + coreMocks.runForks.mockReturnValue(forkSamples); + const runner = new WalltimeRunner(fromPartial({})); + + await runner.runSuite(createSuite()); + + expect(events).toEqual([]); + expect(coreMocks.InstrumentHooks.startBenchmark).not.toHaveBeenCalled(); + expect( + coreMocks.InstrumentHooks.setExecutedBenchmark, + ).not.toHaveBeenCalled(); + expect(coreMocks.runForks.mock.calls).toEqual([ + [NESTED_URI, 3], + [TOP_LEVEL_URI, 3], + ]); + const benchmarks = writtenBenchmarks(); + expect(benchmarks.map((benchmark) => benchmark.uri)).toEqual([ + NESTED_URI, + TOP_LEVEL_URI, ]); + for (const benchmark of benchmarks) { + expect(benchmark.stats.rounds).toBe(forkSamples.length); + expect(benchmark.stats.min_ns).toBe(1_000_000); + expect(benchmark.stats.max_ns).toBe(5_000_000); + } + }); + + it("runs only the isolated bench in a fork and hands its samples over", async () => { + coreMocks.isFork.mockReturnValue(true); + process.env.CODSPEED_ISOLATED_BENCHMARK = TOP_LEVEL_URI; + const runner = new WalltimeRunner(fromPartial({})); + + await runner.runSuite(createSuite()); + + expect(new Set(events)).toEqual(new Set(["fn:top-level bench"])); + expect( + coreMocks.InstrumentHooks.setExecutedBenchmark.mock.calls.map( + ([, uri]) => uri, + ), + ).toEqual([TOP_LEVEL_URI]); + expect(coreMocks.writeForkSamples).toHaveBeenCalledTimes(1); + const [samples] = coreMocks.writeForkSamples.mock.calls[0]; + expect(samples).toHaveLength(BENCH_OPTIONS.iterations); + expect(coreMocks.writeWalltimeResults).not.toHaveBeenCalled(); }); }); diff --git a/packages/vitest-plugin/src/instrument.ts b/packages/vitest-plugin/src/instrument.ts index 24039088..d708c394 100644 --- a/packages/vitest-plugin/src/instrument.ts +++ b/packages/vitest-plugin/src/instrument.ts @@ -1,5 +1,6 @@ import { calculateQuantiles, + collectGarbage, InstrumentHooks, MARKER_TYPE_BENCHMARK_END, MARKER_TYPE_BENCHMARK_START, @@ -28,6 +29,29 @@ export interface TinybenchBench { teardown: TinybenchHook; } +/** A tinybench per-round hook (`beforeEach`/`afterEach`), called with the task as `this`. */ +export type TinybenchRoundHook = ( + this: unknown, + mode: "run" | "warmup", +) => Promise | void; + +/** + * Returns a `beforeEach` hook that runs `hook` and then a full GC. tinybench + * calls `beforeEach` outside the timed call, so the GC starts every round on + * a collected heap without being measured. + */ +export function withGcAfter( + hook: TinybenchRoundHook | undefined, +): TinybenchRoundHook { + return function (this: unknown, mode) { + const result = hook?.call(this, mode); + if (result instanceof Promise) { + return result.then(collectGarbage); + } + collectGarbage(); + }; +} + interface TinybenchStatistics { min: number; max: number; diff --git a/packages/vitest-plugin/src/legacy/walltime.ts b/packages/vitest-plugin/src/legacy/walltime.ts index 8a4f0d7e..cd3b8daf 100644 --- a/packages/vitest-plugin/src/legacy/walltime.ts +++ b/packages/vitest-plugin/src/legacy/walltime.ts @@ -1,4 +1,13 @@ -import { setupCore, wrapWithRootFrame } from "@codspeed/core"; +import { + getWalltimeForks, + isFork, + isGcBetweenRoundsEnabled, + ISOLATED_BENCHMARK_ENV, + runForks, + setupCore, + wrapWithRootFrame, + writeForkSamples, +} from "@codspeed/core"; import { RunnerTaskEventPack, RunnerTaskResultPack, @@ -6,29 +15,70 @@ import { } from "vitest"; import { installInstrumentHooks, + withGcAfter, writeAndLogWalltimeResults, type Tinybench, type TinybenchBench, + type TinybenchRoundHook, } from "../instrument"; -import { patchRootSuiteWithFullFilePath } from "./common"; -import { NodeBenchmarkRunner } from "./compat"; +import { + isVitestTaskBenchmark, + patchRootSuiteWithFullFilePath, +} from "./common"; +import { getBenchFn, NodeBenchmarkRunner, type BenchmarkTask } from "./compat"; import { extractBenchmarkResults } from "./walltime-utils"; /** A tinybench v2 task, whose `fn` is a plain, reassignable property. */ interface TinybenchTask { + name: string; fn: () => unknown; + mode?: string; + opts: { beforeEach?: TinybenchRoundHook }; + loop: (time: number, iterations: number) => Promise; +} + +interface TinybenchLoopResult { + samples?: number[]; + error?: unknown; } +const patchedPrototypes = new WeakSet(); +let activeRunner: WalltimeRunner | null = null; + /** * Lets tinybench run the benches through Vitest's default benchmark execution, * instrumenting each measured loop, then extracts the results from the suite * tree afterwards. */ export class WalltimeRunner extends NodeBenchmarkRunner { - private isTinybenchHookedWithCodspeed = false; private suiteUris = new Map(); /// Suite ID of the currently running suite, to allow constructing the URI in the context of tinybench tasks private currentSuiteId: string | null = null; + /// Forks per benchmark in the parent process, unset when not forking + parentForks: number | undefined; + /// URIs of the benches of the running file, by bench name, for the forks parent + private forkUris = new Map>(); + + isForkParent(): boolean { + return this.parentForks !== undefined; + } + + /** + * URI of a bench in the forks parent. Unlike `getBenchmarkUri`, it doesn't + * depend on the `suite-prepare` update, which Vitest sends throttled and may + * not have arrived when the parent skips the warmup. + */ + forkUri(name: string, fn: unknown): string { + const matches = (this.forkUris.get(name) ?? []).filter( + (bench) => bench.fn === fn, + ); + if (matches.length !== 1) { + throw new Error( + `[CodSpeed] cannot tell which benchmark "${name}" is: ${matches.length} benchmarks share its name and function`, + ); + } + return matches[0].uri; + } async runSuite(suite: RunnerTestSuite): Promise { patchRootSuiteWithFullFilePath(suite); @@ -36,8 +86,30 @@ export class WalltimeRunner extends NodeBenchmarkRunner { setupCore(); + this.parentForks = isFork() ? undefined : getWalltimeForks(); + const benches = benchUris(suite); + if (isFork()) { + const target = process.env[ISOLATED_BENCHMARK_ENV]; + for (const { task, uri } of benches) { + if (uri !== target) task.mode = "skip"; + } + } + this.forkUris.clear(); + if (this.isForkParent()) { + for (const { task, uri } of benches) { + const sameName = this.forkUris.get(task.name) ?? []; + sameName.push({ fn: getBenchFn(task), uri }); + this.forkUris.set(task.name, sameName); + } + } + await super.runSuite(suite); + // The parent writes the merged results of every fork. + if (isFork()) { + return; + } + const benchmarks = await extractBenchmarkResults(suite); if (benchmarks.length === 0) { console.warn( @@ -60,7 +132,7 @@ export class WalltimeRunner extends NodeBenchmarkRunner { } } - private getBenchmarkUri(taskName: string): string { + getBenchmarkUri(taskName: string): string { if (this.currentSuiteId === null) { throw new Error("currentSuiteId is null - something went wrong"); } @@ -72,12 +144,17 @@ export class WalltimeRunner extends NodeBenchmarkRunner { const tinybench = await super.importTinybench(); // `tinybench` is a frozen ES module namespace, so the `Bench` export cannot - // be reassigned. Mutating the shared `Task.prototype` in place is allowed - // and only needs to happen once; the instrumented `Bench` is handed back + // be reassigned. Mutating the shared `Task.prototype` in place is allowed; + // it outlives runner instances, so it is patched once and routes to the + // runner that imported tinybench last. The rounds patch goes on top, so it + // still sees the bench's own `fn`. The instrumented `Bench` is handed back // through a fresh module-shaped object that Vitest destructures from. - if (!this.isTinybenchHookedWithCodspeed) { - this.isTinybenchHookedWithCodspeed = true; - this.patchTaskWithRootFrame(tinybench); + // eslint-disable-next-line @typescript-eslint/no-this-alias + activeRunner = this; + if (!patchedPrototypes.has(tinybench.Task.prototype)) { + patchedPrototypes.add(tinybench.Task.prototype); + patchTaskWithRootFrame(tinybench); + patchTaskRounds(tinybench); } return { @@ -86,28 +163,6 @@ export class WalltimeRunner extends NodeBenchmarkRunner { }; } - /** - * Wrap each task's function with the root frame so collected stacks can be - * attributed to a benchmark. - */ - private patchTaskWithRootFrame(tinybench: Tinybench): void { - const originalRun = tinybench.Task.prototype.run; - - tinybench.Task.prototype.run = async function () { - const task = this as unknown as TinybenchTask; - const originalFn = task.fn; - task.fn = wrapWithRootFrame(() => originalFn.call(task)); - - try { - await originalRun.call(this); - } finally { - task.fn = originalFn; - } - - return this; - }; - } - private createInstrumentedBench( tinybench: Tinybench, ): typeof tinybench.Bench { @@ -119,6 +174,9 @@ export class WalltimeRunner extends NodeBenchmarkRunner { // eslint-disable-next-line @typescript-eslint/no-explicit-any constructor(...benchArgs: any[]) { super(...benchArgs); + if (runner.isForkParent()) { + return; + } installInstrumentHooks(this as unknown as TinybenchBench, (taskName) => runner.getBenchmarkUri(taskName), ); @@ -143,4 +201,113 @@ export class WalltimeRunner extends NodeBenchmarkRunner { } } +/** + * Wrap each task's function with the root frame so collected stacks can be + * attributed to a benchmark. + */ +function patchTaskWithRootFrame(tinybench: Tinybench): void { + const originalRun = tinybench.Task.prototype.run; + + tinybench.Task.prototype.run = async function () { + const task = this as unknown as TinybenchTask; + const originalFn = task.fn; + task.fn = wrapWithRootFrame(() => originalFn.call(task)); + + try { + await originalRun.call(this); + } finally { + task.fn = originalFn; + } + + return this; + }; +} + +/** + * Forks parent: skip warmup and replace the measured loop with the samples + * of every fork. Fork child: hand the measured samples to the parent. + * Otherwise: optionally collect garbage before every round. + */ +function patchTaskRounds(tinybench: Tinybench): void { + const originalWarmup = tinybench.Task.prototype.warmup; + const originalRun = tinybench.Task.prototype.run; + const gcComposed = new WeakSet(); + + const composeGc = (task: TinybenchTask) => { + if (gcComposed.has(task) || !isGcBetweenRoundsEnabled()) { + return; + } + gcComposed.add(task); + task.opts.beforeEach = withGcAfter(task.opts.beforeEach); + }; + + tinybench.Task.prototype.warmup = async function () { + if (activeRunner!.isForkParent()) { + return; + } + composeGc(this as unknown as TinybenchTask); + return originalWarmup.call(this); + }; + + tinybench.Task.prototype.run = async function () { + const task = this as unknown as TinybenchTask; + const originalLoop = task.loop; + + if (activeRunner!.isForkParent()) { + const runner = activeRunner!; + // Read before the root frame patch wraps `fn` for the run. + const fn = task.fn; + // Returned, not thrown: tinybench only reports errors from its loop, and + // a rejected run() would leave Vitest waiting forever. + task.loop = async () => { + try { + return { + samples: runForks( + runner.forkUri(task.name, fn), + runner.parentForks!, + ), + }; + } catch (error) { + return { error }; + } + }; + } else { + composeGc(task); + if (isFork()) { + task.loop = async (time, iterations) => { + const result = await originalLoop.call(task, time, iterations); + if (result.samples) { + // Copied: tinybench sorts and Vitest may truncate the array. + writeForkSamples([...result.samples]); + } + return result; + }; + } + } + + try { + return await originalRun.call(this); + } finally { + task.loop = originalLoop; + } + }; +} + +/** Every bench of `suite` with its URI, built like `extractBenchmarkResults`. */ +function benchUris( + suite: RunnerTestSuite, + parentPath = "", +): Array<{ task: BenchmarkTask; uri: string }> { + const currentPath = parentPath ? `${parentPath}::${suite.name}` : suite.name; + return suite.tasks.flatMap((task) => { + if (task.type === "suite") { + return benchUris(task, currentPath); + } + if (!isVitestTaskBenchmark(task)) { + return []; + } + return [{ task, uri: `${currentPath}::${task.name}` }]; + }); +} + export default WalltimeRunner; diff --git a/packages/vitest-plugin/src/v5/provider.ts b/packages/vitest-plugin/src/v5/provider.ts index c54d2017..3f793507 100644 --- a/packages/vitest-plugin/src/v5/provider.ts +++ b/packages/vitest-plugin/src/v5/provider.ts @@ -1,12 +1,15 @@ import { getGitDir, getInstrumentMode, + getWalltimeForks, InstrumentHooks, + isGcBetweenRoundsEnabled, Isolation, optimizeFunction, optimizeFunctionSync, setupCore, teardownCore, + WALLTIME_FORKS_ENV, wrapWithRootFrame, wrapWithRootFrameSync, type Benchmark, @@ -23,9 +26,11 @@ import type { import { installInstrumentHooks, tinybenchTaskToBenchmark, + withGcAfter, writeAndLogWalltimeResults, type TinybenchBench, type TinybenchOptions, + type TinybenchRoundHook, type TinybenchTask, } from "../instrument"; @@ -231,6 +236,13 @@ async function runWalltime( getUri: (name: string) => string, test: BenchmarkGroup["test"], ): Promise { + if (getWalltimeForks() !== undefined) { + throw new Error( + `[CodSpeed] ${WALLTIME_FORKS_ENV} is not supported with Vitest 5 yet`, + ); + } + + const gcBetweenRounds = isGcBetweenRoundsEnabled(); const { Bench } = await importHostTinybench(); const bench = new Bench({ signal: test.context.signal, @@ -244,7 +256,15 @@ async function runWalltime( bench.add( name, wrapWithRootFrame(() => fn()), - fnOpts, + gcBetweenRounds + ? { + ...fnOpts, + // Vitest's FnHook is typed looser than tinybench's round hook + beforeEach: withGcAfter( + fnOpts?.beforeEach as TinybenchRoundHook | undefined, + ) as TinybenchFnOptions["beforeEach"], + } + : fnOpts, ); } From 8d1f51824a00fe8b36096384bcd39e2c35606d47 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 23 Oct 2026 14:32:48 +0200 Subject: [PATCH 16/21] docs: note that GC between rounds re-baselines walltime results --- CLAUDE.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index 70c10e3f..d2d8258b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -131,4 +131,6 @@ Both are opt-in and only apply to walltime runs. (warmup and measured). It adds `--expose-gc` to the V8 flags and collects in the tinybench `beforeEach` hook, outside the timed call. Vitest 3/4/5. Meant for benchmarks with long rounds: a GC costs milliseconds, so time-based - benchmarks of microsecond functions take far longer to finish. + benchmarks of microsecond functions take far longer to finish. It also + changes absolute times (each round starts on a collected heap), so turning + it on re-baselines a benchmark. From ebd690a34c4de6fcff757bb5c81faeb401a33535 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 23 Oct 2026 14:37:03 +0200 Subject: [PATCH 17/21] refactor(vitest-plugin): drop unused walltime task field --- packages/vitest-plugin/src/legacy/walltime.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/vitest-plugin/src/legacy/walltime.ts b/packages/vitest-plugin/src/legacy/walltime.ts index cd3b8daf..fb3c5185 100644 --- a/packages/vitest-plugin/src/legacy/walltime.ts +++ b/packages/vitest-plugin/src/legacy/walltime.ts @@ -32,7 +32,6 @@ import { extractBenchmarkResults } from "./walltime-utils"; interface TinybenchTask { name: string; fn: () => unknown; - mode?: string; opts: { beforeEach?: TinybenchRoundHook }; loop: (time: number, iterations: number) => Promise; } @@ -132,7 +131,7 @@ export class WalltimeRunner extends NodeBenchmarkRunner { } } - getBenchmarkUri(taskName: string): string { + private getBenchmarkUri(taskName: string): string { if (this.currentSuiteId === null) { throw new Error("currentSuiteId is null - something went wrong"); } From 4e7674d5c0e47f11d2ad835f437c182de9a4af06 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 28 Oct 2026 20:41:17 +0100 Subject: [PATCH 18/21] feat(vitest-plugin): halve the walltime budget of each fork With several forks, every fork paid the full warmup and measurement budget on top of its process startup, so F forks cost F full runs. Each fork now runs half the time and ceil(half) the rounds of both phases. A single fork keeps the full budget. --- CLAUDE.md | 4 ++- packages/core/src/walltime/options.ts | 7 ++++ .../src/__tests__/walltime.test.ts | 15 ++++++++ packages/vitest-plugin/src/legacy/walltime.ts | 35 ++++++++++++++++--- 4 files changed, 56 insertions(+), 5 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index d2d8258b..47808cc5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -126,7 +126,9 @@ Both are opt-in and only apply to walltime runs. runs the benchmark in the benchmark process. Vitest 3/4 only. Like isolation, each child re-runs the Vitest command with `CODSPEED_ISOLATED_BENCHMARK=`; it writes its round samples to the file named by `CODSPEED_FORK_SAMPLES_FILE` - instead of reporting, and the parent merges them. + instead of reporting, and the parent merges them. With `N > 1`, each fork + runs half the warmup and measurement budget (`FORK_BUDGET_FRACTION`: half + the time, rounds rounded up), so forks mostly add process startup. - `CODSPEED_WALLTIME_GC_BETWEEN_ROUNDS=true` runs a full GC before every round (warmup and measured). It adds `--expose-gc` to the V8 flags and collects in the tinybench `beforeEach` hook, outside the timed call. Vitest 3/4/5. Meant diff --git a/packages/core/src/walltime/options.ts b/packages/core/src/walltime/options.ts index 3ee359ff..048568e7 100644 --- a/packages/core/src/walltime/options.ts +++ b/packages/core/src/walltime/options.ts @@ -7,6 +7,13 @@ export const WALLTIME_FORKS_ENV = "CODSPEED_WALLTIME_FORKS"; export const WALLTIME_GC_BETWEEN_ROUNDS_ENV = "CODSPEED_WALLTIME_GC_BETWEEN_ROUNDS"; +/** + * Share of a benchmark's warmup and measurement budget (time and rounds) that + * each fork runs when there are several forks. Every fork pays the process + * startup anyway, so full budgets would make F forks cost F full runs. + */ +export const FORK_BUDGET_FRACTION = 0.5; + /** The number of forks per walltime benchmark, or `undefined` for none. */ export function getWalltimeForks(): number | undefined { const value = process.env[WALLTIME_FORKS_ENV]; diff --git a/packages/vitest-plugin/src/__tests__/walltime.test.ts b/packages/vitest-plugin/src/__tests__/walltime.test.ts index 91975e88..28a9dc0a 100644 --- a/packages/vitest-plugin/src/__tests__/walltime.test.ts +++ b/packages/vitest-plugin/src/__tests__/walltime.test.ts @@ -276,4 +276,19 @@ describe("WalltimeRunner", () => { expect(samples).toHaveLength(BENCH_OPTIONS.iterations); expect(coreMocks.writeWalltimeResults).not.toHaveBeenCalled(); }); + + it("runs half the warmup and measured rounds in each of several forks", async () => { + coreMocks.isFork.mockReturnValue(true); + coreMocks.getWalltimeForks.mockReturnValue(3); + process.env.CODSPEED_ISOLATED_BENCHMARK = TOP_LEVEL_URI; + const runner = new WalltimeRunner(fromPartial({})); + + await runner.runSuite(createSuite()); + + const [samples] = coreMocks.writeForkSamples.mock.calls[0]; + expect(samples).toHaveLength(Math.ceil(BENCH_OPTIONS.iterations / 2)); + // Warmup rounds, measured rounds, and tinybench's async probe. + const calls = Math.ceil(BENCH_OPTIONS.warmupIterations / 2) + Math.ceil(BENCH_OPTIONS.iterations / 2) + 1; + expect(events).toHaveLength(calls); + }); }); diff --git a/packages/vitest-plugin/src/legacy/walltime.ts b/packages/vitest-plugin/src/legacy/walltime.ts index fb3c5185..e337b0df 100644 --- a/packages/vitest-plugin/src/legacy/walltime.ts +++ b/packages/vitest-plugin/src/legacy/walltime.ts @@ -1,4 +1,5 @@ import { + FORK_BUDGET_FRACTION, getWalltimeForks, isFork, isGcBetweenRoundsEnabled, @@ -224,7 +225,8 @@ function patchTaskWithRootFrame(tinybench: Tinybench): void { /** * Forks parent: skip warmup and replace the measured loop with the samples - * of every fork. Fork child: hand the measured samples to the parent. + * of every fork. Fork child: run its share of the warmup and measurement + * budget, then hand the measured samples to the parent. * Otherwise: optionally collect garbage before every round. */ function patchTaskRounds(tinybench: Tinybench): void { @@ -244,8 +246,19 @@ function patchTaskRounds(tinybench: Tinybench): void { if (activeRunner!.isForkParent()) { return; } - composeGc(this as unknown as TinybenchTask); - return originalWarmup.call(this); + const task = this as unknown as TinybenchTask; + composeGc(task); + if (!isFork()) { + return originalWarmup.call(this); + } + const originalLoop = task.loop; + task.loop = (time, iterations) => + originalLoop.call(task, ...forkBudget(time, iterations)); + try { + return await originalWarmup.call(this); + } finally { + task.loop = originalLoop; + } }; tinybench.Task.prototype.run = async function () { @@ -274,7 +287,10 @@ function patchTaskRounds(tinybench: Tinybench): void { composeGc(task); if (isFork()) { task.loop = async (time, iterations) => { - const result = await originalLoop.call(task, time, iterations); + const result = await originalLoop.call( + task, + ...forkBudget(time, iterations), + ); if (result.samples) { // Copied: tinybench sorts and Vitest may truncate the array. writeForkSamples([...result.samples]); @@ -292,6 +308,17 @@ function patchTaskRounds(tinybench: Tinybench): void { }; } +/** The `[time, iterations]` budget of one fork: shared when there are several. */ +function forkBudget(time: number, iterations: number): [number, number] { + if ((getWalltimeForks() ?? 1) <= 1) { + return [time, iterations]; + } + return [ + time * FORK_BUDGET_FRACTION, + Math.ceil(iterations * FORK_BUDGET_FRACTION), + ]; +} + /** Every bench of `suite` with its URI, built like `extractBenchmarkResults`. */ function benchUris( suite: RunnerTestSuite, From ef6aac69ede7c2880091afe546dd2915b414c827 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 28 Oct 2026 20:52:52 +0100 Subject: [PATCH 19/21] fix(core): remove fork samples when a fork fails A failing fork exited the parent through process.exit(), which skipped the cleanup of the samples directory. --- packages/core/src/isolation.ts | 44 +++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/packages/core/src/isolation.ts b/packages/core/src/isolation.ts index 4e563e31..f57b881c 100644 --- a/packages/core/src/isolation.ts +++ b/packages/core/src/isolation.ts @@ -167,7 +167,12 @@ export function runForks(uri: string, forks: number): number[] { let samples: number[] = []; for (let fork = 0; fork < forks; fork++) { const file = join(dir, `${fork}.json`); - runInChildProcess(uri, { [FORK_SAMPLES_FILE_ENV]: file }); + const failure = runChild(uri, { [FORK_SAMPLES_FILE_ENV]: file }); + if (failure !== undefined) { + // process.exit() skips the `finally` below. + rmSync(dir, { recursive: true, force: true }); + process.exit(failure); + } if (!existsSync(file)) { throw new Error( `[CodSpeed] fork ${fork + 1}/${forks} of ${uri} did not run the benchmark`, @@ -209,7 +214,18 @@ interface ForkSamples { * child's code if it fails. Re-exec, not fork(): Node can't fork, and a fork * would only copy the calling thread. */ -function runInChildProcess(uri: string, env: NodeJS.ProcessEnv = {}): void { +function runInChildProcess(uri: string): void { + const failure = runChild(uri); + if (failure !== undefined) { + process.exit(failure); + } +} + +/** Runs the benchmark command limited to `uri`; the exit code if it failed. */ +function runChild( + uri: string, + env: NodeJS.ProcessEnv = {}, +): number | undefined { const [executable, ...args] = isolationCommand(); const result = spawnSync(executable, args, { stdio: "inherit", @@ -218,7 +234,7 @@ function runInChildProcess(uri: string, env: NodeJS.ProcessEnv = {}): void { if (result.error !== undefined) { throw startError(uri, result.error); } - exitOnFailure(result.status, result.signal); + return failureCode(result.status, result.signal); } /** @@ -237,13 +253,13 @@ async function runInChildProcessAsync(uri: string): Promise { child.once("error", (error) => reject(startError(uri, error))); child.once("close", (code, signal) => resolve([code, signal])); const [status, signal] = await promise; - exitOnFailure(status, signal); + const failure = failureCode(status, signal); + if (failure !== undefined) { + process.exit(failure); + } } -function childEnv( - uri: string, - env: NodeJS.ProcessEnv = {}, -): NodeJS.ProcessEnv { +function childEnv(uri: string, env: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { return { ...process.env, ...env, [ISOLATED_BENCHMARK_ENV]: uri }; } @@ -253,16 +269,16 @@ function startError(uri: string, error: Error): Error { ); } -/** Exits with a failed child's code, in the shell convention. */ -function exitOnFailure( +/** A failed child's exit code, in the shell convention. */ +function failureCode( status: number | null, signal: NodeJS.Signals | null, -): void { +): number | undefined { if (status === 0) { - return; + return undefined; } if (signal !== null) { - process.exit(SIGNAL_EXIT_CODE_OFFSET + constants.signals[signal]); + return SIGNAL_EXIT_CODE_OFFSET + constants.signals[signal]; } - process.exit(status ?? GENERIC_FAILURE_EXIT_CODE); + return status ?? GENERIC_FAILURE_EXIT_CODE; } From 8de508688346a3e98a20faea40a03f2b995ae97a Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 28 Oct 2026 20:52:53 +0100 Subject: [PATCH 20/21] fix(vitest-plugin): wait for any thenable before GC between rounds A beforeEach hook returning a promise from another realm or a thenable failed the instanceof check, so the GC ran before the hook finished. --- packages/vitest-plugin/src/instrument.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/vitest-plugin/src/instrument.ts b/packages/vitest-plugin/src/instrument.ts index d708c394..cfb816bc 100644 --- a/packages/vitest-plugin/src/instrument.ts +++ b/packages/vitest-plugin/src/instrument.ts @@ -44,9 +44,10 @@ export function withGcAfter( hook: TinybenchRoundHook | undefined, ): TinybenchRoundHook { return function (this: unknown, mode) { - const result = hook?.call(this, mode); - if (result instanceof Promise) { - return result.then(collectGarbage); + const result: unknown = hook?.call(this, mode); + // Any thenable, not only this realm's Promise, so the GC waits for it. + if (typeof (result as PromiseLike | undefined)?.then === "function") { + return Promise.resolve(result).then(collectGarbage); } collectGarbage(); }; From 4353e5d6a4f581e9e3c46bcd372fbb46766c1d3c Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 28 Oct 2026 21:27:33 +0100 Subject: [PATCH 21/21] fix(core): keep the event loop free while forks run runForks waited for each fork with spawnSync, blocking a Vitest worker's RPC with its main process for the whole run of all forks. --- packages/core/src/isolation.ts | 31 ++++++++++++------- packages/core/tests/fixtures/forks.cjs | 6 ++-- packages/vitest-plugin/src/legacy/walltime.ts | 2 +- 3 files changed, 24 insertions(+), 15 deletions(-) diff --git a/packages/core/src/isolation.ts b/packages/core/src/isolation.ts index f57b881c..7c54766d 100644 --- a/packages/core/src/isolation.ts +++ b/packages/core/src/isolation.ts @@ -161,13 +161,15 @@ function isolationCommand(): string[] { * measures the benchmark like an isolated child and writes its samples to the * file named by {@link FORK_SAMPLES_FILE_ENV} instead of reporting results. */ -export function runForks(uri: string, forks: number): number[] { +export async function runForks(uri: string, forks: number): Promise { const dir = mkdtempSync(join(tmpdir(), "codspeed-forks-")); try { let samples: number[] = []; for (let fork = 0; fork < forks; fork++) { const file = join(dir, `${fork}.json`); - const failure = runChild(uri, { [FORK_SAMPLES_FILE_ENV]: file }); + const failure = await runChildAsync(uri, { + [FORK_SAMPLES_FILE_ENV]: file, + }); if (failure !== undefined) { // process.exit() skips the `finally` below. rmSync(dir, { recursive: true, force: true }); @@ -222,14 +224,11 @@ function runInChildProcess(uri: string): void { } /** Runs the benchmark command limited to `uri`; the exit code if it failed. */ -function runChild( - uri: string, - env: NodeJS.ProcessEnv = {}, -): number | undefined { +function runChild(uri: string): number | undefined { const [executable, ...args] = isolationCommand(); const result = spawnSync(executable, args, { stdio: "inherit", - env: childEnv(uri, env), + env: childEnv(uri), }); if (result.error !== undefined) { throw startError(uri, result.error); @@ -243,20 +242,28 @@ function runChild( * time out otherwise. */ async function runInChildProcessAsync(uri: string): Promise { + const failure = await runChildAsync(uri); + if (failure !== undefined) { + process.exit(failure); + } +} + +/** {@link runChild} without blocking the event loop. */ +async function runChildAsync( + uri: string, + env: NodeJS.ProcessEnv = {}, +): Promise { const [executable, ...args] = isolationCommand(); const child = spawn(executable, args, { stdio: "inherit", - env: childEnv(uri), + env: childEnv(uri, env), }); const { promise, resolve, reject } = Promise.withResolvers<[number | null, NodeJS.Signals | null]>(); child.once("error", (error) => reject(startError(uri, error))); child.once("close", (code, signal) => resolve([code, signal])); const [status, signal] = await promise; - const failure = failureCode(status, signal); - if (failure !== undefined) { - process.exit(failure); - } + return failureCode(status, signal); } function childEnv(uri: string, env: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { diff --git a/packages/core/tests/fixtures/forks.cjs b/packages/core/tests/fixtures/forks.cjs index c5601c65..3e0ec9ab 100644 --- a/packages/core/tests/fixtures/forks.cjs +++ b/packages/core/tests/fixtures/forks.cjs @@ -16,6 +16,8 @@ if (isFork()) { const count = Number(process.env.FORKS_FIXTURE_SAMPLES); writeForkSamples(Array.from({ length: count }, (_, i) => i)); } else { - const samples = runForks("a", Number(process.env.FORKS_FIXTURE_COUNT)); - console.log(`samples ${samples.length}`); + // A rejection ends the process with exit code 1. + runForks("a", Number(process.env.FORKS_FIXTURE_COUNT)).then((samples) => { + console.log(`samples ${samples.length}`); + }); } diff --git a/packages/vitest-plugin/src/legacy/walltime.ts b/packages/vitest-plugin/src/legacy/walltime.ts index e337b0df..9b2666b0 100644 --- a/packages/vitest-plugin/src/legacy/walltime.ts +++ b/packages/vitest-plugin/src/legacy/walltime.ts @@ -274,7 +274,7 @@ function patchTaskRounds(tinybench: Tinybench): void { task.loop = async () => { try { return { - samples: runForks( + samples: await runForks( runner.forkUri(task.name, fn), runner.parentForks!, ),