From 7067a24a442678b27697cf07727d8e50425b786f Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 29 Oct 2026 12:09:22 +0100 Subject: [PATCH 1/8] feat(core): run walltime benchmarks in forks Benchmarks in one process share allocator and JIT state, and rounds within one process are correlated, so a single process gives a biased view of a benchmark. Add the building blocks for integrations to run each walltime benchmark in several fresh processes (forks) and pool their rounds: - `Isolation.walltime(forks?)` picks the mode from the env: parent, fork (`CODSPEED_ISOLATED_BENCHMARK`), or one shared process when `CODSPEED_PROCESS_ISOLATION=false` - the fork count is `CODSPEED_WALLTIME_FORKS`, else the integration's `forks` option, else 3 - `runForks`/`runForksSync` re-run the benchmark command (keeping the V8 flags) once per fork and collect the samples each fork writes with `writeForkSamples`; a failing fork's exit code is passed on - `samplesToResult` builds tinybench-style statistics over the pooled samples Refs COD-3960 --- packages/core/src/index.ts | 1 + packages/core/src/isolation.ts | 223 ++++++++++++++++++++ packages/core/src/walltime/utils.ts | 37 ++++ packages/core/tests/fixtures/isolation.cjs | 36 ++++ packages/core/tests/isolation.integ.test.ts | 187 ++++++++++++++++ 5 files changed, 484 insertions(+) create mode 100644 packages/core/src/isolation.ts create mode 100644 packages/core/tests/fixtures/isolation.cjs create mode 100644 packages/core/tests/isolation.integ.test.ts diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 63ac394f..514abe22 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -51,6 +51,7 @@ export type { SetupInstrumentsResponse, } from "./generated/openapi"; export { getV8Flags, tryIntrospect } from "./introspection"; +export * from "./isolation"; export { getUnsupportedNodeVersionWarning, SUPPORTED_NODE_MAJORS, diff --git a/packages/core/src/isolation.ts b/packages/core/src/isolation.ts new file mode 100644 index 00000000..e3403537 --- /dev/null +++ b/packages/core/src/isolation.ts @@ -0,0 +1,223 @@ +/** + * Process isolation for walltime runs: every benchmark runs in fresh copies + * of the benchmark process (forks), so the allocator state, JIT state, and + * handles that other benchmarks or setup code leave behind can't change its + * measurement. Rounds within one process are correlated (JIT tier-up, heap + * sizing), so the rounds of several forks are pooled into one result. + */ +import { spawn, spawnSync } from "child_process"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; + +/** Isolation is on unless this is `0` or `false`. */ +export const PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION"; +/** Forks per walltime benchmark. Overrides the integration's `forks` option. */ +export const WALLTIME_FORKS_ENV = "CODSPEED_WALLTIME_FORKS"; +/** Forks per walltime benchmark when neither the env nor an option sets it. */ +export const DEFAULT_WALLTIME_FORKS = 3; +/** Set on each fork: the URI of the only benchmark it runs. */ +export const ISOLATED_BENCHMARK_ENV = "CODSPEED_ISOLATED_BENCHMARK"; +/** Set on each fork: the file where it writes its round samples. */ +export const FORK_SAMPLES_FILE_ENV = "CODSPEED_FORK_SAMPLES_FILE"; +/** + * JSON array `[executable, ...args]` of the command that runs the benchmarks + * again. Integrations whose benchmarks run in a worker process (vitest) set it + * from the main process, because re-executing the worker alone can't work. + */ +export const ISOLATION_COMMAND_ENV = "CODSPEED_ISOLATION_COMMAND"; + +/** Walltime settings that every integration exposes next to its own options. */ +export interface WalltimeOptions { + /** + * Fresh processes that each walltime benchmark runs in; their rounds are + * reported together. {@link WALLTIME_FORKS_ENV} overrides it; defaults to + * {@link DEFAULT_WALLTIME_FORKS}. `CODSPEED_PROCESS_ISOLATION=false` runs + * everything in the benchmark process instead. + */ + forks?: number; +} + +export type IsolationMode = + /** All benchmarks are measured in this process. */ + | { kind: "SharedProcess" } + /** Every benchmark is measured in `forks` fresh processes, one after the other. */ + | { kind: "Forks"; forks: number } + /** A fork: measures only `uri` and writes its samples to `samplesFile`. */ + | { kind: "Fork"; uri: string; samplesFile: string }; + +/** + * The isolation of a walltime run, from the environment. `forks` is the + * integration's option, which {@link WALLTIME_FORKS_ENV} overrides. + */ +export function walltimeIsolation(forks?: number): IsolationMode { + const uri = process.env[ISOLATED_BENCHMARK_ENV]; + if (uri !== undefined) { + const samplesFile = process.env[FORK_SAMPLES_FILE_ENV]; + if (samplesFile === undefined) { + throw new Error( + `[CodSpeed] ${ISOLATED_BENCHMARK_ENV} is set without ${FORK_SAMPLES_FILE_ENV}`, + ); + } + return { kind: "Fork", uri, samplesFile }; + } + const isolation = process.env[PROCESS_ISOLATION_ENV]; + if (isolation === "0" || isolation === "false") { + return { kind: "SharedProcess" }; + } + return { kind: "Forks", forks: resolveWalltimeForks(forks) }; +} + +/** + * The number of forks per walltime benchmark: {@link WALLTIME_FORKS_ENV}, + * else `option`, else {@link DEFAULT_WALLTIME_FORKS}. + */ +export function resolveWalltimeForks(option?: number): number { + const env = process.env[WALLTIME_FORKS_ENV]; + if (env !== undefined && env !== "") { + return validForks(env, WALLTIME_FORKS_ENV); + } + if (option !== undefined) { + return validForks(option, "forks"); + } + return DEFAULT_WALLTIME_FORKS; +} + +function validForks(raw: string | number, source: string): number { + const forks = Number(raw); + if (!Number.isInteger(forks) || forks < 1) { + throw new Error( + `[CodSpeed] ${source} must be a positive integer, got \`${raw}\``, + ); + } + return forks; +} + +/** + * Records the command of this process as the one that forks run. Call it + * from the main process of integrations that measure in worker processes. It + * overwrites a command inherited from an outer process, which is not ours. + */ +export function recordIsolationCommand(): void { + process.env[ISOLATION_COMMAND_ENV] = JSON.stringify(currentCommand()); +} + +function currentCommand(): string[] { + // execArgv keeps the V8 flags (e.g. --perf-prof) that the runner set. + return [process.execPath, ...process.execArgv, ...process.argv.slice(1)]; +} + +function isolationCommand(): string[] { + const recorded = process.env[ISOLATION_COMMAND_ENV]; + if (recorded === undefined) { + return currentCommand(); + } + return JSON.parse(recorded) as string[]; +} + +/** + * Runs the benchmark `uri` in `forks` fresh processes, one after the other, + * and returns the round samples of all of them, in milliseconds. Throws when + * a fork fails, without running the next one. + * + * Each fork re-runs the benchmark command, not fork(): Node can't fork, and a + * fork would only copy the calling thread. The fork's event loop stays free + * while it runs: a Vitest worker must keep answering its main process, whose + * calls time out otherwise. + */ +export async function runForks(uri: string, forks: number): Promise { + const dir = mkdtempSync(join(tmpdir(), SAMPLES_DIR_PREFIX)); + try { + let samples: number[] = []; + for (let fork = 0; fork < forks; fork++) { + // Not push(...): spreading a time-based run's samples overflows the stack. + samples = samples.concat(await runFork(uri, join(dir, `${fork}.json`))); + } + return samples; + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +/** {@link runForks} for synchronous runs, which block anyway. */ +export function runForksSync(uri: string, forks: number): number[] { + const dir = mkdtempSync(join(tmpdir(), SAMPLES_DIR_PREFIX)); + try { + let samples: number[] = []; + for (let fork = 0; fork < forks; fork++) { + samples = samples.concat(runForkSync(uri, join(dir, `${fork}.json`))); + } + return samples; + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +const SAMPLES_DIR_PREFIX = "codspeed-forks-"; + +/** Runs one fork of `uri` and returns the samples it wrote to `file`. */ +async function runFork(uri: string, file: string): Promise { + const [executable, ...args] = isolationCommand(); + const child = spawn(executable, args, { + stdio: "inherit", + env: forkEnv(uri, file), + }); + const { promise, resolve, reject } = + Promise.withResolvers<[number | null, NodeJS.Signals | null]>(); + child.once("error", (error) => reject(startError(uri, error))); + child.once("close", (status, signal) => resolve([status, signal])); + const [status, signal] = await promise; + return forkSamples(uri, file, status, signal); +} + +function runForkSync(uri: string, file: string): number[] { + const [executable, ...args] = isolationCommand(); + const result = spawnSync(executable, args, { + stdio: "inherit", + env: forkEnv(uri, file), + }); + if (result.error !== undefined) { + throw startError(uri, result.error); + } + return forkSamples(uri, file, result.status, result.signal); +} + +function forkEnv(uri: string, samplesFile: string): NodeJS.ProcessEnv { + return { + ...process.env, + [ISOLATED_BENCHMARK_ENV]: uri, + [FORK_SAMPLES_FILE_ENV]: samplesFile, + }; +} + +function startError(uri: string, error: Error): Error { + return new Error( + `[CodSpeed] failed to start the process for ${uri}: ${error.message}`, + ); +} + +/** The samples a finished fork wrote; throws if it failed. */ +function forkSamples( + uri: string, + file: string, + status: number | null, + signal: NodeJS.Signals | null, +): number[] { + if (status !== 0) { + throw new Error( + `[CodSpeed] the process for ${uri} failed with ${signal ?? `exit code ${status}`}`, + ); + } + try { + return JSON.parse(readFileSync(file, "utf-8")) as number[]; + } catch (error) { + throw new Error( + `[CodSpeed] the process for ${uri} did not report its samples: ${(error as Error).message}`, + ); + } +} + +/** Hands the round samples (milliseconds) of this fork to its parent. */ +export function writeForkSamples(samplesFile: string, samples: number[]) { + writeFileSync(samplesFile, JSON.stringify(samples)); +} diff --git a/packages/core/src/walltime/utils.ts b/packages/core/src/walltime/utils.ts index fafa9e9e..82362955 100644 --- a/packages/core/src/walltime/utils.ts +++ b/packages/core/src/walltime/utils.ts @@ -18,3 +18,40 @@ export const msToNs = (ms: number) => ms * 1e6; * @returns the seconds */ export const msToS = (ms: number) => ms / 1e3; + +/** tinybench-style statistics of round samples, in milliseconds. */ +export interface RoundsResult { + totalTime: number; + latency: { + min: number; + max: number; + mean: number; + sd: number; + samples: number[]; + }; +} + +/** + * The result of round samples in milliseconds, e.g. the samples of several + * forks of one benchmark. + */ +export function samplesToResult(samples: number[]): RoundsResult { + const n = samples.length; + const totalTime = samples.reduce((sum, ms) => sum + ms, 0); + const mean = totalTime / n; + const variance = + n > 1 + ? samples.reduce((sum, ms) => sum + (ms - mean) ** 2, 0) / (n - 1) + : 0; + return { + totalTime, + latency: { + // Not Math.min(...samples): a time-based run overflows the stack. + min: samples.reduce((a, b) => Math.min(a, b), Infinity), + max: samples.reduce((a, b) => Math.max(a, b), -Infinity), + mean, + sd: Math.sqrt(variance), + samples, + }, + }; +} diff --git a/packages/core/tests/fixtures/isolation.cjs b/packages/core/tests/fixtures/isolation.cjs new file mode 100644 index 00000000..bd2d6d2e --- /dev/null +++ b/packages/core/tests/fixtures/isolation.cjs @@ -0,0 +1,36 @@ +// Run by `isolation.integ.test.ts`. Measures benchmark "a" in forks. +const { + runForks, + runForksSync, + walltimeIsolation, + writeForkSamples, +} = require("../../dist/index.cjs.js"); + +const mode = walltimeIsolation(); + +function measure() { + console.log(`ran ${process.pid} ${JSON.stringify(process.execArgv)}`); + switch (process.env.ISOLATION_FIXTURE_ACTION) { + case "exit": + process.exit(3); + case "no-samples": + process.exit(0); + } + writeForkSamples(mode.samplesFile, [Number(process.pid), 1]); +} + +async function main() { + if (mode.kind === "Fork") { + measure(); + return; + } + const forks = Number(process.env.ISOLATION_FIXTURE_FORKS); + const samples = + process.env.ISOLATION_FIXTURE_SYNC !== undefined + ? runForksSync("a", forks) + : await runForks("a", forks); + console.log(`samples ${samples.length}`); +} + +// A rejection ends the process with exit code 1. +main(); diff --git a/packages/core/tests/isolation.integ.test.ts b/packages/core/tests/isolation.integ.test.ts new file mode 100644 index 00000000..c61d3f45 --- /dev/null +++ b/packages/core/tests/isolation.integ.test.ts @@ -0,0 +1,187 @@ +import { spawnSync } from "child_process"; +import { join } from "path"; +import { + FORK_SAMPLES_FILE_ENV, + ISOLATED_BENCHMARK_ENV, + PROCESS_ISOLATION_ENV, + resolveWalltimeForks, + WALLTIME_FORKS_ENV, + walltimeIsolation, +} from "../src/isolation"; + +const FIXTURE = join(__dirname, "fixtures", "isolation.cjs"); +/** Fails a run whose benchmark process got stuck instead of blocking the suite. */ +const RUN_TIMEOUT_MS = 60_000; +const CHILD_EXIT_CODE = 3; +const RAN_LINE = /^ran (\d+) (.+)$/; +const SAMPLES_LINE = /^samples (\d+)$/; +/** Passed to Node to check that forks keep the flags of their parent. */ +const NODE_FLAG = "--no-warnings"; +/** Each fork of the fixture reports two samples. */ +const SAMPLES_PER_FORK = 2; + +const ENV_KEYS = [ + PROCESS_ISOLATION_ENV, + ISOLATED_BENCHMARK_ENV, + FORK_SAMPLES_FILE_ENV, + WALLTIME_FORKS_ENV, +]; +const savedEnv = Object.fromEntries( + ENV_KEYS.map((key) => [key, process.env[key]]), +); + +function setEnv(env: Record): void { + for (const key of ENV_KEYS) { + delete process.env[key]; + } + for (const [key, value] of Object.entries(env)) { + if (value !== undefined) { + process.env[key] = value; + } + } +} + +afterEach(() => setEnv(savedEnv)); + +describe("resolveWalltimeForks", () => { + it("defaults to 3 forks", () => { + setEnv({}); + expect(resolveWalltimeForks()).toBe(3); + }); + + it("takes the option over the default", () => { + setEnv({}); + expect(resolveWalltimeForks(5)).toBe(5); + }); + + it("takes the env over the option", () => { + setEnv({ [WALLTIME_FORKS_ENV]: "2" }); + expect(resolveWalltimeForks(5)).toBe(2); + }); + + it("ignores an empty env", () => { + setEnv({ [WALLTIME_FORKS_ENV]: "" }); + expect(resolveWalltimeForks(5)).toBe(5); + }); + + it.each(["0", "-1", "1.5", "abc"])("rejects the env %p", (value) => { + setEnv({ [WALLTIME_FORKS_ENV]: value }); + expect(() => resolveWalltimeForks()).toThrow(WALLTIME_FORKS_ENV); + }); + + it.each([0, -1, 1.5, NaN])("rejects the option %p", (value) => { + setEnv({}); + expect(() => resolveWalltimeForks(value)).toThrow("forks"); + }); +}); + +describe("walltimeIsolation", () => { + it.each([ + [undefined, "Forks"], + ["1", "Forks"], + ["true", "Forks"], + ["0", "SharedProcess"], + ["false", "SharedProcess"], + ])("selects %s as %s", (value, kind) => { + setEnv({ [PROCESS_ISOLATION_ENV]: value }); + expect(walltimeIsolation().kind).toBe(kind); + }); + + it("runs everything in this process when isolation is off, whatever the fork count", () => { + setEnv({ [PROCESS_ISOLATION_ENV]: "false", [WALLTIME_FORKS_ENV]: "4" }); + expect(walltimeIsolation(2).kind).toBe("SharedProcess"); + }); + + it("restricts a fork to its benchmark, whatever the isolation setting", () => { + setEnv({ + [PROCESS_ISOLATION_ENV]: "false", + [ISOLATED_BENCHMARK_ENV]: "file::bench", + [FORK_SAMPLES_FILE_ENV]: "samples.json", + }); + expect(walltimeIsolation()).toEqual({ + kind: "Fork", + uri: "file::bench", + samplesFile: "samples.json", + }); + }); +}); + +interface FixtureOptions { + forks: number; + action?: "exit" | "no-samples"; + sync?: boolean; +} + +function runFixture({ forks, action, sync }: FixtureOptions) { + const env: NodeJS.ProcessEnv = { + ...process.env, + ISOLATION_FIXTURE_ACTION: action, + ISOLATION_FIXTURE_FORKS: String(forks), + }; + for (const key of ENV_KEYS) { + delete env[key]; + } + if (sync) { + env.ISOLATION_FIXTURE_SYNC = "1"; + } + const result = spawnSync(process.execPath, [NODE_FLAG, FIXTURE], { + env, + encoding: "utf8", + timeout: RUN_TIMEOUT_MS, + }); + const lines = result.stdout.split("\n"); + const ran = lines + .map((line) => RAN_LINE.exec(line)) + .filter((match): match is RegExpExecArray => match !== null); + const samples = lines + .map((line) => SAMPLES_LINE.exec(line)) + .find((match) => match !== null); + return { + status: result.status, + stderr: result.stderr, + ran, + samples: samples ? Number(samples[1]) : undefined, + }; +} + +describe.each([ + { name: "async", sync: false }, + { name: "sync", sync: true }, +])("runForks ($name run)", ({ sync }) => { + it("runs every fork in a fresh process with the flags of its parent and pools their samples", () => { + const forks = 3; + const { status, ran, samples } = runFixture({ forks, sync }); + + expect(status).toBe(0); + expect(ran).toHaveLength(forks); + expect(new Set(ran.map((match) => match[1])).size).toBe(forks); + for (const match of ran) { + expect(JSON.parse(match[2])).toEqual([NODE_FLAG]); + } + expect(samples).toBe(forks * SAMPLES_PER_FORK); + }); + + it("fails on a failing fork and runs no further fork", () => { + const { status, stderr, ran, samples } = runFixture({ + forks: 3, + sync, + action: "exit", + }); + + expect(status).not.toBe(0); + expect(stderr).toContain(`failed with exit code ${CHILD_EXIT_CODE}`); + expect(ran).toHaveLength(1); + expect(samples).toBeUndefined(); + }); + + it("fails when a fork exits without reporting its samples", () => { + const { status, stderr } = runFixture({ + forks: 2, + sync, + action: "no-samples", + }); + + expect(status).not.toBe(0); + expect(stderr).toContain("did not report its samples"); + }); +}); From 4fa793696bab9dc418f31e449b5bf37d2619a6f5 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 29 Oct 2026 12:09:50 +0100 Subject: [PATCH 2/8] feat(tinybench): run walltime benchmarks in forks `withCodSpeed(bench, { forks })` sets the fork count of walltime runs, and `CODSPEED_WALLTIME_FORKS` overrides it. The parent runs each task in its forks and reports their pooled rounds; a fork runs only its task, hands its samples over and prints no banner or summary. Simulation runs are unchanged. Refs COD-3960 --- packages/tinybench-plugin/src/index.ts | 8 +- packages/tinybench-plugin/src/shared.ts | 51 ++++- packages/tinybench-plugin/src/walltime.ts | 52 +++++- .../tests/fixtures/isolation.mjs | 25 +++ .../tests/isolation.integ.test.ts | 175 ++++++++++++++++++ 5 files changed, 292 insertions(+), 19 deletions(-) create mode 100644 packages/tinybench-plugin/tests/fixtures/isolation.mjs create mode 100644 packages/tinybench-plugin/tests/isolation.integ.test.ts diff --git a/packages/tinybench-plugin/src/index.ts b/packages/tinybench-plugin/src/index.ts index 298f97b9..bb903ae0 100644 --- a/packages/tinybench-plugin/src/index.ts +++ b/packages/tinybench-plugin/src/index.ts @@ -8,6 +8,7 @@ import { SetupInstrumentsResponse, tryIntrospect, wrapWithRootFrameSync, + type WalltimeOptions, } from "@codspeed/core"; import { Bench } from "tinybench"; import { setupCodspeedAnalysisBench } from "./analysis"; @@ -17,7 +18,10 @@ import { setupCodspeedWalltimeBench } from "./walltime"; tryIntrospect(); -export function withCodSpeed(bench: Bench): Bench { +export function withCodSpeed( + bench: Bench, + options: WalltimeOptions = {}, +): Bench { const codspeedRunnerMode = getCodspeedRunnerMode(); if (codspeedRunnerMode === "disabled") { return bench; @@ -52,7 +56,7 @@ export function withCodSpeed(bench: Bench): Bench { if (instrumentMode === "analysis") { setupCodspeedAnalysisBench(bench, rootCallingFile); } else if (instrumentMode === "walltime") { - setupCodspeedWalltimeBench(bench, rootCallingFile); + setupCodspeedWalltimeBench(bench, rootCallingFile, options); } return bench; diff --git a/packages/tinybench-plugin/src/shared.ts b/packages/tinybench-plugin/src/shared.ts index 0fb811c5..9c95324f 100644 --- a/packages/tinybench-plugin/src/shared.ts +++ b/packages/tinybench-plugin/src/shared.ts @@ -1,4 +1,4 @@ -import { setupCore, teardownCore } from "@codspeed/core"; +import { setupCore, teardownCore, type IsolationMode } from "@codspeed/core"; import { Bench, Task } from "tinybench"; import { CapturedTaskData, getTaskData } from "./taskData"; import { getTaskUri } from "./uri"; @@ -8,10 +8,21 @@ declare const __VERSION__: string; export abstract class BaseBenchRunner { protected bench: Bench; protected rootCallingFile: string; - - constructor(bench: Bench, rootCallingFile: string) { + /** Walltime runners pass the isolation of this process; analysis shares it. */ + protected readonly mode: IsolationMode; + + constructor( + bench: Bench, + rootCallingFile: string, + mode: IsolationMode = { kind: "SharedProcess" }, + ) { this.bench = bench; this.rootCallingFile = rootCallingFile; + this.mode = mode; + } + + protected get isChild(): boolean { + return this.mode.kind === "Fork"; } private setupBenchRun(): void { @@ -20,6 +31,10 @@ export abstract class BaseBenchRunner { } private logStart(): void { + // The parent already printed the banner, children stay quiet. + if (this.isChild) { + return; + } console.log( `[CodSpeed] running with @codspeed/tinybench v${__VERSION__} (${this.getModeName()})`, ); @@ -45,7 +60,12 @@ export abstract class BaseBenchRunner { protected finalizeBenchRun(): Task[] { teardownCore(); - console.log(`[CodSpeed] Done running ${this.bench.tasks.length} benches.`); + // The parent already printed the summary line, children stay quiet. + if (!this.isChild) { + console.log( + `[CodSpeed] Done running ${this.bench.tasks.length} benches.`, + ); + } return this.bench.tasks; } @@ -57,10 +77,15 @@ export abstract class BaseBenchRunner { public setupBenchMethods(): void { this.bench.run = async () => { + const tasks = this.selectTasks(); + // A child that owns none of this bench's tasks must not touch the + // instruments, the bench that owns its task does. + if (this.isChild && tasks.length === 0) { + return this.bench.tasks; + } this.setupBenchRun(); - for (const task of this.bench.tasks) { - const uri = this.getTaskUri(task); + for (const { task, uri } of tasks) { await this.runTaskAsync(task, uri); } @@ -68,14 +93,24 @@ export abstract class BaseBenchRunner { }; this.bench.runSync = () => { + const tasks = this.selectTasks(); + if (this.isChild && tasks.length === 0) { + return this.bench.tasks; + } this.setupBenchRun(); - for (const task of this.bench.tasks) { - const uri = this.getTaskUri(task); + for (const { task, uri } of tasks) { this.runTaskSync(task, uri); } return this.finalizeSyncRun(); }; } + + /** The tasks this process measures: all of them, or the isolated one. */ + private selectTasks(): { task: Task; uri: string }[] { + return this.bench.tasks + .map((task) => ({ task, uri: this.getTaskUri(task) })) + .filter(({ uri }) => this.mode.kind !== "Fork" || this.mode.uri === uri); + } } diff --git a/packages/tinybench-plugin/src/walltime.ts b/packages/tinybench-plugin/src/walltime.ts index 90e6b700..7ee4c863 100644 --- a/packages/tinybench-plugin/src/walltime.ts +++ b/packages/tinybench-plugin/src/walltime.ts @@ -1,24 +1,33 @@ import { calculateQuantiles, InstrumentHooks, + walltimeIsolation, MARKER_TYPE_BENCHMARK_END, MARKER_TYPE_BENCHMARK_START, mongoMeasurement, msToNs, msToS, + runForks, + runForksSync, + samplesToResult, + writeForkSamples, writeWalltimeResults, type BenchmarkStats, type Benchmark as CodspeedBenchmark, + type RoundsResult, + type WalltimeOptions, } from "@codspeed/core"; -import { Bench, Task, TaskResult } from "tinybench"; +import { Bench, Task } from "tinybench"; import { getBenchOptions } from "./benchOptions"; import { BaseBenchRunner } from "./shared"; export function setupCodspeedWalltimeBench( bench: Bench, rootCallingFile: string, + options: WalltimeOptions, ): void { - const runner = new WalltimeBenchRunner(bench, rootCallingFile); + const mode = walltimeIsolation(options.forks); + const runner = new WalltimeBenchRunner(bench, rootCallingFile, mode); runner.installInstrumentHooks(); runner.setupBenchMethods(); } @@ -91,6 +100,12 @@ class WalltimeBenchRunner extends BaseBenchRunner { } protected async runTaskAsync(task: Task, uri: string): Promise { + const { mode } = this; + if (mode.kind === "Forks") { + const samples = await runForks(uri, mode.forks); + return this.registerCodspeedBenchmark(task, samplesToResult(samples)); + } + // run the warmup of the task right before its actual run if (getBenchOptions(this.bench).warmup) { await task.warmup(); @@ -103,8 +118,13 @@ class WalltimeBenchRunner extends BaseBenchRunner { this.registerCodspeedBenchmarkFromTask(task); } - // eslint-disable-next-line @typescript-eslint/no-unused-vars - protected runTaskSync(task: Task, _uri: string): void { + protected runTaskSync(task: Task, uri: string): void { + const { mode } = this; + if (mode.kind === "Forks") { + const samples = runForksSync(uri, mode.forks); + return this.registerCodspeedBenchmark(task, samplesToResult(samples)); + } + if (getBenchOptions(this.bench).warmup) { task.warmupSync(); } @@ -123,19 +143,28 @@ class WalltimeBenchRunner extends BaseBenchRunner { } private registerCodspeedBenchmarkFromTask(task: Task): void { - const uri = this.getTaskUri(task); - if (!task.result) { - console.warn(` ⚠ No result data available for ${uri}`); + console.warn( + ` ⚠ No result data available for ${this.getTaskUri(task)}`, + ); return; } + // A fork hands its samples to the parent, which reports the benchmark. + if (this.mode.kind === "Fork") { + writeForkSamples(this.mode.samplesFile, [...task.result.latency.samples]); + return; + } + this.registerCodspeedBenchmark(task, task.result); + } + private registerCodspeedBenchmark(task: Task, result: RoundsResult): void { + const uri = this.getTaskUri(task); const opts = getBenchOptions(this.bench); const warmupIterations = opts.warmup ? (opts.warmupIterations ?? TINYBENCH_WARMUP_DEFAULT) : 0; const stats = convertTinybenchResultToBenchmarkStats( - task.result, + result, warmupIterations, ); @@ -156,6 +185,11 @@ class WalltimeBenchRunner extends BaseBenchRunner { } private finalizeWalltimeRun(isAsync: boolean): Task[] { + // A fork only hands its samples to the parent, which reports the results. + if (this.isChild) { + return this.bench.tasks; + } + // Write results to JSON file using core function if (this.codspeedBenchmarks.length > 0) { writeWalltimeResults(this.codspeedBenchmarks, isAsync); @@ -171,7 +205,7 @@ class WalltimeBenchRunner extends BaseBenchRunner { const TINYBENCH_WARMUP_DEFAULT = 16; function convertTinybenchResultToBenchmarkStats( - result: TaskResult, + result: RoundsResult, warmupIterations: number, ): BenchmarkStats { const { min, max, mean, sd, samples } = result.latency; diff --git a/packages/tinybench-plugin/tests/fixtures/isolation.mjs b/packages/tinybench-plugin/tests/fixtures/isolation.mjs new file mode 100644 index 00000000..6f59d933 --- /dev/null +++ b/packages/tinybench-plugin/tests/fixtures/isolation.mjs @@ -0,0 +1,25 @@ +// Run by `isolation.integ.test.ts`. Prints the PID of every process that runs a benchmark. +import { Bench } from "tinybench"; +import { withCodSpeed } from "../../dist/index.es5.js"; + +const options = process.env.ISOLATION_FIXTURE_FORKS_OPTION; +const bench = withCodSpeed( + new Bench({ iterations: 4, time: 1, warmupTime: 0 }), + options === undefined ? {} : { forks: Number(options) }, +); + +for (const name of ["first", "second"]) { + // tinybench calls the function once when it is added, in every process. + let calls = 0; + bench.add(name, () => { + if (++calls === 2) { + console.log(`ran ${process.pid} ${name}`); + } + }); +} + +if (process.env.ISOLATION_FIXTURE_SYNC !== undefined) { + bench.runSync(); +} else { + await bench.run(); +} diff --git a/packages/tinybench-plugin/tests/isolation.integ.test.ts b/packages/tinybench-plugin/tests/isolation.integ.test.ts new file mode 100644 index 00000000..def38845 --- /dev/null +++ b/packages/tinybench-plugin/tests/isolation.integ.test.ts @@ -0,0 +1,175 @@ +import { getV8Flags } from "@codspeed/core"; +import { spawn } from "child_process"; +import { once } from "events"; +import { mkdtempSync, readdirSync, readFileSync, rmSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; +import { fileURLToPath } from "url"; +import { describe, expect, it } from "vitest"; + +const FIXTURE = fileURLToPath( + new URL("./fixtures/isolation.mjs", import.meta.url), +); +const PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION"; +const WALLTIME_FORKS_ENV = "CODSPEED_WALLTIME_FORKS"; +const BENCHMARKS = ["first", "second"]; +const DEFAULT_FORKS = 3; +/** The fixture's `iterations`: every fork measures at least this many rounds. */ +const MIN_ROUNDS_PER_FORK = 4; +/** Fails a run whose benchmark process got stuck instead of blocking the suite. */ +const RUN_TIMEOUT_MS = 60_000; +const RAN_LINE = /^ran (\d+) (.+)$/; + +interface FixtureRun { + status: number | null; + /** PIDs that ran each benchmark. */ + pidsByName: Map>; + /** PID of the process the test started. */ + parentPid: number; + /** Rounds of each benchmark in the results file. */ + roundsByName: Map; +} + +interface FixtureOptions { + /** `CODSPEED_PROCESS_ISOLATION=false`: every benchmark shares our process. */ + shared?: boolean; + /** `forks` option of `withCodSpeed`. */ + forksOption?: number; + /** `CODSPEED_WALLTIME_FORKS`. */ + forksEnv?: number; + sync?: boolean; +} + +async function runFixture({ + shared, + forksOption, + forksEnv, + sync, +}: FixtureOptions = {}): Promise { + const profileFolder = mkdtempSync(join(tmpdir(), "codspeed-tinybench-")); + try { + const env: NodeJS.ProcessEnv = { + ...process.env, + CODSPEED_ENV: "true", + CODSPEED_RUNNER_MODE: "walltime", + CODSPEED_PROFILE_FOLDER: profileFolder, + }; + delete env[PROCESS_ISOLATION_ENV]; + delete env[WALLTIME_FORKS_ENV]; + if (shared) { + env[PROCESS_ISOLATION_ENV] = "false"; + } + if (forksOption !== undefined) { + env.ISOLATION_FIXTURE_FORKS_OPTION = String(forksOption); + } + if (forksEnv !== undefined) { + env[WALLTIME_FORKS_ENV] = String(forksEnv); + } + if (sync) { + env.ISOLATION_FIXTURE_SYNC = "1"; + } + + // The runner starts Node with these flags, and forks must keep them. Not + // spawnSync: blocking the test worker times out its calls to Vitest. + const child = spawn(process.execPath, [...getV8Flags(), FIXTURE], { + env, + timeout: RUN_TIMEOUT_MS, + }); + let stdout = ""; + child.stdout.setEncoding("utf8").on("data", (chunk) => (stdout += chunk)); + const [status] = (await once(child, "close")) as [number | null]; + + const pidsByName = new Map>(); + for (const line of stdout.split("\n")) { + const match = RAN_LINE.exec(line); + if (!match) continue; + const pids = pidsByName.get(match[2]) ?? new Set(); + pids.add(Number(match[1])); + pidsByName.set(match[2], pids); + } + + const roundsByName = new Map(); + const resultsDir = join(profileFolder, "results"); + for (const file of readdirSync(resultsDir)) { + const data = JSON.parse(readFileSync(join(resultsDir, file), "utf8")); + for (const benchmark of data.benchmarks) { + roundsByName.set(benchmark.name, benchmark.stats.rounds); + } + } + return { status, pidsByName, parentPid: child.pid!, roundsByName }; + } finally { + rmSync(profileFolder, { recursive: true, force: true }); + } +} + +/** PIDs that ran a benchmark besides the one we started. */ +function forkPids(run: FixtureRun, name: string): Set { + const pids = new Set(run.pidsByName.get(name)); + pids.delete(run.parentPid); + return pids; +} + +describe("walltime forks", () => { + it.each([ + { name: "async", sync: false }, + { name: "sync", sync: true }, + ])( + "runs every benchmark in 3 forks by default and pools their rounds ($name run)", + async ({ sync }) => { + const run = await runFixture({ sync }); + + expect(run.status).toBe(0); + for (const name of BENCHMARKS) { + expect(forkPids(run, name).size).toBe(DEFAULT_FORKS); + expect(run.roundsByName.get(name)).toBeGreaterThanOrEqual( + DEFAULT_FORKS * MIN_ROUNDS_PER_FORK, + ); + } + // Forks of different benchmarks are different processes too. + const all = new Set( + BENCHMARKS.flatMap((name) => [...forkPids(run, name)]), + ); + expect(all.size).toBe(DEFAULT_FORKS * BENCHMARKS.length); + }, + RUN_TIMEOUT_MS, + ); + + it( + "takes the fork count from the withCodSpeed option", + async () => { + const run = await runFixture({ forksOption: 2 }); + + expect(run.status).toBe(0); + for (const name of BENCHMARKS) { + expect(forkPids(run, name).size).toBe(2); + } + }, + RUN_TIMEOUT_MS, + ); + + it( + "takes the fork count from the environment over the option", + async () => { + const run = await runFixture({ forksOption: 2, forksEnv: 1 }); + + expect(run.status).toBe(0); + for (const name of BENCHMARKS) { + expect(forkPids(run, name).size).toBe(1); + } + }, + RUN_TIMEOUT_MS, + ); + + it( + "runs every benchmark in the parent process when isolation is disabled", + async () => { + const run = await runFixture({ shared: true, forksEnv: 4 }); + + expect(run.status).toBe(0); + for (const name of BENCHMARKS) { + expect(run.pidsByName.get(name)).toEqual(new Set([run.parentPid])); + } + }, + RUN_TIMEOUT_MS, + ); +}); From e9af7a19c940873d575265e50c7e1dab10a202db Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 29 Oct 2026 12:09:50 +0100 Subject: [PATCH 3/8] feat(vitest): run walltime benchmarks in forks With Vitest 5, each walltime benchmark runs in fresh `vitest bench` processes restricted to its file. The fork count resolves as `CODSPEED_WALLTIME_FORKS`, the benchmark's or run's `forks` option, `codspeedPlugin({ forks })`, then 3. The test timeout is scaled by the env or plugin fork count, since every fork of a test runs within it. Vitest's own report shows zeroed statistics for forked benchmarks; the CodSpeed results hold the pooled rounds. Vitest 3/4 don't fork. Closes COD-3960 --- .../src/__tests__/isolation.test.ts | 159 ++++++++++++++++++ packages/vitest-plugin/src/index.ts | 38 ++++- packages/vitest-plugin/src/instrument.ts | 12 ++ packages/vitest-plugin/src/v5/provider.ts | 84 ++++++++- packages/vitest-plugin/src/vitestBackend.ts | 33 +++- .../tests/fixtures/isolation/pids.bench.ts | 38 +++++ .../tests/fixtures/isolation/vitest.config.ts | 14 ++ packages/vitest-plugin/vitest.config.ts | 3 + 8 files changed, 369 insertions(+), 12 deletions(-) create mode 100644 packages/vitest-plugin/src/__tests__/isolation.test.ts create mode 100644 packages/vitest-plugin/tests/fixtures/isolation/pids.bench.ts create mode 100644 packages/vitest-plugin/tests/fixtures/isolation/vitest.config.ts diff --git a/packages/vitest-plugin/src/__tests__/isolation.test.ts b/packages/vitest-plugin/src/__tests__/isolation.test.ts new file mode 100644 index 00000000..0169373e --- /dev/null +++ b/packages/vitest-plugin/src/__tests__/isolation.test.ts @@ -0,0 +1,159 @@ +import { spawn } from "child_process"; +import { once } from "events"; +import { + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + writeFileSync, +} from "fs"; +import { createRequire } from "module"; +import { tmpdir } from "os"; +import { dirname, join } from "path"; +import { fileURLToPath } from "url"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +const FIXTURE_DIR = fileURLToPath( + new URL("../../tests/fixtures/isolation", import.meta.url), +); +const VITEST_BIN = join( + dirname(createRequire(import.meta.url).resolve("vitest/package.json")), + "vitest.mjs", +); +const PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION"; +const WALLTIME_FORKS_ENV = "CODSPEED_WALLTIME_FORKS"; +const DEFAULT_FORKS = 3; +/** The fixture's `iterations`: every fork measures at least this many rounds. */ +const MIN_ROUNDS_PER_FORK = 4; +const RUN_TIMEOUT_MS = 180_000; +const RAN_LINE = /^ran (\d+) (.+)$/; + +let tmpDir: string; + +beforeEach(() => { + tmpDir = mkdtempSync(join(tmpdir(), "codspeed-isolation-")); +}); + +afterEach(() => { + rmSync(tmpDir, { recursive: true, force: true }); +}); + +interface FixtureOptions { + isolation?: string; + forksEnv?: number; + pluginForks?: number; + firstForks?: number; + failing?: boolean; +} + +async function runFixture(options: FixtureOptions = {}) { + const logFile = join(tmpDir, "ran.log"); + writeFileSync(logFile, ""); + const profileFolder = join(tmpDir, "profile"); + const env: NodeJS.ProcessEnv = { + ...process.env, + CODSPEED_ENV: "true", + CODSPEED_RUNNER_MODE: "walltime", + CODSPEED_PROFILE_FOLDER: profileFolder, + ISOLATION_LOG_FILE: logFile, + [PROCESS_ISOLATION_ENV]: options.isolation, + [WALLTIME_FORKS_ENV]: options.forksEnv?.toString(), + ISOLATION_FIXTURE_PLUGIN_FORKS: options.pluginForks?.toString(), + ISOLATION_FIXTURE_FIRST_FORKS: options.firstForks?.toString(), + ISOLATION_FIXTURE_FAIL: options.failing ? "1" : undefined, + }; + + // Not spawnSync: blocking this worker times out its calls to Vitest. + const child = spawn(process.execPath, [VITEST_BIN, "bench", "--run"], { + cwd: FIXTURE_DIR, + env, + timeout: RUN_TIMEOUT_MS, + stdio: "ignore", + }); + const [status] = (await once(child, "close")) as [number | null]; + + const pidsByName = new Map>(); + for (const line of readFileSync(logFile, "utf8").split("\n")) { + const match = RAN_LINE.exec(line); + if (!match) continue; + const pids = pidsByName.get(match[2]) ?? new Set(); + pids.add(Number(match[1])); + pidsByName.set(match[2], pids); + } + + const roundsByName = new Map(); + const resultsDir = join(profileFolder, "results"); + for (const file of status === 0 ? readdirSync(resultsDir) : []) { + const data = JSON.parse(readFileSync(join(resultsDir, file), "utf8")); + for (const benchmark of data.benchmarks) { + roundsByName.set(benchmark.name, benchmark.stats.rounds); + } + } + return { status, pidsByName, roundsByName }; +} + +describe("walltime forks", () => { + it( + "runs every benchmark in 3 forks by default and pools their rounds", + async () => { + const run = await runFixture(); + + expect(run.status).toBe(0); + for (const name of ["first", "second"]) { + expect(run.pidsByName.get(name)?.size).toBe(DEFAULT_FORKS); + expect(run.roundsByName.get(name)).toBeGreaterThanOrEqual( + DEFAULT_FORKS * MIN_ROUNDS_PER_FORK, + ); + } + }, + RUN_TIMEOUT_MS, + ); + + it( + "takes the benchmark's option over the plugin's", + async () => { + const run = await runFixture({ firstForks: 2, pluginForks: 1 }); + + expect(run.status).toBe(0); + expect(run.pidsByName.get("first")?.size).toBe(2); + expect(run.pidsByName.get("second")?.size).toBe(1); + }, + RUN_TIMEOUT_MS, + ); + + it( + "takes the environment over the benchmark's option", + async () => { + const run = await runFixture({ firstForks: 2, forksEnv: 1 }); + + expect(run.status).toBe(0); + expect(run.pidsByName.get("first")?.size).toBe(1); + expect(run.pidsByName.get("second")?.size).toBe(1); + }, + RUN_TIMEOUT_MS, + ); + + it( + "runs every benchmark in one process when isolation is disabled", + async () => { + const run = await runFixture({ isolation: "false" }); + + expect(run.status).toBe(0); + const pids = new Set( + [...run.pidsByName.values()].flatMap((set) => [...set]), + ); + expect(pids.size).toBe(1); + }, + RUN_TIMEOUT_MS, + ); + + it( + "fails the run when a benchmark throws in its fork", + async () => { + const run = await runFixture({ failing: true, forksEnv: 1 }); + + expect(run.status).not.toBe(0); + }, + RUN_TIMEOUT_MS, + ); +}); diff --git a/packages/vitest-plugin/src/index.ts b/packages/vitest-plugin/src/index.ts index cb3af1ae..ea83d567 100644 --- a/packages/vitest-plugin/src/index.ts +++ b/packages/vitest-plugin/src/index.ts @@ -1,14 +1,19 @@ import { + getGitDir, getInstrumentMode, getV8Flags, InstrumentHooks, + ISOLATED_BENCHMARK_ENV, mongoMeasurement, + recordIsolationCommand, SetupInstrumentsRequestBody, SetupInstrumentsResponse, + type WalltimeOptions, } from "@codspeed/core"; import { join } from "path"; import { Plugin } from "vite"; import { type ViteUserConfig } from "vitest/config"; +import { PLUGIN_FORKS_ENV } from "./instrument"; import { resolveVitestBackend } from "./vitestBackend"; // get this file's directory path from import.meta.url @@ -21,7 +26,11 @@ function resolveFile(name: string): string { return join(__dirname, `${name}.${fileExtension}`); } -export default function codspeedPlugin(): Plugin { +/** + * `options.forks`: fresh processes per walltime benchmark (Vitest 5). A + * benchmark's own `forks` option and `CODSPEED_WALLTIME_FORKS` override it. + */ +export default function codspeedPlugin(options: WalltimeOptions = {}): Plugin { return { name: "codspeed:vitest", apply(_, { mode }) { @@ -37,7 +46,29 @@ export default function codspeedPlugin(): Plugin { return true; }, enforce: "post", - config(userConfig): ViteUserConfig { + config(userConfig: ViteUserConfig): ViteUserConfig { + // Vitest measures in forked workers, whose argv is not the command that + // starts the benchmarks. Record the command of this main process so the + // workers can run it again for each benchmark. Workers inherit the env. + recordIsolationCommand(); + if (options.forks !== undefined) { + process.env[PLUGIN_FORKS_ENV] = String(options.forks); + } + const uri = process.env[ISOLATED_BENCHMARK_ENV]; + if (uri !== undefined) { + const gitDir = getGitDir(userConfig.root ?? process.cwd()); + if (gitDir === undefined) { + throw new Error("Could not find a git repository"); + } + // Replace rather than merge include globs: isolated children must not + // transform and import unrelated benchmark files. + const file = join(gitDir, uri.split("::")[0]) + .replace(/\\/g, "/") + .replace(/[\\*?{}()[\]!+@]/g, "\\$&"); + userConfig.test ??= {}; + userConfig.test.benchmark ??= {}; + userConfig.test.benchmark.include = [file]; + } return { test: { pool: "forks", @@ -45,7 +76,8 @@ export default function codspeedPlugin(): Plugin { ...resolveVitestBackend().getBenchmarkTestConfig( getV8Flags(), resolveFile, - (userConfig as ViteUserConfig).test, + userConfig.test, + options, ), }, }; diff --git a/packages/vitest-plugin/src/instrument.ts b/packages/vitest-plugin/src/instrument.ts index 24039088..77adabdb 100644 --- a/packages/vitest-plugin/src/instrument.ts +++ b/packages/vitest-plugin/src/instrument.ts @@ -51,6 +51,18 @@ export interface TinybenchOptions { iterations?: number; } +/** + * The `forks` option of the plugin, set in the main process for the workers, + * which inherit its env. Not {@link WALLTIME_FORKS_ENV}, which overrides the + * per-benchmark option instead of yielding to it. + */ +export const PLUGIN_FORKS_ENV = "CODSPEED_VITEST_PLUGIN_FORKS"; + +export function getPluginForks(): number | undefined { + const value = process.env[PLUGIN_FORKS_ENV]; + return value === undefined ? undefined : Number(value); +} + /** * Drive the instrumentation window from each bench's run-mode setup/teardown * hooks so it brackets only tinybench's measured loop, excluding the warmup diff --git a/packages/vitest-plugin/src/v5/provider.ts b/packages/vitest-plugin/src/v5/provider.ts index ef38fe0e..8e408495 100644 --- a/packages/vitest-plugin/src/v5/provider.ts +++ b/packages/vitest-plugin/src/v5/provider.ts @@ -2,13 +2,19 @@ import { getGitDir, getInstrumentMode, InstrumentHooks, + walltimeIsolation, optimizeFunction, optimizeFunctionSync, + resolveWalltimeForks, + runForks, + samplesToResult, setupCore, teardownCore, wrapWithRootFrame, wrapWithRootFrameSync, + writeForkSamples, type Benchmark, + type WalltimeOptions, } from "@codspeed/core"; import { createRequire } from "module"; import path from "path"; @@ -20,6 +26,7 @@ import type { BenchRunOptions, } from "vitest"; import { + getPluginForks, installInstrumentHooks, tinybenchTaskToBenchmark, writeAndLogWalltimeResults, @@ -221,6 +228,15 @@ async function runWalltime( getUri: (name: string) => string, test: BenchmarkGroup["test"], ): Promise { + const mode = walltimeIsolation(); + if (mode.kind === "Forks") { + return runForkParent(registrations, options, getUri); + } + // A fork measures only its benchmark; Vitest still gets every result. + const measured = registrations.filter( + ({ name }) => mode.kind !== "Fork" || mode.uri === getUri(name), + ); + const { Bench } = await importHostTinybench(); const bench = new Bench({ signal: test.context.signal, @@ -229,7 +245,7 @@ async function runWalltime( retainSamples: true, }); - for (const { name, fn, fnOpts } of registrations) { + for (const { name, fn, fnOpts } of measured) { // tinybench v6 keeps `fn` private, so the root frame is baked in here bench.add( name, @@ -242,9 +258,56 @@ async function runWalltime( const tasks = await bench.run(); throwOnErroredTasks(tasks); - collectWalltimeResults(bench, getUri); + if (mode.kind === "Fork") { + for (const task of tasks) { + writeForkSamples(mode.samplesFile, [ + ...(task.result?.latency.samples ?? []), + ]); + } + } else { + collectWalltimeResults(bench, getUri); + } - return tasks.map(tinybenchTaskToBenchResult); + const results = new Map( + tasks.map((task) => [task.name, tinybenchTaskToBenchResult(task)]), + ); + return registrations.map( + ({ name }) => results.get(name) ?? analysisResult(name), + ); +} + +/** + * Measures every benchmark in fresh processes and reports their pooled + * rounds. Vitest gets zeroed results, like in analysis mode. + */ +async function runForkParent( + registrations: BenchRegistrationInput[], + options: BenchRunOptions | undefined, + getUri: (name: string) => string, +): Promise { + const { Bench } = await importHostTinybench(); + // Never run: it resolves tinybench's defaults for the reported config. + const bench = new Bench(options); + const benchmarks: Benchmark[] = []; + for (const { name, fnOpts } of registrations) { + const uri = getUri(name); + const forks = resolveWalltimeForks( + (fnOpts as WalltimeOptions | undefined)?.forks ?? + (options as WalltimeOptions | undefined)?.forks ?? + getPluginForks(), + ); + const samples = await runForks(uri, forks); + const benchmark = tinybenchTaskToBenchmark( + { name, result: samplesToResult(samples) }, + uri, + resolvedOptions(bench), + ); + if (benchmark) { + benchmarks.push(benchmark); + } + } + writeAndLogWalltimeResults(benchmarks); + return registrations.map(({ name }) => analysisResult(name)); } /** Fail the test like Vitest's own provider does, instead of returning empty results. */ @@ -260,11 +323,9 @@ function throwOnErroredTasks(tasks: TinybenchTask[]): void { } } -function collectWalltimeResults( - bench: TinybenchWithTasks, - getUri: (name: string) => string, -): void { - const options: TinybenchOptions = { +/** The options the bench runs with, as reported to CodSpeed. */ +function resolvedOptions(bench: TinybenchWithTasks): TinybenchOptions { + return { time: bench.time, iterations: bench.iterations, // A bench with warmup turned off keeps its warmup timings, so they'd @@ -272,6 +333,13 @@ function collectWalltimeResults( warmupTime: bench.warmup ? bench.warmupTime : 0, warmupIterations: bench.warmup ? bench.warmupIterations : 0, }; +} + +function collectWalltimeResults( + bench: TinybenchWithTasks, + getUri: (name: string) => string, +): void { + const options = resolvedOptions(bench); const benchmarks: Benchmark[] = []; for (const task of bench.tasks) { if (task.result?.state !== "completed") continue; diff --git a/packages/vitest-plugin/src/vitestBackend.ts b/packages/vitest-plugin/src/vitestBackend.ts index 46c171d7..0ec52080 100644 --- a/packages/vitest-plugin/src/vitestBackend.ts +++ b/packages/vitest-plugin/src/vitestBackend.ts @@ -1,4 +1,8 @@ -import { getInstrumentMode } from "@codspeed/core"; +import { + getInstrumentMode, + walltimeIsolation, + type WalltimeOptions, +} from "@codspeed/core"; import { join } from "path"; import { type ViteUserConfig } from "vitest/config"; import { version as vitestVersion } from "vitest/node"; @@ -11,6 +15,7 @@ export interface VitestBackend { v8Flags: string[], resolveFile: (name: string) => string, userTestConfig: ViteUserConfig["test"], + walltimeOptions: WalltimeOptions, ): ViteUserConfig["test"]; } @@ -38,6 +43,7 @@ class V5Backend implements VitestBackend { v8Flags: string[], resolveFile: (name: string) => string, userTestConfig: ViteUserConfig["test"], + walltimeOptions: WalltimeOptions, ): ViteUserConfig["test"] { // Vitest's export getter tracker runs inside the measured code; keep it off // unless the user explicitly configured it. @@ -45,6 +51,7 @@ class V5Backend implements VitestBackend { userTestConfig?.benchmark?.suppressExportGetterWarnings ?? true; return { execArgv: v8Flags, + ...forkedTestTimeout(userTestConfig?.testTimeout, walltimeOptions.forks), benchmark: { provider: resolveFile("v5/provider"), suppressExportGetterWarnings, @@ -53,6 +60,30 @@ class V5Backend implements VitestBackend { } } +/** Vitest 5 times a bench test out after max(60 s, `testTimeout`). */ +const MIN_BENCH_TEST_TIMEOUT_MS = 60_000; + +/** + * Every fork of every benchmark of a test runs within that test, so its + * timeout scales with the forks. Per-benchmark `forks` aren't known here. + */ +function forkedTestTimeout( + userTimeout: number | undefined, + forksOption: number | undefined, +): ViteUserConfig["test"] { + if (getInstrumentMode() !== "walltime") { + return {}; + } + const mode = walltimeIsolation(forksOption); + if (mode.kind !== "Forks") { + return {}; + } + return { + testTimeout: + Math.max(MIN_BENCH_TEST_TIMEOUT_MS, userTimeout ?? 0) * mode.forks, + }; +} + /** * Vitest 3/4: `vitest bench` runs in the `"benchmark"` Vite mode, and each * instrument mode has its own `test.runner` (`legacy/analysis`, `legacy/walltime`). diff --git a/packages/vitest-plugin/tests/fixtures/isolation/pids.bench.ts b/packages/vitest-plugin/tests/fixtures/isolation/pids.bench.ts new file mode 100644 index 00000000..be799958 --- /dev/null +++ b/packages/vitest-plugin/tests/fixtures/isolation/pids.bench.ts @@ -0,0 +1,38 @@ +import { appendFileSync } from "fs"; +import { describe, test, type BenchRegistration } from "vitest"; + +// Vitest swallows what workers print, so the PIDs that run each benchmark go +// to a file. `beforeAll` only runs when tinybench measures the benchmark. +function logPid(name: string) { + return () => + appendFileSync( + process.env.ISOLATION_LOG_FILE!, + `ran ${process.pid} ${name}\n`, + ); +} + +const firstForks = process.env.ISOLATION_FIXTURE_FIRST_FORKS; + +describe("isolation", () => { + test("benchmarks", async ({ bench }) => { + const benchmarks: BenchRegistration[] = [ + bench( + "first", + { + beforeAll: logPid("first"), + ...(firstForks && { forks: Number(firstForks) }), + }, + () => {}, + ), + bench("second", { beforeAll: logPid("second") }, () => {}), + ]; + if (process.env.ISOLATION_FIXTURE_FAIL !== undefined) { + benchmarks.push( + bench("failing", () => { + throw new Error("benchmark failed"); + }), + ); + } + await bench.compare(...benchmarks, { iterations: 4, time: 1 }); + }); +}); diff --git a/packages/vitest-plugin/tests/fixtures/isolation/vitest.config.ts b/packages/vitest-plugin/tests/fixtures/isolation/vitest.config.ts new file mode 100644 index 00000000..387bb2bf --- /dev/null +++ b/packages/vitest-plugin/tests/fixtures/isolation/vitest.config.ts @@ -0,0 +1,14 @@ +import { defineConfig } from "vitest/config"; +import codspeedPlugin from "../../../dist/index.mjs"; + +const pluginForks = process.env.ISOLATION_FIXTURE_PLUGIN_FORKS; + +export default defineConfig({ + plugins: [codspeedPlugin(pluginForks ? { forks: Number(pluginForks) } : {})], + define: { + __VERSION__: JSON.stringify("1.0.0"), + }, + test: { + include: ["**/*.bench.ts"], + }, +}); diff --git a/packages/vitest-plugin/vitest.config.ts b/packages/vitest-plugin/vitest.config.ts index 672e3a27..e874589b 100644 --- a/packages/vitest-plugin/vitest.config.ts +++ b/packages/vitest-plugin/vitest.config.ts @@ -10,5 +10,8 @@ export default defineConfig({ test: { exclude: ["**/tests/**/*", "**/.rollup.cache/**/*"], mockReset: true, + benchmark: { + exclude: ["**/node_modules/**", "**/tests/**/*", "**/.rollup.cache/**/*"], + }, }, }); From 7b7de07b74eb24c5958590aa51dd5421149ee221 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 30 Oct 2026 15:27:17 +0100 Subject: [PATCH 4/8] fixup! feat(core): run walltime benchmarks in forks --- packages/core/src/walltime/utils.ts | 21 +++++++++---- .../core/tests/walltimeUtils.integ.test.ts | 31 +++++++++++++++++++ 2 files changed, 46 insertions(+), 6 deletions(-) create mode 100644 packages/core/tests/walltimeUtils.integ.test.ts diff --git a/packages/core/src/walltime/utils.ts b/packages/core/src/walltime/utils.ts index 82362955..baf97181 100644 --- a/packages/core/src/walltime/utils.ts +++ b/packages/core/src/walltime/utils.ts @@ -32,12 +32,22 @@ export interface RoundsResult { } /** - * The result of round samples in milliseconds, e.g. the samples of several - * forks of one benchmark. + * The result of nonempty round samples in milliseconds, e.g. the samples of + * several forks of one benchmark. Throws if no samples were recorded. */ export function samplesToResult(samples: number[]): RoundsResult { const n = samples.length; - const totalTime = samples.reduce((sum, ms) => sum + ms, 0); + if (n === 0) { + throw new Error("[CodSpeed] Cannot compute statistics from empty samples"); + } + let totalTime = 0; + let min = Infinity; + let max = -Infinity; + for (const ms of samples) { + totalTime += ms; + if (ms < min) min = ms; + if (ms > max) max = ms; + } const mean = totalTime / n; const variance = n > 1 @@ -46,9 +56,8 @@ export function samplesToResult(samples: number[]): RoundsResult { return { totalTime, latency: { - // Not Math.min(...samples): a time-based run overflows the stack. - min: samples.reduce((a, b) => Math.min(a, b), Infinity), - max: samples.reduce((a, b) => Math.max(a, b), -Infinity), + min, + max, mean, sd: Math.sqrt(variance), samples, diff --git a/packages/core/tests/walltimeUtils.integ.test.ts b/packages/core/tests/walltimeUtils.integ.test.ts new file mode 100644 index 00000000..132b7559 --- /dev/null +++ b/packages/core/tests/walltimeUtils.integ.test.ts @@ -0,0 +1,31 @@ +import { samplesToResult } from "../src/walltime/utils"; + +describe("samplesToResult", () => { + it("computes pooled statistics", () => { + const samples = [2, 4, 4, 4, 5, 5, 7, 9]; + const { totalTime, latency } = samplesToResult(samples); + expect(totalTime).toBe(40); + expect(latency.min).toBe(2); + expect(latency.max).toBe(9); + expect(latency.mean).toBe(5); + // sample standard deviation: sqrt(32 / 7) + expect(latency.sd).toBeCloseTo(Math.sqrt(32 / 7), 10); + }); + + it("has zero deviation for a single sample", () => { + const { totalTime, latency } = samplesToResult([3]); + expect(totalTime).toBe(3); + expect(latency).toMatchObject({ min: 3, max: 3, mean: 3, sd: 0 }); + }); + + it("handles very large sample counts without overflowing the stack", () => { + const samples = new Array(500_000).fill(1); + expect(samplesToResult(samples).totalTime).toBe(500_000); + }); + + it("rejects empty samples", () => { + expect(() => samplesToResult([])).toThrow( + "[CodSpeed] Cannot compute statistics from empty samples", + ); + }); +}); From 103afb88d71d7f0032463f061b15672aeea5a145 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 30 Oct 2026 15:27:27 +0100 Subject: [PATCH 5/8] fixup! feat(tinybench): run walltime benchmarks in forks --- packages/tinybench-plugin/src/analysis.ts | 4 +++- packages/tinybench-plugin/src/shared.ts | 13 +++++++----- packages/tinybench-plugin/src/walltime.ts | 24 ++++++++++++----------- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/packages/tinybench-plugin/src/analysis.ts b/packages/tinybench-plugin/src/analysis.ts index 65bd6183..54c8d975 100644 --- a/packages/tinybench-plugin/src/analysis.ts +++ b/packages/tinybench-plugin/src/analysis.ts @@ -14,7 +14,9 @@ export function setupCodspeedAnalysisBench( bench: Bench, rootCallingFile: string, ): void { - const runner = new AnalysisBenchRunner(bench, rootCallingFile); + const runner = new AnalysisBenchRunner(bench, rootCallingFile, { + kind: "SharedProcess", + }); runner.setupBenchMethods(); } diff --git a/packages/tinybench-plugin/src/shared.ts b/packages/tinybench-plugin/src/shared.ts index 9c95324f..1e10b4bf 100644 --- a/packages/tinybench-plugin/src/shared.ts +++ b/packages/tinybench-plugin/src/shared.ts @@ -9,20 +9,20 @@ export abstract class BaseBenchRunner { protected bench: Bench; protected rootCallingFile: string; /** Walltime runners pass the isolation of this process; analysis shares it. */ - protected readonly mode: IsolationMode; + protected readonly isolationMode: IsolationMode; constructor( bench: Bench, rootCallingFile: string, - mode: IsolationMode = { kind: "SharedProcess" }, + isolationMode: IsolationMode, ) { this.bench = bench; this.rootCallingFile = rootCallingFile; - this.mode = mode; + this.isolationMode = isolationMode; } protected get isChild(): boolean { - return this.mode.kind === "Fork"; + return this.isolationMode.kind === "Fork"; } private setupBenchRun(): void { @@ -111,6 +111,9 @@ export abstract class BaseBenchRunner { private selectTasks(): { task: Task; uri: string }[] { return this.bench.tasks .map((task) => ({ task, uri: this.getTaskUri(task) })) - .filter(({ uri }) => this.mode.kind !== "Fork" || this.mode.uri === uri); + .filter( + ({ uri }) => + this.isolationMode.kind !== "Fork" || this.isolationMode.uri === uri, + ); } } diff --git a/packages/tinybench-plugin/src/walltime.ts b/packages/tinybench-plugin/src/walltime.ts index 7ee4c863..5787fa51 100644 --- a/packages/tinybench-plugin/src/walltime.ts +++ b/packages/tinybench-plugin/src/walltime.ts @@ -1,7 +1,6 @@ import { calculateQuantiles, InstrumentHooks, - walltimeIsolation, MARKER_TYPE_BENCHMARK_END, MARKER_TYPE_BENCHMARK_START, mongoMeasurement, @@ -10,6 +9,7 @@ import { runForks, runForksSync, samplesToResult, + walltimeIsolation, writeForkSamples, writeWalltimeResults, type BenchmarkStats, @@ -26,8 +26,8 @@ export function setupCodspeedWalltimeBench( rootCallingFile: string, options: WalltimeOptions, ): void { - const mode = walltimeIsolation(options.forks); - const runner = new WalltimeBenchRunner(bench, rootCallingFile, mode); + const isolationMode = walltimeIsolation(options.forks); + const runner = new WalltimeBenchRunner(bench, rootCallingFile, isolationMode); runner.installInstrumentHooks(); runner.setupBenchMethods(); } @@ -100,9 +100,9 @@ class WalltimeBenchRunner extends BaseBenchRunner { } protected async runTaskAsync(task: Task, uri: string): Promise { - const { mode } = this; - if (mode.kind === "Forks") { - const samples = await runForks(uri, mode.forks); + const { isolationMode } = this; + if (isolationMode.kind === "Forks") { + const samples = await runForks(uri, isolationMode.forks); return this.registerCodspeedBenchmark(task, samplesToResult(samples)); } @@ -119,9 +119,9 @@ class WalltimeBenchRunner extends BaseBenchRunner { } protected runTaskSync(task: Task, uri: string): void { - const { mode } = this; - if (mode.kind === "Forks") { - const samples = runForksSync(uri, mode.forks); + const { isolationMode } = this; + if (isolationMode.kind === "Forks") { + const samples = runForksSync(uri, isolationMode.forks); return this.registerCodspeedBenchmark(task, samplesToResult(samples)); } @@ -150,8 +150,10 @@ class WalltimeBenchRunner extends BaseBenchRunner { return; } // A fork hands its samples to the parent, which reports the benchmark. - if (this.mode.kind === "Fork") { - writeForkSamples(this.mode.samplesFile, [...task.result.latency.samples]); + if (this.isolationMode.kind === "Fork") { + writeForkSamples(this.isolationMode.samplesFile, [ + ...task.result.latency.samples, + ]); return; } this.registerCodspeedBenchmark(task, task.result); From 977f82daf25168b21f2b5f1ee5d6f676a84354b2 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 30 Oct 2026 15:27:27 +0100 Subject: [PATCH 6/8] fixup! feat(vitest): run walltime benchmarks in forks --- packages/vitest-plugin/src/index.ts | 44 +++-------- packages/vitest-plugin/src/vitestBackend.ts | 86 +++++++++++++++------ 2 files changed, 73 insertions(+), 57 deletions(-) diff --git a/packages/vitest-plugin/src/index.ts b/packages/vitest-plugin/src/index.ts index ea83d567..4c5a8178 100644 --- a/packages/vitest-plugin/src/index.ts +++ b/packages/vitest-plugin/src/index.ts @@ -1,11 +1,8 @@ import { - getGitDir, getInstrumentMode, getV8Flags, InstrumentHooks, - ISOLATED_BENCHMARK_ENV, mongoMeasurement, - recordIsolationCommand, SetupInstrumentsRequestBody, SetupInstrumentsResponse, type WalltimeOptions, @@ -13,8 +10,10 @@ import { import { join } from "path"; import { Plugin } from "vite"; import { type ViteUserConfig } from "vitest/config"; -import { PLUGIN_FORKS_ENV } from "./instrument"; -import { resolveVitestBackend } from "./vitestBackend"; +import { + resolveVitestBackend, + scaleBenchTimeoutToForks, +} from "./vitestBackend"; // get this file's directory path from import.meta.url const __dirname = new URL(".", import.meta.url).pathname; @@ -26,10 +25,7 @@ function resolveFile(name: string): string { return join(__dirname, `${name}.${fileExtension}`); } -/** - * `options.forks`: fresh processes per walltime benchmark (Vitest 5). A - * benchmark's own `forks` option and `CODSPEED_WALLTIME_FORKS` override it. - */ +/** `options.forks`: see {@link WalltimeOptions}; isolation requires Vitest 5. */ export default function codspeedPlugin(options: WalltimeOptions = {}): Plugin { return { name: "codspeed:vitest", @@ -47,41 +43,23 @@ export default function codspeedPlugin(options: WalltimeOptions = {}): Plugin { }, enforce: "post", config(userConfig: ViteUserConfig): ViteUserConfig { - // Vitest measures in forked workers, whose argv is not the command that - // starts the benchmarks. Record the command of this main process so the - // workers can run it again for each benchmark. Workers inherit the env. - recordIsolationCommand(); - if (options.forks !== undefined) { - process.env[PLUGIN_FORKS_ENV] = String(options.forks); - } - const uri = process.env[ISOLATED_BENCHMARK_ENV]; - if (uri !== undefined) { - const gitDir = getGitDir(userConfig.root ?? process.cwd()); - if (gitDir === undefined) { - throw new Error("Could not find a git repository"); - } - // Replace rather than merge include globs: isolated children must not - // transform and import unrelated benchmark files. - const file = join(gitDir, uri.split("::")[0]) - .replace(/\\/g, "/") - .replace(/[\\*?{}()[\]!+@]/g, "\\$&"); - userConfig.test ??= {}; - userConfig.test.benchmark ??= {}; - userConfig.test.benchmark.include = [file]; - } + const backend = resolveVitestBackend(); + backend.configureIsolation(userConfig, options); return { test: { pool: "forks", globalSetup: [resolveFile("globalSetup")], - ...resolveVitestBackend().getBenchmarkTestConfig( + ...backend.getBenchmarkTestConfig( getV8Flags(), resolveFile, userConfig.test, - options, ), }, }; }, + configureVitest({ project }) { + scaleBenchTimeoutToForks(project.config, options.forks); + }, }; } diff --git a/packages/vitest-plugin/src/vitestBackend.ts b/packages/vitest-plugin/src/vitestBackend.ts index 0ec52080..fd8a6aee 100644 --- a/packages/vitest-plugin/src/vitestBackend.ts +++ b/packages/vitest-plugin/src/vitestBackend.ts @@ -1,21 +1,27 @@ import { + getGitDir, getInstrumentMode, + recordIsolationCommand, walltimeIsolation, type WalltimeOptions, } from "@codspeed/core"; import { join } from "path"; import { type ViteUserConfig } from "vitest/config"; -import { version as vitestVersion } from "vitest/node"; +import { type ResolvedConfig, version as vitestVersion } from "vitest/node"; +import { PLUGIN_FORKS_ENV } from "./instrument"; /** The parts of the Vitest integration that depend on the installed major. */ export interface VitestBackend { isActive(mode: string): boolean; + configureIsolation( + userConfig: ViteUserConfig, + options: WalltimeOptions, + ): void; getBenchmarkTestConfig( v8Flags: string[], resolveFile: (name: string) => string, userTestConfig: ViteUserConfig["test"], - walltimeOptions: WalltimeOptions, ): ViteUserConfig["test"]; } @@ -39,11 +45,33 @@ class V5Backend implements VitestBackend { return getInstrumentMode() !== "disabled"; } + configureIsolation( + userConfig: ViteUserConfig, + options: WalltimeOptions, + ): void { + if (getInstrumentMode() !== "walltime") return; + + const isolationMode = walltimeIsolation(options.forks); + switch (isolationMode.kind) { + case "SharedProcess": + return; + case "Forks": + // Workers cannot re-launch Vitest from their own argv. + recordIsolationCommand(); + if (options.forks !== undefined) { + process.env[PLUGIN_FORKS_ENV] = String(options.forks); + } + return; + case "Fork": + restrictToBenchmarkFile(userConfig, isolationMode.uri); + return; + } + } + getBenchmarkTestConfig( v8Flags: string[], resolveFile: (name: string) => string, userTestConfig: ViteUserConfig["test"], - walltimeOptions: WalltimeOptions, ): ViteUserConfig["test"] { // Vitest's export getter tracker runs inside the measured code; keep it off // unless the user explicitly configured it. @@ -51,7 +79,6 @@ class V5Backend implements VitestBackend { userTestConfig?.benchmark?.suppressExportGetterWarnings ?? true; return { execArgv: v8Flags, - ...forkedTestTimeout(userTestConfig?.testTimeout, walltimeOptions.forks), benchmark: { provider: resolveFile("v5/provider"), suppressExportGetterWarnings, @@ -60,28 +87,36 @@ class V5Backend implements VitestBackend { } } -/** Vitest 5 times a bench test out after max(60 s, `testTimeout`). */ -const MIN_BENCH_TEST_TIMEOUT_MS = 60_000; +function restrictToBenchmarkFile( + userConfig: ViteUserConfig, + uri: string, +): void { + const gitDir = getGitDir(userConfig.root ?? process.cwd()); + if (gitDir === undefined) { + throw new Error("Could not find a git repository"); + } + // Replace rather than merge include globs: isolated children must not + // transform and import unrelated benchmark files. + const file = join(gitDir, uri.split("::")[0]) + .replace(/\\/g, "/") + .replace(/[\\*?{}()[\]!+@]/g, "\\$&"); + userConfig.test ??= {}; + userConfig.test.benchmark ??= {}; + userConfig.test.benchmark.include = [file]; +} -/** - * Every fork of every benchmark of a test runs within that test, so its - * timeout scales with the forks. Per-benchmark `forks` aren't known here. - */ -function forkedTestTimeout( - userTimeout: number | undefined, +export function scaleBenchTimeoutToForks( + config: ResolvedConfig, forksOption: number | undefined, -): ViteUserConfig["test"] { - if (getInstrumentMode() !== "walltime") { - return {}; - } - const mode = walltimeIsolation(forksOption); - if (mode.kind !== "Forks") { - return {}; - } - return { - testTimeout: - Math.max(MIN_BENCH_TEST_TIMEOUT_MS, userTimeout ?? 0) * mode.forks, - }; +): void { + if (!config.benchmark.enabled || getInstrumentMode() !== "walltime") return; + + const isolationMode = walltimeIsolation(forksOption); + if (isolationMode.kind !== "Forks") return; + + // All forks run within the registering test. Per-benchmark forks aren't + // known here; scale Vitest's resolved timeout by the integration's budget. + config.testTimeout *= isolationMode.forks; } /** @@ -95,6 +130,9 @@ class LegacyBackend implements VitestBackend { return mode === "benchmark"; } + // Vitest 3/4 do not support process isolation. + configureIsolation(): void {} + getBenchmarkTestConfig( v8Flags: string[], resolveFile: (name: string) => string, From ad0b5662a5fb194770fb4666e6f6df884711a8e5 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 30 Oct 2026 18:18:51 +0100 Subject: [PATCH 7/8] fixup! feat(core): run walltime benchmarks in forks --- packages/core/src/isolation.ts | 63 +++++++++++++++++----------------- 1 file changed, 31 insertions(+), 32 deletions(-) diff --git a/packages/core/src/isolation.ts b/packages/core/src/isolation.ts index e3403537..a924c697 100644 --- a/packages/core/src/isolation.ts +++ b/packages/core/src/isolation.ts @@ -6,7 +6,7 @@ * sizing), so the rounds of several forks are pooled into one result. */ import { spawn, spawnSync } from "child_process"; -import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "fs"; +import { readFileSync, rmSync, writeFileSync } from "fs"; import { tmpdir } from "os"; import { join } from "path"; @@ -126,37 +126,31 @@ function isolationCommand(): string[] { * calls time out otherwise. */ export async function runForks(uri: string, forks: number): Promise { - const dir = mkdtempSync(join(tmpdir(), SAMPLES_DIR_PREFIX)); - try { - let samples: number[] = []; - for (let fork = 0; fork < forks; fork++) { - // Not push(...): spreading a time-based run's samples overflows the stack. - samples = samples.concat(await runFork(uri, join(dir, `${fork}.json`))); - } - return samples; - } finally { - rmSync(dir, { recursive: true, force: true }); + let samples: number[] = []; + for (let fork = 0; fork < forks; fork++) { + // Not push(...): spreading a time-based run's samples overflows the stack. + samples = samples.concat(await runFork(uri, samplesFile())); } + return samples; } /** {@link runForks} for synchronous runs, which block anyway. */ export function runForksSync(uri: string, forks: number): number[] { - const dir = mkdtempSync(join(tmpdir(), SAMPLES_DIR_PREFIX)); - try { - let samples: number[] = []; - for (let fork = 0; fork < forks; fork++) { - samples = samples.concat(runForkSync(uri, join(dir, `${fork}.json`))); - } - return samples; - } finally { - rmSync(dir, { recursive: true, force: true }); + let samples: number[] = []; + for (let fork = 0; fork < forks; fork++) { + samples = samples.concat(runForkSync(uri, samplesFile())); } + return samples; } -const SAMPLES_DIR_PREFIX = "codspeed-forks-"; +/** Forks run one after the other, so one file per parent process suffices. */ +function samplesFile(): string { + return join(tmpdir(), `codspeed-samples-${process.pid}.json`); +} /** Runs one fork of `uri` and returns the samples it wrote to `file`. */ async function runFork(uri: string, file: string): Promise { + rmSync(file, { force: true }); // a leftover file must not pass as this fork's const [executable, ...args] = isolationCommand(); const child = spawn(executable, args, { stdio: "inherit", @@ -171,6 +165,7 @@ async function runFork(uri: string, file: string): Promise { } function runForkSync(uri: string, file: string): number[] { + rmSync(file, { force: true }); const [executable, ...args] = isolationCommand(); const result = spawnSync(executable, args, { stdio: "inherit", @@ -196,24 +191,28 @@ function startError(uri: string, error: Error): Error { ); } -/** The samples a finished fork wrote; throws if it failed. */ +/** The samples a finished fork wrote, deleting the file; throws if it failed. */ function forkSamples( uri: string, file: string, status: number | null, signal: NodeJS.Signals | null, ): number[] { - if (status !== 0) { - throw new Error( - `[CodSpeed] the process for ${uri} failed with ${signal ?? `exit code ${status}`}`, - ); - } try { - return JSON.parse(readFileSync(file, "utf-8")) as number[]; - } catch (error) { - throw new Error( - `[CodSpeed] the process for ${uri} did not report its samples: ${(error as Error).message}`, - ); + if (status !== 0) { + throw new Error( + `[CodSpeed] the process for ${uri} failed with ${signal ?? `exit code ${status}`}`, + ); + } + try { + return JSON.parse(readFileSync(file, "utf-8")) as number[]; + } catch (error) { + throw new Error( + `[CodSpeed] the process for ${uri} did not report its samples: ${(error as Error).message}`, + ); + } + } finally { + rmSync(file, { force: true }); } } From c376bfd4aa792cc15f44f051b9d86a453c56a13d Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 30 Oct 2026 18:44:13 +0100 Subject: [PATCH 8/8] fixup! feat(core): run walltime benchmarks in forks --- packages/core/src/isolation.ts | 35 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/packages/core/src/isolation.ts b/packages/core/src/isolation.ts index a924c697..105e8c84 100644 --- a/packages/core/src/isolation.ts +++ b/packages/core/src/isolation.ts @@ -129,7 +129,7 @@ export async function runForks(uri: string, forks: number): Promise { let samples: number[] = []; for (let fork = 0; fork < forks; fork++) { // Not push(...): spreading a time-based run's samples overflows the stack. - samples = samples.concat(await runFork(uri, samplesFile())); + samples = samples.concat(await runFork(uri)); } return samples; } @@ -138,50 +138,48 @@ export async function runForks(uri: string, forks: number): Promise { export function runForksSync(uri: string, forks: number): number[] { let samples: number[] = []; for (let fork = 0; fork < forks; fork++) { - samples = samples.concat(runForkSync(uri, samplesFile())); + samples = samples.concat(runForkSync(uri)); } return samples; } /** Forks run one after the other, so one file per parent process suffices. */ -function samplesFile(): string { - return join(tmpdir(), `codspeed-samples-${process.pid}.json`); -} +const SAMPLES_FILE = join(tmpdir(), `codspeed-samples-${process.pid}.json`); -/** Runs one fork of `uri` and returns the samples it wrote to `file`. */ -async function runFork(uri: string, file: string): Promise { - rmSync(file, { force: true }); // a leftover file must not pass as this fork's +/** Runs one fork of `uri` and returns the samples it wrote. */ +async function runFork(uri: string): Promise { + rmSync(SAMPLES_FILE, { force: true }); // a leftover file must not pass as this fork's const [executable, ...args] = isolationCommand(); const child = spawn(executable, args, { stdio: "inherit", - env: forkEnv(uri, file), + env: forkEnv(uri), }); const { promise, resolve, reject } = Promise.withResolvers<[number | null, NodeJS.Signals | null]>(); child.once("error", (error) => reject(startError(uri, error))); child.once("close", (status, signal) => resolve([status, signal])); const [status, signal] = await promise; - return forkSamples(uri, file, status, signal); + return forkSamples(uri, status, signal); } -function runForkSync(uri: string, file: string): number[] { - rmSync(file, { force: true }); +function runForkSync(uri: string): number[] { + rmSync(SAMPLES_FILE, { force: true }); const [executable, ...args] = isolationCommand(); const result = spawnSync(executable, args, { stdio: "inherit", - env: forkEnv(uri, file), + env: forkEnv(uri), }); if (result.error !== undefined) { throw startError(uri, result.error); } - return forkSamples(uri, file, result.status, result.signal); + return forkSamples(uri, result.status, result.signal); } -function forkEnv(uri: string, samplesFile: string): NodeJS.ProcessEnv { +function forkEnv(uri: string): NodeJS.ProcessEnv { return { ...process.env, [ISOLATED_BENCHMARK_ENV]: uri, - [FORK_SAMPLES_FILE_ENV]: samplesFile, + [FORK_SAMPLES_FILE_ENV]: SAMPLES_FILE, }; } @@ -194,7 +192,6 @@ function startError(uri: string, error: Error): Error { /** The samples a finished fork wrote, deleting the file; throws if it failed. */ function forkSamples( uri: string, - file: string, status: number | null, signal: NodeJS.Signals | null, ): number[] { @@ -205,14 +202,14 @@ function forkSamples( ); } try { - return JSON.parse(readFileSync(file, "utf-8")) as number[]; + return JSON.parse(readFileSync(SAMPLES_FILE, "utf-8")) as number[]; } catch (error) { throw new Error( `[CodSpeed] the process for ${uri} did not report its samples: ${(error as Error).message}`, ); } } finally { - rmSync(file, { force: true }); + rmSync(SAMPLES_FILE, { force: true }); } }