feat(vitest-plugin): add walltime forks and GC between rounds

CLAUDE.md

@@ -115,4 +115,24 @@ so exercise all three of these when developing or reviewing a plugin change
 Running these locally outside the CodSpeed runner is expected to log
 `instrument-hooks: failed to write environment.json` and skip actual measurement
 writes — the point is to verify the plugin's control flow and output per mode,
-not to produce real measurements.
+not to produce real measurements.
+
+## Walltime forks and GC between rounds
+
+Both are opt-in and only apply to walltime runs.
+
+- `CODSPEED_WALLTIME_FORKS=<N>` runs each benchmark in `N` fresh processes, one
+  after the other, and reports all their rounds together as one result. Unset
+  runs the benchmark in the benchmark process. Vitest 3/4 only. Like isolation,
+  each child re-runs the Vitest command with `CODSPEED_ISOLATED_BENCHMARK=<uri>`;
+  it writes its round samples to the file named by `CODSPEED_FORK_SAMPLES_FILE`
+  instead of reporting, and the parent merges them. With `N > 1`, each fork
+  runs half the warmup and measurement budget (`FORK_BUDGET_FRACTION`: half
+  the time, rounds rounded up), so forks mostly add process startup.
+- `CODSPEED_WALLTIME_GC_BETWEEN_ROUNDS=true` runs a full GC before every round
+  (warmup and measured). It adds `--expose-gc` to the V8 flags and collects in
+  the tinybench `beforeEach` hook, outside the timed call. Vitest 3/4/5. Meant
+  for benchmarks with long rounds: a GC costs milliseconds, so time-based
+  benchmarks of microsecond functions take far longer to finish. It also
+  changes absolute times (each round starts on a collected heap), so turning
+  it on re-baselines a benchmark.

docs/introspection.md

@@ -13,3 +13,13 @@
 ```bash
 CI=1 CODSPEED_DEBUG=true pnpm --filter with-typescript-esm bench-tinybench
 ```
+
+## Vitest process isolation
+
+Isolated Vitest children restrict benchmark discovery to the file named in
+`CODSPEED_ISOLATED_BENCHMARK`. Other benchmark files are not transformed or imported
+in that child. The parent still discovers the full suite, and each child still
+loads the Vitest configuration and setup files.
+
+Isolated children retain fresh processes and the profiling flags. This avoids
+repeated unrelated file imports; it does not remove Vitest's per-process startup.

examples/with-vitest-v3/src/fibonacci.bench.ts

@@ -18,3 +18,7 @@ describe("fibonacci", () => {
     iterativeFibonacci(20);
   });
 });
+
+bench("top-level recursive fibo 15", () => {
+  recursiveFibonacci(15);
+});

examples/with-vitest-v4/src/fibonacci.bench.ts

@@ -18,3 +18,7 @@ describe("fibonacci", () => {
     iterativeFibonacci(20);
   });
 });
+
+bench("top-level recursive fibo 15", () => {
+  recursiveFibonacci(15);
+});

packages/benchmark.js-plugin/src/index.ts

@@ -1,6 +1,7 @@
 import {
   getCallingFile,
   InstrumentHooks,
+  Isolation,
   mongoMeasurement,
   optimizeFunction,
   optimizeFunctionSync,
@@ -168,63 +169,93 @@ async function runBenchmarks({
   baseUri,
   benchmarkCompletedListeners,
 }: RunBenchmarksOptions): Promise<void> {
-  console.log(`[CodSpeed] running with @codspeed/benchmark.js v${__VERSION__}`);
-  setupCore();
-  for (let i = 0; i < benches.length; i++) {
-    const bench = benches[i];
-    const uri = bench.uri ?? `${baseUri}::unknown_${i}`;
-    const isAsync = bench.options.async || bench.options.defer;
-    let benchPayload;
-    if (bench.options.defer) {
-      benchPayload = () => {
-        return new Promise((resolve, reject) => {
-          (bench.fn as CallableFunction)({ resolve, reject });
-        });
-      };
-    } else if (bench.options.async) {
-      benchPayload = async () => {
-        await (bench.fn as CallableFunction)();
-      };
-    } else {
-      benchPayload = bench.fn as CallableFunction;
-    }
+  const isolation = Isolation.current();
+  const isChild = isolation.isChild();
+  const all = benches.map((bench, i) => ({
+    bench,
+    uri: bench.uri ?? `${baseUri}::unknown_${i}`,
+  }));
+  // A child only measures one benchmark. A run call that owns none of them
+  // must not touch the instruments, the call that owns it does.
+  const measures = all.some(({ uri }) => isolation.shouldRun(uri));
 
-    if (typeof bench.options.setup === "function") {
-      await bench.options.setup();
+  // The parent already printed the banner and the summary, children stay quiet.
+  if (!isChild) {
+    console.log(
+      `[CodSpeed] running with @codspeed/benchmark.js v${__VERSION__}`,
+    );
+  }
+  if (measures) {
+    setupCore();
+  }
+  for (const { bench, uri } of all) {
+    if (isolation.shouldRun(uri)) {
+      await isolation.run(uri, () => measureBenchmark(bench, uri));
     }
+    // Also in children, for benchmarks they skip: a `complete` listener may
+    // start the run that owns the child's benchmark.
+    benchmarkCompletedListeners.forEach((listener) => listener());
+  }
+  if (measures) {
+    teardownCore();
+  }
+  if (!isChild) {
+    console.log(`[CodSpeed] Done running ${benches.length} benches.`);
+  }
+}
 
-    if (isAsync) {
-      await optimizeFunction(benchPayload);
-      await mongoMeasurement.start(uri);
-      global.gc?.();
-      await wrapWithRootFrame(async () => {
-        InstrumentHooks.startBenchmark();
-        await benchPayload();
-        InstrumentHooks.stopBenchmark();
-        InstrumentHooks.setExecutedBenchmark(process.pid, uri);
-      })();
-      await mongoMeasurement.stop(uri);
-    } else {
-      optimizeFunctionSync(benchPayload);
-      await mongoMeasurement.start(uri);
-      wrapWithRootFrameSync(() => {
-        InstrumentHooks.startBenchmark();
-        benchPayload();
-        InstrumentHooks.stopBenchmark();
-        InstrumentHooks.setExecutedBenchmark(process.pid, uri);
-      })();
-      await mongoMeasurement.stop(uri);
-    }
+async function measureBenchmark(
+  bench: BenchmarkWithOptions,
+  uri: string,
+): Promise<void> {
+  const isAsync = bench.options.async || bench.options.defer;
+  let benchPayload;
+  if (bench.options.defer) {
+    benchPayload = () => {
+      const { promise, resolve, reject } = Promise.withResolvers<unknown>();
+      (bench.fn as CallableFunction)({ resolve, reject });
+      return promise;
+    };
+  } else if (bench.options.async) {
+    benchPayload = async () => {
+      await (bench.fn as CallableFunction)();
+    };
+  } else {
+    benchPayload = bench.fn as CallableFunction;
+  }
 
-    if (typeof bench.options.teardown === "function") {
-      await bench.options.teardown();
-    }
+  if (typeof bench.options.setup === "function") {
+    await bench.options.setup();
+  }
 
-    console.log(`    ✔ Measured ${uri}`);
-    benchmarkCompletedListeners.forEach((listener) => listener());
+  if (isAsync) {
+    await optimizeFunction(benchPayload);
+    await mongoMeasurement.start(uri);
+    global.gc?.();
+    await wrapWithRootFrame(async () => {
+      InstrumentHooks.startBenchmark();
+      await benchPayload();
+      InstrumentHooks.stopBenchmark();
+      InstrumentHooks.setExecutedBenchmark(process.pid, uri);
+    })();
+    await mongoMeasurement.stop(uri);
+  } else {
+    optimizeFunctionSync(benchPayload);
+    await mongoMeasurement.start(uri);
+    wrapWithRootFrameSync(() => {
+      InstrumentHooks.startBenchmark();
+      benchPayload();
+      InstrumentHooks.stopBenchmark();
+      InstrumentHooks.setExecutedBenchmark(process.pid, uri);
+    })();
+    await mongoMeasurement.stop(uri);
   }
-  teardownCore();
-  console.log(`[CodSpeed] Done running ${benches.length} benches.`);
+
+  if (typeof bench.options.teardown === "function") {
+    await bench.options.teardown();
+  }
+
+  console.log(`    ✔ Measured ${uri}`);
 }
 
 /**

packages/benchmark.js-plugin/tests/fixtures/isolation.cjs

@@ -0,0 +1,50 @@
+// Run by `isolation.integ.test.ts`. Prints the PID that measures each benchmark.
+const { InstrumentHooks } = require("@codspeed/core");
+const Benchmark = require("benchmark");
+const { withCodSpeed } = require("../../dist/index.cjs.js");
+
+// There is no instrument hook runtime outside the CodSpeed runner, so pretend
+// the hooks are active. The child processes run this file again, so they get
+// the same override.
+InstrumentHooks.isInstrumented = () => true;
+
+// The plugin reports every measured benchmark here, with the PID of the
+// process that measured it.
+const setExecutedBenchmark = InstrumentHooks.setExecutedBenchmark;
+InstrumentHooks.setExecutedBenchmark = (pid, uri) => {
+  setExecutedBenchmark(pid, uri);
+  console.log(`executed ${pid} ${uri}`);
+};
+
+const suite = withCodSpeed(new Benchmark.Suite("isolation"));
+
+suite
+  .add("first", () => {})
+  .add("second", () => {})
+  .add("third", () => {})
+  .add("async", {
+    defer: true,
+    fn: (deferred) => deferred.resolve(),
+  });
+
+if (process.env.ISOLATION_FIXTURE_FAIL !== undefined) {
+  suite.add("failing", () => {
+    throw new Error("benchmark failed");
+  });
+}
+
+// Started from a `complete` listener of the first suite: a child measuring
+// `chained` only reaches it through that listener.
+const chained = withCodSpeed(new Benchmark.Suite("chained"));
+chained.add("chained", () => {});
+let completed = 0;
+suite.on("complete", () => {
+  completed++;
+  if (completed === suite.length) {
+    // After the first run tears down, so the two runs don't overlap.
+    setImmediate(() => chained.run());
+  }
+});
+
+// An unhandled rejection ends the process with exit code 1.
+suite.run();

packages/benchmark.js-plugin/tests/index.integ.test.ts

@@ -11,6 +11,7 @@ jest.mock("@codspeed/core", () => {
   const actual = jest.requireActual("@codspeed/core");
   mockCore.getGitDir = actual.getGitDir;
   mockCore.getCallingFile = actual.getCallingFile;
+  mockCore.Isolation = actual.Isolation;
   mockCore.wrapWithRootFrame = actual.wrapWithRootFrame;
   mockCore.wrapWithRootFrameSync = actual.wrapWithRootFrameSync;
   return mockCore;
@@ -19,6 +20,9 @@ jest.mock("@codspeed/core", () => {
 beforeEach(() => {
   mockReset(mockCore);
   jest.clearAllMocks();
+  // These tests drive the plugin in this process against a mocked core, so
+  // there is no benchmark process to spawn.
+  process.env.CODSPEED_PROCESS_ISOLATION = "false";
 });
 
 const benchOptions: Benchmark.Options = {

packages/benchmark.js-plugin/tests/isolation.integ.test.ts

@@ -0,0 +1,82 @@
+import { getV8Flags } from "@codspeed/core";
+import { spawnSync } from "child_process";
+import { join } from "path";
+
+const FIXTURE = join(__dirname, "fixtures", "isolation.cjs");
+const PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION";
+/** `first`, `second`, `third`, the deferred `async`, and `chained`. */
+const BENCHMARK_COUNT = 5;
+/** Fails a run whose benchmark process got stuck instead of blocking the suite. */
+const RUN_TIMEOUT_MS = 60_000;
+const EXECUTED_LINE = /^executed (\d+) (.+)$/;
+
+interface FixtureOptions {
+  /** Value of CODSPEED_PROCESS_ISOLATION, unset when undefined. */
+  isolation?: string;
+  failing?: boolean;
+}
+
+/** Runs the fixture and returns the PID that measured each benchmark. */
+function runFixture({ isolation, failing }: FixtureOptions = {}) {
+  const env: NodeJS.ProcessEnv = {
+    ...process.env,
+    CODSPEED_ENV: "true",
+    CODSPEED_RUNNER_MODE: "simulation",
+  };
+  delete env[PROCESS_ISOLATION_ENV];
+  if (isolation !== undefined) {
+    env[PROCESS_ISOLATION_ENV] = isolation;
+  }
+  if (failing) {
+    env.ISOLATION_FIXTURE_FAIL = "1";
+  }
+
+  // The runner starts Node with these flags, and children must keep them.
+  const result = spawnSync(process.execPath, [...getV8Flags(), FIXTURE], {
+    env,
+    encoding: "utf8",
+    timeout: RUN_TIMEOUT_MS,
+  });
+
+  const pidsByUri = new Map<string, number>();
+  for (const line of result.stdout.split("\n")) {
+    const match = EXECUTED_LINE.exec(line);
+    if (match) {
+      pidsByUri.set(match[2], Number(match[1]));
+    }
+  }
+  return {
+    status: result.status,
+    stderr: result.stderr,
+    pidsByUri,
+    parentPid: result.pid,
+  };
+}
+
+describe("process isolation", () => {
+  it("measures every benchmark in its own process", () => {
+    const run = runFixture();
+
+    expect(run.status).toBe(0);
+    expect(run.pidsByUri.size).toBe(BENCHMARK_COUNT);
+    const pids = new Set(run.pidsByUri.values());
+    expect(pids.size).toBe(BENCHMARK_COUNT);
+    // The parent only spawns children, it never measures.
+    expect(pids).not.toContain(run.parentPid);
+  });
+
+  it("measures all benchmarks in one process when disabled", () => {
+    const run = runFixture({ isolation: "false" });
+
+    expect(run.status).toBe(0);
+    expect(run.pidsByUri.size).toBe(BENCHMARK_COUNT);
+    expect(new Set(run.pidsByUri.values())).toEqual(new Set([run.parentPid]));
+  });
+
+  it("fails the run when a benchmark throws", () => {
+    const run = runFixture({ isolation: "true", failing: true });
+
+    expect(run.status).not.toBe(0);
+    expect(run.stderr).toContain("benchmark failed");
+  });
+});

packages/core/src/index.ts

@@ -51,6 +51,7 @@ export type {
   SetupInstrumentsResponse,
 } from "./generated/openapi";
 export { getV8Flags, tryIntrospect } from "./introspection";
+export * from "./isolation";
 export {
   getUnsupportedNodeVersionWarning,
   SUPPORTED_NODE_MAJORS,

packages/core/src/introspection.ts

@@ -2,6 +2,7 @@ import { writeFileSync } from "fs";
 import path from "path";
 
 import { getInstrumentMode } from "./runnerMode";
+import { isGcBetweenRoundsEnabled } from "./walltime/options";
 
 const CUSTOM_INTROSPECTION_EXIT_CODE = 0;
 
@@ -49,6 +50,9 @@ export const getV8Flags = () => {
     case "walltime": {
       // Emit the V8 jitdump
       flags.push("--perf-prof");
+      if (isGcBetweenRoundsEnabled()) {
+        flags.push("--expose-gc");
+      }
 
       const v8LogDir = process.env.CODSPEED_V8_LOG;
       if (v8LogDir) {