diff --git a/CLAUDE.md b/CLAUDE.md index 00a76a53..47808cc5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -115,4 +115,24 @@ so exercise all three of these when developing or reviewing a plugin change Running these locally outside the CodSpeed runner is expected to log `instrument-hooks: failed to write environment.json` and skip actual measurement writes — the point is to verify the plugin's control flow and output per mode, -not to produce real measurements. \ No newline at end of file +not to produce real measurements. + +## Walltime forks and GC between rounds + +Both are opt-in and only apply to walltime runs. + +- `CODSPEED_WALLTIME_FORKS=` runs each benchmark in `N` fresh processes, one + after the other, and reports all their rounds together as one result. Unset + runs the benchmark in the benchmark process. Vitest 3/4 only. Like isolation, + each child re-runs the Vitest command with `CODSPEED_ISOLATED_BENCHMARK=`; + it writes its round samples to the file named by `CODSPEED_FORK_SAMPLES_FILE` + instead of reporting, and the parent merges them. With `N > 1`, each fork + runs half the warmup and measurement budget (`FORK_BUDGET_FRACTION`: half + the time, rounds rounded up), so forks mostly add process startup. +- `CODSPEED_WALLTIME_GC_BETWEEN_ROUNDS=true` runs a full GC before every round + (warmup and measured). It adds `--expose-gc` to the V8 flags and collects in + the tinybench `beforeEach` hook, outside the timed call. Vitest 3/4/5. Meant + for benchmarks with long rounds: a GC costs milliseconds, so time-based + benchmarks of microsecond functions take far longer to finish. It also + changes absolute times (each round starts on a collected heap), so turning + it on re-baselines a benchmark. diff --git a/docs/introspection.md b/docs/introspection.md index 9e3fe464..24258ec3 100644 --- a/docs/introspection.md +++ b/docs/introspection.md @@ -13,3 +13,13 @@ ```bash CI=1 CODSPEED_DEBUG=true pnpm --filter with-typescript-esm bench-tinybench ``` + +## Vitest process isolation + +Isolated Vitest children restrict benchmark discovery to the file named in +`CODSPEED_ISOLATED_BENCHMARK`. Other benchmark files are not transformed or imported +in that child. The parent still discovers the full suite, and each child still +loads the Vitest configuration and setup files. + +Isolated children retain fresh processes and the profiling flags. This avoids +repeated unrelated file imports; it does not remove Vitest's per-process startup. diff --git a/examples/with-vitest-v3/src/fibonacci.bench.ts b/examples/with-vitest-v3/src/fibonacci.bench.ts index 227a6e67..1349a1ad 100644 --- a/examples/with-vitest-v3/src/fibonacci.bench.ts +++ b/examples/with-vitest-v3/src/fibonacci.bench.ts @@ -18,3 +18,7 @@ describe("fibonacci", () => { iterativeFibonacci(20); }); }); + +bench("top-level recursive fibo 15", () => { + recursiveFibonacci(15); +}); diff --git a/examples/with-vitest-v4/src/fibonacci.bench.ts b/examples/with-vitest-v4/src/fibonacci.bench.ts index 227a6e67..1349a1ad 100644 --- a/examples/with-vitest-v4/src/fibonacci.bench.ts +++ b/examples/with-vitest-v4/src/fibonacci.bench.ts @@ -18,3 +18,7 @@ describe("fibonacci", () => { iterativeFibonacci(20); }); }); + +bench("top-level recursive fibo 15", () => { + recursiveFibonacci(15); +}); diff --git a/packages/benchmark.js-plugin/src/index.ts b/packages/benchmark.js-plugin/src/index.ts index fa69a35a..49107fea 100644 --- a/packages/benchmark.js-plugin/src/index.ts +++ b/packages/benchmark.js-plugin/src/index.ts @@ -1,6 +1,7 @@ import { getCallingFile, InstrumentHooks, + Isolation, mongoMeasurement, optimizeFunction, optimizeFunctionSync, @@ -168,63 +169,93 @@ async function runBenchmarks({ baseUri, benchmarkCompletedListeners, }: RunBenchmarksOptions): Promise { - console.log(`[CodSpeed] running with @codspeed/benchmark.js v${__VERSION__}`); - setupCore(); - for (let i = 0; i < benches.length; i++) { - const bench = benches[i]; - const uri = bench.uri ?? `${baseUri}::unknown_${i}`; - const isAsync = bench.options.async || bench.options.defer; - let benchPayload; - if (bench.options.defer) { - benchPayload = () => { - return new Promise((resolve, reject) => { - (bench.fn as CallableFunction)({ resolve, reject }); - }); - }; - } else if (bench.options.async) { - benchPayload = async () => { - await (bench.fn as CallableFunction)(); - }; - } else { - benchPayload = bench.fn as CallableFunction; - } + const isolation = Isolation.current(); + const isChild = isolation.isChild(); + const all = benches.map((bench, i) => ({ + bench, + uri: bench.uri ?? `${baseUri}::unknown_${i}`, + })); + // A child only measures one benchmark. A run call that owns none of them + // must not touch the instruments, the call that owns it does. + const measures = all.some(({ uri }) => isolation.shouldRun(uri)); - if (typeof bench.options.setup === "function") { - await bench.options.setup(); + // The parent already printed the banner and the summary, children stay quiet. + if (!isChild) { + console.log( + `[CodSpeed] running with @codspeed/benchmark.js v${__VERSION__}`, + ); + } + if (measures) { + setupCore(); + } + for (const { bench, uri } of all) { + if (isolation.shouldRun(uri)) { + await isolation.run(uri, () => measureBenchmark(bench, uri)); } + // Also in children, for benchmarks they skip: a `complete` listener may + // start the run that owns the child's benchmark. + benchmarkCompletedListeners.forEach((listener) => listener()); + } + if (measures) { + teardownCore(); + } + if (!isChild) { + console.log(`[CodSpeed] Done running ${benches.length} benches.`); + } +} - if (isAsync) { - await optimizeFunction(benchPayload); - await mongoMeasurement.start(uri); - global.gc?.(); - await wrapWithRootFrame(async () => { - InstrumentHooks.startBenchmark(); - await benchPayload(); - InstrumentHooks.stopBenchmark(); - InstrumentHooks.setExecutedBenchmark(process.pid, uri); - })(); - await mongoMeasurement.stop(uri); - } else { - optimizeFunctionSync(benchPayload); - await mongoMeasurement.start(uri); - wrapWithRootFrameSync(() => { - InstrumentHooks.startBenchmark(); - benchPayload(); - InstrumentHooks.stopBenchmark(); - InstrumentHooks.setExecutedBenchmark(process.pid, uri); - })(); - await mongoMeasurement.stop(uri); - } +async function measureBenchmark( + bench: BenchmarkWithOptions, + uri: string, +): Promise { + const isAsync = bench.options.async || bench.options.defer; + let benchPayload; + if (bench.options.defer) { + benchPayload = () => { + const { promise, resolve, reject } = Promise.withResolvers(); + (bench.fn as CallableFunction)({ resolve, reject }); + return promise; + }; + } else if (bench.options.async) { + benchPayload = async () => { + await (bench.fn as CallableFunction)(); + }; + } else { + benchPayload = bench.fn as CallableFunction; + } - if (typeof bench.options.teardown === "function") { - await bench.options.teardown(); - } + if (typeof bench.options.setup === "function") { + await bench.options.setup(); + } - console.log(` ✔ Measured ${uri}`); - benchmarkCompletedListeners.forEach((listener) => listener()); + if (isAsync) { + await optimizeFunction(benchPayload); + await mongoMeasurement.start(uri); + global.gc?.(); + await wrapWithRootFrame(async () => { + InstrumentHooks.startBenchmark(); + await benchPayload(); + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(process.pid, uri); + })(); + await mongoMeasurement.stop(uri); + } else { + optimizeFunctionSync(benchPayload); + await mongoMeasurement.start(uri); + wrapWithRootFrameSync(() => { + InstrumentHooks.startBenchmark(); + benchPayload(); + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(process.pid, uri); + })(); + await mongoMeasurement.stop(uri); } - teardownCore(); - console.log(`[CodSpeed] Done running ${benches.length} benches.`); + + if (typeof bench.options.teardown === "function") { + await bench.options.teardown(); + } + + console.log(` ✔ Measured ${uri}`); } /** diff --git a/packages/benchmark.js-plugin/tests/fixtures/isolation.cjs b/packages/benchmark.js-plugin/tests/fixtures/isolation.cjs new file mode 100644 index 00000000..b1d51e57 --- /dev/null +++ b/packages/benchmark.js-plugin/tests/fixtures/isolation.cjs @@ -0,0 +1,50 @@ +// Run by `isolation.integ.test.ts`. Prints the PID that measures each benchmark. +const { InstrumentHooks } = require("@codspeed/core"); +const Benchmark = require("benchmark"); +const { withCodSpeed } = require("../../dist/index.cjs.js"); + +// There is no instrument hook runtime outside the CodSpeed runner, so pretend +// the hooks are active. The child processes run this file again, so they get +// the same override. +InstrumentHooks.isInstrumented = () => true; + +// The plugin reports every measured benchmark here, with the PID of the +// process that measured it. +const setExecutedBenchmark = InstrumentHooks.setExecutedBenchmark; +InstrumentHooks.setExecutedBenchmark = (pid, uri) => { + setExecutedBenchmark(pid, uri); + console.log(`executed ${pid} ${uri}`); +}; + +const suite = withCodSpeed(new Benchmark.Suite("isolation")); + +suite + .add("first", () => {}) + .add("second", () => {}) + .add("third", () => {}) + .add("async", { + defer: true, + fn: (deferred) => deferred.resolve(), + }); + +if (process.env.ISOLATION_FIXTURE_FAIL !== undefined) { + suite.add("failing", () => { + throw new Error("benchmark failed"); + }); +} + +// Started from a `complete` listener of the first suite: a child measuring +// `chained` only reaches it through that listener. +const chained = withCodSpeed(new Benchmark.Suite("chained")); +chained.add("chained", () => {}); +let completed = 0; +suite.on("complete", () => { + completed++; + if (completed === suite.length) { + // After the first run tears down, so the two runs don't overlap. + setImmediate(() => chained.run()); + } +}); + +// An unhandled rejection ends the process with exit code 1. +suite.run(); diff --git a/packages/benchmark.js-plugin/tests/index.integ.test.ts b/packages/benchmark.js-plugin/tests/index.integ.test.ts index f3e21034..3d2256f5 100644 --- a/packages/benchmark.js-plugin/tests/index.integ.test.ts +++ b/packages/benchmark.js-plugin/tests/index.integ.test.ts @@ -11,6 +11,7 @@ jest.mock("@codspeed/core", () => { const actual = jest.requireActual("@codspeed/core"); mockCore.getGitDir = actual.getGitDir; mockCore.getCallingFile = actual.getCallingFile; + mockCore.Isolation = actual.Isolation; mockCore.wrapWithRootFrame = actual.wrapWithRootFrame; mockCore.wrapWithRootFrameSync = actual.wrapWithRootFrameSync; return mockCore; @@ -19,6 +20,9 @@ jest.mock("@codspeed/core", () => { beforeEach(() => { mockReset(mockCore); jest.clearAllMocks(); + // These tests drive the plugin in this process against a mocked core, so + // there is no benchmark process to spawn. + process.env.CODSPEED_PROCESS_ISOLATION = "false"; }); const benchOptions: Benchmark.Options = { diff --git a/packages/benchmark.js-plugin/tests/isolation.integ.test.ts b/packages/benchmark.js-plugin/tests/isolation.integ.test.ts new file mode 100644 index 00000000..c3d1a9b1 --- /dev/null +++ b/packages/benchmark.js-plugin/tests/isolation.integ.test.ts @@ -0,0 +1,82 @@ +import { getV8Flags } from "@codspeed/core"; +import { spawnSync } from "child_process"; +import { join } from "path"; + +const FIXTURE = join(__dirname, "fixtures", "isolation.cjs"); +const PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION"; +/** `first`, `second`, `third`, the deferred `async`, and `chained`. */ +const BENCHMARK_COUNT = 5; +/** Fails a run whose benchmark process got stuck instead of blocking the suite. */ +const RUN_TIMEOUT_MS = 60_000; +const EXECUTED_LINE = /^executed (\d+) (.+)$/; + +interface FixtureOptions { + /** Value of CODSPEED_PROCESS_ISOLATION, unset when undefined. */ + isolation?: string; + failing?: boolean; +} + +/** Runs the fixture and returns the PID that measured each benchmark. */ +function runFixture({ isolation, failing }: FixtureOptions = {}) { + const env: NodeJS.ProcessEnv = { + ...process.env, + CODSPEED_ENV: "true", + CODSPEED_RUNNER_MODE: "simulation", + }; + delete env[PROCESS_ISOLATION_ENV]; + if (isolation !== undefined) { + env[PROCESS_ISOLATION_ENV] = isolation; + } + if (failing) { + env.ISOLATION_FIXTURE_FAIL = "1"; + } + + // The runner starts Node with these flags, and children must keep them. + const result = spawnSync(process.execPath, [...getV8Flags(), FIXTURE], { + env, + encoding: "utf8", + timeout: RUN_TIMEOUT_MS, + }); + + const pidsByUri = new Map(); + for (const line of result.stdout.split("\n")) { + const match = EXECUTED_LINE.exec(line); + if (match) { + pidsByUri.set(match[2], Number(match[1])); + } + } + return { + status: result.status, + stderr: result.stderr, + pidsByUri, + parentPid: result.pid, + }; +} + +describe("process isolation", () => { + it("measures every benchmark in its own process", () => { + const run = runFixture(); + + expect(run.status).toBe(0); + expect(run.pidsByUri.size).toBe(BENCHMARK_COUNT); + const pids = new Set(run.pidsByUri.values()); + expect(pids.size).toBe(BENCHMARK_COUNT); + // The parent only spawns children, it never measures. + expect(pids).not.toContain(run.parentPid); + }); + + it("measures all benchmarks in one process when disabled", () => { + const run = runFixture({ isolation: "false" }); + + expect(run.status).toBe(0); + expect(run.pidsByUri.size).toBe(BENCHMARK_COUNT); + expect(new Set(run.pidsByUri.values())).toEqual(new Set([run.parentPid])); + }); + + it("fails the run when a benchmark throws", () => { + const run = runFixture({ isolation: "true", failing: true }); + + expect(run.status).not.toBe(0); + expect(run.stderr).toContain("benchmark failed"); + }); +}); diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 63ac394f..514abe22 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -51,6 +51,7 @@ export type { SetupInstrumentsResponse, } from "./generated/openapi"; export { getV8Flags, tryIntrospect } from "./introspection"; +export * from "./isolation"; export { getUnsupportedNodeVersionWarning, SUPPORTED_NODE_MAJORS, diff --git a/packages/core/src/introspection.ts b/packages/core/src/introspection.ts index f4ac29ef..f632b7f0 100644 --- a/packages/core/src/introspection.ts +++ b/packages/core/src/introspection.ts @@ -2,6 +2,7 @@ import { writeFileSync } from "fs"; import path from "path"; import { getInstrumentMode } from "./runnerMode"; +import { isGcBetweenRoundsEnabled } from "./walltime/options"; const CUSTOM_INTROSPECTION_EXIT_CODE = 0; @@ -49,6 +50,9 @@ export const getV8Flags = () => { case "walltime": { // Emit the V8 jitdump flags.push("--perf-prof"); + if (isGcBetweenRoundsEnabled()) { + flags.push("--expose-gc"); + } const v8LogDir = process.env.CODSPEED_V8_LOG; if (v8LogDir) { diff --git a/packages/core/src/isolation.ts b/packages/core/src/isolation.ts new file mode 100644 index 00000000..7c54766d --- /dev/null +++ b/packages/core/src/isolation.ts @@ -0,0 +1,291 @@ +/** + * Process isolation: every benchmark runs in a fresh copy of the benchmark + * process, so the allocator state, JIT state, and handles that other + * benchmarks or setup code leave behind can't change its measurement. + */ +import { spawn, spawnSync } from "child_process"; +import { + existsSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "fs"; +import { constants, tmpdir } from "os"; +import { join } from "path"; + +/** Isolation is on unless this is `0` or `false`. */ +export const PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION"; +/** Set on each child process: the URI of the only benchmark it runs. */ +export const ISOLATED_BENCHMARK_ENV = "CODSPEED_ISOLATED_BENCHMARK"; +/** + * JSON array `[executable, ...args]` of the command that runs the benchmarks + * again. Integrations whose benchmarks run in a worker process (vitest) set it + * from the main process, because re-executing the worker alone can't work. + */ +export const ISOLATION_COMMAND_ENV = "CODSPEED_ISOLATION_COMMAND"; +/** Set on each fork of {@link runForks}: where the fork writes its samples. */ +export const FORK_SAMPLES_FILE_ENV = "CODSPEED_FORK_SAMPLES_FILE"; + +/** Exit code for a child that ended with neither an exit code nor a signal. */ +const GENERIC_FAILURE_EXIT_CODE = 1; +/** Shells report a child killed by signal N as exit code 128 + N. */ +const SIGNAL_EXIT_CODE_OFFSET = 128; + +export type IsolationMode = + /** All benchmarks share this process and its state. */ + | { kind: "SharedProcess" } + /** Every benchmark runs in its own child process. */ + | { kind: "ProcessPerBenchmark" } + /** A child spawned in `ProcessPerBenchmark` mode, restricted to `uri`. */ + | { kind: "IsolatedBenchmark"; uri: string }; + +export class Isolation { + private static instance: Isolation | undefined; + private static readonly sharedInstance = new Isolation({ + kind: "SharedProcess", + }); + + private constructor(readonly mode: IsolationMode) {} + + /** + * The isolation of runs where no instrument measures anything (the hooks are + * not active), so there is nothing to isolate. + */ + static shared(): Isolation { + return Isolation.sharedInstance; + } + + static current(): Isolation { + Isolation.instance ??= new Isolation(modeFromEnv()); + return Isolation.instance; + } + + /** The only benchmark this process may run. It overrides user filters. */ + onlyBenchmark(): string | undefined { + switch (this.mode.kind) { + case "IsolatedBenchmark": + return this.mode.uri; + case "SharedProcess": + case "ProcessPerBenchmark": + return undefined; + } + } + + shouldRun(uri: string): boolean { + const only = this.onlyBenchmark(); + return only === undefined || only === uri; + } + + isChild(): boolean { + return this.onlyBenchmark() !== undefined; + } + + /** + * Runs the benchmark `uri`. `bench` must contain the whole measurement, + * from `startBenchmark` to `setExecutedBenchmark`. + */ + async run(uri: string, bench: () => Promise | void): Promise { + switch (this.mode.kind) { + case "SharedProcess": + case "IsolatedBenchmark": + await bench(); + return; + case "ProcessPerBenchmark": + await runInChildProcessAsync(uri); + return; + } + } + + /** Synchronous variant of {@link Isolation.run}. */ + runSync(uri: string, bench: () => void): void { + switch (this.mode.kind) { + case "SharedProcess": + case "IsolatedBenchmark": + bench(); + return; + case "ProcessPerBenchmark": + runInChildProcess(uri); + return; + } + } +} + +function modeFromEnv(): IsolationMode { + const uri = process.env[ISOLATED_BENCHMARK_ENV]; + if (uri !== undefined) { + return { kind: "IsolatedBenchmark", uri }; + } + + const value = process.env[PROCESS_ISOLATION_ENV]; + switch (value) { + case undefined: + case "1": + case "true": + return { kind: "ProcessPerBenchmark" }; + case "0": + case "false": + return { kind: "SharedProcess" }; + default: + throw new Error( + `${PROCESS_ISOLATION_ENV} must be \`true\` or \`false\`, got \`${value}\``, + ); + } +} + +/** + * Records the command of this process as the one that children run. Call it + * from the main process of integrations that measure in worker processes. It + * overwrites a command inherited from an outer process, which is not ours. + */ +export function recordIsolationCommand(): void { + process.env[ISOLATION_COMMAND_ENV] = JSON.stringify(currentCommand()); +} + +function currentCommand(): string[] { + // execArgv keeps the V8 flags (e.g. --perf-prof) that the runner set. + return [process.execPath, ...process.execArgv, ...process.argv.slice(1)]; +} + +function isolationCommand(): string[] { + const recorded = process.env[ISOLATION_COMMAND_ENV]; + if (recorded === undefined) { + return currentCommand(); + } + return JSON.parse(recorded) as string[]; +} + +/** + * Runs the benchmark `uri` in `forks` fresh processes, one after the other, + * and returns the round samples of all of them, in milliseconds. Each fork + * measures the benchmark like an isolated child and writes its samples to the + * file named by {@link FORK_SAMPLES_FILE_ENV} instead of reporting results. + */ +export async function runForks(uri: string, forks: number): Promise { + const dir = mkdtempSync(join(tmpdir(), "codspeed-forks-")); + try { + let samples: number[] = []; + for (let fork = 0; fork < forks; fork++) { + const file = join(dir, `${fork}.json`); + const failure = await runChildAsync(uri, { + [FORK_SAMPLES_FILE_ENV]: file, + }); + if (failure !== undefined) { + // process.exit() skips the `finally` below. + rmSync(dir, { recursive: true, force: true }); + process.exit(failure); + } + if (!existsSync(file)) { + throw new Error( + `[CodSpeed] fork ${fork + 1}/${forks} of ${uri} did not run the benchmark`, + ); + } + const forkSamples = ( + JSON.parse(readFileSync(file, "utf-8")) as ForkSamples + ).samples; + // Not push(...): spreading a time-based run's samples overflows the stack. + samples = samples.concat(forkSamples); + } + return samples; + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +/** Whether this process is a fork started by {@link runForks}. */ +export function isFork(): boolean { + return process.env[FORK_SAMPLES_FILE_ENV] !== undefined; +} + +/** Hands the round samples (milliseconds) of this fork to its parent. */ +export function writeForkSamples(samples: number[]): void { + const file = process.env[FORK_SAMPLES_FILE_ENV]; + if (file === undefined) { + throw new Error(`[CodSpeed] ${FORK_SAMPLES_FILE_ENV} is not set`); + } + const data: ForkSamples = { samples }; + writeFileSync(file, JSON.stringify(data)); +} + +interface ForkSamples { + samples: number[]; +} + +/** + * Runs the benchmark command again, limited to `uri`, and exits with the + * child's code if it fails. Re-exec, not fork(): Node can't fork, and a fork + * would only copy the calling thread. + */ +function runInChildProcess(uri: string): void { + const failure = runChild(uri); + if (failure !== undefined) { + process.exit(failure); + } +} + +/** Runs the benchmark command limited to `uri`; the exit code if it failed. */ +function runChild(uri: string): number | undefined { + const [executable, ...args] = isolationCommand(); + const result = spawnSync(executable, args, { + stdio: "inherit", + env: childEnv(uri), + }); + if (result.error !== undefined) { + throw startError(uri, result.error); + } + return failureCode(result.status, result.signal); +} + +/** + * {@link runInChildProcess} without blocking the event loop while the child + * runs: a Vitest worker must keep answering its main process, whose calls + * time out otherwise. + */ +async function runInChildProcessAsync(uri: string): Promise { + const failure = await runChildAsync(uri); + if (failure !== undefined) { + process.exit(failure); + } +} + +/** {@link runChild} without blocking the event loop. */ +async function runChildAsync( + uri: string, + env: NodeJS.ProcessEnv = {}, +): Promise { + const [executable, ...args] = isolationCommand(); + const child = spawn(executable, args, { + stdio: "inherit", + env: childEnv(uri, env), + }); + const { promise, resolve, reject } = + Promise.withResolvers<[number | null, NodeJS.Signals | null]>(); + child.once("error", (error) => reject(startError(uri, error))); + child.once("close", (code, signal) => resolve([code, signal])); + const [status, signal] = await promise; + return failureCode(status, signal); +} + +function childEnv(uri: string, env: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { + return { ...process.env, ...env, [ISOLATED_BENCHMARK_ENV]: uri }; +} + +function startError(uri: string, error: Error): Error { + return new Error( + `[CodSpeed] failed to start the process for ${uri}: ${error.message}`, + ); +} + +/** A failed child's exit code, in the shell convention. */ +function failureCode( + status: number | null, + signal: NodeJS.Signals | null, +): number | undefined { + if (status === 0) { + return undefined; + } + if (signal !== null) { + return SIGNAL_EXIT_CODE_OFFSET + constants.signals[signal]; + } + return status ?? GENERIC_FAILURE_EXIT_CODE; +} diff --git a/packages/core/src/walltime/index.ts b/packages/core/src/walltime/index.ts index b2d86059..b2a8b0f4 100644 --- a/packages/core/src/walltime/index.ts +++ b/packages/core/src/walltime/index.ts @@ -60,5 +60,6 @@ export function writeWalltimeResults( } export * from "./interfaces"; +export * from "./options"; export * from "./quantiles"; export * from "./utils"; diff --git a/packages/core/src/walltime/options.ts b/packages/core/src/walltime/options.ts new file mode 100644 index 00000000..048568e7 --- /dev/null +++ b/packages/core/src/walltime/options.ts @@ -0,0 +1,61 @@ +/** + * Number of fresh processes that each walltime benchmark runs in, whose rounds + * are reported together. Unset: the benchmark runs in the benchmark process. + */ +export const WALLTIME_FORKS_ENV = "CODSPEED_WALLTIME_FORKS"; +/** When `true`, a full GC runs before every walltime round, outside its timing. */ +export const WALLTIME_GC_BETWEEN_ROUNDS_ENV = + "CODSPEED_WALLTIME_GC_BETWEEN_ROUNDS"; + +/** + * Share of a benchmark's warmup and measurement budget (time and rounds) that + * each fork runs when there are several forks. Every fork pays the process + * startup anyway, so full budgets would make F forks cost F full runs. + */ +export const FORK_BUDGET_FRACTION = 0.5; + +/** The number of forks per walltime benchmark, or `undefined` for none. */ +export function getWalltimeForks(): number | undefined { + const value = process.env[WALLTIME_FORKS_ENV]; + if (value === undefined || value === "") { + return undefined; + } + const forks = Number(value); + if (!Number.isInteger(forks) || forks < 1) { + throw new Error( + `${WALLTIME_FORKS_ENV} must be a positive integer, got \`${value}\``, + ); + } + return forks; +} + +export function isGcBetweenRoundsEnabled(): boolean { + const value = process.env[WALLTIME_GC_BETWEEN_ROUNDS_ENV]; + switch (value) { + case undefined: + case "": + case "0": + case "false": + return false; + case "1": + case "true": + return true; + default: + throw new Error( + `${WALLTIME_GC_BETWEEN_ROUNDS_ENV} must be \`true\` or \`false\`, got \`${value}\``, + ); + } +} + +/** + * Runs a full GC. Only valid when GC between rounds is enabled, which adds + * `--expose-gc` to the V8 flags. + */ +export function collectGarbage(): void { + if (global.gc === undefined) { + throw new Error( + `[CodSpeed] ${WALLTIME_GC_BETWEEN_ROUNDS_ENV} needs the --expose-gc V8 flag`, + ); + } + global.gc(); +} diff --git a/packages/core/tests/fixtures/forks.cjs b/packages/core/tests/fixtures/forks.cjs new file mode 100644 index 00000000..3e0ec9ab --- /dev/null +++ b/packages/core/tests/fixtures/forks.cjs @@ -0,0 +1,23 @@ +// Run by `isolation.integ.test.ts`. Runs one benchmark through `runForks`. +const { + isFork, + runForks, + writeForkSamples, +} = require("../../dist/index.cjs.js"); + +if (isFork()) { + console.log(`fork ${process.pid}`); + switch (process.env.FORKS_FIXTURE_ACTION) { + case "exit": + process.exit(3); + case "no-samples": + process.exit(0); + } + const count = Number(process.env.FORKS_FIXTURE_SAMPLES); + writeForkSamples(Array.from({ length: count }, (_, i) => i)); +} else { + // A rejection ends the process with exit code 1. + runForks("a", Number(process.env.FORKS_FIXTURE_COUNT)).then((samples) => { + console.log(`samples ${samples.length}`); + }); +} diff --git a/packages/core/tests/fixtures/isolation.cjs b/packages/core/tests/fixtures/isolation.cjs new file mode 100644 index 00000000..46205d3b --- /dev/null +++ b/packages/core/tests/fixtures/isolation.cjs @@ -0,0 +1,27 @@ +// Run by `isolation.integ.test.ts`. Runs two benchmarks through `Isolation`. +const { Isolation } = require("../../dist/index.cjs.js"); + +const isolation = Isolation.current(); + +function bench(uri) { + console.log(`ran ${process.pid} ${uri} ${JSON.stringify(process.execArgv)}`); + switch (process.env.ISOLATION_FIXTURE_ACTION) { + case "exit": + process.exit(3); + case "kill": + process.kill(process.pid, "SIGKILL"); + } +} + +async function main() { + for (const uri of ["a", "b"]) { + if (!isolation.shouldRun(uri)) continue; + if (process.env.ISOLATION_FIXTURE_SYNC !== undefined) { + isolation.runSync(uri, () => bench(uri)); + } else { + await isolation.run(uri, async () => bench(uri)); + } + } +} + +main(); diff --git a/packages/core/tests/isolation.integ.test.ts b/packages/core/tests/isolation.integ.test.ts new file mode 100644 index 00000000..78650ad9 --- /dev/null +++ b/packages/core/tests/isolation.integ.test.ts @@ -0,0 +1,212 @@ +import { spawnSync } from "child_process"; +import { constants } from "os"; +import { join } from "path"; +import { + FORK_SAMPLES_FILE_ENV, + ISOLATED_BENCHMARK_ENV, + Isolation, + PROCESS_ISOLATION_ENV, +} from "../src/isolation"; + +const FIXTURE = join(__dirname, "fixtures", "isolation.cjs"); +/** Fails a run whose benchmark process got stuck instead of blocking the suite. */ +const RUN_TIMEOUT_MS = 60_000; +const SIGNAL_EXIT_CODE_OFFSET = 128; +const CHILD_EXIT_CODE = 3; +const RAN_LINE = /^ran (\d+) (\S+) (.+)$/; +/** Passed to Node to check that children keep the flags of their parent. */ +const NODE_FLAG = "--no-warnings"; + +/** `Isolation.current()` reads the environment once, so load a fresh copy. */ +function currentIsolation(env: Record): Isolation { + const saved = process.env; + process.env = { ...saved, ...env }; + try { + let isolation: Isolation; + jest.isolateModules(() => { + // eslint-disable-next-line @typescript-eslint/no-require-imports + isolation = require("../src/isolation").Isolation.current(); + }); + return isolation!; + } finally { + process.env = saved; + } +} + +describe("Isolation.current", () => { + it.each([ + [undefined, "ProcessPerBenchmark"], + ["1", "ProcessPerBenchmark"], + ["true", "ProcessPerBenchmark"], + ["0", "SharedProcess"], + ["false", "SharedProcess"], + ])("selects %s as %s", (value, kind) => { + expect( + currentIsolation({ + [PROCESS_ISOLATION_ENV]: value, + [ISOLATED_BENCHMARK_ENV]: undefined, + }).mode.kind, + ).toBe(kind); + }); + + it("restricts a child to its benchmark, whatever the isolation setting", () => { + const isolation = currentIsolation({ + [PROCESS_ISOLATION_ENV]: "false", + [ISOLATED_BENCHMARK_ENV]: "file::bench", + }); + + expect(isolation.shouldRun("file::bench")).toBe(true); + // Exact match only: a prefix is another benchmark. + expect(isolation.shouldRun("file::bench 2")).toBe(false); + expect(isolation.shouldRun("file")).toBe(false); + }); + + it("rejects any other value", () => { + expect(() => + currentIsolation({ + [PROCESS_ISOLATION_ENV]: "maybe", + [ISOLATED_BENCHMARK_ENV]: undefined, + }), + ).toThrow(`${PROCESS_ISOLATION_ENV} must be \`true\` or \`false\``); + }); +}); + +interface FixtureOptions { + action?: "exit" | "kill"; + sync?: boolean; +} + +function runFixture({ action, sync }: FixtureOptions = {}) { + const env: NodeJS.ProcessEnv = { + ...process.env, + ISOLATION_FIXTURE_ACTION: action, + }; + delete env[PROCESS_ISOLATION_ENV]; + delete env[ISOLATED_BENCHMARK_ENV]; + if (sync) { + env.ISOLATION_FIXTURE_SYNC = "1"; + } + const result = spawnSync(process.execPath, [NODE_FLAG, FIXTURE], { + env, + encoding: "utf8", + timeout: RUN_TIMEOUT_MS, + }); + const ran = result.stdout + .split("\n") + .map((line) => RAN_LINE.exec(line)) + .filter((match): match is RegExpExecArray => match !== null); + return { status: result.status, ran }; +} + +describe.each([ + { name: "async", sync: false }, + { name: "sync", sync: true }, +])("Isolation child process ($name run)", ({ sync }) => { + it("runs each benchmark in a fresh process with the flags of its parent", () => { + const { status, ran } = runFixture({ sync }); + + expect(status).toBe(0); + expect(ran.map((match) => match[2])).toEqual(["a", "b"]); + expect(new Set(ran.map((match) => match[1])).size).toBe(2); + for (const match of ran) { + expect(JSON.parse(match[3])).toEqual([NODE_FLAG]); + } + }); + + it("exits with the exit code of a failing child", () => { + const { status, ran } = runFixture({ sync, action: "exit" }); + + expect(status).toBe(CHILD_EXIT_CODE); + // The parent stops at the first failure. + expect(ran.map((match) => match[2])).toEqual(["a"]); + }); + + it("exits with 128 + N when a signal N killed the child", () => { + const { status, ran } = runFixture({ sync, action: "kill" }); + + expect(status).toBe(SIGNAL_EXIT_CODE_OFFSET + constants.signals.SIGKILL); + expect(ran.map((match) => match[2])).toEqual(["a"]); + }); +}); + +const FORKS_FIXTURE = join(__dirname, "fixtures", "forks.cjs"); +const FORK_LINE = /^fork (\d+)$/; +const SAMPLES_LINE = /^samples (\d+)$/; +const SAMPLES_PER_FORK = 2; +/** More samples than fit in the arguments of one call. */ +const MANY_SAMPLES_PER_FORK = 300_000; + +function runForksFixture( + count: number, + action?: "exit" | "no-samples", + samplesPerFork = SAMPLES_PER_FORK, +) { + const env: NodeJS.ProcessEnv = { + ...process.env, + FORKS_FIXTURE_COUNT: String(count), + FORKS_FIXTURE_ACTION: action, + FORKS_FIXTURE_SAMPLES: String(samplesPerFork), + }; + delete env[PROCESS_ISOLATION_ENV]; + delete env[ISOLATED_BENCHMARK_ENV]; + delete env[FORK_SAMPLES_FILE_ENV]; + const result = spawnSync(process.execPath, [FORKS_FIXTURE], { + env, + encoding: "utf8", + timeout: RUN_TIMEOUT_MS, + }); + const lines = result.stdout.split("\n"); + const pids = lines + .map((line) => FORK_LINE.exec(line)) + .filter((match): match is RegExpExecArray => match !== null) + .map((match) => match[1]); + const samplesMatch = lines + .map((line) => SAMPLES_LINE.exec(line)) + .find((match) => match !== null); + return { + status: result.status, + stderr: result.stderr, + pids, + sampleCount: samplesMatch ? Number(samplesMatch[1]) : undefined, + }; +} + +describe("runForks", () => { + it("runs each fork in a fresh process and merges their samples", () => { + const forks = 3; + const { status, pids, sampleCount } = runForksFixture(forks); + + expect(status).toBe(0); + expect(new Set(pids).size).toBe(forks); + expect(sampleCount).toBe(forks * SAMPLES_PER_FORK); + }); + + it("merges forks with many samples each", () => { + const forks = 2; + const { status, sampleCount } = runForksFixture( + forks, + undefined, + MANY_SAMPLES_PER_FORK, + ); + + expect(status).toBe(0); + expect(sampleCount).toBe(forks * MANY_SAMPLES_PER_FORK); + }); + + it("fails when a fork wrote no samples", () => { + const { status, stderr, sampleCount } = runForksFixture(2, "no-samples"); + + expect(status).not.toBe(0); + expect(stderr).toContain("did not run the benchmark"); + expect(sampleCount).toBeUndefined(); + }); + + it("exits with the exit code of a failing fork", () => { + const { status, pids, sampleCount } = runForksFixture(2, "exit"); + + expect(status).toBe(CHILD_EXIT_CODE); + // The parent stops at the first failure. + expect(pids).toHaveLength(1); + expect(sampleCount).toBeUndefined(); + }); +}); diff --git a/packages/core/tests/walltimeOptions.integ.test.ts b/packages/core/tests/walltimeOptions.integ.test.ts new file mode 100644 index 00000000..dc11e288 --- /dev/null +++ b/packages/core/tests/walltimeOptions.integ.test.ts @@ -0,0 +1,86 @@ +import { getV8Flags } from "../src/introspection"; +import { + WALLTIME_FORKS_ENV, + WALLTIME_GC_BETWEEN_ROUNDS_ENV, + getWalltimeForks, + isGcBetweenRoundsEnabled, +} from "../src/walltime/options"; + +const EXPOSE_GC_FLAG = "--expose-gc"; +const ENV_KEYS = [ + WALLTIME_FORKS_ENV, + WALLTIME_GC_BETWEEN_ROUNDS_ENV, + "CODSPEED_ENV", + "CODSPEED_RUNNER_MODE", +]; + +const savedEnv = Object.fromEntries( + ENV_KEYS.map((key) => [key, process.env[key]]), +); + +function setEnv(key: string, value: string | undefined): void { + if (value === undefined) { + delete process.env[key]; + } else { + process.env[key] = value; + } +} + +afterEach(() => { + for (const [key, value] of Object.entries(savedEnv)) { + setEnv(key, value); + } +}); + +describe("getWalltimeForks", () => { + it.each([undefined, ""])("returns undefined when unset (%p)", (value) => { + setEnv(WALLTIME_FORKS_ENV, value); + expect(getWalltimeForks()).toBeUndefined(); + }); + + it("returns the number of forks", () => { + setEnv(WALLTIME_FORKS_ENV, "3"); + expect(getWalltimeForks()).toBe(3); + }); + + it.each(["0", "-1", "1.5", "abc"])("rejects %p", (value) => { + setEnv(WALLTIME_FORKS_ENV, value); + expect(() => getWalltimeForks()).toThrow(WALLTIME_FORKS_ENV); + }); +}); + +describe("isGcBetweenRoundsEnabled", () => { + it.each([undefined, "0", "false"])("is disabled for %p", (value) => { + setEnv(WALLTIME_GC_BETWEEN_ROUNDS_ENV, value); + expect(isGcBetweenRoundsEnabled()).toBe(false); + }); + + it.each(["1", "true"])("is enabled for %p", (value) => { + setEnv(WALLTIME_GC_BETWEEN_ROUNDS_ENV, value); + expect(isGcBetweenRoundsEnabled()).toBe(true); + }); + + it("rejects any other value", () => { + setEnv(WALLTIME_GC_BETWEEN_ROUNDS_ENV, "maybe"); + expect(() => isGcBetweenRoundsEnabled()).toThrow( + WALLTIME_GC_BETWEEN_ROUNDS_ENV, + ); + }); +}); + +describe("walltime V8 flags", () => { + beforeEach(() => { + setEnv("CODSPEED_ENV", "1"); + setEnv("CODSPEED_RUNNER_MODE", "walltime"); + }); + + it("exposes the GC when GC between rounds is enabled", () => { + setEnv(WALLTIME_GC_BETWEEN_ROUNDS_ENV, "true"); + expect(getV8Flags()).toContain(EXPOSE_GC_FLAG); + }); + + it("does not expose the GC otherwise", () => { + setEnv(WALLTIME_GC_BETWEEN_ROUNDS_ENV, undefined); + expect(getV8Flags()).not.toContain(EXPOSE_GC_FLAG); + }); +}); diff --git a/packages/tinybench-plugin/src/analysis.ts b/packages/tinybench-plugin/src/analysis.ts index 65bd6183..750925e5 100644 --- a/packages/tinybench-plugin/src/analysis.ts +++ b/packages/tinybench-plugin/src/analysis.ts @@ -1,6 +1,7 @@ import { getCodspeedRunnerMode, InstrumentHooks, + Isolation, mongoMeasurement, optimizeFunction, optimizeFunctionSync, @@ -14,7 +15,11 @@ export function setupCodspeedAnalysisBench( bench: Bench, rootCallingFile: string, ): void { - const runner = new AnalysisBenchRunner(bench, rootCallingFile); + // Without instrumentation nothing is measured, so there is nothing to isolate. + const isolation = InstrumentHooks.isInstrumented() + ? Isolation.current() + : Isolation.shared(); + const runner = new AnalysisBenchRunner(bench, rootCallingFile, isolation); runner.setupBenchMethods(); } diff --git a/packages/tinybench-plugin/src/shared.ts b/packages/tinybench-plugin/src/shared.ts index 0fb811c5..14828bac 100644 --- a/packages/tinybench-plugin/src/shared.ts +++ b/packages/tinybench-plugin/src/shared.ts @@ -1,4 +1,4 @@ -import { setupCore, teardownCore } from "@codspeed/core"; +import { Isolation, setupCore, teardownCore } from "@codspeed/core"; import { Bench, Task } from "tinybench"; import { CapturedTaskData, getTaskData } from "./taskData"; import { getTaskUri } from "./uri"; @@ -8,10 +8,19 @@ declare const __VERSION__: string; export abstract class BaseBenchRunner { protected bench: Bench; protected rootCallingFile: string; - - constructor(bench: Bench, rootCallingFile: string) { + /** Runners that measure pass the isolation of this process; walltime shares it. */ + private readonly isolation: Isolation; + private readonly isChild: boolean; + + constructor( + bench: Bench, + rootCallingFile: string, + isolation: Isolation = Isolation.shared(), + ) { this.bench = bench; this.rootCallingFile = rootCallingFile; + this.isolation = isolation; + this.isChild = isolation.isChild(); } private setupBenchRun(): void { @@ -20,6 +29,10 @@ export abstract class BaseBenchRunner { } private logStart(): void { + // The parent already printed the banner, children stay quiet. + if (this.isChild) { + return; + } console.log( `[CodSpeed] running with @codspeed/tinybench v${__VERSION__} (${this.getModeName()})`, ); @@ -45,7 +58,12 @@ export abstract class BaseBenchRunner { protected finalizeBenchRun(): Task[] { teardownCore(); - console.log(`[CodSpeed] Done running ${this.bench.tasks.length} benches.`); + // The parent already printed the summary line, children stay quiet. + if (!this.isChild) { + console.log( + `[CodSpeed] Done running ${this.bench.tasks.length} benches.`, + ); + } return this.bench.tasks; } @@ -57,25 +75,40 @@ export abstract class BaseBenchRunner { public setupBenchMethods(): void { this.bench.run = async () => { + const tasks = this.selectTasks(); + // A child that owns none of this bench's tasks must not touch the + // instruments, the bench that owns its task does. + if (this.isChild && tasks.length === 0) { + return this.bench.tasks; + } this.setupBenchRun(); - for (const task of this.bench.tasks) { - const uri = this.getTaskUri(task); - await this.runTaskAsync(task, uri); + for (const { task, uri } of tasks) { + await this.isolation.run(uri, () => this.runTaskAsync(task, uri)); } return this.finalizeAsyncRun(); }; this.bench.runSync = () => { + const tasks = this.selectTasks(); + if (this.isChild && tasks.length === 0) { + return this.bench.tasks; + } this.setupBenchRun(); - for (const task of this.bench.tasks) { - const uri = this.getTaskUri(task); - this.runTaskSync(task, uri); + for (const { task, uri } of tasks) { + this.isolation.runSync(uri, () => this.runTaskSync(task, uri)); } return this.finalizeSyncRun(); }; } + + /** The tasks this process measures: all of them, or the isolated one. */ + private selectTasks(): { task: Task; uri: string }[] { + return this.bench.tasks + .map((task) => ({ task, uri: this.getTaskUri(task) })) + .filter(({ uri }) => this.isolation.shouldRun(uri)); + } } diff --git a/packages/tinybench-plugin/tests/fixtures/isolation.mjs b/packages/tinybench-plugin/tests/fixtures/isolation.mjs new file mode 100644 index 00000000..ec2424da --- /dev/null +++ b/packages/tinybench-plugin/tests/fixtures/isolation.mjs @@ -0,0 +1,46 @@ +// Run by `isolation.integ.test.ts`. Prints the PID that measures each benchmark. +import { InstrumentHooks } from "@codspeed/core"; +import { Bench } from "tinybench"; +import { withCodSpeed } from "../../dist/index.es5.js"; + +// There is no instrument hook runtime outside the CodSpeed runner, so pretend +// the hooks are active. The child processes run this file again, so they get +// the same override. +InstrumentHooks.isInstrumented = () => true; + +// The plugin reports every measured benchmark here, with the PID of the +// process that measured it. +const setExecutedBenchmark = InstrumentHooks.setExecutedBenchmark; +InstrumentHooks.setExecutedBenchmark = (pid, uri) => { + setExecutedBenchmark(pid, uri); + console.log(`executed ${pid} ${uri}`); +}; + +// tinybench calls each function once when it is added, so a failing benchmark +// may only throw once the measurement started. +let measuring = false; +const startBenchmark = InstrumentHooks.startBenchmark; +InstrumentHooks.startBenchmark = () => { + measuring = true; + startBenchmark(); +}; + +const bench = withCodSpeed(new Bench({ time: 1, warmupTime: 0 })); + +for (const name of ["first", "second", "third"]) { + bench.add(name, () => {}); +} + +if (process.env.ISOLATION_FIXTURE_FAIL !== undefined) { + bench.add("failing", () => { + if (measuring) { + throw new Error("benchmark failed"); + } + }); +} + +if (process.env.ISOLATION_FIXTURE_SYNC !== undefined) { + bench.runSync(); +} else { + await bench.run(); +} diff --git a/packages/tinybench-plugin/tests/index.integ.test.ts b/packages/tinybench-plugin/tests/index.integ.test.ts index 5e71eeb2..6de91c6e 100644 --- a/packages/tinybench-plugin/tests/index.integ.test.ts +++ b/packages/tinybench-plugin/tests/index.integ.test.ts @@ -41,6 +41,9 @@ vi.mock("@codspeed/core", async (importOriginal) => { beforeEach(() => { process.env.CODSPEED_ENV = "true"; process.env.CODSPEED_RUNNER_MODE = "instrumentation"; + // These tests drive the plugin in this process against a mocked core, so + // there is no benchmark process to spawn. + process.env.CODSPEED_PROCESS_ISOLATION = "false"; vi.clearAllMocks(); }); diff --git a/packages/tinybench-plugin/tests/isolation.integ.test.ts b/packages/tinybench-plugin/tests/isolation.integ.test.ts new file mode 100644 index 00000000..7f6d0059 --- /dev/null +++ b/packages/tinybench-plugin/tests/isolation.integ.test.ts @@ -0,0 +1,103 @@ +import { getV8Flags } from "@codspeed/core"; +import { spawnSync } from "child_process"; +import { fileURLToPath } from "url"; +import { describe, expect, it } from "vitest"; + +const FIXTURE = fileURLToPath( + new URL("./fixtures/isolation.mjs", import.meta.url), +); +const PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION"; +const BENCHMARK_COUNT = 3; +/** Fails a run whose benchmark process got stuck instead of blocking the suite. */ +const RUN_TIMEOUT_MS = 60_000; +const EXECUTED_LINE = /^executed (\d+) (.+)$/; + +interface FixtureRun { + status: number | null; + stderr: string; + /** PID that measured each benchmark, in the order they were measured. */ + pidsByUri: Map; + /** PID of the process the test started. */ + parentPid: number; +} + +interface FixtureOptions { + /** Opt out of isolation: every benchmark shares the process we start. */ + shared?: boolean; + failing?: boolean; + sync?: boolean; +} + +function runFixture({ + shared, + failing, + sync, +}: FixtureOptions = {}): FixtureRun { + const env: NodeJS.ProcessEnv = { + ...process.env, + CODSPEED_ENV: "true", + CODSPEED_RUNNER_MODE: "simulation", + }; + delete env[PROCESS_ISOLATION_ENV]; + if (shared) { + env[PROCESS_ISOLATION_ENV] = "false"; + } + if (failing) { + env.ISOLATION_FIXTURE_FAIL = "1"; + } + if (sync) { + env.ISOLATION_FIXTURE_SYNC = "1"; + } + + // The runner starts Node with these flags, and children must keep them. + const result = spawnSync(process.execPath, [...getV8Flags(), FIXTURE], { + env, + encoding: "utf8", + timeout: RUN_TIMEOUT_MS, + }); + + const pidsByUri = new Map(); + for (const line of result.stdout.split("\n")) { + const match = EXECUTED_LINE.exec(line); + if (match) { + pidsByUri.set(match[2], Number(match[1])); + } + } + return { + status: result.status, + stderr: result.stderr, + pidsByUri, + parentPid: result.pid, + }; +} + +describe.each([ + { name: "async", sync: false }, + { name: "sync", sync: true }, +])("process isolation ($name run)", ({ sync }) => { + it("measures every benchmark in its own process", () => { + const run = runFixture({ sync }); + + expect(run.status).toBe(0); + expect(run.pidsByUri.size).toBe(BENCHMARK_COUNT); + const pids = new Set(run.pidsByUri.values()); + expect(pids.size).toBe(BENCHMARK_COUNT); + // The parent only spawns children, it never measures. + expect(pids).not.toContain(run.parentPid); + }); + + it("measures every benchmark in the parent process when disabled", () => { + const run = runFixture({ shared: true, sync }); + + expect(run.status).toBe(0); + expect(run.pidsByUri.size).toBe(BENCHMARK_COUNT); + expect(new Set(run.pidsByUri.values())).toEqual(new Set([run.parentPid])); + }); + + it("fails the run when a benchmark throws", () => { + const run = runFixture({ failing: true, sync }); + + expect(run.status).not.toBe(0); + expect(run.stderr).toContain("benchmark failed"); + }); +}); diff --git a/packages/vitest-plugin/src/__tests__/instrumented.test.ts b/packages/vitest-plugin/src/__tests__/instrumented.test.ts index ec1a1fd9..2e7f54c6 100644 --- a/packages/vitest-plugin/src/__tests__/instrumented.test.ts +++ b/packages/vitest-plugin/src/__tests__/instrumented.test.ts @@ -6,6 +6,8 @@ import { getBenchFn, getBenchOptions } from "../legacy/compat"; const coreMocks = vi.hoisted(() => { return { InstrumentHooks: { + // A plain function: the mocks above are reset between tests. + isInstrumented: () => true, startBenchmark: vi.fn(), stopBenchmark: vi.fn(), setExecutedBenchmark: vi.fn(), @@ -21,6 +23,10 @@ const coreMocks = vi.hoisted(() => { global.eval = vi.fn(); +// The runner is driven in this process against a mocked core, so there is no +// benchmark process to spawn. +process.env.CODSPEED_PROCESS_ISOLATION = "false"; + vi.mock("@codspeed/core", async (importOriginal) => { const mod = await importOriginal(); return { ...mod, ...coreMocks }; diff --git a/packages/vitest-plugin/src/__tests__/isolation.test.ts b/packages/vitest-plugin/src/__tests__/isolation.test.ts new file mode 100644 index 00000000..30275341 --- /dev/null +++ b/packages/vitest-plugin/src/__tests__/isolation.test.ts @@ -0,0 +1,94 @@ +import { spawnSync } from "child_process"; +import { mkdtempSync, readFileSync, rmSync } from "fs"; +import { createRequire } from "module"; +import { tmpdir } from "os"; +import { dirname, join } from "path"; +import { fileURLToPath } from "url"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +const FIXTURE_DIR = fileURLToPath( + new URL("../../tests/fixtures/isolation", import.meta.url), +); +const PRELOAD = join(FIXTURE_DIR, "stub.mjs"); +const VITEST_BIN = join( + dirname(createRequire(import.meta.url).resolve("vitest/package.json")), + "vitest.mjs", +); +const PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION"; +const BENCHMARK_COUNT = 3; +const RUN_TIMEOUT_MS = 120_000; +const EXECUTED_LINE = /^executed (\d+) (.+)$/; + +let logDir: string; +let logFile: string; + +beforeEach(() => { + logDir = mkdtempSync(join(tmpdir(), "codspeed-isolation-")); + logFile = join(logDir, "executed.log"); +}); + +afterEach(() => { + rmSync(logDir, { recursive: true, force: true }); +}); + +function runFixture({ + isolation, + failing, +}: { isolation?: string; failing?: boolean } = {}) { + const env: NodeJS.ProcessEnv = { + ...process.env, + CODSPEED_ENV: "true", + CODSPEED_RUNNER_MODE: "simulation", + ISOLATION_LOG_FILE: logFile, + NODE_OPTIONS: [process.env.NODE_OPTIONS, `--import ${PRELOAD}`] + .filter(Boolean) + .join(" "), + }; + env[PROCESS_ISOLATION_ENV] = isolation; + if (failing) { + env.ISOLATION_FIXTURE_FAIL = "1"; + } + + const result = spawnSync(process.execPath, [VITEST_BIN, "bench", "--run"], { + cwd: FIXTURE_DIR, + env, + encoding: "utf8", + timeout: RUN_TIMEOUT_MS, + }); + + const pidsByUri = new Map(); + for (const line of readFileSync(logFile, "utf8").split("\n")) { + const match = EXECUTED_LINE.exec(line); + if (match) { + pidsByUri.set(match[2], Number(match[1])); + } + } + return { status: result.status, pidsByUri, parentPid: result.pid }; +} + +describe("process isolation", () => { + it.each([ + { isolation: undefined, processes: BENCHMARK_COUNT }, + { isolation: "false", processes: 1 }, + ])( + "measures benchmarks in $processes processes (isolation=$isolation)", + ({ isolation, processes }) => { + const run = runFixture({ isolation }); + expect(run.status).toBe(0); + expect(run.pidsByUri.size).toBe(BENCHMARK_COUNT); + const pids = new Set(run.pidsByUri.values()); + expect(pids.size).toBe(processes); + expect(pids).not.toContain(run.parentPid); + }, + RUN_TIMEOUT_MS, + ); + + it( + "fails the run when an isolated benchmark throws", + () => { + const run = runFixture({ failing: true }); + expect(run.status).toBe(1); + }, + RUN_TIMEOUT_MS, + ); +}); diff --git a/packages/vitest-plugin/src/__tests__/v5-walltime.test.ts b/packages/vitest-plugin/src/__tests__/v5-walltime.test.ts new file mode 100644 index 00000000..a6838a57 --- /dev/null +++ b/packages/vitest-plugin/src/__tests__/v5-walltime.test.ts @@ -0,0 +1,104 @@ +import type * as Core from "@codspeed/core"; +import { fromPartial } from "@total-typescript/shoehorn"; +import type { + BenchmarkGroup, + BenchRegistrationInput, + BenchRunOptions, +} from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import type * as Instrument from "../instrument"; +import provider from "../v5/provider"; + +const coreMocks = vi.hoisted(() => ({ + InstrumentHooks: { + startBenchmark: vi.fn(), + stopBenchmark: vi.fn(), + setExecutedBenchmark: vi.fn(), + addMarker: vi.fn(), + currentTimestamp: vi.fn(() => 0n), + }, + getInstrumentMode: vi.fn(() => "walltime"), + getGitDir: vi.fn(() => "/repo"), + setupCore: vi.fn(), + teardownCore: vi.fn(), + collectGarbage: vi.fn(), +})); + +vi.mock("@codspeed/core", async (importOriginal) => { + const mod = await importOriginal(); + return { ...mod, ...coreMocks }; +}); + +vi.mock("../instrument", async (importOriginal) => { + const mod = await importOriginal(); + return { ...mod, writeAndLogWalltimeResults: vi.fn() }; +}); + +const ITERATIONS = 3; + +function run(registrations: BenchRegistrationInput[]) { + const options: BenchRunOptions = { + iterations: ITERATIONS, + time: 0, + warmup: false, + }; + return provider.run( + fromPartial({ + test: fromPartial({ + file: { filepath: "/repo/a.bench.ts" }, + fullTestName: "suite > test", + context: {}, + }), + registrations, + options, + }), + ); +} + +describe("v5 walltime provider", () => { + afterEach(() => { + vi.unstubAllEnvs(); + vi.clearAllMocks(); + }); + + it("collects garbage after the user's beforeEach, before every round", async () => { + vi.stubEnv("CODSPEED_WALLTIME_GC_BETWEEN_ROUNDS", "true"); + const calls: string[] = []; + coreMocks.collectGarbage.mockImplementation(() => calls.push("gc")); + + await run([ + fromPartial({ + name: "bench", + fn: () => { + calls.push("fn"); + }, + fnOpts: { beforeEach: () => calls.push("beforeEach") }, + }), + ]); + + expect(calls.length).toBeGreaterThanOrEqual(ITERATIONS * 3); + for (let i = 0; i < calls.length; i += 3) { + expect(calls.slice(i, i + 3)).toEqual(["beforeEach", "gc", "fn"]); + } + }); + + it("does not collect garbage when disabled", async () => { + await run([ + fromPartial({ name: "bench", fn: () => {} }), + ]); + + expect(coreMocks.collectGarbage).not.toHaveBeenCalled(); + }); + + it("rejects forks", async () => { + vi.stubEnv("CODSPEED_WALLTIME_FORKS", "2"); + + await expect( + run([ + fromPartial({ name: "bench", fn: () => {} }), + ]), + ).rejects.toThrow( + "[CodSpeed] CODSPEED_WALLTIME_FORKS is not supported with Vitest 5 yet", + ); + }); +}); diff --git a/packages/vitest-plugin/src/__tests__/walltime.test.ts b/packages/vitest-plugin/src/__tests__/walltime.test.ts new file mode 100644 index 00000000..28a9dc0a --- /dev/null +++ b/packages/vitest-plugin/src/__tests__/walltime.test.ts @@ -0,0 +1,294 @@ +import { fromPartial } from "@total-typescript/shoehorn"; +import { + afterEach, + beforeEach, + describe, + expect, + it, + vi, + type RunnerTestSuite, +} from "vitest"; +import { WalltimeRunner } from "../legacy/walltime"; + +const coreMocks = vi.hoisted(() => { + return { + InstrumentHooks: { + startBenchmark: vi.fn(), + stopBenchmark: vi.fn(), + setExecutedBenchmark: vi.fn(), + addMarker: vi.fn(), + currentTimestamp: vi.fn(() => 0n), + }, + setupCore: vi.fn(), + writeWalltimeResults: vi.fn(), + collectGarbage: vi.fn(), + isGcBetweenRoundsEnabled: vi.fn(() => false), + getWalltimeForks: vi.fn((): number | undefined => undefined), + isFork: vi.fn(() => false), + runForks: vi.fn((): number[] => []), + writeForkSamples: vi.fn(), + }; +}); + +const BENCH_OPTIONS = { + iterations: 3, + time: 0, + warmupIterations: 2, + warmupTime: 0, +}; + +/** Calls of the bench functions and of the mocked GC, in order. */ +const events = vi.hoisted((): string[] => []); + +vi.mock("@codspeed/core", async (importOriginal) => { + const mod = await importOriginal(); + return { ...mod, ...coreMocks }; +}); + +console.warn = vi.fn(); + +// Fully mocked: it loads the Vitest 3/4 runner, absent from the Vitest 5 devDependency. +vi.mock("../legacy/compat", () => { + interface SuiteLike { + id: string; + type: string; + name: string; + mode?: string; + result?: unknown; + tasks: SuiteLike[]; + } + + // Like Vitest, keeps each bench's function next to its task. + const benchFns = new WeakMap void>(); + const getBenchFn = (task: SuiteLike) => { + if (!benchFns.has(task)) { + benchFns.set(task, () => { + events.push(`fn:${task.name}`); + }); + } + return benchFns.get(task)!; + }; + + // Mirrors Vitest's benchmark suite execution: sub-suites run first, then + // the suite's own benches run after a `suite-prepare` event for the suite, + // each as a tinybench `Task` that is warmed up then run. + class NodeBenchmarkRunner { + async importTinybench() { + return import("tinybench"); + } + + async runSuite(suite: SuiteLike): Promise { + await this.runBenchmarkSuite(suite); + } + + private async runBenchmarkSuite(suite: SuiteLike): Promise { + for (const task of suite.tasks) { + if (task.type === "suite") { + await this.runBenchmarkSuite(task); + } + } + + const benches = suite.tasks.filter((task) => task.type === "test"); + if (benches.length === 0) { + return; + } + + await (this as unknown as WalltimeRunner).onTaskUpdate( + [], + [[suite.id, "suite-prepare", undefined]], + ); + const { Bench, Task } = await this.importTinybench(); + for (const task of benches) { + if (task.mode !== undefined && task.mode !== "run") { + continue; + } + const tinybenchTask = new Task( + new Bench({ + iterations: 3, + time: 0, + warmupIterations: 2, + warmupTime: 0, + }), + task.name, + getBenchFn(task), + ); + await tinybenchTask.warmup(); + await tinybenchTask.run(); + task.result = { state: "pass", benchmark: tinybenchTask.result }; + } + } + } + + return { + NodeBenchmarkRunner, + getBenchFn, + getBenchOptions: vi.fn(() => ({ + iterations: 3, + time: 0, + warmupIterations: 2, + warmupTime: 0, + })), + }; +}); + +const FILE_URI = "packages/vitest-plugin/src/__tests__/walltime.test.ts"; + +const NESTED_URI = `${FILE_URI}::nested suite::nested bench`; +const TOP_LEVEL_URI = `${FILE_URI}::top-level bench`; + +function createSuite(): RunnerTestSuite { + return fromPartial({ + id: "file", + type: "suite", + file: { filepath: __filename }, + name: "walltime.test.ts", + tasks: [ + { + id: "nested", + type: "suite", + name: "nested suite", + tasks: [ + { type: "test", name: "nested bench", meta: { benchmark: true } }, + ], + }, + { type: "test", name: "top-level bench", meta: { benchmark: true } }, + ], + }); +} + +function writtenBenchmarks() { + return coreMocks.writeWalltimeResults.mock.calls.flatMap( + ([benchmarks]) => benchmarks, + ); +} + +describe("WalltimeRunner", () => { + beforeEach(() => { + vi.clearAllMocks(); + events.length = 0; + coreMocks.collectGarbage.mockImplementation(() => { + events.push("gc"); + }); + }); + + afterEach(() => { + coreMocks.isGcBetweenRoundsEnabled.mockReturnValue(false); + coreMocks.getWalltimeForks.mockReturnValue(undefined); + coreMocks.isFork.mockReturnValue(false); + delete process.env.CODSPEED_ISOLATED_BENCHMARK; + }); + + it("prefixes top-level and nested bench URIs with the file path", async () => { + const runner = new WalltimeRunner(fromPartial({})); + const suite = fromPartial({ + id: "file", + type: "suite", + file: { filepath: __filename }, + name: "walltime.test.ts", + tasks: [ + { + id: "nested", + type: "suite", + name: "nested suite", + tasks: [ + { type: "test", name: "nested bench", meta: { benchmark: true } }, + ], + }, + { type: "test", name: "top-level bench", meta: { benchmark: true } }, + ], + }); + + await runner.runSuite(suite); + + const uris = coreMocks.InstrumentHooks.setExecutedBenchmark.mock.calls.map( + ([, uri]) => uri, + ); + expect(uris).toEqual([NESTED_URI, TOP_LEVEL_URI]); + }); + + it("collects garbage before every warmup and measured round", async () => { + coreMocks.isGcBetweenRoundsEnabled.mockReturnValue(true); + const runner = new WalltimeRunner(fromPartial({})); + + await runner.runSuite(createSuite()); + + // Every call of the bench function (including tinybench's async probe, + // which also runs `beforeEach` first) is preceded by exactly one GC. + const rounds = BENCH_OPTIONS.warmupIterations + BENCH_OPTIONS.iterations; + for (const name of ["nested bench", "top-level bench"]) { + const benchEvents = events.filter( + (event) => event === "gc" || event === `fn:${name}`, + ); + const fnCalls = benchEvents.filter((event) => event !== "gc").length; + expect(fnCalls).toBeGreaterThanOrEqual(rounds); + } + expect(events.length % 2).toBe(0); + for (let i = 0; i < events.length; i += 2) { + expect(events[i]).toBe("gc"); + expect(events[i + 1]).toMatch(/^fn:/); + } + }); + + it("merges the samples of every fork into one result", async () => { + const forkSamples = [3, 1, 2, 5, 4]; + coreMocks.getWalltimeForks.mockReturnValue(3); + coreMocks.runForks.mockReturnValue(forkSamples); + const runner = new WalltimeRunner(fromPartial({})); + + await runner.runSuite(createSuite()); + + expect(events).toEqual([]); + expect(coreMocks.InstrumentHooks.startBenchmark).not.toHaveBeenCalled(); + expect( + coreMocks.InstrumentHooks.setExecutedBenchmark, + ).not.toHaveBeenCalled(); + expect(coreMocks.runForks.mock.calls).toEqual([ + [NESTED_URI, 3], + [TOP_LEVEL_URI, 3], + ]); + const benchmarks = writtenBenchmarks(); + expect(benchmarks.map((benchmark) => benchmark.uri)).toEqual([ + NESTED_URI, + TOP_LEVEL_URI, + ]); + for (const benchmark of benchmarks) { + expect(benchmark.stats.rounds).toBe(forkSamples.length); + expect(benchmark.stats.min_ns).toBe(1_000_000); + expect(benchmark.stats.max_ns).toBe(5_000_000); + } + }); + + it("runs only the isolated bench in a fork and hands its samples over", async () => { + coreMocks.isFork.mockReturnValue(true); + process.env.CODSPEED_ISOLATED_BENCHMARK = TOP_LEVEL_URI; + const runner = new WalltimeRunner(fromPartial({})); + + await runner.runSuite(createSuite()); + + expect(new Set(events)).toEqual(new Set(["fn:top-level bench"])); + expect( + coreMocks.InstrumentHooks.setExecutedBenchmark.mock.calls.map( + ([, uri]) => uri, + ), + ).toEqual([TOP_LEVEL_URI]); + expect(coreMocks.writeForkSamples).toHaveBeenCalledTimes(1); + const [samples] = coreMocks.writeForkSamples.mock.calls[0]; + expect(samples).toHaveLength(BENCH_OPTIONS.iterations); + expect(coreMocks.writeWalltimeResults).not.toHaveBeenCalled(); + }); + + it("runs half the warmup and measured rounds in each of several forks", async () => { + coreMocks.isFork.mockReturnValue(true); + coreMocks.getWalltimeForks.mockReturnValue(3); + process.env.CODSPEED_ISOLATED_BENCHMARK = TOP_LEVEL_URI; + const runner = new WalltimeRunner(fromPartial({})); + + await runner.runSuite(createSuite()); + + const [samples] = coreMocks.writeForkSamples.mock.calls[0]; + expect(samples).toHaveLength(Math.ceil(BENCH_OPTIONS.iterations / 2)); + // Warmup rounds, measured rounds, and tinybench's async probe. + const calls = Math.ceil(BENCH_OPTIONS.warmupIterations / 2) + Math.ceil(BENCH_OPTIONS.iterations / 2) + 1; + expect(events).toHaveLength(calls); + }); +}); diff --git a/packages/vitest-plugin/src/index.ts b/packages/vitest-plugin/src/index.ts index cb3af1ae..72f48223 100644 --- a/packages/vitest-plugin/src/index.ts +++ b/packages/vitest-plugin/src/index.ts @@ -1,8 +1,11 @@ import { + getGitDir, getInstrumentMode, getV8Flags, InstrumentHooks, + ISOLATED_BENCHMARK_ENV, mongoMeasurement, + recordIsolationCommand, SetupInstrumentsRequestBody, SetupInstrumentsResponse, } from "@codspeed/core"; @@ -37,7 +40,26 @@ export default function codspeedPlugin(): Plugin { return true; }, enforce: "post", - config(userConfig): ViteUserConfig { + config(userConfig: ViteUserConfig): ViteUserConfig { + // Vitest measures in forked workers, whose argv is not the command that + // starts the benchmarks. Record the command of this main process so the + // workers can run it again for each benchmark. Workers inherit the env. + recordIsolationCommand(); + const uri = process.env[ISOLATED_BENCHMARK_ENV]; + if (uri !== undefined) { + const gitDir = getGitDir(userConfig.root ?? process.cwd()); + if (gitDir === undefined) { + throw new Error("Could not find a git repository"); + } + // Replace rather than merge include globs: isolated children must not + // transform and import unrelated benchmark files. + const file = join(gitDir, uri.split("::")[0]) + .replace(/\\/g, "/") + .replace(/[\\*?{}()[\]!+@]/g, "\\$&"); + userConfig.test ??= {}; + userConfig.test.benchmark ??= {}; + userConfig.test.benchmark.include = [file]; + } return { test: { pool: "forks", @@ -45,7 +67,7 @@ export default function codspeedPlugin(): Plugin { ...resolveVitestBackend().getBenchmarkTestConfig( getV8Flags(), resolveFile, - (userConfig as ViteUserConfig).test, + userConfig.test, ), }, }; diff --git a/packages/vitest-plugin/src/instrument.ts b/packages/vitest-plugin/src/instrument.ts index 24039088..cfb816bc 100644 --- a/packages/vitest-plugin/src/instrument.ts +++ b/packages/vitest-plugin/src/instrument.ts @@ -1,5 +1,6 @@ import { calculateQuantiles, + collectGarbage, InstrumentHooks, MARKER_TYPE_BENCHMARK_END, MARKER_TYPE_BENCHMARK_START, @@ -28,6 +29,30 @@ export interface TinybenchBench { teardown: TinybenchHook; } +/** A tinybench per-round hook (`beforeEach`/`afterEach`), called with the task as `this`. */ +export type TinybenchRoundHook = ( + this: unknown, + mode: "run" | "warmup", +) => Promise | void; + +/** + * Returns a `beforeEach` hook that runs `hook` and then a full GC. tinybench + * calls `beforeEach` outside the timed call, so the GC starts every round on + * a collected heap without being measured. + */ +export function withGcAfter( + hook: TinybenchRoundHook | undefined, +): TinybenchRoundHook { + return function (this: unknown, mode) { + const result: unknown = hook?.call(this, mode); + // Any thenable, not only this realm's Promise, so the GC waits for it. + if (typeof (result as PromiseLike | undefined)?.then === "function") { + return Promise.resolve(result).then(collectGarbage); + } + collectGarbage(); + }; +} + interface TinybenchStatistics { min: number; max: number; diff --git a/packages/vitest-plugin/src/legacy/analysis.ts b/packages/vitest-plugin/src/legacy/analysis.ts index f71f0c0a..fea4cc6f 100644 --- a/packages/vitest-plugin/src/legacy/analysis.ts +++ b/packages/vitest-plugin/src/legacy/analysis.ts @@ -1,5 +1,6 @@ import { InstrumentHooks, + Isolation, logDebug, mongoMeasurement, optimizeFunction, @@ -41,63 +42,88 @@ async function runAnalysisBench( suite: RunnerTestSuite, currentSuiteName: string, tinybenchModule: Tinybench, + isolation: Isolation, ) { const uri = `${currentSuiteName}::${benchmark.name}`; - const fn = getBenchFn(benchmark); - - // Constructing a Bench applies tinybench's no-op defaults for the setup and - // teardown hooks and gives them the Task they expect. The bench itself is - // never run: this runner drives the benchmark function directly. - const bench = new tinybenchModule.Bench(getBenchOptions(benchmark)); - const task = new tinybenchModule.Task(bench, benchmark.name, fn); - - await bench.setup(task, "warmup"); - await optimizeFunction(async () => { + if (!isolation.shouldRun(uri)) return; + + await isolation.run(uri, async () => { + const fn = getBenchFn(benchmark); + + // Constructing a Bench applies tinybench's no-op defaults for the setup and + // teardown hooks and gives them the Task they expect. The bench itself is + // never run: this runner drives the benchmark function directly. + const bench = new tinybenchModule.Bench(getBenchOptions(benchmark)); + const task = new tinybenchModule.Task(bench, benchmark.name, fn); + + await bench.setup(task, "warmup"); + await optimizeFunction(async () => { + await callSuiteHook(suite, benchmark, "beforeEach"); + await fn(); + await callSuiteHook(suite, benchmark, "afterEach"); + }); + await bench.teardown(task, "warmup"); + + await bench.setup(task, "run"); await callSuiteHook(suite, benchmark, "beforeEach"); - await fn(); + await mongoMeasurement.start(uri); + global.gc?.(); + await wrapWithRootFrame(async () => { + InstrumentHooks.startBenchmark(); + await fn(); + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(process.pid, uri); + })(); + await mongoMeasurement.stop(uri); await callSuiteHook(suite, benchmark, "afterEach"); + await bench.teardown(task, "run"); + + logCodSpeed(`${uri} done`); }); - await bench.teardown(task, "warmup"); - - await bench.setup(task, "run"); - await callSuiteHook(suite, benchmark, "beforeEach"); - await mongoMeasurement.start(uri); - global.gc?.(); - await wrapWithRootFrame(async () => { - InstrumentHooks.startBenchmark(); - await fn(); - InstrumentHooks.stopBenchmark(); - InstrumentHooks.setExecutedBenchmark(process.pid, uri); - })(); - await mongoMeasurement.stop(uri); - await callSuiteHook(suite, benchmark, "afterEach"); - await bench.teardown(task, "run"); - - logCodSpeed(`${uri} done`); } async function runAnalysisBenchmarkSuite( suite: RunnerTestSuite, tinybenchModule: Tinybench, + isolation: Isolation, parentSuiteName?: string, ) { const currentSuiteName = parentSuiteName ? parentSuiteName + "::" + suite.name : suite.name; - await callSuiteHook(suite, suite, "beforeAll"); + // A process that only starts children leaves the suite hooks to them: + // running them here too would hold their resources (e.g. a port) while a + // child sets them up again. + const runsSuiteHooks = isolation.mode.kind !== "ProcessPerBenchmark"; + if (runsSuiteHooks) { + await callSuiteHook(suite, suite, "beforeAll"); + } for (const task of suite.tasks) { if (task.mode !== "run") continue; if (task.type === "suite") { - await runAnalysisBenchmarkSuite(task, tinybenchModule, currentSuiteName); + await runAnalysisBenchmarkSuite( + task, + tinybenchModule, + isolation, + currentSuiteName, + ); } else if (isVitestTaskBenchmark(task)) { - await runAnalysisBench(task, suite, currentSuiteName, tinybenchModule); + await runAnalysisBench( + task, + suite, + currentSuiteName, + tinybenchModule, + isolation, + ); } } - await callSuiteHook(suite, suite, "afterAll"); + if (runsSuiteHooks) { + await callSuiteHook(suite, suite, "afterAll"); + } } export class AnalysisRunner extends NodeBenchmarkRunner { @@ -107,9 +133,24 @@ export class AnalysisRunner extends NodeBenchmarkRunner { patchRootSuiteWithFullFilePath(suite); - logCodSpeed(`running suite ${suite.name}`); - await runAnalysisBenchmarkSuite(suite, await this.importTinybench()); - logCodSpeed(`running suite ${suite.name} done`); + // Without instrumentation nothing is measured, so there is nothing to isolate. + const isolation = InstrumentHooks.isInstrumented() + ? Isolation.current() + : Isolation.shared(); + // The parent already printed the suite lines, children stay quiet. + const isChild = isolation.isChild(); + + if (!isChild) { + logCodSpeed(`running suite ${suite.name}`); + } + await runAnalysisBenchmarkSuite( + suite, + await this.importTinybench(), + isolation, + ); + if (!isChild) { + logCodSpeed(`running suite ${suite.name} done`); + } teardownCore(); } diff --git a/packages/vitest-plugin/src/legacy/walltime.ts b/packages/vitest-plugin/src/legacy/walltime.ts index 09271df8..9b2666b0 100644 --- a/packages/vitest-plugin/src/legacy/walltime.ts +++ b/packages/vitest-plugin/src/legacy/walltime.ts @@ -1,4 +1,14 @@ -import { setupCore, wrapWithRootFrame } from "@codspeed/core"; +import { + FORK_BUDGET_FRACTION, + getWalltimeForks, + isFork, + isGcBetweenRoundsEnabled, + ISOLATED_BENCHMARK_ENV, + runForks, + setupCore, + wrapWithRootFrame, + writeForkSamples, +} from "@codspeed/core"; import { RunnerTaskEventPack, RunnerTaskResultPack, @@ -6,29 +16,69 @@ import { } from "vitest"; import { installInstrumentHooks, + withGcAfter, writeAndLogWalltimeResults, type Tinybench, type TinybenchBench, + type TinybenchRoundHook, } from "../instrument"; -import { patchRootSuiteWithFullFilePath } from "./common"; -import { NodeBenchmarkRunner } from "./compat"; +import { + isVitestTaskBenchmark, + patchRootSuiteWithFullFilePath, +} from "./common"; +import { getBenchFn, NodeBenchmarkRunner, type BenchmarkTask } from "./compat"; import { extractBenchmarkResults } from "./walltime-utils"; /** A tinybench v2 task, whose `fn` is a plain, reassignable property. */ interface TinybenchTask { + name: string; fn: () => unknown; + opts: { beforeEach?: TinybenchRoundHook }; + loop: (time: number, iterations: number) => Promise; +} + +interface TinybenchLoopResult { + samples?: number[]; + error?: unknown; } +const patchedPrototypes = new WeakSet(); +let activeRunner: WalltimeRunner | null = null; + /** * Lets tinybench run the benches through Vitest's default benchmark execution, * instrumenting each measured loop, then extracts the results from the suite * tree afterwards. */ export class WalltimeRunner extends NodeBenchmarkRunner { - private isTinybenchHookedWithCodspeed = false; private suiteUris = new Map(); /// Suite ID of the currently running suite, to allow constructing the URI in the context of tinybench tasks private currentSuiteId: string | null = null; + /// Forks per benchmark in the parent process, unset when not forking + parentForks: number | undefined; + /// URIs of the benches of the running file, by bench name, for the forks parent + private forkUris = new Map>(); + + isForkParent(): boolean { + return this.parentForks !== undefined; + } + + /** + * URI of a bench in the forks parent. Unlike `getBenchmarkUri`, it doesn't + * depend on the `suite-prepare` update, which Vitest sends throttled and may + * not have arrived when the parent skips the warmup. + */ + forkUri(name: string, fn: unknown): string { + const matches = (this.forkUris.get(name) ?? []).filter( + (bench) => bench.fn === fn, + ); + if (matches.length !== 1) { + throw new Error( + `[CodSpeed] cannot tell which benchmark "${name}" is: ${matches.length} benchmarks share its name and function`, + ); + } + return matches[0].uri; + } async runSuite(suite: RunnerTestSuite): Promise { patchRootSuiteWithFullFilePath(suite); @@ -36,8 +86,30 @@ export class WalltimeRunner extends NodeBenchmarkRunner { setupCore(); + this.parentForks = isFork() ? undefined : getWalltimeForks(); + const benches = benchUris(suite); + if (isFork()) { + const target = process.env[ISOLATED_BENCHMARK_ENV]; + for (const { task, uri } of benches) { + if (uri !== target) task.mode = "skip"; + } + } + this.forkUris.clear(); + if (this.isForkParent()) { + for (const { task, uri } of benches) { + const sameName = this.forkUris.get(task.name) ?? []; + sameName.push({ fn: getBenchFn(task), uri }); + this.forkUris.set(task.name, sameName); + } + } + await super.runSuite(suite); + // The parent writes the merged results of every fork. + if (isFork()) { + return; + } + const benchmarks = await extractBenchmarkResults(suite); if (benchmarks.length === 0) { console.warn( @@ -51,10 +123,10 @@ export class WalltimeRunner extends NodeBenchmarkRunner { private populateBenchmarkUris(suite: RunnerTestSuite, parentPath = ""): void { const currentPath = parentPath !== "" ? `${parentPath}::${suite.name}` : suite.name; + this.suiteUris.set(suite.id, currentPath); for (const task of suite.tasks) { if (task.type === "suite") { - this.suiteUris.set(task.id, `${currentPath}::${task.name}`); this.populateBenchmarkUris(task, currentPath); } } @@ -72,12 +144,17 @@ export class WalltimeRunner extends NodeBenchmarkRunner { const tinybench = await super.importTinybench(); // `tinybench` is a frozen ES module namespace, so the `Bench` export cannot - // be reassigned. Mutating the shared `Task.prototype` in place is allowed - // and only needs to happen once; the instrumented `Bench` is handed back + // be reassigned. Mutating the shared `Task.prototype` in place is allowed; + // it outlives runner instances, so it is patched once and routes to the + // runner that imported tinybench last. The rounds patch goes on top, so it + // still sees the bench's own `fn`. The instrumented `Bench` is handed back // through a fresh module-shaped object that Vitest destructures from. - if (!this.isTinybenchHookedWithCodspeed) { - this.isTinybenchHookedWithCodspeed = true; - this.patchTaskWithRootFrame(tinybench); + // eslint-disable-next-line @typescript-eslint/no-this-alias + activeRunner = this; + if (!patchedPrototypes.has(tinybench.Task.prototype)) { + patchedPrototypes.add(tinybench.Task.prototype); + patchTaskWithRootFrame(tinybench); + patchTaskRounds(tinybench); } return { @@ -86,28 +163,6 @@ export class WalltimeRunner extends NodeBenchmarkRunner { }; } - /** - * Wrap each task's function with the root frame so collected stacks can be - * attributed to a benchmark. - */ - private patchTaskWithRootFrame(tinybench: Tinybench): void { - const originalRun = tinybench.Task.prototype.run; - - tinybench.Task.prototype.run = async function () { - const task = this as unknown as TinybenchTask; - const originalFn = task.fn; - task.fn = wrapWithRootFrame(() => originalFn.call(task)); - - try { - await originalRun.call(this); - } finally { - task.fn = originalFn; - } - - return this; - }; - } - private createInstrumentedBench( tinybench: Tinybench, ): typeof tinybench.Bench { @@ -119,6 +174,9 @@ export class WalltimeRunner extends NodeBenchmarkRunner { // eslint-disable-next-line @typescript-eslint/no-explicit-any constructor(...benchArgs: any[]) { super(...benchArgs); + if (runner.isForkParent()) { + return; + } installInstrumentHooks(this as unknown as TinybenchBench, (taskName) => runner.getBenchmarkUri(taskName), ); @@ -143,4 +201,139 @@ export class WalltimeRunner extends NodeBenchmarkRunner { } } +/** + * Wrap each task's function with the root frame so collected stacks can be + * attributed to a benchmark. + */ +function patchTaskWithRootFrame(tinybench: Tinybench): void { + const originalRun = tinybench.Task.prototype.run; + + tinybench.Task.prototype.run = async function () { + const task = this as unknown as TinybenchTask; + const originalFn = task.fn; + task.fn = wrapWithRootFrame(() => originalFn.call(task)); + + try { + await originalRun.call(this); + } finally { + task.fn = originalFn; + } + + return this; + }; +} + +/** + * Forks parent: skip warmup and replace the measured loop with the samples + * of every fork. Fork child: run its share of the warmup and measurement + * budget, then hand the measured samples to the parent. + * Otherwise: optionally collect garbage before every round. + */ +function patchTaskRounds(tinybench: Tinybench): void { + const originalWarmup = tinybench.Task.prototype.warmup; + const originalRun = tinybench.Task.prototype.run; + const gcComposed = new WeakSet(); + + const composeGc = (task: TinybenchTask) => { + if (gcComposed.has(task) || !isGcBetweenRoundsEnabled()) { + return; + } + gcComposed.add(task); + task.opts.beforeEach = withGcAfter(task.opts.beforeEach); + }; + + tinybench.Task.prototype.warmup = async function () { + if (activeRunner!.isForkParent()) { + return; + } + const task = this as unknown as TinybenchTask; + composeGc(task); + if (!isFork()) { + return originalWarmup.call(this); + } + const originalLoop = task.loop; + task.loop = (time, iterations) => + originalLoop.call(task, ...forkBudget(time, iterations)); + try { + return await originalWarmup.call(this); + } finally { + task.loop = originalLoop; + } + }; + + tinybench.Task.prototype.run = async function () { + const task = this as unknown as TinybenchTask; + const originalLoop = task.loop; + + if (activeRunner!.isForkParent()) { + const runner = activeRunner!; + // Read before the root frame patch wraps `fn` for the run. + const fn = task.fn; + // Returned, not thrown: tinybench only reports errors from its loop, and + // a rejected run() would leave Vitest waiting forever. + task.loop = async () => { + try { + return { + samples: await runForks( + runner.forkUri(task.name, fn), + runner.parentForks!, + ), + }; + } catch (error) { + return { error }; + } + }; + } else { + composeGc(task); + if (isFork()) { + task.loop = async (time, iterations) => { + const result = await originalLoop.call( + task, + ...forkBudget(time, iterations), + ); + if (result.samples) { + // Copied: tinybench sorts and Vitest may truncate the array. + writeForkSamples([...result.samples]); + } + return result; + }; + } + } + + try { + return await originalRun.call(this); + } finally { + task.loop = originalLoop; + } + }; +} + +/** The `[time, iterations]` budget of one fork: shared when there are several. */ +function forkBudget(time: number, iterations: number): [number, number] { + if ((getWalltimeForks() ?? 1) <= 1) { + return [time, iterations]; + } + return [ + time * FORK_BUDGET_FRACTION, + Math.ceil(iterations * FORK_BUDGET_FRACTION), + ]; +} + +/** Every bench of `suite` with its URI, built like `extractBenchmarkResults`. */ +function benchUris( + suite: RunnerTestSuite, + parentPath = "", +): Array<{ task: BenchmarkTask; uri: string }> { + const currentPath = parentPath ? `${parentPath}::${suite.name}` : suite.name; + return suite.tasks.flatMap((task) => { + if (task.type === "suite") { + return benchUris(task, currentPath); + } + if (!isVitestTaskBenchmark(task)) { + return []; + } + return [{ task, uri: `${currentPath}::${task.name}` }]; + }); +} + export default WalltimeRunner; diff --git a/packages/vitest-plugin/src/v5/provider.ts b/packages/vitest-plugin/src/v5/provider.ts index ef38fe0e..3f793507 100644 --- a/packages/vitest-plugin/src/v5/provider.ts +++ b/packages/vitest-plugin/src/v5/provider.ts @@ -1,11 +1,15 @@ import { getGitDir, getInstrumentMode, + getWalltimeForks, InstrumentHooks, + isGcBetweenRoundsEnabled, + Isolation, optimizeFunction, optimizeFunctionSync, setupCore, teardownCore, + WALLTIME_FORKS_ENV, wrapWithRootFrame, wrapWithRootFrameSync, type Benchmark, @@ -22,9 +26,11 @@ import type { import { installInstrumentHooks, tinybenchTaskToBenchmark, + withGcAfter, writeAndLogWalltimeResults, type TinybenchBench, type TinybenchOptions, + type TinybenchRoundHook, type TinybenchTask, } from "../instrument"; @@ -155,12 +161,21 @@ async function runAnalysis( registrations: BenchRegistrationInput[], getUri: (name: string) => string, ): Promise { - const label = InstrumentHooks.isInstrumented() ? "Measured" : "Checked"; + const instrumented = InstrumentHooks.isInstrumented(); + const label = instrumented ? "Measured" : "Checked"; + // Without instrumentation nothing is measured, so there is nothing to isolate. + const isolation = instrumented ? Isolation.current() : Isolation.shared(); const results: BenchResult[] = []; for (const { name, fn, fnOpts } of registrations) { const uri = getUri(name); - await runAnalysisTask(fn, fnOpts, uri); - console.log(`[CodSpeed] ${label} ${uri}`); + // An isolated child still reports every benchmark to Vitest, but it only + // measures its own. + if (isolation.shouldRun(uri)) { + await isolation.run(uri, async () => { + await runAnalysisTask(fn, fnOpts, uri); + console.log(`[CodSpeed] ${label} ${uri}`); + }); + } results.push(analysisResult(name)); } return results; @@ -221,6 +236,13 @@ async function runWalltime( getUri: (name: string) => string, test: BenchmarkGroup["test"], ): Promise { + if (getWalltimeForks() !== undefined) { + throw new Error( + `[CodSpeed] ${WALLTIME_FORKS_ENV} is not supported with Vitest 5 yet`, + ); + } + + const gcBetweenRounds = isGcBetweenRoundsEnabled(); const { Bench } = await importHostTinybench(); const bench = new Bench({ signal: test.context.signal, @@ -234,7 +256,15 @@ async function runWalltime( bench.add( name, wrapWithRootFrame(() => fn()), - fnOpts, + gcBetweenRounds + ? { + ...fnOpts, + // Vitest's FnHook is typed looser than tinybench's round hook + beforeEach: withGcAfter( + fnOpts?.beforeEach as TinybenchRoundHook | undefined, + ) as TinybenchFnOptions["beforeEach"], + } + : fnOpts, ); } diff --git a/packages/vitest-plugin/tests/fixtures/isolation/pids.bench.ts b/packages/vitest-plugin/tests/fixtures/isolation/pids.bench.ts new file mode 100644 index 00000000..c803dcbc --- /dev/null +++ b/packages/vitest-plugin/tests/fixtures/isolation/pids.bench.ts @@ -0,0 +1,19 @@ +import { describe, test } from "vitest"; + +describe("isolation", () => { + test("benchmarks", async ({ bench }) => { + const benchmarks = [ + bench("first", () => {}), + bench("second", () => {}), + bench("third", () => {}), + ]; + if (process.env.ISOLATION_FIXTURE_FAIL !== undefined) { + benchmarks.push( + bench("failing", () => { + throw new Error("benchmark failed"); + }), + ); + } + await bench.compare(...benchmarks); + }); +}); diff --git a/packages/vitest-plugin/tests/fixtures/isolation/stub.mjs b/packages/vitest-plugin/tests/fixtures/isolation/stub.mjs new file mode 100644 index 00000000..bdfd51b6 --- /dev/null +++ b/packages/vitest-plugin/tests/fixtures/isolation/stub.mjs @@ -0,0 +1,16 @@ +// Preloaded with `node --import` into the Vitest process, its workers, and the +// Vitest runs that process isolation starts. +import { InstrumentHooks } from "@codspeed/core"; +import { appendFileSync } from "fs"; + +// There is no instrument hook runtime outside the CodSpeed runner, so pretend +// the hooks are active. +InstrumentHooks.isInstrumented = () => true; + +// Vitest swallows what workers print, so the PID that measures each benchmark +// goes to a file. +const setExecutedBenchmark = InstrumentHooks.setExecutedBenchmark; +InstrumentHooks.setExecutedBenchmark = (pid, uri) => { + setExecutedBenchmark(pid, uri); + appendFileSync(process.env.ISOLATION_LOG_FILE, `executed ${pid} ${uri}\n`); +}; diff --git a/packages/vitest-plugin/tests/fixtures/isolation/vitest.config.ts b/packages/vitest-plugin/tests/fixtures/isolation/vitest.config.ts new file mode 100644 index 00000000..df313ac4 --- /dev/null +++ b/packages/vitest-plugin/tests/fixtures/isolation/vitest.config.ts @@ -0,0 +1,12 @@ +import { defineConfig } from "vitest/config"; +import codspeedPlugin from "../../../dist/index.mjs"; + +export default defineConfig({ + plugins: [codspeedPlugin()], + define: { + __VERSION__: JSON.stringify("1.0.0"), + }, + test: { + include: ["**/*.bench.ts"], + }, +});