Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 7 additions & 5 deletions src/cli/shared.rs
Original file line number Diff line number Diff line change
Expand Up @@ -226,11 +226,13 @@ pub struct WalltimeRunArgs {
/// - `dwarf,<bytes>` sets the user stack size copied per sample; larger values produce
/// deeper call stacks but bigger profiles.
///
/// Inferred from the benchmark command when unset:
/// - Java: `fp`
/// - Go: `fp`
/// - Python: `dwarf,32768`
/// - Others: `dwarf,8192`
/// On Linux, inferred from the original benchmark commands when unset:
/// - Go and Java: `fp` for both perf and samply.
/// - Mixed exec-harness groups: `fp` if any command is detected as Go or Java.
/// - Other commands: DWARF with the profiler's default stack size.
/// Perf uses 8192 bytes (32768 for Python); samply uses 32000 bytes.
///
/// Explicit settings override inference. Samply's unwinding options are Linux-only.
#[arg(
long,
env = "CODSPEED_WALLTIME_UNWINDING_MODE",
Expand Down
4 changes: 2 additions & 2 deletions src/executor/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ pub enum WalltimeProfiler {
Samply,
}

/// Explicit walltime unwinding settings. An absent setting preserves backend inference.
/// Walltime unwinding settings selected explicitly or inferred from the original targets.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum WalltimeUnwinding {
FramePointer,
Expand Down Expand Up @@ -142,7 +142,7 @@ pub struct ExecutorConfig {

pub instruments: Instruments,
pub enable_profiler: bool,
/// Explicit unwinding settings; None preserves the profiler's automatic selection.
/// Resolved unwinding settings; None preserves the profiler's default.
pub walltime_unwinding: Option<WalltimeUnwinding>,

pub simulation_tool: SimulationTool,
Expand Down
8 changes: 0 additions & 8 deletions src/executor/helpers/env.rs
Original file line number Diff line number Diff line change
Expand Up @@ -33,14 +33,6 @@ pub fn get_base_injected_env(
env
}

/// Set the env variable to not warn users about Go's perf unwinding mode when running Go benchmarks
pub fn suppress_go_perf_unwinding_warning() {
// Safety: no multithreading
unsafe {
std::env::set_var("CODSPEED_GO_SUPPRESS_PERF_UNWINDING_MODE_WARNING", "true");
}
}

/// Build the `PATH` value with optional language introspection wrappers prepended.
///
/// When `enable_introspection` is true, the Node.js and Go wrapper script
Expand Down
26 changes: 25 additions & 1 deletion src/executor/orchestrator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ use crate::cli::run::logger::Logger;
use crate::executor::config::BenchmarkTarget;
use crate::executor::config::OrchestratorConfig;
use crate::executor::helpers::profile_folder::create_profile_folder;
use crate::executor::wall_time::profiler::unwinding::resolve_unwinding;
use crate::prelude::*;
use crate::run_environment::{self, RunEnvironment, RunEnvironmentProvider};
use crate::runner_mode::RunnerMode;
Expand Down Expand Up @@ -141,9 +142,32 @@ impl Orchestrator {
}

for (run_part_index, part) in run_parts.into_iter().enumerate() {
let config = self
let mut config = self
.config
.executor_config_for_command(part.command, !part.uses_exec_harness);
if *part.mode == RunnerMode::Walltime && config.enable_profiler {
config.walltime_unwinding = if part.uses_exec_harness {
let commands = exec_targets
.iter()
.filter_map(|target| match target {
BenchmarkTarget::Exec { command, .. } => Some(command),
BenchmarkTarget::Entrypoint { .. } => None,
})
.flatten()
.map(String::as_str);
Comment on lines +153 to +157

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Arguments select the wrong unwinding

Every exec argument is checked as an executable name. For example, codspeed exec -m walltime -- grep go corpus.txt selects frame-pointer unwinding because the pattern go reaches suggested_unwinding as a command. If the native program was built without frame pointers, its call stacks will be incomplete. Every other target in the same recording also gets this setting.

Keep each target's argument list intact and distinguish its executable from ordinary arguments before inferring the runtime.

Prompt To Fix With AI
This is a comment left during a code review.
Path: src/executor/orchestrator.rs
Line: 153-157

Comment:
**Arguments select the wrong unwinding**

Every exec argument is checked as an executable name. For example, `codspeed exec -m walltime -- grep go corpus.txt` selects frame-pointer unwinding because the pattern `go` reaches `suggested_unwinding` as a command. If the native program was built without frame pointers, its call stacks will be incomplete. Every other target in the same recording also gets this setting.

Keep each target's argument list intact and distinguish its executable from ordinary arguments before inferring the runtime.

---

For each issue above, determine whether it is valid and should be fixed. If so, fix it directly.

Fix in Claude Code Fix in Codex

resolve_unwinding(
commands,
config.walltime_unwinding,
self.config.walltime_profiler,
)
} else {
resolve_unwinding(
[config.command.as_str()],
config.walltime_unwinding,
self.config.walltime_profiler,
)
};
}
let mut executor = get_executor_from_mode(part.mode, self.config.walltime_profiler);
let profile_folder =
self.resolve_profile_folder(&executor.name(), run_part_index, total_parts)?;
Expand Down
9 changes: 9 additions & 0 deletions src/executor/wall_time/executor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ use crate::executor::Executor;
use crate::executor::ExecutorConfig;
use crate::executor::ToolStatus;
use crate::executor::config::WalltimeProfiler;
#[cfg(target_os = "linux")]
use crate::executor::config::WalltimeUnwinding;
use crate::executor::helpers::command::CommandBuilder;
use crate::executor::helpers::env::{build_path_env, get_base_injected_env};
use crate::executor::helpers::get_bench_command::get_bench_command;
Expand Down Expand Up @@ -73,6 +75,13 @@ impl WallTimeExecutor {
&execution_context.config,
);
extra_env.insert("PATH".into(), path_value);
#[cfg(target_os = "linux")]
if config.enable_profiler {
extra_env.insert(
"CODSPEED_GO_SUPPRESS_PERF_UNWINDING_MODE_WARNING".into(),
(config.walltime_unwinding == Some(WalltimeUnwinding::FramePointer)).to_string(),
);
}

// We have to write the benchmark command to a script, to ensure proper formatting
// and to not have to manually escape everything.
Expand Down
1 change: 1 addition & 0 deletions src/executor/wall_time/profiler/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

pub mod perf;
pub mod samply;
pub(crate) mod unwinding;

const WALLTIME_METADATA_CURRENT_VERSION: u64 = 1;

Expand Down
74 changes: 4 additions & 70 deletions src/executor/wall_time/profiler/perf/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,7 @@ use crate::executor::ExecutorConfig;
use crate::executor::ToolStatus;
use crate::executor::config::WalltimeUnwinding;
use crate::executor::helpers::command::CommandBuilder;
use crate::executor::helpers::detect_executable::command_has_executable;
use crate::executor::helpers::env::is_codspeed_debug_enabled;
use crate::executor::helpers::env::suppress_go_perf_unwinding_warning;
use crate::executor::helpers::harvest_perf_maps_for_pids::harvest_perf_maps_for_pids;
use crate::executor::helpers::run_with_sudo::wrap_with_sudo;
use crate::executor::shared::fifo::FifoBenchmarkData;
Expand Down Expand Up @@ -39,40 +37,6 @@ pub mod perf_executable;

const PERF_PIPEDATA_FILE_NAME: &str = "perf.pipedata";

/// Unwinding suggested for the benchmarked runtime, if it has a known best setting.
fn suggested_unwinding(command: &str) -> Option<WalltimeUnwinding> {
if command_has_executable(
command,
&["gradle", "gradlew", "java", "maven", "mvn", "mvnw"],
) {
return Some(WalltimeUnwinding::FramePointer);
}
if command_has_executable(command, &["go"]) {
return Some(WalltimeUnwinding::FramePointer);
}
if command_has_executable(command, &["pytest", "uv", "python", "python3"]) {
return Some(WalltimeUnwinding::Dwarf {
stack_size: Some(32 * 1024),
});
}
None
}

fn resolve_unwinding(command: &str, explicit: Option<WalltimeUnwinding>) -> WalltimeUnwinding {
let suggested = suggested_unwinding(command);
let Some(explicit) = explicit else {
return suggested.unwrap_or(WalltimeUnwinding::Dwarf { stack_size: None });
};
if let Some(suggested) = suggested
&& suggested != explicit
{
warn!(
"Overriding suggested unwinding `{suggested}` with `{explicit}`, call stacks may be incomplete"
);
}
explicit
}

pub struct PerfProfiler {
/// Set by [`Profiler::wrap_command`]; used by the FIFO hooks to control event
/// recording on the live `perf record` process.
Expand Down Expand Up @@ -123,13 +87,12 @@ impl Profiler for PerfProfiler {
let perf_fifo = PerfFifo::new()?;
let perf_file_path = profile_folder.join(PERF_PIPEDATA_FILE_NAME);

let unwinding = resolve_unwinding(&config.command, config.walltime_unwinding);
let unwinding = config
.walltime_unwinding
.unwrap_or(WalltimeUnwinding::Dwarf { stack_size: None });

let cg_mode = match unwinding {
WalltimeUnwinding::FramePointer => {
suppress_go_perf_unwinding_warning();
"fp"
}
WalltimeUnwinding::FramePointer => "fp",
WalltimeUnwinding::Dwarf { stack_size } => {
&format!("dwarf,{}", stack_size.unwrap_or(8 * 1024))
}
Expand Down Expand Up @@ -325,32 +288,3 @@ impl BenchmarkData<'_> {
Ok(())
}
}

#[cfg(test)]
mod tests {
use super::*;

#[test]
fn explicit_unwinding_overrides_language_inference() {
let explicit = WalltimeUnwinding::Dwarf {
stack_size: Some(16 * 1024),
};
for (command, inferred) in [
(
"python bench.py",
WalltimeUnwinding::Dwarf {
stack_size: Some(32 * 1024),
},
),
("cargo bench", WalltimeUnwinding::Dwarf { stack_size: None }),
("go test -bench=.", WalltimeUnwinding::FramePointer),
] {
assert_eq!(resolve_unwinding(command, None), inferred);
assert_eq!(resolve_unwinding(command, Some(explicit)), explicit);
}
assert_eq!(
resolve_unwinding("python bench.py", Some(WalltimeUnwinding::FramePointer)),
WalltimeUnwinding::FramePointer,
);
}
}
159 changes: 159 additions & 0 deletions src/executor/wall_time/profiler/unwinding.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
use crate::executor::config::{WalltimeProfiler, WalltimeUnwinding};
use crate::executor::helpers::detect_executable::command_has_executable;
use crate::prelude::*;

/// Resolve against original commands, before exec-harness serializes them.
/// A shared recording uses frame pointers if any target needs them.
pub(crate) fn resolve_unwinding<'a>(
commands: impl IntoIterator<Item = &'a str>,
explicit: Option<WalltimeUnwinding>,
profiler: Option<WalltimeProfiler>,
) -> Option<WalltimeUnwinding> {
// Samply's unwinding controls are Linux-only.
if !cfg!(target_os = "linux") && profiler != Some(WalltimeProfiler::Perf) {
return explicit;
}

let mut suggested = None;
for command in commands {
match suggested_unwinding(command) {
Some(WalltimeUnwinding::FramePointer) => {
suggested = Some(WalltimeUnwinding::FramePointer);
break;
}
Some(unwinding) if profiler == Some(WalltimeProfiler::Perf) => {
suggested = Some(unwinding);
}
_ => {}
}
}
if let (Some(explicit), Some(suggested)) = (explicit, suggested)
&& suggested != explicit
{
warn!(
"Overriding suggested unwinding `{suggested}` with `{explicit}`, call stacks may be incomplete"
);
}
explicit.or(suggested)
}

/// Unwinding suggested for the benchmarked runtime, if it has a known best setting.
fn suggested_unwinding(command: &str) -> Option<WalltimeUnwinding> {
if command_has_executable(
command,
&["gradle", "gradlew", "java", "maven", "mvn", "mvnw"],
) {
return Some(WalltimeUnwinding::FramePointer);
}
if command_has_executable(command, &["go"]) {
return Some(WalltimeUnwinding::FramePointer);
}
if command_has_executable(command, &["pytest", "uv", "python", "python3"]) {
return Some(WalltimeUnwinding::Dwarf {
stack_size: Some(32 * 1024),
});
}
None
}

#[cfg(test)]
mod tests {
use super::*;

#[test]
#[cfg(target_os = "linux")]
fn go_and_java_use_frame_pointers_for_both_profilers() {
for profiler in [
None,
Some(WalltimeProfiler::Perf),
Some(WalltimeProfiler::Samply),
] {
for command in [
"go test -bench=.",
"/usr/bin/java -jar bench.jar",
"./gradlew jmh",
"mvn test",
] {
assert_eq!(
resolve_unwinding([command], None, profiler),
Some(WalltimeUnwinding::FramePointer)
);
}
}
}

#[test]
#[cfg(target_os = "linux")]
fn mixed_exec_targets_use_frame_pointers_in_either_order() {
for profiler in [Some(WalltimeProfiler::Perf), Some(WalltimeProfiler::Samply)] {
for commands in [
["python", "bench.py", "java", "-jar", "bench.jar"],
["java", "-jar", "bench.jar", "python", "bench.py"],
] {
assert_eq!(
resolve_unwinding(commands, None, profiler),
Some(WalltimeUnwinding::FramePointer)
);
}
}
}

#[test]
fn explicit_unwinding_overrides_language_inference() {
for profiler in [Some(WalltimeProfiler::Perf), Some(WalltimeProfiler::Samply)] {
for explicit in [
WalltimeUnwinding::Dwarf { stack_size: None },
WalltimeUnwinding::Dwarf {
stack_size: Some(16 * 1024),
},
WalltimeUnwinding::FramePointer,
] {
for command in [
"go test -bench=.",
"java -jar bench.jar",
"python bench.py",
"cargo bench",
] {
assert_eq!(
resolve_unwinding([command], Some(explicit), profiler),
Some(explicit)
);
}
}
}
}

#[test]
fn dwarf_defaults_remain_backend_specific() {
assert_eq!(
resolve_unwinding(["python bench.py"], None, Some(WalltimeProfiler::Perf)),
Some(WalltimeUnwinding::Dwarf {
stack_size: Some(32 * 1024)
})
);
for profiler in [None, Some(WalltimeProfiler::Samply)] {
assert_eq!(resolve_unwinding(["python bench.py"], None, profiler), None);
}
for profiler in [
None,
Some(WalltimeProfiler::Perf),
Some(WalltimeProfiler::Samply),
] {
assert_eq!(resolve_unwinding(["cargo bench"], None, profiler), None);
}
}

#[test]
#[cfg(not(target_os = "linux"))]
fn samply_keeps_platform_unwinding_defaults() {
assert_eq!(resolve_unwinding(["go test -bench=."], None, None), None);
assert_eq!(
resolve_unwinding(
["java -jar bench.jar"],
None,
Some(WalltimeProfiler::Samply)
),
None
);
}
}
Loading