perf(editor): pipeline Windows/Linux preview readback

apps/desktop-gpui/examples/live-edit-benchmark.rs

@@ -55,7 +55,7 @@ fn percentile(sorted: &[f64], fraction: f64) -> f64 {
 
 fn frame_number(output: &EditorFrameOutput) -> u32 {
     match output {
-        EditorFrameOutput::Rgba(frame) => frame.frame_number,
+        EditorFrameOutput::Rgba(frame) | EditorFrameOutput::Bgra(frame) => frame.frame_number,
         EditorFrameOutput::Nv12(frame) => frame.frame_number,
         #[cfg(target_os = "macos")]
         EditorFrameOutput::Surface(frame) => frame.frame_number,

apps/desktop-gpui/examples/scrub-benchmark.rs

@@ -61,7 +61,7 @@ fn percentile(sorted: &[f64], fraction: f64) -> f64 {
 
 fn frame_number(output: &EditorFrameOutput) -> u32 {
     match output {
-        EditorFrameOutput::Rgba(frame) => frame.frame_number,
+        EditorFrameOutput::Rgba(frame) | EditorFrameOutput::Bgra(frame) => frame.frame_number,
         EditorFrameOutput::Nv12(frame) => frame.frame_number,
         #[cfg(target_os = "macos")]
         EditorFrameOutput::Surface(frame) => frame.frame_number,

apps/desktop-gpui/src/app_windows.rs

@@ -5442,6 +5442,14 @@ fn start_preparing_editor(
                     let Some(image) = image else { continue };
                     editor_window::EditorPreviewFrame::Image(image)
                 }
+                cap_editor::EditorFrameOutput::Bgra(frame) => {
+                    let image = cx
+                        .background_executor()
+                        .spawn(async move { editor_window::bgra_frame_image(frame) })
+                        .await;
+                    let Some(image) = image else { continue };
+                    editor_window::EditorPreviewFrame::Image(image)
+                }
                 #[cfg(target_os = "macos")]
                 cap_editor::EditorFrameOutput::Surface(surface) => {
                     editor_window::surface_preview_frame(surface)
@@ -5658,16 +5666,17 @@ fn load_editor_project_candidate(
                 //
                 let state_cb = editor_window::make_state_callback(playhead);
                 let frame_cb = editor_window::make_frame_callback(frame_tx, frame_stats);
+                let shared_device = shared_wgpu_device().await;
                 #[cfg(target_os = "macos")]
                 let frame_format = cap_editor::EditorFrameFormat::BgraSurface;
                 #[cfg(not(target_os = "macos"))]
-                let frame_format = cap_editor::EditorFrameFormat::Rgba;
+                let frame_format =
+                    editor_window::cpu_preview_frame_format(shared_device.as_ref());
                 if let Some(previous) = previous {
                     return previous
                         .recreate_preparing_candidate(state_cb, frame_cb, frame_format)
                         .await;
                 }
-                let shared_device = shared_wgpu_device().await;
                 let instance = cap_editor::EditorInstance::new_with_startup_inputs(
                     instance_path,
                     state_cb,
@@ -5742,30 +5751,71 @@ fn load_editor_project_candidate(
             return;
         }
 
-        // The pump: convert on the background executor (un-padding plus the
-        // BGRA swap is a few megabytes per frame), deliver on the main one.
         cx.spawn({
             let stats = stats.clone();
             let pump_instance = Arc::downgrade(&instance);
+            let rgba_stream = editor_window::streaming_preview_enabled()
+                .then(|| gpui::StreamingImage::new(gpui::StreamingImageFormat::Rgba8));
+            let bgra_stream = editor_window::streaming_preview_enabled()
+                .then(|| gpui::StreamingImage::new(gpui::StreamingImageFormat::Bgra8));
             async move |cx| {
                 while let Ok((output, layout)) = frame_rx.recv_async().await {
+                    let mut streamed_rgba = None;
+                    let mut streamed_bgra = None;
                     let (frame, number) = match output {
                         cap_editor::EditorFrameOutput::Rgba(rgba) => {
-                            let stats = stats.clone();
                             let number = rgba.frame_number;
-                            let image = cx
-                                .background_executor()
-                                .spawn(
-                                    async move { editor_window::frame_image_timed(&rgba, &stats) },
+                            if rgba_stream.is_some() {
+                                streamed_rgba = Some(rgba);
+                                (
+                                    editor_window::EditorPreviewFrame::Stream(
+                                        rgba_stream.as_ref().expect("stream").clone(),
+                                    ),
+                                    number,
                                 )
-                                .await;
-                            let Some(image) = image else {
-                                tracing::warn!(
-                                    "a rendered frame could not be converted for display"
-                                );
-                                continue;
-                            };
-                            (editor_window::EditorPreviewFrame::Image(image), number)
+                            } else {
+                                let stats = stats.clone();
+                                let image = cx
+                                    .background_executor()
+                                    .spawn(async move {
+                                        editor_window::frame_image_timed(&rgba, &stats)
+                                    })
+                                    .await;
+                                let Some(image) = image else {
+                                    tracing::warn!(
+                                        "a rendered frame could not be converted for display"
+                                    );
+                                    continue;
+                                };
+                                (editor_window::EditorPreviewFrame::Image(image), number)
+                            }
+                        }
+                        cap_editor::EditorFrameOutput::Bgra(bgra) => {
+                            let number = bgra.frame_number;
+                            if bgra_stream.is_some() {
+                                streamed_bgra = Some(bgra);
+                                (
+                                    editor_window::EditorPreviewFrame::Stream(
+                                        bgra_stream.as_ref().expect("stream").clone(),
+                                    ),
+                                    number,
+                                )
+                            } else {
+                                let stats = stats.clone();
+                                let image = cx
+                                    .background_executor()
+                                    .spawn(async move {
+                                        editor_window::bgra_frame_image_timed(bgra, &stats)
+                                    })
+                                    .await;
+                                let Some(image) = image else {
+                                    tracing::warn!(
+                                        "a rendered frame could not be converted for display"
+                                    );
+                                    continue;
+                                };
+                                (editor_window::EditorPreviewFrame::Image(image), number)
+                            }
                         }
                         cap_editor::EditorFrameOutput::Nv12(frame) => {
                             tracing::warn!(
@@ -5785,6 +5835,18 @@ fn load_editor_project_candidate(
                             if !view.holds_instance(&pump_instance) {
                                 return;
                             }
+                            if let (Some(stream), Some(rgba)) =
+                                (&rgba_stream, streamed_rgba.take())
+                                && !editor_window::stream_frame_timed(stream, rgba, &stats)
+                            {
+                                return;
+                            }
+                            if let (Some(stream), Some(bgra)) =
+                                (&bgra_stream, streamed_bgra.take())
+                                && !editor_window::stream_frame_timed(stream, bgra, &stats)
+                            {
+                                return;
+                            }
                             view.frame_arrived(
                                 editor_window::EditorFrame {
                                     frame,

apps/desktop-gpui/src/editor_preparing.rs

@@ -525,9 +525,10 @@ impl PreparingRunner {
         let discard = self.frame_discard.clone();
         #[cfg(target_os = "macos")]
         let frame_format = EditorFrameFormat::BgraSurface;
-        #[cfg(not(target_os = "macos"))]
-        let frame_format = EditorFrameFormat::Rgba;
         let shared_device = crate::app_windows::shared_wgpu_device().await;
+        #[cfg(not(target_os = "macos"))]
+        let frame_format: EditorFrameFormat =
+            crate::editor_window::cpu_preview_frame_format(shared_device.as_ref());
         self.control.accepts_frames.store(true, Ordering::Release);
         let playback = PreparingPlaybackSession::spawn_with_expected_metadata(
             input,

apps/desktop-gpui/src/editor_window.rs

@@ -63,8 +63,8 @@ use gpui::{
     Animation, AnimationExt as _, AppContext as _, Bounds, Context, Entity, FocusHandle,
     FontWeight, Hsla, InteractiveElement, IntoElement, MouseButton, MouseDownEvent, MouseMoveEvent,
     MouseUpEvent, ParentElement, Pixels, Point, Render, RenderImage, SharedString,
-    StatefulInteractiveElement as _, StyleRefinement, Styled, Subscription, WeakEntity, Window,
-    div, point, prelude::FluentBuilder, px, svg,
+    StatefulInteractiveElement as _, StreamingImage, StyleRefinement, Styled, Subscription,
+    WeakEntity, Window, div, point, prelude::FluentBuilder, px, svg,
 };
 
 use crate::i18n::tr;
@@ -593,10 +593,71 @@ pub fn frame_image(frame: &RenderedFrame) -> Option<Arc<RenderImage>> {
     ])))
 }
 
+pub fn bgra_frame_image(frame: RenderedFrame) -> Option<Arc<RenderImage>> {
+    let row_bytes = (frame.width as usize).checked_mul(4)?;
+    if frame.padded_bytes_per_row as usize != row_bytes
+        || frame.data.len() != row_bytes.checked_mul(frame.height as usize)?
+    {
+        return None;
+    }
+    let pixels = Arc::try_unwrap(frame.data).unwrap_or_else(|shared| shared.as_ref().clone());
+    let buffer = image::RgbaImage::from_raw(frame.width, frame.height, pixels)?;
+    Some(Arc::new(RenderImage::new(smallvec::smallvec![
+        image::Frame::new(buffer)
+    ])))
+}
+
+#[cfg(not(target_os = "macos"))]
+pub(crate) fn cpu_preview_frame_format(
+    shared_device: Option<&cap_rendering::SharedWgpuDevice>,
+) -> cap_editor::EditorFrameFormat {
+    if shared_device.is_some_and(|device| !device.is_software_adapter) {
+        cap_editor::EditorFrameFormat::Bgra
+    } else {
+        cap_editor::EditorFrameFormat::Rgba
+    }
+}
+
+pub(crate) fn streaming_preview_enabled() -> bool {
+    static ENABLED: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
+    *ENABLED.get_or_init(|| {
+        match std::env::var_os("CAP_GPUI_STREAMING_IMAGE").as_deref() {
+            Some(value) => value != "0",
+            None => !cfg!(target_os = "macos"),
+        }
+    })
+}
+
+pub fn stream_frame_timed(
+    stream: &StreamingImage,
+    frame: RenderedFrame,
+    stats: &PumpStats,
+) -> bool {
+    let started = Instant::now();
+    let size = gpui::size(
+        gpui::DevicePixels(frame.width as i32),
+        gpui::DevicePixels(frame.height as i32),
+    );
+    let stride = frame.padded_bytes_per_row as usize;
+    let result = match Arc::try_unwrap(frame.data) {
+        Ok(data) => stream.set_frame(size, stride, data).map(drop),
+        Err(data) => stream.copy_frame(size, stride, &data),
+    };
+    stats
+        .convert_nanos
+        .fetch_add(started.elapsed().as_nanos() as u64, Ordering::Relaxed);
+    stats.convert_samples.fetch_add(1, Ordering::Relaxed);
+    if let Err(error) = &result {
+        tracing::warn!(%error, "a rendered frame could not be streamed to the preview");
+    }
+    result.is_ok()
+}
+
 /// What the frame pump hands the window.
 #[derive(Clone)]
 pub(crate) enum EditorPreviewFrame {
     Image(Arc<RenderImage>),
+    Stream(StreamingImage),
     #[cfg(target_os = "macos")]
     Surface(CVPixelBuffer),
 }
@@ -608,6 +669,9 @@ impl EditorPreviewFrame {
             Self::Image(image) => {
                 let _ = window.paint_image(bounds, corners, Arc::clone(image), 0, false);
             }
+            Self::Stream(image) => {
+                let _ = window.paint_streaming_image(bounds, corners, image, false);
+            }
             #[cfg(target_os = "macos")]
             Self::Surface(surface) => {
                 window.paint_surface_fitted(bounds, bounds, corners, surface.clone())
@@ -12520,8 +12584,19 @@ pub(crate) fn surface_preview_frame(frame: cap_rendering::SurfaceFrame) -> Edito
 /// [`frame_image`], timed. The pump calls this so the CPU conversion's cost is
 /// a number rather than a hunch.
 pub fn frame_image_timed(frame: &RenderedFrame, stats: &PumpStats) -> Option<Arc<RenderImage>> {
+    timed_conversion(stats, || frame_image(frame))
+}
+
+pub fn bgra_frame_image_timed(frame: RenderedFrame, stats: &PumpStats) -> Option<Arc<RenderImage>> {
+    timed_conversion(stats, || bgra_frame_image(frame))
+}
+
+fn timed_conversion(
+    stats: &PumpStats,
+    convert: impl FnOnce() -> Option<Arc<RenderImage>>,
+) -> Option<Arc<RenderImage>> {
     let started = Instant::now();
-    let image = frame_image(frame);
+    let image = convert();
     stats
         .convert_nanos
         .fetch_add(started.elapsed().as_nanos() as u64, Ordering::Relaxed);

apps/desktop-gpui/src/picker_probe.rs

@@ -106,7 +106,7 @@ pub fn run(main: WindowHandle<MainWindow>, cx: &mut App) {
             let mut mode = cx.update(|cx| probe_mode(label, cx));
             mode["overlayInactiveBeforeActivation"] = inactive_before.into();
             let screen = screenshot(&output, label, cx).await;
-            mode["backdrop"] = probe_backdrop(&output, label, screen.as_ref(), cx).await;
+            mode["backdrop"] = probe_backdrop(&output, label, kind, screen.as_ref(), cx).await;
             modes.push(mode);
             if kind == TargetType::Window {
                 window_arrangement = Some(probe_window_arrangement(&output, cx).await);
@@ -372,6 +372,7 @@ fn mean_difference(
 async fn probe_backdrop(
     output: &std::ffi::OsStr,
     label: &str,
+    kind: TargetType,
     screen: Option<&image::DynamicImage>,
     cx: &mut AsyncApp,
 ) -> serde_json::Value {
@@ -393,6 +394,13 @@ async fn probe_backdrop(
             .ok()
             .flatten()
     });
+    if !crate::target_overlay::backdrop_suits(Some(kind)) {
+        return serde_json::json!({
+            "skipped": "the picker takes no backdrop in this mode here",
+            "present": image.is_some(),
+            "passed": image.is_none(),
+        });
+    }
     let Some(image) = image else {
         return serde_json::json!({ "present": false, "passed": false });
     };

apps/desktop-gpui/src/target_overlay.rs

@@ -830,7 +830,7 @@ fn more_area_ratios() -> Vec<(u32, u32)> {
 /// so a capture under their panes showed through it as hard-edged blocks;
 /// there the panes sit on the fallback fill, as they do where no capture
 /// is possible.
-fn backdrop_suits(mode: Option<TargetType>) -> bool {
+pub(crate) fn backdrop_suits(mode: Option<TargetType>) -> bool {
     match mode {
         None => false,
         Some(TargetType::Display | TargetType::Window) => !cfg!(windows),

crates/editor/examples/editor-playback-benchmark.rs

@@ -571,6 +571,7 @@ async fn main() {
                 data.len() + metadata_bytes
             }
             EditorFrameOutput::Rgba(frame) => frame.data.len() + 24,
+            EditorFrameOutput::Bgra(frame) => frame.data.len(),
             #[cfg(target_os = "macos")]
             EditorFrameOutput::Surface(_) => return,
         };

crates/editor/examples/imported-music-playback-benchmark.rs

@@ -80,7 +80,9 @@ async fn main() {
         |_| {},
         Box::new(move |frame, _| {
             let number = match frame {
-                EditorFrameOutput::Rgba(frame) => frame.frame_number,
+                EditorFrameOutput::Rgba(frame) | EditorFrameOutput::Bgra(frame) => {
+                    frame.frame_number
+                }
                 EditorFrameOutput::Nv12(frame) => frame.frame_number,
                 #[cfg(target_os = "macos")]
                 EditorFrameOutput::Surface(frame) => frame.frame_number,

crates/editor/examples/scrub-benchmark.rs

@@ -19,7 +19,7 @@ fn arg_value<'a>(args: &'a [String], flag: &str) -> Option<&'a str> {
 
 fn presented_frame(output: &EditorFrameOutput) -> u32 {
     match output {
-        EditorFrameOutput::Rgba(frame) => frame.frame_number,
+        EditorFrameOutput::Rgba(frame) | EditorFrameOutput::Bgra(frame) => frame.frame_number,
         EditorFrameOutput::Nv12(frame) => frame.frame_number,
         #[cfg(target_os = "macos")]
         EditorFrameOutput::Surface(frame) => frame.frame_number,