From 045fd6319a300816dcd83424190d64cca31afb82 Mon Sep 17 00:00:00 2001 From: Aatman Jain Date: Thu, 29 Oct 2026 23:29:45 +0000 Subject: [PATCH 1/9] perf(editor): pipeline Windows/Linux preview readback Overlap GPU BGRA readback with rendering during realtime playback, use GPU BGRA on capable adapters, and stream preview frames into a reused texture on Windows/Linux to avoid per-frame CPU swizzle and atlas uploads. --- .../examples/live-edit-benchmark.rs | 2 +- apps/desktop-gpui/examples/scrub-benchmark.rs | 2 +- apps/desktop-gpui/src/app_windows.rs | 96 +++- apps/desktop-gpui/src/editor_preparing.rs | 5 +- apps/desktop-gpui/src/editor_window.rs | 81 ++- .../examples/editor-playback-benchmark.rs | 1 + crates/editor/examples/scrub-benchmark.rs | 2 +- crates/editor/src/editor.rs | 233 ++++++-- crates/editor/src/preparing_preview.rs | 17 +- .../src/preparing_preview/native_tests.rs | 39 +- crates/editor/src/thumbnail.rs | 1 + crates/rendering/src/frame_pipeline.rs | 506 ++++++++++++++---- crates/rendering/src/lib.rs | 259 ++++++++- 13 files changed, 1064 insertions(+), 180 deletions(-) diff --git a/apps/desktop-gpui/examples/live-edit-benchmark.rs b/apps/desktop-gpui/examples/live-edit-benchmark.rs index 134dc309982..9913acf34c4 100644 --- a/apps/desktop-gpui/examples/live-edit-benchmark.rs +++ b/apps/desktop-gpui/examples/live-edit-benchmark.rs @@ -55,7 +55,7 @@ fn percentile(sorted: &[f64], fraction: f64) -> f64 { fn frame_number(output: &EditorFrameOutput) -> u32 { match output { - EditorFrameOutput::Rgba(frame) => frame.frame_number, + EditorFrameOutput::Rgba(frame) | EditorFrameOutput::Bgra(frame) => frame.frame_number, EditorFrameOutput::Nv12(frame) => frame.frame_number, #[cfg(target_os = "macos")] EditorFrameOutput::Surface(frame) => frame.frame_number, diff --git a/apps/desktop-gpui/examples/scrub-benchmark.rs b/apps/desktop-gpui/examples/scrub-benchmark.rs index 1f3f32c1d6c..72dd5d59045 100644 --- a/apps/desktop-gpui/examples/scrub-benchmark.rs +++ b/apps/desktop-gpui/examples/scrub-benchmark.rs @@ -61,7 +61,7 @@ fn percentile(sorted: &[f64], fraction: f64) -> f64 { fn frame_number(output: &EditorFrameOutput) -> u32 { match output { - EditorFrameOutput::Rgba(frame) => frame.frame_number, + EditorFrameOutput::Rgba(frame) | EditorFrameOutput::Bgra(frame) => frame.frame_number, EditorFrameOutput::Nv12(frame) => frame.frame_number, #[cfg(target_os = "macos")] EditorFrameOutput::Surface(frame) => frame.frame_number, diff --git a/apps/desktop-gpui/src/app_windows.rs b/apps/desktop-gpui/src/app_windows.rs index 885f0cb7b05..8bc0d55e5a2 100644 --- a/apps/desktop-gpui/src/app_windows.rs +++ b/apps/desktop-gpui/src/app_windows.rs @@ -5442,6 +5442,14 @@ fn start_preparing_editor( let Some(image) = image else { continue }; editor_window::EditorPreviewFrame::Image(image) } + cap_editor::EditorFrameOutput::Bgra(frame) => { + let image = cx + .background_executor() + .spawn(async move { editor_window::bgra_frame_image(frame) }) + .await; + let Some(image) = image else { continue }; + editor_window::EditorPreviewFrame::Image(image) + } #[cfg(target_os = "macos")] cap_editor::EditorFrameOutput::Surface(surface) => { editor_window::surface_preview_frame(surface) @@ -5658,16 +5666,17 @@ fn load_editor_project_candidate( // let state_cb = editor_window::make_state_callback(playhead); let frame_cb = editor_window::make_frame_callback(frame_tx, frame_stats); + let shared_device = shared_wgpu_device().await; #[cfg(target_os = "macos")] let frame_format = cap_editor::EditorFrameFormat::BgraSurface; #[cfg(not(target_os = "macos"))] - let frame_format = cap_editor::EditorFrameFormat::Rgba; + let frame_format = + editor_window::cpu_preview_frame_format(shared_device.as_ref()); if let Some(previous) = previous { return previous .recreate_preparing_candidate(state_cb, frame_cb, frame_format) .await; } - let shared_device = shared_wgpu_device().await; let instance = cap_editor::EditorInstance::new_with_startup_inputs( instance_path, state_cb, @@ -5742,30 +5751,71 @@ fn load_editor_project_candidate( return; } - // The pump: convert on the background executor (un-padding plus the - // BGRA swap is a few megabytes per frame), deliver on the main one. cx.spawn({ let stats = stats.clone(); let pump_instance = Arc::downgrade(&instance); + let rgba_stream = editor_window::streaming_preview_enabled() + .then(|| gpui::StreamingImage::new(gpui::StreamingImageFormat::Rgba8)); + let bgra_stream = editor_window::streaming_preview_enabled() + .then(|| gpui::StreamingImage::new(gpui::StreamingImageFormat::Bgra8)); async move |cx| { while let Ok((output, layout)) = frame_rx.recv_async().await { + let mut streamed_rgba = None; + let mut streamed_bgra = None; let (frame, number) = match output { cap_editor::EditorFrameOutput::Rgba(rgba) => { - let stats = stats.clone(); let number = rgba.frame_number; - let image = cx - .background_executor() - .spawn( - async move { editor_window::frame_image_timed(&rgba, &stats) }, + if rgba_stream.is_some() { + streamed_rgba = Some(rgba); + ( + editor_window::EditorPreviewFrame::Stream( + rgba_stream.as_ref().expect("stream").clone(), + ), + number, ) - .await; - let Some(image) = image else { - tracing::warn!( - "a rendered frame could not be converted for display" - ); - continue; - }; - (editor_window::EditorPreviewFrame::Image(image), number) + } else { + let stats = stats.clone(); + let image = cx + .background_executor() + .spawn(async move { + editor_window::frame_image_timed(&rgba, &stats) + }) + .await; + let Some(image) = image else { + tracing::warn!( + "a rendered frame could not be converted for display" + ); + continue; + }; + (editor_window::EditorPreviewFrame::Image(image), number) + } + } + cap_editor::EditorFrameOutput::Bgra(bgra) => { + let number = bgra.frame_number; + if bgra_stream.is_some() { + streamed_bgra = Some(bgra); + ( + editor_window::EditorPreviewFrame::Stream( + bgra_stream.as_ref().expect("stream").clone(), + ), + number, + ) + } else { + let stats = stats.clone(); + let image = cx + .background_executor() + .spawn(async move { + editor_window::bgra_frame_image_timed(bgra, &stats) + }) + .await; + let Some(image) = image else { + tracing::warn!( + "a rendered frame could not be converted for display" + ); + continue; + }; + (editor_window::EditorPreviewFrame::Image(image), number) + } } cap_editor::EditorFrameOutput::Nv12(frame) => { tracing::warn!( @@ -5785,6 +5835,18 @@ fn load_editor_project_candidate( if !view.holds_instance(&pump_instance) { return; } + if let (Some(stream), Some(rgba)) = + (&rgba_stream, streamed_rgba.take()) + && !editor_window::stream_frame_timed(stream, rgba, &stats) + { + return; + } + if let (Some(stream), Some(bgra)) = + (&bgra_stream, streamed_bgra.take()) + && !editor_window::stream_frame_timed(stream, bgra, &stats) + { + return; + } view.frame_arrived( editor_window::EditorFrame { frame, diff --git a/apps/desktop-gpui/src/editor_preparing.rs b/apps/desktop-gpui/src/editor_preparing.rs index 92f6a5faf7a..55a2f2748cc 100644 --- a/apps/desktop-gpui/src/editor_preparing.rs +++ b/apps/desktop-gpui/src/editor_preparing.rs @@ -525,9 +525,10 @@ impl PreparingRunner { let discard = self.frame_discard.clone(); #[cfg(target_os = "macos")] let frame_format = EditorFrameFormat::BgraSurface; - #[cfg(not(target_os = "macos"))] - let frame_format = EditorFrameFormat::Rgba; let shared_device = crate::app_windows::shared_wgpu_device().await; + #[cfg(not(target_os = "macos"))] + let frame_format: EditorFrameFormat = + crate::editor_window::cpu_preview_frame_format(shared_device.as_ref()); self.control.accepts_frames.store(true, Ordering::Release); let playback = PreparingPlaybackSession::spawn_with_expected_metadata( input, diff --git a/apps/desktop-gpui/src/editor_window.rs b/apps/desktop-gpui/src/editor_window.rs index 64edb658fe8..516ae412511 100644 --- a/apps/desktop-gpui/src/editor_window.rs +++ b/apps/desktop-gpui/src/editor_window.rs @@ -63,8 +63,8 @@ use gpui::{ Animation, AnimationExt as _, AppContext as _, Bounds, Context, Entity, FocusHandle, FontWeight, Hsla, InteractiveElement, IntoElement, MouseButton, MouseDownEvent, MouseMoveEvent, MouseUpEvent, ParentElement, Pixels, Point, Render, RenderImage, SharedString, - StatefulInteractiveElement as _, StyleRefinement, Styled, Subscription, WeakEntity, Window, - div, point, prelude::FluentBuilder, px, svg, + StatefulInteractiveElement as _, StreamingImage, StyleRefinement, Styled, Subscription, + WeakEntity, Window, div, point, prelude::FluentBuilder, px, svg, }; use crate::i18n::tr; @@ -593,10 +593,71 @@ pub fn frame_image(frame: &RenderedFrame) -> Option> { ]))) } +pub fn bgra_frame_image(frame: RenderedFrame) -> Option> { + let row_bytes = (frame.width as usize).checked_mul(4)?; + if frame.padded_bytes_per_row as usize != row_bytes + || frame.data.len() != row_bytes.checked_mul(frame.height as usize)? + { + return None; + } + let pixels = Arc::try_unwrap(frame.data).unwrap_or_else(|shared| shared.as_ref().clone()); + let buffer = image::RgbaImage::from_raw(frame.width, frame.height, pixels)?; + Some(Arc::new(RenderImage::new(smallvec::smallvec![ + image::Frame::new(buffer) + ]))) +} + +#[cfg(not(target_os = "macos"))] +pub(crate) fn cpu_preview_frame_format( + shared_device: Option<&cap_rendering::SharedWgpuDevice>, +) -> cap_editor::EditorFrameFormat { + if shared_device.is_some_and(|device| !device.is_software_adapter) { + cap_editor::EditorFrameFormat::Bgra + } else { + cap_editor::EditorFrameFormat::Rgba + } +} + +pub(crate) fn streaming_preview_enabled() -> bool { + static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + *ENABLED.get_or_init(|| { + match std::env::var_os("CAP_GPUI_STREAMING_IMAGE").as_deref() { + Some(value) => value != "0", + None => !cfg!(target_os = "macos"), + } + }) +} + +pub fn stream_frame_timed( + stream: &StreamingImage, + frame: RenderedFrame, + stats: &PumpStats, +) -> bool { + let started = Instant::now(); + let size = gpui::size( + gpui::DevicePixels(frame.width as i32), + gpui::DevicePixels(frame.height as i32), + ); + let stride = frame.padded_bytes_per_row as usize; + let result = match Arc::try_unwrap(frame.data) { + Ok(data) => stream.set_frame(size, stride, data).map(drop), + Err(data) => stream.copy_frame(size, stride, &data), + }; + stats + .convert_nanos + .fetch_add(started.elapsed().as_nanos() as u64, Ordering::Relaxed); + stats.convert_samples.fetch_add(1, Ordering::Relaxed); + if let Err(error) = &result { + tracing::warn!(%error, "a rendered frame could not be streamed to the preview"); + } + result.is_ok() +} + /// What the frame pump hands the window. #[derive(Clone)] pub(crate) enum EditorPreviewFrame { Image(Arc), + Stream(StreamingImage), #[cfg(target_os = "macos")] Surface(CVPixelBuffer), } @@ -608,6 +669,9 @@ impl EditorPreviewFrame { Self::Image(image) => { let _ = window.paint_image(bounds, corners, Arc::clone(image), 0, false); } + Self::Stream(image) => { + let _ = window.paint_streaming_image(bounds, corners, image, false); + } #[cfg(target_os = "macos")] Self::Surface(surface) => { window.paint_surface_fitted(bounds, bounds, corners, surface.clone()) @@ -12520,8 +12584,19 @@ pub(crate) fn surface_preview_frame(frame: cap_rendering::SurfaceFrame) -> Edito /// [`frame_image`], timed. The pump calls this so the CPU conversion's cost is /// a number rather than a hunch. pub fn frame_image_timed(frame: &RenderedFrame, stats: &PumpStats) -> Option> { + timed_conversion(stats, || frame_image(frame)) +} + +pub fn bgra_frame_image_timed(frame: RenderedFrame, stats: &PumpStats) -> Option> { + timed_conversion(stats, || bgra_frame_image(frame)) +} + +fn timed_conversion( + stats: &PumpStats, + convert: impl FnOnce() -> Option>, +) -> Option> { let started = Instant::now(); - let image = frame_image(frame); + let image = convert(); stats .convert_nanos .fetch_add(started.elapsed().as_nanos() as u64, Ordering::Relaxed); diff --git a/crates/editor/examples/editor-playback-benchmark.rs b/crates/editor/examples/editor-playback-benchmark.rs index 0af9f92b602..ad1cf5c058b 100644 --- a/crates/editor/examples/editor-playback-benchmark.rs +++ b/crates/editor/examples/editor-playback-benchmark.rs @@ -571,6 +571,7 @@ async fn main() { data.len() + metadata_bytes } EditorFrameOutput::Rgba(frame) => frame.data.len() + 24, + EditorFrameOutput::Bgra(frame) => frame.data.len(), #[cfg(target_os = "macos")] EditorFrameOutput::Surface(_) => return, }; diff --git a/crates/editor/examples/scrub-benchmark.rs b/crates/editor/examples/scrub-benchmark.rs index e89a889945e..8429b5345e4 100644 --- a/crates/editor/examples/scrub-benchmark.rs +++ b/crates/editor/examples/scrub-benchmark.rs @@ -19,7 +19,7 @@ fn arg_value<'a>(args: &'a [String], flag: &str) -> Option<&'a str> { fn presented_frame(output: &EditorFrameOutput) -> u32 { match output { - EditorFrameOutput::Rgba(frame) => frame.frame_number, + EditorFrameOutput::Rgba(frame) | EditorFrameOutput::Bgra(frame) => frame.frame_number, EditorFrameOutput::Nv12(frame) => frame.frame_number, #[cfg(target_os = "macos")] EditorFrameOutput::Surface(frame) => frame.frame_number, diff --git a/crates/editor/src/editor.rs b/crates/editor/src/editor.rs index 160f8f52a84..655cd181e00 100644 --- a/crates/editor/src/editor.rs +++ b/crates/editor/src/editor.rs @@ -6,7 +6,8 @@ use cap_project::{ClipTransitionType, CursorEvents, ProjectConfiguration}; use cap_rendering::SurfaceFrame; use cap_rendering::{ DecodedSegmentFrames, FrameLayout, FrameRenderStageTimings, FrameRenderer, Nv12RenderedFrame, - ProjectUniforms, RenderVideoConstants, RenderedFrame, RendererLayers, TransitionRenderInput, + ProjectUniforms, RenderVideoConstants, RenderedFrame, RendererLayers, RgbaToBgraReadback, + TransitionRenderInput, }; use tokio::sync::{mpsc, oneshot}; @@ -52,6 +53,8 @@ pub struct RendererTransitionInput { pub enum EditorFrameOutput { Rgba(RenderedFrame), + /// Tightly packed BGRA: `padded_bytes_per_row == width * 4`. + Bgra(RenderedFrame), Nv12(Nv12RenderedFrame), #[cfg(target_os = "macos")] Surface(SurfaceFrame), @@ -61,10 +64,25 @@ pub enum EditorFrameOutput { pub enum EditorFrameFormat { #[default] Rgba, + Bgra, #[cfg(target_os = "macos")] BgraSurface, } +/// `Bgra` needs the adapter to render to and copy from a `Bgra8Unorm` +/// texture; where it cannot, the preview falls back to `Rgba` frames, which +/// every consumer of `Bgra` also accepts. +pub(crate) fn supported_frame_format( + format: EditorFrameFormat, + constants: &RenderVideoConstants, +) -> EditorFrameFormat { + if format == EditorFrameFormat::Bgra && !RgbaToBgraReadback::supported(&constants._adapter) { + tracing::warn!("adapter cannot render BGRA readbacks; preview frames stay RGBA"); + return EditorFrameFormat::Rgba; + } + format +} + pub type RendererLayersReceiver = oneshot::Receiver; pub type EditorFrameCallback = Box; @@ -203,6 +221,7 @@ impl Renderer { output_format, } = self; + let output_format = supported_frame_format(output_format, &render_constants); let mut frame_renderer = FrameRenderer::new(&render_constants); let mut layers = match layers_rx.await { @@ -426,6 +445,7 @@ impl Renderer { let flush_start = Instant::now(); if drained_count > 0 { let _ = frame_renderer.flush_pipeline().await; + let _ = frame_renderer.flush_pipeline_bgra().await; } let flush_duration = if drained_count > 0 { flush_start.elapsed() @@ -440,23 +460,47 @@ impl Renderer { let render_start = Instant::now(); let input_frame_number = current.input.uniforms().frame_number; let frame_layout = current.input.uniforms().frame_layout(); + let realtime = current.realtime; let render_result = match (output_format, current.input) { - (EditorFrameFormat::Rgba, PendingRenderInput::Single(input)) => frame_renderer - .render_immediate_with_timings( - input.segment_frames, - input.uniforms, - &input.cursor, - true, - &mut layers, - ) - .await - .map(|(frame, timings)| { - ( - EditorFrameOutput::Rgba(frame), - PlaybackRenderOutputFormat::Rgba, - timings, - ) - }), + (EditorFrameFormat::Rgba, PendingRenderInput::Single(input)) => { + if realtime { + frame_renderer + .render_with_timings( + input.segment_frames, + input.uniforms, + &input.cursor, + true, + &mut layers, + ) + .await + .map(|(frame, timings)| { + frame.map(|frame| { + ( + EditorFrameOutput::Rgba(frame), + PlaybackRenderOutputFormat::Rgba, + timings, + ) + }) + }) + } else { + frame_renderer + .render_immediate_with_timings( + input.segment_frames, + input.uniforms, + &input.cursor, + true, + &mut layers, + ) + .await + .map(|(frame, timings)| { + Some(( + EditorFrameOutput::Rgba(frame), + PlaybackRenderOutputFormat::Rgba, + timings, + )) + }) + } + } ( EditorFrameFormat::Rgba, PendingRenderInput::Transition { @@ -465,32 +509,122 @@ impl Renderer { kind, progress, }, - ) => frame_renderer - .render_transition_immediate( - TransitionRenderInput { - segment_frames: outgoing.segment_frames, - uniforms: outgoing.uniforms, - cursor: &outgoing.cursor, - render_display: true, - }, - TransitionRenderInput { - segment_frames: incoming.segment_frames, - uniforms: incoming.uniforms, - cursor: &incoming.cursor, - render_display: true, - }, + ) => { + let outgoing = TransitionRenderInput { + segment_frames: outgoing.segment_frames, + uniforms: outgoing.uniforms, + cursor: &outgoing.cursor, + render_display: true, + }; + let incoming = TransitionRenderInput { + segment_frames: incoming.segment_frames, + uniforms: incoming.uniforms, + cursor: &incoming.cursor, + render_display: true, + }; + if realtime { + frame_renderer + .render_transition(outgoing, incoming, kind, progress, &mut layers) + .await + .map(|frame| { + frame.map(|frame| { + ( + EditorFrameOutput::Rgba(frame), + PlaybackRenderOutputFormat::Rgba, + FrameRenderStageTimings::default(), + ) + }) + }) + } else { + frame_renderer + .render_transition_immediate( + outgoing, + incoming, + kind, + progress, + &mut layers, + ) + .await + .map(|frame| { + Some(( + EditorFrameOutput::Rgba(frame), + PlaybackRenderOutputFormat::Rgba, + FrameRenderStageTimings::default(), + )) + }) + } + } + (EditorFrameFormat::Bgra, PendingRenderInput::Single(input)) => { + if realtime { + frame_renderer + .render_bgra_with_timings( + input.segment_frames, + input.uniforms, + &input.cursor, + true, + &mut layers, + ) + .await + .map(|(frame, timings)| { + frame.map(|frame| { + ( + EditorFrameOutput::Bgra(frame), + PlaybackRenderOutputFormat::Bgra, + timings, + ) + }) + }) + } else { + frame_renderer + .render_immediate_bgra_with_timings( + input.segment_frames, + input.uniforms, + &input.cursor, + true, + &mut layers, + ) + .await + .map(|(frame, timings)| { + Some(( + EditorFrameOutput::Bgra(frame), + PlaybackRenderOutputFormat::Bgra, + timings, + )) + }) + } + } + ( + EditorFrameFormat::Bgra, + PendingRenderInput::Transition { + outgoing, + incoming, kind, progress, - &mut layers, - ) - .await - .map(|frame| { - ( - EditorFrameOutput::Rgba(frame), - PlaybackRenderOutputFormat::Rgba, - FrameRenderStageTimings::default(), - ) - }), + }, + ) => { + let outgoing = TransitionRenderInput { + segment_frames: outgoing.segment_frames, + uniforms: outgoing.uniforms, + cursor: &outgoing.cursor, + render_display: true, + }; + let incoming = TransitionRenderInput { + segment_frames: incoming.segment_frames, + uniforms: incoming.uniforms, + cursor: &incoming.cursor, + render_display: true, + }; + frame_renderer + .render_transition_bgra(outgoing, incoming, kind, progress, &mut layers) + .await + .map(|frame| { + Some(( + EditorFrameOutput::Bgra(frame), + PlaybackRenderOutputFormat::Bgra, + FrameRenderStageTimings::default(), + )) + }) + } #[cfg(target_os = "macos")] (EditorFrameFormat::BgraSurface, PendingRenderInput::Single(input)) => { frame_renderer @@ -503,11 +637,11 @@ impl Renderer { ) .await .map(|frame| { - ( + Some(( EditorFrameOutput::Surface(frame), PlaybackRenderOutputFormat::Bgra, FrameRenderStageTimings::default(), - ) + )) }) } #[cfg(target_os = "macos")] @@ -539,18 +673,20 @@ impl Renderer { ) .await .map(|frame| { - ( + Some(( EditorFrameOutput::Surface(frame), PlaybackRenderOutputFormat::Bgra, FrameRenderStageTimings::default(), - ) + )) }), }; match render_result { - Ok((frame, output_format, render_stage_timings)) => { + Ok(Some((frame, output_format, render_stage_timings))) => { let render_duration = render_start.elapsed(); let frame_number = match &frame { - EditorFrameOutput::Rgba(frame) => frame.frame_number, + EditorFrameOutput::Rgba(frame) | EditorFrameOutput::Bgra(frame) => { + frame.frame_number + } EditorFrameOutput::Nv12(frame) => frame.frame_number, #[cfg(target_os = "macos")] EditorFrameOutput::Surface(frame) => frame.frame_number, @@ -574,6 +710,9 @@ impl Renderer { } let _ = current.finished.send(true); } + Ok(None) => { + let _ = current.finished.send(true); + } Err(e) => { tracing::error!(error = %e, "Failed to render frame in editor"); let _ = current.finished.send(false); diff --git a/crates/editor/src/preparing_preview.rs b/crates/editor/src/preparing_preview.rs index f13bc342481..8432d91b497 100644 --- a/crates/editor/src/preparing_preview.rs +++ b/crates/editor/src/preparing_preview.rs @@ -582,6 +582,7 @@ async fn run_preview( ); check_cursor_error(&constants)?; control.check()?; + let frame_format = crate::editor::supported_frame_format(options.frame_format, &constants); let mut renderer = FrameRenderer::new(&constants); let mut cursor_cache = PreviewCursorCache::default(); let mut zoom_cache = VecDeque::new(); @@ -685,11 +686,21 @@ async fn run_preview( ) }; let layout = uniforms.frame_layout(); - let output = match options.frame_format { + let output = match frame_format { EditorFrameFormat::Rgba => renderer .render_immediate(frames, uniforms, &cursor, true, &mut layers) .await .map(EditorFrameOutput::Rgba), + EditorFrameFormat::Bgra => renderer + .render_immediate_bgra_with_timings( + frames, + uniforms, + &cursor, + true, + &mut layers, + ) + .await + .map(|(frame, _)| EditorFrameOutput::Bgra(frame)), #[cfg(target_os = "macos")] EditorFrameFormat::BgraSurface => renderer .render_immediate_bgra_surface(frames, uniforms, &cursor, true, &mut layers) @@ -802,7 +813,9 @@ fn validate_output( layout: FrameLayout, ) -> Result<(), PreparingPreviewError> { let (number, width, height) = match output { - EditorFrameOutput::Rgba(frame) => (frame.frame_number, frame.width, frame.height), + EditorFrameOutput::Rgba(frame) | EditorFrameOutput::Bgra(frame) => { + (frame.frame_number, frame.width, frame.height) + } EditorFrameOutput::Nv12(frame) => (frame.frame_number, frame.width, frame.height), #[cfg(target_os = "macos")] EditorFrameOutput::Surface(frame) => (frame.frame_number, frame.width, frame.height), diff --git a/crates/editor/src/preparing_preview/native_tests.rs b/crates/editor/src/preparing_preview/native_tests.rs index eb62cbb134e..1daa8602c95 100644 --- a/crates/editor/src/preparing_preview/native_tests.rs +++ b/crates/editor/src/preparing_preview/native_tests.rs @@ -792,6 +792,10 @@ async fn composed_parity_with_reference( assert_eq!(frame_format, EditorFrameFormat::Rgba); frame } + EditorFrameOutput::Bgra(frame) => { + assert_eq!(frame_format, EditorFrameFormat::Bgra); + bgra_rgba(frame) + } #[cfg(target_os = "macos")] EditorFrameOutput::Surface(frame) => { assert_eq!(frame_format, EditorFrameFormat::BgraSurface); @@ -1075,6 +1079,32 @@ fn surface_rgba(frame: cap_rendering::SurfaceFrame) -> RenderedFrame { } } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn native_preparing_bgra_readback_matches_default_editor_frame() { + composed_parity_with_reference( + fixture(false), + &[0, 45, 12, 90], + false, + EditorFrameFormat::Bgra, + ) + .await; +} + +fn bgra_rgba(frame: RenderedFrame) -> RenderedFrame { + let row_bytes = frame.width * 4; + assert_eq!(frame.padded_bytes_per_row, row_bytes); + assert_eq!(frame.data.len(), row_bytes as usize * frame.height as usize); + let rgba = frame + .data + .chunks_exact(4) + .flat_map(|pixel| [pixel[2], pixel[1], pixel[0], pixel[3]]) + .collect(); + RenderedFrame { + data: Arc::new(rgba), + ..frame + } +} + #[cfg(target_os = "macos")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn native_preparing_bgra_surface_matches_default_avasset_editor_frame() { @@ -1263,7 +1293,8 @@ async fn native_handoff_reuses_pcm_preserves_audio_across_candidate_disposal_and |_| {}, Box::new(move |output, _| { let frame = match output { - crate::EditorFrameOutput::Rgba(frame) => frame.frame_number, + crate::EditorFrameOutput::Rgba(frame) + | crate::EditorFrameOutput::Bgra(frame) => frame.frame_number, crate::EditorFrameOutput::Nv12(frame) => frame.frame_number, #[cfg(target_os = "macos")] crate::EditorFrameOutput::Surface(frame) => frame.frame_number, @@ -1411,7 +1442,8 @@ async fn native_silent_handoff_opens_and_renders_through_an_equivalent_editor_pa |_| {}, Box::new(move |output, _| { let frame = match output { - crate::EditorFrameOutput::Rgba(frame) => frame.frame_number, + crate::EditorFrameOutput::Rgba(frame) + | crate::EditorFrameOutput::Bgra(frame) => frame.frame_number, crate::EditorFrameOutput::Nv12(frame) => frame.frame_number, #[cfg(target_os = "macos")] crate::EditorFrameOutput::Surface(frame) => frame.frame_number, @@ -1613,7 +1645,8 @@ async fn native_studio_sound_releases_preparing_audio_and_preserves_transport() |_| {}, Box::new(move |output, _| { let frame = match output { - crate::EditorFrameOutput::Rgba(frame) => frame.frame_number, + crate::EditorFrameOutput::Rgba(frame) + | crate::EditorFrameOutput::Bgra(frame) => frame.frame_number, crate::EditorFrameOutput::Nv12(frame) => frame.frame_number, #[cfg(target_os = "macos")] crate::EditorFrameOutput::Surface(frame) => frame.frame_number, diff --git a/crates/editor/src/thumbnail.rs b/crates/editor/src/thumbnail.rs index 9b1abe68a9f..11b29bcca61 100644 --- a/crates/editor/src/thumbnail.rs +++ b/crates/editor/src/thumbnail.rs @@ -268,6 +268,7 @@ mod tests { ffmpeg::init().unwrap(); for format in [ crate::EditorFrameFormat::Rgba, + crate::EditorFrameFormat::Bgra, #[cfg(target_os = "macos")] crate::EditorFrameFormat::BgraSurface, ] { diff --git a/crates/rendering/src/frame_pipeline.rs b/crates/rendering/src/frame_pipeline.rs index 2b9fb639c81..f3d42fec0fa 100644 --- a/crates/rendering/src/frame_pipeline.rs +++ b/crates/rendering/src/frame_pipeline.rs @@ -106,25 +106,6 @@ async fn wait_for_submission( } } -/// The result of a `map_async` whose submission has finished. wgpu runs map -/// callbacks on whichever thread's poll collected them, so when several -/// renderers share a device, another thread can still be about to run this -/// buffer's callback after this thread's own wait returned. -async fn mapped_result( - rx: &mut oneshot::Receiver>, - started: Instant, - frame_number: u32, - timeout: std::time::Duration, -) -> Option> { - match rx.try_recv() { - Ok(result) => return Some(result), - Err(oneshot::error::TryRecvError::Closed) => return None, - Err(oneshot::error::TryRecvError::Empty) => {} - } - let remaining = timeout.checked_sub(readback_waited(started, frame_number))?; - tokio::time::timeout(remaining, rx).await.ok()?.ok() -} - #[derive(Clone, Copy, Debug, Default)] pub struct FinishEncoderTimings { pub wait_previous_duration: std::time::Duration, @@ -1166,18 +1147,17 @@ impl PendingNv12Readback { return Err(self.cancel()); }; - let started = Instant::now(); - let timeout = gpu_buffer_wait_timeout(self.export_deadlines); - let completed = - wait_for_submission(device, &submission, started, self.frame_number, timeout).await; - let mapped = if completed { - mapped_result(&mut rx, started, self.frame_number, timeout).await - } else { - rx.try_recv().ok() - }; - match (completed, mapped) { - (true, Some(Ok(()))) => {} - (_, Some(Err(error))) => { + let completed = wait_for_submission( + device, + &submission, + Instant::now(), + self.frame_number, + gpu_buffer_wait_timeout(self.export_deadlines), + ) + .await; + match (completed, rx.try_recv()) { + (true, Ok(Ok(()))) => {} + (_, Ok(Err(error))) => { self.unmap(); return Err(RenderingError::BufferMapFailed(error)); } @@ -1731,24 +1711,39 @@ impl PendingReadback { device: &wgpu::Device, recycled: &mut RecycledReadbackFrames, ) -> Result { + let active_bytes = self.map(device).await?; + let mut data_vec = if self.export_deadlines { + recycled.take().unwrap_or_default() + } else { + Vec::new() + }; + reserve_readback(&mut data_vec, active_bytes, self.frame_number)?; + let buffer_slice = self.buffer.slice(..active_bytes as u64); + let data = buffer_slice.get_mapped_range(); + data_vec.extend_from_slice(&data); + drop(data); + let recycle = self.export_deadlines && self.recycle_readback_frames; + let padded_bytes_per_row = self.padded_bytes_per_row; + let frame = self.into_frame(data_vec, padded_bytes_per_row); + if recycle { + recycled.keep(&frame.data); + } + Ok(frame) + } + + async fn map(&mut self, device: &wgpu::Device) -> Result { let start_time = Instant::now(); - let timeout = gpu_buffer_wait_timeout(self.export_deadlines); let completed = wait_for_submission( device, &self.submission, start_time, self.frame_number, - timeout, + gpu_buffer_wait_timeout(self.export_deadlines), ) .await; - let mapped = if completed { - mapped_result(&mut self.rx, start_time, self.frame_number, timeout).await - } else { - self.rx.try_recv().ok() - }; - match (completed, mapped) { - (true, Some(Ok(()))) => {} - (_, Some(Err(error))) => { + match (completed, self.rx.try_recv()) { + (true, Ok(Ok(()))) => {} + (_, Ok(Err(error))) => { self.unmap(); return Err(RenderingError::BufferMapFailed(error)); } @@ -1779,17 +1774,10 @@ impl PendingReadback { self.unmap(); return Err(RenderingError::BufferMapWaitingFailed); }; - let mut data_vec = if self.export_deadlines { - recycled.take().unwrap_or_default() - } else { - Vec::new() - }; - reserve_readback(&mut data_vec, active_bytes, self.frame_number)?; - let buffer_slice = self.buffer.slice(..active_bytes as u64); - let data = buffer_slice.get_mapped_range(); - data_vec.extend_from_slice(&data); + Ok(active_bytes) + } - drop(data); + fn into_frame(mut self, data: Vec, padded_bytes_per_row: u32) -> RenderedFrame { self.unmap(); let target_time_ns = @@ -1798,18 +1786,14 @@ impl PendingReadback { if let Some(phase) = self.readiness_phase.take() { phase.finish("frame_ready"); } - let data = Arc::new(data_vec); - if self.export_deadlines && self.recycle_readback_frames { - recycled.keep(&data); - } - Ok(RenderedFrame { - data, - padded_bytes_per_row: self.padded_bytes_per_row, + RenderedFrame { + data: Arc::new(data), + padded_bytes_per_row, width: self.width, height: self.height, frame_number: self.frame_number, target_time_ns, - }) + } } } @@ -1819,6 +1803,298 @@ impl Drop for PendingReadback { } } +fn tight_rows(data: &[u8], row_bytes: usize, stride: usize) -> Option> { + if row_bytes == stride { + return Some(data.to_vec()); + } + if stride == 0 || !data.len().is_multiple_of(stride) { + return None; + } + let mut tight = Vec::with_capacity(row_bytes.checked_mul(data.len() / stride)?); + for row in data.chunks_exact(stride) { + tight.extend_from_slice(row.get(..row_bytes)?); + } + Some(tight) +} + +/// Converts the rendered RGBA frame to BGRA on the GPU before reading it back, +/// so a CPU consumer that wants BGRA (gpui's sprite atlas) gets display-ready, +/// tightly packed bytes from a single copy out of the mapped buffer. +pub struct RgbaToBgraReadback { + pipeline: wgpu::RenderPipeline, + bind_group_layout: wgpu::BindGroupLayout, + target: Option, + source_bind_groups: Vec<(wgpu::Texture, wgpu::BindGroup)>, + pending: Option, +} + +struct BgraReadbackTarget { + width: u32, + height: u32, + view: wgpu::TextureView, + texture: wgpu::Texture, + buffer: Arc, + padded_bytes_per_row: u32, +} + +pub struct PendingBgraReadback(PendingReadback); + +impl RgbaToBgraReadback { + pub fn supported(adapter: &wgpu::Adapter) -> bool { + adapter + .get_texture_format_features(wgpu::TextureFormat::Bgra8Unorm) + .allowed_usages + .contains(wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::COPY_SRC) + } + + pub fn new(device: &wgpu::Device) -> Self { + let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("RGBA to BGRA Readback Blit"), + source: wgpu::ShaderSource::Wgsl(std::borrow::Cow::Borrowed(include_str!( + "shaders/blit_bgra_surface.wgsl" + ))), + }); + let bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("RGBA to BGRA Readback Bind Group Layout"), + entries: &[wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: false }, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + count: None, + }], + }); + let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("RGBA to BGRA Readback Pipeline Layout"), + bind_group_layouts: &[&bind_group_layout], + push_constant_ranges: &[], + }); + let pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("RGBA to BGRA Readback Pipeline"), + layout: Some(&pipeline_layout), + vertex: wgpu::VertexState { + module: &shader, + entry_point: Some("vs_main"), + compilation_options: Default::default(), + buffers: &[], + }, + fragment: Some(wgpu::FragmentState { + module: &shader, + entry_point: Some("fs_main"), + compilation_options: Default::default(), + targets: &[Some(wgpu::ColorTargetState { + format: wgpu::TextureFormat::Bgra8Unorm, + blend: None, + write_mask: wgpu::ColorWrites::ALL, + })], + }), + primitive: wgpu::PrimitiveState::default(), + depth_stencil: None, + multisample: wgpu::MultisampleState::default(), + multiview: None, + cache: None, + }); + + Self { + pipeline, + bind_group_layout, + target: None, + source_bind_groups: Vec::new(), + pending: None, + } + } + + pub fn take_pending(&mut self) -> Option { + self.pending.take() + } + + fn source_bind_group( + &mut self, + device: &wgpu::Device, + source_texture: &wgpu::Texture, + ) -> wgpu::BindGroup { + if let Some((_, bind_group)) = self + .source_bind_groups + .iter() + .find(|(texture, _)| texture == source_texture) + { + return bind_group.clone(); + } + let source_view = source_texture.create_view(&Default::default()); + let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("RGBA to BGRA Readback Bind Group"), + layout: &self.bind_group_layout, + entries: &[wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView(&source_view), + }], + }); + if self.source_bind_groups.len() >= 2 { + self.source_bind_groups.remove(0); + } + self.source_bind_groups + .push((source_texture.clone(), bind_group.clone())); + bind_group + } + + fn ensure_target(&mut self, device: &wgpu::Device, width: u32, height: u32) { + if self + .target + .as_ref() + .is_some_and(|target| target.width == width && target.height == height) + { + return; + } + let padded_bytes_per_row = padded_bytes_per_row((width, height)); + let texture = device.create_texture(&wgpu::TextureDescriptor { + label: Some("BGRA Readback Texture"), + size: wgpu::Extent3d { + width, + height, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Bgra8Unorm, + usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::COPY_SRC, + view_formats: &[], + }); + let buffer = Arc::new(device.create_buffer(&wgpu::BufferDescriptor { + label: Some("BGRA Readback Buffer"), + size: u64::from(padded_bytes_per_row) * u64::from(height), + usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, + mapped_at_creation: false, + })); + self.target = Some(BgraReadbackTarget { + width, + height, + view: texture.create_view(&Default::default()), + texture, + buffer, + padded_bytes_per_row, + }); + } + + pub fn submit( + &mut self, + device: &wgpu::Device, + queue: &wgpu::Queue, + mut encoder: wgpu::CommandEncoder, + source: &wgpu::Texture, + frame_number: u32, + frame_rate: u32, + ) -> Result { + let (width, height) = (source.width(), source.height()); + if width == 0 || height == 0 { + return Err(RenderingError::BufferMapWaitingFailed); + } + let bind_group = self.source_bind_group(device, source); + self.ensure_target(device, width, height); + let target = self + .target + .as_ref() + .expect("BGRA readback target initialized"); + + { + let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("RGBA to BGRA Readback Blit"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &target.view, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + pass.set_pipeline(&self.pipeline); + pass.set_bind_group(0, &bind_group, &[]); + pass.draw(0..3, 0..1); + } + + encoder.copy_texture_to_buffer( + wgpu::TexelCopyTextureInfo { + texture: &target.texture, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + wgpu::TexelCopyBufferInfo { + buffer: &target.buffer, + layout: wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(target.padded_bytes_per_row), + rows_per_image: Some(height), + }, + }, + wgpu::Extent3d { + width, + height, + depth_or_array_layers: 1, + }, + ); + + let submission = queue.submit(std::iter::once(encoder.finish())); + + let (tx, rx) = oneshot::channel(); + target + .buffer + .slice(..u64::from(target.padded_bytes_per_row) * u64::from(height)) + .map_async(wgpu::MapMode::Read, move |result| { + if let Err(e) = tx.send(result) { + tracing::error!("Failed to send map_async result: {:?}", e); + } + }); + + Ok(PendingBgraReadback(PendingReadback { + readiness_phase: None, + rx, + submission, + mapped: true, + buffer: target.buffer.clone(), + padded_bytes_per_row: target.padded_bytes_per_row, + width, + height, + frame_number, + frame_rate, + export_deadlines: false, + recycle_readback_frames: false, + })) + } +} + +impl PendingBgraReadback { + /// The frame as tightly packed BGRA: `padded_bytes_per_row` is always + /// `width * 4`, so the bytes can back an image without un-padding. + pub async fn wait(self, device: &wgpu::Device) -> Result { + let mut pending = self.0; + let active_bytes = pending.map(device).await?; + let row_bytes = pending.width * 4; + let data = pending + .buffer + .slice(..active_bytes as u64) + .get_mapped_range(); + let tight = tight_rows( + &data, + row_bytes as usize, + pending.padded_bytes_per_row as usize, + ); + drop(data); + let Some(tight) = tight else { + pending.unmap(); + return Err(RenderingError::BufferMapWaitingFailed); + }; + Ok(pending.into_frame(tight, row_bytes)) + } +} + const MAX_RECYCLED_READBACK_FRAMES: usize = 8; /// Export readbacks are handed downstream as `Arc>`; keeping a clone @@ -2413,6 +2689,52 @@ pub async fn finish_encoder_bgra_surface( result } +pub async fn finish_encoder_bgra_readback( + session: &mut RenderSession, + readback: &mut RgbaToBgraReadback, + device: &wgpu::Device, + queue: &wgpu::Queue, + uniforms: &ProjectUniforms, + encoder: wgpu::CommandEncoder, +) -> Result, RenderingError> { + let previous = readback.take_pending(); + let pending = readback.submit( + device, + queue, + encoder, + session.current_texture(), + uniforms.frame_number, + uniforms.frame_rate, + )?; + readback.pending = Some(pending); + if let Some(previous) = previous { + match previous.wait(device).await { + Ok(frame) => Ok(Some(frame)), + Err(error) => { + drain_submitted_work(device, queue).await; + Err(error) + } + } + } else { + Ok(None) + } +} + +pub async fn flush_pending_bgra_readback( + readback: &mut RgbaToBgraReadback, + device: &wgpu::Device, + queue: &wgpu::Queue, +) -> Option> { + let pending = readback.take_pending()?; + Some(match pending.wait(device).await { + Ok(frame) => Ok(frame), + Err(error) => { + drain_submitted_work(device, queue).await; + Err(error) + } + }) +} + pub async fn flush_pending_readback( session: &mut RenderSession, device: &wgpu::Device, @@ -2429,41 +2751,6 @@ pub async fn flush_pending_readback( mod readback_output_tests { use super::*; - #[tokio::test] - async fn map_callback_run_by_another_thread_after_the_wait_still_counts() { - let timeout = std::time::Duration::from_secs(5); - let (tx, mut rx) = oneshot::channel(); - let late = std::thread::spawn(move || { - std::thread::sleep(std::time::Duration::from_millis(50)); - tx.send(Ok(())).expect("map receiver"); - }); - assert!(matches!( - mapped_result(&mut rx, Instant::now(), 0, timeout).await, - Some(Ok(())) - )); - late.join().unwrap(); - - let (tx, mut rx) = oneshot::channel::>(); - drop(tx); - assert!( - mapped_result(&mut rx, Instant::now(), 0, timeout) - .await - .is_none() - ); - - let (_tx, mut rx) = oneshot::channel::>(); - assert!( - mapped_result( - &mut rx, - Instant::now(), - 0, - std::time::Duration::from_millis(20) - ) - .await - .is_none() - ); - } - #[test] fn readback_allocation_reserves_frame_and_metadata_without_changing_length() { let mut data = Vec::new(); @@ -2541,6 +2828,37 @@ mod readback_output_tests { } } + #[test] + fn aligned_bgra_rows_are_copied_whole() { + let data: Vec = (0..=255).cycle().take(7_680 * 3).collect(); + assert_eq!(tight_rows(&data, 7_680, 7_680), Some(data)); + } + + #[test] + fn padded_bgra_rows_drop_only_the_padding() { + let stride = 256; + let row_bytes = 3 * 4; + let mut data = vec![0xAA; stride * 2]; + for (index, byte) in data[..row_bytes].iter_mut().enumerate() { + *byte = index as u8; + } + for (index, byte) in data[stride..stride + row_bytes].iter_mut().enumerate() { + *byte = 100 + index as u8; + } + let tight = tight_rows(&data, row_bytes, stride).expect("rows fit"); + assert_eq!(tight.len(), row_bytes * 2); + assert_eq!(&tight[..row_bytes], &data[..row_bytes]); + assert_eq!(&tight[row_bytes..], &data[stride..stride + row_bytes]); + assert!(!tight.contains(&0xAA)); + } + + #[test] + fn malformed_bgra_rows_are_rejected() { + assert_eq!(tight_rows(&[0; 300], 12, 256), None); + assert_eq!(tight_rows(&[0; 512], 260, 256), None); + assert_eq!(tight_rows(&[0; 16], 4, 0), None); + } + #[test] fn oversized_readback_buffers_only_include_active_rows() { assert_eq!( diff --git a/crates/rendering/src/lib.rs b/crates/rendering/src/lib.rs index da01e6295eb..e7cdb6e8eac 100644 --- a/crates/rendering/src/lib.rs +++ b/crates/rendering/src/lib.rs @@ -15,8 +15,8 @@ use decoder::{AsyncVideoDecoderHandle, spawn_decoder}; #[cfg(target_os = "macos")] use frame_pipeline::finish_encoder_bgra_surface; use frame_pipeline::{ - NV12BufferPool, RenderSession, finish_encoder_nv12_pooled, finish_encoder_timed, - flush_pending_readback, + NV12BufferPool, RenderSession, finish_encoder_bgra_readback, finish_encoder_nv12_pooled, + finish_encoder_timed, flush_pending_bgra_readback, flush_pending_readback, }; use futures::future::OptionFuture; use layers::{ @@ -72,6 +72,7 @@ pub use frame_pipeline::Nv12Surface; #[cfg(target_os = "macos")] pub use frame_pipeline::SurfaceFrame; pub use frame_pipeline::{GpuOutputFormat, Nv12RenderedFrame, RenderedFrame, SharedNv12Buffer}; +pub use frame_pipeline::{PendingBgraReadback, RgbaToBgraReadback}; #[cfg(target_os = "macos")] pub use frame_pipeline::{PendingSurface, RgbaToBgraSurfaceConverter}; pub use frame_windows::FrameWindows; @@ -7323,6 +7324,7 @@ pub struct FrameRenderer<'a> { nv12_surface_output: bool, #[cfg(target_os = "macos")] bgra_surface_converter: Option, + bgra_readback: Option, nv12_buffer_pool: NV12BufferPool, transition_compositor: Option, export_deadlines: bool, @@ -7342,6 +7344,7 @@ impl<'a> FrameRenderer<'a> { nv12_surface_output: false, #[cfg(target_os = "macos")] bgra_surface_converter: None, + bgra_readback: None, nv12_buffer_pool: NV12BufferPool::new(6), transition_compositor: None, export_deadlines: false, @@ -7651,6 +7654,216 @@ impl<'a> FrameRenderer<'a> { Ok((frame, timings)) } + /// [`Self::render_immediate_with_timings`], read back as tightly packed + /// BGRA (`padded_bytes_per_row == width * 4`) instead of padded RGBA. + pub async fn render_bgra_with_timings( + &mut self, + segment_frames: DecodedSegmentFrames, + uniforms: ProjectUniforms, + cursor: &CursorEvents, + render_display: bool, + layers: &mut RendererLayers, + ) -> Result<(Option, FrameRenderStageTimings), RenderingError> { + let phase = std::mem::take(&mut self.readiness_first_render) + .then(|| readiness::Phase::start("frame.bgra_readback_first_render")); + let mut last_error = None; + + for attempt in 0..Self::MAX_RENDER_RETRIES { + if attempt > 0 { + tracing::warn!( + frame_number = uniforms.frame_number, + attempt = attempt + 1, + "Retrying BGRA readback frame render after GPU error" + ); + self.reset_session(); + self.bgra_readback = None; + tokio::time::sleep(std::time::Duration::from_millis(100 * (attempt as u64 + 1))) + .await; + } + + let readback = self.bgra_readback.get_or_insert_with(|| { + frame_pipeline::RgbaToBgraReadback::new(&self.constants.device) + }); + let session = self.session.get_or_insert_with(|| { + RenderSession::new( + &self.constants.device, + uniforms.output_size.0, + uniforms.output_size.1, + ) + .with_export_deadlines(self.export_deadlines) + }); + session.update_texture_size( + &self.constants.device, + uniforms.output_size.0, + uniforms.output_size.1, + ); + + let (encoder, mut timings) = encode_frame_with_timings( + self.constants, + &segment_frames, + &uniforms, + cursor, + render_display, + layers, + session, + ) + .await?; + + let finish_start = Instant::now(); + match finish_encoder_bgra_readback( + session, + readback, + &self.constants.device, + &self.constants.queue, + &uniforms, + encoder, + ) + .await + { + Ok(frame) => { + timings.finish_duration = finish_start.elapsed(); + if let Some(phase) = phase { + phase.finish("frame_ready"); + } + return Ok((frame, timings)); + } + Err(RenderingError::BufferMapWaitingFailed) => { + tracing::warn!( + frame_number = uniforms.frame_number, + attempt = attempt + 1, + "GPU buffer mapping failed, will retry" + ); + last_error = Some(RenderingError::BufferMapWaitingFailed); + } + Err(RenderingError::BufferMapFailed(e)) => { + tracing::warn!( + frame_number = uniforms.frame_number, + attempt = attempt + 1, + error = %e, + "GPU buffer async error, will retry" + ); + last_error = Some(RenderingError::BufferMapFailed(e)); + } + Err(e) => return Err(e), + } + } + + Err(last_error.unwrap_or(RenderingError::BufferMapWaitingFailed)) + } + + pub async fn render_immediate_bgra_with_timings( + &mut self, + segment_frames: DecodedSegmentFrames, + uniforms: ProjectUniforms, + cursor: &CursorEvents, + render_display: bool, + layers: &mut RendererLayers, + ) -> Result<(RenderedFrame, FrameRenderStageTimings), RenderingError> { + let (frame, mut timings) = self + .render_bgra_with_timings(segment_frames, uniforms, cursor, render_display, layers) + .await?; + if let Some(frame) = frame { + return Ok((frame, timings)); + } + let flush_start = Instant::now(); + let frame = self + .flush_pipeline_bgra() + .await + .unwrap_or(Err(RenderingError::BufferMapWaitingFailed))?; + timings.immediate_flush_duration = flush_start.elapsed(); + Ok((frame, timings)) + } + + /// [`Self::render_transition_immediate`], read back as tightly packed BGRA. + pub async fn render_transition_bgra( + &mut self, + outgoing: TransitionRenderInput<'_>, + incoming: TransitionRenderInput<'_>, + kind: ClipTransitionType, + progress: f32, + layers: &mut RendererLayers, + ) -> Result { + let mut last_error = None; + + for attempt in 0..Self::MAX_RENDER_RETRIES { + if attempt > 0 { + tracing::warn!( + frame_number = incoming.uniforms.frame_number, + attempt = attempt + 1, + "Retrying BGRA readback transition frame after GPU error" + ); + self.reset_session(); + self.bgra_readback = None; + tokio::time::sleep(std::time::Duration::from_millis(100 * (attempt as u64 + 1))) + .await; + } + + let readback = self.bgra_readback.get_or_insert_with(|| { + frame_pipeline::RgbaToBgraReadback::new(&self.constants.device) + }); + let session = self.session.get_or_insert_with(|| { + RenderSession::new( + &self.constants.device, + incoming.uniforms.output_size.0, + incoming.uniforms.output_size.1, + ) + .with_export_deadlines(self.export_deadlines) + }); + session.update_texture_size( + &self.constants.device, + incoming.uniforms.output_size.0, + incoming.uniforms.output_size.1, + ); + let compositor = self + .transition_compositor + .get_or_insert_with(|| TransitionCompositor::new(&self.constants.device)); + compositor.ensure_size( + &self.constants.device, + incoming.uniforms.output_size.0, + incoming.uniforms.output_size.1, + ); + + let encoder = produce_transition_texture( + self.constants, + &outgoing, + &incoming, + (kind, progress), + layers, + session, + compositor, + ) + .await?; + + match finish_encoder_bgra_readback( + session, + readback, + &self.constants.device, + &self.constants.queue, + &incoming.uniforms, + encoder, + ) + .await + { + Ok(Some(frame)) => return Ok(frame), + Ok(None) => { + if let Some(frame) = self.flush_pipeline_bgra().await { + return frame; + } + continue; + } + Err(RenderingError::BufferMapWaitingFailed) => { + last_error = Some(RenderingError::BufferMapWaitingFailed); + } + Err(RenderingError::BufferMapFailed(error)) => { + last_error = Some(RenderingError::BufferMapFailed(error)); + } + Err(error) => return Err(error), + } + } + + Err(last_error.unwrap_or(RenderingError::BufferMapWaitingFailed)) + } + pub async fn flush_pipeline(&mut self) -> Option> { if let Some(session) = &mut self.session { flush_pending_readback(session, &self.constants.device).await @@ -7659,6 +7872,11 @@ impl<'a> FrameRenderer<'a> { } } + pub async fn flush_pipeline_bgra(&mut self) -> Option> { + let readback = self.bgra_readback.as_mut()?; + flush_pending_bgra_readback(readback, &self.constants.device, &self.constants.queue).await + } + pub async fn render_immediate_nv12( &mut self, segment_frames: DecodedSegmentFrames, @@ -9126,15 +9344,15 @@ impl LayerPasses<'_> { } } -async fn produce_frame_with_timings( +async fn encode_frame_with_timings( constants: &RenderVideoConstants, - segment_frames: DecodedSegmentFrames, - uniforms: ProjectUniforms, + segment_frames: &DecodedSegmentFrames, + uniforms: &ProjectUniforms, cursor: &CursorEvents, render_display: bool, layers: &mut RendererLayers, session: &mut RenderSession, -) -> Result<(Option, FrameRenderStageTimings), RenderingError> { +) -> Result<(wgpu::CommandEncoder, FrameRenderStageTimings), RenderingError> { let mut encoder = constants.device.create_command_encoder( &(wgpu::CommandEncoderDescriptor { label: Some("Render Encoder"), @@ -9145,8 +9363,8 @@ async fn produce_frame_with_timings( let mut timings = layers .prepare_with_encoder_timed( constants, - &uniforms, - &segment_frames, + uniforms, + segment_frames, cursor, &mut encoder, render_display, @@ -9160,11 +9378,34 @@ async fn produce_frame_with_timings( &constants.queue, &mut encoder, session, - &uniforms, + uniforms, render_display, )?; timings.layer_render_duration = layer_render_start.elapsed(); + Ok((encoder, timings)) +} + +async fn produce_frame_with_timings( + constants: &RenderVideoConstants, + segment_frames: DecodedSegmentFrames, + uniforms: ProjectUniforms, + cursor: &CursorEvents, + render_display: bool, + layers: &mut RendererLayers, + session: &mut RenderSession, +) -> Result<(Option, FrameRenderStageTimings), RenderingError> { + let (encoder, mut timings) = encode_frame_with_timings( + constants, + &segment_frames, + &uniforms, + cursor, + render_display, + layers, + session, + ) + .await?; + let finish_start = Instant::now(); let (frame, finish_timings) = finish_encoder_timed( session, From a11789f8233dddd240ccbd4b25d9352d17c20cdf Mon Sep 17 00:00:00 2001 From: Aatman Jain Date: Thu, 29 Oct 2026 23:53:41 +0000 Subject: [PATCH 2/9] fix: address Greptile review for editor preview readback pipeline Double-buffer BGRA GPU readbacks, wait for map callbacks reliably, and flush pipelined frames before seek/immediate renders. --- crates/editor/src/editor.rs | 5 +- crates/rendering/src/frame_pipeline.rs | 110 ++++++++++++++++--------- 2 files changed, 74 insertions(+), 41 deletions(-) diff --git a/crates/editor/src/editor.rs b/crates/editor/src/editor.rs index 655cd181e00..53e5618f7e2 100644 --- a/crates/editor/src/editor.rs +++ b/crates/editor/src/editor.rs @@ -443,11 +443,12 @@ impl Renderer { let queue_wait = current.queued_at.elapsed(); let drain_duration = queue_drain_start.elapsed(); let flush_start = Instant::now(); - if drained_count > 0 { + let flush_pipelines = drained_count > 0 || !current.realtime; + if flush_pipelines { let _ = frame_renderer.flush_pipeline().await; let _ = frame_renderer.flush_pipeline_bgra().await; } - let flush_duration = if drained_count > 0 { + let flush_duration = if flush_pipelines { flush_start.elapsed() } else { std::time::Duration::ZERO diff --git a/crates/rendering/src/frame_pipeline.rs b/crates/rendering/src/frame_pipeline.rs index f3d42fec0fa..8d61634005e 100644 --- a/crates/rendering/src/frame_pipeline.rs +++ b/crates/rendering/src/frame_pipeline.rs @@ -1733,17 +1733,23 @@ impl PendingReadback { async fn map(&mut self, device: &wgpu::Device) -> Result { let start_time = Instant::now(); + let timeout = gpu_buffer_wait_timeout(self.export_deadlines); let completed = wait_for_submission( device, &self.submission, start_time, self.frame_number, - gpu_buffer_wait_timeout(self.export_deadlines), + timeout, ) .await; - match (completed, self.rx.try_recv()) { - (true, Ok(Ok(()))) => {} - (_, Ok(Err(error))) => { + let mapped = if completed { + mapped_result(&mut self.rx, start_time, self.frame_number, timeout).await + } else { + self.rx.try_recv().ok() + }; + match (completed, mapped) { + (true, Some(Ok(()))) => {} + (_, Some(Err(error))) => { self.unmap(); return Err(RenderingError::BufferMapFailed(error)); } @@ -1828,13 +1834,53 @@ pub struct RgbaToBgraReadback { pending: Option, } -struct BgraReadbackTarget { - width: u32, - height: u32, +const BGRA_READBACK_SLOTS: usize = 2; + +struct BgraReadbackSlot { view: wgpu::TextureView, texture: wgpu::Texture, buffer: Arc, +} + +struct BgraReadbackTarget { + width: u32, + height: u32, padded_bytes_per_row: u32, + slots: [BgraReadbackSlot; BGRA_READBACK_SLOTS], + next_slot: usize, +} + +fn new_bgra_readback_slot( + device: &wgpu::Device, + width: u32, + height: u32, + padded_bytes_per_row: u32, +) -> BgraReadbackSlot { + let texture = device.create_texture(&wgpu::TextureDescriptor { + label: Some("BGRA Readback Texture"), + size: wgpu::Extent3d { + width, + height, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Bgra8Unorm, + usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::COPY_SRC, + view_formats: &[], + }); + let buffer = Arc::new(device.create_buffer(&wgpu::BufferDescriptor { + label: Some("BGRA Readback Buffer"), + size: u64::from(padded_bytes_per_row) * u64::from(height), + usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, + mapped_at_creation: false, + })); + BgraReadbackSlot { + view: texture.create_view(&Default::default()), + texture, + buffer, + } } pub struct PendingBgraReadback(PendingReadback); @@ -1949,33 +1995,15 @@ impl RgbaToBgraReadback { return; } let padded_bytes_per_row = padded_bytes_per_row((width, height)); - let texture = device.create_texture(&wgpu::TextureDescriptor { - label: Some("BGRA Readback Texture"), - size: wgpu::Extent3d { - width, - height, - depth_or_array_layers: 1, - }, - mip_level_count: 1, - sample_count: 1, - dimension: wgpu::TextureDimension::D2, - format: wgpu::TextureFormat::Bgra8Unorm, - usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::COPY_SRC, - view_formats: &[], + let slots = std::array::from_fn(|_| { + new_bgra_readback_slot(device, width, height, padded_bytes_per_row) }); - let buffer = Arc::new(device.create_buffer(&wgpu::BufferDescriptor { - label: Some("BGRA Readback Buffer"), - size: u64::from(padded_bytes_per_row) * u64::from(height), - usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, - mapped_at_creation: false, - })); self.target = Some(BgraReadbackTarget { width, height, - view: texture.create_view(&Default::default()), - texture, - buffer, padded_bytes_per_row, + slots, + next_slot: 0, }); } @@ -1996,14 +2024,18 @@ impl RgbaToBgraReadback { self.ensure_target(device, width, height); let target = self .target - .as_ref() + .as_mut() .expect("BGRA readback target initialized"); + let slot_index = target.next_slot; + target.next_slot = (slot_index + 1) % BGRA_READBACK_SLOTS; + let slot = &target.slots[slot_index]; + let padded_bytes_per_row = target.padded_bytes_per_row; { let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { label: Some("RGBA to BGRA Readback Blit"), color_attachments: &[Some(wgpu::RenderPassColorAttachment { - view: &target.view, + view: &slot.view, resolve_target: None, ops: wgpu::Operations { load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT), @@ -2021,16 +2053,16 @@ impl RgbaToBgraReadback { encoder.copy_texture_to_buffer( wgpu::TexelCopyTextureInfo { - texture: &target.texture, + texture: &slot.texture, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All, }, wgpu::TexelCopyBufferInfo { - buffer: &target.buffer, + buffer: &slot.buffer, layout: wgpu::TexelCopyBufferLayout { offset: 0, - bytes_per_row: Some(target.padded_bytes_per_row), + bytes_per_row: Some(padded_bytes_per_row), rows_per_image: Some(height), }, }, @@ -2044,9 +2076,9 @@ impl RgbaToBgraReadback { let submission = queue.submit(std::iter::once(encoder.finish())); let (tx, rx) = oneshot::channel(); - target - .buffer - .slice(..u64::from(target.padded_bytes_per_row) * u64::from(height)) + let readback_buffer = slot.buffer.clone(); + readback_buffer + .slice(..u64::from(padded_bytes_per_row) * u64::from(height)) .map_async(wgpu::MapMode::Read, move |result| { if let Err(e) = tx.send(result) { tracing::error!("Failed to send map_async result: {:?}", e); @@ -2058,8 +2090,8 @@ impl RgbaToBgraReadback { rx, submission, mapped: true, - buffer: target.buffer.clone(), - padded_bytes_per_row: target.padded_bytes_per_row, + buffer: readback_buffer, + padded_bytes_per_row, width, height, frame_number, From 695a37ac1744a693b26c601432c64617d137cf96 Mon Sep 17 00:00:00 2001 From: Aatman Jain Date: Fri, 30 Oct 2026 00:18:32 +0000 Subject: [PATCH 3/9] fix(rendering): restore the bounded map-callback wait for readbacks The PR deleted mapped_result but PendingReadback::map still called it, so cap-rendering did not compile. Restore it, its test, and its use in the NV12 readback so a callback another renderer is about to run on a shared device is still waited for. --- crates/rendering/src/frame_pipeline.rs | 77 ++++++++++++++++++++++---- 1 file changed, 66 insertions(+), 11 deletions(-) diff --git a/crates/rendering/src/frame_pipeline.rs b/crates/rendering/src/frame_pipeline.rs index 8d61634005e..ae6f9c5ab14 100644 --- a/crates/rendering/src/frame_pipeline.rs +++ b/crates/rendering/src/frame_pipeline.rs @@ -106,6 +106,25 @@ async fn wait_for_submission( } } +/// The result of a `map_async` whose submission has finished. wgpu runs map +/// callbacks on whichever thread's poll collected them, so when several +/// renderers share a device, another thread can still be about to run this +/// buffer's callback after this thread's own wait returned. +async fn mapped_result( + rx: &mut oneshot::Receiver>, + started: Instant, + frame_number: u32, + timeout: std::time::Duration, +) -> Option> { + match rx.try_recv() { + Ok(result) => return Some(result), + Err(oneshot::error::TryRecvError::Closed) => return None, + Err(oneshot::error::TryRecvError::Empty) => {} + } + let remaining = timeout.checked_sub(readback_waited(started, frame_number))?; + tokio::time::timeout(remaining, rx).await.ok()?.ok() +} + #[derive(Clone, Copy, Debug, Default)] pub struct FinishEncoderTimings { pub wait_previous_duration: std::time::Duration, @@ -1147,17 +1166,18 @@ impl PendingNv12Readback { return Err(self.cancel()); }; - let completed = wait_for_submission( - device, - &submission, - Instant::now(), - self.frame_number, - gpu_buffer_wait_timeout(self.export_deadlines), - ) - .await; - match (completed, rx.try_recv()) { - (true, Ok(Ok(()))) => {} - (_, Ok(Err(error))) => { + let started = Instant::now(); + let timeout = gpu_buffer_wait_timeout(self.export_deadlines); + let completed = + wait_for_submission(device, &submission, started, self.frame_number, timeout).await; + let mapped = if completed { + mapped_result(&mut rx, started, self.frame_number, timeout).await + } else { + rx.try_recv().ok() + }; + match (completed, mapped) { + (true, Some(Ok(()))) => {} + (_, Some(Err(error))) => { self.unmap(); return Err(RenderingError::BufferMapFailed(error)); } @@ -2783,6 +2803,41 @@ pub async fn flush_pending_readback( mod readback_output_tests { use super::*; + #[tokio::test] + async fn map_callback_run_by_another_thread_after_the_wait_still_counts() { + let timeout = std::time::Duration::from_secs(5); + let (tx, mut rx) = oneshot::channel(); + let late = std::thread::spawn(move || { + std::thread::sleep(std::time::Duration::from_millis(50)); + tx.send(Ok(())).expect("map receiver"); + }); + assert!(matches!( + mapped_result(&mut rx, Instant::now(), 0, timeout).await, + Some(Ok(())) + )); + late.join().unwrap(); + + let (tx, mut rx) = oneshot::channel::>(); + drop(tx); + assert!( + mapped_result(&mut rx, Instant::now(), 0, timeout) + .await + .is_none() + ); + + let (_tx, mut rx) = oneshot::channel::>(); + assert!( + mapped_result( + &mut rx, + Instant::now(), + 0, + std::time::Duration::from_millis(20) + ) + .await + .is_none() + ); + } + #[test] fn readback_allocation_reserves_frame_and_metadata_without_changing_length() { let mut data = Vec::new(); From efd9b58f14b0d5eefff71136c3340843e1dffe83 Mon Sep 17 00:00:00 2001 From: Aatman Jain Date: Fri, 30 Oct 2026 00:25:14 +0000 Subject: [PATCH 4/9] test(rendering): cover BGRA readbacks that overlap a still-mapped buffer Pipelined submits must alternate buffers: with a single slot wgpu rejects the second copy with 'Buffer is still mapped'. The new tests drive eight pipelined frames and a discarded pending readback through the two-slot ring and compare every pixel. --- crates/rendering/src/frame_pipeline.rs | 161 +++++++++++++++++++++++++ 1 file changed, 161 insertions(+) diff --git a/crates/rendering/src/frame_pipeline.rs b/crates/rendering/src/frame_pipeline.rs index ae6f9c5ab14..db83a747150 100644 --- a/crates/rendering/src/frame_pipeline.rs +++ b/crates/rendering/src/frame_pipeline.rs @@ -3027,6 +3027,167 @@ mod recycled_readback_tests { } } +#[cfg(test)] +mod bgra_readback_tests { + use super::*; + + const WIDTH: u32 = 37; + const HEIGHT: u32 = 5; + + fn pixel(frame: u32, x: u32, y: u32) -> [u8; 4] { + [ + ((x * 7 + y * 3 + frame * 29) % 256) as u8, + ((x * 5 + y * 11 + frame * 13) % 256) as u8, + ((x * 3 + y * 17 + frame * 41) % 256) as u8, + 255, + ] + } + + fn write_source(queue: &wgpu::Queue, source: &wgpu::Texture, frame: u32) { + let mut data = Vec::new(); + for y in 0..HEIGHT { + for x in 0..WIDTH { + data.extend_from_slice(&pixel(frame, x, y)); + } + } + queue.write_texture( + source.as_image_copy(), + &data, + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(WIDTH * 4), + rows_per_image: None, + }, + wgpu::Extent3d { + width: WIDTH, + height: HEIGHT, + depth_or_array_layers: 1, + }, + ); + } + + fn assert_frame(frame: &RenderedFrame, number: u32) { + assert_eq!(frame.frame_number, number); + assert_eq!(frame.padded_bytes_per_row, WIDTH * 4); + assert_eq!(frame.data.len(), (WIDTH * HEIGHT * 4) as usize); + for y in 0..HEIGHT { + for x in 0..WIDTH { + let [r, g, b, a] = pixel(number, x, y); + let offset = ((y * WIDTH + x) * 4) as usize; + assert_eq!( + &frame.data[offset..offset + 4], + &[b, g, r, a], + "frame {number} pixel ({x}, {y})" + ); + } + } + } + + async fn device() -> Option<(wgpu::Device, wgpu::Queue)> { + let instance = crate::create_wgpu_instance_sync(); + let adapter = instance + .request_adapter(&wgpu::RequestAdapterOptions::default()) + .await + .ok()?; + if !RgbaToBgraReadback::supported(&adapter) { + return None; + } + adapter + .request_device(&wgpu::DeviceDescriptor::default()) + .await + .ok() + } + + async fn pipelined_step( + device: &wgpu::Device, + queue: &wgpu::Queue, + readback: &mut RgbaToBgraReadback, + source: &wgpu::Texture, + frame: u32, + ) -> Option { + write_source(queue, source, frame); + let previous = readback.take_pending(); + let encoder = device.create_command_encoder(&Default::default()); + let pending = readback + .submit(device, queue, encoder, source, frame, 30) + .expect("submit readback"); + readback.pending = Some(pending); + match previous { + Some(previous) => Some(previous.wait(device).await.expect("previous readback")), + None => None, + } + } + + fn source_texture(device: &wgpu::Device) -> wgpu::Texture { + device.create_texture(&wgpu::TextureDescriptor { + label: Some("bgra readback source"), + size: wgpu::Extent3d { + width: WIDTH, + height: HEIGHT, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba8Unorm, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }) + } + + #[tokio::test] + async fn pipelined_frames_never_reuse_a_buffer_that_is_still_mapped() { + let Some((device, queue)) = device().await else { + eprintln!("no BGRA-capable graphics adapter available, skipping"); + return; + }; + let source = source_texture(&device); + let mut readback = RgbaToBgraReadback::new(&device); + + assert!( + pipelined_step(&device, &queue, &mut readback, &source, 0) + .await + .is_none() + ); + for frame in 1..=8 { + let delivered = pipelined_step(&device, &queue, &mut readback, &source, frame) + .await + .expect("the previous frame is delivered"); + assert_frame(&delivered, frame - 1); + } + let last = flush_pending_bgra_readback(&mut readback, &device, &queue) + .await + .expect("the last frame is still pending") + .expect("last readback"); + assert_frame(&last, 8); + assert!(readback.take_pending().is_none()); + } + + #[tokio::test] + async fn a_discarded_pending_readback_leaves_the_ring_usable() { + let Some((device, queue)) = device().await else { + eprintln!("no BGRA-capable graphics adapter available, skipping"); + return; + }; + let source = source_texture(&device); + let mut readback = RgbaToBgraReadback::new(&device); + + pipelined_step(&device, &queue, &mut readback, &source, 0).await; + drop(readback.take_pending()); + assert!( + pipelined_step(&device, &queue, &mut readback, &source, 1) + .await + .is_none() + ); + for frame in 2..=5 { + let delivered = pipelined_step(&device, &queue, &mut readback, &source, frame) + .await + .expect("the previous frame is delivered"); + assert_frame(&delivered, frame - 1); + } + } +} + #[cfg(all(test, target_os = "macos"))] mod surface_output_tests { use super::*; From 604c69fe207e2a50bf110e89430721c5eb236b89 Mon Sep 17 00:00:00 2001 From: Aatman Jain Date: Fri, 30 Oct 2026 06:33:03 +0000 Subject: [PATCH 5/9] fix(editor): keep pipelined preview frames paired with their layout and drain the last one - pair each delivered frame with the layout it was rendered with instead of the current request's - stop discarding the pending frame when a newer request replaces it, which starved the preview under sustained backlog - discard the pending frame before seeks, paused redraws and thumbnails so they get their own frame - deliver the pending frame when playback ends - keep BGRA transition renders pipelined like RGBA ones and add an immediate variant - cover the new Bgra output in the imported-music benchmark --- .../imported-music-playback-benchmark.rs | 4 +- crates/editor/src/editor.rs | 209 +++++++++++++++--- crates/editor/src/playback.rs | 1 + .../src/preparing_preview/native_tests.rs | 190 ++++++++++++++++ crates/rendering/src/lib.rs | 33 ++- 5 files changed, 399 insertions(+), 38 deletions(-) diff --git a/crates/editor/examples/imported-music-playback-benchmark.rs b/crates/editor/examples/imported-music-playback-benchmark.rs index da8a1250350..a93dbda8fb4 100644 --- a/crates/editor/examples/imported-music-playback-benchmark.rs +++ b/crates/editor/examples/imported-music-playback-benchmark.rs @@ -80,7 +80,9 @@ async fn main() { |_| {}, Box::new(move |frame, _| { let number = match frame { - EditorFrameOutput::Rgba(frame) => frame.frame_number, + EditorFrameOutput::Rgba(frame) | EditorFrameOutput::Bgra(frame) => { + frame.frame_number + } EditorFrameOutput::Nv12(frame) => frame.frame_number, #[cfg(target_os = "macos")] EditorFrameOutput::Surface(frame) => frame.frame_number, diff --git a/crates/editor/src/editor.rs b/crates/editor/src/editor.rs index 53e5618f7e2..498e5745dfd 100644 --- a/crates/editor/src/editor.rs +++ b/crates/editor/src/editor.rs @@ -6,8 +6,8 @@ use cap_project::{ClipTransitionType, CursorEvents, ProjectConfiguration}; use cap_rendering::SurfaceFrame; use cap_rendering::{ DecodedSegmentFrames, FrameLayout, FrameRenderStageTimings, FrameRenderer, Nv12RenderedFrame, - ProjectUniforms, RenderVideoConstants, RenderedFrame, RendererLayers, RgbaToBgraReadback, - TransitionRenderInput, + ProjectUniforms, RenderVideoConstants, RenderedFrame, RendererLayers, RenderingError, + RgbaToBgraReadback, TransitionRenderInput, }; use tokio::sync::{mpsc, oneshot}; @@ -40,6 +40,7 @@ pub enum RendererMessage { queued_at: Instant, realtime: bool, }, + FlushPending, Stop { finished: oneshot::Sender<()>, }, @@ -83,6 +84,41 @@ pub(crate) fn supported_frame_format( format } +impl EditorFrameFormat { + fn pipelines_readback(self) -> bool { + match self { + Self::Rgba | Self::Bgra => true, + #[cfg(target_os = "macos")] + Self::BgraSurface => false, + } + } +} + +struct HeldFrame { + frame_number: u32, + layout: FrameLayout, +} + +fn output_frame_number(output: &EditorFrameOutput) -> u32 { + match output { + EditorFrameOutput::Rgba(frame) | EditorFrameOutput::Bgra(frame) => frame.frame_number, + EditorFrameOutput::Nv12(frame) => frame.frame_number, + #[cfg(target_os = "macos")] + EditorFrameOutput::Surface(frame) => frame.frame_number, + } +} + +fn settled_frame(result: Option>) -> Option { + match result { + Some(Ok(frame)) => Some(frame), + Some(Err(error)) => { + tracing::warn!(%error, "Failed to read back the pending preview frame"); + None + } + None => None, + } +} + pub type RendererLayersReceiver = oneshot::Receiver; pub type EditorFrameCallback = Box; @@ -277,6 +313,7 @@ impl Renderer { } let mut pending_frame: Option = None; + let mut held: Option = None; loop { let frame_to_render = if let Some(pending) = pending_frame.take() { @@ -288,6 +325,7 @@ impl Renderer { continue; } Some(RendererMessage::RenderThumbnail { input, finished }) => { + Self::discard_pending(&mut frame_renderer, output_format, &mut held).await; let result = frame_renderer .render_immediate( input.segment_frames, @@ -337,6 +375,16 @@ impl Renderer { queued_at, realtime, }), + Some(RendererMessage::FlushPending) => { + Self::deliver_pending( + &mut frame_renderer, + output_format, + &mut held, + &mut frame_cb, + ) + .await; + continue; + } Some(RendererMessage::Stop { finished }) => { let _ = finished.send(()); return; @@ -350,10 +398,12 @@ impl Renderer { }; let mut drained_count = 0u32; + let mut flush_after = false; let queue_drain_start = Instant::now(); while let Ok(msg) = rx.try_recv() { match msg { RendererMessage::RenderThumbnail { input, finished } => { + Self::discard_pending(&mut frame_renderer, output_format, &mut held).await; let result = frame_renderer .render_immediate( input.segment_frames, @@ -429,6 +479,10 @@ impl Renderer { }; drained_count += 1; } + RendererMessage::FlushPending => { + flush_after = true; + break; + } RendererMessage::Stop { finished } => { let _ = current.finished.send(false); let _ = finished.send(()); @@ -443,15 +497,11 @@ impl Renderer { let queue_wait = current.queued_at.elapsed(); let drain_duration = queue_drain_start.elapsed(); let flush_start = Instant::now(); - let flush_pipelines = drained_count > 0 || !current.realtime; - if flush_pipelines { - let _ = frame_renderer.flush_pipeline().await; - let _ = frame_renderer.flush_pipeline_bgra().await; - } - let flush_duration = if flush_pipelines { - flush_start.elapsed() - } else { + let flush_duration = if current.realtime { std::time::Duration::ZERO + } else { + Self::discard_pending(&mut frame_renderer, output_format, &mut held).await; + flush_start.elapsed() }; // Playback frames composite with the newest finished camera mask; @@ -462,6 +512,7 @@ impl Renderer { let input_frame_number = current.input.uniforms().frame_number; let frame_layout = current.input.uniforms().frame_layout(); let realtime = current.realtime; + let leaves_pending = realtime && output_format.pipelines_readback(); let render_result = match (output_format, current.input) { (EditorFrameFormat::Rgba, PendingRenderInput::Single(input)) => { if realtime { @@ -615,16 +666,37 @@ impl Renderer { cursor: &incoming.cursor, render_display: true, }; - frame_renderer - .render_transition_bgra(outgoing, incoming, kind, progress, &mut layers) - .await - .map(|frame| { - Some(( - EditorFrameOutput::Bgra(frame), - PlaybackRenderOutputFormat::Bgra, - FrameRenderStageTimings::default(), - )) - }) + if realtime { + frame_renderer + .render_transition_bgra(outgoing, incoming, kind, progress, &mut layers) + .await + .map(|frame| { + frame.map(|frame| { + ( + EditorFrameOutput::Bgra(frame), + PlaybackRenderOutputFormat::Bgra, + FrameRenderStageTimings::default(), + ) + }) + }) + } else { + frame_renderer + .render_transition_immediate_bgra( + outgoing, + incoming, + kind, + progress, + &mut layers, + ) + .await + .map(|frame| { + Some(( + EditorFrameOutput::Bgra(frame), + PlaybackRenderOutputFormat::Bgra, + FrameRenderStageTimings::default(), + )) + }) + } } #[cfg(target_os = "macos")] (EditorFrameFormat::BgraSurface, PendingRenderInput::Single(input)) => { @@ -684,16 +756,20 @@ impl Renderer { match render_result { Ok(Some((frame, output_format, render_stage_timings))) => { let render_duration = render_start.elapsed(); - let frame_number = match &frame { - EditorFrameOutput::Rgba(frame) | EditorFrameOutput::Bgra(frame) => { - frame.frame_number - } - EditorFrameOutput::Nv12(frame) => frame.frame_number, - #[cfg(target_os = "macos")] - EditorFrameOutput::Surface(frame) => frame.frame_number, + let frame_number = output_frame_number(&frame); + let previous = if leaves_pending { + held.replace(HeldFrame { + frame_number: input_frame_number, + layout: frame_layout, + }) + } else { + held.take() }; + let layout = previous + .filter(|previous| previous.frame_number == frame_number) + .map_or(frame_layout, |previous| previous.layout); let callback_start = Instant::now(); - (frame_cb)(frame, frame_layout); + (frame_cb)(frame, layout); let callback_duration = callback_start.elapsed(); if let Some(telemetry) = &telemetry { telemetry.emit(PlaybackTelemetryEvent::RendererFrame { @@ -712,6 +788,12 @@ impl Renderer { let _ = current.finished.send(true); } Ok(None) => { + if leaves_pending { + held = Some(HeldFrame { + frame_number: input_frame_number, + layout: frame_layout, + }); + } let _ = current.finished.send(true); } Err(e) => { @@ -719,6 +801,51 @@ impl Renderer { let _ = current.finished.send(false); } } + + if flush_after { + Self::deliver_pending(&mut frame_renderer, output_format, &mut held, &mut frame_cb) + .await; + } + } + } + + async fn take_pending_output( + frame_renderer: &mut FrameRenderer<'_>, + output_format: EditorFrameFormat, + ) -> Option { + match output_format { + EditorFrameFormat::Rgba => { + settled_frame(frame_renderer.flush_pipeline().await).map(EditorFrameOutput::Rgba) + } + EditorFrameFormat::Bgra => settled_frame(frame_renderer.flush_pipeline_bgra().await) + .map(EditorFrameOutput::Bgra), + #[cfg(target_os = "macos")] + EditorFrameFormat::BgraSurface => None, + } + } + + async fn discard_pending( + frame_renderer: &mut FrameRenderer<'_>, + output_format: EditorFrameFormat, + held: &mut Option, + ) { + let _ = Self::take_pending_output(frame_renderer, output_format).await; + *held = None; + } + + async fn deliver_pending( + frame_renderer: &mut FrameRenderer<'_>, + output_format: EditorFrameFormat, + held: &mut Option, + frame_cb: &mut EditorFrameCallback, + ) { + let held = held.take(); + let Some(frame) = Self::take_pending_output(frame_renderer, output_format).await else { + return; + }; + let frame_number = output_frame_number(&frame); + if let Some(held) = held.filter(|held| held.frame_number == frame_number) { + (frame_cb)(frame, held.layout); } } @@ -836,6 +963,28 @@ impl RendererHandle { } } + #[cfg(test)] + pub(crate) async fn render_frame_realtime_confirmed( + &self, + segment_frames: DecodedSegmentFrames, + uniforms: ProjectUniforms, + cursor: Arc, + ) -> bool { + let (finished_tx, finished_rx) = oneshot::channel(); + let msg = RendererMessage::RenderFrame { + segment_frames, + uniforms, + finished: finished_tx, + cursor, + queued_at: Instant::now(), + realtime: true, + }; + if self.tx.send(msg).await.is_err() { + return false; + } + finished_rx.await.unwrap_or(false) + } + pub async fn render_frame_confirmed( &self, segment_frames: DecodedSegmentFrames, @@ -944,6 +1093,10 @@ impl RendererHandle { finished_rx.blocking_recv().unwrap_or(false) } + pub fn flush_pending_blocking(&self) { + let _ = self.tx.blocking_send(RendererMessage::FlushPending); + } + pub async fn stop(&self) { let (tx, rx) = oneshot::channel(); if self diff --git a/crates/editor/src/playback.rs b/crates/editor/src/playback.rs index 1e4d2efd150..9d034e2f6a2 100644 --- a/crates/editor/src/playback.rs +++ b/crates/editor/src/playback.rs @@ -2120,6 +2120,7 @@ impl Playback { drop(transitioned_audio); stop_tx.send(true).ok(); + self.renderer.flush_pending_blocking(); event_tx.send(PlaybackEvent::Stop).ok(); }; diff --git a/crates/editor/src/preparing_preview/native_tests.rs b/crates/editor/src/preparing_preview/native_tests.rs index 1daa8602c95..a22e89c2909 100644 --- a/crates/editor/src/preparing_preview/native_tests.rs +++ b/crates/editor/src/preparing_preview/native_tests.rs @@ -1079,6 +1079,196 @@ fn surface_rgba(frame: cap_rendering::SurfaceFrame) -> RenderedFrame { } } +async fn pipelined_input( + fixture: &Fixture, + decoders: &RecordingSegmentDecoders, + constants: &RenderVideoConstants, + frame_number: u32, + padding: f64, +) -> (cap_rendering::DecodedSegmentFrames, ProjectUniforms) { + let mut project = fixture.project.clone(); + project.background.padding = padding; + let project = Arc::new(project); + let time = f64::from(frame_number) / 30.0; + let (segment_time, segment) = project.get_segment_time(time).unwrap(); + let frames = decoders + .get_frames_initial( + segment_time as f32, + true, + true, + project.clips[segment.recording_clip as usize].offsets, + ) + .await + .unwrap(); + let duration = project.timeline.as_ref().unwrap().duration(); + let mut zoom = ZoomTransformTimeline::from_project_for_clip( + &project, + &fixture.cursor, + duration, + constants.options.screen_size, + segment.recording_clip, + ); + zoom.ensure_precomputed_until((frame_number as f32 + 1.0) / 30.0); + let uniforms = ProjectUniforms::new( + constants, + &project, + frame_number, + 30, + XY::new(320, 240), + &fixture.cursor, + &frames, + duration, + &zoom, + ); + (frames, uniforms) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn native_pipelined_preview_keeps_layouts_and_drains_the_last_frame() { + let fixture = fixture(false); + let metadata = &fixture.reference_metadata; + let studio = metadata.studio_meta().unwrap(); + let decoders = RecordingSegmentDecoders::new( + metadata, + studio, + SegmentVideoPaths { + display: metadata.project_path.join("display.mp4"), + camera: Some(metadata.project_path.join("camera.mp4")), + }, + 0, + true, + ) + .await + .unwrap(); + let constants = Arc::new( + RenderVideoConstants::new_with_options( + RenderOptions { + screen_size: XY::new(160, 120), + camera_size: Some(XY::new(80, 80)), + preserve_screen_alpha: false, + }, + metadata.clone(), + studio.clone(), + ) + .await + .unwrap(), + ); + + for format in [EditorFrameFormat::Rgba, EditorFrameFormat::Bgra] { + if crate::editor::supported_frame_format(format, &constants) != format { + continue; + } + let mut layers = RendererLayers::new_with_options( + &constants.device, + &constants.queue, + constants.is_software_adapter, + ); + layers.preload_cursor_assets( + &constants, + fixture.project.cursor.use_svg, + fixture.project.cursor.cursor_type(), + ); + let (layers_tx, layers_rx) = tokio::sync::oneshot::channel(); + layers_tx.send(layers).ok(); + let (delivered_tx, mut delivered) = tokio::sync::mpsc::unbounded_channel(); + let renderer = crate::Renderer::spawn_with_format( + constants.clone(), + Box::new(move |output, layout| { + let frame = match output { + EditorFrameOutput::Rgba(frame) | EditorFrameOutput::Bgra(frame) => frame, + _ => panic!("Unexpected output"), + }; + delivered_tx.send((frame.frame_number, layout)).ok(); + }), + layers_rx, + format, + ) + .unwrap(); + + let (first_frames, first_uniforms) = + pipelined_input(&fixture, &decoders, &constants, 0, 0.0).await; + let (second_frames, second_uniforms) = + pipelined_input(&fixture, &decoders, &constants, 45, 25.0).await; + let (third_frames, third_uniforms) = + pipelined_input(&fixture, &decoders, &constants, 12, 0.0).await; + let (seek_frames, seek_uniforms) = + pipelined_input(&fixture, &decoders, &constants, 90, 40.0).await; + let first_layout = first_uniforms.frame_layout(); + let second_layout = second_uniforms.frame_layout(); + let seek_layout = seek_uniforms.frame_layout(); + assert_ne!(first_layout, second_layout); + assert_ne!(second_layout, seek_layout); + + assert!( + renderer + .render_frame_realtime_confirmed( + first_frames, + first_uniforms, + fixture.cursor.clone() + ) + .await + ); + assert!( + delivered.try_recv().is_err(), + "{format:?}: the first frame stays pending" + ); + assert!( + renderer + .render_frame_realtime_confirmed( + second_frames, + second_uniforms, + fixture.cursor.clone() + ) + .await + ); + let (frame_number, layout) = + tokio::time::timeout(Duration::from_secs(15), delivered.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!(frame_number, 0, "{format:?}"); + assert_eq!(layout, first_layout, "{format:?}: pixels and layout agree"); + + tokio::task::block_in_place(|| renderer.flush_pending_blocking()); + let (frame_number, layout) = + tokio::time::timeout(Duration::from_secs(15), delivered.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!(frame_number, 45, "{format:?}: the last frame is drained"); + assert_eq!(layout, second_layout, "{format:?}"); + + assert!( + renderer + .render_frame_realtime_confirmed( + third_frames, + third_uniforms, + fixture.cursor.clone() + ) + .await + ); + assert!( + renderer + .render_frame_confirmed(seek_frames, seek_uniforms, fixture.cursor.clone()) + .await + ); + let (frame_number, layout) = + tokio::time::timeout(Duration::from_secs(15), delivered.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!(frame_number, 90, "{format:?}: a seek shows its own frame"); + assert_eq!(layout, seek_layout, "{format:?}"); + assert!( + tokio::time::timeout(Duration::from_millis(300), delivered.recv()) + .await + .is_err(), + "{format:?}: the playback frame a seek replaced is not shown" + ); + renderer.stop().await; + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn native_preparing_bgra_readback_matches_default_editor_frame() { composed_parity_with_reference( diff --git a/crates/rendering/src/lib.rs b/crates/rendering/src/lib.rs index e7cdb6e8eac..786cbd06dd1 100644 --- a/crates/rendering/src/lib.rs +++ b/crates/rendering/src/lib.rs @@ -7774,7 +7774,7 @@ impl<'a> FrameRenderer<'a> { Ok((frame, timings)) } - /// [`Self::render_transition_immediate`], read back as tightly packed BGRA. + /// [`Self::render_transition`], read back as tightly packed BGRA. pub async fn render_transition_bgra( &mut self, outgoing: TransitionRenderInput<'_>, @@ -7782,7 +7782,7 @@ impl<'a> FrameRenderer<'a> { kind: ClipTransitionType, progress: f32, layers: &mut RendererLayers, - ) -> Result { + ) -> Result, RenderingError> { let mut last_error = None; for attempt in 0..Self::MAX_RENDER_RETRIES { @@ -7844,13 +7844,7 @@ impl<'a> FrameRenderer<'a> { ) .await { - Ok(Some(frame)) => return Ok(frame), - Ok(None) => { - if let Some(frame) = self.flush_pipeline_bgra().await { - return frame; - } - continue; - } + Ok(frame) => return Ok(frame), Err(RenderingError::BufferMapWaitingFailed) => { last_error = Some(RenderingError::BufferMapWaitingFailed); } @@ -7864,6 +7858,27 @@ impl<'a> FrameRenderer<'a> { Err(last_error.unwrap_or(RenderingError::BufferMapWaitingFailed)) } + /// [`Self::render_transition_immediate`], read back as tightly packed BGRA. + pub async fn render_transition_immediate_bgra( + &mut self, + outgoing: TransitionRenderInput<'_>, + incoming: TransitionRenderInput<'_>, + kind: ClipTransitionType, + progress: f32, + layers: &mut RendererLayers, + ) -> Result { + if let Some(frame) = self + .render_transition_bgra(outgoing, incoming, kind, progress, layers) + .await? + { + return Ok(frame); + } + + self.flush_pipeline_bgra() + .await + .unwrap_or(Err(RenderingError::BufferMapWaitingFailed)) + } + pub async fn flush_pipeline(&mut self) -> Option> { if let Some(session) = &mut self.session { flush_pending_readback(session, &self.constants.device).await From 538b58cc729485bd7dbed579b44970d6c8867877 Mon Sep 17 00:00:00 2001 From: Aatman Jain Date: Fri, 30 Oct 2026 10:28:08 +0000 Subject: [PATCH 6/9] fix(desktop-gpui): expect no picker backdrop where the overlay skips it --- apps/desktop-gpui/src/picker_probe.rs | 10 +++++++++- apps/desktop-gpui/src/target_overlay.rs | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/apps/desktop-gpui/src/picker_probe.rs b/apps/desktop-gpui/src/picker_probe.rs index fcc7e77285f..5dbcaeb5d23 100644 --- a/apps/desktop-gpui/src/picker_probe.rs +++ b/apps/desktop-gpui/src/picker_probe.rs @@ -106,7 +106,7 @@ pub fn run(main: WindowHandle, cx: &mut App) { let mut mode = cx.update(|cx| probe_mode(label, cx)); mode["overlayInactiveBeforeActivation"] = inactive_before.into(); let screen = screenshot(&output, label, cx).await; - mode["backdrop"] = probe_backdrop(&output, label, screen.as_ref(), cx).await; + mode["backdrop"] = probe_backdrop(&output, label, kind, screen.as_ref(), cx).await; modes.push(mode); if kind == TargetType::Window { window_arrangement = Some(probe_window_arrangement(&output, cx).await); @@ -372,6 +372,7 @@ fn mean_difference( async fn probe_backdrop( output: &std::ffi::OsStr, label: &str, + kind: TargetType, screen: Option<&image::DynamicImage>, cx: &mut AsyncApp, ) -> serde_json::Value { @@ -393,6 +394,13 @@ async fn probe_backdrop( .ok() .flatten() }); + if !crate::target_overlay::backdrop_suits(Some(kind)) { + return serde_json::json!({ + "skipped": "the picker takes no backdrop in this mode here", + "present": image.is_some(), + "passed": image.is_none(), + }); + } let Some(image) = image else { return serde_json::json!({ "present": false, "passed": false }); }; diff --git a/apps/desktop-gpui/src/target_overlay.rs b/apps/desktop-gpui/src/target_overlay.rs index 28cc9b5b91f..74165eab287 100644 --- a/apps/desktop-gpui/src/target_overlay.rs +++ b/apps/desktop-gpui/src/target_overlay.rs @@ -830,7 +830,7 @@ fn more_area_ratios() -> Vec<(u32, u32)> { /// so a capture under their panes showed through it as hard-edged blocks; /// there the panes sit on the fallback fill, as they do where no capture /// is possible. -fn backdrop_suits(mode: Option) -> bool { +pub(crate) fn backdrop_suits(mode: Option) -> bool { match mode { None => false, Some(TargetType::Display | TargetType::Window) => !cfg!(windows), From f97cef59c26b9c8b94084b3b9c8126c20a0e6234 Mon Sep 17 00:00:00 2001 From: Aatman Jain Date: Fri, 30 Oct 2026 10:23:07 +0000 Subject: [PATCH 7/9] fix(ci): configure Hyprland 0.57 through hyprland.lua in the Linux harness Hyprland 0.57 dropped hyprland.conf and only reads hyprland.lua, so the harness's monitor mode, session export and waybar were ignored and the output came up at vkms's preferred 1024x768. The screenshot scenario then reported screenshot_size=1024x768 instead of yes. Write the same settings as Lua and use the Lua form of the focus dispatcher. --- scripts/linux-desktop-harness/arch-up.sh | 17 ++++++++++------- .../scenario-screenshot.sh | 2 +- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/scripts/linux-desktop-harness/arch-up.sh b/scripts/linux-desktop-harness/arch-up.sh index 6a1a8028862..f3a883510d6 100755 --- a/scripts/linux-desktop-harness/arch-up.sh +++ b/scripts/linux-desktop-harness/arch-up.sh @@ -23,13 +23,16 @@ common=(LIBSEAT_BACKEND=seatd XDG_SESSION_TYPE=wayland) case "$desktop" in hyprland) - cat >"$HOME/.config/hypr/hyprland.conf" <"$HOME/.config/hypr/hyprland.lua" <"$HOME/.config/hypr/xdph.conf" < /^(Cap|so\.cap\.Cap)$/.test(w.get_wm_class() || '') && w.showing_on_its_workspace()).forEach(w => w.activate(global.display.get_current_time_roundtrip()))" \ >/dev/null 2>&1 elif [[ -n ${HYPRLAND_INSTANCE_SIGNATURE:-} ]]; then - hyprctl dispatch focuswindow class:Cap >/dev/null 2>&1 + hyprctl dispatch "hl.dsp.focus({ window = 'class:Cap' })" >/dev/null 2>&1 fi } for _ in $(seq 60); do From 8ef6d571eff9d794ab37f4f7fe58f207a50bf47d Mon Sep 17 00:00:00 2001 From: Aatman Jain Date: Fri, 30 Oct 2026 10:38:26 +0000 Subject: [PATCH 8/9] fix(verify): encode the 320x180 compositing codec case at the Web preset bitrate At 80 kbps the paravirtualised VideoToolbox HEVC encoder on the CI macOS VMs occasionally left a blurred camera code cell at margin 0.17, below the reader's 0.18 threshold. 138,240 bps is the Web export preset (0.08 bpp), still below the Social preset the verify exports use. --- crates/verify/tests/codec.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/verify/tests/codec.rs b/crates/verify/tests/codec.rs index d2efa9de025..bbff4a6afe8 100644 --- a/crates/verify/tests/codec.rs +++ b/crates/verify/tests/codec.rs @@ -215,7 +215,11 @@ fn video_code_survives_compositing_scaling_rounding_bubble_and_blur() { let dir = tempfile::tempdir().unwrap(); // 640x360 is a typical verification output; 320x180 puts the screen at // a quarter of its source size and the camera code cells at ~6 px. - for (out_w, out_h, bitrate) in [(640, 360, 150_000), (320, 180, 80_000)] { + // 320x180 is encoded at the Web export preset (0.08 bpp): at 80 kbps the + // paravirtualised VideoToolbox HEVC encoder on the CI macOS VMs + // occasionally left a blurred camera cell at margin 0.17, below the + // reader's 0.18, while this host's hardware encoder read it at 0.31. + for (out_w, out_h, bitrate) in [(640, 360, 150_000), (320, 180, 138_240)] { for blur in [false, true] { let pairs: Vec<(u32, u32)> = (0..40).map(|i| (i * 3 + 100, i / 2 + 7)).collect(); let frames: Vec = pairs From 247ed79ae38cda5a23dd3156057fcaacedb481f7 Mon Sep 17 00:00:00 2001 From: Aatman Jain Date: Fri, 30 Oct 2026 11:02:49 +0000 Subject: [PATCH 9/9] fix(ci): walk GIF blocks when measuring the exported GIF's length --- .../linux-desktop-harness/scenario-export.sh | 30 +++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/scripts/linux-desktop-harness/scenario-export.sh b/scripts/linux-desktop-harness/scenario-export.sh index d73efcf77af..ec76b57331e 100755 --- a/scripts/linux-desktop-harness/scenario-export.sh +++ b/scripts/linux-desktop-harness/scenario-export.sh @@ -60,12 +60,32 @@ gif_frames_and_seconds() { import sys data = open(sys.argv[1], "rb").read() + + +def skip_sub_blocks(at): + while data[at]: + at += data[at] + 1 + return at + 1 + + if data[:6] in (b"GIF87a", b"GIF89a"): - frames, centiseconds, at = 0, 0, data.find(b"\x21\xf9\x04") - while at != -1 and at + 6 <= len(data): - frames += 1 - centiseconds += data[at + 4] | data[at + 5] << 8 - at = data.find(b"\x21\xf9\x04", at + 8) + frames, centiseconds, delay = 0, 0, 0 + flags = data[10] + at = 13 + (3 << ((flags & 7) + 1) if flags & 0x80 else 0) + while at < len(data) and data[at] != 0x3B: + if data[at] == 0x21: + if data[at + 1] == 0xF9: + delay = data[at + 4] | data[at + 5] << 8 + at = skip_sub_blocks(at + 2) + elif data[at] == 0x2C: + frames += 1 + centiseconds += delay + delay = 0 + local = data[at + 9] + at += 10 + (3 << ((local & 7) + 1) if local & 0x80 else 0) + at = skip_sub_blocks(at + 1) + else: + break print(frames, f"{centiseconds / 100:.2f}") PY }