diff --git a/apps/desktop-gpui/examples/live-edit-benchmark.rs b/apps/desktop-gpui/examples/live-edit-benchmark.rs index 134dc309982..9913acf34c4 100644 --- a/apps/desktop-gpui/examples/live-edit-benchmark.rs +++ b/apps/desktop-gpui/examples/live-edit-benchmark.rs @@ -55,7 +55,7 @@ fn percentile(sorted: &[f64], fraction: f64) -> f64 { fn frame_number(output: &EditorFrameOutput) -> u32 { match output { - EditorFrameOutput::Rgba(frame) => frame.frame_number, + EditorFrameOutput::Rgba(frame) | EditorFrameOutput::Bgra(frame) => frame.frame_number, EditorFrameOutput::Nv12(frame) => frame.frame_number, #[cfg(target_os = "macos")] EditorFrameOutput::Surface(frame) => frame.frame_number, diff --git a/apps/desktop-gpui/examples/scrub-benchmark.rs b/apps/desktop-gpui/examples/scrub-benchmark.rs index 1f3f32c1d6c..72dd5d59045 100644 --- a/apps/desktop-gpui/examples/scrub-benchmark.rs +++ b/apps/desktop-gpui/examples/scrub-benchmark.rs @@ -61,7 +61,7 @@ fn percentile(sorted: &[f64], fraction: f64) -> f64 { fn frame_number(output: &EditorFrameOutput) -> u32 { match output { - EditorFrameOutput::Rgba(frame) => frame.frame_number, + EditorFrameOutput::Rgba(frame) | EditorFrameOutput::Bgra(frame) => frame.frame_number, EditorFrameOutput::Nv12(frame) => frame.frame_number, #[cfg(target_os = "macos")] EditorFrameOutput::Surface(frame) => frame.frame_number, diff --git a/apps/desktop-gpui/src/app_windows.rs b/apps/desktop-gpui/src/app_windows.rs index 885f0cb7b05..8bc0d55e5a2 100644 --- a/apps/desktop-gpui/src/app_windows.rs +++ b/apps/desktop-gpui/src/app_windows.rs @@ -5442,6 +5442,14 @@ fn start_preparing_editor( let Some(image) = image else { continue }; editor_window::EditorPreviewFrame::Image(image) } + cap_editor::EditorFrameOutput::Bgra(frame) => { + let image = cx + .background_executor() + .spawn(async move { editor_window::bgra_frame_image(frame) }) + .await; + let Some(image) = image else { continue }; + editor_window::EditorPreviewFrame::Image(image) + } #[cfg(target_os = "macos")] cap_editor::EditorFrameOutput::Surface(surface) => { editor_window::surface_preview_frame(surface) @@ -5658,16 +5666,17 @@ fn load_editor_project_candidate( // let state_cb = editor_window::make_state_callback(playhead); let frame_cb = editor_window::make_frame_callback(frame_tx, frame_stats); + let shared_device = shared_wgpu_device().await; #[cfg(target_os = "macos")] let frame_format = cap_editor::EditorFrameFormat::BgraSurface; #[cfg(not(target_os = "macos"))] - let frame_format = cap_editor::EditorFrameFormat::Rgba; + let frame_format = + editor_window::cpu_preview_frame_format(shared_device.as_ref()); if let Some(previous) = previous { return previous .recreate_preparing_candidate(state_cb, frame_cb, frame_format) .await; } - let shared_device = shared_wgpu_device().await; let instance = cap_editor::EditorInstance::new_with_startup_inputs( instance_path, state_cb, @@ -5742,30 +5751,71 @@ fn load_editor_project_candidate( return; } - // The pump: convert on the background executor (un-padding plus the - // BGRA swap is a few megabytes per frame), deliver on the main one. cx.spawn({ let stats = stats.clone(); let pump_instance = Arc::downgrade(&instance); + let rgba_stream = editor_window::streaming_preview_enabled() + .then(|| gpui::StreamingImage::new(gpui::StreamingImageFormat::Rgba8)); + let bgra_stream = editor_window::streaming_preview_enabled() + .then(|| gpui::StreamingImage::new(gpui::StreamingImageFormat::Bgra8)); async move |cx| { while let Ok((output, layout)) = frame_rx.recv_async().await { + let mut streamed_rgba = None; + let mut streamed_bgra = None; let (frame, number) = match output { cap_editor::EditorFrameOutput::Rgba(rgba) => { - let stats = stats.clone(); let number = rgba.frame_number; - let image = cx - .background_executor() - .spawn( - async move { editor_window::frame_image_timed(&rgba, &stats) }, + if rgba_stream.is_some() { + streamed_rgba = Some(rgba); + ( + editor_window::EditorPreviewFrame::Stream( + rgba_stream.as_ref().expect("stream").clone(), + ), + number, ) - .await; - let Some(image) = image else { - tracing::warn!( - "a rendered frame could not be converted for display" - ); - continue; - }; - (editor_window::EditorPreviewFrame::Image(image), number) + } else { + let stats = stats.clone(); + let image = cx + .background_executor() + .spawn(async move { + editor_window::frame_image_timed(&rgba, &stats) + }) + .await; + let Some(image) = image else { + tracing::warn!( + "a rendered frame could not be converted for display" + ); + continue; + }; + (editor_window::EditorPreviewFrame::Image(image), number) + } + } + cap_editor::EditorFrameOutput::Bgra(bgra) => { + let number = bgra.frame_number; + if bgra_stream.is_some() { + streamed_bgra = Some(bgra); + ( + editor_window::EditorPreviewFrame::Stream( + bgra_stream.as_ref().expect("stream").clone(), + ), + number, + ) + } else { + let stats = stats.clone(); + let image = cx + .background_executor() + .spawn(async move { + editor_window::bgra_frame_image_timed(bgra, &stats) + }) + .await; + let Some(image) = image else { + tracing::warn!( + "a rendered frame could not be converted for display" + ); + continue; + }; + (editor_window::EditorPreviewFrame::Image(image), number) + } } cap_editor::EditorFrameOutput::Nv12(frame) => { tracing::warn!( @@ -5785,6 +5835,18 @@ fn load_editor_project_candidate( if !view.holds_instance(&pump_instance) { return; } + if let (Some(stream), Some(rgba)) = + (&rgba_stream, streamed_rgba.take()) + && !editor_window::stream_frame_timed(stream, rgba, &stats) + { + return; + } + if let (Some(stream), Some(bgra)) = + (&bgra_stream, streamed_bgra.take()) + && !editor_window::stream_frame_timed(stream, bgra, &stats) + { + return; + } view.frame_arrived( editor_window::EditorFrame { frame, diff --git a/apps/desktop-gpui/src/editor_preparing.rs b/apps/desktop-gpui/src/editor_preparing.rs index 92f6a5faf7a..55a2f2748cc 100644 --- a/apps/desktop-gpui/src/editor_preparing.rs +++ b/apps/desktop-gpui/src/editor_preparing.rs @@ -525,9 +525,10 @@ impl PreparingRunner { let discard = self.frame_discard.clone(); #[cfg(target_os = "macos")] let frame_format = EditorFrameFormat::BgraSurface; - #[cfg(not(target_os = "macos"))] - let frame_format = EditorFrameFormat::Rgba; let shared_device = crate::app_windows::shared_wgpu_device().await; + #[cfg(not(target_os = "macos"))] + let frame_format: EditorFrameFormat = + crate::editor_window::cpu_preview_frame_format(shared_device.as_ref()); self.control.accepts_frames.store(true, Ordering::Release); let playback = PreparingPlaybackSession::spawn_with_expected_metadata( input, diff --git a/apps/desktop-gpui/src/editor_window.rs b/apps/desktop-gpui/src/editor_window.rs index 64edb658fe8..516ae412511 100644 --- a/apps/desktop-gpui/src/editor_window.rs +++ b/apps/desktop-gpui/src/editor_window.rs @@ -63,8 +63,8 @@ use gpui::{ Animation, AnimationExt as _, AppContext as _, Bounds, Context, Entity, FocusHandle, FontWeight, Hsla, InteractiveElement, IntoElement, MouseButton, MouseDownEvent, MouseMoveEvent, MouseUpEvent, ParentElement, Pixels, Point, Render, RenderImage, SharedString, - StatefulInteractiveElement as _, StyleRefinement, Styled, Subscription, WeakEntity, Window, - div, point, prelude::FluentBuilder, px, svg, + StatefulInteractiveElement as _, StreamingImage, StyleRefinement, Styled, Subscription, + WeakEntity, Window, div, point, prelude::FluentBuilder, px, svg, }; use crate::i18n::tr; @@ -593,10 +593,71 @@ pub fn frame_image(frame: &RenderedFrame) -> Option> { ]))) } +pub fn bgra_frame_image(frame: RenderedFrame) -> Option> { + let row_bytes = (frame.width as usize).checked_mul(4)?; + if frame.padded_bytes_per_row as usize != row_bytes + || frame.data.len() != row_bytes.checked_mul(frame.height as usize)? + { + return None; + } + let pixels = Arc::try_unwrap(frame.data).unwrap_or_else(|shared| shared.as_ref().clone()); + let buffer = image::RgbaImage::from_raw(frame.width, frame.height, pixels)?; + Some(Arc::new(RenderImage::new(smallvec::smallvec![ + image::Frame::new(buffer) + ]))) +} + +#[cfg(not(target_os = "macos"))] +pub(crate) fn cpu_preview_frame_format( + shared_device: Option<&cap_rendering::SharedWgpuDevice>, +) -> cap_editor::EditorFrameFormat { + if shared_device.is_some_and(|device| !device.is_software_adapter) { + cap_editor::EditorFrameFormat::Bgra + } else { + cap_editor::EditorFrameFormat::Rgba + } +} + +pub(crate) fn streaming_preview_enabled() -> bool { + static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + *ENABLED.get_or_init(|| { + match std::env::var_os("CAP_GPUI_STREAMING_IMAGE").as_deref() { + Some(value) => value != "0", + None => !cfg!(target_os = "macos"), + } + }) +} + +pub fn stream_frame_timed( + stream: &StreamingImage, + frame: RenderedFrame, + stats: &PumpStats, +) -> bool { + let started = Instant::now(); + let size = gpui::size( + gpui::DevicePixels(frame.width as i32), + gpui::DevicePixels(frame.height as i32), + ); + let stride = frame.padded_bytes_per_row as usize; + let result = match Arc::try_unwrap(frame.data) { + Ok(data) => stream.set_frame(size, stride, data).map(drop), + Err(data) => stream.copy_frame(size, stride, &data), + }; + stats + .convert_nanos + .fetch_add(started.elapsed().as_nanos() as u64, Ordering::Relaxed); + stats.convert_samples.fetch_add(1, Ordering::Relaxed); + if let Err(error) = &result { + tracing::warn!(%error, "a rendered frame could not be streamed to the preview"); + } + result.is_ok() +} + /// What the frame pump hands the window. #[derive(Clone)] pub(crate) enum EditorPreviewFrame { Image(Arc), + Stream(StreamingImage), #[cfg(target_os = "macos")] Surface(CVPixelBuffer), } @@ -608,6 +669,9 @@ impl EditorPreviewFrame { Self::Image(image) => { let _ = window.paint_image(bounds, corners, Arc::clone(image), 0, false); } + Self::Stream(image) => { + let _ = window.paint_streaming_image(bounds, corners, image, false); + } #[cfg(target_os = "macos")] Self::Surface(surface) => { window.paint_surface_fitted(bounds, bounds, corners, surface.clone()) @@ -12520,8 +12584,19 @@ pub(crate) fn surface_preview_frame(frame: cap_rendering::SurfaceFrame) -> Edito /// [`frame_image`], timed. The pump calls this so the CPU conversion's cost is /// a number rather than a hunch. pub fn frame_image_timed(frame: &RenderedFrame, stats: &PumpStats) -> Option> { + timed_conversion(stats, || frame_image(frame)) +} + +pub fn bgra_frame_image_timed(frame: RenderedFrame, stats: &PumpStats) -> Option> { + timed_conversion(stats, || bgra_frame_image(frame)) +} + +fn timed_conversion( + stats: &PumpStats, + convert: impl FnOnce() -> Option>, +) -> Option> { let started = Instant::now(); - let image = frame_image(frame); + let image = convert(); stats .convert_nanos .fetch_add(started.elapsed().as_nanos() as u64, Ordering::Relaxed); diff --git a/apps/desktop-gpui/src/picker_probe.rs b/apps/desktop-gpui/src/picker_probe.rs index fcc7e77285f..5dbcaeb5d23 100644 --- a/apps/desktop-gpui/src/picker_probe.rs +++ b/apps/desktop-gpui/src/picker_probe.rs @@ -106,7 +106,7 @@ pub fn run(main: WindowHandle, cx: &mut App) { let mut mode = cx.update(|cx| probe_mode(label, cx)); mode["overlayInactiveBeforeActivation"] = inactive_before.into(); let screen = screenshot(&output, label, cx).await; - mode["backdrop"] = probe_backdrop(&output, label, screen.as_ref(), cx).await; + mode["backdrop"] = probe_backdrop(&output, label, kind, screen.as_ref(), cx).await; modes.push(mode); if kind == TargetType::Window { window_arrangement = Some(probe_window_arrangement(&output, cx).await); @@ -372,6 +372,7 @@ fn mean_difference( async fn probe_backdrop( output: &std::ffi::OsStr, label: &str, + kind: TargetType, screen: Option<&image::DynamicImage>, cx: &mut AsyncApp, ) -> serde_json::Value { @@ -393,6 +394,13 @@ async fn probe_backdrop( .ok() .flatten() }); + if !crate::target_overlay::backdrop_suits(Some(kind)) { + return serde_json::json!({ + "skipped": "the picker takes no backdrop in this mode here", + "present": image.is_some(), + "passed": image.is_none(), + }); + } let Some(image) = image else { return serde_json::json!({ "present": false, "passed": false }); }; diff --git a/apps/desktop-gpui/src/target_overlay.rs b/apps/desktop-gpui/src/target_overlay.rs index 28cc9b5b91f..74165eab287 100644 --- a/apps/desktop-gpui/src/target_overlay.rs +++ b/apps/desktop-gpui/src/target_overlay.rs @@ -830,7 +830,7 @@ fn more_area_ratios() -> Vec<(u32, u32)> { /// so a capture under their panes showed through it as hard-edged blocks; /// there the panes sit on the fallback fill, as they do where no capture /// is possible. -fn backdrop_suits(mode: Option) -> bool { +pub(crate) fn backdrop_suits(mode: Option) -> bool { match mode { None => false, Some(TargetType::Display | TargetType::Window) => !cfg!(windows), diff --git a/crates/editor/examples/editor-playback-benchmark.rs b/crates/editor/examples/editor-playback-benchmark.rs index 0af9f92b602..ad1cf5c058b 100644 --- a/crates/editor/examples/editor-playback-benchmark.rs +++ b/crates/editor/examples/editor-playback-benchmark.rs @@ -571,6 +571,7 @@ async fn main() { data.len() + metadata_bytes } EditorFrameOutput::Rgba(frame) => frame.data.len() + 24, + EditorFrameOutput::Bgra(frame) => frame.data.len(), #[cfg(target_os = "macos")] EditorFrameOutput::Surface(_) => return, }; diff --git a/crates/editor/examples/imported-music-playback-benchmark.rs b/crates/editor/examples/imported-music-playback-benchmark.rs index da8a1250350..a93dbda8fb4 100644 --- a/crates/editor/examples/imported-music-playback-benchmark.rs +++ b/crates/editor/examples/imported-music-playback-benchmark.rs @@ -80,7 +80,9 @@ async fn main() { |_| {}, Box::new(move |frame, _| { let number = match frame { - EditorFrameOutput::Rgba(frame) => frame.frame_number, + EditorFrameOutput::Rgba(frame) | EditorFrameOutput::Bgra(frame) => { + frame.frame_number + } EditorFrameOutput::Nv12(frame) => frame.frame_number, #[cfg(target_os = "macos")] EditorFrameOutput::Surface(frame) => frame.frame_number, diff --git a/crates/editor/examples/scrub-benchmark.rs b/crates/editor/examples/scrub-benchmark.rs index e89a889945e..8429b5345e4 100644 --- a/crates/editor/examples/scrub-benchmark.rs +++ b/crates/editor/examples/scrub-benchmark.rs @@ -19,7 +19,7 @@ fn arg_value<'a>(args: &'a [String], flag: &str) -> Option<&'a str> { fn presented_frame(output: &EditorFrameOutput) -> u32 { match output { - EditorFrameOutput::Rgba(frame) => frame.frame_number, + EditorFrameOutput::Rgba(frame) | EditorFrameOutput::Bgra(frame) => frame.frame_number, EditorFrameOutput::Nv12(frame) => frame.frame_number, #[cfg(target_os = "macos")] EditorFrameOutput::Surface(frame) => frame.frame_number, diff --git a/crates/editor/src/editor.rs b/crates/editor/src/editor.rs index 160f8f52a84..498e5745dfd 100644 --- a/crates/editor/src/editor.rs +++ b/crates/editor/src/editor.rs @@ -6,7 +6,8 @@ use cap_project::{ClipTransitionType, CursorEvents, ProjectConfiguration}; use cap_rendering::SurfaceFrame; use cap_rendering::{ DecodedSegmentFrames, FrameLayout, FrameRenderStageTimings, FrameRenderer, Nv12RenderedFrame, - ProjectUniforms, RenderVideoConstants, RenderedFrame, RendererLayers, TransitionRenderInput, + ProjectUniforms, RenderVideoConstants, RenderedFrame, RendererLayers, RenderingError, + RgbaToBgraReadback, TransitionRenderInput, }; use tokio::sync::{mpsc, oneshot}; @@ -39,6 +40,7 @@ pub enum RendererMessage { queued_at: Instant, realtime: bool, }, + FlushPending, Stop { finished: oneshot::Sender<()>, }, @@ -52,6 +54,8 @@ pub struct RendererTransitionInput { pub enum EditorFrameOutput { Rgba(RenderedFrame), + /// Tightly packed BGRA: `padded_bytes_per_row == width * 4`. + Bgra(RenderedFrame), Nv12(Nv12RenderedFrame), #[cfg(target_os = "macos")] Surface(SurfaceFrame), @@ -61,10 +65,60 @@ pub enum EditorFrameOutput { pub enum EditorFrameFormat { #[default] Rgba, + Bgra, #[cfg(target_os = "macos")] BgraSurface, } +/// `Bgra` needs the adapter to render to and copy from a `Bgra8Unorm` +/// texture; where it cannot, the preview falls back to `Rgba` frames, which +/// every consumer of `Bgra` also accepts. +pub(crate) fn supported_frame_format( + format: EditorFrameFormat, + constants: &RenderVideoConstants, +) -> EditorFrameFormat { + if format == EditorFrameFormat::Bgra && !RgbaToBgraReadback::supported(&constants._adapter) { + tracing::warn!("adapter cannot render BGRA readbacks; preview frames stay RGBA"); + return EditorFrameFormat::Rgba; + } + format +} + +impl EditorFrameFormat { + fn pipelines_readback(self) -> bool { + match self { + Self::Rgba | Self::Bgra => true, + #[cfg(target_os = "macos")] + Self::BgraSurface => false, + } + } +} + +struct HeldFrame { + frame_number: u32, + layout: FrameLayout, +} + +fn output_frame_number(output: &EditorFrameOutput) -> u32 { + match output { + EditorFrameOutput::Rgba(frame) | EditorFrameOutput::Bgra(frame) => frame.frame_number, + EditorFrameOutput::Nv12(frame) => frame.frame_number, + #[cfg(target_os = "macos")] + EditorFrameOutput::Surface(frame) => frame.frame_number, + } +} + +fn settled_frame(result: Option>) -> Option { + match result { + Some(Ok(frame)) => Some(frame), + Some(Err(error)) => { + tracing::warn!(%error, "Failed to read back the pending preview frame"); + None + } + None => None, + } +} + pub type RendererLayersReceiver = oneshot::Receiver; pub type EditorFrameCallback = Box; @@ -203,6 +257,7 @@ impl Renderer { output_format, } = self; + let output_format = supported_frame_format(output_format, &render_constants); let mut frame_renderer = FrameRenderer::new(&render_constants); let mut layers = match layers_rx.await { @@ -258,6 +313,7 @@ impl Renderer { } let mut pending_frame: Option = None; + let mut held: Option = None; loop { let frame_to_render = if let Some(pending) = pending_frame.take() { @@ -269,6 +325,7 @@ impl Renderer { continue; } Some(RendererMessage::RenderThumbnail { input, finished }) => { + Self::discard_pending(&mut frame_renderer, output_format, &mut held).await; let result = frame_renderer .render_immediate( input.segment_frames, @@ -318,6 +375,16 @@ impl Renderer { queued_at, realtime, }), + Some(RendererMessage::FlushPending) => { + Self::deliver_pending( + &mut frame_renderer, + output_format, + &mut held, + &mut frame_cb, + ) + .await; + continue; + } Some(RendererMessage::Stop { finished }) => { let _ = finished.send(()); return; @@ -331,10 +398,12 @@ impl Renderer { }; let mut drained_count = 0u32; + let mut flush_after = false; let queue_drain_start = Instant::now(); while let Ok(msg) = rx.try_recv() { match msg { RendererMessage::RenderThumbnail { input, finished } => { + Self::discard_pending(&mut frame_renderer, output_format, &mut held).await; let result = frame_renderer .render_immediate( input.segment_frames, @@ -410,6 +479,10 @@ impl Renderer { }; drained_count += 1; } + RendererMessage::FlushPending => { + flush_after = true; + break; + } RendererMessage::Stop { finished } => { let _ = current.finished.send(false); let _ = finished.send(()); @@ -424,13 +497,11 @@ impl Renderer { let queue_wait = current.queued_at.elapsed(); let drain_duration = queue_drain_start.elapsed(); let flush_start = Instant::now(); - if drained_count > 0 { - let _ = frame_renderer.flush_pipeline().await; - } - let flush_duration = if drained_count > 0 { - flush_start.elapsed() - } else { + let flush_duration = if current.realtime { std::time::Duration::ZERO + } else { + Self::discard_pending(&mut frame_renderer, output_format, &mut held).await; + flush_start.elapsed() }; // Playback frames composite with the newest finished camera mask; @@ -440,23 +511,48 @@ impl Renderer { let render_start = Instant::now(); let input_frame_number = current.input.uniforms().frame_number; let frame_layout = current.input.uniforms().frame_layout(); + let realtime = current.realtime; + let leaves_pending = realtime && output_format.pipelines_readback(); let render_result = match (output_format, current.input) { - (EditorFrameFormat::Rgba, PendingRenderInput::Single(input)) => frame_renderer - .render_immediate_with_timings( - input.segment_frames, - input.uniforms, - &input.cursor, - true, - &mut layers, - ) - .await - .map(|(frame, timings)| { - ( - EditorFrameOutput::Rgba(frame), - PlaybackRenderOutputFormat::Rgba, - timings, - ) - }), + (EditorFrameFormat::Rgba, PendingRenderInput::Single(input)) => { + if realtime { + frame_renderer + .render_with_timings( + input.segment_frames, + input.uniforms, + &input.cursor, + true, + &mut layers, + ) + .await + .map(|(frame, timings)| { + frame.map(|frame| { + ( + EditorFrameOutput::Rgba(frame), + PlaybackRenderOutputFormat::Rgba, + timings, + ) + }) + }) + } else { + frame_renderer + .render_immediate_with_timings( + input.segment_frames, + input.uniforms, + &input.cursor, + true, + &mut layers, + ) + .await + .map(|(frame, timings)| { + Some(( + EditorFrameOutput::Rgba(frame), + PlaybackRenderOutputFormat::Rgba, + timings, + )) + }) + } + } ( EditorFrameFormat::Rgba, PendingRenderInput::Transition { @@ -465,32 +561,143 @@ impl Renderer { kind, progress, }, - ) => frame_renderer - .render_transition_immediate( - TransitionRenderInput { - segment_frames: outgoing.segment_frames, - uniforms: outgoing.uniforms, - cursor: &outgoing.cursor, - render_display: true, - }, - TransitionRenderInput { - segment_frames: incoming.segment_frames, - uniforms: incoming.uniforms, - cursor: &incoming.cursor, - render_display: true, - }, + ) => { + let outgoing = TransitionRenderInput { + segment_frames: outgoing.segment_frames, + uniforms: outgoing.uniforms, + cursor: &outgoing.cursor, + render_display: true, + }; + let incoming = TransitionRenderInput { + segment_frames: incoming.segment_frames, + uniforms: incoming.uniforms, + cursor: &incoming.cursor, + render_display: true, + }; + if realtime { + frame_renderer + .render_transition(outgoing, incoming, kind, progress, &mut layers) + .await + .map(|frame| { + frame.map(|frame| { + ( + EditorFrameOutput::Rgba(frame), + PlaybackRenderOutputFormat::Rgba, + FrameRenderStageTimings::default(), + ) + }) + }) + } else { + frame_renderer + .render_transition_immediate( + outgoing, + incoming, + kind, + progress, + &mut layers, + ) + .await + .map(|frame| { + Some(( + EditorFrameOutput::Rgba(frame), + PlaybackRenderOutputFormat::Rgba, + FrameRenderStageTimings::default(), + )) + }) + } + } + (EditorFrameFormat::Bgra, PendingRenderInput::Single(input)) => { + if realtime { + frame_renderer + .render_bgra_with_timings( + input.segment_frames, + input.uniforms, + &input.cursor, + true, + &mut layers, + ) + .await + .map(|(frame, timings)| { + frame.map(|frame| { + ( + EditorFrameOutput::Bgra(frame), + PlaybackRenderOutputFormat::Bgra, + timings, + ) + }) + }) + } else { + frame_renderer + .render_immediate_bgra_with_timings( + input.segment_frames, + input.uniforms, + &input.cursor, + true, + &mut layers, + ) + .await + .map(|(frame, timings)| { + Some(( + EditorFrameOutput::Bgra(frame), + PlaybackRenderOutputFormat::Bgra, + timings, + )) + }) + } + } + ( + EditorFrameFormat::Bgra, + PendingRenderInput::Transition { + outgoing, + incoming, kind, progress, - &mut layers, - ) - .await - .map(|frame| { - ( - EditorFrameOutput::Rgba(frame), - PlaybackRenderOutputFormat::Rgba, - FrameRenderStageTimings::default(), - ) - }), + }, + ) => { + let outgoing = TransitionRenderInput { + segment_frames: outgoing.segment_frames, + uniforms: outgoing.uniforms, + cursor: &outgoing.cursor, + render_display: true, + }; + let incoming = TransitionRenderInput { + segment_frames: incoming.segment_frames, + uniforms: incoming.uniforms, + cursor: &incoming.cursor, + render_display: true, + }; + if realtime { + frame_renderer + .render_transition_bgra(outgoing, incoming, kind, progress, &mut layers) + .await + .map(|frame| { + frame.map(|frame| { + ( + EditorFrameOutput::Bgra(frame), + PlaybackRenderOutputFormat::Bgra, + FrameRenderStageTimings::default(), + ) + }) + }) + } else { + frame_renderer + .render_transition_immediate_bgra( + outgoing, + incoming, + kind, + progress, + &mut layers, + ) + .await + .map(|frame| { + Some(( + EditorFrameOutput::Bgra(frame), + PlaybackRenderOutputFormat::Bgra, + FrameRenderStageTimings::default(), + )) + }) + } + } #[cfg(target_os = "macos")] (EditorFrameFormat::BgraSurface, PendingRenderInput::Single(input)) => { frame_renderer @@ -503,11 +710,11 @@ impl Renderer { ) .await .map(|frame| { - ( + Some(( EditorFrameOutput::Surface(frame), PlaybackRenderOutputFormat::Bgra, FrameRenderStageTimings::default(), - ) + )) }) } #[cfg(target_os = "macos")] @@ -539,24 +746,30 @@ impl Renderer { ) .await .map(|frame| { - ( + Some(( EditorFrameOutput::Surface(frame), PlaybackRenderOutputFormat::Bgra, FrameRenderStageTimings::default(), - ) + )) }), }; match render_result { - Ok((frame, output_format, render_stage_timings)) => { + Ok(Some((frame, output_format, render_stage_timings))) => { let render_duration = render_start.elapsed(); - let frame_number = match &frame { - EditorFrameOutput::Rgba(frame) => frame.frame_number, - EditorFrameOutput::Nv12(frame) => frame.frame_number, - #[cfg(target_os = "macos")] - EditorFrameOutput::Surface(frame) => frame.frame_number, + let frame_number = output_frame_number(&frame); + let previous = if leaves_pending { + held.replace(HeldFrame { + frame_number: input_frame_number, + layout: frame_layout, + }) + } else { + held.take() }; + let layout = previous + .filter(|previous| previous.frame_number == frame_number) + .map_or(frame_layout, |previous| previous.layout); let callback_start = Instant::now(); - (frame_cb)(frame, frame_layout); + (frame_cb)(frame, layout); let callback_duration = callback_start.elapsed(); if let Some(telemetry) = &telemetry { telemetry.emit(PlaybackTelemetryEvent::RendererFrame { @@ -574,11 +787,65 @@ impl Renderer { } let _ = current.finished.send(true); } + Ok(None) => { + if leaves_pending { + held = Some(HeldFrame { + frame_number: input_frame_number, + layout: frame_layout, + }); + } + let _ = current.finished.send(true); + } Err(e) => { tracing::error!(error = %e, "Failed to render frame in editor"); let _ = current.finished.send(false); } } + + if flush_after { + Self::deliver_pending(&mut frame_renderer, output_format, &mut held, &mut frame_cb) + .await; + } + } + } + + async fn take_pending_output( + frame_renderer: &mut FrameRenderer<'_>, + output_format: EditorFrameFormat, + ) -> Option { + match output_format { + EditorFrameFormat::Rgba => { + settled_frame(frame_renderer.flush_pipeline().await).map(EditorFrameOutput::Rgba) + } + EditorFrameFormat::Bgra => settled_frame(frame_renderer.flush_pipeline_bgra().await) + .map(EditorFrameOutput::Bgra), + #[cfg(target_os = "macos")] + EditorFrameFormat::BgraSurface => None, + } + } + + async fn discard_pending( + frame_renderer: &mut FrameRenderer<'_>, + output_format: EditorFrameFormat, + held: &mut Option, + ) { + let _ = Self::take_pending_output(frame_renderer, output_format).await; + *held = None; + } + + async fn deliver_pending( + frame_renderer: &mut FrameRenderer<'_>, + output_format: EditorFrameFormat, + held: &mut Option, + frame_cb: &mut EditorFrameCallback, + ) { + let held = held.take(); + let Some(frame) = Self::take_pending_output(frame_renderer, output_format).await else { + return; + }; + let frame_number = output_frame_number(&frame); + if let Some(held) = held.filter(|held| held.frame_number == frame_number) { + (frame_cb)(frame, held.layout); } } @@ -696,6 +963,28 @@ impl RendererHandle { } } + #[cfg(test)] + pub(crate) async fn render_frame_realtime_confirmed( + &self, + segment_frames: DecodedSegmentFrames, + uniforms: ProjectUniforms, + cursor: Arc, + ) -> bool { + let (finished_tx, finished_rx) = oneshot::channel(); + let msg = RendererMessage::RenderFrame { + segment_frames, + uniforms, + finished: finished_tx, + cursor, + queued_at: Instant::now(), + realtime: true, + }; + if self.tx.send(msg).await.is_err() { + return false; + } + finished_rx.await.unwrap_or(false) + } + pub async fn render_frame_confirmed( &self, segment_frames: DecodedSegmentFrames, @@ -804,6 +1093,10 @@ impl RendererHandle { finished_rx.blocking_recv().unwrap_or(false) } + pub fn flush_pending_blocking(&self) { + let _ = self.tx.blocking_send(RendererMessage::FlushPending); + } + pub async fn stop(&self) { let (tx, rx) = oneshot::channel(); if self diff --git a/crates/editor/src/playback.rs b/crates/editor/src/playback.rs index 1e4d2efd150..9d034e2f6a2 100644 --- a/crates/editor/src/playback.rs +++ b/crates/editor/src/playback.rs @@ -2120,6 +2120,7 @@ impl Playback { drop(transitioned_audio); stop_tx.send(true).ok(); + self.renderer.flush_pending_blocking(); event_tx.send(PlaybackEvent::Stop).ok(); }; diff --git a/crates/editor/src/preparing_preview.rs b/crates/editor/src/preparing_preview.rs index f13bc342481..8432d91b497 100644 --- a/crates/editor/src/preparing_preview.rs +++ b/crates/editor/src/preparing_preview.rs @@ -582,6 +582,7 @@ async fn run_preview( ); check_cursor_error(&constants)?; control.check()?; + let frame_format = crate::editor::supported_frame_format(options.frame_format, &constants); let mut renderer = FrameRenderer::new(&constants); let mut cursor_cache = PreviewCursorCache::default(); let mut zoom_cache = VecDeque::new(); @@ -685,11 +686,21 @@ async fn run_preview( ) }; let layout = uniforms.frame_layout(); - let output = match options.frame_format { + let output = match frame_format { EditorFrameFormat::Rgba => renderer .render_immediate(frames, uniforms, &cursor, true, &mut layers) .await .map(EditorFrameOutput::Rgba), + EditorFrameFormat::Bgra => renderer + .render_immediate_bgra_with_timings( + frames, + uniforms, + &cursor, + true, + &mut layers, + ) + .await + .map(|(frame, _)| EditorFrameOutput::Bgra(frame)), #[cfg(target_os = "macos")] EditorFrameFormat::BgraSurface => renderer .render_immediate_bgra_surface(frames, uniforms, &cursor, true, &mut layers) @@ -802,7 +813,9 @@ fn validate_output( layout: FrameLayout, ) -> Result<(), PreparingPreviewError> { let (number, width, height) = match output { - EditorFrameOutput::Rgba(frame) => (frame.frame_number, frame.width, frame.height), + EditorFrameOutput::Rgba(frame) | EditorFrameOutput::Bgra(frame) => { + (frame.frame_number, frame.width, frame.height) + } EditorFrameOutput::Nv12(frame) => (frame.frame_number, frame.width, frame.height), #[cfg(target_os = "macos")] EditorFrameOutput::Surface(frame) => (frame.frame_number, frame.width, frame.height), diff --git a/crates/editor/src/preparing_preview/native_tests.rs b/crates/editor/src/preparing_preview/native_tests.rs index eb62cbb134e..a22e89c2909 100644 --- a/crates/editor/src/preparing_preview/native_tests.rs +++ b/crates/editor/src/preparing_preview/native_tests.rs @@ -792,6 +792,10 @@ async fn composed_parity_with_reference( assert_eq!(frame_format, EditorFrameFormat::Rgba); frame } + EditorFrameOutput::Bgra(frame) => { + assert_eq!(frame_format, EditorFrameFormat::Bgra); + bgra_rgba(frame) + } #[cfg(target_os = "macos")] EditorFrameOutput::Surface(frame) => { assert_eq!(frame_format, EditorFrameFormat::BgraSurface); @@ -1075,6 +1079,222 @@ fn surface_rgba(frame: cap_rendering::SurfaceFrame) -> RenderedFrame { } } +async fn pipelined_input( + fixture: &Fixture, + decoders: &RecordingSegmentDecoders, + constants: &RenderVideoConstants, + frame_number: u32, + padding: f64, +) -> (cap_rendering::DecodedSegmentFrames, ProjectUniforms) { + let mut project = fixture.project.clone(); + project.background.padding = padding; + let project = Arc::new(project); + let time = f64::from(frame_number) / 30.0; + let (segment_time, segment) = project.get_segment_time(time).unwrap(); + let frames = decoders + .get_frames_initial( + segment_time as f32, + true, + true, + project.clips[segment.recording_clip as usize].offsets, + ) + .await + .unwrap(); + let duration = project.timeline.as_ref().unwrap().duration(); + let mut zoom = ZoomTransformTimeline::from_project_for_clip( + &project, + &fixture.cursor, + duration, + constants.options.screen_size, + segment.recording_clip, + ); + zoom.ensure_precomputed_until((frame_number as f32 + 1.0) / 30.0); + let uniforms = ProjectUniforms::new( + constants, + &project, + frame_number, + 30, + XY::new(320, 240), + &fixture.cursor, + &frames, + duration, + &zoom, + ); + (frames, uniforms) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn native_pipelined_preview_keeps_layouts_and_drains_the_last_frame() { + let fixture = fixture(false); + let metadata = &fixture.reference_metadata; + let studio = metadata.studio_meta().unwrap(); + let decoders = RecordingSegmentDecoders::new( + metadata, + studio, + SegmentVideoPaths { + display: metadata.project_path.join("display.mp4"), + camera: Some(metadata.project_path.join("camera.mp4")), + }, + 0, + true, + ) + .await + .unwrap(); + let constants = Arc::new( + RenderVideoConstants::new_with_options( + RenderOptions { + screen_size: XY::new(160, 120), + camera_size: Some(XY::new(80, 80)), + preserve_screen_alpha: false, + }, + metadata.clone(), + studio.clone(), + ) + .await + .unwrap(), + ); + + for format in [EditorFrameFormat::Rgba, EditorFrameFormat::Bgra] { + if crate::editor::supported_frame_format(format, &constants) != format { + continue; + } + let mut layers = RendererLayers::new_with_options( + &constants.device, + &constants.queue, + constants.is_software_adapter, + ); + layers.preload_cursor_assets( + &constants, + fixture.project.cursor.use_svg, + fixture.project.cursor.cursor_type(), + ); + let (layers_tx, layers_rx) = tokio::sync::oneshot::channel(); + layers_tx.send(layers).ok(); + let (delivered_tx, mut delivered) = tokio::sync::mpsc::unbounded_channel(); + let renderer = crate::Renderer::spawn_with_format( + constants.clone(), + Box::new(move |output, layout| { + let frame = match output { + EditorFrameOutput::Rgba(frame) | EditorFrameOutput::Bgra(frame) => frame, + _ => panic!("Unexpected output"), + }; + delivered_tx.send((frame.frame_number, layout)).ok(); + }), + layers_rx, + format, + ) + .unwrap(); + + let (first_frames, first_uniforms) = + pipelined_input(&fixture, &decoders, &constants, 0, 0.0).await; + let (second_frames, second_uniforms) = + pipelined_input(&fixture, &decoders, &constants, 45, 25.0).await; + let (third_frames, third_uniforms) = + pipelined_input(&fixture, &decoders, &constants, 12, 0.0).await; + let (seek_frames, seek_uniforms) = + pipelined_input(&fixture, &decoders, &constants, 90, 40.0).await; + let first_layout = first_uniforms.frame_layout(); + let second_layout = second_uniforms.frame_layout(); + let seek_layout = seek_uniforms.frame_layout(); + assert_ne!(first_layout, second_layout); + assert_ne!(second_layout, seek_layout); + + assert!( + renderer + .render_frame_realtime_confirmed( + first_frames, + first_uniforms, + fixture.cursor.clone() + ) + .await + ); + assert!( + delivered.try_recv().is_err(), + "{format:?}: the first frame stays pending" + ); + assert!( + renderer + .render_frame_realtime_confirmed( + second_frames, + second_uniforms, + fixture.cursor.clone() + ) + .await + ); + let (frame_number, layout) = + tokio::time::timeout(Duration::from_secs(15), delivered.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!(frame_number, 0, "{format:?}"); + assert_eq!(layout, first_layout, "{format:?}: pixels and layout agree"); + + tokio::task::block_in_place(|| renderer.flush_pending_blocking()); + let (frame_number, layout) = + tokio::time::timeout(Duration::from_secs(15), delivered.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!(frame_number, 45, "{format:?}: the last frame is drained"); + assert_eq!(layout, second_layout, "{format:?}"); + + assert!( + renderer + .render_frame_realtime_confirmed( + third_frames, + third_uniforms, + fixture.cursor.clone() + ) + .await + ); + assert!( + renderer + .render_frame_confirmed(seek_frames, seek_uniforms, fixture.cursor.clone()) + .await + ); + let (frame_number, layout) = + tokio::time::timeout(Duration::from_secs(15), delivered.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!(frame_number, 90, "{format:?}: a seek shows its own frame"); + assert_eq!(layout, seek_layout, "{format:?}"); + assert!( + tokio::time::timeout(Duration::from_millis(300), delivered.recv()) + .await + .is_err(), + "{format:?}: the playback frame a seek replaced is not shown" + ); + renderer.stop().await; + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn native_preparing_bgra_readback_matches_default_editor_frame() { + composed_parity_with_reference( + fixture(false), + &[0, 45, 12, 90], + false, + EditorFrameFormat::Bgra, + ) + .await; +} + +fn bgra_rgba(frame: RenderedFrame) -> RenderedFrame { + let row_bytes = frame.width * 4; + assert_eq!(frame.padded_bytes_per_row, row_bytes); + assert_eq!(frame.data.len(), row_bytes as usize * frame.height as usize); + let rgba = frame + .data + .chunks_exact(4) + .flat_map(|pixel| [pixel[2], pixel[1], pixel[0], pixel[3]]) + .collect(); + RenderedFrame { + data: Arc::new(rgba), + ..frame + } +} + #[cfg(target_os = "macos")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn native_preparing_bgra_surface_matches_default_avasset_editor_frame() { @@ -1263,7 +1483,8 @@ async fn native_handoff_reuses_pcm_preserves_audio_across_candidate_disposal_and |_| {}, Box::new(move |output, _| { let frame = match output { - crate::EditorFrameOutput::Rgba(frame) => frame.frame_number, + crate::EditorFrameOutput::Rgba(frame) + | crate::EditorFrameOutput::Bgra(frame) => frame.frame_number, crate::EditorFrameOutput::Nv12(frame) => frame.frame_number, #[cfg(target_os = "macos")] crate::EditorFrameOutput::Surface(frame) => frame.frame_number, @@ -1411,7 +1632,8 @@ async fn native_silent_handoff_opens_and_renders_through_an_equivalent_editor_pa |_| {}, Box::new(move |output, _| { let frame = match output { - crate::EditorFrameOutput::Rgba(frame) => frame.frame_number, + crate::EditorFrameOutput::Rgba(frame) + | crate::EditorFrameOutput::Bgra(frame) => frame.frame_number, crate::EditorFrameOutput::Nv12(frame) => frame.frame_number, #[cfg(target_os = "macos")] crate::EditorFrameOutput::Surface(frame) => frame.frame_number, @@ -1613,7 +1835,8 @@ async fn native_studio_sound_releases_preparing_audio_and_preserves_transport() |_| {}, Box::new(move |output, _| { let frame = match output { - crate::EditorFrameOutput::Rgba(frame) => frame.frame_number, + crate::EditorFrameOutput::Rgba(frame) + | crate::EditorFrameOutput::Bgra(frame) => frame.frame_number, crate::EditorFrameOutput::Nv12(frame) => frame.frame_number, #[cfg(target_os = "macos")] crate::EditorFrameOutput::Surface(frame) => frame.frame_number, diff --git a/crates/editor/src/thumbnail.rs b/crates/editor/src/thumbnail.rs index 9b1abe68a9f..11b29bcca61 100644 --- a/crates/editor/src/thumbnail.rs +++ b/crates/editor/src/thumbnail.rs @@ -268,6 +268,7 @@ mod tests { ffmpeg::init().unwrap(); for format in [ crate::EditorFrameFormat::Rgba, + crate::EditorFrameFormat::Bgra, #[cfg(target_os = "macos")] crate::EditorFrameFormat::BgraSurface, ] { diff --git a/crates/rendering/src/frame_pipeline.rs b/crates/rendering/src/frame_pipeline.rs index 2b9fb639c81..db83a747150 100644 --- a/crates/rendering/src/frame_pipeline.rs +++ b/crates/rendering/src/frame_pipeline.rs @@ -1731,6 +1731,27 @@ impl PendingReadback { device: &wgpu::Device, recycled: &mut RecycledReadbackFrames, ) -> Result { + let active_bytes = self.map(device).await?; + let mut data_vec = if self.export_deadlines { + recycled.take().unwrap_or_default() + } else { + Vec::new() + }; + reserve_readback(&mut data_vec, active_bytes, self.frame_number)?; + let buffer_slice = self.buffer.slice(..active_bytes as u64); + let data = buffer_slice.get_mapped_range(); + data_vec.extend_from_slice(&data); + drop(data); + let recycle = self.export_deadlines && self.recycle_readback_frames; + let padded_bytes_per_row = self.padded_bytes_per_row; + let frame = self.into_frame(data_vec, padded_bytes_per_row); + if recycle { + recycled.keep(&frame.data); + } + Ok(frame) + } + + async fn map(&mut self, device: &wgpu::Device) -> Result { let start_time = Instant::now(); let timeout = gpu_buffer_wait_timeout(self.export_deadlines); let completed = wait_for_submission( @@ -1779,17 +1800,10 @@ impl PendingReadback { self.unmap(); return Err(RenderingError::BufferMapWaitingFailed); }; - let mut data_vec = if self.export_deadlines { - recycled.take().unwrap_or_default() - } else { - Vec::new() - }; - reserve_readback(&mut data_vec, active_bytes, self.frame_number)?; - let buffer_slice = self.buffer.slice(..active_bytes as u64); - let data = buffer_slice.get_mapped_range(); - data_vec.extend_from_slice(&data); + Ok(active_bytes) + } - drop(data); + fn into_frame(mut self, data: Vec, padded_bytes_per_row: u32) -> RenderedFrame { self.unmap(); let target_time_ns = @@ -1798,18 +1812,14 @@ impl PendingReadback { if let Some(phase) = self.readiness_phase.take() { phase.finish("frame_ready"); } - let data = Arc::new(data_vec); - if self.export_deadlines && self.recycle_readback_frames { - recycled.keep(&data); - } - Ok(RenderedFrame { - data, - padded_bytes_per_row: self.padded_bytes_per_row, + RenderedFrame { + data: Arc::new(data), + padded_bytes_per_row, width: self.width, height: self.height, frame_number: self.frame_number, target_time_ns, - }) + } } } @@ -1819,6 +1829,324 @@ impl Drop for PendingReadback { } } +fn tight_rows(data: &[u8], row_bytes: usize, stride: usize) -> Option> { + if row_bytes == stride { + return Some(data.to_vec()); + } + if stride == 0 || !data.len().is_multiple_of(stride) { + return None; + } + let mut tight = Vec::with_capacity(row_bytes.checked_mul(data.len() / stride)?); + for row in data.chunks_exact(stride) { + tight.extend_from_slice(row.get(..row_bytes)?); + } + Some(tight) +} + +/// Converts the rendered RGBA frame to BGRA on the GPU before reading it back, +/// so a CPU consumer that wants BGRA (gpui's sprite atlas) gets display-ready, +/// tightly packed bytes from a single copy out of the mapped buffer. +pub struct RgbaToBgraReadback { + pipeline: wgpu::RenderPipeline, + bind_group_layout: wgpu::BindGroupLayout, + target: Option, + source_bind_groups: Vec<(wgpu::Texture, wgpu::BindGroup)>, + pending: Option, +} + +const BGRA_READBACK_SLOTS: usize = 2; + +struct BgraReadbackSlot { + view: wgpu::TextureView, + texture: wgpu::Texture, + buffer: Arc, +} + +struct BgraReadbackTarget { + width: u32, + height: u32, + padded_bytes_per_row: u32, + slots: [BgraReadbackSlot; BGRA_READBACK_SLOTS], + next_slot: usize, +} + +fn new_bgra_readback_slot( + device: &wgpu::Device, + width: u32, + height: u32, + padded_bytes_per_row: u32, +) -> BgraReadbackSlot { + let texture = device.create_texture(&wgpu::TextureDescriptor { + label: Some("BGRA Readback Texture"), + size: wgpu::Extent3d { + width, + height, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Bgra8Unorm, + usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::COPY_SRC, + view_formats: &[], + }); + let buffer = Arc::new(device.create_buffer(&wgpu::BufferDescriptor { + label: Some("BGRA Readback Buffer"), + size: u64::from(padded_bytes_per_row) * u64::from(height), + usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, + mapped_at_creation: false, + })); + BgraReadbackSlot { + view: texture.create_view(&Default::default()), + texture, + buffer, + } +} + +pub struct PendingBgraReadback(PendingReadback); + +impl RgbaToBgraReadback { + pub fn supported(adapter: &wgpu::Adapter) -> bool { + adapter + .get_texture_format_features(wgpu::TextureFormat::Bgra8Unorm) + .allowed_usages + .contains(wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::COPY_SRC) + } + + pub fn new(device: &wgpu::Device) -> Self { + let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("RGBA to BGRA Readback Blit"), + source: wgpu::ShaderSource::Wgsl(std::borrow::Cow::Borrowed(include_str!( + "shaders/blit_bgra_surface.wgsl" + ))), + }); + let bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("RGBA to BGRA Readback Bind Group Layout"), + entries: &[wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: false }, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + count: None, + }], + }); + let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("RGBA to BGRA Readback Pipeline Layout"), + bind_group_layouts: &[&bind_group_layout], + push_constant_ranges: &[], + }); + let pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("RGBA to BGRA Readback Pipeline"), + layout: Some(&pipeline_layout), + vertex: wgpu::VertexState { + module: &shader, + entry_point: Some("vs_main"), + compilation_options: Default::default(), + buffers: &[], + }, + fragment: Some(wgpu::FragmentState { + module: &shader, + entry_point: Some("fs_main"), + compilation_options: Default::default(), + targets: &[Some(wgpu::ColorTargetState { + format: wgpu::TextureFormat::Bgra8Unorm, + blend: None, + write_mask: wgpu::ColorWrites::ALL, + })], + }), + primitive: wgpu::PrimitiveState::default(), + depth_stencil: None, + multisample: wgpu::MultisampleState::default(), + multiview: None, + cache: None, + }); + + Self { + pipeline, + bind_group_layout, + target: None, + source_bind_groups: Vec::new(), + pending: None, + } + } + + pub fn take_pending(&mut self) -> Option { + self.pending.take() + } + + fn source_bind_group( + &mut self, + device: &wgpu::Device, + source_texture: &wgpu::Texture, + ) -> wgpu::BindGroup { + if let Some((_, bind_group)) = self + .source_bind_groups + .iter() + .find(|(texture, _)| texture == source_texture) + { + return bind_group.clone(); + } + let source_view = source_texture.create_view(&Default::default()); + let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("RGBA to BGRA Readback Bind Group"), + layout: &self.bind_group_layout, + entries: &[wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView(&source_view), + }], + }); + if self.source_bind_groups.len() >= 2 { + self.source_bind_groups.remove(0); + } + self.source_bind_groups + .push((source_texture.clone(), bind_group.clone())); + bind_group + } + + fn ensure_target(&mut self, device: &wgpu::Device, width: u32, height: u32) { + if self + .target + .as_ref() + .is_some_and(|target| target.width == width && target.height == height) + { + return; + } + let padded_bytes_per_row = padded_bytes_per_row((width, height)); + let slots = std::array::from_fn(|_| { + new_bgra_readback_slot(device, width, height, padded_bytes_per_row) + }); + self.target = Some(BgraReadbackTarget { + width, + height, + padded_bytes_per_row, + slots, + next_slot: 0, + }); + } + + pub fn submit( + &mut self, + device: &wgpu::Device, + queue: &wgpu::Queue, + mut encoder: wgpu::CommandEncoder, + source: &wgpu::Texture, + frame_number: u32, + frame_rate: u32, + ) -> Result { + let (width, height) = (source.width(), source.height()); + if width == 0 || height == 0 { + return Err(RenderingError::BufferMapWaitingFailed); + } + let bind_group = self.source_bind_group(device, source); + self.ensure_target(device, width, height); + let target = self + .target + .as_mut() + .expect("BGRA readback target initialized"); + let slot_index = target.next_slot; + target.next_slot = (slot_index + 1) % BGRA_READBACK_SLOTS; + let slot = &target.slots[slot_index]; + let padded_bytes_per_row = target.padded_bytes_per_row; + + { + let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("RGBA to BGRA Readback Blit"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &slot.view, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + pass.set_pipeline(&self.pipeline); + pass.set_bind_group(0, &bind_group, &[]); + pass.draw(0..3, 0..1); + } + + encoder.copy_texture_to_buffer( + wgpu::TexelCopyTextureInfo { + texture: &slot.texture, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + wgpu::TexelCopyBufferInfo { + buffer: &slot.buffer, + layout: wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(padded_bytes_per_row), + rows_per_image: Some(height), + }, + }, + wgpu::Extent3d { + width, + height, + depth_or_array_layers: 1, + }, + ); + + let submission = queue.submit(std::iter::once(encoder.finish())); + + let (tx, rx) = oneshot::channel(); + let readback_buffer = slot.buffer.clone(); + readback_buffer + .slice(..u64::from(padded_bytes_per_row) * u64::from(height)) + .map_async(wgpu::MapMode::Read, move |result| { + if let Err(e) = tx.send(result) { + tracing::error!("Failed to send map_async result: {:?}", e); + } + }); + + Ok(PendingBgraReadback(PendingReadback { + readiness_phase: None, + rx, + submission, + mapped: true, + buffer: readback_buffer, + padded_bytes_per_row, + width, + height, + frame_number, + frame_rate, + export_deadlines: false, + recycle_readback_frames: false, + })) + } +} + +impl PendingBgraReadback { + /// The frame as tightly packed BGRA: `padded_bytes_per_row` is always + /// `width * 4`, so the bytes can back an image without un-padding. + pub async fn wait(self, device: &wgpu::Device) -> Result { + let mut pending = self.0; + let active_bytes = pending.map(device).await?; + let row_bytes = pending.width * 4; + let data = pending + .buffer + .slice(..active_bytes as u64) + .get_mapped_range(); + let tight = tight_rows( + &data, + row_bytes as usize, + pending.padded_bytes_per_row as usize, + ); + drop(data); + let Some(tight) = tight else { + pending.unmap(); + return Err(RenderingError::BufferMapWaitingFailed); + }; + Ok(pending.into_frame(tight, row_bytes)) + } +} + const MAX_RECYCLED_READBACK_FRAMES: usize = 8; /// Export readbacks are handed downstream as `Arc>`; keeping a clone @@ -2413,6 +2741,52 @@ pub async fn finish_encoder_bgra_surface( result } +pub async fn finish_encoder_bgra_readback( + session: &mut RenderSession, + readback: &mut RgbaToBgraReadback, + device: &wgpu::Device, + queue: &wgpu::Queue, + uniforms: &ProjectUniforms, + encoder: wgpu::CommandEncoder, +) -> Result, RenderingError> { + let previous = readback.take_pending(); + let pending = readback.submit( + device, + queue, + encoder, + session.current_texture(), + uniforms.frame_number, + uniforms.frame_rate, + )?; + readback.pending = Some(pending); + if let Some(previous) = previous { + match previous.wait(device).await { + Ok(frame) => Ok(Some(frame)), + Err(error) => { + drain_submitted_work(device, queue).await; + Err(error) + } + } + } else { + Ok(None) + } +} + +pub async fn flush_pending_bgra_readback( + readback: &mut RgbaToBgraReadback, + device: &wgpu::Device, + queue: &wgpu::Queue, +) -> Option> { + let pending = readback.take_pending()?; + Some(match pending.wait(device).await { + Ok(frame) => Ok(frame), + Err(error) => { + drain_submitted_work(device, queue).await; + Err(error) + } + }) +} + pub async fn flush_pending_readback( session: &mut RenderSession, device: &wgpu::Device, @@ -2541,6 +2915,37 @@ mod readback_output_tests { } } + #[test] + fn aligned_bgra_rows_are_copied_whole() { + let data: Vec = (0..=255).cycle().take(7_680 * 3).collect(); + assert_eq!(tight_rows(&data, 7_680, 7_680), Some(data)); + } + + #[test] + fn padded_bgra_rows_drop_only_the_padding() { + let stride = 256; + let row_bytes = 3 * 4; + let mut data = vec![0xAA; stride * 2]; + for (index, byte) in data[..row_bytes].iter_mut().enumerate() { + *byte = index as u8; + } + for (index, byte) in data[stride..stride + row_bytes].iter_mut().enumerate() { + *byte = 100 + index as u8; + } + let tight = tight_rows(&data, row_bytes, stride).expect("rows fit"); + assert_eq!(tight.len(), row_bytes * 2); + assert_eq!(&tight[..row_bytes], &data[..row_bytes]); + assert_eq!(&tight[row_bytes..], &data[stride..stride + row_bytes]); + assert!(!tight.contains(&0xAA)); + } + + #[test] + fn malformed_bgra_rows_are_rejected() { + assert_eq!(tight_rows(&[0; 300], 12, 256), None); + assert_eq!(tight_rows(&[0; 512], 260, 256), None); + assert_eq!(tight_rows(&[0; 16], 4, 0), None); + } + #[test] fn oversized_readback_buffers_only_include_active_rows() { assert_eq!( @@ -2622,6 +3027,167 @@ mod recycled_readback_tests { } } +#[cfg(test)] +mod bgra_readback_tests { + use super::*; + + const WIDTH: u32 = 37; + const HEIGHT: u32 = 5; + + fn pixel(frame: u32, x: u32, y: u32) -> [u8; 4] { + [ + ((x * 7 + y * 3 + frame * 29) % 256) as u8, + ((x * 5 + y * 11 + frame * 13) % 256) as u8, + ((x * 3 + y * 17 + frame * 41) % 256) as u8, + 255, + ] + } + + fn write_source(queue: &wgpu::Queue, source: &wgpu::Texture, frame: u32) { + let mut data = Vec::new(); + for y in 0..HEIGHT { + for x in 0..WIDTH { + data.extend_from_slice(&pixel(frame, x, y)); + } + } + queue.write_texture( + source.as_image_copy(), + &data, + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(WIDTH * 4), + rows_per_image: None, + }, + wgpu::Extent3d { + width: WIDTH, + height: HEIGHT, + depth_or_array_layers: 1, + }, + ); + } + + fn assert_frame(frame: &RenderedFrame, number: u32) { + assert_eq!(frame.frame_number, number); + assert_eq!(frame.padded_bytes_per_row, WIDTH * 4); + assert_eq!(frame.data.len(), (WIDTH * HEIGHT * 4) as usize); + for y in 0..HEIGHT { + for x in 0..WIDTH { + let [r, g, b, a] = pixel(number, x, y); + let offset = ((y * WIDTH + x) * 4) as usize; + assert_eq!( + &frame.data[offset..offset + 4], + &[b, g, r, a], + "frame {number} pixel ({x}, {y})" + ); + } + } + } + + async fn device() -> Option<(wgpu::Device, wgpu::Queue)> { + let instance = crate::create_wgpu_instance_sync(); + let adapter = instance + .request_adapter(&wgpu::RequestAdapterOptions::default()) + .await + .ok()?; + if !RgbaToBgraReadback::supported(&adapter) { + return None; + } + adapter + .request_device(&wgpu::DeviceDescriptor::default()) + .await + .ok() + } + + async fn pipelined_step( + device: &wgpu::Device, + queue: &wgpu::Queue, + readback: &mut RgbaToBgraReadback, + source: &wgpu::Texture, + frame: u32, + ) -> Option { + write_source(queue, source, frame); + let previous = readback.take_pending(); + let encoder = device.create_command_encoder(&Default::default()); + let pending = readback + .submit(device, queue, encoder, source, frame, 30) + .expect("submit readback"); + readback.pending = Some(pending); + match previous { + Some(previous) => Some(previous.wait(device).await.expect("previous readback")), + None => None, + } + } + + fn source_texture(device: &wgpu::Device) -> wgpu::Texture { + device.create_texture(&wgpu::TextureDescriptor { + label: Some("bgra readback source"), + size: wgpu::Extent3d { + width: WIDTH, + height: HEIGHT, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba8Unorm, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }) + } + + #[tokio::test] + async fn pipelined_frames_never_reuse_a_buffer_that_is_still_mapped() { + let Some((device, queue)) = device().await else { + eprintln!("no BGRA-capable graphics adapter available, skipping"); + return; + }; + let source = source_texture(&device); + let mut readback = RgbaToBgraReadback::new(&device); + + assert!( + pipelined_step(&device, &queue, &mut readback, &source, 0) + .await + .is_none() + ); + for frame in 1..=8 { + let delivered = pipelined_step(&device, &queue, &mut readback, &source, frame) + .await + .expect("the previous frame is delivered"); + assert_frame(&delivered, frame - 1); + } + let last = flush_pending_bgra_readback(&mut readback, &device, &queue) + .await + .expect("the last frame is still pending") + .expect("last readback"); + assert_frame(&last, 8); + assert!(readback.take_pending().is_none()); + } + + #[tokio::test] + async fn a_discarded_pending_readback_leaves_the_ring_usable() { + let Some((device, queue)) = device().await else { + eprintln!("no BGRA-capable graphics adapter available, skipping"); + return; + }; + let source = source_texture(&device); + let mut readback = RgbaToBgraReadback::new(&device); + + pipelined_step(&device, &queue, &mut readback, &source, 0).await; + drop(readback.take_pending()); + assert!( + pipelined_step(&device, &queue, &mut readback, &source, 1) + .await + .is_none() + ); + for frame in 2..=5 { + let delivered = pipelined_step(&device, &queue, &mut readback, &source, frame) + .await + .expect("the previous frame is delivered"); + assert_frame(&delivered, frame - 1); + } + } +} + #[cfg(all(test, target_os = "macos"))] mod surface_output_tests { use super::*; diff --git a/crates/rendering/src/lib.rs b/crates/rendering/src/lib.rs index da01e6295eb..786cbd06dd1 100644 --- a/crates/rendering/src/lib.rs +++ b/crates/rendering/src/lib.rs @@ -15,8 +15,8 @@ use decoder::{AsyncVideoDecoderHandle, spawn_decoder}; #[cfg(target_os = "macos")] use frame_pipeline::finish_encoder_bgra_surface; use frame_pipeline::{ - NV12BufferPool, RenderSession, finish_encoder_nv12_pooled, finish_encoder_timed, - flush_pending_readback, + NV12BufferPool, RenderSession, finish_encoder_bgra_readback, finish_encoder_nv12_pooled, + finish_encoder_timed, flush_pending_bgra_readback, flush_pending_readback, }; use futures::future::OptionFuture; use layers::{ @@ -72,6 +72,7 @@ pub use frame_pipeline::Nv12Surface; #[cfg(target_os = "macos")] pub use frame_pipeline::SurfaceFrame; pub use frame_pipeline::{GpuOutputFormat, Nv12RenderedFrame, RenderedFrame, SharedNv12Buffer}; +pub use frame_pipeline::{PendingBgraReadback, RgbaToBgraReadback}; #[cfg(target_os = "macos")] pub use frame_pipeline::{PendingSurface, RgbaToBgraSurfaceConverter}; pub use frame_windows::FrameWindows; @@ -7323,6 +7324,7 @@ pub struct FrameRenderer<'a> { nv12_surface_output: bool, #[cfg(target_os = "macos")] bgra_surface_converter: Option, + bgra_readback: Option, nv12_buffer_pool: NV12BufferPool, transition_compositor: Option, export_deadlines: bool, @@ -7342,6 +7344,7 @@ impl<'a> FrameRenderer<'a> { nv12_surface_output: false, #[cfg(target_os = "macos")] bgra_surface_converter: None, + bgra_readback: None, nv12_buffer_pool: NV12BufferPool::new(6), transition_compositor: None, export_deadlines: false, @@ -7651,6 +7654,231 @@ impl<'a> FrameRenderer<'a> { Ok((frame, timings)) } + /// [`Self::render_immediate_with_timings`], read back as tightly packed + /// BGRA (`padded_bytes_per_row == width * 4`) instead of padded RGBA. + pub async fn render_bgra_with_timings( + &mut self, + segment_frames: DecodedSegmentFrames, + uniforms: ProjectUniforms, + cursor: &CursorEvents, + render_display: bool, + layers: &mut RendererLayers, + ) -> Result<(Option, FrameRenderStageTimings), RenderingError> { + let phase = std::mem::take(&mut self.readiness_first_render) + .then(|| readiness::Phase::start("frame.bgra_readback_first_render")); + let mut last_error = None; + + for attempt in 0..Self::MAX_RENDER_RETRIES { + if attempt > 0 { + tracing::warn!( + frame_number = uniforms.frame_number, + attempt = attempt + 1, + "Retrying BGRA readback frame render after GPU error" + ); + self.reset_session(); + self.bgra_readback = None; + tokio::time::sleep(std::time::Duration::from_millis(100 * (attempt as u64 + 1))) + .await; + } + + let readback = self.bgra_readback.get_or_insert_with(|| { + frame_pipeline::RgbaToBgraReadback::new(&self.constants.device) + }); + let session = self.session.get_or_insert_with(|| { + RenderSession::new( + &self.constants.device, + uniforms.output_size.0, + uniforms.output_size.1, + ) + .with_export_deadlines(self.export_deadlines) + }); + session.update_texture_size( + &self.constants.device, + uniforms.output_size.0, + uniforms.output_size.1, + ); + + let (encoder, mut timings) = encode_frame_with_timings( + self.constants, + &segment_frames, + &uniforms, + cursor, + render_display, + layers, + session, + ) + .await?; + + let finish_start = Instant::now(); + match finish_encoder_bgra_readback( + session, + readback, + &self.constants.device, + &self.constants.queue, + &uniforms, + encoder, + ) + .await + { + Ok(frame) => { + timings.finish_duration = finish_start.elapsed(); + if let Some(phase) = phase { + phase.finish("frame_ready"); + } + return Ok((frame, timings)); + } + Err(RenderingError::BufferMapWaitingFailed) => { + tracing::warn!( + frame_number = uniforms.frame_number, + attempt = attempt + 1, + "GPU buffer mapping failed, will retry" + ); + last_error = Some(RenderingError::BufferMapWaitingFailed); + } + Err(RenderingError::BufferMapFailed(e)) => { + tracing::warn!( + frame_number = uniforms.frame_number, + attempt = attempt + 1, + error = %e, + "GPU buffer async error, will retry" + ); + last_error = Some(RenderingError::BufferMapFailed(e)); + } + Err(e) => return Err(e), + } + } + + Err(last_error.unwrap_or(RenderingError::BufferMapWaitingFailed)) + } + + pub async fn render_immediate_bgra_with_timings( + &mut self, + segment_frames: DecodedSegmentFrames, + uniforms: ProjectUniforms, + cursor: &CursorEvents, + render_display: bool, + layers: &mut RendererLayers, + ) -> Result<(RenderedFrame, FrameRenderStageTimings), RenderingError> { + let (frame, mut timings) = self + .render_bgra_with_timings(segment_frames, uniforms, cursor, render_display, layers) + .await?; + if let Some(frame) = frame { + return Ok((frame, timings)); + } + let flush_start = Instant::now(); + let frame = self + .flush_pipeline_bgra() + .await + .unwrap_or(Err(RenderingError::BufferMapWaitingFailed))?; + timings.immediate_flush_duration = flush_start.elapsed(); + Ok((frame, timings)) + } + + /// [`Self::render_transition`], read back as tightly packed BGRA. + pub async fn render_transition_bgra( + &mut self, + outgoing: TransitionRenderInput<'_>, + incoming: TransitionRenderInput<'_>, + kind: ClipTransitionType, + progress: f32, + layers: &mut RendererLayers, + ) -> Result, RenderingError> { + let mut last_error = None; + + for attempt in 0..Self::MAX_RENDER_RETRIES { + if attempt > 0 { + tracing::warn!( + frame_number = incoming.uniforms.frame_number, + attempt = attempt + 1, + "Retrying BGRA readback transition frame after GPU error" + ); + self.reset_session(); + self.bgra_readback = None; + tokio::time::sleep(std::time::Duration::from_millis(100 * (attempt as u64 + 1))) + .await; + } + + let readback = self.bgra_readback.get_or_insert_with(|| { + frame_pipeline::RgbaToBgraReadback::new(&self.constants.device) + }); + let session = self.session.get_or_insert_with(|| { + RenderSession::new( + &self.constants.device, + incoming.uniforms.output_size.0, + incoming.uniforms.output_size.1, + ) + .with_export_deadlines(self.export_deadlines) + }); + session.update_texture_size( + &self.constants.device, + incoming.uniforms.output_size.0, + incoming.uniforms.output_size.1, + ); + let compositor = self + .transition_compositor + .get_or_insert_with(|| TransitionCompositor::new(&self.constants.device)); + compositor.ensure_size( + &self.constants.device, + incoming.uniforms.output_size.0, + incoming.uniforms.output_size.1, + ); + + let encoder = produce_transition_texture( + self.constants, + &outgoing, + &incoming, + (kind, progress), + layers, + session, + compositor, + ) + .await?; + + match finish_encoder_bgra_readback( + session, + readback, + &self.constants.device, + &self.constants.queue, + &incoming.uniforms, + encoder, + ) + .await + { + Ok(frame) => return Ok(frame), + Err(RenderingError::BufferMapWaitingFailed) => { + last_error = Some(RenderingError::BufferMapWaitingFailed); + } + Err(RenderingError::BufferMapFailed(error)) => { + last_error = Some(RenderingError::BufferMapFailed(error)); + } + Err(error) => return Err(error), + } + } + + Err(last_error.unwrap_or(RenderingError::BufferMapWaitingFailed)) + } + + /// [`Self::render_transition_immediate`], read back as tightly packed BGRA. + pub async fn render_transition_immediate_bgra( + &mut self, + outgoing: TransitionRenderInput<'_>, + incoming: TransitionRenderInput<'_>, + kind: ClipTransitionType, + progress: f32, + layers: &mut RendererLayers, + ) -> Result { + if let Some(frame) = self + .render_transition_bgra(outgoing, incoming, kind, progress, layers) + .await? + { + return Ok(frame); + } + + self.flush_pipeline_bgra() + .await + .unwrap_or(Err(RenderingError::BufferMapWaitingFailed)) + } + pub async fn flush_pipeline(&mut self) -> Option> { if let Some(session) = &mut self.session { flush_pending_readback(session, &self.constants.device).await @@ -7659,6 +7887,11 @@ impl<'a> FrameRenderer<'a> { } } + pub async fn flush_pipeline_bgra(&mut self) -> Option> { + let readback = self.bgra_readback.as_mut()?; + flush_pending_bgra_readback(readback, &self.constants.device, &self.constants.queue).await + } + pub async fn render_immediate_nv12( &mut self, segment_frames: DecodedSegmentFrames, @@ -9126,15 +9359,15 @@ impl LayerPasses<'_> { } } -async fn produce_frame_with_timings( +async fn encode_frame_with_timings( constants: &RenderVideoConstants, - segment_frames: DecodedSegmentFrames, - uniforms: ProjectUniforms, + segment_frames: &DecodedSegmentFrames, + uniforms: &ProjectUniforms, cursor: &CursorEvents, render_display: bool, layers: &mut RendererLayers, session: &mut RenderSession, -) -> Result<(Option, FrameRenderStageTimings), RenderingError> { +) -> Result<(wgpu::CommandEncoder, FrameRenderStageTimings), RenderingError> { let mut encoder = constants.device.create_command_encoder( &(wgpu::CommandEncoderDescriptor { label: Some("Render Encoder"), @@ -9145,8 +9378,8 @@ async fn produce_frame_with_timings( let mut timings = layers .prepare_with_encoder_timed( constants, - &uniforms, - &segment_frames, + uniforms, + segment_frames, cursor, &mut encoder, render_display, @@ -9160,11 +9393,34 @@ async fn produce_frame_with_timings( &constants.queue, &mut encoder, session, - &uniforms, + uniforms, render_display, )?; timings.layer_render_duration = layer_render_start.elapsed(); + Ok((encoder, timings)) +} + +async fn produce_frame_with_timings( + constants: &RenderVideoConstants, + segment_frames: DecodedSegmentFrames, + uniforms: ProjectUniforms, + cursor: &CursorEvents, + render_display: bool, + layers: &mut RendererLayers, + session: &mut RenderSession, +) -> Result<(Option, FrameRenderStageTimings), RenderingError> { + let (encoder, mut timings) = encode_frame_with_timings( + constants, + &segment_frames, + &uniforms, + cursor, + render_display, + layers, + session, + ) + .await?; + let finish_start = Instant::now(); let (frame, finish_timings) = finish_encoder_timed( session, diff --git a/crates/verify/tests/codec.rs b/crates/verify/tests/codec.rs index d2efa9de025..bbff4a6afe8 100644 --- a/crates/verify/tests/codec.rs +++ b/crates/verify/tests/codec.rs @@ -215,7 +215,11 @@ fn video_code_survives_compositing_scaling_rounding_bubble_and_blur() { let dir = tempfile::tempdir().unwrap(); // 640x360 is a typical verification output; 320x180 puts the screen at // a quarter of its source size and the camera code cells at ~6 px. - for (out_w, out_h, bitrate) in [(640, 360, 150_000), (320, 180, 80_000)] { + // 320x180 is encoded at the Web export preset (0.08 bpp): at 80 kbps the + // paravirtualised VideoToolbox HEVC encoder on the CI macOS VMs + // occasionally left a blurred camera cell at margin 0.17, below the + // reader's 0.18, while this host's hardware encoder read it at 0.31. + for (out_w, out_h, bitrate) in [(640, 360, 150_000), (320, 180, 138_240)] { for blur in [false, true] { let pairs: Vec<(u32, u32)> = (0..40).map(|i| (i * 3 + 100, i / 2 + 7)).collect(); let frames: Vec = pairs diff --git a/scripts/linux-desktop-harness/arch-up.sh b/scripts/linux-desktop-harness/arch-up.sh index 6a1a8028862..f3a883510d6 100755 --- a/scripts/linux-desktop-harness/arch-up.sh +++ b/scripts/linux-desktop-harness/arch-up.sh @@ -23,13 +23,16 @@ common=(LIBSEAT_BACKEND=seatd XDG_SESSION_TYPE=wayland) case "$desktop" in hyprland) - cat >"$HOME/.config/hypr/hyprland.conf" <"$HOME/.config/hypr/hyprland.lua" <"$HOME/.config/hypr/xdph.conf" < /^(Cap|so\.cap\.Cap)$/.test(w.get_wm_class() || '') && w.showing_on_its_workspace()).forEach(w => w.activate(global.display.get_current_time_roundtrip()))" \ >/dev/null 2>&1 elif [[ -n ${HYPRLAND_INSTANCE_SIGNATURE:-} ]]; then - hyprctl dispatch focuswindow class:Cap >/dev/null 2>&1 + hyprctl dispatch "hl.dsp.focus({ window = 'class:Cap' })" >/dev/null 2>&1 fi } for _ in $(seq 60); do