From 800e0581d90d0e5bceb9d312c98a31ab8010260d Mon Sep 17 00:00:00 2001 From: Aatman Jain Date: Thu, 29 Oct 2026 23:20:46 +0000 Subject: [PATCH 1/8] perf(recording): reduce Windows CPU copies and fix studio fps cap Use capture fps for non-fragmented studio muxing, reuse NV12 readback buffers, downscale camera preview on Win/Linux, deepen D3D capture pools, and feed UYVY camera frames through without swizzle copies. --- apps/desktop-gpui/src/feeds.rs | 2 +- crates/recording/src/capture_pipeline.rs | 3 +- crates/recording/src/output_pipeline/win.rs | 258 +++--------------- .../src/sources/screen_capture/windows.rs | 6 + crates/scap-direct3d/src/lib.rs | 2 +- crates/video-decode/src/media_foundation.rs | 19 +- 6 files changed, 65 insertions(+), 225 deletions(-) diff --git a/apps/desktop-gpui/src/feeds.rs b/apps/desktop-gpui/src/feeds.rs index 14ce0bd60b0..c6764cb6581 100644 --- a/apps/desktop-gpui/src/feeds.rs +++ b/apps/desktop-gpui/src/feeds.rs @@ -562,7 +562,7 @@ fn run_camera_preview_worker(config: CameraPreviewWorkerConfig) { .publisher .fail("Requested camera blur is unavailable".to_string()); } - let max_dims = blur_mode.map(|_| crate::camera_blur_portable::MAX_DIMS); + let max_dims = Some(crate::camera_blur_portable::MAX_DIMS); let Some((mut image, dims)) = camera_preview_image(&frame.inner, &mut scaler, requested_mirror, max_dims) else { diff --git a/crates/recording/src/capture_pipeline.rs b/crates/recording/src/capture_pipeline.rs index d935e0e30f2..1525fc7fbc8 100644 --- a/crates/recording/src/capture_pipeline.rs +++ b/crates/recording/src/capture_pipeline.rs @@ -325,6 +325,7 @@ impl MakeCapturePipeline for screen_capture::Direct3DCapture { } } else { let d3d_device = screen_capture.d3d_device.clone(); + let frame_rate = screen_capture.capture_frame_rate(); let bitrate_multiplier = if ultra { 0.3f32 } else { 0.15f32 }; OutputPipeline::builder(output_path.clone()) @@ -335,7 +336,7 @@ impl MakeCapturePipeline for screen_capture::Direct3DCapture { pixel_format: screen_capture::Direct3DCapture::PIXEL_FORMAT.as_dxgi(), d3d_device, bitrate_multiplier, - frame_rate: 30u32, + frame_rate, output_size: output_size.map(|(w, h)| windows::Graphics::SizeInt32 { Width: w as i32, Height: h as i32, diff --git a/crates/recording/src/output_pipeline/win.rs b/crates/recording/src/output_pipeline/win.rs index faab33d6e76..814443b6e65 100644 --- a/crates/recording/src/output_pipeline/win.rs +++ b/crates/recording/src/output_pipeline/win.rs @@ -1096,25 +1096,16 @@ impl AudioMuxer for WindowsCameraMuxer { } pub struct CameraBuffers { - uyvy_buffer: Vec, flip_buffer: Vec, } impl CameraBuffers { pub fn new() -> Self { Self { - uyvy_buffer: Vec::new(), flip_buffer: Vec::new(), } } - fn ensure_uyvy_capacity(&mut self, size: usize) -> &mut [u8] { - if self.uyvy_buffer.len() < size { - self.uyvy_buffer.resize(size, 0); - } - &mut self.uyvy_buffer[..size] - } - fn ensure_flip_capacity(&mut self, size: usize) -> &mut [u8] { if self.flip_buffer.len() < size { self.flip_buffer.resize(size, 0); @@ -1259,94 +1250,6 @@ fn compact_primary_stride(pixel_format: cap_camera_windows::PixelFormat, width: } } -fn convert_uyvy_row_to_yuyv_into(src: &[u8], dst: &mut [u8]) { - let src_len = src.len().min(dst.len()); - - #[cfg(target_arch = "x86_64")] - { - if is_x86_feature_detected!("ssse3") { - unsafe { - convert_uyvy_to_yuyv_ssse3(src, dst, src_len); - } - return; - } - } - - convert_uyvy_to_yuyv_scalar(src, dst, src_len); -} - -fn convert_uyvy_to_yuyv_into( - src: &[u8], - dst: &mut [u8], - width: u32, - height: u32, - src_stride: usize, - flip: bool, -) { - let row_bytes = (width * 2) as usize; - let height = height as usize; - - for row in 0..height { - let src_row = if flip { height - 1 - row } else { row }; - let src_start = src_row.saturating_mul(src_stride); - let dst_start = row.saturating_mul(row_bytes); - let src_end = src_start.saturating_add(row_bytes).min(src.len()); - let dst_end = dst_start.saturating_add(row_bytes).min(dst.len()); - - if src_end <= src_start || dst_end <= dst_start || dst_end - dst_start < row_bytes { - continue; - } - - convert_uyvy_row_to_yuyv_into(&src[src_start..src_end], &mut dst[dst_start..dst_end]); - } -} - -#[cfg(target_arch = "x86_64")] -#[target_feature(enable = "ssse3")] -unsafe fn convert_uyvy_to_yuyv_ssse3(src: &[u8], dst: &mut [u8], len: usize) { - use std::arch::x86_64::*; - - unsafe { - let shuffle_mask = _mm_setr_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); - - let mut i = 0; - let simd_end = len & !15; - - while i < simd_end { - let chunk = _mm_loadu_si128(src.as_ptr().add(i) as *const __m128i); - let shuffled = _mm_shuffle_epi8(chunk, shuffle_mask); - _mm_storeu_si128(dst.as_mut_ptr().add(i) as *mut __m128i, shuffled); - i += 16; - } - - convert_uyvy_to_yuyv_scalar(&src[i..], &mut dst[i..], len - i); - } -} - -fn convert_uyvy_to_yuyv_scalar(src: &[u8], dst: &mut [u8], len: usize) { - for i in (0..len).step_by(4) { - if i + 3 < src.len() && i + 3 < dst.len() { - dst[i] = src[i + 1]; - dst[i + 1] = src[i]; - dst[i + 2] = src[i + 3]; - dst[i + 3] = src[i + 2]; - } - } -} - -fn convert_uyvy_to_yuyv( - src: &[u8], - width: u32, - height: u32, - src_stride: usize, - flip: bool, -) -> Vec { - let total_bytes = (width * height * 2) as usize; - let mut dst = vec![0u8; total_bytes]; - convert_uyvy_to_yuyv_into(src, &mut dst, width, height, src_stride, flip); - dst -} - pub(crate) fn camera_frame_to_ffmpeg( frame: &NativeCameraFrame, mjpeg_decoder: &mut MjpegDecoder, @@ -1393,37 +1296,19 @@ pub(crate) fn camera_frame_to_ffmpeg( ) })?; - let converted_data_storage; - let (final_data, final_format): (&[u8], ffmpeg::format::Pixel) = - if frame.pixel_format == cap_camera_windows::PixelFormat::UYVY422 { - converted_data_storage = convert_uyvy_to_yuyv( - data, - frame.width, - frame.height, - layout.primary_stride, - frame.is_bottom_up, - ); - ( - converted_data_storage.as_slice(), - ffmpeg::format::Pixel::YUYV422, - ) - } else { - (data, ffmpeg_format) - }; - - let mut ffmpeg_frame = ffmpeg::frame::Video::new(final_format, frame.width, frame.height); - let flip = frame.is_bottom_up && frame.pixel_format != cap_camera_windows::PixelFormat::UYVY422; + let mut ffmpeg_frame = ffmpeg::frame::Video::new(ffmpeg_format, frame.width, frame.height); + let flip = frame.is_bottom_up; match frame.pixel_format { cap_camera_windows::PixelFormat::NV12 => { let uv_height = height / 2; let y_size = layout.primary_stride * height; let uv_size = layout.secondary_stride * uv_height; - if final_data.len() >= y_size + uv_size { + if data.len() >= y_size + uv_size { let stride_y = ffmpeg_frame.stride(0); let stride_uv = ffmpeg_frame.stride(1); copy_plane( - &final_data[..y_size], + &data[..y_size], ffmpeg_frame.data_mut(0), width, height, @@ -1432,7 +1317,7 @@ pub(crate) fn camera_frame_to_ffmpeg( flip, ); copy_plane( - &final_data[y_size..y_size + uv_size], + &data[y_size..y_size + uv_size], ffmpeg_frame.data_mut(1), width, uv_height, @@ -1446,11 +1331,11 @@ pub(crate) fn camera_frame_to_ffmpeg( let uv_height = height / 2; let y_size = layout.primary_stride * height; let uv_size = layout.secondary_stride * uv_height; - if final_data.len() >= y_size + uv_size { + if data.len() >= y_size + uv_size { let stride_y = ffmpeg_frame.stride(0); let stride_uv = ffmpeg_frame.stride(1); copy_plane( - &final_data[..y_size], + &data[..y_size], ffmpeg_frame.data_mut(0), width, height, @@ -1458,7 +1343,7 @@ pub(crate) fn camera_frame_to_ffmpeg( stride_y, flip, ); - let uv_data = &final_data[y_size..y_size + uv_size]; + let uv_data = &data[y_size..y_size + uv_size]; let dest = ffmpeg_frame.data_mut(1); for row in 0..uv_height { let src_row = if flip { uv_height - 1 - row } else { row }; @@ -1480,10 +1365,10 @@ pub(crate) fn camera_frame_to_ffmpeg( } else { layout.primary_stride * height }; - if final_data.len() >= size { + if data.len() >= size { let stride = ffmpeg_frame.stride(0); copy_plane( - final_data, + data, ffmpeg_frame.data_mut(0), row_bytes, height, @@ -1500,10 +1385,10 @@ pub(crate) fn camera_frame_to_ffmpeg( cap_camera_windows::PixelFormat::ARGB | cap_camera_windows::PixelFormat::RGB32 => { let row_bytes = width * 4; let size = layout.primary_stride * height; - if final_data.len() >= size { + if data.len() >= size { let stride = ffmpeg_frame.stride(0); copy_plane( - final_data, + data, ffmpeg_frame.data_mut(0), row_bytes, height, @@ -1516,10 +1401,10 @@ pub(crate) fn camera_frame_to_ffmpeg( cap_camera_windows::PixelFormat::RGB24 | cap_camera_windows::PixelFormat::BGR24 => { let row_bytes = width * 3; let size = layout.primary_stride * height; - if final_data.len() >= size { + if data.len() >= size { let stride = ffmpeg_frame.stride(0); copy_plane( - final_data, + data, ffmpeg_frame.data_mut(0), row_bytes, height, @@ -1533,12 +1418,12 @@ pub(crate) fn camera_frame_to_ffmpeg( let uv_height = height / 2; let y_size = layout.primary_stride * height; let uv_size = layout.secondary_stride * uv_height; - if final_data.len() >= y_size + uv_size * 2 { + if data.len() >= y_size + uv_size * 2 { let stride_y = ffmpeg_frame.stride(0); let stride_u = ffmpeg_frame.stride(1); let stride_v = ffmpeg_frame.stride(2); copy_plane( - &final_data[..y_size], + &data[..y_size], ffmpeg_frame.data_mut(0), width, height, @@ -1547,7 +1432,7 @@ pub(crate) fn camera_frame_to_ffmpeg( flip, ); copy_plane( - &final_data[y_size..y_size + uv_size], + &data[y_size..y_size + uv_size], ffmpeg_frame.data_mut(1), width / 2, uv_height, @@ -1556,7 +1441,7 @@ pub(crate) fn camera_frame_to_ffmpeg( flip, ); copy_plane( - &final_data[y_size + uv_size..y_size + uv_size * 2], + &data[y_size + uv_size..y_size + uv_size * 2], ffmpeg_frame.data_mut(2), width / 2, uv_height, @@ -1570,12 +1455,12 @@ pub(crate) fn camera_frame_to_ffmpeg( let uv_height = height / 2; let y_size = layout.primary_stride * height; let uv_size = layout.secondary_stride * uv_height; - if final_data.len() >= y_size + uv_size * 2 { + if data.len() >= y_size + uv_size * 2 { let stride_y = ffmpeg_frame.stride(0); let stride_u = ffmpeg_frame.stride(1); let stride_v = ffmpeg_frame.stride(2); copy_plane( - &final_data[..y_size], + &data[..y_size], ffmpeg_frame.data_mut(0), width, height, @@ -1584,7 +1469,7 @@ pub(crate) fn camera_frame_to_ffmpeg( flip, ); copy_plane( - &final_data[y_size + uv_size..y_size + uv_size * 2], + &data[y_size + uv_size..y_size + uv_size * 2], ffmpeg_frame.data_mut(1), width / 2, uv_height, @@ -1593,7 +1478,7 @@ pub(crate) fn camera_frame_to_ffmpeg( flip, ); copy_plane( - &final_data[y_size..y_size + uv_size], + &data[y_size..y_size + uv_size], ffmpeg_frame.data_mut(2), width / 2, uv_height, @@ -1842,88 +1727,27 @@ pub fn upload_mf_buffer_to_texture( ) })?; - let needs_uyvy_conversion = frame.pixel_format == cap_camera_windows::PixelFormat::UYVY422; - let needs_flip = frame.is_bottom_up; let mut row_pitch = layout.primary_stride as u32; - let data: &[u8] = match (needs_uyvy_conversion, needs_flip) { - (false, false) => original_data, - (true, false) => { - let uyvy_size = (frame.width * frame.height * 2) as usize; - let dst = buffers.ensure_uyvy_capacity(uyvy_size); - convert_uyvy_to_yuyv_into( - original_data, - dst, - frame.width, - frame.height, - layout.primary_stride, - false, - ); - row_pitch = compact_primary_stride( - cap_camera_windows::PixelFormat::YUYV422, - frame.width as usize, - ) as u32; - dst - } - (false, true) => { - let flip_size = flip_buffer_size( - frame.width as usize, - frame.height as usize, - frame.pixel_format, - ); - let dst = buffers.ensure_flip_capacity(flip_size); - flip_camera_buffer_into( - original_data, - dst, - frame.width as usize, - frame.height as usize, - frame.pixel_format, - layout, - ); - row_pitch = compact_primary_stride(frame.pixel_format, frame.width as usize) as u32; - dst - } - (true, true) => { - let uyvy_size = (frame.width * frame.height * 2) as usize; - let flip_size = flip_buffer_size( - frame.width as usize, - frame.height as usize, - frame.pixel_format, - ); - - if buffers.uyvy_buffer.len() < uyvy_size { - buffers.uyvy_buffer.resize(uyvy_size, 0); - } - if buffers.flip_buffer.len() < flip_size { - buffers.flip_buffer.resize(flip_size, 0); - } - - convert_uyvy_to_yuyv_into( - original_data, - &mut buffers.uyvy_buffer[..uyvy_size], - frame.width, - frame.height, - layout.primary_stride, - false, - ); - - flip_camera_buffer_into( - &buffers.uyvy_buffer[..uyvy_size], - &mut buffers.flip_buffer[..flip_size], - frame.width as usize, - frame.height as usize, - cap_camera_windows::PixelFormat::YUYV422, - CameraBufferLayout { - primary_stride: frame.width as usize * 2, - secondary_stride: frame.width as usize * 2, - }, - ); - row_pitch = compact_primary_stride( - cap_camera_windows::PixelFormat::YUYV422, - frame.width as usize, - ) as u32; - &buffers.flip_buffer[..flip_size] - } + let data: &[u8] = if frame.is_bottom_up { + let flip_size = flip_buffer_size( + frame.width as usize, + frame.height as usize, + frame.pixel_format, + ); + let dst = buffers.ensure_flip_capacity(flip_size); + flip_camera_buffer_into( + original_data, + dst, + frame.width as usize, + frame.height as usize, + frame.pixel_format, + layout, + ); + row_pitch = compact_primary_stride(frame.pixel_format, frame.width as usize) as u32; + dst + } else { + original_data }; let texture_desc = D3D11_TEXTURE2D_DESC { diff --git a/crates/recording/src/sources/screen_capture/windows.rs b/crates/recording/src/sources/screen_capture/windows.rs index ef8a9dbd343..60c7c7cca6a 100644 --- a/crates/recording/src/sources/screen_capture/windows.rs +++ b/crates/recording/src/sources/screen_capture/windows.rs @@ -466,6 +466,12 @@ pub struct VideoSourceConfig { settings: scap_direct3d::Settings, pub d3d_device: ID3D11Device, } + +impl VideoSourceConfig { + pub(crate) fn capture_frame_rate(&self) -> u32 { + self.video_info.fps().max(1) + } +} pub struct VideoSource { video_info: VideoInfo, ctrl_tx: std::sync::mpsc::SyncSender, diff --git a/crates/scap-direct3d/src/lib.rs b/crates/scap-direct3d/src/lib.rs index 92e37f9d342..3f79ae851b4 100644 --- a/crates/scap-direct3d/src/lib.rs +++ b/crates/scap-direct3d/src/lib.rs @@ -582,7 +582,7 @@ impl Capturer { let frame_pool_size = settings .fps - .map(|fps| ((fps as f32 / 30.0 * 2.0).ceil() as i32).clamp(2, 4)) + .map(|fps| ((fps as f32 / 30.0 * 2.0).ceil() as i32).clamp(4, 6)) .unwrap_or(2); let item_size = item.Size().map_err(NewCapturerError::ItemSize)?; diff --git a/crates/video-decode/src/media_foundation.rs b/crates/video-decode/src/media_foundation.rs index 5b4de1c8067..4ab85401a72 100644 --- a/crates/video-decode/src/media_foundation.rs +++ b/crates/video-decode/src/media_foundation.rs @@ -600,6 +600,7 @@ pub struct MediaFoundationDecoder { // How the stream stopped, reported once the queued readbacks are drained. readback_end: Option>, frame_pool: FramePool, + nv12_byte_pool: Vec>, plane_converter: Nv12PlaneConverter, capabilities: MFDecoderCapabilities, // COM can unload the decoder DLL, so release every COM field before its apartment shuts down. @@ -692,6 +693,7 @@ impl MediaFoundationDecoder { readback_queue: VecDeque::new(), readback_end: None, frame_pool: FramePool::new(), + nv12_byte_pool: Vec::with_capacity(READBACK_DEPTH), plane_converter, capabilities, _init_guard: guard, @@ -759,7 +761,7 @@ impl MediaFoundationDecoder { Ok(Some(NV12Sample { pts, data: data? })) } - unsafe fn map_staging(&self, staging: &ID3D11Texture2D) -> Result { + unsafe fn map_staging(&mut self, staging: &ID3D11Texture2D) -> Result { let mut mapped = D3D11_MAPPED_SUBRESOURCE::default(); unsafe { self.d3d11_context @@ -773,11 +775,18 @@ impl MediaFoundationDecoder { let y_size = (y_stride * y_height) as usize; let uv_size = (y_stride * uv_height) as usize; - let total_size = y_size + uv_size; - - let mut data = vec![0u8; total_size]; + let copy_len = y_size + uv_size; + + let mut data = self + .nv12_byte_pool + .pop() + .unwrap_or_else(|| Vec::with_capacity(copy_len)); + if data.capacity() < copy_len { + data.reserve(copy_len - data.capacity()); + } unsafe { - std::ptr::copy_nonoverlapping(mapped.pData as *const u8, data.as_mut_ptr(), total_size); + data.set_len(copy_len); + std::ptr::copy_nonoverlapping(mapped.pData as *const u8, data.as_mut_ptr(), copy_len); self.d3d11_context.Unmap(staging, 0); } From cbf38f7d9328f4c0ff6eb67babf1d474e397a11d Mon Sep 17 00:00:00 2001 From: Aatman Jain Date: Thu, 29 Oct 2026 23:53:56 +0000 Subject: [PATCH 2/8] fix(recording): restore UYVY conversion for D3D camera upload Re-swizzle UYVY to YUY2 for Media Foundation texture uploads while keeping FFmpeg on native UYVY with padded stride handling. --- crates/recording/src/output_pipeline/win.rs | 195 +++++++++++++++++--- 1 file changed, 166 insertions(+), 29 deletions(-) diff --git a/crates/recording/src/output_pipeline/win.rs b/crates/recording/src/output_pipeline/win.rs index 814443b6e65..a0d741daabb 100644 --- a/crates/recording/src/output_pipeline/win.rs +++ b/crates/recording/src/output_pipeline/win.rs @@ -1096,16 +1096,25 @@ impl AudioMuxer for WindowsCameraMuxer { } pub struct CameraBuffers { + uyvy_buffer: Vec, flip_buffer: Vec, } impl CameraBuffers { pub fn new() -> Self { Self { + uyvy_buffer: Vec::new(), flip_buffer: Vec::new(), } } + fn ensure_uyvy_capacity(&mut self, size: usize) -> &mut [u8] { + if self.uyvy_buffer.len() < size { + self.uyvy_buffer.resize(size, 0); + } + &mut self.uyvy_buffer[..size] + } + fn ensure_flip_capacity(&mut self, size: usize) -> &mut [u8] { if self.flip_buffer.len() < size { self.flip_buffer.resize(size, 0); @@ -1250,6 +1259,81 @@ fn compact_primary_stride(pixel_format: cap_camera_windows::PixelFormat, width: } } +fn convert_uyvy_row_to_yuyv_into(src: &[u8], dst: &mut [u8]) { + let src_len = src.len().min(dst.len()); + + #[cfg(target_arch = "x86_64")] + { + if is_x86_feature_detected!("ssse3") { + unsafe { + convert_uyvy_to_yuyv_ssse3(src, dst, src_len); + } + return; + } + } + + convert_uyvy_to_yuyv_scalar(src, dst, src_len); +} + +fn convert_uyvy_to_yuyv_into( + src: &[u8], + dst: &mut [u8], + width: u32, + height: u32, + src_stride: usize, + flip: bool, +) { + let row_bytes = (width * 2) as usize; + let height = height as usize; + + for row in 0..height { + let src_row = if flip { height - 1 - row } else { row }; + let src_start = src_row.saturating_mul(src_stride); + let dst_start = row.saturating_mul(row_bytes); + let src_end = src_start.saturating_add(row_bytes).min(src.len()); + let dst_end = dst_start.saturating_add(row_bytes).min(dst.len()); + + if src_end <= src_start || dst_end <= dst_start || dst_end - dst_start < row_bytes { + continue; + } + + convert_uyvy_row_to_yuyv_into(&src[src_start..src_end], &mut dst[dst_start..dst_end]); + } +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "ssse3")] +unsafe fn convert_uyvy_to_yuyv_ssse3(src: &[u8], dst: &mut [u8], len: usize) { + use std::arch::x86_64::*; + + unsafe { + let shuffle_mask = _mm_setr_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); + + let mut i = 0; + let simd_end = len & !15; + + while i < simd_end { + let chunk = _mm_loadu_si128(src.as_ptr().add(i) as *const __m128i); + let shuffled = _mm_shuffle_epi8(chunk, shuffle_mask); + _mm_storeu_si128(dst.as_mut_ptr().add(i) as *mut __m128i, shuffled); + i += 16; + } + + convert_uyvy_to_yuyv_scalar(&src[i..], &mut dst[i..], len - i); + } +} + +fn convert_uyvy_to_yuyv_scalar(src: &[u8], dst: &mut [u8], len: usize) { + for i in (0..len).step_by(4) { + if i + 3 < src.len() && i + 3 < dst.len() { + dst[i] = src[i + 1]; + dst[i + 1] = src[i]; + dst[i + 2] = src[i + 3]; + dst[i + 3] = src[i + 2]; + } + } +} + pub(crate) fn camera_frame_to_ffmpeg( frame: &NativeCameraFrame, mjpeg_decoder: &mut MjpegDecoder, @@ -1360,11 +1444,7 @@ pub(crate) fn camera_frame_to_ffmpeg( } cap_camera_windows::PixelFormat::YUYV422 | cap_camera_windows::PixelFormat::UYVY422 => { let row_bytes = width * 2; - let size = if frame.pixel_format == cap_camera_windows::PixelFormat::UYVY422 { - row_bytes * height - } else { - layout.primary_stride * height - }; + let size = layout.primary_stride * height; if data.len() >= size { let stride = ffmpeg_frame.stride(0); copy_plane( @@ -1372,11 +1452,7 @@ pub(crate) fn camera_frame_to_ffmpeg( ffmpeg_frame.data_mut(0), row_bytes, height, - if frame.pixel_format == cap_camera_windows::PixelFormat::UYVY422 { - row_bytes - } else { - layout.primary_stride - }, + layout.primary_stride, stride, flip, ); @@ -1727,27 +1803,88 @@ pub fn upload_mf_buffer_to_texture( ) })?; + let needs_uyvy_conversion = frame.pixel_format == cap_camera_windows::PixelFormat::UYVY422; + let needs_flip = frame.is_bottom_up; let mut row_pitch = layout.primary_stride as u32; - let data: &[u8] = if frame.is_bottom_up { - let flip_size = flip_buffer_size( - frame.width as usize, - frame.height as usize, - frame.pixel_format, - ); - let dst = buffers.ensure_flip_capacity(flip_size); - flip_camera_buffer_into( - original_data, - dst, - frame.width as usize, - frame.height as usize, - frame.pixel_format, - layout, - ); - row_pitch = compact_primary_stride(frame.pixel_format, frame.width as usize) as u32; - dst - } else { - original_data + let data: &[u8] = match (needs_uyvy_conversion, needs_flip) { + (false, false) => original_data, + (true, false) => { + let uyvy_size = (frame.width * frame.height * 2) as usize; + let dst = buffers.ensure_uyvy_capacity(uyvy_size); + convert_uyvy_to_yuyv_into( + original_data, + dst, + frame.width, + frame.height, + layout.primary_stride, + false, + ); + row_pitch = compact_primary_stride( + cap_camera_windows::PixelFormat::YUYV422, + frame.width as usize, + ) as u32; + dst + } + (false, true) => { + let flip_size = flip_buffer_size( + frame.width as usize, + frame.height as usize, + frame.pixel_format, + ); + let dst = buffers.ensure_flip_capacity(flip_size); + flip_camera_buffer_into( + original_data, + dst, + frame.width as usize, + frame.height as usize, + frame.pixel_format, + layout, + ); + row_pitch = compact_primary_stride(frame.pixel_format, frame.width as usize) as u32; + dst + } + (true, true) => { + let uyvy_size = (frame.width * frame.height * 2) as usize; + let flip_size = flip_buffer_size( + frame.width as usize, + frame.height as usize, + frame.pixel_format, + ); + + if buffers.uyvy_buffer.len() < uyvy_size { + buffers.uyvy_buffer.resize(uyvy_size, 0); + } + if buffers.flip_buffer.len() < flip_size { + buffers.flip_buffer.resize(flip_size, 0); + } + + convert_uyvy_to_yuyv_into( + original_data, + &mut buffers.uyvy_buffer[..uyvy_size], + frame.width, + frame.height, + layout.primary_stride, + false, + ); + + flip_camera_buffer_into( + &buffers.uyvy_buffer[..uyvy_size], + &mut buffers.flip_buffer[..flip_size], + frame.width as usize, + frame.height as usize, + cap_camera_windows::PixelFormat::YUYV422, + CameraBufferLayout { + primary_stride: frame.width as usize * 2, + secondary_stride: frame.width as usize * 2, + }, + ); + row_pitch = compact_primary_stride( + cap_camera_windows::PixelFormat::YUYV422, + frame.width as usize, + ) as u32; + &buffers.flip_buffer[..flip_size] + } }; let texture_desc = D3D11_TEXTURE2D_DESC { From 564cb04a0c3cde01dbdc4bee519465b5bb8e3181 Mon Sep 17 00:00:00 2001 From: Aatman Jain Date: Fri, 30 Oct 2026 00:00:10 +0000 Subject: [PATCH 3/8] fix(recording): address Greptile camera and decode feedback Cap preview downscale to blur bubble only, match fragmented camera muxer to UYVY422 frames, and drop the non-functional NV12 byte pool. --- apps/desktop-gpui/src/feeds.rs | 2 +- .../src/output_pipeline/win_fragmented_m4s.rs | 2 +- crates/video-decode/src/media_foundation.rs | 10 +--------- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/apps/desktop-gpui/src/feeds.rs b/apps/desktop-gpui/src/feeds.rs index c6764cb6581..14ce0bd60b0 100644 --- a/apps/desktop-gpui/src/feeds.rs +++ b/apps/desktop-gpui/src/feeds.rs @@ -562,7 +562,7 @@ fn run_camera_preview_worker(config: CameraPreviewWorkerConfig) { .publisher .fail("Requested camera blur is unavailable".to_string()); } - let max_dims = Some(crate::camera_blur_portable::MAX_DIMS); + let max_dims = blur_mode.map(|_| crate::camera_blur_portable::MAX_DIMS); let Some((mut image, dims)) = camera_preview_image(&frame.inner, &mut scaler, requested_mirror, max_dims) else { diff --git a/crates/recording/src/output_pipeline/win_fragmented_m4s.rs b/crates/recording/src/output_pipeline/win_fragmented_m4s.rs index b517b575903..e0e78d6646a 100644 --- a/crates/recording/src/output_pipeline/win_fragmented_m4s.rs +++ b/crates/recording/src/output_pipeline/win_fragmented_m4s.rs @@ -737,7 +737,7 @@ impl WindowsFragmentedM4SCameraMuxer { let mut updated = false; let desired_pixel_format = match frame.pixel_format { - cap_camera_windows::PixelFormat::UYVY422 => Pixel::YUYV422, + cap_camera_windows::PixelFormat::UYVY422 => Pixel::UYVY422, _ => self.video_config.pixel_format, }; diff --git a/crates/video-decode/src/media_foundation.rs b/crates/video-decode/src/media_foundation.rs index 4ab85401a72..53738476d34 100644 --- a/crates/video-decode/src/media_foundation.rs +++ b/crates/video-decode/src/media_foundation.rs @@ -600,7 +600,6 @@ pub struct MediaFoundationDecoder { // How the stream stopped, reported once the queued readbacks are drained. readback_end: Option>, frame_pool: FramePool, - nv12_byte_pool: Vec>, plane_converter: Nv12PlaneConverter, capabilities: MFDecoderCapabilities, // COM can unload the decoder DLL, so release every COM field before its apartment shuts down. @@ -693,7 +692,6 @@ impl MediaFoundationDecoder { readback_queue: VecDeque::new(), readback_end: None, frame_pool: FramePool::new(), - nv12_byte_pool: Vec::with_capacity(READBACK_DEPTH), plane_converter, capabilities, _init_guard: guard, @@ -777,13 +775,7 @@ impl MediaFoundationDecoder { let uv_size = (y_stride * uv_height) as usize; let copy_len = y_size + uv_size; - let mut data = self - .nv12_byte_pool - .pop() - .unwrap_or_else(|| Vec::with_capacity(copy_len)); - if data.capacity() < copy_len { - data.reserve(copy_len - data.capacity()); - } + let mut data = Vec::with_capacity(copy_len); unsafe { data.set_len(copy_len); std::ptr::copy_nonoverlapping(mapped.pData as *const u8, data.as_mut_ptr(), copy_len); From a4c0fbf3947fc416df84cc64f8dff1bd153ff422 Mon Sep 17 00:00:00 2001 From: Aatman Jain Date: Fri, 30 Oct 2026 00:19:46 +0000 Subject: [PATCH 4/8] fix(recording): copy decoder readback without set_len on a fresh Vec clippy::uninit_vec (deny) rejects with_capacity + set_len in the Media Foundation NV12 readback, which failed the Windows clippy job in cap-video-decode before cap-recording was reached. Copy the mapped bytes with to_vec (allocation plus memcpy, no zero fill) and take map_staging back to &self now that the byte pool is gone. --- crates/video-decode/src/media_foundation.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/crates/video-decode/src/media_foundation.rs b/crates/video-decode/src/media_foundation.rs index 53738476d34..592480562c0 100644 --- a/crates/video-decode/src/media_foundation.rs +++ b/crates/video-decode/src/media_foundation.rs @@ -759,7 +759,7 @@ impl MediaFoundationDecoder { Ok(Some(NV12Sample { pts, data: data? })) } - unsafe fn map_staging(&mut self, staging: &ID3D11Texture2D) -> Result { + unsafe fn map_staging(&self, staging: &ID3D11Texture2D) -> Result { let mut mapped = D3D11_MAPPED_SUBRESOURCE::default(); unsafe { self.d3d11_context @@ -773,14 +773,13 @@ impl MediaFoundationDecoder { let y_size = (y_stride * y_height) as usize; let uv_size = (y_stride * uv_height) as usize; - let copy_len = y_size + uv_size; + let total_size = y_size + uv_size; - let mut data = Vec::with_capacity(copy_len); - unsafe { - data.set_len(copy_len); - std::ptr::copy_nonoverlapping(mapped.pData as *const u8, data.as_mut_ptr(), copy_len); + let data = unsafe { + let data = std::slice::from_raw_parts(mapped.pData as *const u8, total_size).to_vec(); self.d3d11_context.Unmap(staging, 0); - } + data + }; Ok(NV12Data { data: Arc::new(data), From 7e6b3c3afdf2a6ac870330e8c6e6a46125554670 Mon Sep 17 00:00:00 2001 From: Aatman Jain Date: Fri, 30 Oct 2026 00:19:46 +0000 Subject: [PATCH 5/8] fix(recording): separate capture_frame_rate impl from VideoSource --- crates/recording/src/sources/screen_capture/windows.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/recording/src/sources/screen_capture/windows.rs b/crates/recording/src/sources/screen_capture/windows.rs index 60c7c7cca6a..0ae3f117a91 100644 --- a/crates/recording/src/sources/screen_capture/windows.rs +++ b/crates/recording/src/sources/screen_capture/windows.rs @@ -472,6 +472,7 @@ impl VideoSourceConfig { self.video_info.fps().max(1) } } + pub struct VideoSource { video_info: VideoInfo, ctrl_tx: std::sync::mpsc::SyncSender, From c8eff37ddb8c973deb3246327a00e30b3de3c0cd Mon Sep 17 00:00:00 2001 From: Aatman Jain Date: Fri, 30 Oct 2026 10:23:07 +0000 Subject: [PATCH 6/8] fix(ci): configure Hyprland 0.57 through hyprland.lua in the Linux harness Hyprland 0.57 dropped hyprland.conf and only reads hyprland.lua, so the harness's monitor mode, session export and waybar were ignored and the output came up at vkms's preferred 1024x768. The screenshot scenario then reported screenshot_size=1024x768 instead of yes. Write the same settings as Lua and use the Lua form of the focus dispatcher. --- scripts/linux-desktop-harness/arch-up.sh | 17 ++++++++++------- .../scenario-screenshot.sh | 2 +- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/scripts/linux-desktop-harness/arch-up.sh b/scripts/linux-desktop-harness/arch-up.sh index 6a1a8028862..f3a883510d6 100755 --- a/scripts/linux-desktop-harness/arch-up.sh +++ b/scripts/linux-desktop-harness/arch-up.sh @@ -23,13 +23,16 @@ common=(LIBSEAT_BACKEND=seatd XDG_SESSION_TYPE=wayland) case "$desktop" in hyprland) - cat >"$HOME/.config/hypr/hyprland.conf" <"$HOME/.config/hypr/hyprland.lua" <"$HOME/.config/hypr/xdph.conf" < /^(Cap|so\.cap\.Cap)$/.test(w.get_wm_class() || '') && w.showing_on_its_workspace()).forEach(w => w.activate(global.display.get_current_time_roundtrip()))" \ >/dev/null 2>&1 elif [[ -n ${HYPRLAND_INSTANCE_SIGNATURE:-} ]]; then - hyprctl dispatch focuswindow class:Cap >/dev/null 2>&1 + hyprctl dispatch "hl.dsp.focus({ window = 'class:Cap' })" >/dev/null 2>&1 fi } for _ in $(seq 60); do From 131c4027d2cb898ce7a6be2e625d15574864802f Mon Sep 17 00:00:00 2001 From: Aatman Jain Date: Fri, 30 Oct 2026 10:38:26 +0000 Subject: [PATCH 7/8] fix(verify): encode the 320x180 compositing codec case at the Web preset bitrate At 80 kbps the paravirtualised VideoToolbox HEVC encoder on the CI macOS VMs occasionally left a blurred camera code cell at margin 0.17, below the reader's 0.18 threshold. 138,240 bps is the Web export preset (0.08 bpp), still below the Social preset the verify exports use. --- crates/verify/tests/codec.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/verify/tests/codec.rs b/crates/verify/tests/codec.rs index d2efa9de025..bbff4a6afe8 100644 --- a/crates/verify/tests/codec.rs +++ b/crates/verify/tests/codec.rs @@ -215,7 +215,11 @@ fn video_code_survives_compositing_scaling_rounding_bubble_and_blur() { let dir = tempfile::tempdir().unwrap(); // 640x360 is a typical verification output; 320x180 puts the screen at // a quarter of its source size and the camera code cells at ~6 px. - for (out_w, out_h, bitrate) in [(640, 360, 150_000), (320, 180, 80_000)] { + // 320x180 is encoded at the Web export preset (0.08 bpp): at 80 kbps the + // paravirtualised VideoToolbox HEVC encoder on the CI macOS VMs + // occasionally left a blurred camera cell at margin 0.17, below the + // reader's 0.18, while this host's hardware encoder read it at 0.31. + for (out_w, out_h, bitrate) in [(640, 360, 150_000), (320, 180, 138_240)] { for blur in [false, true] { let pairs: Vec<(u32, u32)> = (0..40).map(|i| (i * 3 + 100, i / 2 + 7)).collect(); let frames: Vec = pairs From 18f03112ca96f135a95e7e555d94fd20f7af8271 Mon Sep 17 00:00:00 2001 From: Aatman Jain Date: Fri, 30 Oct 2026 11:02:49 +0000 Subject: [PATCH 8/8] fix(ci): walk GIF blocks when measuring the exported GIF's length --- .../linux-desktop-harness/scenario-export.sh | 30 +++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/scripts/linux-desktop-harness/scenario-export.sh b/scripts/linux-desktop-harness/scenario-export.sh index d73efcf77af..ec76b57331e 100755 --- a/scripts/linux-desktop-harness/scenario-export.sh +++ b/scripts/linux-desktop-harness/scenario-export.sh @@ -60,12 +60,32 @@ gif_frames_and_seconds() { import sys data = open(sys.argv[1], "rb").read() + + +def skip_sub_blocks(at): + while data[at]: + at += data[at] + 1 + return at + 1 + + if data[:6] in (b"GIF87a", b"GIF89a"): - frames, centiseconds, at = 0, 0, data.find(b"\x21\xf9\x04") - while at != -1 and at + 6 <= len(data): - frames += 1 - centiseconds += data[at + 4] | data[at + 5] << 8 - at = data.find(b"\x21\xf9\x04", at + 8) + frames, centiseconds, delay = 0, 0, 0 + flags = data[10] + at = 13 + (3 << ((flags & 7) + 1) if flags & 0x80 else 0) + while at < len(data) and data[at] != 0x3B: + if data[at] == 0x21: + if data[at + 1] == 0xF9: + delay = data[at + 4] | data[at + 5] << 8 + at = skip_sub_blocks(at + 2) + elif data[at] == 0x2C: + frames += 1 + centiseconds += delay + delay = 0 + local = data[at + 9] + at += 10 + (3 << ((local & 7) + 1) if local & 0x80 else 0) + at = skip_sub_blocks(at + 1) + else: + break print(frames, f"{centiseconds / 100:.2f}") PY }