perf(recording): Windows fragmented M4S readback overlap and GPU scale

.github/workflows/instant-reliability.yml

@@ -316,9 +316,19 @@ jobs:
         run: |
           set -euo pipefail
           sudo apt-get update -qq
-          # v4l2loopback needs videodev, which is in the extra kernel modules.
-          sudo apt-get install -y -qq "linux-modules-extra-$(uname -r)" v4l2loopback-dkms xvfb x11-utils xdotool \
-            dbus ffmpeg pulseaudio pulseaudio-utils libasound2-plugins >/dev/null
+          packages=("linux-headers-$(uname -r)" v4l2loopback-dkms xvfb x11-utils xdotool
+            dbus ffmpeg pulseaudio pulseaudio-utils libasound2-plugins)
+          # v4l2loopback needs videodev. Newer runner kernels ship it in the
+          # base modules package and have no linux-modules-extra package.
+          extra="linux-modules-extra-$(uname -r)"
+          if ! sudo modprobe videodev 2>/dev/null && apt-cache show "$extra" >/dev/null 2>&1; then
+            packages+=("$extra")
+          fi
+          sudo apt-get install -y -qq "${packages[@]}" >/dev/null
+          if ! sudo modprobe videodev; then
+            echo "::error::videodev is not available for kernel $(uname -r), so v4l2loopback cannot load"
+            exit 1
+          fi
           chmod +x "$RUNNER_TEMP/build/cap-gpui" "$RUNNER_TEMP/build/real-app-tests"
           sudo modprobe v4l2loopback video_nr=42 card_label="Cap Test Camera" exclusive_caps=1
           sudo udevadm settle --timeout=10

crates/enc-ffmpeg/src/mux/segmented_stream.rs

@@ -231,6 +231,10 @@ pub struct SegmentedVideoEncoderConfig {
     /// [`SegmentedVideoEncoder::queue_hw_pixel_buffer`]) before falling back
     /// to the software-frame path. Ignored on other platforms.
     pub prefer_videotoolbox_hw_input: bool,
+    /// When set, pixel format conversion and scaling run outside the encoder
+    /// (for example via [`cap_frame_converter::AsyncConverterPool`]) so
+    /// readback and GPU conversion can overlap encoding.
+    pub prefer_external_conversion: bool,
 }
 
 impl Default for SegmentedVideoEncoderConfig {
@@ -241,6 +245,7 @@ impl Default for SegmentedVideoEncoderConfig {
             bpp: H264EncoderBuilder::QUALITY_BPP,
             output_size: None,
             prefer_videotoolbox_hw_input: false,
+            prefer_external_conversion: false,
         }
     }
 }
@@ -333,6 +338,10 @@ impl SegmentedVideoEncoder {
             builder = builder.with_output_size(width, height)?;
         }
 
+        if config.prefer_external_conversion {
+            builder = builder.with_external_conversion();
+        }
+
         #[cfg(target_os = "macos")]
         let hw_attempt = if config.prefer_videotoolbox_hw_input {
             match builder.clone().build_videotoolbox_hw_input(&mut output) {
@@ -480,6 +489,21 @@ impl SegmentedVideoEncoder {
         Ok(())
     }
 
+    pub fn conversion_requirements(&self) -> crate::video::h264::ConversionRequirements {
+        self.encoder.conversion_requirements()
+    }
+
+    pub fn queue_preconverted_frame(
+        &mut self,
+        frame: frame::Video,
+        timestamp: Duration,
+    ) -> Result<(), QueueFrameError> {
+        self.encoder
+            .queue_preconverted_frame_to(frame, timestamp, &mut self.mux)?;
+        self.mux.frame_queued();
+        Ok(())
+    }
+
     pub fn finish(&mut self) -> Result<(), FinishError> {
         let encoder = &mut self.encoder;
         self.mux

crates/enc-ffmpeg/src/video/h264.rs

@@ -1167,9 +1167,18 @@ impl H264Encoder {
 
     pub fn queue_preconverted_frame(
         &mut self,
-        mut frame: frame::Video,
+        frame: frame::Video,
         timestamp: Duration,
         output: &mut format::context::Output,
+    ) -> Result<(), QueueFrameError> {
+        self.queue_preconverted_frame_to(frame, timestamp, output)
+    }
+
+    pub fn queue_preconverted_frame_to<S: PacketSink + ?Sized>(
+        &mut self,
+        mut frame: frame::Video,
+        timestamp: Duration,
+        output: &mut S,
     ) -> Result<(), QueueFrameError> {
         trace!(
             "Encoding pre-converted frame: format={:?}, size={}x{}, expected={:?} {}x{}",

crates/frame-converter/src/d3d11.rs

@@ -533,6 +533,16 @@ impl D3D11Converter {
                 })?
         };
 
+        if is_rgb_format(config.input_format) && !is_rgb_format(config.output_format) {
+            let output_color_space =
+                windows::Win32::Graphics::Direct3D11::D3D11_VIDEO_PROCESSOR_COLOR_SPACE {
+                    _bitfield: YCBCR_MATRIX_BT709,
+                };
+            unsafe {
+                video_context.VideoProcessorSetOutputColorSpace(&processor, &output_color_space);
+            }
+        }
+
         let (input_texture, input_shared_handle) = create_shared_texture(
             &device,
             config.input_width,
@@ -873,6 +883,15 @@ impl FrameConverter for D3D11Converter {
     }
 }
 
+// `D3D11_VIDEO_PROCESSOR_COLOR_SPACE.YCbCr_Matrix` (bit 2): the video processor
+// converts RGB to YCbCr with BT.601 unless told otherwise, while every encoder
+// stream is tagged BT.709.
+const YCBCR_MATRIX_BT709: u32 = 1 << 2;
+
+fn is_rgb_format(pixel: Pixel) -> bool {
+    matches!(pixel, Pixel::BGRA | Pixel::RGBA)
+}
+
 fn pixel_to_dxgi(pixel: Pixel) -> Result<DXGI_FORMAT, ConvertError> {
     match pixel {
         Pixel::NV12 => Ok(DXGI_FORMAT_NV12),

crates/frame-converter/src/lib.rs

@@ -7,6 +7,9 @@ pub use pool::*;
 mod frame_pool;
 pub use frame_pool::*;
 
+mod reorder;
+pub use reorder::*;
+
 #[cfg(target_os = "macos")]
 mod videotoolbox;
 #[cfg(target_os = "macos")]