From 06857b0d0010adcc11044ffe28ed790717557341 Mon Sep 17 00:00:00 2001 From: "Aatman Jain (AJ)" <257399449+AatmanAJ@users.noreply.github.com> Date: Tue, 27 Oct 2026 14:46:20 +0000 Subject: [PATCH 1/4] feat(rendering): loop video and GIF backgrounds behind the recording An image background whose file is a video (mp4, mov, m4v, webm, mkv) or a GIF now plays and loops instead of showing its first frame. Each one is decoded on its own thread with FFmpeg (VideoToolbox on macOS, D3D11VA on Windows, software elsewhere), scaled once to RGBA no larger than the output needs (max 1920px), and read ahead between frames. The start of the clip stays decoded so a loop restarts without a seek, and short clips and most GIFs stay entirely in memory. The GPUI editor's image pane now also accepts videos and shows their first frame as the thumbnail. --- apps/desktop-gpui/src/editor_sidebar.rs | 36 +- .../rendering/src/decoder/frame_converter.rs | 2 +- crates/rendering/src/decoder/mod.rs | 1 + crates/rendering/src/layers/background.rs | 268 ++++++- crates/rendering/src/layers/mod.rs | 2 + .../rendering/src/layers/video_background.rs | 678 ++++++++++++++++++ crates/rendering/src/lib.rs | 3 +- 7 files changed, 951 insertions(+), 39 deletions(-) create mode 100644 crates/rendering/src/layers/video_background.rs diff --git a/apps/desktop-gpui/src/editor_sidebar.rs b/apps/desktop-gpui/src/editor_sidebar.rs index ee004dbae73..79e607f7b1c 100644 --- a/apps/desktop-gpui/src/editor_sidebar.rs +++ b/apps/desktop-gpui/src/editor_sidebar.rs @@ -192,6 +192,11 @@ pub const DEFAULT_BACKGROUND_ROUNDING: f64 = 7.5; pub const DEFAULT_NOISE_SCALE: f32 = 3.; /// `BACKGROUND_IMAGE_EXTENSIONS` (`:250-257`). pub const BACKGROUND_IMAGE_EXTENSIONS: [&str; 6] = ["jpg", "jpeg", "png", "gif", "webp", "bmp"]; +/// What the recording editor's image pane accepts: the images above plus +/// videos, which loop behind the recording. +pub const BACKGROUND_MEDIA_EXTENSIONS: [&str; 10] = [ + "jpg", "jpeg", "png", "gif", "webp", "bmp", "mp4", "mov", "m4v", "webm", +]; /// `CURRENT_DESKTOP_BACKGROUND_BASENAME` (`:350`, and `recording.rs:91`). pub const CURRENT_DESKTOP_BACKGROUND_BASENAME: &str = "current-desktop-background"; /// `recording.rs:92-93`. @@ -1972,6 +1977,10 @@ impl EditorWindow { /// side. The same rule as `library::decode_thumbnail`: the atlas would /// otherwise hold a multi-megapixel original per tile. fn decode_scaled_rgba(path: &Path, max: u32) -> Option { + if is_background_video(path) { + let (pixels, width, height) = cap_rendering::video_background_poster(path, max)?; + return image::RgbaImage::from_raw(width, height, pixels); + } let bytes = std::fs::read(path).ok()?; let format = image::guess_format(&bytes).ok()?; let (decoded, width, height) = if format == image::ImageFormat::Jpeg { @@ -2009,6 +2018,12 @@ fn decode_scaled_rgba(path: &Path, max: u32) -> Option { }) } +/// GIFs decode as images here; only the first frame is needed for a tile. +fn is_background_video(path: &Path) -> bool { + let path = path.to_string_lossy(); + cap_rendering::is_video_background(&path) && !path.to_lowercase().ends_with(".gif") +} + fn decode_jpeg_thumbnail(bytes: &[u8], max: u32) -> Option<(image::DynamicImage, u32, u32)> { crate::jpeg_thumbnail::decode_jpeg_thumbnail(bytes, |_, _| (max, max)) } @@ -2437,14 +2452,14 @@ impl EditorWindow { self.sidebar.picker_task = Some(cx.spawn_in(window, async move |this, cx| { #[cfg(target_os = "linux")] let picked = crate::platform::open_file_panel_async( - &[("Image", &BACKGROUND_IMAGE_EXTENSIONS[..])], + &[("Image or video", &BACKGROUND_MEDIA_EXTENSIONS[..])], None, cx, ) .await; #[cfg(not(target_os = "linux"))] let picked = cx - .update(|_, _| crate::platform::open_image_panel(&BACKGROUND_IMAGE_EXTENSIONS)) + .update(|_, _| crate::platform::open_image_panel(&BACKGROUND_MEDIA_EXTENSIONS)) .ok() .flatten(); @@ -2457,9 +2472,9 @@ impl EditorWindow { .and_then(|extension| extension.to_str()) .map(str::to_lowercase); if !extension.is_some_and(|extension| { - BACKGROUND_IMAGE_EXTENSIONS.contains(&extension.as_str()) + BACKGROUND_MEDIA_EXTENSIONS.contains(&extension.as_str()) }) { - return Err("Invalid image file type".to_string()); + return Err("Invalid image or video file type".to_string()); } let timestamp = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -3134,7 +3149,7 @@ impl EditorWindow { .child( div() .text_color(Hsla::from(theme.editor.text_1)) - .child("Click to select or drag and drop image"), + .child("Click to select an image, GIF or video"), ) .on_click(cx.listener(|this, _, window, cx| { this.pick_background_image(window, cx); @@ -4285,6 +4300,17 @@ mod tests { assert!(pixel[2].abs_diff(200) <= 3); } + #[test] + fn videos_get_a_poster_while_gifs_decode_as_images() { + assert!(is_background_video(Path::new("/bg/loop.mp4"))); + assert!(is_background_video(Path::new("/bg/loop.MOV"))); + assert!(!is_background_video(Path::new("/bg/loop.gif"))); + assert!(!is_background_video(Path::new("/bg/still.png"))); + for extension in BACKGROUND_IMAGE_EXTENSIONS { + assert!(BACKGROUND_MEDIA_EXTENSIONS.contains(&extension)); + } + } + #[test] fn built_in_wallpapers_reject_unknown_and_traversal_identifiers() { assert_eq!(wallpaper_path("unknown/wallpaper"), None); diff --git a/crates/rendering/src/decoder/frame_converter.rs b/crates/rendering/src/decoder/frame_converter.rs index 7c9e66c02c6..d9b2931e828 100644 --- a/crates/rendering/src/decoder/frame_converter.rs +++ b/crates/rendering/src/decoder/frame_converter.rs @@ -185,7 +185,7 @@ impl FrameConverter { /// swscale converts YUV with BT.601 unless told otherwise: switch the source /// matrix to BT.709 and keep the ranges and adjustments it chose. -fn use_bt709_source(context: &mut software::scaling::Context) { +pub(crate) fn use_bt709_source(context: &mut software::scaling::Context) { unsafe { let mut inv_table = std::ptr::null_mut(); let mut table = std::ptr::null_mut(); diff --git a/crates/rendering/src/decoder/mod.rs b/crates/rendering/src/decoder/mod.rs index 186ad71f131..77733b411ff 100644 --- a/crates/rendering/src/decoder/mod.rs +++ b/crates/rendering/src/decoder/mod.rs @@ -21,6 +21,7 @@ use std::sync::{ mod avassetreader; mod ffmpeg; mod frame_converter; +pub(crate) use frame_converter::use_bt709_source; #[cfg(target_os = "windows")] mod media_foundation; #[cfg(test)] diff --git a/crates/rendering/src/layers/background.rs b/crates/rendering/src/layers/background.rs index 62e3c212f2b..8686cb263f5 100644 --- a/crates/rendering/src/layers/background.rs +++ b/crates/rendering/src/layers/background.rs @@ -4,13 +4,20 @@ use image::GenericImageView; use serde::{Deserialize, Serialize}; use specta::Type; use std::collections::HashMap; +use std::path::PathBuf; use std::sync::Arc; use tokio::sync::{Mutex, RwLock}; use wgpu::{include_wgsl, util::DeviceExt}; use crate::{ProjectUniforms, RenderVideoConstants, RenderingError, create_shader_render_pipeline}; -use super::AnimatedGradientLayer; +use super::{ + AnimatedGradientLayer, + video_background::{ + VideoBackgroundFrame, VideoBackgroundSource, is_video_background, + video_background_max_dimension, + }, +}; const MAX_BACKGROUND_DIMENSION: u32 = 2560; @@ -266,6 +273,7 @@ pub enum Background { Gradient(Gradient), AnimatedGradient(AnimatedGradientConfig), Image { path: String }, + Video { path: String }, } impl From for Background { @@ -322,6 +330,7 @@ fn background_source_is_empty(source: &BackgroundSource) -> bool { impl Background { fn image_or_white(path: Option) -> Self { match path { + Some(path) if is_video_background(&path) => Background::Video { path }, Some(path) => Background::Image { path }, None => Background::Color([1.0, 1.0, 1.0, 1.0]), } @@ -375,6 +384,7 @@ pub enum Inner { output_size: (u32, u32), bind_group: wgpu::BindGroup, }, + Video(Box), ColorOrGradient { value: ColorOrGradient, #[allow(unused)] @@ -389,6 +399,8 @@ pub struct BackgroundLayer { /// derived from a static background (e.g. its blurred copy). generation: u64, resolved_image_path: ResolvedImagePath, + /// A video that failed to open, so it is not reopened every frame. + failed_video_path: Option, image_pipeline: ImageBackgroundPipeline, color_pipeline: GradientOrColorPipeline, } @@ -399,6 +411,7 @@ impl BackgroundLayer { inner: None, generation: 0, resolved_image_path: ResolvedImagePath::default(), + failed_video_path: None, image_pipeline: ImageBackgroundPipeline::new(device), color_pipeline: GradientOrColorPipeline::new(device), } @@ -462,35 +475,10 @@ impl BackgroundLayer { } }; - let output_ar = - uniforms.output_size.1 as f32 / uniforms.output_size.0 as f32; - let image_ar = texture.height() as f32 / texture.width() as f32; - - let y_height = if output_ar < image_ar { - ((image_ar - output_ar) / 2.0) / image_ar - } else { - 0.0 - }; - - let x_width = if output_ar > image_ar { - let output_ar = 1.0 / output_ar; - let image_ar = 1.0 / image_ar; - - ((image_ar - output_ar) / 2.0) / image_ar - } else { - 0.0 - }; - - let image_uniforms = ImageBackgroundUniforms { - output_size: [ - uniforms.output_size.0 as f32, - uniforms.output_size.1 as f32, - ], - padding: 0.0, - x_width, - y_height, - _padding: 0.0, - }; + let image_uniforms = ImageBackgroundUniforms::cover( + uniforms.output_size, + (texture.width(), texture.height()), + ); let uniform_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor { @@ -515,6 +503,9 @@ impl BackgroundLayer { } }; } + Background::Video { path } => { + self.prepare_video(constants, uniforms, path).await; + } Background::Color(color) => match &self.inner { Some(Inner::ColorOrGradient { value: ColorOrGradient::Color(current_color), @@ -550,6 +541,85 @@ impl BackgroundLayer { Ok(()) } + async fn prepare_video( + &mut self, + constants: &RenderVideoConstants, + uniforms: &ProjectUniforms, + path: String, + ) { + let device = &constants.device; + let max_dimension = video_background_max_dimension( + uniforms.output_size, + device.limits().max_texture_dimension_2d, + ); + + let reusable = matches!( + &self.inner, + Some(Inner::Video(video)) + if video.path == path && video.max_dimension == max_dimension + ); + if !reusable { + if self.failed_video_path.as_ref() == Some(&path) { + self.use_white_fallback(device); + return; + } + match VideoBackgroundSource::open(PathBuf::from(&path), max_dimension).await { + Ok(source) => { + self.failed_video_path = None; + self.generation += 1; + self.inner = Some(Inner::Video(Box::new(VideoBackground { + path, + max_dimension, + source, + current: None, + texture: None, + }))); + } + Err(error) => { + tracing::warn!( + "Failed to open background video '{}': {}. Falling back to solid color.", + path, + error + ); + self.failed_video_path = Some(path); + self.use_white_fallback(device); + return; + } + } + } + + let Some(Inner::Video(video)) = &mut self.inner else { + return; + }; + let seconds = f64::from(uniforms.frame_number) / f64::from(uniforms.frame_rate.max(1)); + if let Some(frame) = video.source.frame_at(seconds).await { + video.show( + device, + &constants.queue, + &self.image_pipeline, + uniforms.output_size, + frame, + ); + } + } + + fn use_white_fallback(&mut self, device: &wgpu::Device) { + let white = [1.0, 1.0, 1.0, 1.0]; + if matches!( + &self.inner, + Some(Inner::ColorOrGradient { value: ColorOrGradient::Color(color), .. }) if *color == white + ) { + return; + } + let buffer = GradientOrColorUniforms::from(Background::Color(white)).to_buffer(device); + self.generation += 1; + self.inner = Some(Inner::ColorOrGradient { + value: ColorOrGradient::Color(white), + bind_group: self.color_pipeline.bind_group(device, &buffer), + buffer, + }); + } + /// Identifies the current background while it renders the same pixels /// every frame; `None` for animated backgrounds. pub fn static_generation(&self) -> Option { @@ -570,7 +640,17 @@ impl BackgroundLayer { layer.render(pass); return; } - if let Some(Inner::Image { bind_group, .. }) = &self.inner { + let video_bind_group = match &self.inner { + Some(Inner::Video(video)) => match &video.texture { + Some(texture) => Some(&texture.bind_group), + None => return, + }, + _ => None, + }; + if let Some(bind_group) = video_bind_group { + pass.set_pipeline(&self.image_pipeline.render_pipeline); + pass.set_bind_group(0, bind_group, &[]); + } else if let Some(Inner::Image { bind_group, .. }) = &self.inner { pass.set_pipeline(&self.image_pipeline.render_pipeline); pass.set_bind_group(0, bind_group, &[]); } else if let Some(Inner::ColorOrGradient { bind_group, .. }) = &self.inner { @@ -584,6 +664,97 @@ impl BackgroundLayer { } } +pub struct VideoBackground { + path: String, + max_dimension: u32, + source: VideoBackgroundSource, + current: Option>, + texture: Option, +} + +struct VideoBackgroundTexture { + texture: wgpu::Texture, + output_size: (u32, u32), + bind_group: wgpu::BindGroup, +} + +impl VideoBackground { + fn show( + &mut self, + device: &wgpu::Device, + queue: &wgpu::Queue, + pipeline: &ImageBackgroundPipeline, + output_size: (u32, u32), + frame: Arc, + ) { + let reusable = self.texture.as_ref().is_some_and(|current| { + current.output_size == output_size + && current.texture.width() == frame.width + && current.texture.height() == frame.height + }); + if !reusable { + let texture = device.create_texture(&wgpu::TextureDescriptor { + label: Some("Background Video Texture"), + size: wgpu::Extent3d { + width: frame.width, + height: frame.height, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba8Unorm, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }); + let uniforms = ImageBackgroundUniforms::cover(output_size, (frame.width, frame.height)); + let uniform_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("Video Background Uniforms"), + contents: bytemuck::cast_slice(&[uniforms]), + usage: wgpu::BufferUsages::UNIFORM, + }); + let view = texture.create_view(&wgpu::TextureViewDescriptor::default()); + self.texture = Some(VideoBackgroundTexture { + bind_group: pipeline.bind_group(device, &uniform_buffer, &view), + texture, + output_size, + }); + self.current = None; + } + + if self + .current + .as_ref() + .is_some_and(|current| Arc::ptr_eq(current, &frame)) + { + return; + } + let Some(target) = &self.texture else { + return; + }; + queue.write_texture( + wgpu::TexelCopyTextureInfo { + texture: &target.texture, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + &frame.data, + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(4 * frame.width), + rows_per_image: Some(frame.height), + }, + wgpu::Extent3d { + width: frame.width, + height: frame.height, + depth_or_array_layers: 1, + }, + ); + self.current = Some(frame); + } +} + pub struct ImageBackgroundPipeline { pub bind_group_layout: wgpu::BindGroupLayout, pub render_pipeline: wgpu::RenderPipeline, @@ -599,6 +770,37 @@ pub struct ImageBackgroundUniforms { _padding: f32, // For alignment } +impl ImageBackgroundUniforms { + /// Crops the texture to fill the output without stretching it. + fn cover(output_size: (u32, u32), texture_size: (u32, u32)) -> Self { + let output_ar = output_size.1 as f32 / output_size.0 as f32; + let image_ar = texture_size.1 as f32 / texture_size.0 as f32; + + let y_height = if output_ar < image_ar { + ((image_ar - output_ar) / 2.0) / image_ar + } else { + 0.0 + }; + + let x_width = if output_ar > image_ar { + let output_ar = 1.0 / output_ar; + let image_ar = 1.0 / image_ar; + + ((image_ar - output_ar) / 2.0) / image_ar + } else { + 0.0 + }; + + Self { + output_size: [output_size.0 as f32, output_size.1 as f32], + padding: 0.0, + x_width, + y_height, + _padding: 0.0, + } + } +} + impl ImageBackgroundPipeline { pub fn new(device: &wgpu::Device) -> Self { let bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { @@ -791,7 +993,9 @@ impl From for GradientOrColorUniforms { noise_scale, _padding: 0.0, }, - Background::Image { .. } | Background::AnimatedGradient(_) => { + Background::Image { .. } + | Background::Video { .. } + | Background::AnimatedGradient(_) => { unreachable!("Textured backgrounds should be handled separately") } } diff --git a/crates/rendering/src/layers/mod.rs b/crates/rendering/src/layers/mod.rs index 9143787bd3d..ed805429bdb 100644 --- a/crates/rendering/src/layers/mod.rs +++ b/crates/rendering/src/layers/mod.rs @@ -14,6 +14,7 @@ mod keyboard; mod mask; mod notch; mod text; +mod video_background; use std::sync::OnceLock; @@ -140,6 +141,7 @@ pub use keyboard::*; pub use mask::*; pub use notch::*; pub use text::*; +pub use video_background::{is_video_background, video_background_poster}; #[cfg(test)] mod font_tests { diff --git a/crates/rendering/src/layers/video_background.rs b/crates/rendering/src/layers/video_background.rs new file mode 100644 index 00000000000..0cc3a1be2da --- /dev/null +++ b/crates/rendering/src/layers/video_background.rs @@ -0,0 +1,678 @@ +use std::{ + collections::VecDeque, + path::{Path, PathBuf}, + sync::{Arc, mpsc}, + time::Duration, +}; + +use ffmpeg::{format, frame, software}; +use tokio::sync::oneshot; + +#[cfg(any(target_os = "macos", target_os = "windows"))] +use ffmpeg::sys::AVHWDeviceType; + +const VIDEO_EXTENSIONS: &[&str] = &["mp4", "mov", "m4v", "webm", "mkv", "gif"]; + +/// Frames from the start of the clip stay decoded so a loop restarts +/// without a seek. Short clips and most GIFs fit entirely. +const HEAD_BUDGET_BYTES: usize = 32 * 1024 * 1024; +const MIN_HEAD_FRAMES: usize = 2; +const PREFETCH_FRAMES: usize = 3; +/// A request this far past the decoded frames seeks instead of decoding +/// through everything in between. +const SEEK_THRESHOLD_SECS: f64 = 1.5; +const TIME_EPSILON: f64 = 1e-4; +const OPEN_TIMEOUT: Duration = Duration::from_secs(15); +const FRAME_TIMEOUT: Duration = Duration::from_secs(5); +const MAX_DECODE_ERRORS: usize = 16; + +pub fn is_video_background(path: &str) -> bool { + Path::new(path) + .extension() + .and_then(|extension| extension.to_str()) + .is_some_and(|extension| { + VIDEO_EXTENSIONS + .iter() + .any(|video| extension.eq_ignore_ascii_case(video)) + }) +} + +/// Picks the decode size from a few fixed steps, so resizing the preview +/// only reopens the video when the output crosses a step. +pub fn video_background_max_dimension(output_size: (u32, u32), device_limit: u32) -> u32 { + let longest = output_size.0.max(output_size.1); + [640, 960, 1280, 1920] + .into_iter() + .find(|step| *step >= longest) + .unwrap_or(1920) + .min(device_limit) +} + +/// The first frame of a background video as tightly packed RGBA, for +/// thumbnails. +pub fn video_background_poster(path: &Path, max_dimension: u32) -> Option<(Vec, u32, u32)> { + let reader = Reader::open(path.to_path_buf(), max_dimension, 0) + .inspect_err(|error| tracing::warn!("Failed to read background video poster: {error}")) + .ok()?; + let frame = reader.head.first()?; + Some((frame.data.clone(), frame.width, frame.height)) +} + +pub struct VideoBackgroundFrame { + pub data: Vec, + pub width: u32, + pub height: u32, + time: f64, +} + +struct FrameRequest { + seconds: f64, + reply: oneshot::Sender>>, +} + +/// A looping video or GIF decoded on its own thread. Frames come back as +/// RGBA scaled to fit `max_dimension`, and the thread reads ahead between +/// requests so playback and export rarely wait on a decode. +pub struct VideoBackgroundSource { + requests: mpsc::Sender, +} + +impl VideoBackgroundSource { + pub async fn open(path: PathBuf, max_dimension: u32) -> Result { + Self::open_with_head_budget(path, max_dimension, HEAD_BUDGET_BYTES).await + } + + async fn open_with_head_budget( + path: PathBuf, + max_dimension: u32, + head_budget_bytes: usize, + ) -> Result { + let (ready_tx, ready_rx) = oneshot::channel(); + let (requests, receiver) = mpsc::channel(); + + std::thread::Builder::new() + .name("video-background".to_string()) + .spawn( + move || match Reader::open(path, max_dimension, head_budget_bytes) { + Ok(reader) => { + let _ = ready_tx.send(Ok(())); + reader.run(receiver); + } + Err(error) => { + let _ = ready_tx.send(Err(error)); + } + }, + ) + .map_err(|error| format!("spawn video background thread / {error}"))?; + + tokio::time::timeout(OPEN_TIMEOUT, ready_rx) + .await + .map_err(|_| "timed out opening video background".to_string())? + .map_err(|_| "video background thread exited".to_string())??; + + Ok(Self { requests }) + } + + pub async fn frame_at(&self, seconds: f64) -> Option> { + let (reply, response) = oneshot::channel(); + self.requests.send(FrameRequest { seconds, reply }).ok()?; + tokio::time::timeout(FRAME_TIMEOUT, response) + .await + .ok()? + .ok()? + } +} + +struct Scaler { + context: software::scaling::Context, + input: (format::Pixel, u32, u32), + output: frame::Video, +} + +struct Reader { + path: PathBuf, + max_dimension: u32, + decoder: cap_video_decode::FFmpegDecoder, + seconds_per_tick: f64, + start_ticks: i64, + frame_interval: f64, + duration: f64, + scaler: Option, + head: Vec>, + head_is_whole_clip: bool, + ring: VecDeque>, + at_end: bool, + last_time: Option, + last_served: f64, + unreachable_anchor: Option, +} + +fn hardware_device() -> Option { + #[cfg(target_os = "windows")] + { + Some(AVHWDeviceType::AV_HWDEVICE_TYPE_D3D11VA) + } + #[cfg(target_os = "macos")] + { + Some(AVHWDeviceType::AV_HWDEVICE_TYPE_VIDEOTOOLBOX) + } + #[cfg(not(any(target_os = "windows", target_os = "macos")))] + { + None + } +} + +fn open_decoder(path: &Path) -> Result { + let is_gif = path + .extension() + .and_then(|extension| extension.to_str()) + .is_some_and(|extension| extension.eq_ignore_ascii_case("gif")); + let hardware = if is_gif { None } else { hardware_device() }; + cap_video_decode::FFmpegDecoder::new(path, hardware) +} + +fn covering_frame<'a>( + frames: impl IntoIterator>, + time: f64, + reaches_end: bool, +) -> Option<&'a Arc> { + let mut frames = frames.into_iter().peekable(); + let first = frames.peek()?; + if first.time > time + TIME_EPSILON { + return None; + } + while let Some(frame) = frames.next() { + match frames.peek() { + Some(next) if next.time <= time + TIME_EPSILON => continue, + Some(_) => return Some(frame), + None => return reaches_end.then_some(frame), + } + } + None +} + +impl Reader { + fn open(path: PathBuf, max_dimension: u32, head_budget_bytes: usize) -> Result { + let probe = format::input(&path).map_err(|error| format!("open video / {error}"))?; + let stream = probe + .streams() + .best(ffmpeg::media::Type::Video) + .ok_or_else(|| "no video stream".to_string())?; + let time_base = stream.time_base(); + let seconds_per_tick = + f64::from(time_base.numerator()) / f64::from(time_base.denominator().max(1)); + let rate = stream.avg_frame_rate(); + let frame_interval = if rate.numerator() > 0 && rate.denominator() > 0 { + f64::from(rate.denominator()) / f64::from(rate.numerator()) + } else { + 1.0 / 30.0 + }; + let duration = if stream.duration() > 0 { + stream.duration() as f64 * seconds_per_tick + } else if probe.duration() > 0 { + probe.duration() as f64 / f64::from(ffmpeg::ffi::AV_TIME_BASE) + } else { + 0.0 + }; + let start_ticks = match stream.start_time() { + ffmpeg::ffi::AV_NOPTS_VALUE => 0, + start => start, + }; + drop(probe); + + let mut reader = Self { + decoder: open_decoder(&path)?, + path, + max_dimension: max_dimension.max(2), + seconds_per_tick, + start_ticks, + frame_interval, + duration, + scaler: None, + head: Vec::new(), + head_is_whole_clip: false, + ring: VecDeque::new(), + at_end: false, + last_time: None, + last_served: 0.0, + unreachable_anchor: None, + }; + reader.fill_head(head_budget_bytes)?; + Ok(reader) + } + + fn fill_head(&mut self, budget_bytes: usize) -> Result<(), String> { + let mut bytes = 0; + while self.head.len() < MIN_HEAD_FRAMES || bytes < budget_bytes { + let Some(frame) = self.decode_next() else { + self.head_is_whole_clip = true; + break; + }; + bytes += frame.data.len(); + self.head.push(frame); + } + let Some(last) = self.head.last() else { + return Err("video has no frames".to_string()); + }; + self.ring.push_back(last.clone()); + Ok(()) + } + + fn run(mut self, requests: mpsc::Receiver) { + loop { + let request = if self.wants_prefetch() { + match requests.try_recv() { + Ok(request) => Some(request), + Err(mpsc::TryRecvError::Empty) => None, + Err(mpsc::TryRecvError::Disconnected) => return, + } + } else { + match requests.recv() { + Ok(request) => Some(request), + Err(_) => return, + } + }; + + match request { + Some(request) => { + let frame = self.frame_at(request.seconds); + let _ = request.reply.send(frame); + } + None => self.prefetch(), + } + } + } + + fn clip_time(&self, seconds: f64) -> f64 { + if !seconds.is_finite() || seconds <= 0.0 { + 0.0 + } else if self.duration > 0.0 { + seconds.rem_euclid(self.duration) + } else { + seconds + } + } + + fn frame_at(&mut self, seconds: f64) -> Option> { + let time = self.clip_time(seconds); + self.last_served = time; + + if self.head_is_whole_clip { + return covering_frame(&self.head, time, true) + .or_else(|| self.head.first()) + .cloned(); + } + if let Some(frame) = covering_frame(&self.head, time, false) { + return Some(frame.clone()); + } + if time < self.head[0].time { + return Some(self.head[0].clone()); + } + if let Some(frame) = covering_frame(&self.ring, time, self.at_end) { + return Some(frame.clone()); + } + + let close_ahead = self.ring.back().is_some_and(|last| { + !self.at_end && time > last.time && time - last.time < SEEK_THRESHOLD_SECS + }); + if !close_ahead { + self.seek(time); + } + self.decode_until(time) + } + + fn decode_until(&mut self, time: f64) -> Option> { + loop { + self.trim_before(time); + if let Some(frame) = covering_frame(&self.ring, time, self.at_end) { + return Some(frame.clone()); + } + if self.at_end { + return self.ring.back().cloned(); + } + match self.decode_next() { + Some(frame) => self.ring.push_back(frame), + None => self.at_end = true, + } + } + } + + fn trim_before(&mut self, time: f64) { + while self.ring.len() >= 2 && self.ring[1].time <= time + TIME_EPSILON { + self.ring.pop_front(); + } + } + + /// Where reading ahead continues from: just after the cached head while + /// the head is playing, otherwise the last frame handed out. + fn prefetch_anchor(&self) -> f64 { + let head_end = self.head.last().map_or(0.0, |frame| frame.time); + self.last_served.max(head_end) + } + + fn ring_is_positioned(&self, anchor: f64) -> bool { + match (self.ring.front(), self.ring.back()) { + (Some(first), Some(last)) => { + first.time <= anchor + TIME_EPSILON + && (self.at_end || anchor - last.time < SEEK_THRESHOLD_SECS) + } + _ => false, + } + } + + fn wants_prefetch(&self) -> bool { + if self.head_is_whole_clip { + return false; + } + let anchor = self.prefetch_anchor(); + if !self.ring_is_positioned(anchor) { + return self.unreachable_anchor != Some(anchor); + } + !self.at_end + && self + .ring + .iter() + .filter(|frame| frame.time > anchor + TIME_EPSILON) + .count() + < PREFETCH_FRAMES + } + + fn prefetch(&mut self) { + let anchor = self.prefetch_anchor(); + if !self.ring_is_positioned(anchor) { + self.seek(anchor); + self.decode_until(anchor); + if !self.ring_is_positioned(anchor) { + self.unreachable_anchor = Some(anchor); + } + return; + } + self.trim_before(anchor); + match self.decode_next() { + Some(frame) => self.ring.push_back(frame), + None => self.at_end = true, + } + } + + fn seek(&mut self, time: f64) { + self.ring.clear(); + self.at_end = false; + self.last_time = None; + let target = time + self.start_ticks as f64 * self.seconds_per_tick; + if time <= TIME_EPSILON || self.decoder.reset(target as f32).is_err() { + match open_decoder(&self.path) { + Ok(decoder) => self.decoder = decoder, + Err(error) => { + tracing::warn!("Failed to reopen video background: {error}"); + self.at_end = true; + } + } + } + } + + fn decode_next(&mut self) -> Option> { + let mut errors = 0; + loop { + let decoded = match self.decoder.frames().next() { + Some(Ok(decoded)) => decoded, + Some(Err(error)) => { + errors += 1; + if errors >= MAX_DECODE_ERRORS { + tracing::warn!("Video background decode failed: {error}"); + return self.reached_end(); + } + continue; + } + None => return self.reached_end(), + }; + + let time = match decoded.timestamp().or(decoded.pts()) { + Some(ticks) => (ticks - self.start_ticks) as f64 * self.seconds_per_tick, + None => self + .last_time + .map_or(0.0, |last| last + self.frame_interval), + }; + if let Some(last) = self.last_time + && time > last + { + self.frame_interval = time - last; + } + self.last_time = Some(time); + + let Some((data, width, height)) = self.scale(&decoded) else { + errors += 1; + if errors >= MAX_DECODE_ERRORS { + return self.reached_end(); + } + continue; + }; + return Some(Arc::new(VideoBackgroundFrame { + data, + width, + height, + time: time.max(0.0), + })); + } + } + + fn reached_end(&mut self) -> Option> { + if let Some(last) = self.last_time { + let end = last + self.frame_interval; + if end > 0.0 { + self.duration = end; + } + } + None + } + + fn output_size(&self, width: u32, height: u32) -> (u32, u32) { + let longest = width.max(height).max(1); + let scale = (f64::from(self.max_dimension) / f64::from(longest)).min(1.0); + let even = |value: u32| ((f64::from(value) * scale).round() as u32).max(2) & !1; + (even(width), even(height)) + } + + fn scale(&mut self, decoded: &frame::Video) -> Option<(Vec, u32, u32)> { + let input = (decoded.format(), decoded.width(), decoded.height()); + if input.1 == 0 || input.2 == 0 { + return None; + } + + if self + .scaler + .as_ref() + .is_none_or(|scaler| scaler.input != input) + { + let (width, height) = self.output_size(input.1, input.2); + let flags = if width < input.1 { + software::scaling::Flags::AREA + } else { + software::scaling::Flags::BILINEAR + }; + let mut context = software::scaling::Context::get( + input.0, + input.1, + input.2, + format::Pixel::RGBA, + width, + height, + flags, + ) + .ok()?; + if decoded.color_space() == ffmpeg::util::color::Space::BT709 { + crate::decoder::use_bt709_source(&mut context); + } + self.scaler = Some(Scaler { + context, + input, + output: frame::Video::new(format::Pixel::RGBA, width, height), + }); + } + + let scaler = self.scaler.as_mut()?; + scaler.context.run(decoded, &mut scaler.output).ok()?; + + let width = scaler.output.width(); + let height = scaler.output.height(); + let row_bytes = width as usize * 4; + let stride = scaler.output.stride(0); + let source = scaler.output.data(0); + let mut data = Vec::with_capacity(row_bytes * height as usize); + for row in 0..height as usize { + let start = row * stride; + data.extend_from_slice(&source[start..start + row_bytes]); + } + Some((data, width, height)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use image::{Delay, Frame, Rgba, RgbaImage, codecs::gif::GifEncoder}; + + const COLOURS: [[u8; 3]; 4] = [[255, 0, 0], [0, 255, 0], [0, 0, 255], [255, 255, 0]]; + + fn write_gif(path: &Path, width: u32, height: u32, frames_per_colour: usize) { + let file = std::fs::File::create(path).unwrap(); + let mut encoder = GifEncoder::new(file); + encoder + .set_repeat(image::codecs::gif::Repeat::Infinite) + .unwrap(); + for colour in COLOURS { + for _ in 0..frames_per_colour { + let image = RgbaImage::from_pixel( + width, + height, + Rgba([colour[0], colour[1], colour[2], 255]), + ); + encoder + .encode_frame(Frame::from_parts( + image, + 0, + 0, + Delay::from_numer_denom_ms(100, 1), + )) + .unwrap(); + } + } + } + + fn colour_index(frame: &VideoBackgroundFrame) -> usize { + let pixel = &frame.data[..3]; + COLOURS + .iter() + .position(|colour| { + colour + .iter() + .zip(pixel) + .all(|(expected, actual)| expected.abs_diff(*actual) < 40) + }) + .unwrap_or_else(|| panic!("unexpected colour {pixel:?}")) + } + + fn runtime() -> tokio::runtime::Runtime { + tokio::runtime::Builder::new_current_thread() + .enable_time() + .build() + .unwrap() + } + + #[test] + fn recognises_video_and_gif_paths() { + assert!(is_video_background("/a/b/clip.MP4")); + assert!(is_video_background("loop.gif")); + assert!(is_video_background("c.webm")); + assert!(!is_video_background("wallpaper.jpg")); + assert!(!is_video_background("no-extension")); + } + + #[test] + fn decode_size_steps_with_the_output() { + assert_eq!(video_background_max_dimension((600, 400), 8192), 640); + assert_eq!(video_background_max_dimension((1280, 720), 8192), 1280); + assert_eq!(video_background_max_dimension((3840, 2160), 8192), 1920); + assert_eq!(video_background_max_dimension((3840, 2160), 1024), 1024); + } + + #[test] + fn small_gif_plays_and_loops_from_memory() { + ffmpeg::init().unwrap(); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("loop.gif"); + write_gif(&path, 64, 32, 1); + + runtime().block_on(async { + let source = VideoBackgroundSource::open(path, 1920).await.unwrap(); + for (seconds, expected) in [ + (0.0, 0), + (0.05, 0), + (0.15, 1), + (0.25, 2), + (0.35, 3), + (0.45, 0), + (1.05, 2), + (0.15, 1), + ] { + let frame = source.frame_at(seconds).await.unwrap(); + assert_eq!((frame.width, frame.height), (64, 32)); + assert_eq!(colour_index(&frame), expected, "at {seconds}s"); + } + }); + } + + #[test] + fn long_clip_streams_seeks_and_loops() { + ffmpeg::init().unwrap(); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("long.gif"); + write_gif(&path, 96, 48, 10); + + runtime().block_on(async { + let frame_bytes = 64 * 32 * 4; + let source = VideoBackgroundSource::open_with_head_budget(path, 64, frame_bytes * 5) + .await + .unwrap(); + let mut seconds = 0.0; + while seconds < 9.0 { + let frame = source.frame_at(seconds).await.unwrap(); + assert_eq!((frame.width, frame.height), (64, 32)); + let expected = ((seconds % 4.0) / 1.0) as usize; + assert_eq!(colour_index(&frame), expected, "at {seconds}s"); + seconds += 0.25; + } + for (seconds, expected) in [(3.5, 3), (0.5, 0), (2.5, 2), (1.5, 1), (5.5, 1)] { + let frame = source.frame_at(seconds).await.unwrap(); + assert_eq!(colour_index(&frame), expected, "at {seconds}s"); + } + }); + } + + #[test] + fn poster_is_the_first_frame_scaled_down() { + ffmpeg::init().unwrap(); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("poster.gif"); + write_gif(&path, 96, 48, 1); + + let (data, width, height) = video_background_poster(&path, 48).unwrap(); + assert_eq!((width, height), (48, 24)); + let frame = VideoBackgroundFrame { + data, + width, + height, + time: 0.0, + }; + assert_eq!(colour_index(&frame), 0); + } + + #[test] + fn missing_file_fails_to_open() { + ffmpeg::init().unwrap(); + let result = runtime().block_on(VideoBackgroundSource::open( + PathBuf::from("/definitely/not/here.mp4"), + 640, + )); + assert!(result.is_err()); + } +} diff --git a/crates/rendering/src/lib.rs b/crates/rendering/src/lib.rs index da01e6295eb..ce6ab48b5f6 100644 --- a/crates/rendering/src/lib.rs +++ b/crates/rendering/src/lib.rs @@ -76,7 +76,8 @@ pub use frame_pipeline::{GpuOutputFormat, Nv12RenderedFrame, RenderedFrame, Shar pub use frame_pipeline::{PendingSurface, RgbaToBgraSurfaceConverter}; pub use frame_windows::FrameWindows; pub use layers::{ - BackgroundTextureCache, clean_background_path, last_caption_center, new_font_system, + BackgroundTextureCache, clean_background_path, is_video_background, last_caption_center, + new_font_system, video_background_poster, }; pub use managed_segment::{ ManagedRecordingSegmentDecoders, ManagedSegmentDecoderStatus, ManagedSegmentStopHandles, From 5395a57c6f8594678f32e82988bf60ac646810d2 Mon Sep 17 00:00:00 2001 From: "Aatman Jain (AJ)" <257399449+AatmanAJ@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:09:32 +0000 Subject: [PATCH 2/4] fix(rendering): decode forward instead of seeking when a video background request lands on the newest frame --- .../rendering/src/layers/video_background.rs | 37 +++++++++++++++++-- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/crates/rendering/src/layers/video_background.rs b/crates/rendering/src/layers/video_background.rs index 0cc3a1be2da..65e2f3f7421 100644 --- a/crates/rendering/src/layers/video_background.rs +++ b/crates/rendering/src/layers/video_background.rs @@ -145,6 +145,8 @@ struct Reader { last_time: Option, last_served: f64, unreachable_anchor: Option, + #[cfg(test)] + seeks: usize, } fn hardware_device() -> Option { @@ -236,6 +238,8 @@ impl Reader { last_time: None, last_served: 0.0, unreachable_anchor: None, + #[cfg(test)] + seeks: 0, }; reader.fill_head(head_budget_bytes)?; Ok(reader) @@ -312,9 +316,16 @@ impl Reader { return Some(frame.clone()); } - let close_ahead = self.ring.back().is_some_and(|last| { - !self.at_end && time > last.time && time - last.time < SEEK_THRESHOLD_SECS - }); + // A request on the newest frame isn't covered until the frame after it + // is decoded, so it counts as ahead rather than as a reason to seek. + let close_ahead = match (self.ring.front(), self.ring.back()) { + (Some(first), Some(last)) => { + !self.at_end + && time + TIME_EPSILON >= first.time + && time - last.time < SEEK_THRESHOLD_SECS + } + _ => false, + }; if !close_ahead { self.seek(time); } @@ -395,6 +406,10 @@ impl Reader { } fn seek(&mut self, time: f64) { + #[cfg(test)] + { + self.seeks += 1; + } self.ring.clear(); self.at_end = false; self.last_time = None; @@ -648,6 +663,22 @@ mod tests { }); } + #[test] + fn playing_forward_without_read_ahead_never_seeks() { + ffmpeg::init().unwrap(); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("forward.gif"); + write_gif(&path, 96, 48, 10); + + let mut reader = Reader::open(path, 64, 64 * 32 * 4 * 5).unwrap(); + for frame in 0..40 { + let seconds = frame as f64 / 10.0; + let served = reader.frame_at(seconds).unwrap(); + assert_eq!(colour_index(&served), frame / 10, "at {seconds}s"); + } + assert_eq!(reader.seeks, 0); + } + #[test] fn poster_is_the_first_frame_scaled_down() { ffmpeg::init().unwrap(); From afefd9bcafe244630fd4a93324385d40031696a1 Mon Sep 17 00:00:00 2001 From: "Aatman Jain (AJ)" <257399449+AatmanAJ@users.noreply.github.com> Date: Wed, 28 Oct 2026 14:05:29 +0000 Subject: [PATCH 3/4] perf(rendering): scale video backgrounds on four threads and skip frames that are never shown --- .../rendering/src/decoder/frame_converter.rs | 19 +- crates/rendering/src/decoder/mod.rs | 2 +- .../rendering/src/layers/video_background.rs | 293 +++++++++++++++--- 3 files changed, 261 insertions(+), 53 deletions(-) diff --git a/crates/rendering/src/decoder/frame_converter.rs b/crates/rendering/src/decoder/frame_converter.rs index d9b2931e828..66fef38cb43 100644 --- a/crates/rendering/src/decoder/frame_converter.rs +++ b/crates/rendering/src/decoder/frame_converter.rs @@ -186,13 +186,21 @@ impl FrameConverter { /// swscale converts YUV with BT.601 unless told otherwise: switch the source /// matrix to BT.709 and keep the ranges and adjustments it chose. pub(crate) fn use_bt709_source(context: &mut software::scaling::Context) { + unsafe { use_bt709_source_raw(context.as_mut_ptr()) } +} + +/// [`use_bt709_source`] for a context created through the raw swscale API. +/// +/// # Safety +/// `context` must point to an initialised `SwsContext`. +pub(crate) unsafe fn use_bt709_source_raw(context: *mut ffmpeg::ffi::SwsContext) { unsafe { let mut inv_table = std::ptr::null_mut(); let mut table = std::ptr::null_mut(); let (mut src_range, mut dst_range) = (0, 0); let (mut brightness, mut contrast, mut saturation) = (0, 0, 0); if ffmpeg::ffi::sws_getColorspaceDetails( - context.as_mut_ptr(), + context, &mut inv_table, &mut src_range, &mut table, @@ -206,14 +214,7 @@ pub(crate) fn use_bt709_source(context: &mut software::scaling::Context) { } let bt709 = ffmpeg::ffi::sws_getCoefficients(ffmpeg::ffi::SWS_CS_ITU709); ffmpeg::ffi::sws_setColorspaceDetails( - context.as_mut_ptr(), - bt709, - src_range, - table, - dst_range, - brightness, - contrast, - saturation, + context, bt709, src_range, table, dst_range, brightness, contrast, saturation, ); } } diff --git a/crates/rendering/src/decoder/mod.rs b/crates/rendering/src/decoder/mod.rs index 77733b411ff..9247285db13 100644 --- a/crates/rendering/src/decoder/mod.rs +++ b/crates/rendering/src/decoder/mod.rs @@ -21,7 +21,7 @@ use std::sync::{ mod avassetreader; mod ffmpeg; mod frame_converter; -pub(crate) use frame_converter::use_bt709_source; +pub(crate) use frame_converter::use_bt709_source_raw; #[cfg(target_os = "windows")] mod media_foundation; #[cfg(test)] diff --git a/crates/rendering/src/layers/video_background.rs b/crates/rendering/src/layers/video_background.rs index 65e2f3f7421..7a79ea0555c 100644 --- a/crates/rendering/src/layers/video_background.rs +++ b/crates/rendering/src/layers/video_background.rs @@ -24,6 +24,9 @@ const SEEK_THRESHOLD_SECS: f64 = 1.5; const TIME_EPSILON: f64 = 1e-4; const OPEN_TIMEOUT: Duration = Duration::from_secs(15); const FRAME_TIMEOUT: Duration = Duration::from_secs(5); +/// Downscaling a 4K frame is the most expensive step; past four threads it +/// stops paying off and starts competing with the recording's own decoders. +const SCALER_THREADS: usize = 4; const MAX_DECODE_ERRORS: usize = 16; pub fn is_video_background(path: &str) -> bool { @@ -123,12 +126,89 @@ impl VideoBackgroundSource { } } +/// Converts decoded frames to RGBA at the output size. Built through the raw +/// swscale API because only a context configured before initialisation can +/// spread the conversion over several threads. struct Scaler { - context: software::scaling::Context, + context: *mut ffmpeg::ffi::SwsContext, input: (format::Pixel, u32, u32), output: frame::Video, } +// SAFETY: the context is created, used and freed by the reader thread that +// owns the scaler; nothing else holds the pointer. +unsafe impl Send for Scaler {} + +impl Scaler { + fn new( + input: (format::Pixel, u32, u32), + (width, height): (u32, u32), + bt709: bool, + ) -> Option { + let flags = if width < input.1 { + software::scaling::Flags::AREA + } else { + software::scaling::Flags::BILINEAR + }; + let source: ffmpeg::ffi::AVPixelFormat = input.0.into(); + let threads = std::thread::available_parallelism() + .map_or(1, |count| count.get()) + .min(SCALER_THREADS) as i64; + unsafe { + let context = ffmpeg::ffi::sws_alloc_context(); + if context.is_null() { + return None; + } + let scaler = Self { + context, + input, + output: frame::Video::new(format::Pixel::RGBA, width, height), + }; + let options = [ + (c"srcw", i64::from(input.1)), + (c"srch", i64::from(input.2)), + (c"src_format", source as i64), + (c"dstw", i64::from(width)), + (c"dsth", i64::from(height)), + ( + c"dst_format", + ffmpeg::ffi::AVPixelFormat::AV_PIX_FMT_RGBA as i64, + ), + (c"sws_flags", i64::from(flags.bits())), + ]; + for (name, value) in options { + if ffmpeg::ffi::av_opt_set_int(context.cast(), name.as_ptr(), value, 0) < 0 { + return None; + } + } + // Without threading support this stays a single-threaded scaler. + ffmpeg::ffi::av_opt_set_int(context.cast(), c"threads".as_ptr(), threads, 0); + if ffmpeg::ffi::sws_init_context(context, std::ptr::null_mut(), std::ptr::null_mut()) + < 0 + { + return None; + } + if bt709 { + crate::decoder::use_bt709_source_raw(context); + } + Some(scaler) + } + } + + fn run(&mut self, decoded: &frame::Video) -> bool { + unsafe { + ffmpeg::ffi::sws_scale_frame(self.context, self.output.as_mut_ptr(), decoded.as_ptr()) + >= 0 + } + } +} + +impl Drop for Scaler { + fn drop(&mut self) { + unsafe { ffmpeg::ffi::sws_freeContext(self.context) }; + } +} + struct Reader { path: PathBuf, max_dimension: u32, @@ -144,9 +224,17 @@ struct Reader { at_end: bool, last_time: Option, last_served: f64, + /// Spacing of recent requests, when they come further apart than the + /// clip's frames (a 60 fps clip behind a 30 fps export). + request_interval: Option, + /// Decoded but not yet scaled: the frame after the newest one in `ring`, + /// held so a frame is only scaled once we know it will be shown. + pending: Option<(f64, frame::Video)>, unreachable_anchor: Option, #[cfg(test)] seeks: usize, + #[cfg(test)] + scaled: usize, } fn hardware_device() -> Option { @@ -237,9 +325,13 @@ impl Reader { at_end: false, last_time: None, last_served: 0.0, + request_interval: None, + pending: None, unreachable_anchor: None, #[cfg(test)] seeks: 0, + #[cfg(test)] + scaled: 0, }; reader.fill_head(head_budget_bytes)?; Ok(reader) @@ -299,6 +391,9 @@ impl Reader { fn frame_at(&mut self, seconds: f64) -> Option> { let time = self.clip_time(seconds); + let step = time - self.last_served; + self.request_interval = + (step > self.frame_interval * 1.5 && step < SEEK_THRESHOLD_SECS).then_some(step); self.last_served = time; if self.head_is_whole_clip { @@ -341,7 +436,7 @@ impl Reader { if self.at_end { return self.ring.back().cloned(); } - match self.decode_next() { + match self.next_shown(|frame_time| frame_time.max(time)) { Some(frame) => self.ring.push_back(frame), None => self.at_end = true, } @@ -399,7 +494,14 @@ impl Reader { return; } self.trim_before(anchor); - match self.decode_next() { + let interval = self.request_interval; + let next = self.next_shown(|frame_time| match interval { + Some(interval) if frame_time > anchor => { + anchor + ((frame_time - anchor) / interval - TIME_EPSILON).ceil() * interval + } + _ => frame_time.max(anchor), + }); + match next { Some(frame) => self.ring.push_back(frame), None => self.at_end = true, } @@ -411,6 +513,7 @@ impl Reader { self.seeks += 1; } self.ring.clear(); + self.pending = None; self.at_end = false; self.last_time = None; let target = time + self.start_ticks as f64 * self.seconds_per_tick; @@ -425,7 +528,59 @@ impl Reader { } } + /// The next frame, decoded and scaled. fn decode_next(&mut self) -> Option> { + self.next_shown(|frame_time| frame_time) + } + + /// Decodes forward and returns the next frame that will actually be + /// shown. `first_request_at` maps a frame's time to the first request + /// expected at or after it; a frame whose successor starts by then is + /// never shown, so it's dropped without being scaled. That keeps seeks + /// (which decode forward from a keyframe) and clips with more frames than + /// the output from paying for conversions nobody sees. + fn next_shown( + &mut self, + first_request_at: impl Fn(f64) -> f64, + ) -> Option> { + let mut errors = 0; + loop { + let (time, decoded) = match self.pending.take() { + Some(pending) => pending, + None => self.decode_raw()?, + }; + self.pending = self.decode_raw(); + if let Some((next_time, _)) = &self.pending + && *next_time <= first_request_at(time) + TIME_EPSILON + { + continue; + } + #[cfg(test)] + { + self.scaled += 1; + } + match self.scale(&decoded) { + Some((data, width, height)) => { + return Some(Arc::new(VideoBackgroundFrame { + data, + width, + height, + time, + })); + } + None => { + errors += 1; + if errors >= MAX_DECODE_ERRORS { + self.pending = None; + return self.reached_end(); + } + } + } + } + } + + /// The next decoded frame and its time in the clip, before scaling. + fn decode_raw(&mut self) -> Option<(f64, frame::Video)> { let mut errors = 0; loop { let decoded = match self.decoder.frames().next() { @@ -453,24 +608,11 @@ impl Reader { self.frame_interval = time - last; } self.last_time = Some(time); - - let Some((data, width, height)) = self.scale(&decoded) else { - errors += 1; - if errors >= MAX_DECODE_ERRORS { - return self.reached_end(); - } - continue; - }; - return Some(Arc::new(VideoBackgroundFrame { - data, - width, - height, - time: time.max(0.0), - })); + return Some((time.max(0.0), decoded)); } } - fn reached_end(&mut self) -> Option> { + fn reached_end(&mut self) -> Option { if let Some(last) = self.last_time { let end = last + self.frame_interval; if end > 0.0 { @@ -498,34 +640,15 @@ impl Reader { .as_ref() .is_none_or(|scaler| scaler.input != input) { - let (width, height) = self.output_size(input.1, input.2); - let flags = if width < input.1 { - software::scaling::Flags::AREA - } else { - software::scaling::Flags::BILINEAR - }; - let mut context = software::scaling::Context::get( - input.0, - input.1, - input.2, - format::Pixel::RGBA, - width, - height, - flags, - ) - .ok()?; - if decoded.color_space() == ffmpeg::util::color::Space::BT709 { - crate::decoder::use_bt709_source(&mut context); - } - self.scaler = Some(Scaler { - context, - input, - output: frame::Video::new(format::Pixel::RGBA, width, height), - }); + let size = self.output_size(input.1, input.2); + let bt709 = decoded.color_space() == ffmpeg::util::color::Space::BT709; + self.scaler = Some(Scaler::new(input, size, bt709)?); } let scaler = self.scaler.as_mut()?; - scaler.context.run(decoded, &mut scaler.output).ok()?; + if !scaler.run(decoded) { + return None; + } let width = scaler.output.width(); let height = scaler.output.height(); @@ -679,6 +802,90 @@ mod tests { assert_eq!(reader.seeks, 0); } + #[test] + fn seeking_scales_only_the_frame_it_lands_on() { + ffmpeg::init().unwrap(); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("seek.gif"); + write_gif(&path, 96, 48, 10); + + let mut reader = Reader::open(path, 64, 64 * 32 * 4 * 2).unwrap(); + let after_head = reader.scaled; + let served = reader.frame_at(3.05).unwrap(); + assert_eq!(colour_index(&served), 3); + assert_eq!(reader.seeks, 1); + assert!( + reader.scaled - after_head <= 2, + "scaled {} frames to show one", + reader.scaled - after_head + ); + } + + #[test] + fn sparse_requests_skip_frames_between_them() { + ffmpeg::init().unwrap(); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("sparse.gif"); + write_gif(&path, 96, 48, 10); + + let mut reader = Reader::open(path, 64, 64 * 32 * 4 * 2).unwrap(); + let after_head = reader.scaled; + for request in 0..20 { + let seconds = request as f64 * 0.2; + let served = reader.frame_at(seconds).unwrap(); + assert_eq!(colour_index(&served), request / 5, "at {seconds}s"); + while reader.wants_prefetch() { + reader.prefetch(); + } + } + assert_eq!(reader.seeks, 0); + let scaled = reader.scaled - after_head; + assert!( + scaled <= 24, + "scaled {scaled} frames for 20 requests over 40 frames" + ); + } + + #[test] + fn threaded_scaler_matches_a_single_threaded_one() { + ffmpeg::init().unwrap(); + let (width, height) = (1280, 720); + let mut source = frame::Video::new(format::Pixel::NV12, width, height); + for plane in 0..2 { + let stride = source.stride(plane); + let rows = if plane == 0 { height } else { height / 2 } as usize; + let data = source.data_mut(plane); + for row in 0..rows { + for column in 0..width as usize { + data[row * stride + column] = ((row * 7 + column * 3 + plane * 50) % 256) as u8; + } + } + } + let mut threaded = + Scaler::new((format::Pixel::NV12, width, height), (640, 360), true).unwrap(); + assert!(threaded.run(&source)); + + let mut reference = software::scaling::Context::get( + format::Pixel::NV12, + width, + height, + format::Pixel::RGBA, + 640, + 360, + software::scaling::Flags::AREA, + ) + .unwrap(); + unsafe { crate::decoder::use_bt709_source_raw(reference.as_mut_ptr()) }; + let mut expected = frame::Video::new(format::Pixel::RGBA, 640, 360); + reference.run(&source, &mut expected).unwrap(); + + for row in 0..360 { + let actual = &threaded.output.data(0)[row * threaded.output.stride(0)..][..640 * 4]; + let wanted = &expected.data(0)[row * expected.stride(0)..][..640 * 4]; + assert_eq!(actual, wanted, "row {row}"); + } + } + #[test] fn poster_is_the_first_frame_scaled_down() { ffmpeg::init().unwrap(); From c975955090ef3e758b6d5bf17fe5bea6677cb440 Mon Sep 17 00:00:00 2001 From: "Aatman Jain (AJ)" <257399449+AatmanAJ@users.noreply.github.com> Date: Wed, 28 Oct 2026 18:09:49 +0000 Subject: [PATCH 4/4] fix(rendering): cover skipped gaps, hold a GIF's last frame, and honour rotation and pixel aspect in video backgrounds --- .../rendering/src/layers/video_background.rs | 286 ++++++++++++++---- 1 file changed, 232 insertions(+), 54 deletions(-) diff --git a/crates/rendering/src/layers/video_background.rs b/crates/rendering/src/layers/video_background.rs index 7a79ea0555c..53724526e2c 100644 --- a/crates/rendering/src/layers/video_background.rs +++ b/crates/rendering/src/layers/video_background.rs @@ -51,8 +51,6 @@ pub fn video_background_max_dimension(output_size: (u32, u32), device_limit: u32 .min(device_limit) } -/// The first frame of a background video as tightly packed RGBA, for -/// thumbnails. pub fn video_background_poster(path: &Path, max_dimension: u32) -> Option<(Vec, u32, u32)> { let reader = Reader::open(path.to_path_buf(), max_dimension, 0) .inspect_err(|error| tracing::warn!("Failed to read background video poster: {error}")) @@ -66,6 +64,10 @@ pub struct VideoBackgroundFrame { pub width: u32, pub height: u32, time: f64, + /// When the next decoded frame starts. Frames skipped by read-ahead leave + /// gaps between cached frames, so coverage can't be taken from the next + /// cached frame. + end: f64, } struct FrameRequest { @@ -73,9 +75,6 @@ struct FrameRequest { reply: oneshot::Sender>>, } -/// A looping video or GIF decoded on its own thread. Frames come back as -/// RGBA scaled to fit `max_dimension`, and the thread reads ahead between -/// requests so playback and export rarely wait on a decode. pub struct VideoBackgroundSource { requests: mpsc::Sender, } @@ -126,9 +125,8 @@ impl VideoBackgroundSource { } } -/// Converts decoded frames to RGBA at the output size. Built through the raw -/// swscale API because only a context configured before initialisation can -/// spread the conversion over several threads. +/// Built through the raw swscale API: only a context configured before +/// initialisation can spread the conversion over several threads. struct Scaler { context: *mut ffmpeg::ffi::SwsContext, input: (format::Pixel, u32, u32), @@ -224,12 +222,12 @@ struct Reader { at_end: bool, last_time: Option, last_served: f64, - /// Spacing of recent requests, when they come further apart than the - /// clip's frames (a 60 fps clip behind a 30 fps export). + /// Only set while requests come further apart than the clip's frames. request_interval: Option, - /// Decoded but not yet scaled: the frame after the newest one in `ring`, - /// held so a frame is only scaled once we know it will be shown. pending: Option<(f64, frame::Video)>, + last_frame_duration: Option, + quarter_turns: u32, + sample_aspect: (i32, i32), unreachable_anchor: Option, #[cfg(test)] seeks: usize, @@ -264,21 +262,84 @@ fn open_decoder(path: &Path) -> Result fn covering_frame<'a>( frames: impl IntoIterator>, time: f64, - reaches_end: bool, ) -> Option<&'a Arc> { - let mut frames = frames.into_iter().peekable(); - let first = frames.peek()?; - if first.time > time + TIME_EPSILON { - return None; + frames + .into_iter() + .take_while(|frame| frame.time <= time + TIME_EPSILON) + .last() + .filter(|frame| frame.end > time + TIME_EPSILON) +} + +/// Phones store portrait video as landscape pixels plus a display matrix. +fn display_quarter_turns(stream: &ffmpeg::format::stream::Stream) -> u32 { + unsafe { + let parameters = stream.parameters().as_ptr(); + let side_data = ffmpeg::ffi::av_packet_side_data_get( + (*parameters).coded_side_data, + (*parameters).nb_coded_side_data, + ffmpeg::ffi::AVPacketSideDataType::AV_PKT_DATA_DISPLAYMATRIX, + ); + if side_data.is_null() || (*side_data).size < 9 * 4 { + return 0; + } + quarter_turns_from_matrix(&*((*side_data).data as *const [i32; 9])) } - while let Some(frame) = frames.next() { - match frames.peek() { - Some(next) if next.time <= time + TIME_EPSILON => continue, - Some(_) => return Some(frame), - None => return reaches_end.then_some(frame), +} + +fn quarter_turns_from_matrix(matrix: &[i32; 9]) -> u32 { + let counter_clockwise = unsafe { ffmpeg::ffi::av_display_rotation_get(matrix.as_ptr()) }; + if !counter_clockwise.is_finite() { + return 0; + } + ((-counter_clockwise / 90.0).round() as i64).rem_euclid(4) as u32 +} + +fn rotate_rgba(data: &[u8], width: u32, height: u32, quarter_turns: u32) -> (Vec, u32, u32) { + let (width, height) = (width as usize, height as usize); + let (out_width, out_height) = if quarter_turns % 2 == 1 { + (height, width) + } else { + (width, height) + }; + let mut out = vec![0u8; data.len()]; + const TILE: usize = 64; + for tile_y in (0..height).step_by(TILE) { + for tile_x in (0..width).step_by(TILE) { + for y in tile_y..(tile_y + TILE).min(height) { + for x in tile_x..(tile_x + TILE).min(width) { + let (to_x, to_y) = match quarter_turns { + 1 => (height - 1 - y, x), + 2 => (width - 1 - x, height - 1 - y), + 3 => (y, width - 1 - x), + _ => (x, y), + }; + let from = (y * width + x) * 4; + let to = (to_y * out_width + to_x) * 4; + out[to..to + 4].copy_from_slice(&data[from..from + 4]); + } + } } } - None + (out, out_width as u32, out_height as u32) +} + +fn output_size( + width: u32, + height: u32, + sample_aspect: (i32, i32), + max_dimension: u32, +) -> (u32, u32) { + let display_width = match sample_aspect { + (numerator, denominator) if numerator > 0 && denominator > 0 => { + f64::from(width) * f64::from(numerator) / f64::from(denominator) + } + _ => f64::from(width), + }; + let height = f64::from(height); + let longest = display_width.max(height).max(1.0); + let scale = (f64::from(max_dimension) / longest).min(1.0); + let even = |value: f64| ((value * scale).round() as u32).max(2) & !1; + (even(display_width), even(height)) } impl Reader { @@ -304,6 +365,11 @@ impl Reader { } else { 0.0 }; + let quarter_turns = display_quarter_turns(&stream); + let sample_aspect = unsafe { + let ratio = (*stream.parameters().as_ptr()).sample_aspect_ratio; + (ratio.num, ratio.den) + }; let start_ticks = match stream.start_time() { ffmpeg::ffi::AV_NOPTS_VALUE => 0, start => start, @@ -327,6 +393,9 @@ impl Reader { last_served: 0.0, request_interval: None, pending: None, + last_frame_duration: None, + quarter_turns, + sample_aspect, unreachable_anchor: None, #[cfg(test)] seeks: 0, @@ -397,30 +466,27 @@ impl Reader { self.last_served = time; if self.head_is_whole_clip { - return covering_frame(&self.head, time, true) + return covering_frame(&self.head, time) .or_else(|| self.head.first()) .cloned(); } - if let Some(frame) = covering_frame(&self.head, time, false) { + if let Some(frame) = covering_frame(&self.head, time) { return Some(frame.clone()); } if time < self.head[0].time { return Some(self.head[0].clone()); } - if let Some(frame) = covering_frame(&self.ring, time, self.at_end) { + if let Some(frame) = covering_frame(&self.ring, time) { return Some(frame.clone()); } - // A request on the newest frame isn't covered until the frame after it - // is decoded, so it counts as ahead rather than as a reason to seek. - let close_ahead = match (self.ring.front(), self.ring.back()) { - (Some(first), Some(last)) => { - !self.at_end - && time + TIME_EPSILON >= first.time - && time - last.time < SEEK_THRESHOLD_SECS - } - _ => false, - }; + // Anything before the newest frame's end that isn't covered landed in + // a gap read-ahead skipped, and only a seek can go back for it. + let close_ahead = self.ring.back().is_some_and(|last| { + !self.at_end + && time + TIME_EPSILON >= last.end + && time - last.time < SEEK_THRESHOLD_SECS + }); if !close_ahead { self.seek(time); } @@ -430,9 +496,15 @@ impl Reader { fn decode_until(&mut self, time: f64) -> Option> { loop { self.trim_before(time); - if let Some(frame) = covering_frame(&self.ring, time, self.at_end) { + if let Some(frame) = covering_frame(&self.ring, time) { return Some(frame.clone()); } + // A seek can land after the target; decoding on would never come back. + if let Some(first) = self.ring.front() + && first.time > time + TIME_EPSILON + { + return Some(first.clone()); + } if self.at_end { return self.ring.back().cloned(); } @@ -528,17 +600,14 @@ impl Reader { } } - /// The next frame, decoded and scaled. fn decode_next(&mut self) -> Option> { self.next_shown(|frame_time| frame_time) } - /// Decodes forward and returns the next frame that will actually be - /// shown. `first_request_at` maps a frame's time to the first request - /// expected at or after it; a frame whose successor starts by then is - /// never shown, so it's dropped without being scaled. That keeps seeks - /// (which decode forward from a keyframe) and clips with more frames than - /// the output from paying for conversions nobody sees. + /// A frame whose successor starts by `first_request_at(frame time)` is + /// never shown, so it's dropped unscaled: decoding forward from a keyframe + /// after a seek, or a clip with more frames than the output, would + /// otherwise pay for conversions nobody sees. fn next_shown( &mut self, first_request_at: impl Fn(f64) -> f64, @@ -561,11 +630,16 @@ impl Reader { } match self.scale(&decoded) { Some((data, width, height)) => { + let end = self + .pending + .as_ref() + .map_or(f64::INFINITY, |(next_time, _)| *next_time); return Some(Arc::new(VideoBackgroundFrame { data, width, height, time, + end, })); } None => { @@ -579,7 +653,6 @@ impl Reader { } } - /// The next decoded frame and its time in the clip, before scaling. fn decode_raw(&mut self) -> Option<(f64, frame::Video)> { let mut errors = 0; loop { @@ -608,13 +681,22 @@ impl Reader { self.frame_interval = time - last; } self.last_time = Some(time); + let ticks = unsafe { (*decoded.as_ptr()).duration }; + self.last_frame_duration = (ticks > 0).then_some(ticks as f64 * self.seconds_per_tick); return Some((time.max(0.0), decoded)); } } + /// Container durations can be missing or wrong, and a GIF can hold its + /// last frame far longer than the others, so the last frame's own + /// duration wins over both. fn reached_end(&mut self) -> Option { if let Some(last) = self.last_time { - let end = last + self.frame_interval; + let end = match self.last_frame_duration { + Some(duration) => last + duration, + None if self.duration > last + TIME_EPSILON => self.duration, + None => last + self.frame_interval, + }; if end > 0.0 { self.duration = end; } @@ -622,13 +704,6 @@ impl Reader { None } - fn output_size(&self, width: u32, height: u32) -> (u32, u32) { - let longest = width.max(height).max(1); - let scale = (f64::from(self.max_dimension) / f64::from(longest)).min(1.0); - let even = |value: u32| ((f64::from(value) * scale).round() as u32).max(2) & !1; - (even(width), even(height)) - } - fn scale(&mut self, decoded: &frame::Video) -> Option<(Vec, u32, u32)> { let input = (decoded.format(), decoded.width(), decoded.height()); if input.1 == 0 || input.2 == 0 { @@ -640,7 +715,14 @@ impl Reader { .as_ref() .is_none_or(|scaler| scaler.input != input) { - let size = self.output_size(input.1, input.2); + // Frames copied out of a hardware surface lose their aspect ratio. + let sample_aspect = match decoded.aspect_ratio() { + ratio if ratio.numerator() > 0 && ratio.denominator() > 0 => { + (ratio.numerator(), ratio.denominator()) + } + _ => self.sample_aspect, + }; + let size = output_size(input.1, input.2, sample_aspect, self.max_dimension); let bt709 = decoded.color_space() == ffmpeg::util::color::Space::BT709; self.scaler = Some(Scaler::new(input, size, bt709)?); } @@ -660,6 +742,9 @@ impl Reader { let start = row * stride; data.extend_from_slice(&source[start..start + row_bytes]); } + if self.quarter_turns != 0 { + return Some(rotate_rgba(&data, width, height, self.quarter_turns)); + } Some((data, width, height)) } } @@ -886,6 +971,98 @@ mod tests { } } + fn write_gif_frames(path: &Path, frames: &[(usize, u32)]) { + let file = std::fs::File::create(path).unwrap(); + let mut encoder = GifEncoder::new(file); + encoder + .set_repeat(image::codecs::gif::Repeat::Infinite) + .unwrap(); + for &(colour, delay_ms) in frames { + let [r, g, b] = COLOURS[colour]; + let image = RgbaImage::from_pixel(96, 48, Rgba([r, g, b, 255])); + encoder + .encode_frame(Frame::from_parts( + image, + 0, + 0, + Delay::from_numer_denom_ms(delay_ms, 1), + )) + .unwrap(); + } + } + + #[test] + fn request_in_a_gap_read_ahead_skipped_gets_its_own_frame() { + ffmpeg::init().unwrap(); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("gaps.gif"); + let frames: Vec<_> = (0..40).map(|index| (index % 4, 100)).collect(); + write_gif_frames(&path, &frames); + + let mut reader = Reader::open(path, 64, 64 * 32 * 4 * 2).unwrap(); + for seconds in [0.5, 1.0] { + reader.frame_at(seconds).unwrap(); + while reader.wants_prefetch() { + reader.prefetch(); + } + } + for (seconds, frame) in [(1.7, 17), (2.05, 20), (2.3, 23)] { + let served = reader.frame_at(seconds).unwrap(); + assert_eq!(colour_index(&served), frame % 4, "at {seconds}s"); + } + } + + #[test] + fn gif_loops_after_its_last_frame_is_held() { + ffmpeg::init().unwrap(); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("held.gif"); + write_gif_frames(&path, &[(0, 100), (1, 100), (2, 2000)]); + + let mut reader = Reader::open(path, 64, HEAD_BUDGET_BYTES).unwrap(); + for (seconds, expected) in [(0.05, 0), (0.15, 1), (1.0, 2), (2.1, 2), (2.25, 0)] { + let served = reader.frame_at(seconds).unwrap(); + assert_eq!(colour_index(&served), expected, "at {seconds}s"); + } + } + + #[test] + fn display_matrix_turns_into_clockwise_quarter_turns() { + // av_display_rotation_set takes a clockwise angle; phones tag portrait + // video with a clockwise 90°. + for (clockwise, expected) in [(0.0, 0), (90.0, 1), (180.0, 2), (-90.0, 3)] { + let mut matrix = [0i32; 9]; + unsafe { ffmpeg::ffi::av_display_rotation_set(matrix.as_mut_ptr(), clockwise) }; + assert_eq!(quarter_turns_from_matrix(&matrix), expected, "{clockwise}°"); + } + } + + #[test] + fn rotation_moves_pixels_clockwise() { + let pixels: Vec = (0..6u8).flat_map(|value| [value; 4]).collect(); + let at = |data: &[u8], width: u32, x: u32, y: u32| data[((y * width + x) * 4) as usize]; + + let (turned, width, height) = rotate_rgba(&pixels, 3, 2, 1); + assert_eq!((width, height), (2, 3)); + assert_eq!([at(&turned, 2, 0, 0), at(&turned, 2, 1, 0)], [3, 0]); + assert_eq!([at(&turned, 2, 0, 2), at(&turned, 2, 1, 2)], [5, 2]); + + let (flipped, width, _) = rotate_rgba(&pixels, 3, 2, 2); + assert_eq!(at(&flipped, width, 0, 0), 5); + + let (back, width, height) = rotate_rgba(&pixels, 3, 2, 3); + assert_eq!((width, height), (2, 3)); + assert_eq!([at(&back, 2, 0, 0), at(&back, 2, 1, 0)], [2, 5]); + } + + #[test] + fn non_square_pixels_keep_their_display_shape() { + assert_eq!(output_size(720, 576, (64, 45), 1920), (1024, 576)); + assert_eq!(output_size(1920, 1080, (1, 1), 960), (960, 540)); + assert_eq!(output_size(1920, 1080, (0, 1), 960), (960, 540)); + assert_eq!(output_size(1440, 1080, (4, 3), 1920), (1920, 1080)); + } + #[test] fn poster_is_the_first_frame_scaled_down() { ffmpeg::init().unwrap(); @@ -900,6 +1077,7 @@ mod tests { width, height, time: 0.0, + end: f64::INFINITY, }; assert_eq!(colour_index(&frame), 0); }