From 13009652b0fa848a230b78fd74293597b196b062 Mon Sep 17 00:00:00 2001 From: Todd Beauchamp Date: Thu, 22 Oct 2026 14:33:56 -0400 Subject: [PATCH] gpui: shrink large images off the UI thread Paint used to box-filter an oversized image inline the first time it was drawn at each downscale level, which took 11 ms for a 2048px avatar on an M5 Max. Images over 128x128 pixels are now shrunk on the background executor, every frame at once. Until the copy is uploaded, paint draws the nearest copy already in the atlas, or nothing, and the views that painted the image are notified when it lands. drop_image cancels a pending copy. The filter itself is about 5x faster with bit-identical output: fixed-size 2x2 blocks over chunked rows, and no per-channel division for fully opaque blocks. --- crates/gpui/src/assets.rs | 65 ++++++--- crates/gpui/src/elements/img.rs | 163 +++++++++++++++++++---- crates/gpui/src/window.rs | 229 +++++++++++++++++++++++++++----- 3 files changed, 380 insertions(+), 77 deletions(-) diff --git a/crates/gpui/src/assets.rs b/crates/gpui/src/assets.rs index 31783dda6c..eea232788a 100644 --- a/crates/gpui/src/assets.rs +++ b/crates/gpui/src/assets.rs @@ -269,33 +269,31 @@ pub(crate) fn downscale_bgra(bytes: &[u8], source: Size, level: u8 let mut height = source.height.0.max(0) as usize; let mut pixels = Cow::Borrowed(bytes); for _ in 0..level { - if width <= 1 && height <= 1 { + if width == 0 || height == 0 || (width <= 1 && height <= 1) { break; } let half_width = width.div_ceil(2); let half_height = height.div_ceil(2); let mut half = vec![0u8; half_width * half_height * 4]; - for y in 0..half_height { - for x in 0..half_width { - let mut color = [0u32; 3]; - let mut alpha = 0u32; - let mut count = 0u32; - for source_y in (2 * y)..(2 * y + 2).min(height) { - for source_x in (2 * x)..(2 * x + 2).min(width) { - let pixel = &pixels[(source_y * width + source_x) * 4..][..4]; - let a = u32::from(pixel[3]); - for (sum, channel) in color.iter_mut().zip(&pixel[..3]) { - *sum += u32::from(*channel) * a; - } - alpha += a; - count += 1; - } + for (y, out_row) in half.chunks_exact_mut(half_width * 4).enumerate() { + let out_row = out_row.as_chunks_mut::<4>().0; + let row = |row: usize| pixels[row * width * 4..][..width * 4].as_chunks::<4>().0; + let (top_pairs, top_rest) = row(2 * y).as_chunks::<2>(); + if 2 * y + 1 < height { + let (bottom_pairs, bottom_rest) = row(2 * y + 1).as_chunks::<2>(); + for ((out, [a, b]), [c, d]) in out_row.iter_mut().zip(top_pairs).zip(bottom_pairs) { + *out = average_bgra([a, b, c, d]); } - let out = &mut half[(y * half_width + x) * 4..][..4]; - for (channel, sum) in out.iter_mut().zip(color) { - *channel = (sum + alpha / 2).checked_div(alpha).unwrap_or(0) as u8; + if let ([a], [c]) = (top_rest, bottom_rest) { + out_row[half_width - 1] = average_bgra([a, c]); + } + } else { + for (out, [a, b]) in out_row.iter_mut().zip(top_pairs) { + *out = average_bgra([a, b]); + } + if let [a] = top_rest { + out_row[half_width - 1] = average_bgra([a]); } - out[3] = ((alpha + count / 2) / count) as u8; } } pixels = Cow::Owned(half); @@ -305,6 +303,33 @@ pub(crate) fn downscale_bgra(bytes: &[u8], source: Size, level: u8 pixels.into_owned() } +/// The alpha-weighted average of `N` straight-alpha BGRA pixels, rounded to nearest. +/// +/// A fixed `N` lets the compiler unroll the sums. Fully opaque blocks, the common case for +/// photos and avatars, skip the per-channel division by total alpha. +#[inline(always)] +fn average_bgra(pixels: [&[u8; 4]; N]) -> [u8; 4] { + let count = N as u32; + let alpha: u32 = pixels.iter().map(|pixel| u32::from(pixel[3])).sum(); + let mut out = [0u8; 4]; + if alpha == 255 * count { + for (channel, out) in out[..3].iter_mut().enumerate() { + let sum: u32 = pixels.iter().map(|pixel| u32::from(pixel[channel])).sum(); + *out = ((sum + count / 2) / count) as u8; + } + } else { + for (channel, out) in out[..3].iter_mut().enumerate() { + let sum: u32 = pixels + .iter() + .map(|pixel| u32::from(pixel[channel]) * u32::from(pixel[3])) + .sum(); + *out = (sum + alpha / 2).checked_div(alpha).unwrap_or(0) as u8; + } + } + out[3] = ((alpha + count / 2) / count) as u8; + out +} + /// A cached and processed image, in BGRA format pub struct RenderImage { /// The ID associated with this image diff --git a/crates/gpui/src/elements/img.rs b/crates/gpui/src/elements/img.rs index 7d0b95ce48..283b0c4ab6 100644 --- a/crates/gpui/src/elements/img.rs +++ b/crates/gpui/src/elements/img.rs @@ -800,7 +800,10 @@ impl From for ImageCacheError { #[cfg(test)] mod tests { use super::*; - use crate::{ParentElement as _, TestAppContext, canvas, div, point, px, size}; + use crate::{ + Context, Entity, ParentElement as _, Render, Scene, TestApp, TestAppContext, TestAppWindow, + VisualTestContext, Window, canvas, div, point, px, size, + }; use image::{Frame, ImageBuffer, Rgba}; const TEST_IMG_ID: &str = "test-img"; @@ -959,34 +962,47 @@ mod tests { assert_eq!(corner_radius, Some(expected_corner_radius)); } - #[gpui::test] - fn oversized_image_uploads_a_copy_near_its_draw_size(cx: &mut TestAppContext) { - let window = cx.add_empty_window(); - let image = test_image_with_size(512, 512); - window.draw(point(px(0.), px(0.)), size(px(100.), px(100.)), |_, _| { + /// Draws `image` at `side` x `side` device pixels and returns the size of every tile the + /// frame painted. + fn draw_image_at( + window: &mut VisualTestContext, + image: &Arc, + side: f32, + ) -> Vec<(i32, i32)> { + let side = side / window.update(|window, _| window.scale_factor()); + window.draw(point(px(0.), px(0.)), size(px(600.), px(600.)), |_, _| { img(ImageSource::Render(image.clone())) - .size(px(32.)) + .size(px(side)) .into_any_element() }); + window.update(|window, _| painted_tile_sizes(&window.rendered_frame.scene)) + } - let (tile_size, target) = window.update(|window, _| { - let sprite = window - .rendered_frame - .scene - .polychrome_sprites - .last() - .expect("image should paint a sprite"); - (sprite.tile.bounds.size, 32. * window.scale_factor()) - }); - // The 512px source would leave bilinear sampling reading 4 of every 256 pixels. - for side in [tile_size.width.0, tile_size.height.0] { - let side = side as f32; - assert!( - side >= target && side < 2. * target, - "a {target}px draw uploaded a {side}px tile" - ); - } + fn painted_tile_sizes(scene: &Scene) -> Vec<(i32, i32)> { + scene + .polychrome_sprites + .iter() + .map(|sprite| { + let size = sprite.tile.bounds.size; + (size.width.0, size.height.0) + }) + .collect() + } + + #[gpui::test] + fn oversized_image_is_shrunk_off_the_ui_thread(cx: &mut TestAppContext) { + let window = cx.add_empty_window(); + let image = test_image_with_size(512, 512); + + // Shrinking 512px to 32px takes about a millisecond, too long for paint. Until the + // copy is ready there is no tile of this image to draw. + assert_eq!(draw_image_at(window, &image, 32.), vec![]); + assert!(!window.update(|window, _| window.has_image_atlas_entry(&image))); + + window.run_until_parked(); assert!(window.update(|window, _| window.has_image_atlas_entry(&image))); + // The 512px source would leave bilinear sampling reading 4 of every 256 pixels. + assert_eq!(draw_image_at(window, &image, 32.), vec![(32, 32)]); window.update(|window, _| window.drop_image(image.clone()).unwrap()); assert!( @@ -995,6 +1011,105 @@ mod tests { ); } + #[gpui::test] + fn resized_image_keeps_its_last_copy_until_the_new_one_is_ready(cx: &mut TestAppContext) { + let window = cx.add_empty_window(); + let image = test_image_with_size(512, 512); + draw_image_at(window, &image, 256.); + window.run_until_parked(); + assert_eq!(draw_image_at(window, &image, 256.), vec![(256, 256)]); + + assert_eq!( + draw_image_at(window, &image, 32.), + vec![(256, 256)], + "the 256px copy stands in while the 32px one is made" + ); + window.run_until_parked(); + assert_eq!(draw_image_at(window, &image, 32.), vec![(32, 32)]); + } + + #[gpui::test] + fn dropping_an_image_cancels_its_pending_copy(cx: &mut TestAppContext) { + let window = cx.add_empty_window(); + let image = test_image_with_size(512, 512); + draw_image_at(window, &image, 32.); + window.update(|window, _| window.drop_image(image.clone()).unwrap()); + + window.run_until_parked(); + assert!( + !window.update(|window, _| window.has_image_atlas_entry(&image)), + "a copy finished after the drop should not reach the atlas" + ); + } + + #[gpui::test] + fn animated_image_shrinks_every_frame_at_once(cx: &mut TestAppContext) { + let window = cx.add_empty_window(); + let frame = Frame::new(ImageBuffer::from_pixel(512, 512, Rgba([0, 0, 0, 0]))); + let image = Arc::new(RenderImage::new(SmallVec::from_elem(frame, 3))); + draw_image_at(window, &image, 32.); + + window.run_until_parked(); + // Only frame 0 was painted, but every frame has a copy, so playback never waits. + assert!(window.update(|window, _| window.has_image_atlas_entry(&image))); + } + + #[gpui::test] + fn small_image_is_shrunk_during_paint(cx: &mut TestAppContext) { + let window = cx.add_empty_window(); + let image = test_image_with_size(128, 128); + assert_eq!(draw_image_at(window, &image, 16.), vec![(16, 16)]); + } + + struct CachedImage { + image: Arc, + side: Pixels, + } + + impl Render for CachedImage { + fn render(&mut self, _window: &mut Window, _cx: &mut Context) -> impl IntoElement { + img(ImageSource::Render(self.image.clone())).size(self.side) + } + } + + struct CachedImageParent { + child: Entity, + side: Pixels, + } + + impl Render for CachedImageParent { + fn render(&mut self, _window: &mut Window, _cx: &mut Context) -> impl IntoElement { + let child_style = div().size(self.side).style().clone(); + div() + .size(px(100.)) + .child(self.child.clone().cached(child_style)) + } + } + + #[test] + fn finished_copy_repaints_the_cached_view_that_drew_the_image() { + let mut app = TestApp::new(); + let image = test_image_with_size(512, 512); + let mut window = app.open_window(|window, cx| { + let side = px(32. / window.scale_factor()); + CachedImageParent { + child: cx.new(|_| CachedImage { + image: image.clone(), + side, + }), + side, + } + }); + let tile_sizes = |window: &mut TestAppWindow| { + window.update(|_, window, _| painted_tile_sizes(&window.rendered_frame.scene)) + }; + // Opening the window painted the child before its copy existed. The child is cached, + // so only a notify makes it paint again. + app.run_until_parked(); + window.draw(); + assert_eq!(tile_sizes(&mut window), vec![(32, 32)]); + } + #[gpui::test] fn image_drawn_at_its_own_size_uploads_the_original(cx: &mut TestAppContext) { let window = cx.add_empty_window(); diff --git a/crates/gpui/src/window.rs b/crates/gpui/src/window.rs index b4bc22ac09..ab1a36f66b 100644 --- a/crates/gpui/src/window.rs +++ b/crates/gpui/src/window.rs @@ -1,24 +1,24 @@ use crate::{ Action, AnyDrag, AnyElement, AnyImageCache, AnyTooltip, AnyView, App, AppContext, Arena, Asset, - AsyncWindowContext, AtlasTile, AvailableSpace, BackdropFilter, Background, BorderStyle, Bounds, - BoxShadow, Capslock, Context, Corners, CursorHideMode, CursorStyle, Decorations, DevicePixels, - DispatchActionListener, DispatchNodeId, DispatchTree, DisplayId, Edges, Effect, Entity, - EntityId, EventEmitter, FileDropEvent, Filter, FilterBoundary, FontId, Global, GlobalElementId, - GlyphId, GlyphRenderMode, GpuSpecs, InputHandler, IntoElement, IsZero, KeyBinding, KeyContext, - KeyDownEvent, KeyEvent, Keystroke, KeystrokeEvent, LayoutId, Lerp, LineLayoutIndex, Modifiers, - ModifiersChangedEvent, MonochromeSprite, Motion, MouseButton, MouseEvent, MouseMoveEvent, - MouseUpEvent, PaintSurface, Path, Pixels, PlatformAtlas, PlatformDisplay, PlatformInput, - PlatformInputHandler, PlatformWindow, Point, PolychromeSprite, Priority, PromptButton, - PromptLevel, Quad, RasterizedGlyphFormat, Render, RenderGlyphParams, RenderImage, - RenderImageParams, RenderSvgParams, Replay, ResizeEdge, ResolvedDirection, - SMOOTH_SVG_SCALE_FACTOR, SUBPIXEL_VARIANTS_X, SUBPIXEL_VARIANTS_Y, ScaledFilter, ScaledPixels, - Scene, Shadow, SharedString, Size, StrikethroughStyle, Style, SubpixelSprite, SubscriberSet, - Subscription, SystemWindowTab, SystemWindowTabController, TabStopMap, TaffyLayoutEngine, Task, - TextInputConfiguration, TextInputStateChange, TextRenderingMode, TextStyle, - TextStyleRefinement, ThermalState, TransformationMatrix, Transition, TransitionState, - Underline, UnderlineStyle, UnicodeBidi, WindowAppearance, WindowBackgroundAppearance, - WindowBounds, WindowControls, WindowDecorations, WindowOptions, WindowParams, WindowTextSystem, - downscale_bgra, downscaled_image_size, + AsyncApp, AsyncWindowContext, AtlasTile, AvailableSpace, BackdropFilter, Background, + BorderStyle, Bounds, BoxShadow, Capslock, Context, Corners, CursorHideMode, CursorStyle, + Decorations, DevicePixels, DispatchActionListener, DispatchNodeId, DispatchTree, DisplayId, + Edges, Effect, Entity, EntityId, EventEmitter, FileDropEvent, Filter, FilterBoundary, FontId, + Global, GlobalElementId, GlyphId, GlyphRenderMode, GpuSpecs, ImageId, InputHandler, + IntoElement, IsZero, KeyBinding, KeyContext, KeyDownEvent, KeyEvent, Keystroke, KeystrokeEvent, + LayoutId, Lerp, LineLayoutIndex, Modifiers, ModifiersChangedEvent, MonochromeSprite, Motion, + MouseButton, MouseEvent, MouseMoveEvent, MouseUpEvent, PaintSurface, Path, Pixels, + PlatformAtlas, PlatformDisplay, PlatformInput, PlatformInputHandler, PlatformWindow, Point, + PolychromeSprite, Priority, PromptButton, PromptLevel, Quad, RasterizedGlyphFormat, Render, + RenderGlyphParams, RenderImage, RenderImageParams, RenderSvgParams, Replay, ResizeEdge, + ResolvedDirection, SMOOTH_SVG_SCALE_FACTOR, SUBPIXEL_VARIANTS_X, SUBPIXEL_VARIANTS_Y, + ScaledFilter, ScaledPixels, Scene, Shadow, SharedString, Size, StrikethroughStyle, Style, + SubpixelSprite, SubscriberSet, Subscription, SystemWindowTab, SystemWindowTabController, + TabStopMap, TaffyLayoutEngine, Task, TextInputConfiguration, TextInputStateChange, + TextRenderingMode, TextStyle, TextStyleRefinement, ThermalState, TransformationMatrix, + Transition, TransitionState, Underline, UnderlineStyle, UnicodeBidi, WindowAppearance, + WindowBackgroundAppearance, WindowBounds, WindowControls, WindowDecorations, WindowOptions, + WindowParams, WindowTextSystem, downscale_bgra, downscaled_image_size, gestures::{GestureTuning, RecognizedTouchGesture, TouchGestureRecognizer}, image_downscale_level, interactive::TouchEvent, @@ -1302,6 +1302,11 @@ pub struct Window { is_resizable: bool, is_minimizable: bool, sprite_atlas: Arc, + /// Box-filtered copies of oversized images being made off the UI thread, by image and + /// downscale level. See [`Window::paint_image`]. + pending_image_downscales: FxHashMap<(ImageId, u8), PendingImageDownscale>, + /// Runs those copies and brings each back to this window when it is done. + async_app: AsyncApp, text_system: Arc, text_rendering_mode: Rc>, rem_size: Pixels, @@ -1388,6 +1393,18 @@ pub struct Window { pub(crate) a11y: A11y, } +/// A box-filtered copy of every frame of an image, being made on the background executor. +struct PendingImageDownscale { + /// The views that painted the image while it was pending, to repaint once it is uploaded. + /// Empty when it was painted outside any view; the whole window is redrawn then. + views: SmallVec<[EntityId; 1]>, + task: Task<()>, +} + +/// Images of up to this many pixels are shrunk during paint, which takes microseconds, so +/// they never miss a frame. Larger ones are shrunk on the background executor. +const INLINE_IMAGE_DOWNSCALE_MAX_PIXELS: i64 = 128 * 128; + #[derive(Clone, Debug, Default)] struct ModifierState { modifiers: Modifiers, @@ -2067,6 +2084,8 @@ impl Window { is_resizable, is_minimizable, sprite_atlas, + pending_image_downscales: FxHashMap::default(), + async_app: cx.to_async(), text_system, text_rendering_mode: cx.text_rendering_mode.clone(), rem_size: px(16.), @@ -5105,6 +5124,10 @@ impl Window { /// The visible region rendered is `bounds.intersect(&image_bounds)`, with `corner_radii` /// applied to `bounds`. /// `corner_smoothing` ranges from circular corners at `0.0` to maximum smoothing at `1.0`. + /// + /// An image drawn well below its pixel size is drawn from a box-filtered copy. A large + /// image's copy is made off the UI thread: until it is ready, the image is drawn from a copy + /// already uploaded at another size, or not at all, and the view is repainted after. pub fn paint_image( &mut self, bounds: Bounds, @@ -5138,21 +5161,38 @@ impl Window { downscale_level, }; - let tile = self + let downscale_inline = downscale_level == 0 + || i64::from(source_size.width.0) * i64::from(source_size.height.0) + <= INLINE_IMAGE_DOWNSCALE_MAX_PIXELS; + let tile = if downscale_inline { + self.sprite_atlas + .get_or_insert_with(¶ms.into(), &mut || { + let bytes = data + .as_bytes(frame_index) + .expect("It's the caller's job to pass a valid frame index"); + if downscale_level == 0 { + return Ok(Some((source_size, Cow::Borrowed(bytes)))); + } + Ok(Some(( + downscaled_image_size(source_size, downscale_level), + Cow::Owned(downscale_bgra(bytes, source_size, downscale_level)), + ))) + })? + .expect("Callback above only returns Some") + } else if let Some(tile) = self .sprite_atlas - .get_or_insert_with(¶ms.into(), &mut || { - let bytes = data - .as_bytes(frame_index) - .expect("It's the caller's job to pass a valid frame index"); - if downscale_level == 0 { - return Ok(Some((source_size, Cow::Borrowed(bytes)))); - } - Ok(Some(( - downscaled_image_size(source_size, downscale_level), - Cow::Owned(downscale_bgra(bytes, source_size, downscale_level)), - ))) - })? - .expect("Callback above only returns Some"); + .get_or_insert_with(¶ms.clone().into(), &mut || Ok(None))? + { + tile + } else { + // Shrinking a large image takes milliseconds, so it happens off the UI thread. Until + // it lands, draw a copy already uploaded at another size, or nothing. + self.downscale_image_in_background(&data, downscale_level); + let Some(tile) = self.nearest_uploaded_image_tile(params, source_size)? else { + return Ok(()); + }; + tile + }; let visible_bounds_snapped = self.snap_bounds(visible_bounds); @@ -5234,8 +5274,131 @@ impl Window { }); } + /// Starts making a box-filtered copy of every frame of `data` at `downscale_level` on the + /// background executor, unless one is already being made. The views that painted the image + /// meanwhile are notified once the copy is in the atlas, so they paint it. + fn downscale_image_in_background(&mut self, data: &Arc, downscale_level: u8) { + let view = self.rendered_entity_stack.last().copied(); + let key = (data.id, downscale_level); + if let Some(pending) = self.pending_image_downscales.get_mut(&key) { + if let Some(view) = view + && !pending.views.contains(&view) + { + pending.views.push(view); + } + return; + } + + let mut cx = self.async_app.clone(); + let background = cx.background_executor().clone(); + let handle = self.handle; + let data = data.clone(); + let task = cx.foreground_executor().clone().spawn(async move { + let frames = background + .spawn(async move { + (0..data.frame_count()) + .map(|frame_index| { + let size = data.size(frame_index); + let bytes = data + .as_bytes(frame_index) + .expect("frame_index is below frame_count"); + ( + downscaled_image_size(size, downscale_level), + downscale_bgra(bytes, size, downscale_level), + ) + }) + .collect::>() + }) + .await; + handle + .update(&mut cx, |_, window, cx| { + window.finish_image_downscale(key, frames, cx) + }) + .ok(); + }); + self.pending_image_downscales.insert( + key, + PendingImageDownscale { + views: view.into_iter().collect(), + task, + }, + ); + } + + /// Uploads the frames a background downscale made and repaints the views waiting on them. + fn finish_image_downscale( + &mut self, + (image_id, downscale_level): (ImageId, u8), + frames: Vec<(Size, Vec)>, + cx: &mut App, + ) { + // A missing entry means `drop_image` ran after the copy was made; it stays out. + let Some(pending) = self + .pending_image_downscales + .remove(&(image_id, downscale_level)) + else { + return; + }; + for (frame_index, (size, bytes)) in frames.iter().enumerate() { + let params = RenderImageParams { + image_id, + frame_index, + downscale_level, + }; + self.sprite_atlas + .get_or_insert_with(¶ms.into(), &mut || { + Ok(Some((*size, Cow::Borrowed(bytes.as_slice())))) + }) + .log_err(); + } + if pending.views.is_empty() { + // Painted outside any view, as tests do: redraw everything. + self.refresh(); + } + for view in pending.views { + cx.notify(view); + } + // This runs inside that task, so let it finish rather than cancel it. + pending.task.detach(); + } + + /// The tile of the frame `params` names at the downscale level nearest its own that is + /// already in the atlas, if any. + fn nearest_uploaded_image_tile( + &self, + params: RenderImageParams, + source_size: Size, + ) -> Result> { + let wanted = params.downscale_level; + let max_level = max_image_downscale_level(source_size); + for distance in 1..=wanted.max(max_level - wanted.min(max_level)) { + // A larger copy aliases a little; a smaller one blurs a little. Neither for long. + let levels = [wanted.checked_sub(distance), wanted.checked_add(distance)]; + for downscale_level in levels.into_iter().flatten() { + if downscale_level > max_level { + continue; + } + let key = RenderImageParams { + downscale_level, + ..params + } + .into(); + if let Some(tile) = self + .sprite_atlas + .get_or_insert_with(&key, &mut || Ok(None))? + { + return Ok(Some(tile)); + } + } + } + Ok(None) + } + /// Removes an image from the sprite atlas. pub fn drop_image(&mut self, data: Arc) -> Result<()> { + // Cancels any copy still being made, so it never reaches the atlas. + self.pending_image_downscales + .retain(|(image_id, _), _| *image_id != data.id); for frame_index in 0..data.frame_count() { // Every size the image was painted at left its own downscaled copy. for downscale_level in 0..=max_image_downscale_level(data.size(frame_index)) {