diff --git a/crates/gpui/src/assets.rs b/crates/gpui/src/assets.rs index 31783dda6c..eea232788a 100644 --- a/crates/gpui/src/assets.rs +++ b/crates/gpui/src/assets.rs @@ -269,33 +269,31 @@ pub(crate) fn downscale_bgra(bytes: &[u8], source: Size, level: u8 let mut height = source.height.0.max(0) as usize; let mut pixels = Cow::Borrowed(bytes); for _ in 0..level { - if width <= 1 && height <= 1 { + if width == 0 || height == 0 || (width <= 1 && height <= 1) { break; } let half_width = width.div_ceil(2); let half_height = height.div_ceil(2); let mut half = vec![0u8; half_width * half_height * 4]; - for y in 0..half_height { - for x in 0..half_width { - let mut color = [0u32; 3]; - let mut alpha = 0u32; - let mut count = 0u32; - for source_y in (2 * y)..(2 * y + 2).min(height) { - for source_x in (2 * x)..(2 * x + 2).min(width) { - let pixel = &pixels[(source_y * width + source_x) * 4..][..4]; - let a = u32::from(pixel[3]); - for (sum, channel) in color.iter_mut().zip(&pixel[..3]) { - *sum += u32::from(*channel) * a; - } - alpha += a; - count += 1; - } + for (y, out_row) in half.chunks_exact_mut(half_width * 4).enumerate() { + let out_row = out_row.as_chunks_mut::<4>().0; + let row = |row: usize| pixels[row * width * 4..][..width * 4].as_chunks::<4>().0; + let (top_pairs, top_rest) = row(2 * y).as_chunks::<2>(); + if 2 * y + 1 < height { + let (bottom_pairs, bottom_rest) = row(2 * y + 1).as_chunks::<2>(); + for ((out, [a, b]), [c, d]) in out_row.iter_mut().zip(top_pairs).zip(bottom_pairs) { + *out = average_bgra([a, b, c, d]); } - let out = &mut half[(y * half_width + x) * 4..][..4]; - for (channel, sum) in out.iter_mut().zip(color) { - *channel = (sum + alpha / 2).checked_div(alpha).unwrap_or(0) as u8; + if let ([a], [c]) = (top_rest, bottom_rest) { + out_row[half_width - 1] = average_bgra([a, c]); + } + } else { + for (out, [a, b]) in out_row.iter_mut().zip(top_pairs) { + *out = average_bgra([a, b]); + } + if let [a] = top_rest { + out_row[half_width - 1] = average_bgra([a]); } - out[3] = ((alpha + count / 2) / count) as u8; } } pixels = Cow::Owned(half); @@ -305,6 +303,33 @@ pub(crate) fn downscale_bgra(bytes: &[u8], source: Size, level: u8 pixels.into_owned() } +/// The alpha-weighted average of `N` straight-alpha BGRA pixels, rounded to nearest. +/// +/// A fixed `N` lets the compiler unroll the sums. Fully opaque blocks, the common case for +/// photos and avatars, skip the per-channel division by total alpha. +#[inline(always)] +fn average_bgra(pixels: [&[u8; 4]; N]) -> [u8; 4] { + let count = N as u32; + let alpha: u32 = pixels.iter().map(|pixel| u32::from(pixel[3])).sum(); + let mut out = [0u8; 4]; + if alpha == 255 * count { + for (channel, out) in out[..3].iter_mut().enumerate() { + let sum: u32 = pixels.iter().map(|pixel| u32::from(pixel[channel])).sum(); + *out = ((sum + count / 2) / count) as u8; + } + } else { + for (channel, out) in out[..3].iter_mut().enumerate() { + let sum: u32 = pixels + .iter() + .map(|pixel| u32::from(pixel[channel]) * u32::from(pixel[3])) + .sum(); + *out = (sum + alpha / 2).checked_div(alpha).unwrap_or(0) as u8; + } + } + out[3] = ((alpha + count / 2) / count) as u8; + out +} + /// A cached and processed image, in BGRA format pub struct RenderImage { /// The ID associated with this image diff --git a/crates/gpui/src/elements/img.rs b/crates/gpui/src/elements/img.rs index 7d0b95ce48..283b0c4ab6 100644 --- a/crates/gpui/src/elements/img.rs +++ b/crates/gpui/src/elements/img.rs @@ -800,7 +800,10 @@ impl From for ImageCacheError { #[cfg(test)] mod tests { use super::*; - use crate::{ParentElement as _, TestAppContext, canvas, div, point, px, size}; + use crate::{ + Context, Entity, ParentElement as _, Render, Scene, TestApp, TestAppContext, TestAppWindow, + VisualTestContext, Window, canvas, div, point, px, size, + }; use image::{Frame, ImageBuffer, Rgba}; const TEST_IMG_ID: &str = "test-img"; @@ -959,34 +962,47 @@ mod tests { assert_eq!(corner_radius, Some(expected_corner_radius)); } - #[gpui::test] - fn oversized_image_uploads_a_copy_near_its_draw_size(cx: &mut TestAppContext) { - let window = cx.add_empty_window(); - let image = test_image_with_size(512, 512); - window.draw(point(px(0.), px(0.)), size(px(100.), px(100.)), |_, _| { + /// Draws `image` at `side` x `side` device pixels and returns the size of every tile the + /// frame painted. + fn draw_image_at( + window: &mut VisualTestContext, + image: &Arc, + side: f32, + ) -> Vec<(i32, i32)> { + let side = side / window.update(|window, _| window.scale_factor()); + window.draw(point(px(0.), px(0.)), size(px(600.), px(600.)), |_, _| { img(ImageSource::Render(image.clone())) - .size(px(32.)) + .size(px(side)) .into_any_element() }); + window.update(|window, _| painted_tile_sizes(&window.rendered_frame.scene)) + } - let (tile_size, target) = window.update(|window, _| { - let sprite = window - .rendered_frame - .scene - .polychrome_sprites - .last() - .expect("image should paint a sprite"); - (sprite.tile.bounds.size, 32. * window.scale_factor()) - }); - // The 512px source would leave bilinear sampling reading 4 of every 256 pixels. - for side in [tile_size.width.0, tile_size.height.0] { - let side = side as f32; - assert!( - side >= target && side < 2. * target, - "a {target}px draw uploaded a {side}px tile" - ); - } + fn painted_tile_sizes(scene: &Scene) -> Vec<(i32, i32)> { + scene + .polychrome_sprites + .iter() + .map(|sprite| { + let size = sprite.tile.bounds.size; + (size.width.0, size.height.0) + }) + .collect() + } + + #[gpui::test] + fn oversized_image_is_shrunk_off_the_ui_thread(cx: &mut TestAppContext) { + let window = cx.add_empty_window(); + let image = test_image_with_size(512, 512); + + // Shrinking 512px to 32px takes about a millisecond, too long for paint. Until the + // copy is ready there is no tile of this image to draw. + assert_eq!(draw_image_at(window, &image, 32.), vec![]); + assert!(!window.update(|window, _| window.has_image_atlas_entry(&image))); + + window.run_until_parked(); assert!(window.update(|window, _| window.has_image_atlas_entry(&image))); + // The 512px source would leave bilinear sampling reading 4 of every 256 pixels. + assert_eq!(draw_image_at(window, &image, 32.), vec![(32, 32)]); window.update(|window, _| window.drop_image(image.clone()).unwrap()); assert!( @@ -995,6 +1011,105 @@ mod tests { ); } + #[gpui::test] + fn resized_image_keeps_its_last_copy_until_the_new_one_is_ready(cx: &mut TestAppContext) { + let window = cx.add_empty_window(); + let image = test_image_with_size(512, 512); + draw_image_at(window, &image, 256.); + window.run_until_parked(); + assert_eq!(draw_image_at(window, &image, 256.), vec![(256, 256)]); + + assert_eq!( + draw_image_at(window, &image, 32.), + vec![(256, 256)], + "the 256px copy stands in while the 32px one is made" + ); + window.run_until_parked(); + assert_eq!(draw_image_at(window, &image, 32.), vec![(32, 32)]); + } + + #[gpui::test] + fn dropping_an_image_cancels_its_pending_copy(cx: &mut TestAppContext) { + let window = cx.add_empty_window(); + let image = test_image_with_size(512, 512); + draw_image_at(window, &image, 32.); + window.update(|window, _| window.drop_image(image.clone()).unwrap()); + + window.run_until_parked(); + assert!( + !window.update(|window, _| window.has_image_atlas_entry(&image)), + "a copy finished after the drop should not reach the atlas" + ); + } + + #[gpui::test] + fn animated_image_shrinks_every_frame_at_once(cx: &mut TestAppContext) { + let window = cx.add_empty_window(); + let frame = Frame::new(ImageBuffer::from_pixel(512, 512, Rgba([0, 0, 0, 0]))); + let image = Arc::new(RenderImage::new(SmallVec::from_elem(frame, 3))); + draw_image_at(window, &image, 32.); + + window.run_until_parked(); + // Only frame 0 was painted, but every frame has a copy, so playback never waits. + assert!(window.update(|window, _| window.has_image_atlas_entry(&image))); + } + + #[gpui::test] + fn small_image_is_shrunk_during_paint(cx: &mut TestAppContext) { + let window = cx.add_empty_window(); + let image = test_image_with_size(128, 128); + assert_eq!(draw_image_at(window, &image, 16.), vec![(16, 16)]); + } + + struct CachedImage { + image: Arc, + side: Pixels, + } + + impl Render for CachedImage { + fn render(&mut self, _window: &mut Window, _cx: &mut Context) -> impl IntoElement { + img(ImageSource::Render(self.image.clone())).size(self.side) + } + } + + struct CachedImageParent { + child: Entity, + side: Pixels, + } + + impl Render for CachedImageParent { + fn render(&mut self, _window: &mut Window, _cx: &mut Context) -> impl IntoElement { + let child_style = div().size(self.side).style().clone(); + div() + .size(px(100.)) + .child(self.child.clone().cached(child_style)) + } + } + + #[test] + fn finished_copy_repaints_the_cached_view_that_drew_the_image() { + let mut app = TestApp::new(); + let image = test_image_with_size(512, 512); + let mut window = app.open_window(|window, cx| { + let side = px(32. / window.scale_factor()); + CachedImageParent { + child: cx.new(|_| CachedImage { + image: image.clone(), + side, + }), + side, + } + }); + let tile_sizes = |window: &mut TestAppWindow| { + window.update(|_, window, _| painted_tile_sizes(&window.rendered_frame.scene)) + }; + // Opening the window painted the child before its copy existed. The child is cached, + // so only a notify makes it paint again. + app.run_until_parked(); + window.draw(); + assert_eq!(tile_sizes(&mut window), vec![(32, 32)]); + } + #[gpui::test] fn image_drawn_at_its_own_size_uploads_the_original(cx: &mut TestAppContext) { let window = cx.add_empty_window(); diff --git a/crates/gpui/src/window.rs b/crates/gpui/src/window.rs index b4bc22ac09..ab1a36f66b 100644 --- a/crates/gpui/src/window.rs +++ b/crates/gpui/src/window.rs @@ -1,24 +1,24 @@ use crate::{ Action, AnyDrag, AnyElement, AnyImageCache, AnyTooltip, AnyView, App, AppContext, Arena, Asset, - AsyncWindowContext, AtlasTile, AvailableSpace, BackdropFilter, Background, BorderStyle, Bounds, - BoxShadow, Capslock, Context, Corners, CursorHideMode, CursorStyle, Decorations, DevicePixels, - DispatchActionListener, DispatchNodeId, DispatchTree, DisplayId, Edges, Effect, Entity, - EntityId, EventEmitter, FileDropEvent, Filter, FilterBoundary, FontId, Global, GlobalElementId, - GlyphId, GlyphRenderMode, GpuSpecs, InputHandler, IntoElement, IsZero, KeyBinding, KeyContext, - KeyDownEvent, KeyEvent, Keystroke, KeystrokeEvent, LayoutId, Lerp, LineLayoutIndex, Modifiers, - ModifiersChangedEvent, MonochromeSprite, Motion, MouseButton, MouseEvent, MouseMoveEvent, - MouseUpEvent, PaintSurface, Path, Pixels, PlatformAtlas, PlatformDisplay, PlatformInput, - PlatformInputHandler, PlatformWindow, Point, PolychromeSprite, Priority, PromptButton, - PromptLevel, Quad, RasterizedGlyphFormat, Render, RenderGlyphParams, RenderImage, - RenderImageParams, RenderSvgParams, Replay, ResizeEdge, ResolvedDirection, - SMOOTH_SVG_SCALE_FACTOR, SUBPIXEL_VARIANTS_X, SUBPIXEL_VARIANTS_Y, ScaledFilter, ScaledPixels, - Scene, Shadow, SharedString, Size, StrikethroughStyle, Style, SubpixelSprite, SubscriberSet, - Subscription, SystemWindowTab, SystemWindowTabController, TabStopMap, TaffyLayoutEngine, Task, - TextInputConfiguration, TextInputStateChange, TextRenderingMode, TextStyle, - TextStyleRefinement, ThermalState, TransformationMatrix, Transition, TransitionState, - Underline, UnderlineStyle, UnicodeBidi, WindowAppearance, WindowBackgroundAppearance, - WindowBounds, WindowControls, WindowDecorations, WindowOptions, WindowParams, WindowTextSystem, - downscale_bgra, downscaled_image_size, + AsyncApp, AsyncWindowContext, AtlasTile, AvailableSpace, BackdropFilter, Background, + BorderStyle, Bounds, BoxShadow, Capslock, Context, Corners, CursorHideMode, CursorStyle, + Decorations, DevicePixels, DispatchActionListener, DispatchNodeId, DispatchTree, DisplayId, + Edges, Effect, Entity, EntityId, EventEmitter, FileDropEvent, Filter, FilterBoundary, FontId, + Global, GlobalElementId, GlyphId, GlyphRenderMode, GpuSpecs, ImageId, InputHandler, + IntoElement, IsZero, KeyBinding, KeyContext, KeyDownEvent, KeyEvent, Keystroke, KeystrokeEvent, + LayoutId, Lerp, LineLayoutIndex, Modifiers, ModifiersChangedEvent, MonochromeSprite, Motion, + MouseButton, MouseEvent, MouseMoveEvent, MouseUpEvent, PaintSurface, Path, Pixels, + PlatformAtlas, PlatformDisplay, PlatformInput, PlatformInputHandler, PlatformWindow, Point, + PolychromeSprite, Priority, PromptButton, PromptLevel, Quad, RasterizedGlyphFormat, Render, + RenderGlyphParams, RenderImage, RenderImageParams, RenderSvgParams, Replay, ResizeEdge, + ResolvedDirection, SMOOTH_SVG_SCALE_FACTOR, SUBPIXEL_VARIANTS_X, SUBPIXEL_VARIANTS_Y, + ScaledFilter, ScaledPixels, Scene, Shadow, SharedString, Size, StrikethroughStyle, Style, + SubpixelSprite, SubscriberSet, Subscription, SystemWindowTab, SystemWindowTabController, + TabStopMap, TaffyLayoutEngine, Task, TextInputConfiguration, TextInputStateChange, + TextRenderingMode, TextStyle, TextStyleRefinement, ThermalState, TransformationMatrix, + Transition, TransitionState, Underline, UnderlineStyle, UnicodeBidi, WindowAppearance, + WindowBackgroundAppearance, WindowBounds, WindowControls, WindowDecorations, WindowOptions, + WindowParams, WindowTextSystem, downscale_bgra, downscaled_image_size, gestures::{GestureTuning, RecognizedTouchGesture, TouchGestureRecognizer}, image_downscale_level, interactive::TouchEvent, @@ -1302,6 +1302,11 @@ pub struct Window { is_resizable: bool, is_minimizable: bool, sprite_atlas: Arc, + /// Box-filtered copies of oversized images being made off the UI thread, by image and + /// downscale level. See [`Window::paint_image`]. + pending_image_downscales: FxHashMap<(ImageId, u8), PendingImageDownscale>, + /// Runs those copies and brings each back to this window when it is done. + async_app: AsyncApp, text_system: Arc, text_rendering_mode: Rc>, rem_size: Pixels, @@ -1388,6 +1393,18 @@ pub struct Window { pub(crate) a11y: A11y, } +/// A box-filtered copy of every frame of an image, being made on the background executor. +struct PendingImageDownscale { + /// The views that painted the image while it was pending, to repaint once it is uploaded. + /// Empty when it was painted outside any view; the whole window is redrawn then. + views: SmallVec<[EntityId; 1]>, + task: Task<()>, +} + +/// Images of up to this many pixels are shrunk during paint, which takes microseconds, so +/// they never miss a frame. Larger ones are shrunk on the background executor. +const INLINE_IMAGE_DOWNSCALE_MAX_PIXELS: i64 = 128 * 128; + #[derive(Clone, Debug, Default)] struct ModifierState { modifiers: Modifiers, @@ -2067,6 +2084,8 @@ impl Window { is_resizable, is_minimizable, sprite_atlas, + pending_image_downscales: FxHashMap::default(), + async_app: cx.to_async(), text_system, text_rendering_mode: cx.text_rendering_mode.clone(), rem_size: px(16.), @@ -5105,6 +5124,10 @@ impl Window { /// The visible region rendered is `bounds.intersect(&image_bounds)`, with `corner_radii` /// applied to `bounds`. /// `corner_smoothing` ranges from circular corners at `0.0` to maximum smoothing at `1.0`. + /// + /// An image drawn well below its pixel size is drawn from a box-filtered copy. A large + /// image's copy is made off the UI thread: until it is ready, the image is drawn from a copy + /// already uploaded at another size, or not at all, and the view is repainted after. pub fn paint_image( &mut self, bounds: Bounds, @@ -5138,21 +5161,38 @@ impl Window { downscale_level, }; - let tile = self + let downscale_inline = downscale_level == 0 + || i64::from(source_size.width.0) * i64::from(source_size.height.0) + <= INLINE_IMAGE_DOWNSCALE_MAX_PIXELS; + let tile = if downscale_inline { + self.sprite_atlas + .get_or_insert_with(¶ms.into(), &mut || { + let bytes = data + .as_bytes(frame_index) + .expect("It's the caller's job to pass a valid frame index"); + if downscale_level == 0 { + return Ok(Some((source_size, Cow::Borrowed(bytes)))); + } + Ok(Some(( + downscaled_image_size(source_size, downscale_level), + Cow::Owned(downscale_bgra(bytes, source_size, downscale_level)), + ))) + })? + .expect("Callback above only returns Some") + } else if let Some(tile) = self .sprite_atlas - .get_or_insert_with(¶ms.into(), &mut || { - let bytes = data - .as_bytes(frame_index) - .expect("It's the caller's job to pass a valid frame index"); - if downscale_level == 0 { - return Ok(Some((source_size, Cow::Borrowed(bytes)))); - } - Ok(Some(( - downscaled_image_size(source_size, downscale_level), - Cow::Owned(downscale_bgra(bytes, source_size, downscale_level)), - ))) - })? - .expect("Callback above only returns Some"); + .get_or_insert_with(¶ms.clone().into(), &mut || Ok(None))? + { + tile + } else { + // Shrinking a large image takes milliseconds, so it happens off the UI thread. Until + // it lands, draw a copy already uploaded at another size, or nothing. + self.downscale_image_in_background(&data, downscale_level); + let Some(tile) = self.nearest_uploaded_image_tile(params, source_size)? else { + return Ok(()); + }; + tile + }; let visible_bounds_snapped = self.snap_bounds(visible_bounds); @@ -5234,8 +5274,131 @@ impl Window { }); } + /// Starts making a box-filtered copy of every frame of `data` at `downscale_level` on the + /// background executor, unless one is already being made. The views that painted the image + /// meanwhile are notified once the copy is in the atlas, so they paint it. + fn downscale_image_in_background(&mut self, data: &Arc, downscale_level: u8) { + let view = self.rendered_entity_stack.last().copied(); + let key = (data.id, downscale_level); + if let Some(pending) = self.pending_image_downscales.get_mut(&key) { + if let Some(view) = view + && !pending.views.contains(&view) + { + pending.views.push(view); + } + return; + } + + let mut cx = self.async_app.clone(); + let background = cx.background_executor().clone(); + let handle = self.handle; + let data = data.clone(); + let task = cx.foreground_executor().clone().spawn(async move { + let frames = background + .spawn(async move { + (0..data.frame_count()) + .map(|frame_index| { + let size = data.size(frame_index); + let bytes = data + .as_bytes(frame_index) + .expect("frame_index is below frame_count"); + ( + downscaled_image_size(size, downscale_level), + downscale_bgra(bytes, size, downscale_level), + ) + }) + .collect::>() + }) + .await; + handle + .update(&mut cx, |_, window, cx| { + window.finish_image_downscale(key, frames, cx) + }) + .ok(); + }); + self.pending_image_downscales.insert( + key, + PendingImageDownscale { + views: view.into_iter().collect(), + task, + }, + ); + } + + /// Uploads the frames a background downscale made and repaints the views waiting on them. + fn finish_image_downscale( + &mut self, + (image_id, downscale_level): (ImageId, u8), + frames: Vec<(Size, Vec)>, + cx: &mut App, + ) { + // A missing entry means `drop_image` ran after the copy was made; it stays out. + let Some(pending) = self + .pending_image_downscales + .remove(&(image_id, downscale_level)) + else { + return; + }; + for (frame_index, (size, bytes)) in frames.iter().enumerate() { + let params = RenderImageParams { + image_id, + frame_index, + downscale_level, + }; + self.sprite_atlas + .get_or_insert_with(¶ms.into(), &mut || { + Ok(Some((*size, Cow::Borrowed(bytes.as_slice())))) + }) + .log_err(); + } + if pending.views.is_empty() { + // Painted outside any view, as tests do: redraw everything. + self.refresh(); + } + for view in pending.views { + cx.notify(view); + } + // This runs inside that task, so let it finish rather than cancel it. + pending.task.detach(); + } + + /// The tile of the frame `params` names at the downscale level nearest its own that is + /// already in the atlas, if any. + fn nearest_uploaded_image_tile( + &self, + params: RenderImageParams, + source_size: Size, + ) -> Result> { + let wanted = params.downscale_level; + let max_level = max_image_downscale_level(source_size); + for distance in 1..=wanted.max(max_level - wanted.min(max_level)) { + // A larger copy aliases a little; a smaller one blurs a little. Neither for long. + let levels = [wanted.checked_sub(distance), wanted.checked_add(distance)]; + for downscale_level in levels.into_iter().flatten() { + if downscale_level > max_level { + continue; + } + let key = RenderImageParams { + downscale_level, + ..params + } + .into(); + if let Some(tile) = self + .sprite_atlas + .get_or_insert_with(&key, &mut || Ok(None))? + { + return Ok(Some(tile)); + } + } + } + Ok(None) + } + /// Removes an image from the sprite atlas. pub fn drop_image(&mut self, data: Arc) -> Result<()> { + // Cancels any copy still being made, so it never reaches the atlas. + self.pending_image_downscales + .retain(|(image_id, _), _| *image_id != data.id); for frame_index in 0..data.frame_count() { // Every size the image was painted at left its own downscaled copy. for downscale_level in 0..=max_image_downscale_level(data.size(frame_index)) {