Skip to content

Shrink large images off the UI thread #20

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
toddbeauchamp wants to merge 1 commit into from
Closed

Shrink large images off the UI thread #20

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 45 additions & 20 deletions crates/gpui/src/assets.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -269,33 +269,31 @@ pub(crate) fn downscale_bgra(bytes: &[u8], source: Size<DevicePixels>, level: u8
let mut height = source.height.0.max(0) as usize;
let mut pixels = Cow::Borrowed(bytes);
for _ in 0..level {
if width <= 1 && height <= 1 {
if width == 0 || height == 0 || (width <= 1 && height <= 1) {
break;
}
let half_width = width.div_ceil(2);
let half_height = height.div_ceil(2);
let mut half = vec![0u8; half_width * half_height * 4];
for y in 0..half_height {
for x in 0..half_width {
let mut color = [0u32; 3];
let mut alpha = 0u32;
let mut count = 0u32;
for source_y in (2 * y)..(2 * y + 2).min(height) {
for source_x in (2 * x)..(2 * x + 2).min(width) {
let pixel = &pixels[(source_y * width + source_x) * 4..][..4];
let a = u32::from(pixel[3]);
for (sum, channel) in color.iter_mut().zip(&pixel[..3]) {
*sum += u32::from(*channel) * a;
}
alpha += a;
count += 1;
}
for (y, out_row) in half.chunks_exact_mut(half_width * 4).enumerate() {
let out_row = out_row.as_chunks_mut::<4>().0;
let row = |row: usize| pixels[row * width * 4..][..width * 4].as_chunks::<4>().0;
let (top_pairs, top_rest) = row(2 * y).as_chunks::<2>();
if 2 * y + 1 < height {
let (bottom_pairs, bottom_rest) = row(2 * y + 1).as_chunks::<2>();
for ((out, [a, b]), [c, d]) in out_row.iter_mut().zip(top_pairs).zip(bottom_pairs) {
*out = average_bgra([a, b, c, d]);
}
let out = &mut half[(y * half_width + x) * 4..][..4];
for (channel, sum) in out.iter_mut().zip(color) {
*channel = (sum + alpha / 2).checked_div(alpha).unwrap_or(0) as u8;
if let ([a], [c]) = (top_rest, bottom_rest) {
out_row[half_width - 1] = average_bgra([a, c]);
}
} else {
for (out, [a, b]) in out_row.iter_mut().zip(top_pairs) {
*out = average_bgra([a, b]);
}
if let [a] = top_rest {
out_row[half_width - 1] = average_bgra([a]);
}
out[3] = ((alpha + count / 2) / count) as u8;
}
}
pixels = Cow::Owned(half);
Expand All @@ -305,6 +303,33 @@ pub(crate) fn downscale_bgra(bytes: &[u8], source: Size<DevicePixels>, level: u8
pixels.into_owned()
}

/// The alpha-weighted average of `N` straight-alpha BGRA pixels, rounded to nearest.
///
/// A fixed `N` lets the compiler unroll the sums. Fully opaque blocks, the common case for
/// photos and avatars, skip the per-channel division by total alpha.
#[inline(always)]
fn average_bgra<const N: usize>(pixels: [&[u8; 4]; N]) -> [u8; 4] {
let count = N as u32;
let alpha: u32 = pixels.iter().map(|pixel| u32::from(pixel[3])).sum();
let mut out = [0u8; 4];
if alpha == 255 * count {
for (channel, out) in out[..3].iter_mut().enumerate() {
let sum: u32 = pixels.iter().map(|pixel| u32::from(pixel[channel])).sum();
*out = ((sum + count / 2) / count) as u8;
}
} else {
for (channel, out) in out[..3].iter_mut().enumerate() {
let sum: u32 = pixels
.iter()
.map(|pixel| u32::from(pixel[channel]) * u32::from(pixel[3]))
.sum();
*out = (sum + alpha / 2).checked_div(alpha).unwrap_or(0) as u8;
}
}
out[3] = ((alpha + count / 2) / count) as u8;
out
}

/// A cached and processed image, in BGRA format
pub struct RenderImage {
/// The ID associated with this image
Expand Down
163 changes: 139 additions & 24 deletions crates/gpui/src/elements/img.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -800,7 +800,10 @@ impl From<image::ImageError> for ImageCacheError {
#[cfg(test)]
mod tests {
use super::*;
use crate::{ParentElement as _, TestAppContext, canvas, div, point, px, size};
use crate::{
Context, Entity, ParentElement as _, Render, Scene, TestApp, TestAppContext, TestAppWindow,
VisualTestContext, Window, canvas, div, point, px, size,
};
use image::{Frame, ImageBuffer, Rgba};

const TEST_IMG_ID: &str = "test-img";
Expand Down Expand Up @@ -959,34 +962,47 @@ mod tests {
assert_eq!(corner_radius, Some(expected_corner_radius));
}

#[gpui::test]
fn oversized_image_uploads_a_copy_near_its_draw_size(cx: &mut TestAppContext) {
let window = cx.add_empty_window();
let image = test_image_with_size(512, 512);
window.draw(point(px(0.), px(0.)), size(px(100.), px(100.)), |_, _| {
/// Draws `image` at `side` x `side` device pixels and returns the size of every tile the
/// frame painted.
fn draw_image_at(
window: &mut VisualTestContext,
image: &Arc<RenderImage>,
side: f32,
) -> Vec<(i32, i32)> {
let side = side / window.update(|window, _| window.scale_factor());
window.draw(point(px(0.), px(0.)), size(px(600.), px(600.)), |_, _| {
img(ImageSource::Render(image.clone()))
.size(px(32.))
.size(px(side))
.into_any_element()
});
window.update(|window, _| painted_tile_sizes(&window.rendered_frame.scene))
}

let (tile_size, target) = window.update(|window, _| {
let sprite = window
.rendered_frame
.scene
.polychrome_sprites
.last()
.expect("image should paint a sprite");
(sprite.tile.bounds.size, 32. * window.scale_factor())
});
// The 512px source would leave bilinear sampling reading 4 of every 256 pixels.
for side in [tile_size.width.0, tile_size.height.0] {
let side = side as f32;
assert!(
side >= target && side < 2. * target,
"a {target}px draw uploaded a {side}px tile"
);
}
fn painted_tile_sizes(scene: &Scene) -> Vec<(i32, i32)> {
scene
.polychrome_sprites
.iter()
.map(|sprite| {
let size = sprite.tile.bounds.size;
(size.width.0, size.height.0)
})
.collect()
}

#[gpui::test]
fn oversized_image_is_shrunk_off_the_ui_thread(cx: &mut TestAppContext) {
let window = cx.add_empty_window();
let image = test_image_with_size(512, 512);

// Shrinking 512px to 32px takes about a millisecond, too long for paint. Until the
// copy is ready there is no tile of this image to draw.
assert_eq!(draw_image_at(window, &image, 32.), vec![]);
assert!(!window.update(|window, _| window.has_image_atlas_entry(&image)));

window.run_until_parked();
assert!(window.update(|window, _| window.has_image_atlas_entry(&image)));
// The 512px source would leave bilinear sampling reading 4 of every 256 pixels.
assert_eq!(draw_image_at(window, &image, 32.), vec![(32, 32)]);

window.update(|window, _| window.drop_image(image.clone()).unwrap());
assert!(
Expand All @@ -995,6 +1011,105 @@ mod tests {
);
}

#[gpui::test]
fn resized_image_keeps_its_last_copy_until_the_new_one_is_ready(cx: &mut TestAppContext) {
let window = cx.add_empty_window();
let image = test_image_with_size(512, 512);
draw_image_at(window, &image, 256.);
window.run_until_parked();
assert_eq!(draw_image_at(window, &image, 256.), vec![(256, 256)]);

assert_eq!(
draw_image_at(window, &image, 32.),
vec![(256, 256)],
"the 256px copy stands in while the 32px one is made"
);
window.run_until_parked();
assert_eq!(draw_image_at(window, &image, 32.), vec![(32, 32)]);
}

#[gpui::test]
fn dropping_an_image_cancels_its_pending_copy(cx: &mut TestAppContext) {
let window = cx.add_empty_window();
let image = test_image_with_size(512, 512);
draw_image_at(window, &image, 32.);
window.update(|window, _| window.drop_image(image.clone()).unwrap());

window.run_until_parked();
assert!(
!window.update(|window, _| window.has_image_atlas_entry(&image)),
"a copy finished after the drop should not reach the atlas"
);
}

#[gpui::test]
fn animated_image_shrinks_every_frame_at_once(cx: &mut TestAppContext) {
let window = cx.add_empty_window();
let frame = Frame::new(ImageBuffer::from_pixel(512, 512, Rgba([0, 0, 0, 0])));
let image = Arc::new(RenderImage::new(SmallVec::from_elem(frame, 3)));
draw_image_at(window, &image, 32.);

window.run_until_parked();
// Only frame 0 was painted, but every frame has a copy, so playback never waits.
assert!(window.update(|window, _| window.has_image_atlas_entry(&image)));
}

#[gpui::test]
fn small_image_is_shrunk_during_paint(cx: &mut TestAppContext) {
let window = cx.add_empty_window();
let image = test_image_with_size(128, 128);
assert_eq!(draw_image_at(window, &image, 16.), vec![(16, 16)]);
}

struct CachedImage {
image: Arc<RenderImage>,
side: Pixels,
}

impl Render for CachedImage {
fn render(&mut self, _window: &mut Window, _cx: &mut Context<Self>) -> impl IntoElement {
img(ImageSource::Render(self.image.clone())).size(self.side)
}
}

struct CachedImageParent {
child: Entity<CachedImage>,
side: Pixels,
}

impl Render for CachedImageParent {
fn render(&mut self, _window: &mut Window, _cx: &mut Context<Self>) -> impl IntoElement {
let child_style = div().size(self.side).style().clone();
div()
.size(px(100.))
.child(self.child.clone().cached(child_style))
}
}

#[test]
fn finished_copy_repaints_the_cached_view_that_drew_the_image() {
let mut app = TestApp::new();
let image = test_image_with_size(512, 512);
let mut window = app.open_window(|window, cx| {
let side = px(32. / window.scale_factor());
CachedImageParent {
child: cx.new(|_| CachedImage {
image: image.clone(),
side,
}),
side,
}
});
let tile_sizes = |window: &mut TestAppWindow<CachedImageParent>| {
window.update(|_, window, _| painted_tile_sizes(&window.rendered_frame.scene))
};
// Opening the window painted the child before its copy existed. The child is cached,
// so only a notify makes it paint again.
app.run_until_parked();
window.draw();
assert_eq!(tile_sizes(&mut window), vec![(32, 32)]);
}

#[gpui::test]
fn image_drawn_at_its_own_size_uploads_the_original(cx: &mut TestAppContext) {
let window = cx.add_empty_window();
Expand Down
Loading
Loading