cgen: optimize etc__swizzle_ycck for 444 bgrx/rgbx

name                                             old speed     new speed     delta

wuffs_jpeg_decode_552k_24bpp_444/clang11         227MB/s ± 0%  275MB/s ± 0%  +20.76%  (p=0.008 n=5+5)
wuffs_jpeg_decode_552k_24bpp_444/gcc10           223MB/s ± 0%  249MB/s ± 0%  +11.51%  (p=0.008 n=5+5)
This commit is contained in:
Nigel Tao committed 2023-06-07 22:25:02 +10:00
1 parent a3e8531ecb
commit ca2909b6d7
3 files changed
+522 -30

No files matched your search

+26
View File
@@ -253,6 +253,32 @@ wuffs_base__color_ycc__as__color_u32(uint8_t yy, uint8_t cb, uint8_t cr) {
((0x00FF0000 & bb32) >> 16);
}
// wuffs_base__color_ycc__as__color_u32_abgr is like
// wuffs_base__color_ycc__as__color_u32 but the uint32_t returned is in
// 0xAABBGGRR order, not 0xAARRGGBB.
static inline uint32_t //
wuffs_base__color_ycc__as__color_u32_abgr(uint8_t yy, uint8_t cb, uint8_t cr) {
uint32_t yy32 = (((uint32_t)yy) << 16) | (1 << 15);
uint32_t cb32 = (((uint32_t)cb) - 0x80);
uint32_t cr32 = (((uint32_t)cr) - 0x80);
uint32_t rr32 = yy32 + (0x166E9 * cr32);
uint32_t gg32 = yy32 - (0x0581A * cb32) - (0x0B6D2 * cr32);
uint32_t bb32 = yy32 + (0x1C5A2 * cb32);
if (rr32 >> 24) {
rr32 = ~((uint32_t)(((int32_t)rr32) >> 31));
}
if (gg32 >> 24) {
gg32 = ~((uint32_t)(((int32_t)gg32) >> 31));
}
if (bb32 >> 24) {
bb32 = ~((uint32_t)(((int32_t)bb32) >> 31));
}
return 0xFF000000 | //
((0x00FF0000 & bb32) >> 0) | //
((0x00FF0000 & gg32) >> 8) | //
((0x00FF0000 & rr32) >> 16);
}
// --------
typedef uint8_t wuffs_base__pixel_blend;
+235 -15
View File
@@ -564,7 +564,10 @@ wuffs_base__pixel_swizzler__swizzle_ycc__general__box_filter(
uint32_t inv_h2,
uint32_t inv_v0,
uint32_t inv_v1,
uint32_t inv_v2) {
uint32_t inv_v2,
uint32_t half_width_for_2to1,
uint32_t half_height_for_2to1,
uint8_t* scratch_buffer_2k_ptr) {
// Convert an inv_h or inv_v value from {1, 2, 3, 4} to {12, 6, 4, 3}.
uint32_t h0_out_of_12 = 12u / inv_h0;
uint32_t h1_out_of_12 = 12u / inv_h1;
@@ -637,6 +640,196 @@ wuffs_base__pixel_swizzler__swizzle_ycc__general__box_filter(
// --------
// Specializes wuffs_base__pixel_swizzler__swizzle_ycc__general__box_filter.
static void //
wuffs_base__pixel_swizzler__swizzle_ycc__bgrx__box_filter(
wuffs_base__pixel_buffer* dst,
uint32_t width,
uint32_t height,
const uint8_t* src_ptr0,
const uint8_t* src_ptr1,
const uint8_t* src_ptr2,
uint32_t stride0,
uint32_t stride1,
uint32_t stride2,
uint32_t inv_h0,
uint32_t inv_h1,
uint32_t inv_h2,
uint32_t inv_v0,
uint32_t inv_v1,
uint32_t inv_v2,
uint32_t half_width_for_2to1,
uint32_t half_height_for_2to1,
uint8_t* scratch_buffer_2k_ptr) {
uint32_t h0_out_of_12 = 12u / inv_h0;
uint32_t h1_out_of_12 = 12u / inv_h1;
uint32_t h2_out_of_12 = 12u / inv_h2;
uint32_t v0_out_of_12 = 12u / inv_v0;
uint32_t v1_out_of_12 = 12u / inv_v1;
uint32_t v2_out_of_12 = 12u / inv_v2;
uint32_t iy0 = 0u;
uint32_t iy1 = 0u;
uint32_t iy2 = 0u;
uint32_t y = 0u;
while (true) {
const uint8_t* src_iter0 = src_ptr0;
const uint8_t* src_iter1 = src_ptr1;
const uint8_t* src_iter2 = src_ptr2;
size_t dst_stride = dst->private_impl.planes[0].stride;
uint8_t* dst_iter =
dst->private_impl.planes[0].ptr + (dst_stride * ((size_t)y));
uint32_t ix0 = 0u;
uint32_t ix1 = 0u;
uint32_t ix2 = 0u;
uint32_t x = 0u;
while (true) {
wuffs_base__poke_u32le__no_bounds_check( //
dst_iter, wuffs_base__color_ycc__as__color_u32( //
*src_iter0, *src_iter1, *src_iter2));
dst_iter += 4;
if ((x + 1u) == width) {
break;
}
x = x + 1u;
ix0 += h0_out_of_12;
if (ix0 >= 12u) {
ix0 = 0u;
src_iter0++;
}
ix1 += h1_out_of_12;
if (ix1 >= 12u) {
ix1 = 0u;
src_iter1++;
}
ix2 += h2_out_of_12;
if (ix2 >= 12u) {
ix2 = 0u;
src_iter2++;
}
}
if ((y + 1u) == height) {
break;
}
y = y + 1u;
iy0 += v0_out_of_12;
if (iy0 >= 12u) {
iy0 = 0u;
src_ptr0 += stride0;
}
iy1 += v1_out_of_12;
if (iy1 >= 12u) {
iy1 = 0u;
src_ptr1 += stride1;
}
iy2 += v2_out_of_12;
if (iy2 >= 12u) {
iy2 = 0u;
src_ptr2 += stride2;
}
}
}
// Specializes wuffs_base__pixel_swizzler__swizzle_ycc__general__box_filter.
static void //
wuffs_base__pixel_swizzler__swizzle_ycc__rgbx__box_filter(
wuffs_base__pixel_buffer* dst,
uint32_t width,
uint32_t height,
const uint8_t* src_ptr0,
const uint8_t* src_ptr1,
const uint8_t* src_ptr2,
uint32_t stride0,
uint32_t stride1,
uint32_t stride2,
uint32_t inv_h0,
uint32_t inv_h1,
uint32_t inv_h2,
uint32_t inv_v0,
uint32_t inv_v1,
uint32_t inv_v2,
uint32_t half_width_for_2to1,
uint32_t half_height_for_2to1,
uint8_t* scratch_buffer_2k_ptr) {
uint32_t h0_out_of_12 = 12u / inv_h0;
uint32_t h1_out_of_12 = 12u / inv_h1;
uint32_t h2_out_of_12 = 12u / inv_h2;
uint32_t v0_out_of_12 = 12u / inv_v0;
uint32_t v1_out_of_12 = 12u / inv_v1;
uint32_t v2_out_of_12 = 12u / inv_v2;
uint32_t iy0 = 0u;
uint32_t iy1 = 0u;
uint32_t iy2 = 0u;
uint32_t y = 0u;
while (true) {
const uint8_t* src_iter0 = src_ptr0;
const uint8_t* src_iter1 = src_ptr1;
const uint8_t* src_iter2 = src_ptr2;
size_t dst_stride = dst->private_impl.planes[0].stride;
uint8_t* dst_iter =
dst->private_impl.planes[0].ptr + (dst_stride * ((size_t)y));
uint32_t ix0 = 0u;
uint32_t ix1 = 0u;
uint32_t ix2 = 0u;
uint32_t x = 0u;
while (true) {
wuffs_base__poke_u32le__no_bounds_check( //
dst_iter, wuffs_base__color_ycc__as__color_u32_abgr( //
*src_iter0, *src_iter1, *src_iter2));
dst_iter += 4;
if ((x + 1u) == width) {
break;
}
x = x + 1u;
ix0 += h0_out_of_12;
if (ix0 >= 12u) {
ix0 = 0u;
src_iter0++;
}
ix1 += h1_out_of_12;
if (ix1 >= 12u) {
ix1 = 0u;
src_iter1++;
}
ix2 += h2_out_of_12;
if (ix2 >= 12u) {
ix2 = 0u;
src_iter2++;
}
}
if ((y + 1u) == height) {
break;
}
y = y + 1u;
iy0 += v0_out_of_12;
if (iy0 >= 12u) {
iy0 = 0u;
src_ptr0 += stride0;
}
iy1 += v1_out_of_12;
if (iy1 >= 12u) {
iy1 = 0u;
src_ptr1 += stride1;
}
iy2 += v2_out_of_12;
if (iy2 >= 12u) {
iy2 = 0u;
src_ptr2 += stride2;
}
}
}
// --------
// wuffs_base__pixel_swizzler__flattened_length is like
// wuffs_base__table__flattened_length but returns uint64_t (not size_t) and
// also accounts for subsampling.
@@ -784,27 +977,54 @@ wuffs_base__pixel_swizzler__swizzle_ycck(
return wuffs_base__make_status(NULL);
}
void (*func)(wuffs_base__pixel_buffer * dst, //
uint32_t width, //
uint32_t height, //
const uint8_t* src_ptr0, //
const uint8_t* src_ptr1, //
const uint8_t* src_ptr2, //
uint32_t stride0, //
uint32_t stride1, //
uint32_t stride2, //
uint32_t inv_h0, //
uint32_t inv_h1, //
uint32_t inv_h2, //
uint32_t inv_v0, //
uint32_t inv_v1, //
uint32_t inv_v2, //
uint32_t half_width_for_2to1, //
uint32_t half_height_for_2to1, //
uint8_t* scratch_buffer_2k_ptr) =
&wuffs_base__pixel_swizzler__swizzle_ycc__general__box_filter;
if (triangle_filter_for_2to1 &&
(wuffs_base__pixel_swizzler__has_triangle_upsampler(inv_h0, inv_v0) ||
wuffs_base__pixel_swizzler__has_triangle_upsampler(inv_h1, inv_v1) ||
wuffs_base__pixel_swizzler__has_triangle_upsampler(inv_h2, inv_v2))) {
wuffs_base__pixel_swizzler__swizzle_ycc__general__triangle_filter(
dst, width, height, //
src0.ptr, src1.ptr, src2.ptr, //
stride0, stride1, stride2, //
inv_h0, inv_h1, inv_h2, //
inv_v0, inv_v1, inv_v2, //
half_width_for_2to1, half_height_for_2to1, //
scratch_buffer_2k.ptr);
func = &wuffs_base__pixel_swizzler__swizzle_ycc__general__triangle_filter;
} else {
wuffs_base__pixel_swizzler__swizzle_ycc__general__box_filter(
dst, width, height, //
src0.ptr, src1.ptr, src2.ptr, //
stride0, stride1, stride2, //
inv_h0, inv_h1, inv_h2, //
inv_v0, inv_v1, inv_v2);
switch (dst->pixcfg.private_impl.pixfmt.repr) {
case WUFFS_BASE__PIXEL_FORMAT__BGRA_NONPREMUL:
case WUFFS_BASE__PIXEL_FORMAT__BGRA_PREMUL:
case WUFFS_BASE__PIXEL_FORMAT__BGRX:
func = &wuffs_base__pixel_swizzler__swizzle_ycc__bgrx__box_filter;
break;
case WUFFS_BASE__PIXEL_FORMAT__RGBA_NONPREMUL:
case WUFFS_BASE__PIXEL_FORMAT__RGBA_PREMUL:
case WUFFS_BASE__PIXEL_FORMAT__RGBX:
func = &wuffs_base__pixel_swizzler__swizzle_ycc__rgbx__box_filter;
break;
}
}
(*func)(dst, width, height, //
src0.ptr, src1.ptr, src2.ptr, //
stride0, stride1, stride2, //
inv_h0, inv_h1, inv_h2, //
inv_v0, inv_v1, inv_v2, //
half_width_for_2to1, half_height_for_2to1, //
scratch_buffer_2k.ptr);
return wuffs_base__make_status(NULL);
}
+261 -15
View File
@@ -3846,6 +3846,32 @@ wuffs_base__color_ycc__as__color_u32(uint8_t yy, uint8_t cb, uint8_t cr) {
((0x00FF0000 & bb32) >> 16);
}
// wuffs_base__color_ycc__as__color_u32_abgr is like
// wuffs_base__color_ycc__as__color_u32 but the uint32_t returned is in
// 0xAABBGGRR order, not 0xAARRGGBB.
static inline uint32_t //
wuffs_base__color_ycc__as__color_u32_abgr(uint8_t yy, uint8_t cb, uint8_t cr) {
uint32_t yy32 = (((uint32_t)yy) << 16) | (1 << 15);
uint32_t cb32 = (((uint32_t)cb) - 0x80);
uint32_t cr32 = (((uint32_t)cr) - 0x80);
uint32_t rr32 = yy32 + (0x166E9 * cr32);
uint32_t gg32 = yy32 - (0x0581A * cb32) - (0x0B6D2 * cr32);
uint32_t bb32 = yy32 + (0x1C5A2 * cb32);
if (rr32 >> 24) {
rr32 = ~((uint32_t)(((int32_t)rr32) >> 31));
}
if (gg32 >> 24) {
gg32 = ~((uint32_t)(((int32_t)gg32) >> 31));
}
if (bb32 >> 24) {
bb32 = ~((uint32_t)(((int32_t)bb32) >> 31));
}
return 0xFF000000 | //
((0x00FF0000 & bb32) >> 0) | //
((0x00FF0000 & gg32) >> 8) | //
((0x00FF0000 & rr32) >> 16);
}
// --------
typedef uint8_t wuffs_base__pixel_blend;
@@ -23258,7 +23284,10 @@ wuffs_base__pixel_swizzler__swizzle_ycc__general__box_filter(
uint32_t inv_h2,
uint32_t inv_v0,
uint32_t inv_v1,
uint32_t inv_v2) {
uint32_t inv_v2,
uint32_t half_width_for_2to1,
uint32_t half_height_for_2to1,
uint8_t* scratch_buffer_2k_ptr) {
// Convert an inv_h or inv_v value from {1, 2, 3, 4} to {12, 6, 4, 3}.
uint32_t h0_out_of_12 = 12u / inv_h0;
uint32_t h1_out_of_12 = 12u / inv_h1;
@@ -23331,6 +23360,196 @@ wuffs_base__pixel_swizzler__swizzle_ycc__general__box_filter(
// --------
// Specializes wuffs_base__pixel_swizzler__swizzle_ycc__general__box_filter.
static void //
wuffs_base__pixel_swizzler__swizzle_ycc__bgrx__box_filter(
wuffs_base__pixel_buffer* dst,
uint32_t width,
uint32_t height,
const uint8_t* src_ptr0,
const uint8_t* src_ptr1,
const uint8_t* src_ptr2,
uint32_t stride0,
uint32_t stride1,
uint32_t stride2,
uint32_t inv_h0,
uint32_t inv_h1,
uint32_t inv_h2,
uint32_t inv_v0,
uint32_t inv_v1,
uint32_t inv_v2,
uint32_t half_width_for_2to1,
uint32_t half_height_for_2to1,
uint8_t* scratch_buffer_2k_ptr) {
uint32_t h0_out_of_12 = 12u / inv_h0;
uint32_t h1_out_of_12 = 12u / inv_h1;
uint32_t h2_out_of_12 = 12u / inv_h2;
uint32_t v0_out_of_12 = 12u / inv_v0;
uint32_t v1_out_of_12 = 12u / inv_v1;
uint32_t v2_out_of_12 = 12u / inv_v2;
uint32_t iy0 = 0u;
uint32_t iy1 = 0u;
uint32_t iy2 = 0u;
uint32_t y = 0u;
while (true) {
const uint8_t* src_iter0 = src_ptr0;
const uint8_t* src_iter1 = src_ptr1;
const uint8_t* src_iter2 = src_ptr2;
size_t dst_stride = dst->private_impl.planes[0].stride;
uint8_t* dst_iter =
dst->private_impl.planes[0].ptr + (dst_stride * ((size_t)y));
uint32_t ix0 = 0u;
uint32_t ix1 = 0u;
uint32_t ix2 = 0u;
uint32_t x = 0u;
while (true) {
wuffs_base__poke_u32le__no_bounds_check( //
dst_iter, wuffs_base__color_ycc__as__color_u32( //
*src_iter0, *src_iter1, *src_iter2));
dst_iter += 4;
if ((x + 1u) == width) {
break;
}
x = x + 1u;
ix0 += h0_out_of_12;
if (ix0 >= 12u) {
ix0 = 0u;
src_iter0++;
}
ix1 += h1_out_of_12;
if (ix1 >= 12u) {
ix1 = 0u;
src_iter1++;
}
ix2 += h2_out_of_12;
if (ix2 >= 12u) {
ix2 = 0u;
src_iter2++;
}
}
if ((y + 1u) == height) {
break;
}
y = y + 1u;
iy0 += v0_out_of_12;
if (iy0 >= 12u) {
iy0 = 0u;
src_ptr0 += stride0;
}
iy1 += v1_out_of_12;
if (iy1 >= 12u) {
iy1 = 0u;
src_ptr1 += stride1;
}
iy2 += v2_out_of_12;
if (iy2 >= 12u) {
iy2 = 0u;
src_ptr2 += stride2;
}
}
}
// Specializes wuffs_base__pixel_swizzler__swizzle_ycc__general__box_filter.
static void //
wuffs_base__pixel_swizzler__swizzle_ycc__rgbx__box_filter(
wuffs_base__pixel_buffer* dst,
uint32_t width,
uint32_t height,
const uint8_t* src_ptr0,
const uint8_t* src_ptr1,
const uint8_t* src_ptr2,
uint32_t stride0,
uint32_t stride1,
uint32_t stride2,
uint32_t inv_h0,
uint32_t inv_h1,
uint32_t inv_h2,
uint32_t inv_v0,
uint32_t inv_v1,
uint32_t inv_v2,
uint32_t half_width_for_2to1,
uint32_t half_height_for_2to1,
uint8_t* scratch_buffer_2k_ptr) {
uint32_t h0_out_of_12 = 12u / inv_h0;
uint32_t h1_out_of_12 = 12u / inv_h1;
uint32_t h2_out_of_12 = 12u / inv_h2;
uint32_t v0_out_of_12 = 12u / inv_v0;
uint32_t v1_out_of_12 = 12u / inv_v1;
uint32_t v2_out_of_12 = 12u / inv_v2;
uint32_t iy0 = 0u;
uint32_t iy1 = 0u;
uint32_t iy2 = 0u;
uint32_t y = 0u;
while (true) {
const uint8_t* src_iter0 = src_ptr0;
const uint8_t* src_iter1 = src_ptr1;
const uint8_t* src_iter2 = src_ptr2;
size_t dst_stride = dst->private_impl.planes[0].stride;
uint8_t* dst_iter =
dst->private_impl.planes[0].ptr + (dst_stride * ((size_t)y));
uint32_t ix0 = 0u;
uint32_t ix1 = 0u;
uint32_t ix2 = 0u;
uint32_t x = 0u;
while (true) {
wuffs_base__poke_u32le__no_bounds_check( //
dst_iter, wuffs_base__color_ycc__as__color_u32_abgr( //
*src_iter0, *src_iter1, *src_iter2));
dst_iter += 4;
if ((x + 1u) == width) {
break;
}
x = x + 1u;
ix0 += h0_out_of_12;
if (ix0 >= 12u) {
ix0 = 0u;
src_iter0++;
}
ix1 += h1_out_of_12;
if (ix1 >= 12u) {
ix1 = 0u;
src_iter1++;
}
ix2 += h2_out_of_12;
if (ix2 >= 12u) {
ix2 = 0u;
src_iter2++;
}
}
if ((y + 1u) == height) {
break;
}
y = y + 1u;
iy0 += v0_out_of_12;
if (iy0 >= 12u) {
iy0 = 0u;
src_ptr0 += stride0;
}
iy1 += v1_out_of_12;
if (iy1 >= 12u) {
iy1 = 0u;
src_ptr1 += stride1;
}
iy2 += v2_out_of_12;
if (iy2 >= 12u) {
iy2 = 0u;
src_ptr2 += stride2;
}
}
}
// --------
// wuffs_base__pixel_swizzler__flattened_length is like
// wuffs_base__table__flattened_length but returns uint64_t (not size_t) and
// also accounts for subsampling.
@@ -23478,28 +23697,55 @@ wuffs_base__pixel_swizzler__swizzle_ycck(
return wuffs_base__make_status(NULL);
}
void (*func)(wuffs_base__pixel_buffer * dst, //
uint32_t width, //
uint32_t height, //
const uint8_t* src_ptr0, //
const uint8_t* src_ptr1, //
const uint8_t* src_ptr2, //
uint32_t stride0, //
uint32_t stride1, //
uint32_t stride2, //
uint32_t inv_h0, //
uint32_t inv_h1, //
uint32_t inv_h2, //
uint32_t inv_v0, //
uint32_t inv_v1, //
uint32_t inv_v2, //
uint32_t half_width_for_2to1, //
uint32_t half_height_for_2to1, //
uint8_t* scratch_buffer_2k_ptr) =
&wuffs_base__pixel_swizzler__swizzle_ycc__general__box_filter;
if (triangle_filter_for_2to1 &&
(wuffs_base__pixel_swizzler__has_triangle_upsampler(inv_h0, inv_v0) ||
wuffs_base__pixel_swizzler__has_triangle_upsampler(inv_h1, inv_v1) ||
wuffs_base__pixel_swizzler__has_triangle_upsampler(inv_h2, inv_v2))) {
wuffs_base__pixel_swizzler__swizzle_ycc__general__triangle_filter(
dst, width, height, //
src0.ptr, src1.ptr, src2.ptr, //
stride0, stride1, stride2, //
inv_h0, inv_h1, inv_h2, //
inv_v0, inv_v1, inv_v2, //
half_width_for_2to1, half_height_for_2to1, //
scratch_buffer_2k.ptr);
func = &wuffs_base__pixel_swizzler__swizzle_ycc__general__triangle_filter;
} else {
wuffs_base__pixel_swizzler__swizzle_ycc__general__box_filter(
dst, width, height, //
src0.ptr, src1.ptr, src2.ptr, //
stride0, stride1, stride2, //
inv_h0, inv_h1, inv_h2, //
inv_v0, inv_v1, inv_v2);
switch (dst->pixcfg.private_impl.pixfmt.repr) {
case WUFFS_BASE__PIXEL_FORMAT__BGRA_NONPREMUL:
case WUFFS_BASE__PIXEL_FORMAT__BGRA_PREMUL:
case WUFFS_BASE__PIXEL_FORMAT__BGRX:
func = &wuffs_base__pixel_swizzler__swizzle_ycc__bgrx__box_filter;
break;
case WUFFS_BASE__PIXEL_FORMAT__RGBA_NONPREMUL:
case WUFFS_BASE__PIXEL_FORMAT__RGBA_PREMUL:
case WUFFS_BASE__PIXEL_FORMAT__RGBX:
func = &wuffs_base__pixel_swizzler__swizzle_ycc__rgbx__box_filter;
break;
}
}
(*func)(dst, width, height, //
src0.ptr, src1.ptr, src2.ptr, //
stride0, stride1, stride2, //
inv_h0, inv_h1, inv_h2, //
inv_v0, inv_v1, inv_v2, //
half_width_for_2to1, half_height_for_2to1, //
scratch_buffer_2k.ptr);
return wuffs_base__make_status(NULL);
}