cgen: optimize etc__swizzle_ycck for 420 bgrx/rgbx

name                                             old speed     new speed     delta

wuffs_jpeg_decode_30k_24bpp_progressive/clang11  172MB/s ± 0%  200MB/s ± 0%  +15.74%  (p=0.008 n=5+5)
wuffs_jpeg_decode_30k_24bpp_sequential/clang11   292MB/s ± 0%  362MB/s ± 0%  +23.88%  (p=0.008 n=5+5)
wuffs_jpeg_decode_77k_24bpp/clang11              256MB/s ± 0%  307MB/s ± 0%  +19.79%  (p=0.008 n=5+5)
wuffs_jpeg_decode_552k_24bpp_420/clang11         264MB/s ± 0%  318MB/s ± 0%  +20.32%  (p=0.008 n=5+5)
wuffs_jpeg_decode_4002k_24bpp/clang11            269MB/s ± 0%  320MB/s ± 0%  +19.13%  (p=0.008 n=5+5)

wuffs_jpeg_decode_30k_24bpp_progressive/gcc10    202MB/s ± 0%  224MB/s ± 0%  +11.01%  (p=0.008 n=5+5)
wuffs_jpeg_decode_30k_24bpp_sequential/gcc10     350MB/s ± 0%  454MB/s ± 0%  +29.83%  (p=0.008 n=5+5)
wuffs_jpeg_decode_77k_24bpp/gcc10                248MB/s ± 0%  311MB/s ± 0%  +24.97%  (p=0.008 n=5+5)
wuffs_jpeg_decode_552k_24bpp_420/gcc10           261MB/s ± 0%  327MB/s ± 0%  +25.49%  (p=0.008 n=5+5)
wuffs_jpeg_decode_4002k_24bpp/gcc10              259MB/s ± 0%  324MB/s ± 0%  +25.38%  (p=0.008 n=5+5)
This commit is contained in:
Nigel Tao committed 2023-06-09 19:21:28 +10:00
1 parent aec9003ea0
commit 379000a711
3 files changed
+369 -12

No files matched your search

+20 -1
View File
@@ -523,10 +523,14 @@ wuffs_base__pixel_swizzler__swizzle_ycc__general__triangle_filter(
first_column, //
(total_src_len2 >= half_width_for_2to1));
// ¡ dst_iter = etc
for (; x < end; x++) {
// ¡ BEGIN set_color_u32_at
wuffs_base__pixel_buffer__set_color_u32_at(
dst, x, y,
wuffs_base__color_ycc__as__color_u32(*up0++, *up1++, *up2++));
// ¡ END set_color_u32_at
}
}
@@ -827,7 +831,22 @@ wuffs_base__pixel_swizzler__swizzle_ycck(
(wuffs_base__pixel_swizzler__has_triangle_upsampler(inv_h0, inv_v0) ||
wuffs_base__pixel_swizzler__has_triangle_upsampler(inv_h1, inv_v1) ||
wuffs_base__pixel_swizzler__has_triangle_upsampler(inv_h2, inv_v2))) {
func = &wuffs_base__pixel_swizzler__swizzle_ycc__general__triangle_filter;
switch (dst->pixcfg.private_impl.pixfmt.repr) {
case WUFFS_BASE__PIXEL_FORMAT__BGRA_NONPREMUL:
case WUFFS_BASE__PIXEL_FORMAT__BGRA_PREMUL:
case WUFFS_BASE__PIXEL_FORMAT__BGRX:
func = &wuffs_base__pixel_swizzler__swizzle_ycc__bgrx__triangle_filter;
break;
case WUFFS_BASE__PIXEL_FORMAT__RGBA_NONPREMUL:
case WUFFS_BASE__PIXEL_FORMAT__RGBA_PREMUL:
case WUFFS_BASE__PIXEL_FORMAT__RGBX:
func = &wuffs_base__pixel_swizzler__swizzle_ycc__rgbx__triangle_filter;
break;
default:
func =
&wuffs_base__pixel_swizzler__swizzle_ycc__general__triangle_filter;
break;
}
} else {
switch (dst->pixcfg.private_impl.pixfmt.repr) {
+57 -10
View File
@@ -22,6 +22,13 @@ import (
func insertBasePixConvSubmoduleYcckC(buf *buffer) error {
src := embedBasePixConvSubmoduleYcckC.Trim()
generalTriangleFilter, err := extractCFunc(src, ""+
"static void //\n"+
"wuffs_base__pixel_swizzler__swizzle_ycc__general__triangle_filter(\n")
if err != nil {
return err
}
generalBoxFilter, err := extractCFunc(src, ""+
"static void //\n"+
"wuffs_base__pixel_swizzler__swizzle_ycc__general__box_filter(\n")
@@ -40,15 +47,24 @@ func insertBasePixConvSubmoduleYcckC(buf *buffer) error {
// ----
const dstIterEtc = "" +
const dstIterEtcSansX = "" +
"size_t dst_stride = dst->private_impl.planes[0].stride;\n" +
"uint8_t* dst_iter =\n" +
" dst->private_impl.planes[0].ptr + (dst_stride * ((size_t)y));\n"
const setColorU32At = "" +
const dstIterWithX = "" +
"size_t dst_stride = dst->private_impl.planes[0].stride;\n" +
"uint8_t* dst_iter =\n" +
" dst->private_impl.planes[0].ptr + (dst_stride * ((size_t)y)) + (4 * ((size_t)x));\n"
const setColorU32AtSrcIter = "" +
"wuffs_base__poke_u32le__no_bounds_check(\n" +
" dst_iter, wuffs_base__color_ycc__as__color_u32_FLAVOR(\n" +
" *src_iter0, *src_iter1, *src_iter2));\n" +
"dst_iter += 4;\n"
const setColorU32AtUp = "" +
"wuffs_base__poke_u32le__no_bounds_check(\n" +
" dst_iter, wuffs_base__color_ycc__as__color_u32_FLAVOR(\n" +
" *up0++, *up1++, *up2++));\n" +
"dst_iter += 4;\n"
const useIxHv11 = "" +
"src_iter0++;\n" +
"src_iter1++;\n" +
@@ -58,6 +74,37 @@ func insertBasePixConvSubmoduleYcckC(buf *buffer) error {
"src_ptr1 += stride1;\n" +
"src_ptr2 += stride2;\n"
// ----
generalTriangleFilterPatches := []struct {
newFuncNameLine string
patchMap map[string]string
}{{
"wuffs_base__pixel_swizzler__swizzle_ycc__bgrx__triangle_filter(\n",
map[string]string{
"// ¡ dst_iter = etc\n": dstIterWithX,
"// ¡ BEGIN set_color_u32_at\n": strings.Replace(setColorU32AtUp, "_FLAVOR", "", 1),
},
}, {
"wuffs_base__pixel_swizzler__swizzle_ycc__rgbx__triangle_filter(\n",
map[string]string{
"// ¡ dst_iter = etc\n": dstIterWithX,
"// ¡ BEGIN set_color_u32_at\n": strings.Replace(setColorU32AtUp, "_FLAVOR", "_abgr", 1),
},
}}
for _, p := range generalTriangleFilterPatches {
const oldFuncNameLine = "wuffs_base__pixel_swizzler__swizzle_ycc__general__triangle_filter(\n"
p.patchMap[oldFuncNameLine] = p.newFuncNameLine
if err := patchCFunc(buf, generalTriangleFilter, oldFuncNameLine, p.patchMap); err != nil {
return err
}
}
// ----
generalBoxFilterPatches := []struct {
newFuncNameLine string
patchMap map[string]string
@@ -65,22 +112,22 @@ func insertBasePixConvSubmoduleYcckC(buf *buffer) error {
"wuffs_base__pixel_swizzler__swizzle_ycc__bgrx__box_filter(\n",
map[string]string{
"// ¡ dst_iter = etc\n": dstIterEtc,
"// ¡ BEGIN set_color_u32_at\n": strings.Replace(setColorU32At, "_FLAVOR", "", 1),
"// ¡ dst_iter = etc\n": dstIterEtcSansX,
"// ¡ BEGIN set_color_u32_at\n": strings.Replace(setColorU32AtSrcIter, "_FLAVOR", "", 1),
},
}, {
"wuffs_base__pixel_swizzler__swizzle_ycc__rgbx__box_filter(\n",
map[string]string{
"// ¡ dst_iter = etc\n": dstIterEtc,
"// ¡ BEGIN set_color_u32_at\n": strings.Replace(setColorU32At, "_FLAVOR", "_abgr", 1),
"// ¡ dst_iter = etc\n": dstIterEtcSansX,
"// ¡ BEGIN set_color_u32_at\n": strings.Replace(setColorU32AtSrcIter, "_FLAVOR", "_abgr", 1),
},
}, {
"wuffs_base__pixel_swizzler__swizzle_ycc__bgrx__hv11(\n",
map[string]string{
"// ¡ dst_iter = etc\n": dstIterEtc,
"// ¡ BEGIN set_color_u32_at\n": strings.Replace(setColorU32At, "_FLAVOR", "", 1),
"// ¡ dst_iter = etc\n": dstIterEtcSansX,
"// ¡ BEGIN set_color_u32_at\n": strings.Replace(setColorU32AtSrcIter, "_FLAVOR", "", 1),
"// ¡ BEGIN declare iy\n": "",
"// ¡ BEGIN declare ix\n": "",
"// ¡ BEGIN use ix\n": useIxHv11,
@@ -90,8 +137,8 @@ func insertBasePixConvSubmoduleYcckC(buf *buffer) error {
"wuffs_base__pixel_swizzler__swizzle_ycc__rgbx__hv11(\n",
map[string]string{
"// ¡ dst_iter = etc\n": dstIterEtc,
"// ¡ BEGIN set_color_u32_at\n": strings.Replace(setColorU32At, "_FLAVOR", "_abgr", 1),
"// ¡ dst_iter = etc\n": dstIterEtcSansX,
"// ¡ BEGIN set_color_u32_at\n": strings.Replace(setColorU32AtSrcIter, "_FLAVOR", "_abgr", 1),
"// ¡ BEGIN declare iy\n": "",
"// ¡ BEGIN declare ix\n": "",
"// ¡ BEGIN use ix\n": useIxHv11,
+292 -1
View File
@@ -23360,6 +23360,282 @@ wuffs_base__pixel_swizzler__swizzle_ycc__general__box_filter(
// --------
// Specializes wuffs_base__pixel_swizzler__swizzle_ycc__general__triangle_filter.
static void //
wuffs_base__pixel_swizzler__swizzle_ycc__bgrx__triangle_filter(
wuffs_base__pixel_buffer* dst,
uint32_t width,
uint32_t height,
const uint8_t* src_ptr0,
const uint8_t* src_ptr1,
const uint8_t* src_ptr2,
uint32_t stride0,
uint32_t stride1,
uint32_t stride2,
uint32_t inv_h0,
uint32_t inv_h1,
uint32_t inv_h2,
uint32_t inv_v0,
uint32_t inv_v1,
uint32_t inv_v2,
uint32_t half_width_for_2to1,
uint32_t half_height_for_2to1,
uint8_t* scratch_buffer_2k_ptr) {
wuffs_base__pixel_swizzler__swizzle_ycc__upsample_func upfunc0 =
wuffs_base__pixel_swizzler__swizzle_ycc__upsample_funcs
[(inv_h0 - 1u) & 3u][(inv_v0 - 1u) & 3u];
wuffs_base__pixel_swizzler__swizzle_ycc__upsample_func upfunc1 =
wuffs_base__pixel_swizzler__swizzle_ycc__upsample_funcs
[(inv_h1 - 1u) & 3u][(inv_v1 - 1u) & 3u];
wuffs_base__pixel_swizzler__swizzle_ycc__upsample_func upfunc2 =
wuffs_base__pixel_swizzler__swizzle_ycc__upsample_funcs
[(inv_h2 - 1u) & 3u][(inv_v2 - 1u) & 3u];
uint32_t h1v2_bias = 1u;
wuffs_base__pixel_swizzler__swizzle_ycc__general__triangle_filter_edge_row(
dst, width, 0u, //
src_ptr0, src_ptr1, src_ptr2, //
stride0, stride1, stride2, //
inv_h0, inv_h1, inv_h2, //
inv_v0, inv_v1, inv_v2, //
half_width_for_2to1, //
h1v2_bias, //
scratch_buffer_2k_ptr, //
upfunc0, upfunc1, upfunc2);
h1v2_bias = 2u;
bool last_row = height == 2u * half_height_for_2to1;
uint32_t y_max_excl = last_row ? (height - 1u) : height;
uint32_t y;
for (y = 1u; y < y_max_excl; y++) {
const uint8_t* src0_major = src_ptr0 + ((y / inv_v0) * (size_t)stride0);
const uint8_t* src0_minor =
(inv_v0 != 2u)
? src0_major
: ((y & 1u) ? (src0_major + stride0) : (src0_major - stride0));
const uint8_t* src1_major = src_ptr1 + ((y / inv_v1) * (size_t)stride1);
const uint8_t* src1_minor =
(inv_v1 != 2u)
? src1_major
: ((y & 1u) ? (src1_major + stride1) : (src1_major - stride1));
const uint8_t* src2_major = src_ptr2 + ((y / inv_v2) * (size_t)stride2);
const uint8_t* src2_minor =
(inv_v2 != 2u)
? src2_major
: ((y & 1u) ? (src2_major + stride2) : (src2_major - stride2));
uint32_t total_src_len0 = 0u;
uint32_t total_src_len1 = 0u;
uint32_t total_src_len2 = 0u;
uint32_t x = 0u;
while (x < width) {
bool first_column = x == 0u;
uint32_t end = x + 672u;
if (end > width) {
end = width;
}
uint32_t src_len0 = ((end - x) + inv_h0 - 1u) / inv_h0;
uint32_t src_len1 = ((end - x) + inv_h1 - 1u) / inv_h1;
uint32_t src_len2 = ((end - x) + inv_h2 - 1u) / inv_h2;
total_src_len0 += src_len0;
total_src_len1 += src_len1;
total_src_len2 += src_len2;
const uint8_t* up0 = (*upfunc0)( //
scratch_buffer_2k_ptr + (0u * 672u), //
src0_major + (x / inv_h0), //
src0_minor + (x / inv_h0), //
src_len0, //
h1v2_bias, //
first_column, //
(total_src_len0 >= half_width_for_2to1));
const uint8_t* up1 = (*upfunc1)( //
scratch_buffer_2k_ptr + (1u * 672u), //
src1_major + (x / inv_h1), //
src1_minor + (x / inv_h1), //
src_len1, //
h1v2_bias, //
first_column, //
(total_src_len1 >= half_width_for_2to1));
const uint8_t* up2 = (*upfunc2)( //
scratch_buffer_2k_ptr + (2u * 672u), //
src2_major + (x / inv_h2), //
src2_minor + (x / inv_h2), //
src_len2, //
h1v2_bias, //
first_column, //
(total_src_len2 >= half_width_for_2to1));
size_t dst_stride = dst->private_impl.planes[0].stride;
uint8_t* dst_iter =
dst->private_impl.planes[0].ptr + (dst_stride * ((size_t)y)) + (4 * ((size_t)x));
for (; x < end; x++) {
wuffs_base__poke_u32le__no_bounds_check(
dst_iter, wuffs_base__color_ycc__as__color_u32(
*up0++, *up1++, *up2++));
dst_iter += 4;
}
}
h1v2_bias ^= 3u;
}
if (y_max_excl != height) {
wuffs_base__pixel_swizzler__swizzle_ycc__general__triangle_filter_edge_row(
dst, width, height - 1u, //
src_ptr0, src_ptr1, src_ptr2, //
stride0, stride1, stride2, //
inv_h0, inv_h1, inv_h2, //
inv_v0, inv_v1, inv_v2, //
half_width_for_2to1, //
h1v2_bias, //
scratch_buffer_2k_ptr, //
upfunc0, upfunc1, upfunc2);
}
}
// Specializes wuffs_base__pixel_swizzler__swizzle_ycc__general__triangle_filter.
static void //
wuffs_base__pixel_swizzler__swizzle_ycc__rgbx__triangle_filter(
wuffs_base__pixel_buffer* dst,
uint32_t width,
uint32_t height,
const uint8_t* src_ptr0,
const uint8_t* src_ptr1,
const uint8_t* src_ptr2,
uint32_t stride0,
uint32_t stride1,
uint32_t stride2,
uint32_t inv_h0,
uint32_t inv_h1,
uint32_t inv_h2,
uint32_t inv_v0,
uint32_t inv_v1,
uint32_t inv_v2,
uint32_t half_width_for_2to1,
uint32_t half_height_for_2to1,
uint8_t* scratch_buffer_2k_ptr) {
wuffs_base__pixel_swizzler__swizzle_ycc__upsample_func upfunc0 =
wuffs_base__pixel_swizzler__swizzle_ycc__upsample_funcs
[(inv_h0 - 1u) & 3u][(inv_v0 - 1u) & 3u];
wuffs_base__pixel_swizzler__swizzle_ycc__upsample_func upfunc1 =
wuffs_base__pixel_swizzler__swizzle_ycc__upsample_funcs
[(inv_h1 - 1u) & 3u][(inv_v1 - 1u) & 3u];
wuffs_base__pixel_swizzler__swizzle_ycc__upsample_func upfunc2 =
wuffs_base__pixel_swizzler__swizzle_ycc__upsample_funcs
[(inv_h2 - 1u) & 3u][(inv_v2 - 1u) & 3u];
uint32_t h1v2_bias = 1u;
wuffs_base__pixel_swizzler__swizzle_ycc__general__triangle_filter_edge_row(
dst, width, 0u, //
src_ptr0, src_ptr1, src_ptr2, //
stride0, stride1, stride2, //
inv_h0, inv_h1, inv_h2, //
inv_v0, inv_v1, inv_v2, //
half_width_for_2to1, //
h1v2_bias, //
scratch_buffer_2k_ptr, //
upfunc0, upfunc1, upfunc2);
h1v2_bias = 2u;
bool last_row = height == 2u * half_height_for_2to1;
uint32_t y_max_excl = last_row ? (height - 1u) : height;
uint32_t y;
for (y = 1u; y < y_max_excl; y++) {
const uint8_t* src0_major = src_ptr0 + ((y / inv_v0) * (size_t)stride0);
const uint8_t* src0_minor =
(inv_v0 != 2u)
? src0_major
: ((y & 1u) ? (src0_major + stride0) : (src0_major - stride0));
const uint8_t* src1_major = src_ptr1 + ((y / inv_v1) * (size_t)stride1);
const uint8_t* src1_minor =
(inv_v1 != 2u)
? src1_major
: ((y & 1u) ? (src1_major + stride1) : (src1_major - stride1));
const uint8_t* src2_major = src_ptr2 + ((y / inv_v2) * (size_t)stride2);
const uint8_t* src2_minor =
(inv_v2 != 2u)
? src2_major
: ((y & 1u) ? (src2_major + stride2) : (src2_major - stride2));
uint32_t total_src_len0 = 0u;
uint32_t total_src_len1 = 0u;
uint32_t total_src_len2 = 0u;
uint32_t x = 0u;
while (x < width) {
bool first_column = x == 0u;
uint32_t end = x + 672u;
if (end > width) {
end = width;
}
uint32_t src_len0 = ((end - x) + inv_h0 - 1u) / inv_h0;
uint32_t src_len1 = ((end - x) + inv_h1 - 1u) / inv_h1;
uint32_t src_len2 = ((end - x) + inv_h2 - 1u) / inv_h2;
total_src_len0 += src_len0;
total_src_len1 += src_len1;
total_src_len2 += src_len2;
const uint8_t* up0 = (*upfunc0)( //
scratch_buffer_2k_ptr + (0u * 672u), //
src0_major + (x / inv_h0), //
src0_minor + (x / inv_h0), //
src_len0, //
h1v2_bias, //
first_column, //
(total_src_len0 >= half_width_for_2to1));
const uint8_t* up1 = (*upfunc1)( //
scratch_buffer_2k_ptr + (1u * 672u), //
src1_major + (x / inv_h1), //
src1_minor + (x / inv_h1), //
src_len1, //
h1v2_bias, //
first_column, //
(total_src_len1 >= half_width_for_2to1));
const uint8_t* up2 = (*upfunc2)( //
scratch_buffer_2k_ptr + (2u * 672u), //
src2_major + (x / inv_h2), //
src2_minor + (x / inv_h2), //
src_len2, //
h1v2_bias, //
first_column, //
(total_src_len2 >= half_width_for_2to1));
size_t dst_stride = dst->private_impl.planes[0].stride;
uint8_t* dst_iter =
dst->private_impl.planes[0].ptr + (dst_stride * ((size_t)y)) + (4 * ((size_t)x));
for (; x < end; x++) {
wuffs_base__poke_u32le__no_bounds_check(
dst_iter, wuffs_base__color_ycc__as__color_u32_abgr(
*up0++, *up1++, *up2++));
dst_iter += 4;
}
}
h1v2_bias ^= 3u;
}
if (y_max_excl != height) {
wuffs_base__pixel_swizzler__swizzle_ycc__general__triangle_filter_edge_row(
dst, width, height - 1u, //
src_ptr0, src_ptr1, src_ptr2, //
stride0, stride1, stride2, //
inv_h0, inv_h1, inv_h2, //
inv_v0, inv_v1, inv_v2, //
half_width_for_2to1, //
h1v2_bias, //
scratch_buffer_2k_ptr, //
upfunc0, upfunc1, upfunc2);
}
}
// Specializes wuffs_base__pixel_swizzler__swizzle_ycc__general__box_filter.
static void //
wuffs_base__pixel_swizzler__swizzle_ycc__bgrx__box_filter(
@@ -23835,7 +24111,22 @@ wuffs_base__pixel_swizzler__swizzle_ycck(
(wuffs_base__pixel_swizzler__has_triangle_upsampler(inv_h0, inv_v0) ||
wuffs_base__pixel_swizzler__has_triangle_upsampler(inv_h1, inv_v1) ||
wuffs_base__pixel_swizzler__has_triangle_upsampler(inv_h2, inv_v2))) {
func = &wuffs_base__pixel_swizzler__swizzle_ycc__general__triangle_filter;
switch (dst->pixcfg.private_impl.pixfmt.repr) {
case WUFFS_BASE__PIXEL_FORMAT__BGRA_NONPREMUL:
case WUFFS_BASE__PIXEL_FORMAT__BGRA_PREMUL:
case WUFFS_BASE__PIXEL_FORMAT__BGRX:
func = &wuffs_base__pixel_swizzler__swizzle_ycc__bgrx__triangle_filter;
break;
case WUFFS_BASE__PIXEL_FORMAT__RGBA_NONPREMUL:
case WUFFS_BASE__PIXEL_FORMAT__RGBA_PREMUL:
case WUFFS_BASE__PIXEL_FORMAT__RGBX:
func = &wuffs_base__pixel_swizzler__swizzle_ycc__rgbx__triangle_filter;
break;
default:
func =
&wuffs_base__pixel_swizzler__swizzle_ycc__general__triangle_filter;
break;
}
} else {
switch (dst->pixcfg.private_impl.pixfmt.repr) {