From 92cda94f6690e800525161f181264b0589fcd54d Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Tue, 19 Jan 2021 21:25:52 +1100 Subject: [PATCH] Add png.decoder.filter_4_distance_?_sse128 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit wuffs_png_decode_40k_24bpp/clang9 128MB/s ± 0% 131MB/s ± 0% +2.38% (p=0.008 n=5+5) wuffs_png_decode_552k_32bpp/clang9 169MB/s ± 0% 258MB/s ± 0% +52.72% (p=0.008 n=5+5) wuffs_png_decode_4002k_24bpp/clang9 129MB/s ± 1% 131MB/s ± 0% +1.79% (p=0.008 n=5+5) wuffs_png_decode_filt_4_dist_3/clang9 131MB/s ± 0% 540MB/s ± 0% +311.86% (p=0.016 n=4+5) wuffs_png_decode_filt_4_dist_4/clang9 138MB/s ± 0% 784MB/s ± 0% +469.66% (p=0.008 n=5+5) wuffs_png_decode_40k_24bpp/gcc10 131MB/s ± 0% 135MB/s ± 0% +3.48% (p=0.008 n=5+5) wuffs_png_decode_552k_32bpp/gcc10 167MB/s ± 0% 275MB/s ± 0% +65.10% (p=0.008 n=5+5) wuffs_png_decode_4002k_24bpp/gcc10 134MB/s ± 0% 136MB/s ± 0% +1.78% (p=0.008 n=5+5) wuffs_png_decode_filt_4_dist_3/gcc10 95.8MB/s ± 0% 594.7MB/s ± 0% +520.73% (p=0.008 n=5+5) wuffs_png_decode_filt_4_dist_4/gcc10 91.5MB/s ± 0% 795.3MB/s ± 0% +769.14% (p=0.008 n=5+5) --- lang/builtin/builtin.go | 8 ++ release/c/wuffs-unsupported-snapshot.c | 162 +++++++++++++++++++++++++ std/png/decode_filter_sse128.wuffs | 145 ++++++++++++++++++++++ std/png/decode_png.wuffs | 8 +- 4 files changed, 321 insertions(+), 2 deletions(-) diff --git a/lang/builtin/builtin.go b/lang/builtin/builtin.go index 60d804b0..66ba7721 100644 --- a/lang/builtin/builtin.go +++ b/lang/builtin/builtin.go @@ -539,10 +539,18 @@ var Funcs = []string{ "sse128_i.create_mm_set1_epi8(a: u8) sse128_i", + "sse128_i._mm_abs_epi16() sse128_i", + "sse128_i._mm_add_epi16(b: sse128_i) sse128_i", "sse128_i._mm_add_epi8(b: sse128_i) sse128_i", "sse128_i._mm_and_si128(b: sse128_i) sse128_i", "sse128_i._mm_avg_epu8(b: sse128_i) sse128_i", + "sse128_i._mm_blendv_epi8(b: sse128_i, mask: sse128_i) sse128_i", + "sse128_i._mm_cmpeq_epi16(b: sse128_i) sse128_i", + "sse128_i._mm_min_epi16(b: sse128_i) sse128_i", + "sse128_i._mm_packus_epi16(b: sse128_i) sse128_i", + "sse128_i._mm_sub_epi16(b: sse128_i) sse128_i", "sse128_i._mm_sub_epi8(b: sse128_i) sse128_i", + "sse128_i._mm_unpacklo_epi8(b: sse128_i) sse128_i", "sse128_i._mm_xor_si128(b: sse128_i) sse128_i", } diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index d6ed9d9c..ee2cac85 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -30306,6 +30306,22 @@ wuffs_png__decoder__filter_3_distance_4_sse128( wuffs_base__slice_u8 a_prev); #endif // defined(WUFFS_BASE__CPU_ARCH__X86_64) +#if defined(WUFFS_BASE__CPU_ARCH__X86_64) +static wuffs_base__empty_struct +wuffs_png__decoder__filter_4_distance_3_sse128( + wuffs_png__decoder* self, + wuffs_base__slice_u8 a_curr, + wuffs_base__slice_u8 a_prev); +#endif // defined(WUFFS_BASE__CPU_ARCH__X86_64) + +#if defined(WUFFS_BASE__CPU_ARCH__X86_64) +static wuffs_base__empty_struct +wuffs_png__decoder__filter_4_distance_4_sse128( + wuffs_png__decoder* self, + wuffs_base__slice_u8 a_curr, + wuffs_base__slice_u8 a_prev); +#endif // defined(WUFFS_BASE__CPU_ARCH__X86_64) + static wuffs_base__empty_struct wuffs_png__decoder__choose_filter_implementations( wuffs_png__decoder* self); @@ -31144,6 +31160,146 @@ wuffs_png__decoder__filter_3_distance_4_sse128( } #endif // defined(WUFFS_BASE__CPU_ARCH__X86_64) +// -------- func png.decoder.filter_4_distance_3_sse128 + +#if defined(WUFFS_BASE__CPU_ARCH__X86_64) +#if defined(__GNUC__) +__attribute__((target("sse4.2"))) +#endif +static wuffs_base__empty_struct +wuffs_png__decoder__filter_4_distance_3_sse128( + wuffs_png__decoder* self, + wuffs_base__slice_u8 a_curr, + wuffs_base__slice_u8 a_prev) { + wuffs_base__slice_u8 v_c = {0}; + wuffs_base__slice_u8 v_p = {0}; + __m128i v_x128 = {0}; + __m128i v_a128 = {0}; + __m128i v_b128 = {0}; + __m128i v_c128 = {0}; + __m128i v_p128 = {0}; + __m128i v_pa128 = {0}; + __m128i v_pb128 = {0}; + __m128i v_pc128 = {0}; + __m128i v_smallest128 = {0}; + __m128i v_z128 = {0}; + + { + wuffs_base__slice_u8 i_slice_c = a_curr; + v_c = i_slice_c; + wuffs_base__slice_u8 i_slice_p = a_prev; + v_p = i_slice_p; + i_slice_c.len = ((size_t)(wuffs_base__u64__min(i_slice_c.len, i_slice_p.len))); + v_c.len = 4; + v_p.len = 4; + uint8_t* i_end0_c = i_slice_c.ptr + wuffs_base__iterate_total_advance(i_slice_c.len, 4, 3); + while (v_c.ptr < i_end0_c) { + (v_b128 = _mm_cvtsi32_si128((int)(wuffs_base__peek_u32le__no_bounds_check(v_p.ptr))), wuffs_base__make_empty_struct()); + v_b128 = _mm_unpacklo_epi8(v_b128, v_z128); + v_pa128 = _mm_sub_epi16(v_b128, v_c128); + v_pb128 = _mm_sub_epi16(v_a128, v_c128); + v_pc128 = _mm_add_epi16(v_pa128, v_pb128); + v_pa128 = _mm_abs_epi16(v_pa128); + v_pb128 = _mm_abs_epi16(v_pb128); + v_pc128 = _mm_abs_epi16(v_pc128); + v_smallest128 = _mm_min_epi16(v_pc128, _mm_min_epi16(v_pb128, v_pa128)); + v_p128 = _mm_blendv_epi8(_mm_blendv_epi8(v_c128, v_b128, _mm_cmpeq_epi16(v_smallest128, v_pb128)), v_a128, _mm_cmpeq_epi16(v_smallest128, v_pa128)); + (v_x128 = _mm_cvtsi32_si128((int)(wuffs_base__peek_u32le__no_bounds_check(v_c.ptr))), wuffs_base__make_empty_struct()); + v_x128 = _mm_unpacklo_epi8(v_x128, v_z128); + v_x128 = _mm_add_epi8(v_x128, v_p128); + v_a128 = v_x128; + v_c128 = v_b128; + v_x128 = _mm_packus_epi16(v_x128, v_x128); + (wuffs_base__poke_u24le__no_bounds_check(v_c.ptr, ((uint32_t)(_mm_cvtsi128_si32(v_x128)))), wuffs_base__make_empty_struct()); + v_c.ptr += 3; + v_p.ptr += 3; + } + v_c.len = 3; + v_p.len = 3; + uint8_t* i_end1_c = i_slice_c.ptr + ((i_slice_c.len / 3) * 3); + while (v_c.ptr < i_end1_c) { + (v_b128 = _mm_cvtsi32_si128((int)(wuffs_base__peek_u24le__no_bounds_check(v_p.ptr))), wuffs_base__make_empty_struct()); + v_b128 = _mm_unpacklo_epi8(v_b128, v_z128); + v_pa128 = _mm_sub_epi16(v_b128, v_c128); + v_pb128 = _mm_sub_epi16(v_a128, v_c128); + v_pc128 = _mm_add_epi16(v_pa128, v_pb128); + v_pa128 = _mm_abs_epi16(v_pa128); + v_pb128 = _mm_abs_epi16(v_pb128); + v_pc128 = _mm_abs_epi16(v_pc128); + v_smallest128 = _mm_min_epi16(v_pc128, _mm_min_epi16(v_pb128, v_pa128)); + v_p128 = _mm_blendv_epi8(_mm_blendv_epi8(v_c128, v_b128, _mm_cmpeq_epi16(v_smallest128, v_pb128)), v_a128, _mm_cmpeq_epi16(v_smallest128, v_pa128)); + (v_x128 = _mm_cvtsi32_si128((int)(wuffs_base__peek_u24le__no_bounds_check(v_c.ptr))), wuffs_base__make_empty_struct()); + v_x128 = _mm_unpacklo_epi8(v_x128, v_z128); + v_x128 = _mm_add_epi8(v_x128, v_p128); + v_x128 = _mm_packus_epi16(v_x128, v_x128); + (wuffs_base__poke_u24le__no_bounds_check(v_c.ptr, ((uint32_t)(_mm_cvtsi128_si32(v_x128)))), wuffs_base__make_empty_struct()); + v_c.ptr += 3; + v_p.ptr += 3; + } + } + return wuffs_base__make_empty_struct(); +} +#endif // defined(WUFFS_BASE__CPU_ARCH__X86_64) + +// -------- func png.decoder.filter_4_distance_4_sse128 + +#if defined(WUFFS_BASE__CPU_ARCH__X86_64) +#if defined(__GNUC__) +__attribute__((target("sse4.2"))) +#endif +static wuffs_base__empty_struct +wuffs_png__decoder__filter_4_distance_4_sse128( + wuffs_png__decoder* self, + wuffs_base__slice_u8 a_curr, + wuffs_base__slice_u8 a_prev) { + wuffs_base__slice_u8 v_c = {0}; + wuffs_base__slice_u8 v_p = {0}; + __m128i v_x128 = {0}; + __m128i v_a128 = {0}; + __m128i v_b128 = {0}; + __m128i v_c128 = {0}; + __m128i v_p128 = {0}; + __m128i v_pa128 = {0}; + __m128i v_pb128 = {0}; + __m128i v_pc128 = {0}; + __m128i v_smallest128 = {0}; + __m128i v_z128 = {0}; + + { + wuffs_base__slice_u8 i_slice_c = a_curr; + v_c = i_slice_c; + wuffs_base__slice_u8 i_slice_p = a_prev; + v_p = i_slice_p; + i_slice_c.len = ((size_t)(wuffs_base__u64__min(i_slice_c.len, i_slice_p.len))); + v_c.len = 4; + v_p.len = 4; + uint8_t* i_end0_c = i_slice_c.ptr + ((i_slice_c.len / 4) * 4); + while (v_c.ptr < i_end0_c) { + (v_b128 = _mm_cvtsi32_si128((int)(wuffs_base__peek_u32le__no_bounds_check(v_p.ptr))), wuffs_base__make_empty_struct()); + v_b128 = _mm_unpacklo_epi8(v_b128, v_z128); + v_pa128 = _mm_sub_epi16(v_b128, v_c128); + v_pb128 = _mm_sub_epi16(v_a128, v_c128); + v_pc128 = _mm_add_epi16(v_pa128, v_pb128); + v_pa128 = _mm_abs_epi16(v_pa128); + v_pb128 = _mm_abs_epi16(v_pb128); + v_pc128 = _mm_abs_epi16(v_pc128); + v_smallest128 = _mm_min_epi16(v_pc128, _mm_min_epi16(v_pb128, v_pa128)); + v_p128 = _mm_blendv_epi8(_mm_blendv_epi8(v_c128, v_b128, _mm_cmpeq_epi16(v_smallest128, v_pb128)), v_a128, _mm_cmpeq_epi16(v_smallest128, v_pa128)); + (v_x128 = _mm_cvtsi32_si128((int)(wuffs_base__peek_u32le__no_bounds_check(v_c.ptr))), wuffs_base__make_empty_struct()); + v_x128 = _mm_unpacklo_epi8(v_x128, v_z128); + v_x128 = _mm_add_epi8(v_x128, v_p128); + v_a128 = v_x128; + v_c128 = v_b128; + v_x128 = _mm_packus_epi16(v_x128, v_x128); + (wuffs_base__poke_u32le__no_bounds_check(v_c.ptr, ((uint32_t)(_mm_cvtsi128_si32(v_x128)))), wuffs_base__make_empty_struct()); + v_c.ptr += 4; + v_p.ptr += 4; + } + } + return wuffs_base__make_empty_struct(); +} +#endif // defined(WUFFS_BASE__CPU_ARCH__X86_64) + // -------- func png.decoder.set_quirk_enabled WUFFS_BASE__MAYBE_STATIC wuffs_base__empty_struct @@ -31632,6 +31788,9 @@ wuffs_png__decoder__choose_filter_implementations( self->private_impl.choosy_filter_3 = ( &wuffs_png__decoder__filter_3_distance_3_fallback); self->private_impl.choosy_filter_4 = ( +#if defined(WUFFS_BASE__CPU_ARCH__X86_64) + wuffs_base__cpu_arch__have_sse128() ? &wuffs_png__decoder__filter_4_distance_3_sse128 : +#endif &wuffs_png__decoder__filter_4_distance_3_fallback); } else if (self->private_impl.f_filter_distance == 4) { self->private_impl.choosy_filter_1 = ( @@ -31645,6 +31804,9 @@ wuffs_png__decoder__choose_filter_implementations( #endif &wuffs_png__decoder__filter_3_distance_4_fallback); self->private_impl.choosy_filter_4 = ( +#if defined(WUFFS_BASE__CPU_ARCH__X86_64) + wuffs_base__cpu_arch__have_sse128() ? &wuffs_png__decoder__filter_4_distance_4_sse128 : +#endif &wuffs_png__decoder__filter_4_distance_4_fallback); } return wuffs_base__make_empty_struct(); diff --git a/std/png/decode_filter_sse128.wuffs b/std/png/decode_filter_sse128.wuffs index ac189841..5d447d5d 100644 --- a/std/png/decode_filter_sse128.wuffs +++ b/std/png/decode_filter_sse128.wuffs @@ -118,3 +118,148 @@ pri func decoder.filter_3_distance_4_sse128!(curr: slice base.u8, prev: slice ba } } } + +// -------- + +// Filter 4: Paeth. + +pri func decoder.filter_4_distance_3_sse128!(curr: slice base.u8, prev: slice base.u8), + choose cpu_arch >= sse128, +{ + // See the comments in filter_4_distance_4_sse128 for an explanation of how + // this works. That function's single loop is copied twice here, once with + // "length: 4" and once with "length: 3". It's generally faster to load 4 + // bytes at a time instead of 3. + // + // Differences between that function and this one are marked with a §. + + var c : slice base.u8 + var p : slice base.u8 + var x128 : base.sse128_i + var a128 : base.sse128_i + var b128 : base.sse128_i + var c128 : base.sse128_i + var p128 : base.sse128_i + var pa128 : base.sse128_i + var pb128 : base.sse128_i + var pc128 : base.sse128_i + var smallest128 : base.sse128_i + var z128 : base.sse128_i + + // § The advance is 3, not 4. + iterate (c = args.curr, p = args.prev)(length: 4, advance: 3, unroll: 1) { + b128.load_u32!(a: p.peek_u32le()) + b128 = b128._mm_unpacklo_epi8(b: z128) + pa128 = b128._mm_sub_epi16(b: c128) + pb128 = a128._mm_sub_epi16(b: c128) + pc128 = pa128._mm_add_epi16(b: pb128) + pa128 = pa128._mm_abs_epi16() + pb128 = pb128._mm_abs_epi16() + pc128 = pc128._mm_abs_epi16() + smallest128 = pc128._mm_min_epi16(b: pb128._mm_min_epi16(b: pa128)) + p128 = c128._mm_blendv_epi8( + b: b128, + mask: smallest128._mm_cmpeq_epi16(b: pb128))._mm_blendv_epi8( + b: a128, + mask: smallest128._mm_cmpeq_epi16(b: pa128)) + x128.load_u32!(a: c.peek_u32le()) + x128 = x128._mm_unpacklo_epi8(b: z128) + x128 = x128._mm_add_epi8(b: p128) + a128 = x128 + c128 = b128 + x128 = x128._mm_packus_epi16(b: x128) + // § poke_u24le replaces poke_u32le. + c.poke_u24le!(a: x128.truncate_u32()) + + // § The length and advance are both 3, not 4. + } else (length: 3, advance: 3, unroll: 1) { + // § peek_u24le_as_u32 replaces peek_u32le. + b128.load_u32!(a: p.peek_u24le_as_u32()) + b128 = b128._mm_unpacklo_epi8(b: z128) + pa128 = b128._mm_sub_epi16(b: c128) + pb128 = a128._mm_sub_epi16(b: c128) + pc128 = pa128._mm_add_epi16(b: pb128) + pa128 = pa128._mm_abs_epi16() + pb128 = pb128._mm_abs_epi16() + pc128 = pc128._mm_abs_epi16() + smallest128 = pc128._mm_min_epi16(b: pb128._mm_min_epi16(b: pa128)) + p128 = c128._mm_blendv_epi8( + b: b128, + mask: smallest128._mm_cmpeq_epi16(b: pb128))._mm_blendv_epi8( + b: a128, + mask: smallest128._mm_cmpeq_epi16(b: pa128)) + // § peek_u24le_as_u32 replaces peek_u32le. + x128.load_u32!(a: c.peek_u24le_as_u32()) + x128 = x128._mm_unpacklo_epi8(b: z128) + x128 = x128._mm_add_epi8(b: p128) + // § These assignments are unnecessary; this is the last iteration. + // a128 = x128 + // c128 = b128 + x128 = x128._mm_packus_epi16(b: x128) + // § poke_u24le replaces poke_u32le. + c.poke_u24le!(a: x128.truncate_u32()) + } +} + +pri func decoder.filter_4_distance_4_sse128!(curr: slice base.u8, prev: slice base.u8), + choose cpu_arch >= sse128, +{ + var c : slice base.u8 + var p : slice base.u8 + var x128 : base.sse128_i + var a128 : base.sse128_i + var b128 : base.sse128_i + var c128 : base.sse128_i + var p128 : base.sse128_i + var pa128 : base.sse128_i + var pb128 : base.sse128_i + var pc128 : base.sse128_i + var smallest128 : base.sse128_i + var z128 : base.sse128_i + + iterate (c = args.curr, p = args.prev)(length: 4, advance: 4, unroll: 1) { + // Load the pixel from the row above. + b128.load_u32!(a: p.peek_u32le()) + + // Convert from u8 to i16 by unpacking it with zeroes. + b128 = b128._mm_unpacklo_epi8(b: z128) + + // Compute: + // - pa128 = (m128 - a128) + // - pb128 = (m128 - b128) + // - pc128 = (m128 - c128) + // where m128 = (a128 + b128 - c128). + pa128 = b128._mm_sub_epi16(b: c128) + pb128 = a128._mm_sub_epi16(b: c128) + pc128 = pa128._mm_add_epi16(b: pb128) + + // Compute the smallest absolute value of pa128, pb128 and pc128. + pa128 = pa128._mm_abs_epi16() + pb128 = pb128._mm_abs_epi16() + pc128 = pc128._mm_abs_epi16() + smallest128 = pc128._mm_min_epi16(b: pb128._mm_min_epi16(b: pa128)) + + // The predictor, p128, is whichever of a128, b128 or c128 such that + // pa128, pb128 or pc128 matches this smallest absolute value. Ties are + // broken in favor of a128 then b128 then c128. + // + // The a._mm_blendv_epi8(b, mask) method picks b when mask is true. + p128 = c128._mm_blendv_epi8( + b: b128, + mask: smallest128._mm_cmpeq_epi16(b: pb128))._mm_blendv_epi8( + b: a128, + mask: smallest128._mm_cmpeq_epi16(b: pa128)) + + // Add the predictor to the residual and, for the next iteration, set + // its previous pixels, a128 and c128, to x128 and b128. + // + // Again, convert between u8 and i16 by unpacking and re-packing. + x128.load_u32!(a: c.peek_u32le()) + x128 = x128._mm_unpacklo_epi8(b: z128) + x128 = x128._mm_add_epi8(b: p128) + a128 = x128 + c128 = b128 + x128 = x128._mm_packus_epi16(b: x128) + c.poke_u32le!(a: x128.truncate_u32()) + } +} diff --git a/std/png/decode_png.wuffs b/std/png/decode_png.wuffs index 0dc152e9..ddbcc084 100644 --- a/std/png/decode_png.wuffs +++ b/std/png/decode_png.wuffs @@ -251,7 +251,9 @@ pri func decoder.choose_filter_implementations!() { if this.filter_distance == 3 { choose filter_1 = [filter_1_distance_3_fallback] choose filter_3 = [filter_3_distance_3_fallback] - choose filter_4 = [filter_4_distance_3_fallback] + choose filter_4 = [ + filter_4_distance_3_sse128, + filter_4_distance_3_fallback] } else if this.filter_distance == 4 { choose filter_1 = [ filter_1_distance_4_sse128, @@ -259,7 +261,9 @@ pri func decoder.choose_filter_implementations!() { choose filter_3 = [ filter_3_distance_4_sse128, filter_3_distance_4_fallback] - choose filter_4 = [filter_4_distance_4_fallback] + choose filter_4 = [ + filter_4_distance_4_sse128, + filter_4_distance_4_fallback] } }