Add png.decoder.filter_4_distance_?_sse128

wuffs_png_decode_40k_24bpp/clang9       128MB/s ± 0%    131MB/s ± 0%    +2.38%  (p=0.008 n=5+5)
wuffs_png_decode_552k_32bpp/clang9      169MB/s ± 0%    258MB/s ± 0%   +52.72%  (p=0.008 n=5+5)
wuffs_png_decode_4002k_24bpp/clang9     129MB/s ± 1%    131MB/s ± 0%    +1.79%  (p=0.008 n=5+5)

wuffs_png_decode_filt_4_dist_3/clang9   131MB/s ± 0%    540MB/s ± 0%  +311.86%  (p=0.016 n=4+5)
wuffs_png_decode_filt_4_dist_4/clang9   138MB/s ± 0%    784MB/s ± 0%  +469.66%  (p=0.008 n=5+5)

wuffs_png_decode_40k_24bpp/gcc10        131MB/s ± 0%    135MB/s ± 0%    +3.48%  (p=0.008 n=5+5)
wuffs_png_decode_552k_32bpp/gcc10       167MB/s ± 0%    275MB/s ± 0%   +65.10%  (p=0.008 n=5+5)
wuffs_png_decode_4002k_24bpp/gcc10      134MB/s ± 0%    136MB/s ± 0%    +1.78%  (p=0.008 n=5+5)

wuffs_png_decode_filt_4_dist_3/gcc10   95.8MB/s ± 0%  594.7MB/s ± 0%  +520.73%  (p=0.008 n=5+5)
wuffs_png_decode_filt_4_dist_4/gcc10   91.5MB/s ± 0%  795.3MB/s ± 0%  +769.14%  (p=0.008 n=5+5)
This commit is contained in:
Nigel Tao committed 2021-01-19 21:55:03 +11:00
1 parent a1788a0808
commit 92cda94f66
4 files changed
+321 -2

No files matched your search

+8
View File
@@ -539,10 +539,18 @@ var Funcs = []string{
"sse128_i.create_mm_set1_epi8(a: u8) sse128_i",
"sse128_i._mm_abs_epi16() sse128_i",
"sse128_i._mm_add_epi16(b: sse128_i) sse128_i",
"sse128_i._mm_add_epi8(b: sse128_i) sse128_i",
"sse128_i._mm_and_si128(b: sse128_i) sse128_i",
"sse128_i._mm_avg_epu8(b: sse128_i) sse128_i",
"sse128_i._mm_blendv_epi8(b: sse128_i, mask: sse128_i) sse128_i",
"sse128_i._mm_cmpeq_epi16(b: sse128_i) sse128_i",
"sse128_i._mm_min_epi16(b: sse128_i) sse128_i",
"sse128_i._mm_packus_epi16(b: sse128_i) sse128_i",
"sse128_i._mm_sub_epi16(b: sse128_i) sse128_i",
"sse128_i._mm_sub_epi8(b: sse128_i) sse128_i",
"sse128_i._mm_unpacklo_epi8(b: sse128_i) sse128_i",
"sse128_i._mm_xor_si128(b: sse128_i) sse128_i",
}
+162
View File
@@ -30306,6 +30306,22 @@ wuffs_png__decoder__filter_3_distance_4_sse128(
wuffs_base__slice_u8 a_prev);
#endif // defined(WUFFS_BASE__CPU_ARCH__X86_64)
#if defined(WUFFS_BASE__CPU_ARCH__X86_64)
static wuffs_base__empty_struct
wuffs_png__decoder__filter_4_distance_3_sse128(
wuffs_png__decoder* self,
wuffs_base__slice_u8 a_curr,
wuffs_base__slice_u8 a_prev);
#endif // defined(WUFFS_BASE__CPU_ARCH__X86_64)
#if defined(WUFFS_BASE__CPU_ARCH__X86_64)
static wuffs_base__empty_struct
wuffs_png__decoder__filter_4_distance_4_sse128(
wuffs_png__decoder* self,
wuffs_base__slice_u8 a_curr,
wuffs_base__slice_u8 a_prev);
#endif // defined(WUFFS_BASE__CPU_ARCH__X86_64)
static wuffs_base__empty_struct
wuffs_png__decoder__choose_filter_implementations(
wuffs_png__decoder* self);
@@ -31144,6 +31160,146 @@ wuffs_png__decoder__filter_3_distance_4_sse128(
}
#endif // defined(WUFFS_BASE__CPU_ARCH__X86_64)
// -------- func png.decoder.filter_4_distance_3_sse128
#if defined(WUFFS_BASE__CPU_ARCH__X86_64)
#if defined(__GNUC__)
__attribute__((target("sse4.2")))
#endif
static wuffs_base__empty_struct
wuffs_png__decoder__filter_4_distance_3_sse128(
wuffs_png__decoder* self,
wuffs_base__slice_u8 a_curr,
wuffs_base__slice_u8 a_prev) {
wuffs_base__slice_u8 v_c = {0};
wuffs_base__slice_u8 v_p = {0};
__m128i v_x128 = {0};
__m128i v_a128 = {0};
__m128i v_b128 = {0};
__m128i v_c128 = {0};
__m128i v_p128 = {0};
__m128i v_pa128 = {0};
__m128i v_pb128 = {0};
__m128i v_pc128 = {0};
__m128i v_smallest128 = {0};
__m128i v_z128 = {0};
{
wuffs_base__slice_u8 i_slice_c = a_curr;
v_c = i_slice_c;
wuffs_base__slice_u8 i_slice_p = a_prev;
v_p = i_slice_p;
i_slice_c.len = ((size_t)(wuffs_base__u64__min(i_slice_c.len, i_slice_p.len)));
v_c.len = 4;
v_p.len = 4;
uint8_t* i_end0_c = i_slice_c.ptr + wuffs_base__iterate_total_advance(i_slice_c.len, 4, 3);
while (v_c.ptr < i_end0_c) {
(v_b128 = _mm_cvtsi32_si128((int)(wuffs_base__peek_u32le__no_bounds_check(v_p.ptr))), wuffs_base__make_empty_struct());
v_b128 = _mm_unpacklo_epi8(v_b128, v_z128);
v_pa128 = _mm_sub_epi16(v_b128, v_c128);
v_pb128 = _mm_sub_epi16(v_a128, v_c128);
v_pc128 = _mm_add_epi16(v_pa128, v_pb128);
v_pa128 = _mm_abs_epi16(v_pa128);
v_pb128 = _mm_abs_epi16(v_pb128);
v_pc128 = _mm_abs_epi16(v_pc128);
v_smallest128 = _mm_min_epi16(v_pc128, _mm_min_epi16(v_pb128, v_pa128));
v_p128 = _mm_blendv_epi8(_mm_blendv_epi8(v_c128, v_b128, _mm_cmpeq_epi16(v_smallest128, v_pb128)), v_a128, _mm_cmpeq_epi16(v_smallest128, v_pa128));
(v_x128 = _mm_cvtsi32_si128((int)(wuffs_base__peek_u32le__no_bounds_check(v_c.ptr))), wuffs_base__make_empty_struct());
v_x128 = _mm_unpacklo_epi8(v_x128, v_z128);
v_x128 = _mm_add_epi8(v_x128, v_p128);
v_a128 = v_x128;
v_c128 = v_b128;
v_x128 = _mm_packus_epi16(v_x128, v_x128);
(wuffs_base__poke_u24le__no_bounds_check(v_c.ptr, ((uint32_t)(_mm_cvtsi128_si32(v_x128)))), wuffs_base__make_empty_struct());
v_c.ptr += 3;
v_p.ptr += 3;
}
v_c.len = 3;
v_p.len = 3;
uint8_t* i_end1_c = i_slice_c.ptr + ((i_slice_c.len / 3) * 3);
while (v_c.ptr < i_end1_c) {
(v_b128 = _mm_cvtsi32_si128((int)(wuffs_base__peek_u24le__no_bounds_check(v_p.ptr))), wuffs_base__make_empty_struct());
v_b128 = _mm_unpacklo_epi8(v_b128, v_z128);
v_pa128 = _mm_sub_epi16(v_b128, v_c128);
v_pb128 = _mm_sub_epi16(v_a128, v_c128);
v_pc128 = _mm_add_epi16(v_pa128, v_pb128);
v_pa128 = _mm_abs_epi16(v_pa128);
v_pb128 = _mm_abs_epi16(v_pb128);
v_pc128 = _mm_abs_epi16(v_pc128);
v_smallest128 = _mm_min_epi16(v_pc128, _mm_min_epi16(v_pb128, v_pa128));
v_p128 = _mm_blendv_epi8(_mm_blendv_epi8(v_c128, v_b128, _mm_cmpeq_epi16(v_smallest128, v_pb128)), v_a128, _mm_cmpeq_epi16(v_smallest128, v_pa128));
(v_x128 = _mm_cvtsi32_si128((int)(wuffs_base__peek_u24le__no_bounds_check(v_c.ptr))), wuffs_base__make_empty_struct());
v_x128 = _mm_unpacklo_epi8(v_x128, v_z128);
v_x128 = _mm_add_epi8(v_x128, v_p128);
v_x128 = _mm_packus_epi16(v_x128, v_x128);
(wuffs_base__poke_u24le__no_bounds_check(v_c.ptr, ((uint32_t)(_mm_cvtsi128_si32(v_x128)))), wuffs_base__make_empty_struct());
v_c.ptr += 3;
v_p.ptr += 3;
}
}
return wuffs_base__make_empty_struct();
}
#endif // defined(WUFFS_BASE__CPU_ARCH__X86_64)
// -------- func png.decoder.filter_4_distance_4_sse128
#if defined(WUFFS_BASE__CPU_ARCH__X86_64)
#if defined(__GNUC__)
__attribute__((target("sse4.2")))
#endif
static wuffs_base__empty_struct
wuffs_png__decoder__filter_4_distance_4_sse128(
wuffs_png__decoder* self,
wuffs_base__slice_u8 a_curr,
wuffs_base__slice_u8 a_prev) {
wuffs_base__slice_u8 v_c = {0};
wuffs_base__slice_u8 v_p = {0};
__m128i v_x128 = {0};
__m128i v_a128 = {0};
__m128i v_b128 = {0};
__m128i v_c128 = {0};
__m128i v_p128 = {0};
__m128i v_pa128 = {0};
__m128i v_pb128 = {0};
__m128i v_pc128 = {0};
__m128i v_smallest128 = {0};
__m128i v_z128 = {0};
{
wuffs_base__slice_u8 i_slice_c = a_curr;
v_c = i_slice_c;
wuffs_base__slice_u8 i_slice_p = a_prev;
v_p = i_slice_p;
i_slice_c.len = ((size_t)(wuffs_base__u64__min(i_slice_c.len, i_slice_p.len)));
v_c.len = 4;
v_p.len = 4;
uint8_t* i_end0_c = i_slice_c.ptr + ((i_slice_c.len / 4) * 4);
while (v_c.ptr < i_end0_c) {
(v_b128 = _mm_cvtsi32_si128((int)(wuffs_base__peek_u32le__no_bounds_check(v_p.ptr))), wuffs_base__make_empty_struct());
v_b128 = _mm_unpacklo_epi8(v_b128, v_z128);
v_pa128 = _mm_sub_epi16(v_b128, v_c128);
v_pb128 = _mm_sub_epi16(v_a128, v_c128);
v_pc128 = _mm_add_epi16(v_pa128, v_pb128);
v_pa128 = _mm_abs_epi16(v_pa128);
v_pb128 = _mm_abs_epi16(v_pb128);
v_pc128 = _mm_abs_epi16(v_pc128);
v_smallest128 = _mm_min_epi16(v_pc128, _mm_min_epi16(v_pb128, v_pa128));
v_p128 = _mm_blendv_epi8(_mm_blendv_epi8(v_c128, v_b128, _mm_cmpeq_epi16(v_smallest128, v_pb128)), v_a128, _mm_cmpeq_epi16(v_smallest128, v_pa128));
(v_x128 = _mm_cvtsi32_si128((int)(wuffs_base__peek_u32le__no_bounds_check(v_c.ptr))), wuffs_base__make_empty_struct());
v_x128 = _mm_unpacklo_epi8(v_x128, v_z128);
v_x128 = _mm_add_epi8(v_x128, v_p128);
v_a128 = v_x128;
v_c128 = v_b128;
v_x128 = _mm_packus_epi16(v_x128, v_x128);
(wuffs_base__poke_u32le__no_bounds_check(v_c.ptr, ((uint32_t)(_mm_cvtsi128_si32(v_x128)))), wuffs_base__make_empty_struct());
v_c.ptr += 4;
v_p.ptr += 4;
}
}
return wuffs_base__make_empty_struct();
}
#endif // defined(WUFFS_BASE__CPU_ARCH__X86_64)
// -------- func png.decoder.set_quirk_enabled
WUFFS_BASE__MAYBE_STATIC wuffs_base__empty_struct
@@ -31632,6 +31788,9 @@ wuffs_png__decoder__choose_filter_implementations(
self->private_impl.choosy_filter_3 = (
&wuffs_png__decoder__filter_3_distance_3_fallback);
self->private_impl.choosy_filter_4 = (
#if defined(WUFFS_BASE__CPU_ARCH__X86_64)
wuffs_base__cpu_arch__have_sse128() ? &wuffs_png__decoder__filter_4_distance_3_sse128 :
#endif
&wuffs_png__decoder__filter_4_distance_3_fallback);
} else if (self->private_impl.f_filter_distance == 4) {
self->private_impl.choosy_filter_1 = (
@@ -31645,6 +31804,9 @@ wuffs_png__decoder__choose_filter_implementations(
#endif
&wuffs_png__decoder__filter_3_distance_4_fallback);
self->private_impl.choosy_filter_4 = (
#if defined(WUFFS_BASE__CPU_ARCH__X86_64)
wuffs_base__cpu_arch__have_sse128() ? &wuffs_png__decoder__filter_4_distance_4_sse128 :
#endif
&wuffs_png__decoder__filter_4_distance_4_fallback);
}
return wuffs_base__make_empty_struct();
+145
View File
@@ -118,3 +118,148 @@ pri func decoder.filter_3_distance_4_sse128!(curr: slice base.u8, prev: slice ba
}
}
}
// --------
// Filter 4: Paeth.
pri func decoder.filter_4_distance_3_sse128!(curr: slice base.u8, prev: slice base.u8),
choose cpu_arch >= sse128,
{
// See the comments in filter_4_distance_4_sse128 for an explanation of how
// this works. That function's single loop is copied twice here, once with
// "length: 4" and once with "length: 3". It's generally faster to load 4
// bytes at a time instead of 3.
//
// Differences between that function and this one are marked with a §.
var c : slice base.u8
var p : slice base.u8
var x128 : base.sse128_i
var a128 : base.sse128_i
var b128 : base.sse128_i
var c128 : base.sse128_i
var p128 : base.sse128_i
var pa128 : base.sse128_i
var pb128 : base.sse128_i
var pc128 : base.sse128_i
var smallest128 : base.sse128_i
var z128 : base.sse128_i
// § The advance is 3, not 4.
iterate (c = args.curr, p = args.prev)(length: 4, advance: 3, unroll: 1) {
b128.load_u32!(a: p.peek_u32le())
b128 = b128._mm_unpacklo_epi8(b: z128)
pa128 = b128._mm_sub_epi16(b: c128)
pb128 = a128._mm_sub_epi16(b: c128)
pc128 = pa128._mm_add_epi16(b: pb128)
pa128 = pa128._mm_abs_epi16()
pb128 = pb128._mm_abs_epi16()
pc128 = pc128._mm_abs_epi16()
smallest128 = pc128._mm_min_epi16(b: pb128._mm_min_epi16(b: pa128))
p128 = c128._mm_blendv_epi8(
b: b128,
mask: smallest128._mm_cmpeq_epi16(b: pb128))._mm_blendv_epi8(
b: a128,
mask: smallest128._mm_cmpeq_epi16(b: pa128))
x128.load_u32!(a: c.peek_u32le())
x128 = x128._mm_unpacklo_epi8(b: z128)
x128 = x128._mm_add_epi8(b: p128)
a128 = x128
c128 = b128
x128 = x128._mm_packus_epi16(b: x128)
// § poke_u24le replaces poke_u32le.
c.poke_u24le!(a: x128.truncate_u32())
// § The length and advance are both 3, not 4.
} else (length: 3, advance: 3, unroll: 1) {
// § peek_u24le_as_u32 replaces peek_u32le.
b128.load_u32!(a: p.peek_u24le_as_u32())
b128 = b128._mm_unpacklo_epi8(b: z128)
pa128 = b128._mm_sub_epi16(b: c128)
pb128 = a128._mm_sub_epi16(b: c128)
pc128 = pa128._mm_add_epi16(b: pb128)
pa128 = pa128._mm_abs_epi16()
pb128 = pb128._mm_abs_epi16()
pc128 = pc128._mm_abs_epi16()
smallest128 = pc128._mm_min_epi16(b: pb128._mm_min_epi16(b: pa128))
p128 = c128._mm_blendv_epi8(
b: b128,
mask: smallest128._mm_cmpeq_epi16(b: pb128))._mm_blendv_epi8(
b: a128,
mask: smallest128._mm_cmpeq_epi16(b: pa128))
// § peek_u24le_as_u32 replaces peek_u32le.
x128.load_u32!(a: c.peek_u24le_as_u32())
x128 = x128._mm_unpacklo_epi8(b: z128)
x128 = x128._mm_add_epi8(b: p128)
// § These assignments are unnecessary; this is the last iteration.
// a128 = x128
// c128 = b128
x128 = x128._mm_packus_epi16(b: x128)
// § poke_u24le replaces poke_u32le.
c.poke_u24le!(a: x128.truncate_u32())
}
}
pri func decoder.filter_4_distance_4_sse128!(curr: slice base.u8, prev: slice base.u8),
choose cpu_arch >= sse128,
{
var c : slice base.u8
var p : slice base.u8
var x128 : base.sse128_i
var a128 : base.sse128_i
var b128 : base.sse128_i
var c128 : base.sse128_i
var p128 : base.sse128_i
var pa128 : base.sse128_i
var pb128 : base.sse128_i
var pc128 : base.sse128_i
var smallest128 : base.sse128_i
var z128 : base.sse128_i
iterate (c = args.curr, p = args.prev)(length: 4, advance: 4, unroll: 1) {
// Load the pixel from the row above.
b128.load_u32!(a: p.peek_u32le())
// Convert from u8 to i16 by unpacking it with zeroes.
b128 = b128._mm_unpacklo_epi8(b: z128)
// Compute:
// - pa128 = (m128 - a128)
// - pb128 = (m128 - b128)
// - pc128 = (m128 - c128)
// where m128 = (a128 + b128 - c128).
pa128 = b128._mm_sub_epi16(b: c128)
pb128 = a128._mm_sub_epi16(b: c128)
pc128 = pa128._mm_add_epi16(b: pb128)
// Compute the smallest absolute value of pa128, pb128 and pc128.
pa128 = pa128._mm_abs_epi16()
pb128 = pb128._mm_abs_epi16()
pc128 = pc128._mm_abs_epi16()
smallest128 = pc128._mm_min_epi16(b: pb128._mm_min_epi16(b: pa128))
// The predictor, p128, is whichever of a128, b128 or c128 such that
// pa128, pb128 or pc128 matches this smallest absolute value. Ties are
// broken in favor of a128 then b128 then c128.
//
// The a._mm_blendv_epi8(b, mask) method picks b when mask is true.
p128 = c128._mm_blendv_epi8(
b: b128,
mask: smallest128._mm_cmpeq_epi16(b: pb128))._mm_blendv_epi8(
b: a128,
mask: smallest128._mm_cmpeq_epi16(b: pa128))
// Add the predictor to the residual and, for the next iteration, set
// its previous pixels, a128 and c128, to x128 and b128.
//
// Again, convert between u8 and i16 by unpacking and re-packing.
x128.load_u32!(a: c.peek_u32le())
x128 = x128._mm_unpacklo_epi8(b: z128)
x128 = x128._mm_add_epi8(b: p128)
a128 = x128
c128 = b128
x128 = x128._mm_packus_epi16(b: x128)
c.poke_u32le!(a: x128.truncate_u32())
}
}
+6 -2
View File
@@ -251,7 +251,9 @@ pri func decoder.choose_filter_implementations!() {
if this.filter_distance == 3 {
choose filter_1 = [filter_1_distance_3_fallback]
choose filter_3 = [filter_3_distance_3_fallback]
choose filter_4 = [filter_4_distance_3_fallback]
choose filter_4 = [
filter_4_distance_3_sse128,
filter_4_distance_3_fallback]
} else if this.filter_distance == 4 {
choose filter_1 = [
filter_1_distance_4_sse128,
@@ -259,7 +261,9 @@ pri func decoder.choose_filter_implementations!() {
choose filter_3 = [
filter_3_distance_4_sse128,
filter_3_distance_4_fallback]
choose filter_4 = [filter_4_distance_4_fallback]
choose filter_4 = [
filter_4_distance_4_sse128,
filter_4_distance_4_fallback]
}
}