From b61a8273541dd26a4025669cf00f980791c538e9 Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Tue, 26 Jan 2021 14:12:09 +1100 Subject: [PATCH] Optimize std/adler32 for SSE4.2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On a mid-range x86_64 laptop: wuffs_adler32_10k/clang9 1.35GB/s ± 0% 10.78GB/s ± 1% +700.65% (p=0.008 n=5+5) wuffs_adler32_100k/clang9 1.35GB/s ± 0% 12.88GB/s ± 0% +856.09% (p=0.008 n=5+5) wuffs_adler32_10k/gcc10 1.85GB/s ± 0% 11.21GB/s ± 0% +506.34% (p=0.008 n=5+5) wuffs_adler32_100k/gcc10 1.84GB/s ± 0% 11.43GB/s ± 0% +521.51% (p=0.008 n=5+5) mimic_adler32_10k 1.76GB/s ± 0% 1.76GB/s ± 0% ~ (p=0.841 n=5+5) mimic_adler32_100k 1.72GB/s ± 0% 1.72GB/s ± 0% ~ (p=0.222 n=5+5) wuffs_zlib_decode_10k/clang9 124MB/s ± 0% 139MB/s ± 0% +12.29% (p=0.016 n=4+5) wuffs_zlib_decode_100k/clang9 148MB/s ± 0% 155MB/s ± 0% +4.21% (p=0.008 n=5+5) wuffs_zlib_decode_10k/gcc10 133MB/s ± 1% 141MB/s ± 0% +5.75% (p=0.008 n=5+5) wuffs_zlib_decode_100k/gcc10 159MB/s ± 0% 172MB/s ± 0% +7.77% (p=0.016 n=5+4) mimic_zlib_decode_10k 133MB/s ± 0% 133MB/s ± 0% +0.38% (p=0.008 n=5+5) mimic_zlib_decode_100k 160MB/s ± 0% 160MB/s ± 0% ~ (p=0.841 n=5+5) wuffs_png_decode_image_19k_8bpp/clang9 90.6MB/s ± 0% 94.7MB/s ± 0% +4.58% (p=0.016 n=5+4) wuffs_png_decode_image_40k_24bpp/clang9 129MB/s ± 0% 135MB/s ± 0% +4.49% (p=0.008 n=5+5) wuffs_png_decode_image_77k_8bpp/clang9 335MB/s ± 0% 351MB/s ± 0% +4.62% (p=0.008 n=5+5) wuffs_png_decode_image_552k_32bpp_ignore_checksum/clang9 322MB/s ± 0% 320MB/s ± 1% -0.71% (p=0.016 n=4+5) wuffs_png_decode_image_552k_32bpp_verify_checksum/clang9 256MB/s ± 0% 308MB/s ± 0% +20.24% (p=0.008 n=5+5) wuffs_png_decode_image_4002k_24bpp/clang9 131MB/s ± 0% 138MB/s ± 0% +5.24% (p=0.008 n=5+5) wuffs_png_decode_image_19k_8bpp/gcc10 95.1MB/s ± 0% 100.8MB/s ± 1% +6.08% (p=0.008 n=5+5) wuffs_png_decode_image_40k_24bpp/gcc10 131MB/s ± 0% 136MB/s ± 0% +3.65% (p=0.008 n=5+5) wuffs_png_decode_image_77k_8bpp/gcc10 350MB/s ± 1% 371MB/s ± 0% +6.06% (p=0.008 n=5+5) wuffs_png_decode_image_552k_32bpp_ignore_checksum/gcc10 327MB/s ± 0% 333MB/s ± 0% +1.66% (p=0.008 n=5+5) wuffs_png_decode_image_552k_32bpp_verify_checksum/gcc10 274MB/s ± 0% 318MB/s ± 0% +15.93% (p=0.008 n=5+5) wuffs_png_decode_image_4002k_24bpp/gcc10 132MB/s ± 0% 138MB/s ± 0% +4.73% (p=0.008 n=5+5) mimic_png_decode_image_19k_8bpp 58.1MB/s ± 0% 58.2MB/s ± 0% +0.27% (p=0.008 n=5+5) mimic_png_decode_image_40k_24bpp 73.0MB/s ± 1% 73.3MB/s ± 0% +0.45% (p=0.008 n=5+5) mimic_png_decode_image_77k_8bpp 177MB/s ± 0% 177MB/s ± 0% ~ (p=0.421 n=5+5) mimic_png_decode_image_552k_32bpp_ignore_checksum skipped mimic_png_decode_image_552k_32bpp_verify_checksum 146MB/s ± 0% 146MB/s ± 0% ~ (p=0.310 n=5+5) mimic_png_decode_image_4002k_24bpp 104MB/s ± 0% 104MB/s ± 0% ~ (p=0.056 n=5+5) --- internal/cgen/cgen.go | 11 ++ internal/cgen/statement.go | 2 +- lang/check/optimize.go | 23 +++ release/c/wuffs-unsupported-snapshot.c | 147 ++++++++++++++++++- std/adler32/common_adler32.wuffs | 186 +++++++++++++++++++++++-- 5 files changed, 354 insertions(+), 15 deletions(-) diff --git a/internal/cgen/cgen.go b/internal/cgen/cgen.go index 74da71e7..329daa9a 100644 --- a/internal/cgen/cgen.go +++ b/internal/cgen/cgen.go @@ -913,6 +913,17 @@ func (g *gen) forEachStatus(b *buffer, v visibility, f func(*gen, *buffer, *a.St return nil } +func (g *gen) findAstFunc(qqid t.QQID) *a.Func { + for _, file := range g.files { + for _, tld := range file.TopLevelDecls() { + if (tld.Kind() == a.KFunc) && (tld.AsFunc().QQID() == qqid) { + return tld.AsFunc() + } + } + } + return nil +} + func (g *gen) cName(name string) string { return cName(name, g.pkgPrefix) } diff --git a/internal/cgen/statement.go b/internal/cgen/statement.go index fa8ac8cd..e83d1d42 100644 --- a/internal/cgen/statement.go +++ b/internal/cgen/statement.go @@ -238,7 +238,7 @@ func (g *gen) writeStatementChoose(b *buffer, n *a.Choose, depth uint32) error { if n.Name() == id { suffix = "__choosy_default" } - caMacro, caName, _ := cpuArchCNames(g.funks[t.QQID{recv[0], recv[1], id}].astFunc.Asserts()) + caMacro, caName, _ := cpuArchCNames(g.findAstFunc(t.QQID{recv[0], recv[1], id}).Asserts()) if caMacro == "" { b.printf("&%s%s__%s%s", g.pkgPrefix, recv.Str(g.tm), id.Str(g.tm), suffix) conclusive = true diff --git a/lang/check/optimize.go b/lang/check/optimize.go index 4c4202eb..15173639 100644 --- a/lang/check/optimize.go +++ b/lang/check/optimize.go @@ -53,6 +53,29 @@ func (q *checker) optimizeIOMethodAdvance(receiver *a.Expr, advance *big.Int, ad return q.optimizeIOMethodAdvanceExpr(receiver, advanceExpr, update) } + // Check if receiver looks like "a[i .. j]" where i and j are constants and + // ((j - i) >= advance). + if receiver.Operator() == t.IDDotDot { + icv := (*big.Int)(nil) + if i := receiver.MHS().AsExpr(); i == nil { + icv = zero + } else if i.ConstValue() != nil { + icv = i.ConstValue() + } + + jcv := (*big.Int)(nil) + if j := receiver.RHS().AsExpr(); (j != nil) && (j.ConstValue() != nil) { + jcv = j.ConstValue() + } + + if (icv != nil) && (jcv != nil) { + n := big.NewInt(0).Sub(jcv, icv) + if n.Cmp(advance) >= 0 { + retOK = true + } + } + } + retErr = q.facts.update(func(x *a.Expr) (*a.Expr, error) { // TODO: update (discard?) any facts that merely mention // receiver.length(), even if they aren't an exact match. diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 82638aae..ca9ad79d 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -5551,6 +5551,10 @@ struct wuffs_adler32__hasher__struct { uint32_t f_state; bool f_started; + + wuffs_base__empty_struct (*choosy_up)( + wuffs_adler32__hasher* self, + wuffs_base__slice_u8 a_x); } private_impl; #ifdef __cplusplus @@ -17156,6 +17160,23 @@ wuffs_base__ascii__longest_valid_prefix(const uint8_t* s_ptr, size_t s_len) { // ---------------- Private Function Prototypes +static wuffs_base__empty_struct +wuffs_adler32__hasher__up( + wuffs_adler32__hasher* self, + wuffs_base__slice_u8 a_x); + +static wuffs_base__empty_struct +wuffs_adler32__hasher__up__choosy_default( + wuffs_adler32__hasher* self, + wuffs_base__slice_u8 a_x); + +#if defined(WUFFS_BASE__CPU_ARCH__X86_64) +static wuffs_base__empty_struct +wuffs_adler32__hasher__up_sse42( + wuffs_adler32__hasher* self, + wuffs_base__slice_u8 a_x); +#endif // defined(WUFFS_BASE__CPU_ARCH__X86_64) + // ---------------- VTables const wuffs_base__hasher_u32__func_ptrs @@ -17208,6 +17229,8 @@ wuffs_adler32__hasher__initialize( } } + self->private_impl.choosy_up = &wuffs_adler32__hasher__up__choosy_default; + self->private_impl.magic = WUFFS_BASE__MAGIC; self->private_impl.vtable_for__wuffs_base__hasher_u32.vtable_name = wuffs_base__hasher_u32__vtable_name; @@ -17261,15 +17284,37 @@ wuffs_adler32__hasher__update_u32( return 0; } + if ( ! self->private_impl.f_started) { + self->private_impl.f_started = true; + self->private_impl.f_state = 1; + self->private_impl.choosy_up = ( +#if defined(WUFFS_BASE__CPU_ARCH__X86_64) + wuffs_base__cpu_arch__have_sse42() ? &wuffs_adler32__hasher__up_sse42 : +#endif + self->private_impl.choosy_up); + } + wuffs_adler32__hasher__up(self, a_x); + return self->private_impl.f_state; +} + +// -------- func adler32.hasher.up + +static wuffs_base__empty_struct +wuffs_adler32__hasher__up( + wuffs_adler32__hasher* self, + wuffs_base__slice_u8 a_x) { + return (*self->private_impl.choosy_up)(self, a_x); +} + +static wuffs_base__empty_struct +wuffs_adler32__hasher__up__choosy_default( + wuffs_adler32__hasher* self, + wuffs_base__slice_u8 a_x) { uint32_t v_s1 = 0; uint32_t v_s2 = 0; wuffs_base__slice_u8 v_remaining = {0}; wuffs_base__slice_u8 v_p = {0}; - if ( ! self->private_impl.f_started) { - self->private_impl.f_started = true; - self->private_impl.f_state = 1; - } v_s1 = ((self->private_impl.f_state) & 0xFFFF); v_s2 = ((self->private_impl.f_state) >> (32 - (16))); while (((uint64_t)(a_x.len)) > 0) { @@ -17323,9 +17368,101 @@ wuffs_adler32__hasher__update_u32( a_x = v_remaining; } self->private_impl.f_state = (((v_s2 & 65535) << 16) | (v_s1 & 65535)); - return self->private_impl.f_state; + return wuffs_base__make_empty_struct(); } +// -------- func adler32.hasher.up_sse42 + +#if defined(WUFFS_BASE__CPU_ARCH__X86_64) +#if defined(__GNUC__) +__attribute__((target("sse4.2"))) +#endif +static wuffs_base__empty_struct +wuffs_adler32__hasher__up_sse42( + wuffs_adler32__hasher* self, + wuffs_base__slice_u8 a_x) { + uint32_t v_s1 = 0; + uint32_t v_s2 = 0; + wuffs_base__slice_u8 v_remaining = {0}; + wuffs_base__slice_u8 v_p = {0}; + __m128i v_zeroes = {0}; + __m128i v_ones = {0}; + __m128i v_weights__left = {0}; + __m128i v_weights_right = {0}; + __m128i v_p__left = {0}; + __m128i v_p_right = {0}; + __m128i v_v1 = {0}; + __m128i v_v2 = {0}; + __m128i v_v2j = {0}; + __m128i v_v2k = {0}; + uint32_t v_num_iterate_bytes = 0; + uint64_t v_tail_index = 0; + + v_zeroes = _mm_set1_epi16((int16_t)(0)); + v_ones = _mm_set1_epi16((int16_t)(1)); + v_weights__left = _mm_set_epi8((int8_t)(17), (int8_t)(18), (int8_t)(19), (int8_t)(20), (int8_t)(21), (int8_t)(22), (int8_t)(23), (int8_t)(24), (int8_t)(25), (int8_t)(26), (int8_t)(27), (int8_t)(28), (int8_t)(29), (int8_t)(30), (int8_t)(31), (int8_t)(32)); + v_weights_right = _mm_set_epi8((int8_t)(1), (int8_t)(2), (int8_t)(3), (int8_t)(4), (int8_t)(5), (int8_t)(6), (int8_t)(7), (int8_t)(8), (int8_t)(9), (int8_t)(10), (int8_t)(11), (int8_t)(12), (int8_t)(13), (int8_t)(14), (int8_t)(15), (int8_t)(16)); + v_s1 = ((self->private_impl.f_state) & 0xFFFF); + v_s2 = ((self->private_impl.f_state) >> (32 - (16))); + while (((uint64_t)(a_x.len)) > 0) { + v_remaining = wuffs_base__slice_u8__subslice_j(a_x, 0); + if (((uint64_t)(a_x.len)) > 5536) { + v_remaining = wuffs_base__slice_u8__subslice_i(a_x, 5536); + a_x = wuffs_base__slice_u8__subslice_j(a_x, 5536); + } + v_num_iterate_bytes = ((uint32_t)((((uint64_t)(a_x.len)) & 4294967264))); + v_s2 += (v_s1 * v_num_iterate_bytes); + v_v1 = _mm_setzero_si128(); + v_v2j = _mm_setzero_si128(); + v_v2k = _mm_setzero_si128(); + { + wuffs_base__slice_u8 i_slice_p = a_x; + v_p.ptr = i_slice_p.ptr; + v_p.len = 32; + uint8_t* i_end0_p = v_p.ptr + (((i_slice_p.len - (size_t)(v_p.ptr - i_slice_p.ptr)) / 32) * 32); + while (v_p.ptr < i_end0_p) { + v_p__left = _mm_lddqu_si128((const __m128i*)(const void*)(wuffs_base__slice_u8__subslice_j(v_p, 16).ptr)); + v_p_right = _mm_lddqu_si128((const __m128i*)(const void*)(wuffs_base__slice_u8__subslice_ij(v_p, 16, 32).ptr)); + v_v2j = _mm_add_epi32(v_v2j, v_v1); + v_v1 = _mm_add_epi32(v_v1, _mm_sad_epu8(v_p__left, v_zeroes)); + v_v1 = _mm_add_epi32(v_v1, _mm_sad_epu8(v_p_right, v_zeroes)); + v_v2k = _mm_add_epi32(v_v2k, _mm_madd_epi16(v_ones, _mm_maddubs_epi16(v_p__left, v_weights__left))); + v_v2k = _mm_add_epi32(v_v2k, _mm_madd_epi16(v_ones, _mm_maddubs_epi16(v_p_right, v_weights_right))); + v_p.ptr += 32; + } + v_p.len = 0; + } + v_v1 = _mm_add_epi32(v_v1, _mm_shuffle_epi32(v_v1, (int32_t)(177))); + v_v1 = _mm_add_epi32(v_v1, _mm_shuffle_epi32(v_v1, (int32_t)(78))); + v_s1 += ((uint32_t)(_mm_cvtsi128_si32(v_v1))); + v_v2 = _mm_add_epi32(v_v2k, _mm_slli_epi32(v_v2j, (int32_t)(5))); + v_v2 = _mm_add_epi32(v_v2, _mm_shuffle_epi32(v_v2, (int32_t)(177))); + v_v2 = _mm_add_epi32(v_v2, _mm_shuffle_epi32(v_v2, (int32_t)(78))); + v_s2 += ((uint32_t)(_mm_cvtsi128_si32(v_v2))); + v_tail_index = (((uint64_t)(a_x.len)) & 18446744073709551584u); + if (v_tail_index < ((uint64_t)(a_x.len))) { + { + wuffs_base__slice_u8 i_slice_p = wuffs_base__slice_u8__subslice_i(a_x, v_tail_index); + v_p.ptr = i_slice_p.ptr; + v_p.len = 1; + uint8_t* i_end0_p = i_slice_p.ptr + i_slice_p.len; + while (v_p.ptr < i_end0_p) { + v_s1 += ((uint32_t)(v_p.ptr[0])); + v_s2 += v_s1; + v_p.ptr += 1; + } + v_p.len = 0; + } + } + v_s1 %= 65521; + v_s2 %= 65521; + a_x = v_remaining; + } + self->private_impl.f_state = (((v_s2 & 65535) << 16) | (v_s1 & 65535)); + return wuffs_base__make_empty_struct(); +} +#endif // defined(WUFFS_BASE__CPU_ARCH__X86_64) + #endif // !defined(WUFFS_CONFIG__MODULES) || defined(WUFFS_CONFIG__MODULE__ADLER32) #if !defined(WUFFS_CONFIG__MODULES) || defined(WUFFS_CONFIG__MODULE__BMP) diff --git a/std/adler32/common_adler32.wuffs b/std/adler32/common_adler32.wuffs index dd7d66c9..74bbba47 100644 --- a/std/adler32/common_adler32.wuffs +++ b/std/adler32/common_adler32.wuffs @@ -22,6 +22,18 @@ pub func hasher.set_quirk_enabled!(quirk: base.u32, enabled: base.bool) { } pub func hasher.update_u32!(x: slice base.u8) base.u32 { + if not this.started { + this.started = true + this.state = 1 + choose up = [up_sse42] + } + this.up!(x: args.x) + return this.state +} + +pri func hasher.up!(x: slice base.u8), + choosy, +{ // The Adler-32 checksum's magic 65521 and 5552 numbers are discussed in // this package's README.md. @@ -30,14 +42,8 @@ pub func hasher.update_u32!(x: slice base.u8) base.u32 { var remaining : slice base.u8 var p : slice base.u8 - if not this.started { - this.started = true - this.state = 1 - } - s1 = this.state.low_bits(n: 16) s2 = this.state.high_bits(n: 16) - while args.x.length() > 0 { remaining = args.x[.. 0] if args.x.length() > 5552 { @@ -45,6 +51,7 @@ pub func hasher.update_u32!(x: slice base.u8) base.u32 { args.x = args.x[.. 5552] } + // The SIMD versions of this function replace this simple iterate loop. iterate (p = args.x)(length: 1, advance: 1, unroll: 8) { s1 ~mod+= p[0] as base.u32 s2 ~mod+= s1 @@ -52,10 +59,171 @@ pub func hasher.update_u32!(x: slice base.u8) base.u32 { s1 %= 65521 s2 %= 65521 - args.x = remaining } endwhile - this.state = ((s2 & 0xFFFF) << 16) | (s1 & 0xFFFF) - return this.state +} + +pri func hasher.up_sse42!(x: slice base.u8), + choose cpu_arch >= x86_sse42, +{ + // These variables are the same as the non-SIMD version. + var s1 : base.u32 + var s2 : base.u32 + var remaining : slice base.u8 + var p : slice base.u8 + + // The remaining variables are specific to the SIMD version. + var zeroes : base.x86_m128i + var ones : base.x86_m128i + var weights__left : base.x86_m128i + var weights_right : base.x86_m128i + var p__left : base.x86_m128i + var p_right : base.x86_m128i + var v1 : base.x86_m128i + var v2 : base.x86_m128i + var v2j : base.x86_m128i + var v2k : base.x86_m128i + var num_iterate_bytes : base.u32 + var tail_index : base.u64 + + // zeroes and ones are uniform 8×u16 vectors. + zeroes = zeroes.create_mm_set1_epi16(a: 0) + ones = ones.create_mm_set1_epi16(a: 1) + + // weights__left and weights_right, little-endian, form the sequence 32, + // 31, 30, ..., 1. + weights__left = weights__left.create_mm_set_epi8( + e15: 0x11, e14: 0x12, e13: 0x13, e12: 0x14, + e11: 0x15, e10: 0x16, e9: 0x17, e8: 0x18, + e7: 0x19, e6: 0x1A, e5: 0x1B, e4: 0x1C, + e3: 0x1D, e2: 0x1E, e1: 0x1F, e0: 0x20) + weights_right = weights_right.create_mm_set_epi8( + e15: 0x01, e14: 0x02, e13: 0x03, e12: 0x04, + e11: 0x05, e10: 0x06, e9: 0x07, e8: 0x08, + e7: 0x09, e6: 0x0A, e5: 0x0B, e4: 0x0C, + e3: 0x0D, e2: 0x0E, e1: 0x0F, e0: 0x10) + + // Just like the non-SIMD version, split the u32 state into two u16 halves + // and loop over args.x up to almost-5552 bytes at a time. The slightly + // smaller 5536 is the largest multiple of 32 less than non-SIMD's 5552. + s1 = this.state.low_bits(n: 16) + s2 = this.state.high_bits(n: 16) + while args.x.length() > 0 { + remaining = args.x[.. 0] + if args.x.length() > 5536 { + remaining = args.x[5536 ..] + args.x = args.x[.. 5536] + } + + // The s1 state is the sum of the input bytes and the s2 state is the + // sum of the s1 state at each 1-byte step. Inside the iterate loop + // below, but starting fresh at each outer while loop iteration, s1 + // consists of three parts (called s1i, s1j and s1k): + // - s1i: the initial value, before any 32-byte iterations. + // - s1j: the total contribution from previous 32-byte iterations. + // - s1k: the contribution due to the current 32-byte iteration. + // + // The upcoming iterate loop (at 32 bytes per iteration) encompasses + // num_iterate_bytes 1-byte steps. We hoist the total s1i contribution, + // (s1i * num_iterate_bytes) out here. + num_iterate_bytes = (args.x.length() & 0xFFFF_FFE0) as base.u32 + s2 ~mod+= (s1 ~mod* num_iterate_bytes) + + // Zero-initialize some 4×u32 vectors associated with the two state + // variables s1 and s2. The iterate loop accumulates 4 parallel u32 + // sums in each vector. A post-iterate step merges the four u32 sums + // into a single u32 sum. + v1 = v1.create_mm_setzero_si128() + v2j = v2j.create_mm_setzero_si128() + v2k = v2k.create_mm_setzero_si128() + + // The inner loop. + iterate (p = args.x)(length: 32, advance: 32, unroll: 1) { + // Split the 32-byte p into left and right halves. SSE4.2 works + // with 16-byte registers. + // + // Let p__left = [16×u8: p00, p01, p02, ..., p15] + // Let p_right = [16×u8: p16, p17, p18, ..., p31] + p__left.load_slice128!(a: p[.. 16]) + p_right.load_slice128!(a: p[16 .. 32]) + + // For v2j, we need to calculate the sums of the s1j terms for each + // of p's 32 elements. This is simply 32 times the same number, + // that number being the sum of v1's four u32 accumulators. We + // add v1 now and multiply by 32 later, outside the inner loop. + v2j = v2j._mm_add_epi32(b: v1) + + // For v1, we need to add the elements of p. Computing the sum of + // absolute differences (_mm_sad_epu8) with zero just sums the + // elements. p__left._mm_sad_epu8(b: zeroes) equals + // [2×u64: p00 + p01 + ... + p07, p08 + p09 + ... + p15] + // This is equivalent (little-endian) to: + // [4×u32: p00 + p01 + ... + p07, 0, p08 + p09 + ... + p15, 0] + // We accumulate those "sum of p__left elements" in v1, and ditto + // for the p_right elements. + v1 = v1._mm_add_epi32(b: p__left._mm_sad_epu8(b: zeroes)) + v1 = v1._mm_add_epi32(b: p_right._mm_sad_epu8(b: zeroes)) + + // For v2k, we need to calculate a weighted sum: ((32 * p00) + (31 + // * p01) + (30 * p02) + ... + (1 * p31)), which splits naturally + // into weighted sums of the left half and of the right half. + // + // The _mm_maddubs_epi16 call (vertically multiply u8 columns and + // then horizontally sum u16 pairs) with the left half produces: + // [8×u16: ((32*p00)+(31*p01)), + // ((30*p02)+(29*p03)), + // ... + // ((18*p14)+(17*p15))] + // + // The ones._mm_madd_epi16(b: etc) call is likewise a multiply-add + // (note that it's "madd" not "add"). Multiplying by 1 is a no-op, + // so this sums u16 pairs to produce u32 values: + // [4×u32: ((32*p00)+(31*p01)+(30*p02)+(29*p03)), + // ((28*p04)+(27*p05)+(26*p06)+(25*p07)), + // ... + // ((20*p12)+(19*p13)+(18*p14)+(17*p15))] + // + // Ditto again for the p_right elements. + v2k = v2k._mm_add_epi32(b: ones._mm_madd_epi16(b: + p__left._mm_maddubs_epi16(b: weights__left))) + v2k = v2k._mm_add_epi32(b: ones._mm_madd_epi16(b: + p_right._mm_maddubs_epi16(b: weights_right))) + } + + // Merge the four per-u32-lane sums (v1) into the single u32 sum (s1). + // Starting with a 4×u32 vector [x0, x1, x2, x3]: + // - shuffling with 0b1011_0001 gives [x1, x0, x3, x2]. + // - adding gives [x0+x1, x0+x1, x2+x3, x2+x3]. + // - shuffling with 0b0100_1110 gives [x2+x3, x2+x3, x0+x1, x0+x1]. + // - adding gives [x0+x1+x2+x3, ditto, ditto, ditto]. + // The truncate_u32 call extracts the first u32: x0+x1+x2+x3. + v1 = v1._mm_add_epi32(b: v1._mm_shuffle_epi32(imm8: 0xB1)) // 0b1011_0001 + v1 = v1._mm_add_epi32(b: v1._mm_shuffle_epi32(imm8: 0x4E)) // 0b0100_1110 + s1 ~mod+= v1.truncate_u32() + + // Combine v2j and v2k. The slli (shift logical left immediate) by 5 + // multiplies v2j's four u32 elements each by 32, alluded to earlier. + v2 = v2k._mm_add_epi32(b: v2j._mm_slli_epi32(imm8: 5)) + + // Similarly merge v2 (a 4×u32 vector) into s2 (a scalar u32 scalar). + v2 = v2._mm_add_epi32(b: v2._mm_shuffle_epi32(imm8: 0xB1)) // 0b1011_0001 + v2 = v2._mm_add_epi32(b: v2._mm_shuffle_epi32(imm8: 0x4E)) // 0b0100_1110 + s2 ~mod+= v2.truncate_u32() + + // Handle the tail of args.x that wasn't a complete 32-byte chunk. + tail_index = args.x.length() & 0xFFFF_FFFF_FFFF_FFE0 // And-not 32. + if tail_index < args.x.length() { + iterate (p = args.x[tail_index ..])(length: 1, advance: 1, unroll: 1) { + s1 ~mod+= p[0] as base.u32 + s2 ~mod+= s1 + } + } + + // The rest of this function is the same as the non-SIMD version. + s1 %= 65521 + s2 %= 65521 + args.x = remaining + } endwhile + this.state = ((s2 & 0xFFFF) << 16) | (s1 & 0xFFFF) }