Optimize std/adler32 for SSE4.2

On a mid-range x86_64 laptop:

wuffs_adler32_10k/clang9                                  1.35GB/s ± 0%  10.78GB/s ± 1%  +700.65%  (p=0.008 n=5+5)
wuffs_adler32_100k/clang9                                 1.35GB/s ± 0%  12.88GB/s ± 0%  +856.09%  (p=0.008 n=5+5)

wuffs_adler32_10k/gcc10                                   1.85GB/s ± 0%  11.21GB/s ± 0%  +506.34%  (p=0.008 n=5+5)
wuffs_adler32_100k/gcc10                                  1.84GB/s ± 0%  11.43GB/s ± 0%  +521.51%  (p=0.008 n=5+5)

mimic_adler32_10k                                         1.76GB/s ± 0%   1.76GB/s ± 0%      ~     (p=0.841 n=5+5)
mimic_adler32_100k                                        1.72GB/s ± 0%   1.72GB/s ± 0%      ~     (p=0.222 n=5+5)

wuffs_zlib_decode_10k/clang9                               124MB/s ± 0%    139MB/s ± 0%   +12.29%  (p=0.016 n=4+5)
wuffs_zlib_decode_100k/clang9                              148MB/s ± 0%    155MB/s ± 0%    +4.21%  (p=0.008 n=5+5)

wuffs_zlib_decode_10k/gcc10                                133MB/s ± 1%    141MB/s ± 0%    +5.75%  (p=0.008 n=5+5)
wuffs_zlib_decode_100k/gcc10                               159MB/s ± 0%    172MB/s ± 0%    +7.77%  (p=0.016 n=5+4)

mimic_zlib_decode_10k                                      133MB/s ± 0%    133MB/s ± 0%    +0.38%  (p=0.008 n=5+5)
mimic_zlib_decode_100k                                     160MB/s ± 0%    160MB/s ± 0%      ~     (p=0.841 n=5+5)

wuffs_png_decode_image_19k_8bpp/clang9                    90.6MB/s ± 0%   94.7MB/s ± 0%    +4.58%  (p=0.016 n=5+4)
wuffs_png_decode_image_40k_24bpp/clang9                    129MB/s ± 0%    135MB/s ± 0%    +4.49%  (p=0.008 n=5+5)
wuffs_png_decode_image_77k_8bpp/clang9                     335MB/s ± 0%    351MB/s ± 0%    +4.62%  (p=0.008 n=5+5)
wuffs_png_decode_image_552k_32bpp_ignore_checksum/clang9   322MB/s ± 0%    320MB/s ± 1%    -0.71%  (p=0.016 n=4+5)
wuffs_png_decode_image_552k_32bpp_verify_checksum/clang9   256MB/s ± 0%    308MB/s ± 0%   +20.24%  (p=0.008 n=5+5)
wuffs_png_decode_image_4002k_24bpp/clang9                  131MB/s ± 0%    138MB/s ± 0%    +5.24%  (p=0.008 n=5+5)

wuffs_png_decode_image_19k_8bpp/gcc10                     95.1MB/s ± 0%  100.8MB/s ± 1%    +6.08%  (p=0.008 n=5+5)
wuffs_png_decode_image_40k_24bpp/gcc10                     131MB/s ± 0%    136MB/s ± 0%    +3.65%  (p=0.008 n=5+5)
wuffs_png_decode_image_77k_8bpp/gcc10                      350MB/s ± 1%    371MB/s ± 0%    +6.06%  (p=0.008 n=5+5)
wuffs_png_decode_image_552k_32bpp_ignore_checksum/gcc10    327MB/s ± 0%    333MB/s ± 0%    +1.66%  (p=0.008 n=5+5)
wuffs_png_decode_image_552k_32bpp_verify_checksum/gcc10    274MB/s ± 0%    318MB/s ± 0%   +15.93%  (p=0.008 n=5+5)
wuffs_png_decode_image_4002k_24bpp/gcc10                   132MB/s ± 0%    138MB/s ± 0%    +4.73%  (p=0.008 n=5+5)

mimic_png_decode_image_19k_8bpp                           58.1MB/s ± 0%   58.2MB/s ± 0%    +0.27%  (p=0.008 n=5+5)
mimic_png_decode_image_40k_24bpp                          73.0MB/s ± 1%   73.3MB/s ± 0%    +0.45%  (p=0.008 n=5+5)
mimic_png_decode_image_77k_8bpp                            177MB/s ± 0%    177MB/s ± 0%      ~     (p=0.421 n=5+5)
mimic_png_decode_image_552k_32bpp_ignore_checksum               skipped
mimic_png_decode_image_552k_32bpp_verify_checksum          146MB/s ± 0%    146MB/s ± 0%      ~     (p=0.310 n=5+5)
mimic_png_decode_image_4002k_24bpp                         104MB/s ± 0%    104MB/s ± 0%      ~     (p=0.056 n=5+5)
This commit is contained in:
Nigel Tao committed 2021-01-26 22:31:54 +11:00
1 parent cee57bc4ae
commit b61a827354
5 files changed
+354 -15

No files matched your search

+11
View File
@@ -913,6 +913,17 @@ func (g *gen) forEachStatus(b *buffer, v visibility, f func(*gen, *buffer, *a.St
return nil
}
func (g *gen) findAstFunc(qqid t.QQID) *a.Func {
for _, file := range g.files {
for _, tld := range file.TopLevelDecls() {
if (tld.Kind() == a.KFunc) && (tld.AsFunc().QQID() == qqid) {
return tld.AsFunc()
}
}
}
return nil
}
func (g *gen) cName(name string) string {
return cName(name, g.pkgPrefix)
}
+1 -1
View File
@@ -238,7 +238,7 @@ func (g *gen) writeStatementChoose(b *buffer, n *a.Choose, depth uint32) error {
if n.Name() == id {
suffix = "__choosy_default"
}
caMacro, caName, _ := cpuArchCNames(g.funks[t.QQID{recv[0], recv[1], id}].astFunc.Asserts())
caMacro, caName, _ := cpuArchCNames(g.findAstFunc(t.QQID{recv[0], recv[1], id}).Asserts())
if caMacro == "" {
b.printf("&%s%s__%s%s", g.pkgPrefix, recv.Str(g.tm), id.Str(g.tm), suffix)
conclusive = true
+23
View File
@@ -53,6 +53,29 @@ func (q *checker) optimizeIOMethodAdvance(receiver *a.Expr, advance *big.Int, ad
return q.optimizeIOMethodAdvanceExpr(receiver, advanceExpr, update)
}
// Check if receiver looks like "a[i .. j]" where i and j are constants and
// ((j - i) >= advance).
if receiver.Operator() == t.IDDotDot {
icv := (*big.Int)(nil)
if i := receiver.MHS().AsExpr(); i == nil {
icv = zero
} else if i.ConstValue() != nil {
icv = i.ConstValue()
}
jcv := (*big.Int)(nil)
if j := receiver.RHS().AsExpr(); (j != nil) && (j.ConstValue() != nil) {
jcv = j.ConstValue()
}
if (icv != nil) && (jcv != nil) {
n := big.NewInt(0).Sub(jcv, icv)
if n.Cmp(advance) >= 0 {
retOK = true
}
}
}
retErr = q.facts.update(func(x *a.Expr) (*a.Expr, error) {
// TODO: update (discard?) any facts that merely mention
// receiver.length(), even if they aren't an exact match.
+142 -5
View File
@@ -5551,6 +5551,10 @@ struct wuffs_adler32__hasher__struct {
uint32_t f_state;
bool f_started;
wuffs_base__empty_struct (*choosy_up)(
wuffs_adler32__hasher* self,
wuffs_base__slice_u8 a_x);
} private_impl;
#ifdef __cplusplus
@@ -17156,6 +17160,23 @@ wuffs_base__ascii__longest_valid_prefix(const uint8_t* s_ptr, size_t s_len) {
// ---------------- Private Function Prototypes
static wuffs_base__empty_struct
wuffs_adler32__hasher__up(
wuffs_adler32__hasher* self,
wuffs_base__slice_u8 a_x);
static wuffs_base__empty_struct
wuffs_adler32__hasher__up__choosy_default(
wuffs_adler32__hasher* self,
wuffs_base__slice_u8 a_x);
#if defined(WUFFS_BASE__CPU_ARCH__X86_64)
static wuffs_base__empty_struct
wuffs_adler32__hasher__up_sse42(
wuffs_adler32__hasher* self,
wuffs_base__slice_u8 a_x);
#endif // defined(WUFFS_BASE__CPU_ARCH__X86_64)
// ---------------- VTables
const wuffs_base__hasher_u32__func_ptrs
@@ -17208,6 +17229,8 @@ wuffs_adler32__hasher__initialize(
}
}
self->private_impl.choosy_up = &wuffs_adler32__hasher__up__choosy_default;
self->private_impl.magic = WUFFS_BASE__MAGIC;
self->private_impl.vtable_for__wuffs_base__hasher_u32.vtable_name =
wuffs_base__hasher_u32__vtable_name;
@@ -17261,15 +17284,37 @@ wuffs_adler32__hasher__update_u32(
return 0;
}
if ( ! self->private_impl.f_started) {
self->private_impl.f_started = true;
self->private_impl.f_state = 1;
self->private_impl.choosy_up = (
#if defined(WUFFS_BASE__CPU_ARCH__X86_64)
wuffs_base__cpu_arch__have_sse42() ? &wuffs_adler32__hasher__up_sse42 :
#endif
self->private_impl.choosy_up);
}
wuffs_adler32__hasher__up(self, a_x);
return self->private_impl.f_state;
}
// -------- func adler32.hasher.up
static wuffs_base__empty_struct
wuffs_adler32__hasher__up(
wuffs_adler32__hasher* self,
wuffs_base__slice_u8 a_x) {
return (*self->private_impl.choosy_up)(self, a_x);
}
static wuffs_base__empty_struct
wuffs_adler32__hasher__up__choosy_default(
wuffs_adler32__hasher* self,
wuffs_base__slice_u8 a_x) {
uint32_t v_s1 = 0;
uint32_t v_s2 = 0;
wuffs_base__slice_u8 v_remaining = {0};
wuffs_base__slice_u8 v_p = {0};
if ( ! self->private_impl.f_started) {
self->private_impl.f_started = true;
self->private_impl.f_state = 1;
}
v_s1 = ((self->private_impl.f_state) & 0xFFFF);
v_s2 = ((self->private_impl.f_state) >> (32 - (16)));
while (((uint64_t)(a_x.len)) > 0) {
@@ -17323,9 +17368,101 @@ wuffs_adler32__hasher__update_u32(
a_x = v_remaining;
}
self->private_impl.f_state = (((v_s2 & 65535) << 16) | (v_s1 & 65535));
return self->private_impl.f_state;
return wuffs_base__make_empty_struct();
}
// -------- func adler32.hasher.up_sse42
#if defined(WUFFS_BASE__CPU_ARCH__X86_64)
#if defined(__GNUC__)
__attribute__((target("sse4.2")))
#endif
static wuffs_base__empty_struct
wuffs_adler32__hasher__up_sse42(
wuffs_adler32__hasher* self,
wuffs_base__slice_u8 a_x) {
uint32_t v_s1 = 0;
uint32_t v_s2 = 0;
wuffs_base__slice_u8 v_remaining = {0};
wuffs_base__slice_u8 v_p = {0};
__m128i v_zeroes = {0};
__m128i v_ones = {0};
__m128i v_weights__left = {0};
__m128i v_weights_right = {0};
__m128i v_p__left = {0};
__m128i v_p_right = {0};
__m128i v_v1 = {0};
__m128i v_v2 = {0};
__m128i v_v2j = {0};
__m128i v_v2k = {0};
uint32_t v_num_iterate_bytes = 0;
uint64_t v_tail_index = 0;
v_zeroes = _mm_set1_epi16((int16_t)(0));
v_ones = _mm_set1_epi16((int16_t)(1));
v_weights__left = _mm_set_epi8((int8_t)(17), (int8_t)(18), (int8_t)(19), (int8_t)(20), (int8_t)(21), (int8_t)(22), (int8_t)(23), (int8_t)(24), (int8_t)(25), (int8_t)(26), (int8_t)(27), (int8_t)(28), (int8_t)(29), (int8_t)(30), (int8_t)(31), (int8_t)(32));
v_weights_right = _mm_set_epi8((int8_t)(1), (int8_t)(2), (int8_t)(3), (int8_t)(4), (int8_t)(5), (int8_t)(6), (int8_t)(7), (int8_t)(8), (int8_t)(9), (int8_t)(10), (int8_t)(11), (int8_t)(12), (int8_t)(13), (int8_t)(14), (int8_t)(15), (int8_t)(16));
v_s1 = ((self->private_impl.f_state) & 0xFFFF);
v_s2 = ((self->private_impl.f_state) >> (32 - (16)));
while (((uint64_t)(a_x.len)) > 0) {
v_remaining = wuffs_base__slice_u8__subslice_j(a_x, 0);
if (((uint64_t)(a_x.len)) > 5536) {
v_remaining = wuffs_base__slice_u8__subslice_i(a_x, 5536);
a_x = wuffs_base__slice_u8__subslice_j(a_x, 5536);
}
v_num_iterate_bytes = ((uint32_t)((((uint64_t)(a_x.len)) & 4294967264)));
v_s2 += (v_s1 * v_num_iterate_bytes);
v_v1 = _mm_setzero_si128();
v_v2j = _mm_setzero_si128();
v_v2k = _mm_setzero_si128();
{
wuffs_base__slice_u8 i_slice_p = a_x;
v_p.ptr = i_slice_p.ptr;
v_p.len = 32;
uint8_t* i_end0_p = v_p.ptr + (((i_slice_p.len - (size_t)(v_p.ptr - i_slice_p.ptr)) / 32) * 32);
while (v_p.ptr < i_end0_p) {
v_p__left = _mm_lddqu_si128((const __m128i*)(const void*)(wuffs_base__slice_u8__subslice_j(v_p, 16).ptr));
v_p_right = _mm_lddqu_si128((const __m128i*)(const void*)(wuffs_base__slice_u8__subslice_ij(v_p, 16, 32).ptr));
v_v2j = _mm_add_epi32(v_v2j, v_v1);
v_v1 = _mm_add_epi32(v_v1, _mm_sad_epu8(v_p__left, v_zeroes));
v_v1 = _mm_add_epi32(v_v1, _mm_sad_epu8(v_p_right, v_zeroes));
v_v2k = _mm_add_epi32(v_v2k, _mm_madd_epi16(v_ones, _mm_maddubs_epi16(v_p__left, v_weights__left)));
v_v2k = _mm_add_epi32(v_v2k, _mm_madd_epi16(v_ones, _mm_maddubs_epi16(v_p_right, v_weights_right)));
v_p.ptr += 32;
}
v_p.len = 0;
}
v_v1 = _mm_add_epi32(v_v1, _mm_shuffle_epi32(v_v1, (int32_t)(177)));
v_v1 = _mm_add_epi32(v_v1, _mm_shuffle_epi32(v_v1, (int32_t)(78)));
v_s1 += ((uint32_t)(_mm_cvtsi128_si32(v_v1)));
v_v2 = _mm_add_epi32(v_v2k, _mm_slli_epi32(v_v2j, (int32_t)(5)));
v_v2 = _mm_add_epi32(v_v2, _mm_shuffle_epi32(v_v2, (int32_t)(177)));
v_v2 = _mm_add_epi32(v_v2, _mm_shuffle_epi32(v_v2, (int32_t)(78)));
v_s2 += ((uint32_t)(_mm_cvtsi128_si32(v_v2)));
v_tail_index = (((uint64_t)(a_x.len)) & 18446744073709551584u);
if (v_tail_index < ((uint64_t)(a_x.len))) {
{
wuffs_base__slice_u8 i_slice_p = wuffs_base__slice_u8__subslice_i(a_x, v_tail_index);
v_p.ptr = i_slice_p.ptr;
v_p.len = 1;
uint8_t* i_end0_p = i_slice_p.ptr + i_slice_p.len;
while (v_p.ptr < i_end0_p) {
v_s1 += ((uint32_t)(v_p.ptr[0]));
v_s2 += v_s1;
v_p.ptr += 1;
}
v_p.len = 0;
}
}
v_s1 %= 65521;
v_s2 %= 65521;
a_x = v_remaining;
}
self->private_impl.f_state = (((v_s2 & 65535) << 16) | (v_s1 & 65535));
return wuffs_base__make_empty_struct();
}
#endif // defined(WUFFS_BASE__CPU_ARCH__X86_64)
#endif // !defined(WUFFS_CONFIG__MODULES) || defined(WUFFS_CONFIG__MODULE__ADLER32)
#if !defined(WUFFS_CONFIG__MODULES) || defined(WUFFS_CONFIG__MODULE__BMP)
+177 -9
View File
@@ -22,6 +22,18 @@ pub func hasher.set_quirk_enabled!(quirk: base.u32, enabled: base.bool) {
}
pub func hasher.update_u32!(x: slice base.u8) base.u32 {
if not this.started {
this.started = true
this.state = 1
choose up = [up_sse42]
}
this.up!(x: args.x)
return this.state
}
pri func hasher.up!(x: slice base.u8),
choosy,
{
// The Adler-32 checksum's magic 65521 and 5552 numbers are discussed in
// this package's README.md.
@@ -30,14 +42,8 @@ pub func hasher.update_u32!(x: slice base.u8) base.u32 {
var remaining : slice base.u8
var p : slice base.u8
if not this.started {
this.started = true
this.state = 1
}
s1 = this.state.low_bits(n: 16)
s2 = this.state.high_bits(n: 16)
while args.x.length() > 0 {
remaining = args.x[.. 0]
if args.x.length() > 5552 {
@@ -45,6 +51,7 @@ pub func hasher.update_u32!(x: slice base.u8) base.u32 {
args.x = args.x[.. 5552]
}
// The SIMD versions of this function replace this simple iterate loop.
iterate (p = args.x)(length: 1, advance: 1, unroll: 8) {
s1 ~mod+= p[0] as base.u32
s2 ~mod+= s1
@@ -52,10 +59,171 @@ pub func hasher.update_u32!(x: slice base.u8) base.u32 {
s1 %= 65521
s2 %= 65521
args.x = remaining
} endwhile
this.state = ((s2 & 0xFFFF) << 16) | (s1 & 0xFFFF)
return this.state
}
pri func hasher.up_sse42!(x: slice base.u8),
choose cpu_arch >= x86_sse42,
{
// These variables are the same as the non-SIMD version.
var s1 : base.u32
var s2 : base.u32
var remaining : slice base.u8
var p : slice base.u8
// The remaining variables are specific to the SIMD version.
var zeroes : base.x86_m128i
var ones : base.x86_m128i
var weights__left : base.x86_m128i
var weights_right : base.x86_m128i
var p__left : base.x86_m128i
var p_right : base.x86_m128i
var v1 : base.x86_m128i
var v2 : base.x86_m128i
var v2j : base.x86_m128i
var v2k : base.x86_m128i
var num_iterate_bytes : base.u32
var tail_index : base.u64
// zeroes and ones are uniform 8×u16 vectors.
zeroes = zeroes.create_mm_set1_epi16(a: 0)
ones = ones.create_mm_set1_epi16(a: 1)
// weights__left and weights_right, little-endian, form the sequence 32,
// 31, 30, ..., 1.
weights__left = weights__left.create_mm_set_epi8(
e15: 0x11, e14: 0x12, e13: 0x13, e12: 0x14,
e11: 0x15, e10: 0x16, e9: 0x17, e8: 0x18,
e7: 0x19, e6: 0x1A, e5: 0x1B, e4: 0x1C,
e3: 0x1D, e2: 0x1E, e1: 0x1F, e0: 0x20)
weights_right = weights_right.create_mm_set_epi8(
e15: 0x01, e14: 0x02, e13: 0x03, e12: 0x04,
e11: 0x05, e10: 0x06, e9: 0x07, e8: 0x08,
e7: 0x09, e6: 0x0A, e5: 0x0B, e4: 0x0C,
e3: 0x0D, e2: 0x0E, e1: 0x0F, e0: 0x10)
// Just like the non-SIMD version, split the u32 state into two u16 halves
// and loop over args.x up to almost-5552 bytes at a time. The slightly
// smaller 5536 is the largest multiple of 32 less than non-SIMD's 5552.
s1 = this.state.low_bits(n: 16)
s2 = this.state.high_bits(n: 16)
while args.x.length() > 0 {
remaining = args.x[.. 0]
if args.x.length() > 5536 {
remaining = args.x[5536 ..]
args.x = args.x[.. 5536]
}
// The s1 state is the sum of the input bytes and the s2 state is the
// sum of the s1 state at each 1-byte step. Inside the iterate loop
// below, but starting fresh at each outer while loop iteration, s1
// consists of three parts (called s1i, s1j and s1k):
// - s1i: the initial value, before any 32-byte iterations.
// - s1j: the total contribution from previous 32-byte iterations.
// - s1k: the contribution due to the current 32-byte iteration.
//
// The upcoming iterate loop (at 32 bytes per iteration) encompasses
// num_iterate_bytes 1-byte steps. We hoist the total s1i contribution,
// (s1i * num_iterate_bytes) out here.
num_iterate_bytes = (args.x.length() & 0xFFFF_FFE0) as base.u32
s2 ~mod+= (s1 ~mod* num_iterate_bytes)
// Zero-initialize some 4×u32 vectors associated with the two state
// variables s1 and s2. The iterate loop accumulates 4 parallel u32
// sums in each vector. A post-iterate step merges the four u32 sums
// into a single u32 sum.
v1 = v1.create_mm_setzero_si128()
v2j = v2j.create_mm_setzero_si128()
v2k = v2k.create_mm_setzero_si128()
// The inner loop.
iterate (p = args.x)(length: 32, advance: 32, unroll: 1) {
// Split the 32-byte p into left and right halves. SSE4.2 works
// with 16-byte registers.
//
// Let p__left = [16×u8: p00, p01, p02, ..., p15]
// Let p_right = [16×u8: p16, p17, p18, ..., p31]
p__left.load_slice128!(a: p[.. 16])
p_right.load_slice128!(a: p[16 .. 32])
// For v2j, we need to calculate the sums of the s1j terms for each
// of p's 32 elements. This is simply 32 times the same number,
// that number being the sum of v1's four u32 accumulators. We
// add v1 now and multiply by 32 later, outside the inner loop.
v2j = v2j._mm_add_epi32(b: v1)
// For v1, we need to add the elements of p. Computing the sum of
// absolute differences (_mm_sad_epu8) with zero just sums the
// elements. p__left._mm_sad_epu8(b: zeroes) equals
// [2×u64: p00 + p01 + ... + p07, p08 + p09 + ... + p15]
// This is equivalent (little-endian) to:
// [4×u32: p00 + p01 + ... + p07, 0, p08 + p09 + ... + p15, 0]
// We accumulate those "sum of p__left elements" in v1, and ditto
// for the p_right elements.
v1 = v1._mm_add_epi32(b: p__left._mm_sad_epu8(b: zeroes))
v1 = v1._mm_add_epi32(b: p_right._mm_sad_epu8(b: zeroes))
// For v2k, we need to calculate a weighted sum: ((32 * p00) + (31
// * p01) + (30 * p02) + ... + (1 * p31)), which splits naturally
// into weighted sums of the left half and of the right half.
//
// The _mm_maddubs_epi16 call (vertically multiply u8 columns and
// then horizontally sum u16 pairs) with the left half produces:
// [8×u16: ((32*p00)+(31*p01)),
// ((30*p02)+(29*p03)),
// ...
// ((18*p14)+(17*p15))]
//
// The ones._mm_madd_epi16(b: etc) call is likewise a multiply-add
// (note that it's "madd" not "add"). Multiplying by 1 is a no-op,
// so this sums u16 pairs to produce u32 values:
// [4×u32: ((32*p00)+(31*p01)+(30*p02)+(29*p03)),
// ((28*p04)+(27*p05)+(26*p06)+(25*p07)),
// ...
// ((20*p12)+(19*p13)+(18*p14)+(17*p15))]
//
// Ditto again for the p_right elements.
v2k = v2k._mm_add_epi32(b: ones._mm_madd_epi16(b:
p__left._mm_maddubs_epi16(b: weights__left)))
v2k = v2k._mm_add_epi32(b: ones._mm_madd_epi16(b:
p_right._mm_maddubs_epi16(b: weights_right)))
}
// Merge the four per-u32-lane sums (v1) into the single u32 sum (s1).
// Starting with a 4×u32 vector [x0, x1, x2, x3]:
// - shuffling with 0b1011_0001 gives [x1, x0, x3, x2].
// - adding gives [x0+x1, x0+x1, x2+x3, x2+x3].
// - shuffling with 0b0100_1110 gives [x2+x3, x2+x3, x0+x1, x0+x1].
// - adding gives [x0+x1+x2+x3, ditto, ditto, ditto].
// The truncate_u32 call extracts the first u32: x0+x1+x2+x3.
v1 = v1._mm_add_epi32(b: v1._mm_shuffle_epi32(imm8: 0xB1)) // 0b1011_0001
v1 = v1._mm_add_epi32(b: v1._mm_shuffle_epi32(imm8: 0x4E)) // 0b0100_1110
s1 ~mod+= v1.truncate_u32()
// Combine v2j and v2k. The slli (shift logical left immediate) by 5
// multiplies v2j's four u32 elements each by 32, alluded to earlier.
v2 = v2k._mm_add_epi32(b: v2j._mm_slli_epi32(imm8: 5))
// Similarly merge v2 (a 4×u32 vector) into s2 (a scalar u32 scalar).
v2 = v2._mm_add_epi32(b: v2._mm_shuffle_epi32(imm8: 0xB1)) // 0b1011_0001
v2 = v2._mm_add_epi32(b: v2._mm_shuffle_epi32(imm8: 0x4E)) // 0b0100_1110
s2 ~mod+= v2.truncate_u32()
// Handle the tail of args.x that wasn't a complete 32-byte chunk.
tail_index = args.x.length() & 0xFFFF_FFFF_FFFF_FFE0 // And-not 32.
if tail_index < args.x.length() {
iterate (p = args.x[tail_index ..])(length: 1, advance: 1, unroll: 1) {
s1 ~mod+= p[0] as base.u32
s2 ~mod+= s1
}
}
// The rest of this function is the same as the non-SIMD version.
s1 %= 65521
s2 %= 65521
args.x = remaining
} endwhile
this.state = ((s2 & 0xFFFF) << 16) | (s1 & 0xFFFF)
}