From 010b34ed38801091c3d45740059b1a9caa03612e Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Mon, 15 Feb 2021 23:52:21 +1100 Subject: [PATCH] Add std/adler32 hasher.up_arm_neon MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On a Raspberry Pi 4 (32-bit armv7l) with -march=native and -mfpu=neon ("native" means "armv8-a+crc+simd"): wuffs_adler32_10k/clang9 1.21GB/s ± 0% 6.58GB/s ± 1% +442.43% (p=0.008 n=5+5) wuffs_adler32_100k/clang9 1.20GB/s ± 0% 6.58GB/s ± 0% +449.37% (p=0.008 n=5+5) wuffs_adler32_10k/gcc8 1.24GB/s ± 0% 4.31GB/s ± 0% +248.87% (p=0.008 n=5+5) wuffs_adler32_100k/gcc8 1.22GB/s ± 0% 4.29GB/s ± 0% +251.09% (p=0.008 n=5+5) mimic_adler32_10k 1.40GB/s ± 0% 1.40GB/s ± 0% -0.04% (p=0.008 n=5+5) mimic_adler32_100k 1.35GB/s ± 0% 1.38GB/s ± 0% +2.77% (p=0.008 n=5+5) wuffs_zlib_decode_10k/clang9 85.3MB/s ± 0% 91.1MB/s ± 0% +6.83% (p=0.008 n=5+5) wuffs_zlib_decode_100k/clang9 107MB/s ± 0% 113MB/s ± 0% +5.20% (p=0.008 n=5+5) wuffs_zlib_decode_10k/gcc8 86.1MB/s ± 0% 91.0MB/s ± 0% +5.69% (p=0.008 n=5+5) wuffs_zlib_decode_100k/gcc8 104MB/s ± 0% 111MB/s ± 0% +7.08% (p=0.016 n=5+4) mimic_zlib_decode_10k 108MB/s ± 0% 108MB/s ± 0% ~ (p=0.548 n=5+5) mimic_zlib_decode_100k 142MB/s ± 0% 141MB/s ± 1% ~ (p=0.310 n=5+5) wuffs_png_decode_image_19k_8bpp/clang9 69.9MB/s ± 0% 74.3MB/s ± 0% +6.37% (p=0.016 n=5+4) wuffs_png_decode_image_40k_24bpp/clang9 99.4MB/s ± 0% 104.3MB/s ± 0% +4.92% (p=0.008 n=5+5) wuffs_png_decode_image_77k_8bpp/clang9 266MB/s ± 0% 280MB/s ± 0% +5.21% (p=0.008 n=5+5) wuffs_png_decode_image_552k_32bpp_ignore_checksum/clang9 165MB/s ± 0% 169MB/s ± 0% +2.45% (p=0.008 n=5+5) wuffs_png_decode_image_552k_32bpp_verify_checksum/clang9 146MB/s ± 0% 163MB/s ± 0% +11.86% (p=0.008 n=5+5) wuffs_png_decode_image_4002k_24bpp/clang9 93.2MB/s ± 0% 97.7MB/s ± 0% +4.78% (p=0.008 n=5+5) wuffs_png_decode_image_19k_8bpp/gcc8 70.3MB/s ± 0% 74.3MB/s ± 0% +5.70% (p=0.008 n=5+5) wuffs_png_decode_image_40k_24bpp/gcc8 101MB/s ± 0% 107MB/s ± 0% +5.67% (p=0.008 n=5+5) wuffs_png_decode_image_77k_8bpp/gcc8 247MB/s ± 0% 259MB/s ± 0% +4.71% (p=0.008 n=5+5) wuffs_png_decode_image_552k_32bpp_ignore_checksum/gcc8 162MB/s ± 0% 162MB/s ± 0% +0.09% (p=0.032 n=5+4) wuffs_png_decode_image_552k_32bpp_verify_checksum/gcc8 140MB/s ± 0% 155MB/s ± 0% +10.67% (p=0.008 n=5+5) wuffs_png_decode_image_4002k_24bpp/gcc8 94.1MB/s ± 0% 99.4MB/s ± 0% +5.57% (p=0.008 n=5+5) mimic_png_decode_image_19k_8bpp 43.6MB/s ± 0% 43.3MB/s ± 1% ~ (p=0.167 n=5+5) mimic_png_decode_image_40k_24bpp 54.6MB/s ± 0% 54.6MB/s ± 0% ~ (p=1.000 n=5+5) mimic_png_decode_image_77k_8bpp 125MB/s ± 0% 123MB/s ± 0% -1.03% (p=0.016 n=5+4) mimic_png_decode_image_552k_32bpp_ignore_checksum skipped mimic_png_decode_image_552k_32bpp_verify_checksum 102MB/s ± 0% 101MB/s ± 0% -0.43% (p=0.008 n=5+5) mimic_png_decode_image_4002k_24bpp 81.3MB/s ± 0% 81.8MB/s ± 0% +0.62% (p=0.008 n=5+5) --- internal/cgen/builtin.go | 216 +++++++++++++++++++++++-- lang/builtin/builtin.go | 63 ++++++++ lang/check/bounds.go | 8 +- lang/token/list.go | 22 ++- release/c/wuffs-unsupported-snapshot.c | 114 +++++++++++++ std/adler32/common_adler32.wuffs | 62 +++---- std/adler32/common_up_arm_neon.wuffs | 194 ++++++++++++++++++++++ 7 files changed, 629 insertions(+), 50 deletions(-) create mode 100644 std/adler32/common_up_arm_neon.wuffs diff --git a/internal/cgen/builtin.go b/internal/cgen/builtin.go index beec84d1..ce3de1df 100644 --- a/internal/cgen/builtin.go +++ b/internal/cgen/builtin.go @@ -448,6 +448,9 @@ func (g *gen) writeBuiltinTokenWriter(b *buffer, recv *a.Expr, method t.ID, args } func (g *gen) writeBuiltinCPUArch(b *buffer, recv *a.Expr, method t.ID, args []*a.Node, sideEffectsOnly bool, depth uint32) error { + armCRC32U32 := recv.MType().Eq(typeExprARMCRC32U32) + armNeon := recv.MType().Eq(typeExprARMNeon64) || recv.MType().Eq(typeExprARMNeon128) + switch method { case t.IDLoadU32, t.IDLoadU64, t.IDLoadSlice128: if !sideEffectsOnly { @@ -521,16 +524,37 @@ func (g *gen) writeBuiltinCPUArch(b *buffer, recv *a.Expr, method t.ID, args []* b.writes(")))") } return nil - } - armCRC32U32 := recv.MType().Eq(typeExprARMCRC32U32) - armNeon := recv.MType().Eq(typeExprARMNeon64) || recv.MType().Eq(typeExprARMNeon128) + case t.IDCreateSlice64, t.IDCreateSlice128: + if armNeon { + switch method { + case t.IDCreateSlice64: + b.writes("vld1_u8(") + case t.IDCreateSlice128: + b.writes("vld1q_u8(") + } + } + if err := g.writeExpr(b, args[0].AsArg().Value(), false, depth); err != nil { + return err + } + b.writes(".ptr)") + return nil + } methodStr := method.Str(g.tm) vreinterpretU8Uxx, vreinterpretUxxU8, vreinterpretClose := "", "", "" if armNeon { vreinterpretClose = ")" switch { + case methodStr == "create_vdupq_n_u16": + vreinterpretU8Uxx = "vreinterpretq_u8_u16(" + vreinterpretUxxU8 = "(" + case methodStr == "create_vdupq_n_u32": + vreinterpretU8Uxx = "vreinterpretq_u8_u32(" + vreinterpretUxxU8 = "(" + case methodStr == "create_vdupq_n_u64": + vreinterpretU8Uxx = "vreinterpretq_u8_u64(" + vreinterpretUxxU8 = "(" case methodStr == "vmovn_u16": vreinterpretU8Uxx = "(" vreinterpretUxxU8 = "vreinterpretq_u16_u8(" @@ -563,13 +587,46 @@ func (g *gen) writeBuiltinCPUArch(b *buffer, recv *a.Expr, method t.ID, args []* } } - const create = "create" + const ( + create = "create" + createLiteralU = "create_literal_u" + ) if methodStr == "value" { return g.writeExpr(b, recv, false, depth) } else if methodStr == create { return g.writeExpr(b, args[0].AsArg().Value(), false, depth) + } else if strings.HasPrefix(methodStr, createLiteralU) { + methodStr = methodStr[len(createLiteralU):] + switch methodStr { + case "16x4": + b.writes("vreinterpret_u8_u16") + case "16x8": + b.writes("vreinterpretq_u8_u16") + case "32x2": + b.writes("vreinterpret_u8_u32") + case "32x4": + b.writes("vreinterpretq_u8_u32") + case "64x1": + b.writes("vreinterpret_u8_u64") + case "64x2": + b.writes("vreinterpretq_u8_u64") + } + b.writes("((uint") + b.writes(methodStr) + b.writes("_t){") + for i, o := range args { + if i > 0 { + b.writes(", ") + } + if err := g.writeExpr(b, o.AsArg().Value(), false, depth); err != nil { + return err + } + } + b.writes("})") + return nil + } else if strings.HasPrefix(methodStr, create) { methodStr = methodStr[len(create):] if armNeon && (methodStr != "") && (methodStr[0] == '_') { @@ -602,9 +659,133 @@ func (g *gen) writeBuiltinCPUArch(b *buffer, recv *a.Expr, method t.ID, args []* } b.writes(after) } + b.writes(")") b.writes(vreinterpretClose) + return nil - } else { + } + + switch methodStr { + case "vaddw_u8": + b.writes("vreinterpretq_u8_u16(") + b.writes(methodStr) + b.writes("(vreinterpretq_u16_u8(") + if err := g.writeExpr(b, recv, false, depth); err != nil { + return err + } + b.writes("), ") + if err := g.writeExpr(b, args[0].AsArg().Value(), false, depth); err != nil { + return err + } + b.writes("))") + return nil + + case "vget_low_u16", "vget_high_u16": + b.writes("vreinterpret_u8_u16(") + b.writes(methodStr) + b.writes("(vreinterpretq_u16_u8(") + if err := g.writeExpr(b, recv, false, depth); err != nil { + return err + } + b.writes(")))") + return nil + + case "vget_low_u32", "vget_high_u32": + b.writes("vreinterpret_u8_u32(") + b.writes(methodStr) + b.writes("(vreinterpretq_u32_u8(") + if err := g.writeExpr(b, recv, false, depth); err != nil { + return err + } + b.writes(")))") + return nil + + case "vget_low_u64", "vget_high_u64": + b.writes("vreinterpret_u8_u64(") + b.writes(methodStr) + b.writes("(vreinterpretq_u64_u8(") + if err := g.writeExpr(b, recv, false, depth); err != nil { + return err + } + b.writes(")))") + return nil + + case "vget_low_u8", "vget_high_u8": + b.writes(methodStr) + b.writes("(") + if err := g.writeExpr(b, recv, false, depth); err != nil { + return err + } + b.writes(")") + return nil + + case "vmlal_u16": + b.writes("vreinterpretq_u8_u32(") + b.writes(methodStr) + b.writes("(vreinterpretq_u32_u8(") + if err := g.writeExpr(b, recv, false, depth); err != nil { + return err + } + b.writes(")") + for i := range args { + b.writes(", vreinterpret_u16_u8(") + if err := g.writeExpr(b, args[i].AsArg().Value(), false, depth); err != nil { + return err + } + b.writes(")") + } + b.writes("))") + + case "vshlq_n_u32": + b.writes("vreinterpretq_u8_u32(") + b.writes(methodStr) + b.writes("(vreinterpretq_u32_u8(") + if err := g.writeExpr(b, recv, false, depth); err != nil { + return err + } + b.writes("), ") + if err := g.writeExpr(b, args[0].AsArg().Value(), false, depth); err != nil { + return err + } + b.writes("))") + + case "vpadalq_u16": + b.writes("vreinterpretq_u8_u32(vpadalq_u16(vreinterpretq_u32_u8(") + if err := g.writeExpr(b, recv, false, depth); err != nil { + return err + } + b.writes("), vreinterpretq_u16_u8(") + if err := g.writeExpr(b, args[0].AsArg().Value(), false, depth); err != nil { + return err + } + b.writes(")))") + return nil + + case "vpadalq_u32": + b.writes("vreinterpretq_u8_u64(vpadalq_u32(vreinterpretq_u64_u8(") + if err := g.writeExpr(b, recv, false, depth); err != nil { + return err + } + b.writes("), vreinterpretq_u32_u8(") + if err := g.writeExpr(b, args[0].AsArg().Value(), false, depth); err != nil { + return err + } + b.writes(")))") + return nil + + case "vpadalq_u8": + b.writes("vreinterpretq_u8_u16(vpadalq_u8(vreinterpretq_u16_u8(") + if err := g.writeExpr(b, recv, false, depth); err != nil { + return err + } + b.writes("), ") + if err := g.writeExpr(b, args[0].AsArg().Value(), false, depth); err != nil { + return err + } + b.writes("))") + return nil + + default: postArgsAfter := "" if armCRC32U32 { b.writeb('_') @@ -612,14 +793,24 @@ func (g *gen) writeBuiltinCPUArch(b *buffer, recv *a.Expr, method t.ID, args []* // TODO: generate this table automatically? postArgsAfter = ")" switch methodStr { - case "vabdl_u16", "vaddl_u16", - "vabdq_u32", "vcleq_u32": - b.writes("vreinterpretq_u8_u32(") - case "vabdl_u32", "vaddl_u32": - b.writes("vreinterpretq_u8_u64(") + case "vpadd_u16": + b.writes("vreinterpret_u8_u16(") + case "vpadd_u32": + b.writes("vreinterpret_u8_u32(") case "vabdl_u8", "vaddl_u8", - "vabdq_u16", "vcleq_u16": + "vabdq_u16", "vcleq_u16", + "vaddq_u16", + "vpaddlq_u8": b.writes("vreinterpretq_u8_u16(") + case "vabdl_u16", "vaddl_u16", + "vabdq_u32", "vcleq_u32", + "vaddq_u32", + "vpaddlq_u16": + b.writes("vreinterpretq_u8_u32(") + case "vabdl_u32", "vaddl_u32", + "vaddq_u64", + "vpaddlq_u32": + b.writes("vreinterpretq_u8_u64(") default: postArgsAfter = "" } @@ -656,10 +847,9 @@ func (g *gen) writeBuiltinCPUArch(b *buffer, recv *a.Expr, method t.ID, args []* } b.writes(after) } + b.writes(")") b.writes(postArgsAfter) } - - b.writes(")") return nil } diff --git a/lang/builtin/builtin.go b/lang/builtin/builtin.go index ef8018cc..8c822fbe 100644 --- a/lang/builtin/builtin.go +++ b/lang/builtin/builtin.go @@ -548,6 +548,18 @@ var Funcs = []string{ // TODO: generate these methods automatically? + "arm_neon_64.create_slice64(a: slice base.u8) arm_neon_64", + + "arm_neon_64.create_literal_u8x8(" + + "a0: u8, a1: u8, a2: u8, a3: u8," + + "a4: u8, a5: u8, a6: u8, a7: u8) arm_neon_64", + "arm_neon_64.create_literal_u16x4(" + + "a0: u16, a1: u16, a2: u16, a3: u16) arm_neon_64", + "arm_neon_64.create_literal_u32x2(" + + "a0: u32, a1: u32) arm_neon_64", + "arm_neon_64.create_literal_u64x1(" + + "a0: u64) arm_neon_64", + "arm_neon_64.create_vdup_n_u16(a: u16) arm_neon_64", "arm_neon_64.create_vdup_n_u32(a: u32) arm_neon_64", "arm_neon_64.create_vdup_n_u64(a: u64) arm_neon_64", @@ -571,19 +583,70 @@ var Funcs = []string{ "arm_neon_64.vhadd_u16(b: arm_neon_64) arm_neon_64", "arm_neon_64.vhadd_u32(b: arm_neon_64) arm_neon_64", "arm_neon_64.vhadd_u8(b: arm_neon_64) arm_neon_64", + "arm_neon_64.vpadd_u16(b: arm_neon_64) arm_neon_64", + "arm_neon_64.vpadd_u32(b: arm_neon_64) arm_neon_64", + "arm_neon_64.vpadd_u8(b: arm_neon_64) arm_neon_64", // ---- arm_neon_128 + "arm_neon_128.create_slice128(a: slice base.u8) arm_neon_128", + + "arm_neon_128.create_literal_u8x16(" + + "a0: u8, a1: u8, a2: u8, a3: u8," + + "a4: u8, a5: u8, a6: u8, a7: u8," + + "a8: u8, a9: u8, a10: u8, a11: u8," + + "a12: u8, a13: u8, a14: u8, a15: u8) arm_neon_128", + "arm_neon_128.create_literal_u16x8(" + + "a0: u16, a1: u16, a2: u16, a3: u16," + + "a4: u16, a5: u16, a6: u16, a7: u16) arm_neon_128", + "arm_neon_128.create_literal_u32x4(" + + "a0: u32, a1: u32, a2: u32, a3: u32) arm_neon_128", + "arm_neon_128.create_literal_u64x2(" + + "a0: u64, a1: u64) arm_neon_128", + + "arm_neon_128.create_vdupq_n_u16(a: u16) arm_neon_128", + "arm_neon_128.create_vdupq_n_u32(a: u32) arm_neon_128", + "arm_neon_128.create_vdupq_n_u64(a: u64) arm_neon_128", + "arm_neon_128.create_vdupq_n_u8(a: u8) arm_neon_128", + "arm_neon_128.vabdq_u16(b: arm_neon_128) arm_neon_128", "arm_neon_128.vabdq_u32(b: arm_neon_128) arm_neon_128", "arm_neon_128.vabdq_u8(b: arm_neon_128) arm_neon_128", + "arm_neon_128.vaddq_u16(b: arm_neon_128) arm_neon_128", + "arm_neon_128.vaddq_u32(b: arm_neon_128) arm_neon_128", + "arm_neon_128.vaddq_u64(b: arm_neon_128) arm_neon_128", + "arm_neon_128.vaddq_u8(b: arm_neon_128) arm_neon_128", + "arm_neon_128.vaddw_u16(b: arm_neon_64) arm_neon_128", + "arm_neon_128.vaddw_u32(b: arm_neon_64) arm_neon_128", + "arm_neon_128.vaddw_u8(b: arm_neon_64) arm_neon_128", + "arm_neon_128.vpadalq_u16(b: arm_neon_128) arm_neon_128", + "arm_neon_128.vpadalq_u32(b: arm_neon_128) arm_neon_128", + "arm_neon_128.vpadalq_u8(b: arm_neon_128) arm_neon_128", + "arm_neon_128.vpaddlq_u16() arm_neon_128", + "arm_neon_128.vpaddlq_u32() arm_neon_128", + "arm_neon_128.vpaddlq_u8() arm_neon_128", "arm_neon_128.vandq_u8(b: arm_neon_128) arm_neon_128", "arm_neon_128.vcleq_u16(b: arm_neon_128) arm_neon_128", "arm_neon_128.vcleq_u32(b: arm_neon_128) arm_neon_128", "arm_neon_128.vcleq_u8(b: arm_neon_128) arm_neon_128", + "arm_neon_128.vget_high_u16() arm_neon_64", + "arm_neon_128.vget_high_u32() arm_neon_64", + "arm_neon_128.vget_high_u64() arm_neon_64", + "arm_neon_128.vget_high_u8() arm_neon_64", + "arm_neon_128.vget_low_u16() arm_neon_64", + "arm_neon_128.vget_low_u32() arm_neon_64", + "arm_neon_128.vget_low_u64() arm_neon_64", + "arm_neon_128.vget_low_u8() arm_neon_64", + "arm_neon_128.vmlal_u16(b: arm_neon_64, c: arm_neon_64) arm_neon_128", + "arm_neon_128.vmlal_u32(b: arm_neon_64, c: arm_neon_64) arm_neon_128", + "arm_neon_128.vmlal_u8(b: arm_neon_64, c: arm_neon_64) arm_neon_128", "arm_neon_128.vmovn_u16() arm_neon_64", "arm_neon_128.vmovn_u32() arm_neon_64", "arm_neon_128.vmovn_u64() arm_neon_64", + "arm_neon_128.vshlq_n_u16(b: u32) arm_neon_128", + "arm_neon_128.vshlq_n_u32(b: u32) arm_neon_128", + "arm_neon_128.vshlq_n_u64(b: u32) arm_neon_128", + "arm_neon_128.vshlq_n_u8(b: u32) arm_neon_128", // ---- x86_m128i diff --git a/lang/check/bounds.go b/lang/check/bounds.go index c35208a7..553facd2 100644 --- a/lang/check/bounds.go +++ b/lang/check/bounds.go @@ -1440,7 +1440,10 @@ var ioMethodAdvances = [...]struct { } var lsMethodAdvances = [...]*big.Int{ - // 128 bits is 16 bytes. 256 bits is 32 bytes. 512 bits is 64 bytes. + // 64 bits is 8 bytes. + // 128 bits is 16 bytes. + // 256 bits is 32 bytes. + // 512 bits is 64 bytes. t.IDLoadSlice128 - t.IDLoadSlice128: sixteen, t.IDLoadSlice256 - t.IDLoadSlice128: thirtyTwo, @@ -1449,6 +1452,9 @@ var lsMethodAdvances = [...]*big.Int{ t.IDStoreSlice128 - t.IDLoadSlice128: sixteen, t.IDStoreSlice256 - t.IDLoadSlice128: thirtyTwo, t.IDStoreSlice512 - t.IDLoadSlice128: sixtyFour, + + t.IDCreateSlice64 - t.IDLoadSlice128: eight, + t.IDCreateSlice128 - t.IDLoadSlice128: sixteen, } func makeConstValueExpr(tm *t.Map, cv *big.Int) (*a.Expr, error) { diff --git a/lang/token/list.go b/lang/token/list.go index dcedf8ef..d73fb860 100644 --- a/lang/token/list.go +++ b/lang/token/list.go @@ -695,14 +695,17 @@ const ( IDStoreSlice256 = ID(0x38B) IDStoreSlice512 = ID(0x38C) - IDCreateMMSet1EPI8 = ID(0x390) - IDCreateMMSet1EPI16 = ID(0x391) - IDCreateMMSet1EPI32 = ID(0x392) - IDCreateMMSet1EPI64X = ID(0x393) - IDCreateMMSetEPI8 = ID(0x394) - IDCreateMMSetEPI16 = ID(0x395) - IDCreateMMSetEPI32 = ID(0x396) - IDCreateMMSetEPI64X = ID(0x397) + IDCreateSlice64 = ID(0x390) + IDCreateSlice128 = ID(0x391) + + IDCreateMMSet1EPI8 = ID(0x3A0) + IDCreateMMSet1EPI16 = ID(0x3A1) + IDCreateMMSet1EPI32 = ID(0x3A2) + IDCreateMMSet1EPI64X = ID(0x3A3) + IDCreateMMSetEPI8 = ID(0x3A4) + IDCreateMMSetEPI16 = ID(0x3A5) + IDCreateMMSetEPI32 = ID(0x3A6) + IDCreateMMSetEPI64X = ID(0x3A7) ) var builtInsByID = [nBuiltInIDs]string{ @@ -1114,6 +1117,9 @@ var builtInsByID = [nBuiltInIDs]string{ IDStoreSlice256: "store_slice256", IDStoreSlice512: "store_slice512", + IDCreateSlice64: "create_slice64", + IDCreateSlice128: "create_slice128", + IDCreateMMSet1EPI8: "create_mm_set1_epi8", IDCreateMMSet1EPI16: "create_mm_set1_epi16", IDCreateMMSet1EPI32: "create_mm_set1_epi32", diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 05afd04e..88603c56 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -17665,6 +17665,13 @@ wuffs_adler32__hasher__up_sse42( wuffs_base__slice_u8 a_x); #endif // defined(WUFFS_BASE__CPU_ARCH__X86_64) +#if defined(WUFFS_BASE__CPU_ARCH__ARM_NEON) +static wuffs_base__empty_struct +wuffs_adler32__hasher__up_arm_neon( + wuffs_adler32__hasher* self, + wuffs_base__slice_u8 a_x); +#endif // defined(WUFFS_BASE__CPU_ARCH__ARM_NEON) + // ---------------- VTables const wuffs_base__hasher_u32__func_ptrs @@ -17776,6 +17783,9 @@ wuffs_adler32__hasher__update_u32( self->private_impl.f_started = true; self->private_impl.f_state = 1; self->private_impl.choosy_up = ( +#if defined(WUFFS_BASE__CPU_ARCH__ARM_NEON) + wuffs_base__cpu_arch__have_arm_neon() ? &wuffs_adler32__hasher__up_arm_neon : +#endif #if defined(WUFFS_BASE__CPU_ARCH__X86_64) wuffs_base__cpu_arch__have_sse42() ? &wuffs_adler32__hasher__up_sse42 : #endif @@ -17951,6 +17961,110 @@ wuffs_adler32__hasher__up_sse42( } #endif // defined(WUFFS_BASE__CPU_ARCH__X86_64) +// -------- func adler32.hasher.up_arm_neon + +#if defined(WUFFS_BASE__CPU_ARCH__ARM_NEON) +static wuffs_base__empty_struct +wuffs_adler32__hasher__up_arm_neon( + wuffs_adler32__hasher* self, + wuffs_base__slice_u8 a_x) { + uint32_t v_s1 = 0; + uint32_t v_s2 = 0; + wuffs_base__slice_u8 v_remaining = {0}; + wuffs_base__slice_u8 v_p = {0}; + uint8x16_t v_p__left = {0}; + uint8x16_t v_p_right = {0}; + uint8x16_t v_v1 = {0}; + uint8x16_t v_v2 = {0}; + uint8x16_t v_col0 = {0}; + uint8x16_t v_col1 = {0}; + uint8x16_t v_col2 = {0}; + uint8x16_t v_col3 = {0}; + uint8x8_t v_sum1 = {0}; + uint8x8_t v_sum2 = {0}; + uint8x8_t v_sum12 = {0}; + uint32_t v_num_iterate_bytes = 0; + uint64_t v_tail_index = 0; + + v_s1 = ((self->private_impl.f_state) & 0xFFFF); + v_s2 = ((self->private_impl.f_state) >> (32 - (16))); + while ((((uint64_t)(a_x.len)) > 0) && ((15 & ((uint32_t)(0xFFF & (uintptr_t)(a_x.ptr)))) != 0)) { + v_s1 += ((uint32_t)(a_x.ptr[0])); + v_s2 += v_s1; + a_x = wuffs_base__slice_u8__subslice_i(a_x, 1); + } + v_s1 %= 65521; + v_s2 %= 65521; + while (((uint64_t)(a_x.len)) > 0) { + v_remaining = wuffs_base__slice_u8__subslice_j(a_x, 0); + if (((uint64_t)(a_x.len)) > 5536) { + v_remaining = wuffs_base__slice_u8__subslice_i(a_x, 5536); + a_x = wuffs_base__slice_u8__subslice_j(a_x, 5536); + } + v_num_iterate_bytes = ((uint32_t)((((uint64_t)(a_x.len)) & 4294967264))); + v_s2 += ((uint32_t)(v_s1 * v_num_iterate_bytes)); + v_v1 = vreinterpretq_u8_u32(vdupq_n_u32(0)); + v_v2 = vreinterpretq_u8_u32(vdupq_n_u32(0)); + v_col0 = vreinterpretq_u8_u16(vdupq_n_u16(0)); + v_col1 = vreinterpretq_u8_u16(vdupq_n_u16(0)); + v_col2 = vreinterpretq_u8_u16(vdupq_n_u16(0)); + v_col3 = vreinterpretq_u8_u16(vdupq_n_u16(0)); + { + wuffs_base__slice_u8 i_slice_p = a_x; + v_p.ptr = i_slice_p.ptr; + v_p.len = 32; + uint8_t* i_end0_p = v_p.ptr + (((i_slice_p.len - (size_t)(v_p.ptr - i_slice_p.ptr)) / 32) * 32); + while (v_p.ptr < i_end0_p) { + v_p__left = vld1q_u8(wuffs_base__slice_u8__subslice_j(v_p, 16).ptr); + v_p_right = vld1q_u8(wuffs_base__slice_u8__subslice_ij(v_p, 16, 32).ptr); + v_v2 = vreinterpretq_u8_u32(vaddq_u32(vreinterpretq_u32_u8(v_v2), vreinterpretq_u32_u8(v_v1))); + v_v1 = vreinterpretq_u8_u32(vpadalq_u16(vreinterpretq_u32_u8(v_v1), vreinterpretq_u16_u8(vreinterpretq_u8_u16(vpadalq_u8(vreinterpretq_u16_u8(vreinterpretq_u8_u16(vpaddlq_u8(v_p__left))), v_p_right))))); + v_col0 = vreinterpretq_u8_u16(vaddw_u8(vreinterpretq_u16_u8(v_col0), vget_low_u8(v_p__left))); + v_col1 = vreinterpretq_u8_u16(vaddw_u8(vreinterpretq_u16_u8(v_col1), vget_high_u8(v_p__left))); + v_col2 = vreinterpretq_u8_u16(vaddw_u8(vreinterpretq_u16_u8(v_col2), vget_low_u8(v_p_right))); + v_col3 = vreinterpretq_u8_u16(vaddw_u8(vreinterpretq_u16_u8(v_col3), vget_high_u8(v_p_right))); + v_p.ptr += 32; + } + v_p.len = 0; + } + v_v2 = vreinterpretq_u8_u32(vshlq_n_u32(vreinterpretq_u32_u8(v_v2), 5)); + v_v2 = vreinterpretq_u8_u32(vmlal_u16(vreinterpretq_u32_u8(v_v2), vreinterpret_u16_u8(vreinterpret_u8_u16(vget_low_u16(vreinterpretq_u16_u8(v_col0)))), vreinterpret_u16_u8(vreinterpret_u8_u16((uint16x4_t){32, 31, 30, 29})))); + v_v2 = vreinterpretq_u8_u32(vmlal_u16(vreinterpretq_u32_u8(v_v2), vreinterpret_u16_u8(vreinterpret_u8_u16(vget_high_u16(vreinterpretq_u16_u8(v_col0)))), vreinterpret_u16_u8(vreinterpret_u8_u16((uint16x4_t){28, 27, 26, 25})))); + v_v2 = vreinterpretq_u8_u32(vmlal_u16(vreinterpretq_u32_u8(v_v2), vreinterpret_u16_u8(vreinterpret_u8_u16(vget_low_u16(vreinterpretq_u16_u8(v_col1)))), vreinterpret_u16_u8(vreinterpret_u8_u16((uint16x4_t){24, 23, 22, 21})))); + v_v2 = vreinterpretq_u8_u32(vmlal_u16(vreinterpretq_u32_u8(v_v2), vreinterpret_u16_u8(vreinterpret_u8_u16(vget_high_u16(vreinterpretq_u16_u8(v_col1)))), vreinterpret_u16_u8(vreinterpret_u8_u16((uint16x4_t){20, 19, 18, 17})))); + v_v2 = vreinterpretq_u8_u32(vmlal_u16(vreinterpretq_u32_u8(v_v2), vreinterpret_u16_u8(vreinterpret_u8_u16(vget_low_u16(vreinterpretq_u16_u8(v_col2)))), vreinterpret_u16_u8(vreinterpret_u8_u16((uint16x4_t){16, 15, 14, 13})))); + v_v2 = vreinterpretq_u8_u32(vmlal_u16(vreinterpretq_u32_u8(v_v2), vreinterpret_u16_u8(vreinterpret_u8_u16(vget_high_u16(vreinterpretq_u16_u8(v_col2)))), vreinterpret_u16_u8(vreinterpret_u8_u16((uint16x4_t){12, 11, 10, 9})))); + v_v2 = vreinterpretq_u8_u32(vmlal_u16(vreinterpretq_u32_u8(v_v2), vreinterpret_u16_u8(vreinterpret_u8_u16(vget_low_u16(vreinterpretq_u16_u8(v_col3)))), vreinterpret_u16_u8(vreinterpret_u8_u16((uint16x4_t){8, 7, 6, 5})))); + v_v2 = vreinterpretq_u8_u32(vmlal_u16(vreinterpretq_u32_u8(v_v2), vreinterpret_u16_u8(vreinterpret_u8_u16(vget_high_u16(vreinterpretq_u16_u8(v_col3)))), vreinterpret_u16_u8(vreinterpret_u8_u16((uint16x4_t){4, 3, 2, 1})))); + v_sum1 = vreinterpret_u8_u32(vpadd_u32(vreinterpret_u32_u8(vreinterpret_u8_u32(vget_low_u32(vreinterpretq_u32_u8(v_v1)))), vreinterpret_u32_u8(vreinterpret_u8_u32(vget_high_u32(vreinterpretq_u32_u8(v_v1)))))); + v_sum2 = vreinterpret_u8_u32(vpadd_u32(vreinterpret_u32_u8(vreinterpret_u8_u32(vget_low_u32(vreinterpretq_u32_u8(v_v2)))), vreinterpret_u32_u8(vreinterpret_u8_u32(vget_high_u32(vreinterpretq_u32_u8(v_v2)))))); + v_sum12 = vreinterpret_u8_u32(vpadd_u32(vreinterpret_u32_u8(v_sum1), vreinterpret_u32_u8(v_sum2))); + v_s1 += vget_lane_u32(vreinterpret_u32_u8(v_sum12), 0); + v_s2 += vget_lane_u32(vreinterpret_u32_u8(v_sum12), 1); + v_tail_index = (((uint64_t)(a_x.len)) & 18446744073709551584u); + if (v_tail_index < ((uint64_t)(a_x.len))) { + { + wuffs_base__slice_u8 i_slice_p = wuffs_base__slice_u8__subslice_i(a_x, v_tail_index); + v_p.ptr = i_slice_p.ptr; + v_p.len = 1; + uint8_t* i_end0_p = i_slice_p.ptr + i_slice_p.len; + while (v_p.ptr < i_end0_p) { + v_s1 += ((uint32_t)(v_p.ptr[0])); + v_s2 += v_s1; + v_p.ptr += 1; + } + v_p.len = 0; + } + } + v_s1 %= 65521; + v_s2 %= 65521; + a_x = v_remaining; + } + self->private_impl.f_state = (((v_s2 & 65535) << 16) | (v_s1 & 65535)); + return wuffs_base__make_empty_struct(); +} +#endif // defined(WUFFS_BASE__CPU_ARCH__ARM_NEON) + #endif // !defined(WUFFS_CONFIG__MODULES) || defined(WUFFS_CONFIG__MODULE__ADLER32) #if !defined(WUFFS_CONFIG__MODULES) || defined(WUFFS_CONFIG__MODULE__BMP) diff --git a/std/adler32/common_adler32.wuffs b/std/adler32/common_adler32.wuffs index 8121c771..0ad80542 100644 --- a/std/adler32/common_adler32.wuffs +++ b/std/adler32/common_adler32.wuffs @@ -25,7 +25,9 @@ pub func hasher.update_u32!(x: slice base.u8) base.u32 { if not this.started { this.started = true this.state = 1 - choose up = [up_sse42] + choose up = [ + up_arm_neon, + up_sse42] } this.up!(x: args.x) return this.state @@ -74,20 +76,22 @@ pri func hasher.up_sse42!(x: slice base.u8), var p : slice base.u8 // The remaining variables are specific to the SIMD version. - var zeroes : base.x86_m128i - var ones : base.x86_m128i - var weights__left : base.x86_m128i - var weights_right : base.x86_m128i - var p__left : base.x86_m128i - var p_right : base.x86_m128i - var v1 : base.x86_m128i - var v2 : base.x86_m128i - var v2j : base.x86_m128i - var v2k : base.x86_m128i + + var zeroes : base.x86_m128i + var ones : base.x86_m128i + var weights__left : base.x86_m128i + var weights_right : base.x86_m128i + var p__left : base.x86_m128i + var p_right : base.x86_m128i + var v1 : base.x86_m128i + var v2 : base.x86_m128i + var v2j : base.x86_m128i + var v2k : base.x86_m128i + var num_iterate_bytes : base.u32 var tail_index : base.u64 - // zeroes and ones are uniform 8×u16 vectors. + // zeroes and ones are uniform u16×8 vectors. zeroes = zeroes.create_mm_set1_epi16(a: 0) ones = ones.create_mm_set1_epi16(a: 1) @@ -104,11 +108,13 @@ pri func hasher.up_sse42!(x: slice base.u8), e7: 0x09, e6: 0x0A, e5: 0x0B, e4: 0x0C, e3: 0x0D, e2: 0x0E, e1: 0x0F, e0: 0x10) - // Just like the non-SIMD version, split the u32 state into two u16 halves - // and loop over args.x up to almost-5552 bytes at a time. The slightly - // smaller 5536 is the largest multiple of 32 less than non-SIMD's 5552. + // Decompose this.state. s1 = this.state.low_bits(n: 16) s2 = this.state.high_bits(n: 16) + + // Just like the non-SIMD version, loop over args.x up to almost-5552 bytes + // at a time. The slightly smaller 5536 is the largest multiple of 32 less + // than non-SIMD's 5552. while args.x.length() > 0 { remaining = args.x[.. 0] if args.x.length() > 5536 { @@ -130,8 +136,8 @@ pri func hasher.up_sse42!(x: slice base.u8), num_iterate_bytes = (args.x.length() & 0xFFFF_FFE0) as base.u32 s2 ~mod+= (s1 ~mod* num_iterate_bytes) - // Zero-initialize some 4×u32 vectors associated with the two state - // variables s1 and s2. The iterate loop accumulates 4 parallel u32 + // Zero-initialize some u32×4 vectors associated with the two state + // variables s1 and s2. The iterate loop accumulates four parallel u32 // sums in each vector. A post-iterate step merges the four u32 sums // into a single u32 sum. v1 = v1.create_mm_setzero_si128() @@ -143,23 +149,23 @@ pri func hasher.up_sse42!(x: slice base.u8), // Split the 32-byte p into left and right halves. SSE4.2 works // with 16-byte registers. // - // Let p__left = [16×u8: p00, p01, p02, ..., p15] - // Let p_right = [16×u8: p16, p17, p18, ..., p31] + // Let p__left = [u8×16: p00, p01, p02, ..., p15] + // Let p_right = [u8×16: p16, p17, p18, ..., p31] p__left.load_slice128!(a: p[.. 16]) p_right.load_slice128!(a: p[16 .. 32]) // For v2j, we need to calculate the sums of the s1j terms for each // of p's 32 elements. This is simply 32 times the same number, - // that number being the sum of v1's four u32 accumulators. We - // add v1 now and multiply by 32 later, outside the inner loop. + // that number being the sum of v1's four u32 accumulators. We add + // v1 now and multiply by 32 later, outside the inner loop. v2j = v2j._mm_add_epi32(b: v1) // For v1, we need to add the elements of p. Computing the sum of // absolute differences (_mm_sad_epu8) with zero just sums the // elements. p__left._mm_sad_epu8(b: zeroes) equals - // [2×u64: p00 + p01 + ... + p07, p08 + p09 + ... + p15] + // [u64×2: p00 + p01 + ... + p07, p08 + p09 + ... + p15] // This is equivalent (little-endian) to: - // [4×u32: p00 + p01 + ... + p07, 0, p08 + p09 + ... + p15, 0] + // [u32×4: p00 + p01 + ... + p07, 0, p08 + p09 + ... + p15, 0] // We accumulate those "sum of p__left elements" in v1, and ditto // for the p_right elements. v1 = v1._mm_add_epi32(b: p__left._mm_sad_epu8(b: zeroes)) @@ -171,7 +177,7 @@ pri func hasher.up_sse42!(x: slice base.u8), // // The _mm_maddubs_epi16 call (vertically multiply u8 columns and // then horizontally sum u16 pairs) with the left half produces: - // [8×u16: ((32*p00)+(31*p01)), + // [u16×8: ((32*p00)+(31*p01)), // ((30*p02)+(29*p03)), // ... // ((18*p14)+(17*p15))] @@ -179,7 +185,7 @@ pri func hasher.up_sse42!(x: slice base.u8), // The ones._mm_madd_epi16(b: etc) call is likewise a multiply-add // (note that it's "madd" not "add"). Multiplying by 1 is a no-op, // so this sums u16 pairs to produce u32 values: - // [4×u32: ((32*p00)+(31*p01)+(30*p02)+(29*p03)), + // [u32×4: ((32*p00)+(31*p01)+(30*p02)+(29*p03)), // ((28*p04)+(27*p05)+(26*p06)+(25*p07)), // ... // ((20*p12)+(19*p13)+(18*p14)+(17*p15))] @@ -191,8 +197,8 @@ pri func hasher.up_sse42!(x: slice base.u8), p_right._mm_maddubs_epi16(b: weights_right))) } - // Merge the four per-u32-lane sums (v1) into the single u32 sum (s1). - // Starting with a 4×u32 vector [x0, x1, x2, x3]: + // Merge the four parallel u32 sums (v1) into the single u32 sum (s1). + // Starting with a u32×4 vector [x0, x1, x2, x3]: // - shuffling with 0b1011_0001 gives [x1, x0, x3, x2]. // - adding gives [x0+x1, x0+x1, x2+x3, x2+x3]. // - shuffling with 0b0100_1110 gives [x2+x3, x2+x3, x0+x1, x0+x1]. @@ -206,7 +212,7 @@ pri func hasher.up_sse42!(x: slice base.u8), // multiplies v2j's four u32 elements each by 32, alluded to earlier. v2 = v2k._mm_add_epi32(b: v2j._mm_slli_epi32(imm8: 5)) - // Similarly merge v2 (a 4×u32 vector) into s2 (a scalar u32 scalar). + // Similarly merge v2 (a u32×4 vector) into s2 (a u32 scalar). v2 = v2._mm_add_epi32(b: v2._mm_shuffle_epi32(imm8: 0b1011_0001)) v2 = v2._mm_add_epi32(b: v2._mm_shuffle_epi32(imm8: 0b0100_1110)) s2 ~mod+= v2.truncate_u32() diff --git a/std/adler32/common_up_arm_neon.wuffs b/std/adler32/common_up_arm_neon.wuffs new file mode 100644 index 00000000..821ae0bf --- /dev/null +++ b/std/adler32/common_up_arm_neon.wuffs @@ -0,0 +1,194 @@ +// Copyright 2021 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pri func hasher.up_arm_neon!(x: slice base.u8), + choose cpu_arch >= arm_neon, +{ + // These variables are the same as the non-SIMD version. + var s1 : base.u32 + var s2 : base.u32 + var remaining : slice base.u8 + var p : slice base.u8 + + // The remaining variables are specific to the SIMD version. + + var p__left : base.arm_neon_128 + var p_right : base.arm_neon_128 + var v1 : base.arm_neon_128 + var v2 : base.arm_neon_128 + var col0 : base.arm_neon_128 + var col1 : base.arm_neon_128 + var col2 : base.arm_neon_128 + var col3 : base.arm_neon_128 + var sum1 : base.arm_neon_64 + var sum2 : base.arm_neon_64 + var sum12 : base.arm_neon_64 + + var num_iterate_bytes : base.u32 + var tail_index : base.u64 + + // Decompose this.state. + s1 = this.state.low_bits(n: 16) + s2 = this.state.high_bits(n: 16) + + // Align to a 16-byte boundary. + while (args.x.length() > 0) and ((15 & args.x.uintptr_low_12_bits()) <> 0) { + s1 ~mod+= args.x[0] as base.u32 + s2 ~mod+= s1 + args.x = args.x[1 ..] + } endwhile + s1 %= 65521 + s2 %= 65521 + + // Just like the non-SIMD version, loop over args.x up to almost-5552 bytes + // at a time. The slightly smaller 5536 is the largest multiple of 32 less + // than non-SIMD's 5552. + while args.x.length() > 0 { + remaining = args.x[.. 0] + if args.x.length() > 5536 { + remaining = args.x[5536 ..] + args.x = args.x[.. 5536] + } + + // The s1 state is the sum of the input bytes and the s2 state is the + // sum of the s1 state at each 1-byte step. Inside the iterate loop + // below, but starting fresh at each outer while loop iteration, s1 + // consists of three parts (called s1i, s1j and s1k): + // - s1i: the initial value, before any 32-byte iterations. + // - s1j: the total contribution from previous 32-byte iterations. + // - s1k: the contribution due to the current 32-byte iteration. + // + // The upcoming iterate loop (at 32 bytes per iteration) encompasses + // num_iterate_bytes 1-byte steps. We hoist the total s1i contribution, + // (s1i * num_iterate_bytes) out here. + num_iterate_bytes = (args.x.length() & 0xFFFF_FFE0) as base.u32 + s2 ~mod+= (s1 ~mod* num_iterate_bytes) + + // Zero-initialize some u32×4 vectors associated with the two state + // variables s1 and s2. The iterate loop accumulates four parallel u32 + // sums in each vector. A post-iterate step merges the four u32 sums + // into a single u32 sum. + v1 = v1.create_vdupq_n_u32(a: 0) + v2 = v2.create_vdupq_n_u32(a: 0) + col0 = col0.create_vdupq_n_u16(a: 0) + col1 = col1.create_vdupq_n_u16(a: 0) + col2 = col2.create_vdupq_n_u16(a: 0) + col3 = col3.create_vdupq_n_u16(a: 0) + + // The inner loop. + iterate (p = args.x)(length: 32, advance: 32, unroll: 1) { + // Split the 32-byte p into left and right halves. NEON works with + // 16-byte registers. + // + // Let p__left = [u8×16: p00, p01, p02, ..., p15] + // Let p_right = [u8×16: p16, p17, p18, ..., p31] + p__left = p__left.create_slice128(a: p[.. 16]) + p_right = p_right.create_slice128(a: p[16 .. 32]) + + // For v2j, we need to calculate the sums of the s1j terms for each + // of p's 32 elements. This is simply 32 times the same number, + // that number being the sum of v1's four u32 accumulators. We add + // v1 now and multiply by 32 later, outside the inner loop. + v2 = v2.vaddq_u32(b: v1) + + // For v1, we need to add the elements of p. + // + // p__left.vpaddlq_u8() is: + // [u16×8: p00 + p01, p02 + p03, ..., p14 + p15] + // + // Combining (vpadalq_u8) that with p_right gives: + // [u16×8: p00 + p01 + p16 + p17, + // p02 + p03 + p18 + p19, + // ... + // p14 + p15 + p30 + p31] + // + // Pair-wise summing (and widening) again sets v1 to: + // [u32×4: Σ (p00 + p01 + p02 + p03 + p16 + p17 + p18 + p19), + // Σ (p04 + p05 + p06 + p07 + p20 + p21 + p22 + p23), + // ... + // Σ (p12 + p13 + p14 + p15 + p28 + p29 + p30 + p31)] + v1 = v1.vpadalq_u16(b: p__left.vpaddlq_u8().vpadalq_u8(b: p_right)) + + // Accumulate column sums: + // col0 = [u16×8: Σ p00, Σ p01, ..., Σ p07] + // col1 = [u16×8: Σ p08, Σ p09, ..., Σ p15] + // col2 = [u16×8: Σ p16, Σ p17, ..., Σ p23] + // col3 = [u16×8: Σ p24, Σ p25, ..., Σ p31] + col0 = col0.vaddw_u8(b: p__left.vget_low_u8()) + col1 = col1.vaddw_u8(b: p__left.vget_high_u8()) + col2 = col2.vaddw_u8(b: p_right.vget_low_u8()) + col3 = col3.vaddw_u8(b: p_right.vget_high_u8()) + } + + // Multiply v2j's four u32 elements each by 32, alluded to earlier. + v2 = v2.vshlq_n_u32(b: 5) + + // Add the v2k contributions in eight u32×4 vectors: + // [u32×4: 0x20 * (Σ p00), 0x1F * (Σ p01), ..., 0x1D * (Σ p03)] + // [u32×4: 0x1C * (Σ p04), 0x1B * (Σ p05), ..., 0x19 * (Σ p07)] + // ... + // [u32×4: 0x04 * (Σ p28), 0x03 * (Σ p29), ..., 0x01 * (Σ p31)] + v2 = v2.vmlal_u16( + b: col0.vget_low_u16(), + c: sum2.create_literal_u16x4(a0: 0x20, a1: 0x1F, a2: 0x1E, a3: 0x1D)) + v2 = v2.vmlal_u16( + b: col0.vget_high_u16(), + c: sum2.create_literal_u16x4(a0: 0x1C, a1: 0x1B, a2: 0x1A, a3: 0x19)) + v2 = v2.vmlal_u16( + b: col1.vget_low_u16(), + c: sum2.create_literal_u16x4(a0: 0x18, a1: 0x17, a2: 0x16, a3: 0x15)) + v2 = v2.vmlal_u16( + b: col1.vget_high_u16(), + c: sum2.create_literal_u16x4(a0: 0x14, a1: 0x13, a2: 0x12, a3: 0x11)) + v2 = v2.vmlal_u16( + b: col2.vget_low_u16(), + c: sum2.create_literal_u16x4(a0: 0x10, a1: 0x0F, a2: 0x0E, a3: 0x0D)) + v2 = v2.vmlal_u16( + b: col2.vget_high_u16(), + c: sum2.create_literal_u16x4(a0: 0x0C, a1: 0x0B, a2: 0x0A, a3: 0x09)) + v2 = v2.vmlal_u16( + b: col3.vget_low_u16(), + c: sum2.create_literal_u16x4(a0: 0x08, a1: 0x07, a2: 0x06, a3: 0x05)) + v2 = v2.vmlal_u16( + b: col3.vget_high_u16(), + c: sum2.create_literal_u16x4(a0: 0x04, a1: 0x03, a2: 0x02, a3: 0x01)) + + // Merge the four parallel u32 sums (v1) into the single u32 sum (s1) + // and ditto for v2. Starting with [u32×4: xx_0, xx_1, xx_2, xx_3]: + // sum1 = [u32×2: v1_0 + v1_1, v1_2 + v1_3] + // sum2 = [u32×2: v2_0 + v2_1, v2_2 + v2_3] + // sum12 = [u32×2: v1_0 + v1_1 + v1_2 + v1_3, + // v2_0 + v2_1 + v2_2 + v2_3] + sum1 = v1.vget_low_u32().vpadd_u32(b: v1.vget_high_u32()) + sum2 = v2.vget_low_u32().vpadd_u32(b: v2.vget_high_u32()) + sum12 = sum1.vpadd_u32(b: sum2) + s1 ~mod+= sum12.vget_lane_u32(b: 0) + s2 ~mod+= sum12.vget_lane_u32(b: 1) + + // Handle the tail of args.x that wasn't a complete 32-byte chunk. + tail_index = args.x.length() & 0xFFFF_FFFF_FFFF_FFE0 // And-not 32. + if tail_index < args.x.length() { + iterate (p = args.x[tail_index ..])(length: 1, advance: 1, unroll: 1) { + s1 ~mod+= p[0] as base.u32 + s2 ~mod+= s1 + } + } + + // The rest of this function is the same as the non-SIMD version. + s1 %= 65521 + s2 %= 65521 + args.x = remaining + } endwhile + this.state = ((s2 & 0xFFFF) << 16) | (s1 & 0xFFFF) +}