From 3b5d8fdfbac3d2ac627a3575c5c6f6ccccd8c945 Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Wed, 24 Feb 2021 16:13:59 +1100 Subject: [PATCH] Use arm_neon_uAxB types and methods --- internal/cgen/builtin.go | 72 ++++++-- lang/builtin/builtin.go | 31 ++++ release/c/wuffs-unsupported-snapshot.c | 228 ++++++++++++------------- std/adler32/common_up_arm_neon.wuffs | 55 +++--- std/png/decode_filter_arm_neon.wuffs | 97 ++++++----- 5 files changed, 279 insertions(+), 204 deletions(-) diff --git a/internal/cgen/builtin.go b/internal/cgen/builtin.go index 3d3a24f9..dfd6f9ba 100644 --- a/internal/cgen/builtin.go +++ b/internal/cgen/builtin.go @@ -775,29 +775,69 @@ func (g *gen) writeBuiltinCPUArch(b *buffer, recv *a.Expr, method t.ID, args []* func (g *gen) writeBuiltinCPUArchARMNeon(b *buffer, recv *a.Expr, method t.ID, args []*a.Node, sideEffectsOnly bool, depth uint32) error { methodStr := method.Str(g.tm) if strings.HasPrefix(methodStr, "make_") { - fName := "" - switch methodStr { - case "make_u8x8_repeat": - fName = "vdup_n_u8" - case "make_u16x4_repeat": - fName = "vdup_n_u16" - case "make_u32x2_repeat": - fName = "vdup_n_u32" - case "make_u64x1_repeat": - fName = "vdup_n_u64" - default: - return fmt.Errorf("internal error: unsupported cpu_arch method %q", methodStr) + before, after, ptr := "", ")", false + if strings.HasSuffix(methodStr, "_multiple") { + after = "})" + switch methodStr { + case "make_u8x8_multiple": + before = "((uint8x8_t){" + case "make_u16x4_multiple": + before = "((uint16x4_t){" + case "make_u32x2_multiple": + before = "((uint32x2_t){" + case "make_u64x1_multiple": + before = "((uint64x1_t){" + case "make_u8x16_multiple": + before = "((uint8x16_t){" + case "make_u16x8_multiple": + before = "((uint16x8_t){" + case "make_u32x4_multiple": + before = "((uint32x4_t){" + case "make_u64x2_multiple": + before = "((uint64x2_t){" + } + } else { + switch methodStr { + case "make_u8x8_repeat": + before = "vdup_n_u8(" + case "make_u16x4_repeat": + before = "vdup_n_u16(" + case "make_u32x2_repeat": + before = "vdup_n_u32(" + case "make_u64x1_repeat": + before = "vdup_n_u64(" + case "make_u8x16_repeat": + before = "vdupq_n_u8(" + case "make_u16x8_repeat": + before = "vdupq_n_u16(" + case "make_u32x4_repeat": + before = "vdupq_n_u32(" + case "make_u64x2_repeat": + before = "vdupq_n_u64(" + case "make_u8x8_slice64": + before, ptr = "vld1_u8(", true + case "make_u8x16_slice128": + before, ptr = "vld1q_u8(", true + default: + return fmt.Errorf("internal error: unsupported cpu_arch method %q", methodStr) + } } - b.printf("%s(", fName) + b.writes(before) for i, o := range args { if i > 0 { b.writes(", ") } - if err := g.writeExpr(b, o.AsArg().Value(), false, depth); err != nil { - return err + if ptr { + if err := g.writeExprDotPtr(b, o.AsArg().Value(), false, depth); err != nil { + return err + } + } else { + if err := g.writeExpr(b, o.AsArg().Value(), false, depth); err != nil { + return err + } } } - b.writes(")") + b.writes(after) return nil } else if strings.HasPrefix(methodStr, "as_") { diff --git a/lang/builtin/builtin.go b/lang/builtin/builtin.go index ad8794d3..d9553da0 100644 --- a/lang/builtin/builtin.go +++ b/lang/builtin/builtin.go @@ -568,11 +568,42 @@ var funcsOther = [...]string{ // ---- arm_neon_utility + "arm_neon_utility.make_u8x8_multiple(" + + "a00: u8, a01: u8, a02: u8, a03: u8," + + "a04: u8, a05: u8, a06: u8, a07: u8) arm_neon_u8x8", + "arm_neon_utility.make_u16x4_multiple(" + + "a00: u16, a01: u16, a02: u16, a03: u16) arm_neon_u16x4", + "arm_neon_utility.make_u32x2_multiple(" + + "a00: u32, a01: u32) arm_neon_u32x2", + "arm_neon_utility.make_u64x1_multiple(" + + "a00: u64) arm_neon_u64x1", + + "arm_neon_utility.make_u8x16_multiple(" + + "a00: u8, a01: u8, a02: u8, a03: u8," + + "a04: u8, a05: u8, a06: u8, a07: u8," + + "a08: u8, a09: u8, a10: u8, a11: u8," + + "a12: u8, a13: u8, a14: u8, a15: u8) arm_neon_u8x16", + "arm_neon_utility.make_u16x8_multiple(" + + "a00: u16, a01: u16, a02: u16, a03: u16," + + "a04: u16, a05: u16, a06: u16, a07: u16) arm_neon_u16x8", + "arm_neon_utility.make_u32x4_multiple(" + + "a00: u32, a01: u32, a02: u32, a03: u32) arm_neon_u32x4", + "arm_neon_utility.make_u64x2_multiple(" + + "a00: u64, a01: u64) arm_neon_u64x2", + "arm_neon_utility.make_u8x8_repeat(a: u8) arm_neon_u8x8", "arm_neon_utility.make_u16x4_repeat(a: u16) arm_neon_u16x4", "arm_neon_utility.make_u32x2_repeat(a: u32) arm_neon_u32x2", "arm_neon_utility.make_u64x1_repeat(a: u64) arm_neon_u64x1", + "arm_neon_utility.make_u8x16_repeat(a: u8) arm_neon_u8x16", + "arm_neon_utility.make_u16x8_repeat(a: u16) arm_neon_u16x8", + "arm_neon_utility.make_u32x4_repeat(a: u32) arm_neon_u32x4", + "arm_neon_utility.make_u64x2_repeat(a: u64) arm_neon_u64x2", + + "arm_neon_utility.make_u8x8_slice64(a: slice base.u8) arm_neon_u8x8", + "arm_neon_utility.make_u8x16_slice128(a: slice base.u8) arm_neon_u8x16", + // ---- arm_neon_uAxB.as_uCxD "arm_neon_u8x8.as_u16x4() arm_neon_u16x4", diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 6c60a224..3be2495d 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -17835,15 +17835,15 @@ wuffs_adler32__hasher__up_arm_neon( wuffs_base__slice_u8 v_p = {0}; uint8x16_t v_p__left = {0}; uint8x16_t v_p_right = {0}; - uint8x16_t v_v1 = {0}; - uint8x16_t v_v2 = {0}; - uint8x16_t v_col0 = {0}; - uint8x16_t v_col1 = {0}; - uint8x16_t v_col2 = {0}; - uint8x16_t v_col3 = {0}; - uint8x8_t v_sum1 = {0}; - uint8x8_t v_sum2 = {0}; - uint8x8_t v_sum12 = {0}; + uint32x4_t v_v1 = {0}; + uint32x4_t v_v2 = {0}; + uint16x8_t v_col0 = {0}; + uint16x8_t v_col1 = {0}; + uint16x8_t v_col2 = {0}; + uint16x8_t v_col3 = {0}; + uint32x2_t v_sum1 = {0}; + uint32x2_t v_sum2 = {0}; + uint32x2_t v_sum12 = {0}; uint32_t v_num_iterate_bytes = 0; uint64_t v_tail_index = 0; @@ -17864,44 +17864,44 @@ wuffs_adler32__hasher__up_arm_neon( } v_num_iterate_bytes = ((uint32_t)((((uint64_t)(a_x.len)) & 4294967264))); v_s2 += ((uint32_t)(v_s1 * v_num_iterate_bytes)); - v_v1 = vreinterpretq_u8_u32(vdupq_n_u32(0)); - v_v2 = vreinterpretq_u8_u32(vdupq_n_u32(0)); - v_col0 = vreinterpretq_u8_u16(vdupq_n_u16(0)); - v_col1 = vreinterpretq_u8_u16(vdupq_n_u16(0)); - v_col2 = vreinterpretq_u8_u16(vdupq_n_u16(0)); - v_col3 = vreinterpretq_u8_u16(vdupq_n_u16(0)); + v_v1 = vdupq_n_u32(0); + v_v2 = vdupq_n_u32(0); + v_col0 = vdupq_n_u16(0); + v_col1 = vdupq_n_u16(0); + v_col2 = vdupq_n_u16(0); + v_col3 = vdupq_n_u16(0); { wuffs_base__slice_u8 i_slice_p = a_x; v_p.ptr = i_slice_p.ptr; v_p.len = 32; uint8_t* i_end0_p = v_p.ptr + (((i_slice_p.len - (size_t)(v_p.ptr - i_slice_p.ptr)) / 32) * 32); while (v_p.ptr < i_end0_p) { - v_p__left = vld1q_u8(wuffs_base__slice_u8__subslice_j(v_p, 16).ptr); - v_p_right = vld1q_u8(wuffs_base__slice_u8__subslice_ij(v_p, 16, 32).ptr); - v_v2 = vreinterpretq_u8_u32(vaddq_u32(vreinterpretq_u32_u8(v_v2), vreinterpretq_u32_u8(v_v1))); - v_v1 = vreinterpretq_u8_u32(vpadalq_u16(vreinterpretq_u32_u8(v_v1), vreinterpretq_u16_u8(vreinterpretq_u8_u16(vpadalq_u8(vreinterpretq_u16_u8(vreinterpretq_u8_u16(vpaddlq_u8(v_p__left))), v_p_right))))); - v_col0 = vreinterpretq_u8_u16(vaddw_u8(vreinterpretq_u16_u8(v_col0), vget_low_u8(v_p__left))); - v_col1 = vreinterpretq_u8_u16(vaddw_u8(vreinterpretq_u16_u8(v_col1), vget_high_u8(v_p__left))); - v_col2 = vreinterpretq_u8_u16(vaddw_u8(vreinterpretq_u16_u8(v_col2), vget_low_u8(v_p_right))); - v_col3 = vreinterpretq_u8_u16(vaddw_u8(vreinterpretq_u16_u8(v_col3), vget_high_u8(v_p_right))); + v_p__left = vld1q_u8(v_p.ptr); + v_p_right = vld1q_u8(v_p.ptr + 16); + v_v2 = vaddq_u32(v_v2, v_v1); + v_v1 = vpadalq_u16(v_v1, vpadalq_u8(vpaddlq_u8(v_p__left), v_p_right)); + v_col0 = vaddw_u8(v_col0, vget_low_u8(v_p__left)); + v_col1 = vaddw_u8(v_col1, vget_high_u8(v_p__left)); + v_col2 = vaddw_u8(v_col2, vget_low_u8(v_p_right)); + v_col3 = vaddw_u8(v_col3, vget_high_u8(v_p_right)); v_p.ptr += 32; } v_p.len = 0; } - v_v2 = vreinterpretq_u8_u32(vshlq_n_u32(vreinterpretq_u32_u8(v_v2), 5)); - v_v2 = vreinterpretq_u8_u32(vmlal_u16(vreinterpretq_u32_u8(v_v2), vreinterpret_u16_u8(vreinterpret_u8_u16(vget_low_u16(vreinterpretq_u16_u8(v_col0)))), vreinterpret_u16_u8(vreinterpret_u8_u16((uint16x4_t){32, 31, 30, 29})))); - v_v2 = vreinterpretq_u8_u32(vmlal_u16(vreinterpretq_u32_u8(v_v2), vreinterpret_u16_u8(vreinterpret_u8_u16(vget_high_u16(vreinterpretq_u16_u8(v_col0)))), vreinterpret_u16_u8(vreinterpret_u8_u16((uint16x4_t){28, 27, 26, 25})))); - v_v2 = vreinterpretq_u8_u32(vmlal_u16(vreinterpretq_u32_u8(v_v2), vreinterpret_u16_u8(vreinterpret_u8_u16(vget_low_u16(vreinterpretq_u16_u8(v_col1)))), vreinterpret_u16_u8(vreinterpret_u8_u16((uint16x4_t){24, 23, 22, 21})))); - v_v2 = vreinterpretq_u8_u32(vmlal_u16(vreinterpretq_u32_u8(v_v2), vreinterpret_u16_u8(vreinterpret_u8_u16(vget_high_u16(vreinterpretq_u16_u8(v_col1)))), vreinterpret_u16_u8(vreinterpret_u8_u16((uint16x4_t){20, 19, 18, 17})))); - v_v2 = vreinterpretq_u8_u32(vmlal_u16(vreinterpretq_u32_u8(v_v2), vreinterpret_u16_u8(vreinterpret_u8_u16(vget_low_u16(vreinterpretq_u16_u8(v_col2)))), vreinterpret_u16_u8(vreinterpret_u8_u16((uint16x4_t){16, 15, 14, 13})))); - v_v2 = vreinterpretq_u8_u32(vmlal_u16(vreinterpretq_u32_u8(v_v2), vreinterpret_u16_u8(vreinterpret_u8_u16(vget_high_u16(vreinterpretq_u16_u8(v_col2)))), vreinterpret_u16_u8(vreinterpret_u8_u16((uint16x4_t){12, 11, 10, 9})))); - v_v2 = vreinterpretq_u8_u32(vmlal_u16(vreinterpretq_u32_u8(v_v2), vreinterpret_u16_u8(vreinterpret_u8_u16(vget_low_u16(vreinterpretq_u16_u8(v_col3)))), vreinterpret_u16_u8(vreinterpret_u8_u16((uint16x4_t){8, 7, 6, 5})))); - v_v2 = vreinterpretq_u8_u32(vmlal_u16(vreinterpretq_u32_u8(v_v2), vreinterpret_u16_u8(vreinterpret_u8_u16(vget_high_u16(vreinterpretq_u16_u8(v_col3)))), vreinterpret_u16_u8(vreinterpret_u8_u16((uint16x4_t){4, 3, 2, 1})))); - v_sum1 = vreinterpret_u8_u32(vpadd_u32(vreinterpret_u32_u8(vreinterpret_u8_u32(vget_low_u32(vreinterpretq_u32_u8(v_v1)))), vreinterpret_u32_u8(vreinterpret_u8_u32(vget_high_u32(vreinterpretq_u32_u8(v_v1)))))); - v_sum2 = vreinterpret_u8_u32(vpadd_u32(vreinterpret_u32_u8(vreinterpret_u8_u32(vget_low_u32(vreinterpretq_u32_u8(v_v2)))), vreinterpret_u32_u8(vreinterpret_u8_u32(vget_high_u32(vreinterpretq_u32_u8(v_v2)))))); - v_sum12 = vreinterpret_u8_u32(vpadd_u32(vreinterpret_u32_u8(v_sum1), vreinterpret_u32_u8(v_sum2))); - v_s1 += vget_lane_u32(vreinterpret_u32_u8(v_sum12), 0); - v_s2 += vget_lane_u32(vreinterpret_u32_u8(v_sum12), 1); + v_v2 = vshlq_n_u32(v_v2, 5); + v_v2 = vmlal_u16(v_v2, vget_low_u16(v_col0), ((uint16x4_t){32, 31, 30, 29})); + v_v2 = vmlal_u16(v_v2, vget_high_u16(v_col0), ((uint16x4_t){28, 27, 26, 25})); + v_v2 = vmlal_u16(v_v2, vget_low_u16(v_col1), ((uint16x4_t){24, 23, 22, 21})); + v_v2 = vmlal_u16(v_v2, vget_high_u16(v_col1), ((uint16x4_t){20, 19, 18, 17})); + v_v2 = vmlal_u16(v_v2, vget_low_u16(v_col2), ((uint16x4_t){16, 15, 14, 13})); + v_v2 = vmlal_u16(v_v2, vget_high_u16(v_col2), ((uint16x4_t){12, 11, 10, 9})); + v_v2 = vmlal_u16(v_v2, vget_low_u16(v_col3), ((uint16x4_t){8, 7, 6, 5})); + v_v2 = vmlal_u16(v_v2, vget_high_u16(v_col3), ((uint16x4_t){4, 3, 2, 1})); + v_sum1 = vpadd_u32(vget_low_u32(v_v1), vget_high_u32(v_v1)); + v_sum2 = vpadd_u32(vget_low_u32(v_v2), vget_high_u32(v_v2)); + v_sum12 = vpadd_u32(v_sum1, v_sum2); + v_s1 += vget_lane_u32(v_sum12, 0); + v_s2 += vget_lane_u32(v_sum12, 1); v_tail_index = (((uint64_t)(a_x.len)) & 18446744073709551584u); if (v_tail_index < ((uint64_t)(a_x.len))) { { @@ -31732,13 +31732,13 @@ wuffs_png__decoder__filter_4_distance_3_arm_neon( uint8x8_t v_fb = {0}; uint8x8_t v_fc = {0}; uint8x8_t v_fx = {0}; - uint8x16_t v_fafb = {0}; - uint8x16_t v_fcfc = {0}; - uint8x16_t v_pa = {0}; - uint8x16_t v_pb = {0}; - uint8x16_t v_pc = {0}; - uint8x16_t v_cmpab = {0}; - uint8x16_t v_cmpac = {0}; + uint16x8_t v_fafb = {0}; + uint16x8_t v_fcfc = {0}; + uint16x8_t v_pa = {0}; + uint16x8_t v_pb = {0}; + uint16x8_t v_pc = {0}; + uint16x8_t v_cmpab = {0}; + uint16x8_t v_cmpac = {0}; uint8x8_t v_picka = {0}; uint8x8_t v_pickb = {0}; @@ -31754,15 +31754,15 @@ wuffs_png__decoder__filter_4_distance_3_arm_neon( while (v_curr.ptr < i_end0_curr) { v_fb = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u32le__no_bounds_check(v_prev.ptr))); v_fx = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u32le__no_bounds_check(v_curr.ptr))); - v_fafb = vreinterpretq_u8_u16(vaddl_u8(v_fa, v_fb)); - v_fcfc = vreinterpretq_u8_u16(vaddl_u8(v_fc, v_fc)); - v_pa = vreinterpretq_u8_u16(vabdl_u8(v_fb, v_fc)); - v_pb = vreinterpretq_u8_u16(vabdl_u8(v_fa, v_fc)); - v_pc = vreinterpretq_u8_u16(vabdq_u16(vreinterpretq_u16_u8(v_fafb), vreinterpretq_u16_u8(v_fcfc))); - v_cmpab = vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pa), vreinterpretq_u16_u8(v_pb))); - v_cmpac = vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pa), vreinterpretq_u16_u8(v_pc))); - v_picka = vmovn_u16(vreinterpretq_u16_u8(vandq_u8(v_cmpab, v_cmpac))); - v_pickb = vmovn_u16(vreinterpretq_u16_u8(vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pb), vreinterpretq_u16_u8(v_pc))))); + v_fafb = vaddl_u8(v_fa, v_fb); + v_fcfc = vaddl_u8(v_fc, v_fc); + v_pa = vabdl_u8(v_fb, v_fc); + v_pb = vabdl_u8(v_fa, v_fc); + v_pc = vabdq_u16(v_fafb, v_fcfc); + v_cmpab = vcleq_u16(v_pa, v_pb); + v_cmpac = vcleq_u16(v_pa, v_pc); + v_picka = vmovn_u16(vandq_u16(v_cmpab, v_cmpac)); + v_pickb = vmovn_u16(vcleq_u16(v_pb, v_pc)); v_fx = vadd_u8(v_fx, vbsl_u8(v_picka, v_fa, vbsl_u8(v_pickb, v_fb, v_fc))); wuffs_base__poke_u24le__no_bounds_check(v_curr.ptr, vget_lane_u32(vreinterpret_u32_u8(v_fx), 0)); v_fc = v_fb; @@ -31771,15 +31771,15 @@ wuffs_png__decoder__filter_4_distance_3_arm_neon( v_prev.ptr += 3; v_fb = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u32le__no_bounds_check(v_prev.ptr))); v_fx = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u32le__no_bounds_check(v_curr.ptr))); - v_fafb = vreinterpretq_u8_u16(vaddl_u8(v_fa, v_fb)); - v_fcfc = vreinterpretq_u8_u16(vaddl_u8(v_fc, v_fc)); - v_pa = vreinterpretq_u8_u16(vabdl_u8(v_fb, v_fc)); - v_pb = vreinterpretq_u8_u16(vabdl_u8(v_fa, v_fc)); - v_pc = vreinterpretq_u8_u16(vabdq_u16(vreinterpretq_u16_u8(v_fafb), vreinterpretq_u16_u8(v_fcfc))); - v_cmpab = vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pa), vreinterpretq_u16_u8(v_pb))); - v_cmpac = vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pa), vreinterpretq_u16_u8(v_pc))); - v_picka = vmovn_u16(vreinterpretq_u16_u8(vandq_u8(v_cmpab, v_cmpac))); - v_pickb = vmovn_u16(vreinterpretq_u16_u8(vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pb), vreinterpretq_u16_u8(v_pc))))); + v_fafb = vaddl_u8(v_fa, v_fb); + v_fcfc = vaddl_u8(v_fc, v_fc); + v_pa = vabdl_u8(v_fb, v_fc); + v_pb = vabdl_u8(v_fa, v_fc); + v_pc = vabdq_u16(v_fafb, v_fcfc); + v_cmpab = vcleq_u16(v_pa, v_pb); + v_cmpac = vcleq_u16(v_pa, v_pc); + v_picka = vmovn_u16(vandq_u16(v_cmpab, v_cmpac)); + v_pickb = vmovn_u16(vcleq_u16(v_pb, v_pc)); v_fx = vadd_u8(v_fx, vbsl_u8(v_picka, v_fa, vbsl_u8(v_pickb, v_fb, v_fc))); wuffs_base__poke_u24le__no_bounds_check(v_curr.ptr, vget_lane_u32(vreinterpret_u32_u8(v_fx), 0)); v_fc = v_fb; @@ -31793,15 +31793,15 @@ wuffs_png__decoder__filter_4_distance_3_arm_neon( while (v_curr.ptr < i_end1_curr) { v_fb = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u32le__no_bounds_check(v_prev.ptr))); v_fx = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u32le__no_bounds_check(v_curr.ptr))); - v_fafb = vreinterpretq_u8_u16(vaddl_u8(v_fa, v_fb)); - v_fcfc = vreinterpretq_u8_u16(vaddl_u8(v_fc, v_fc)); - v_pa = vreinterpretq_u8_u16(vabdl_u8(v_fb, v_fc)); - v_pb = vreinterpretq_u8_u16(vabdl_u8(v_fa, v_fc)); - v_pc = vreinterpretq_u8_u16(vabdq_u16(vreinterpretq_u16_u8(v_fafb), vreinterpretq_u16_u8(v_fcfc))); - v_cmpab = vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pa), vreinterpretq_u16_u8(v_pb))); - v_cmpac = vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pa), vreinterpretq_u16_u8(v_pc))); - v_picka = vmovn_u16(vreinterpretq_u16_u8(vandq_u8(v_cmpab, v_cmpac))); - v_pickb = vmovn_u16(vreinterpretq_u16_u8(vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pb), vreinterpretq_u16_u8(v_pc))))); + v_fafb = vaddl_u8(v_fa, v_fb); + v_fcfc = vaddl_u8(v_fc, v_fc); + v_pa = vabdl_u8(v_fb, v_fc); + v_pb = vabdl_u8(v_fa, v_fc); + v_pc = vabdq_u16(v_fafb, v_fcfc); + v_cmpab = vcleq_u16(v_pa, v_pb); + v_cmpac = vcleq_u16(v_pa, v_pc); + v_picka = vmovn_u16(vandq_u16(v_cmpab, v_cmpac)); + v_pickb = vmovn_u16(vcleq_u16(v_pb, v_pc)); v_fx = vadd_u8(v_fx, vbsl_u8(v_picka, v_fa, vbsl_u8(v_pickb, v_fb, v_fc))); wuffs_base__poke_u24le__no_bounds_check(v_curr.ptr, vget_lane_u32(vreinterpret_u32_u8(v_fx), 0)); v_fc = v_fb; @@ -31815,15 +31815,15 @@ wuffs_png__decoder__filter_4_distance_3_arm_neon( while (v_curr.ptr < i_end2_curr) { v_fb = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u24le__no_bounds_check(v_prev.ptr))); v_fx = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u24le__no_bounds_check(v_curr.ptr))); - v_fafb = vreinterpretq_u8_u16(vaddl_u8(v_fa, v_fb)); - v_fcfc = vreinterpretq_u8_u16(vaddl_u8(v_fc, v_fc)); - v_pa = vreinterpretq_u8_u16(vabdl_u8(v_fb, v_fc)); - v_pb = vreinterpretq_u8_u16(vabdl_u8(v_fa, v_fc)); - v_pc = vreinterpretq_u8_u16(vabdq_u16(vreinterpretq_u16_u8(v_fafb), vreinterpretq_u16_u8(v_fcfc))); - v_cmpab = vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pa), vreinterpretq_u16_u8(v_pb))); - v_cmpac = vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pa), vreinterpretq_u16_u8(v_pc))); - v_picka = vmovn_u16(vreinterpretq_u16_u8(vandq_u8(v_cmpab, v_cmpac))); - v_pickb = vmovn_u16(vreinterpretq_u16_u8(vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pb), vreinterpretq_u16_u8(v_pc))))); + v_fafb = vaddl_u8(v_fa, v_fb); + v_fcfc = vaddl_u8(v_fc, v_fc); + v_pa = vabdl_u8(v_fb, v_fc); + v_pb = vabdl_u8(v_fa, v_fc); + v_pc = vabdq_u16(v_fafb, v_fcfc); + v_cmpab = vcleq_u16(v_pa, v_pb); + v_cmpac = vcleq_u16(v_pa, v_pc); + v_picka = vmovn_u16(vandq_u16(v_cmpab, v_cmpac)); + v_pickb = vmovn_u16(vcleq_u16(v_pb, v_pc)); v_fx = vadd_u8(v_fx, vbsl_u8(v_picka, v_fa, vbsl_u8(v_pickb, v_fb, v_fc))); wuffs_base__poke_u24le__no_bounds_check(v_curr.ptr, vget_lane_u32(vreinterpret_u32_u8(v_fx), 0)); v_curr.ptr += 3; @@ -31850,13 +31850,13 @@ wuffs_png__decoder__filter_4_distance_4_arm_neon( uint8x8_t v_fb = {0}; uint8x8_t v_fc = {0}; uint8x8_t v_fx = {0}; - uint8x16_t v_fafb = {0}; - uint8x16_t v_fcfc = {0}; - uint8x16_t v_pa = {0}; - uint8x16_t v_pb = {0}; - uint8x16_t v_pc = {0}; - uint8x16_t v_cmpab = {0}; - uint8x16_t v_cmpac = {0}; + uint16x8_t v_fafb = {0}; + uint16x8_t v_fcfc = {0}; + uint16x8_t v_pa = {0}; + uint16x8_t v_pb = {0}; + uint16x8_t v_pc = {0}; + uint16x8_t v_cmpab = {0}; + uint16x8_t v_cmpac = {0}; uint8x8_t v_picka = {0}; uint8x8_t v_pickb = {0}; @@ -31872,15 +31872,15 @@ wuffs_png__decoder__filter_4_distance_4_arm_neon( while (v_curr.ptr < i_end0_curr) { v_fb = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u32le__no_bounds_check(v_prev.ptr))); v_fx = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u32le__no_bounds_check(v_curr.ptr))); - v_fafb = vreinterpretq_u8_u16(vaddl_u8(v_fa, v_fb)); - v_fcfc = vreinterpretq_u8_u16(vaddl_u8(v_fc, v_fc)); - v_pa = vreinterpretq_u8_u16(vabdl_u8(v_fb, v_fc)); - v_pb = vreinterpretq_u8_u16(vabdl_u8(v_fa, v_fc)); - v_pc = vreinterpretq_u8_u16(vabdq_u16(vreinterpretq_u16_u8(v_fafb), vreinterpretq_u16_u8(v_fcfc))); - v_cmpab = vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pa), vreinterpretq_u16_u8(v_pb))); - v_cmpac = vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pa), vreinterpretq_u16_u8(v_pc))); - v_picka = vmovn_u16(vreinterpretq_u16_u8(vandq_u8(v_cmpab, v_cmpac))); - v_pickb = vmovn_u16(vreinterpretq_u16_u8(vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pb), vreinterpretq_u16_u8(v_pc))))); + v_fafb = vaddl_u8(v_fa, v_fb); + v_fcfc = vaddl_u8(v_fc, v_fc); + v_pa = vabdl_u8(v_fb, v_fc); + v_pb = vabdl_u8(v_fa, v_fc); + v_pc = vabdq_u16(v_fafb, v_fcfc); + v_cmpab = vcleq_u16(v_pa, v_pb); + v_cmpac = vcleq_u16(v_pa, v_pc); + v_picka = vmovn_u16(vandq_u16(v_cmpab, v_cmpac)); + v_pickb = vmovn_u16(vcleq_u16(v_pb, v_pc)); v_fx = vadd_u8(v_fx, vbsl_u8(v_picka, v_fa, vbsl_u8(v_pickb, v_fb, v_fc))); wuffs_base__poke_u32le__no_bounds_check(v_curr.ptr, vget_lane_u32(vreinterpret_u32_u8(v_fx), 0)); v_fc = v_fb; @@ -31889,15 +31889,15 @@ wuffs_png__decoder__filter_4_distance_4_arm_neon( v_prev.ptr += 4; v_fb = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u32le__no_bounds_check(v_prev.ptr))); v_fx = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u32le__no_bounds_check(v_curr.ptr))); - v_fafb = vreinterpretq_u8_u16(vaddl_u8(v_fa, v_fb)); - v_fcfc = vreinterpretq_u8_u16(vaddl_u8(v_fc, v_fc)); - v_pa = vreinterpretq_u8_u16(vabdl_u8(v_fb, v_fc)); - v_pb = vreinterpretq_u8_u16(vabdl_u8(v_fa, v_fc)); - v_pc = vreinterpretq_u8_u16(vabdq_u16(vreinterpretq_u16_u8(v_fafb), vreinterpretq_u16_u8(v_fcfc))); - v_cmpab = vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pa), vreinterpretq_u16_u8(v_pb))); - v_cmpac = vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pa), vreinterpretq_u16_u8(v_pc))); - v_picka = vmovn_u16(vreinterpretq_u16_u8(vandq_u8(v_cmpab, v_cmpac))); - v_pickb = vmovn_u16(vreinterpretq_u16_u8(vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pb), vreinterpretq_u16_u8(v_pc))))); + v_fafb = vaddl_u8(v_fa, v_fb); + v_fcfc = vaddl_u8(v_fc, v_fc); + v_pa = vabdl_u8(v_fb, v_fc); + v_pb = vabdl_u8(v_fa, v_fc); + v_pc = vabdq_u16(v_fafb, v_fcfc); + v_cmpab = vcleq_u16(v_pa, v_pb); + v_cmpac = vcleq_u16(v_pa, v_pc); + v_picka = vmovn_u16(vandq_u16(v_cmpab, v_cmpac)); + v_pickb = vmovn_u16(vcleq_u16(v_pb, v_pc)); v_fx = vadd_u8(v_fx, vbsl_u8(v_picka, v_fa, vbsl_u8(v_pickb, v_fb, v_fc))); wuffs_base__poke_u32le__no_bounds_check(v_curr.ptr, vget_lane_u32(vreinterpret_u32_u8(v_fx), 0)); v_fc = v_fb; @@ -31911,15 +31911,15 @@ wuffs_png__decoder__filter_4_distance_4_arm_neon( while (v_curr.ptr < i_end1_curr) { v_fb = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u32le__no_bounds_check(v_prev.ptr))); v_fx = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u32le__no_bounds_check(v_curr.ptr))); - v_fafb = vreinterpretq_u8_u16(vaddl_u8(v_fa, v_fb)); - v_fcfc = vreinterpretq_u8_u16(vaddl_u8(v_fc, v_fc)); - v_pa = vreinterpretq_u8_u16(vabdl_u8(v_fb, v_fc)); - v_pb = vreinterpretq_u8_u16(vabdl_u8(v_fa, v_fc)); - v_pc = vreinterpretq_u8_u16(vabdq_u16(vreinterpretq_u16_u8(v_fafb), vreinterpretq_u16_u8(v_fcfc))); - v_cmpab = vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pa), vreinterpretq_u16_u8(v_pb))); - v_cmpac = vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pa), vreinterpretq_u16_u8(v_pc))); - v_picka = vmovn_u16(vreinterpretq_u16_u8(vandq_u8(v_cmpab, v_cmpac))); - v_pickb = vmovn_u16(vreinterpretq_u16_u8(vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pb), vreinterpretq_u16_u8(v_pc))))); + v_fafb = vaddl_u8(v_fa, v_fb); + v_fcfc = vaddl_u8(v_fc, v_fc); + v_pa = vabdl_u8(v_fb, v_fc); + v_pb = vabdl_u8(v_fa, v_fc); + v_pc = vabdq_u16(v_fafb, v_fcfc); + v_cmpab = vcleq_u16(v_pa, v_pb); + v_cmpac = vcleq_u16(v_pa, v_pc); + v_picka = vmovn_u16(vandq_u16(v_cmpab, v_cmpac)); + v_pickb = vmovn_u16(vcleq_u16(v_pb, v_pc)); v_fx = vadd_u8(v_fx, vbsl_u8(v_picka, v_fa, vbsl_u8(v_pickb, v_fb, v_fc))); wuffs_base__poke_u32le__no_bounds_check(v_curr.ptr, vget_lane_u32(vreinterpret_u32_u8(v_fx), 0)); v_fc = v_fb; diff --git a/std/adler32/common_up_arm_neon.wuffs b/std/adler32/common_up_arm_neon.wuffs index 821ae0bf..47d7eabb 100644 --- a/std/adler32/common_up_arm_neon.wuffs +++ b/std/adler32/common_up_arm_neon.wuffs @@ -23,17 +23,18 @@ pri func hasher.up_arm_neon!(x: slice base.u8), // The remaining variables are specific to the SIMD version. - var p__left : base.arm_neon_128 - var p_right : base.arm_neon_128 - var v1 : base.arm_neon_128 - var v2 : base.arm_neon_128 - var col0 : base.arm_neon_128 - var col1 : base.arm_neon_128 - var col2 : base.arm_neon_128 - var col3 : base.arm_neon_128 - var sum1 : base.arm_neon_64 - var sum2 : base.arm_neon_64 - var sum12 : base.arm_neon_64 + var util : base.arm_neon_utility + var p__left : base.arm_neon_u8x16 + var p_right : base.arm_neon_u8x16 + var v1 : base.arm_neon_u32x4 + var v2 : base.arm_neon_u32x4 + var col0 : base.arm_neon_u16x8 + var col1 : base.arm_neon_u16x8 + var col2 : base.arm_neon_u16x8 + var col3 : base.arm_neon_u16x8 + var sum1 : base.arm_neon_u32x2 + var sum2 : base.arm_neon_u32x2 + var sum12 : base.arm_neon_u32x2 var num_iterate_bytes : base.u32 var tail_index : base.u64 @@ -79,12 +80,12 @@ pri func hasher.up_arm_neon!(x: slice base.u8), // variables s1 and s2. The iterate loop accumulates four parallel u32 // sums in each vector. A post-iterate step merges the four u32 sums // into a single u32 sum. - v1 = v1.create_vdupq_n_u32(a: 0) - v2 = v2.create_vdupq_n_u32(a: 0) - col0 = col0.create_vdupq_n_u16(a: 0) - col1 = col1.create_vdupq_n_u16(a: 0) - col2 = col2.create_vdupq_n_u16(a: 0) - col3 = col3.create_vdupq_n_u16(a: 0) + v1 = util.make_u32x4_repeat(a: 0) + v2 = util.make_u32x4_repeat(a: 0) + col0 = util.make_u16x8_repeat(a: 0) + col1 = util.make_u16x8_repeat(a: 0) + col2 = util.make_u16x8_repeat(a: 0) + col3 = util.make_u16x8_repeat(a: 0) // The inner loop. iterate (p = args.x)(length: 32, advance: 32, unroll: 1) { @@ -93,8 +94,8 @@ pri func hasher.up_arm_neon!(x: slice base.u8), // // Let p__left = [u8×16: p00, p01, p02, ..., p15] // Let p_right = [u8×16: p16, p17, p18, ..., p31] - p__left = p__left.create_slice128(a: p[.. 16]) - p_right = p_right.create_slice128(a: p[16 .. 32]) + p__left = util.make_u8x16_slice128(a: p[.. 16]) + p_right = util.make_u8x16_slice128(a: p[16 .. 32]) // For v2j, we need to calculate the sums of the s1j terms for each // of p's 32 elements. This is simply 32 times the same number, @@ -141,28 +142,28 @@ pri func hasher.up_arm_neon!(x: slice base.u8), // [u32×4: 0x04 * (Σ p28), 0x03 * (Σ p29), ..., 0x01 * (Σ p31)] v2 = v2.vmlal_u16( b: col0.vget_low_u16(), - c: sum2.create_literal_u16x4(a0: 0x20, a1: 0x1F, a2: 0x1E, a3: 0x1D)) + c: util.make_u16x4_multiple(a00: 0x20, a01: 0x1F, a02: 0x1E, a03: 0x1D)) v2 = v2.vmlal_u16( b: col0.vget_high_u16(), - c: sum2.create_literal_u16x4(a0: 0x1C, a1: 0x1B, a2: 0x1A, a3: 0x19)) + c: util.make_u16x4_multiple(a00: 0x1C, a01: 0x1B, a02: 0x1A, a03: 0x19)) v2 = v2.vmlal_u16( b: col1.vget_low_u16(), - c: sum2.create_literal_u16x4(a0: 0x18, a1: 0x17, a2: 0x16, a3: 0x15)) + c: util.make_u16x4_multiple(a00: 0x18, a01: 0x17, a02: 0x16, a03: 0x15)) v2 = v2.vmlal_u16( b: col1.vget_high_u16(), - c: sum2.create_literal_u16x4(a0: 0x14, a1: 0x13, a2: 0x12, a3: 0x11)) + c: util.make_u16x4_multiple(a00: 0x14, a01: 0x13, a02: 0x12, a03: 0x11)) v2 = v2.vmlal_u16( b: col2.vget_low_u16(), - c: sum2.create_literal_u16x4(a0: 0x10, a1: 0x0F, a2: 0x0E, a3: 0x0D)) + c: util.make_u16x4_multiple(a00: 0x10, a01: 0x0F, a02: 0x0E, a03: 0x0D)) v2 = v2.vmlal_u16( b: col2.vget_high_u16(), - c: sum2.create_literal_u16x4(a0: 0x0C, a1: 0x0B, a2: 0x0A, a3: 0x09)) + c: util.make_u16x4_multiple(a00: 0x0C, a01: 0x0B, a02: 0x0A, a03: 0x09)) v2 = v2.vmlal_u16( b: col3.vget_low_u16(), - c: sum2.create_literal_u16x4(a0: 0x08, a1: 0x07, a2: 0x06, a3: 0x05)) + c: util.make_u16x4_multiple(a00: 0x08, a01: 0x07, a02: 0x06, a03: 0x05)) v2 = v2.vmlal_u16( b: col3.vget_high_u16(), - c: sum2.create_literal_u16x4(a0: 0x04, a1: 0x03, a2: 0x02, a3: 0x01)) + c: util.make_u16x4_multiple(a00: 0x04, a01: 0x03, a02: 0x02, a03: 0x01)) // Merge the four parallel u32 sums (v1) into the single u32 sum (s1) // and ditto for v2. Starting with [u32×4: xx_0, xx_1, xx_2, xx_3]: diff --git a/std/png/decode_filter_arm_neon.wuffs b/std/png/decode_filter_arm_neon.wuffs index 08858396..484832a4 100644 --- a/std/png/decode_filter_arm_neon.wuffs +++ b/std/png/decode_filter_arm_neon.wuffs @@ -55,24 +55,25 @@ pri func decoder.filter_3_distance_4_arm_neon!(curr: slice base.u8, prev: slice var curr : slice base.u8 var prev : slice base.u8 - var fa : base.arm_neon_64 - var fb : base.arm_neon_64 - var fx : base.arm_neon_64 + var util : base.arm_neon_utility + var fa : base.arm_neon_u8x8 + var fb : base.arm_neon_u8x8 + var fx : base.arm_neon_u8x8 if args.prev.length() == 0 { iterate (curr = args.curr)(length: 4, advance: 4, unroll: 2) { - fx = fx.create_vdup_n_u32(a: curr.peek_u32le()) + fx = util.make_u32x2_repeat(a: curr.peek_u32le()).as_u8x8() fx = fx.vadd_u8(b: fa.vhadd_u8(b: fb)) - curr.poke_u32le!(a: fx.vget_lane_u32(b: 0)) + curr.poke_u32le!(a: fx.as_u32x2().vget_lane_u32(b: 0)) fa = fx } } else { iterate (curr = args.curr, prev = args.prev)(length: 4, advance: 4, unroll: 2) { - fb = fb.create_vdup_n_u32(a: prev.peek_u32le()) - fx = fx.create_vdup_n_u32(a: curr.peek_u32le()) + fb = util.make_u32x2_repeat(a: prev.peek_u32le()).as_u8x8() + fx = util.make_u32x2_repeat(a: curr.peek_u32le()).as_u8x8() fx = fx.vadd_u8(b: fa.vhadd_u8(b: fb)) - curr.poke_u32le!(a: fx.vget_lane_u32(b: 0)) + curr.poke_u32le!(a: fx.as_u32x2().vget_lane_u32(b: 0)) fa = fx } } @@ -95,24 +96,25 @@ pri func decoder.filter_4_distance_3_arm_neon!(curr: slice base.u8, prev: slice var curr : slice base.u8 var prev : slice base.u8 - var fa : base.arm_neon_64 - var fb : base.arm_neon_64 - var fc : base.arm_neon_64 - var fx : base.arm_neon_64 - var fafb : base.arm_neon_128 - var fcfc : base.arm_neon_128 - var pa : base.arm_neon_128 - var pb : base.arm_neon_128 - var pc : base.arm_neon_128 - var cmpab : base.arm_neon_128 - var cmpac : base.arm_neon_128 - var picka : base.arm_neon_64 - var pickb : base.arm_neon_64 + var util : base.arm_neon_utility + var fa : base.arm_neon_u8x8 + var fb : base.arm_neon_u8x8 + var fc : base.arm_neon_u8x8 + var fx : base.arm_neon_u8x8 + var fafb : base.arm_neon_u16x8 + var fcfc : base.arm_neon_u16x8 + var pa : base.arm_neon_u16x8 + var pb : base.arm_neon_u16x8 + var pc : base.arm_neon_u16x8 + var cmpab : base.arm_neon_u16x8 + var cmpac : base.arm_neon_u16x8 + var picka : base.arm_neon_u8x8 + var pickb : base.arm_neon_u8x8 // § The advance is 3, not 4. iterate (curr = args.curr, prev = args.prev)(length: 4, advance: 3, unroll: 2) { - fb = fb.create_vdup_n_u32(a: prev.peek_u32le()) - fx = fx.create_vdup_n_u32(a: curr.peek_u32le()) + fb = util.make_u32x2_repeat(a: prev.peek_u32le()).as_u8x8() + fx = util.make_u32x2_repeat(a: curr.peek_u32le()).as_u8x8() fafb = fa.vaddl_u8(b: fb) fcfc = fc.vaddl_u8(b: fc) pa = fb.vabdl_u8(b: fc) @@ -120,20 +122,20 @@ pri func decoder.filter_4_distance_3_arm_neon!(curr: slice base.u8, prev: slice pc = fafb.vabdq_u16(b: fcfc) cmpab = pa.vcleq_u16(b: pb) cmpac = pa.vcleq_u16(b: pc) - picka = cmpab.vandq_u8(b: cmpac).vmovn_u16() + picka = cmpab.vandq_u16(b: cmpac).vmovn_u16() pickb = pb.vcleq_u16(b: pc).vmovn_u16() fx = fx.vadd_u8( b: picka.vbsl_u8(b: fa, c: pickb.vbsl_u8(b: fb, c: fc))) // § poke_u24le replaces poke_u32le. - curr.poke_u24le!(a: fx.vget_lane_u32(b: 0)) + curr.poke_u24le!(a: fx.as_u32x2().vget_lane_u32(b: 0)) fc = fb fa = fx } else (length: 3, advance: 3, unroll: 1) { // § peek_u24le_as_u32 replaces peek_u32le. - fb = fb.create_vdup_n_u32(a: prev.peek_u24le_as_u32()) + fb = util.make_u32x2_repeat(a: prev.peek_u24le_as_u32()).as_u8x8() // § peek_u24le_as_u32 replaces peek_u32le. - fx = fx.create_vdup_n_u32(a: curr.peek_u24le_as_u32()) + fx = util.make_u32x2_repeat(a: curr.peek_u24le_as_u32()).as_u8x8() fafb = fa.vaddl_u8(b: fb) fcfc = fc.vaddl_u8(b: fc) pa = fb.vabdl_u8(b: fc) @@ -141,13 +143,13 @@ pri func decoder.filter_4_distance_3_arm_neon!(curr: slice base.u8, prev: slice pc = fafb.vabdq_u16(b: fcfc) cmpab = pa.vcleq_u16(b: pb) cmpac = pa.vcleq_u16(b: pc) - picka = cmpab.vandq_u8(b: cmpac).vmovn_u16() + picka = cmpab.vandq_u16(b: cmpac).vmovn_u16() pickb = pb.vcleq_u16(b: pc).vmovn_u16() fx = fx.vadd_u8( b: picka.vbsl_u8(b: fa, c: pickb.vbsl_u8(b: fb, c: fc))) // § poke_u24le replaces poke_u32le. - curr.poke_u24le!(a: fx.vget_lane_u32(b: 0)) + curr.poke_u24le!(a: fx.as_u32x2().vget_lane_u32(b: 0)) // § These assignments are unnecessary; this is the last iteration. // fc = fb // fa = fx @@ -160,23 +162,24 @@ pri func decoder.filter_4_distance_4_arm_neon!(curr: slice base.u8, prev: slice var curr : slice base.u8 var prev : slice base.u8 - var fa : base.arm_neon_64 - var fb : base.arm_neon_64 - var fc : base.arm_neon_64 - var fx : base.arm_neon_64 - var fafb : base.arm_neon_128 - var fcfc : base.arm_neon_128 - var pa : base.arm_neon_128 - var pb : base.arm_neon_128 - var pc : base.arm_neon_128 - var cmpab : base.arm_neon_128 - var cmpac : base.arm_neon_128 - var picka : base.arm_neon_64 - var pickb : base.arm_neon_64 + var util : base.arm_neon_utility + var fa : base.arm_neon_u8x8 + var fb : base.arm_neon_u8x8 + var fc : base.arm_neon_u8x8 + var fx : base.arm_neon_u8x8 + var fafb : base.arm_neon_u16x8 + var fcfc : base.arm_neon_u16x8 + var pa : base.arm_neon_u16x8 + var pb : base.arm_neon_u16x8 + var pc : base.arm_neon_u16x8 + var cmpab : base.arm_neon_u16x8 + var cmpac : base.arm_neon_u16x8 + var picka : base.arm_neon_u8x8 + var pickb : base.arm_neon_u8x8 iterate (curr = args.curr, prev = args.prev)(length: 4, advance: 4, unroll: 2) { - fb = fb.create_vdup_n_u32(a: prev.peek_u32le()) - fx = fx.create_vdup_n_u32(a: curr.peek_u32le()) + fb = util.make_u32x2_repeat(a: prev.peek_u32le()).as_u8x8() + fx = util.make_u32x2_repeat(a: curr.peek_u32le()).as_u8x8() fafb = fa.vaddl_u8(b: fb) // fafb = (fa + fb) fcfc = fc.vaddl_u8(b: fc) // fcfc = (fc + fc) @@ -188,15 +191,15 @@ pri func decoder.filter_4_distance_4_arm_neon!(curr: slice base.u8, prev: slice cmpab = pa.vcleq_u16(b: pb) // cmpab = (pa <= pb) cmpac = pa.vcleq_u16(b: pc) // cmpac = (pa <= pc) - picka = cmpab.vandq_u8(b: cmpac).vmovn_u16() // picka = ((pa <= pb) && (pa <= pc)) - pickb = pb.vcleq_u16(b: pc).vmovn_u16() // pickb = (pb <= pc) + picka = cmpab.vandq_u16(b: cmpac).vmovn_u16() // picka = ((pa <= pb) && (pa <= pc)) + pickb = pb.vcleq_u16(b: pc).vmovn_u16() // pickb = (pb <= pc) // Add the predictor to the residual. fx = fx.vadd_u8( b: picka.vbsl_u8(b: fa, c: pickb.vbsl_u8(b: fb, c: fc))) - curr.poke_u32le!(a: fx.vget_lane_u32(b: 0)) + curr.poke_u32le!(a: fx.as_u32x2().vget_lane_u32(b: 0)) fc = fb fa = fx }