Use arm_neon_uAxB types and methods

This commit is contained in:
Nigel Tao committed 2021-02-24 16:48:09 +11:00
1 parent 1f9b6ea535
commit 3b5d8fdfba
5 files changed
+279 -204

No files matched your search

+56 -16
View File
@@ -775,29 +775,69 @@ func (g *gen) writeBuiltinCPUArch(b *buffer, recv *a.Expr, method t.ID, args []*
func (g *gen) writeBuiltinCPUArchARMNeon(b *buffer, recv *a.Expr, method t.ID, args []*a.Node, sideEffectsOnly bool, depth uint32) error {
methodStr := method.Str(g.tm)
if strings.HasPrefix(methodStr, "make_") {
fName := ""
switch methodStr {
case "make_u8x8_repeat":
fName = "vdup_n_u8"
case "make_u16x4_repeat":
fName = "vdup_n_u16"
case "make_u32x2_repeat":
fName = "vdup_n_u32"
case "make_u64x1_repeat":
fName = "vdup_n_u64"
default:
return fmt.Errorf("internal error: unsupported cpu_arch method %q", methodStr)
before, after, ptr := "", ")", false
if strings.HasSuffix(methodStr, "_multiple") {
after = "})"
switch methodStr {
case "make_u8x8_multiple":
before = "((uint8x8_t){"
case "make_u16x4_multiple":
before = "((uint16x4_t){"
case "make_u32x2_multiple":
before = "((uint32x2_t){"
case "make_u64x1_multiple":
before = "((uint64x1_t){"
case "make_u8x16_multiple":
before = "((uint8x16_t){"
case "make_u16x8_multiple":
before = "((uint16x8_t){"
case "make_u32x4_multiple":
before = "((uint32x4_t){"
case "make_u64x2_multiple":
before = "((uint64x2_t){"
}
} else {
switch methodStr {
case "make_u8x8_repeat":
before = "vdup_n_u8("
case "make_u16x4_repeat":
before = "vdup_n_u16("
case "make_u32x2_repeat":
before = "vdup_n_u32("
case "make_u64x1_repeat":
before = "vdup_n_u64("
case "make_u8x16_repeat":
before = "vdupq_n_u8("
case "make_u16x8_repeat":
before = "vdupq_n_u16("
case "make_u32x4_repeat":
before = "vdupq_n_u32("
case "make_u64x2_repeat":
before = "vdupq_n_u64("
case "make_u8x8_slice64":
before, ptr = "vld1_u8(", true
case "make_u8x16_slice128":
before, ptr = "vld1q_u8(", true
default:
return fmt.Errorf("internal error: unsupported cpu_arch method %q", methodStr)
}
}
b.printf("%s(", fName)
b.writes(before)
for i, o := range args {
if i > 0 {
b.writes(", ")
}
if err := g.writeExpr(b, o.AsArg().Value(), false, depth); err != nil {
return err
if ptr {
if err := g.writeExprDotPtr(b, o.AsArg().Value(), false, depth); err != nil {
return err
}
} else {
if err := g.writeExpr(b, o.AsArg().Value(), false, depth); err != nil {
return err
}
}
}
b.writes(")")
b.writes(after)
return nil
} else if strings.HasPrefix(methodStr, "as_") {
+31
View File
@@ -568,11 +568,42 @@ var funcsOther = [...]string{
// ---- arm_neon_utility
"arm_neon_utility.make_u8x8_multiple(" +
"a00: u8, a01: u8, a02: u8, a03: u8," +
"a04: u8, a05: u8, a06: u8, a07: u8) arm_neon_u8x8",
"arm_neon_utility.make_u16x4_multiple(" +
"a00: u16, a01: u16, a02: u16, a03: u16) arm_neon_u16x4",
"arm_neon_utility.make_u32x2_multiple(" +
"a00: u32, a01: u32) arm_neon_u32x2",
"arm_neon_utility.make_u64x1_multiple(" +
"a00: u64) arm_neon_u64x1",
"arm_neon_utility.make_u8x16_multiple(" +
"a00: u8, a01: u8, a02: u8, a03: u8," +
"a04: u8, a05: u8, a06: u8, a07: u8," +
"a08: u8, a09: u8, a10: u8, a11: u8," +
"a12: u8, a13: u8, a14: u8, a15: u8) arm_neon_u8x16",
"arm_neon_utility.make_u16x8_multiple(" +
"a00: u16, a01: u16, a02: u16, a03: u16," +
"a04: u16, a05: u16, a06: u16, a07: u16) arm_neon_u16x8",
"arm_neon_utility.make_u32x4_multiple(" +
"a00: u32, a01: u32, a02: u32, a03: u32) arm_neon_u32x4",
"arm_neon_utility.make_u64x2_multiple(" +
"a00: u64, a01: u64) arm_neon_u64x2",
"arm_neon_utility.make_u8x8_repeat(a: u8) arm_neon_u8x8",
"arm_neon_utility.make_u16x4_repeat(a: u16) arm_neon_u16x4",
"arm_neon_utility.make_u32x2_repeat(a: u32) arm_neon_u32x2",
"arm_neon_utility.make_u64x1_repeat(a: u64) arm_neon_u64x1",
"arm_neon_utility.make_u8x16_repeat(a: u8) arm_neon_u8x16",
"arm_neon_utility.make_u16x8_repeat(a: u16) arm_neon_u16x8",
"arm_neon_utility.make_u32x4_repeat(a: u32) arm_neon_u32x4",
"arm_neon_utility.make_u64x2_repeat(a: u64) arm_neon_u64x2",
"arm_neon_utility.make_u8x8_slice64(a: slice base.u8) arm_neon_u8x8",
"arm_neon_utility.make_u8x16_slice128(a: slice base.u8) arm_neon_u8x16",
// ---- arm_neon_uAxB.as_uCxD
"arm_neon_u8x8.as_u16x4() arm_neon_u16x4",
+114 -114
View File
@@ -17835,15 +17835,15 @@ wuffs_adler32__hasher__up_arm_neon(
wuffs_base__slice_u8 v_p = {0};
uint8x16_t v_p__left = {0};
uint8x16_t v_p_right = {0};
uint8x16_t v_v1 = {0};
uint8x16_t v_v2 = {0};
uint8x16_t v_col0 = {0};
uint8x16_t v_col1 = {0};
uint8x16_t v_col2 = {0};
uint8x16_t v_col3 = {0};
uint8x8_t v_sum1 = {0};
uint8x8_t v_sum2 = {0};
uint8x8_t v_sum12 = {0};
uint32x4_t v_v1 = {0};
uint32x4_t v_v2 = {0};
uint16x8_t v_col0 = {0};
uint16x8_t v_col1 = {0};
uint16x8_t v_col2 = {0};
uint16x8_t v_col3 = {0};
uint32x2_t v_sum1 = {0};
uint32x2_t v_sum2 = {0};
uint32x2_t v_sum12 = {0};
uint32_t v_num_iterate_bytes = 0;
uint64_t v_tail_index = 0;
@@ -17864,44 +17864,44 @@ wuffs_adler32__hasher__up_arm_neon(
}
v_num_iterate_bytes = ((uint32_t)((((uint64_t)(a_x.len)) & 4294967264)));
v_s2 += ((uint32_t)(v_s1 * v_num_iterate_bytes));
v_v1 = vreinterpretq_u8_u32(vdupq_n_u32(0));
v_v2 = vreinterpretq_u8_u32(vdupq_n_u32(0));
v_col0 = vreinterpretq_u8_u16(vdupq_n_u16(0));
v_col1 = vreinterpretq_u8_u16(vdupq_n_u16(0));
v_col2 = vreinterpretq_u8_u16(vdupq_n_u16(0));
v_col3 = vreinterpretq_u8_u16(vdupq_n_u16(0));
v_v1 = vdupq_n_u32(0);
v_v2 = vdupq_n_u32(0);
v_col0 = vdupq_n_u16(0);
v_col1 = vdupq_n_u16(0);
v_col2 = vdupq_n_u16(0);
v_col3 = vdupq_n_u16(0);
{
wuffs_base__slice_u8 i_slice_p = a_x;
v_p.ptr = i_slice_p.ptr;
v_p.len = 32;
uint8_t* i_end0_p = v_p.ptr + (((i_slice_p.len - (size_t)(v_p.ptr - i_slice_p.ptr)) / 32) * 32);
while (v_p.ptr < i_end0_p) {
v_p__left = vld1q_u8(wuffs_base__slice_u8__subslice_j(v_p, 16).ptr);
v_p_right = vld1q_u8(wuffs_base__slice_u8__subslice_ij(v_p, 16, 32).ptr);
v_v2 = vreinterpretq_u8_u32(vaddq_u32(vreinterpretq_u32_u8(v_v2), vreinterpretq_u32_u8(v_v1)));
v_v1 = vreinterpretq_u8_u32(vpadalq_u16(vreinterpretq_u32_u8(v_v1), vreinterpretq_u16_u8(vreinterpretq_u8_u16(vpadalq_u8(vreinterpretq_u16_u8(vreinterpretq_u8_u16(vpaddlq_u8(v_p__left))), v_p_right)))));
v_col0 = vreinterpretq_u8_u16(vaddw_u8(vreinterpretq_u16_u8(v_col0), vget_low_u8(v_p__left)));
v_col1 = vreinterpretq_u8_u16(vaddw_u8(vreinterpretq_u16_u8(v_col1), vget_high_u8(v_p__left)));
v_col2 = vreinterpretq_u8_u16(vaddw_u8(vreinterpretq_u16_u8(v_col2), vget_low_u8(v_p_right)));
v_col3 = vreinterpretq_u8_u16(vaddw_u8(vreinterpretq_u16_u8(v_col3), vget_high_u8(v_p_right)));
v_p__left = vld1q_u8(v_p.ptr);
v_p_right = vld1q_u8(v_p.ptr + 16);
v_v2 = vaddq_u32(v_v2, v_v1);
v_v1 = vpadalq_u16(v_v1, vpadalq_u8(vpaddlq_u8(v_p__left), v_p_right));
v_col0 = vaddw_u8(v_col0, vget_low_u8(v_p__left));
v_col1 = vaddw_u8(v_col1, vget_high_u8(v_p__left));
v_col2 = vaddw_u8(v_col2, vget_low_u8(v_p_right));
v_col3 = vaddw_u8(v_col3, vget_high_u8(v_p_right));
v_p.ptr += 32;
}
v_p.len = 0;
}
v_v2 = vreinterpretq_u8_u32(vshlq_n_u32(vreinterpretq_u32_u8(v_v2), 5));
v_v2 = vreinterpretq_u8_u32(vmlal_u16(vreinterpretq_u32_u8(v_v2), vreinterpret_u16_u8(vreinterpret_u8_u16(vget_low_u16(vreinterpretq_u16_u8(v_col0)))), vreinterpret_u16_u8(vreinterpret_u8_u16((uint16x4_t){32, 31, 30, 29}))));
v_v2 = vreinterpretq_u8_u32(vmlal_u16(vreinterpretq_u32_u8(v_v2), vreinterpret_u16_u8(vreinterpret_u8_u16(vget_high_u16(vreinterpretq_u16_u8(v_col0)))), vreinterpret_u16_u8(vreinterpret_u8_u16((uint16x4_t){28, 27, 26, 25}))));
v_v2 = vreinterpretq_u8_u32(vmlal_u16(vreinterpretq_u32_u8(v_v2), vreinterpret_u16_u8(vreinterpret_u8_u16(vget_low_u16(vreinterpretq_u16_u8(v_col1)))), vreinterpret_u16_u8(vreinterpret_u8_u16((uint16x4_t){24, 23, 22, 21}))));
v_v2 = vreinterpretq_u8_u32(vmlal_u16(vreinterpretq_u32_u8(v_v2), vreinterpret_u16_u8(vreinterpret_u8_u16(vget_high_u16(vreinterpretq_u16_u8(v_col1)))), vreinterpret_u16_u8(vreinterpret_u8_u16((uint16x4_t){20, 19, 18, 17}))));
v_v2 = vreinterpretq_u8_u32(vmlal_u16(vreinterpretq_u32_u8(v_v2), vreinterpret_u16_u8(vreinterpret_u8_u16(vget_low_u16(vreinterpretq_u16_u8(v_col2)))), vreinterpret_u16_u8(vreinterpret_u8_u16((uint16x4_t){16, 15, 14, 13}))));
v_v2 = vreinterpretq_u8_u32(vmlal_u16(vreinterpretq_u32_u8(v_v2), vreinterpret_u16_u8(vreinterpret_u8_u16(vget_high_u16(vreinterpretq_u16_u8(v_col2)))), vreinterpret_u16_u8(vreinterpret_u8_u16((uint16x4_t){12, 11, 10, 9}))));
v_v2 = vreinterpretq_u8_u32(vmlal_u16(vreinterpretq_u32_u8(v_v2), vreinterpret_u16_u8(vreinterpret_u8_u16(vget_low_u16(vreinterpretq_u16_u8(v_col3)))), vreinterpret_u16_u8(vreinterpret_u8_u16((uint16x4_t){8, 7, 6, 5}))));
v_v2 = vreinterpretq_u8_u32(vmlal_u16(vreinterpretq_u32_u8(v_v2), vreinterpret_u16_u8(vreinterpret_u8_u16(vget_high_u16(vreinterpretq_u16_u8(v_col3)))), vreinterpret_u16_u8(vreinterpret_u8_u16((uint16x4_t){4, 3, 2, 1}))));
v_sum1 = vreinterpret_u8_u32(vpadd_u32(vreinterpret_u32_u8(vreinterpret_u8_u32(vget_low_u32(vreinterpretq_u32_u8(v_v1)))), vreinterpret_u32_u8(vreinterpret_u8_u32(vget_high_u32(vreinterpretq_u32_u8(v_v1))))));
v_sum2 = vreinterpret_u8_u32(vpadd_u32(vreinterpret_u32_u8(vreinterpret_u8_u32(vget_low_u32(vreinterpretq_u32_u8(v_v2)))), vreinterpret_u32_u8(vreinterpret_u8_u32(vget_high_u32(vreinterpretq_u32_u8(v_v2))))));
v_sum12 = vreinterpret_u8_u32(vpadd_u32(vreinterpret_u32_u8(v_sum1), vreinterpret_u32_u8(v_sum2)));
v_s1 += vget_lane_u32(vreinterpret_u32_u8(v_sum12), 0);
v_s2 += vget_lane_u32(vreinterpret_u32_u8(v_sum12), 1);
v_v2 = vshlq_n_u32(v_v2, 5);
v_v2 = vmlal_u16(v_v2, vget_low_u16(v_col0), ((uint16x4_t){32, 31, 30, 29}));
v_v2 = vmlal_u16(v_v2, vget_high_u16(v_col0), ((uint16x4_t){28, 27, 26, 25}));
v_v2 = vmlal_u16(v_v2, vget_low_u16(v_col1), ((uint16x4_t){24, 23, 22, 21}));
v_v2 = vmlal_u16(v_v2, vget_high_u16(v_col1), ((uint16x4_t){20, 19, 18, 17}));
v_v2 = vmlal_u16(v_v2, vget_low_u16(v_col2), ((uint16x4_t){16, 15, 14, 13}));
v_v2 = vmlal_u16(v_v2, vget_high_u16(v_col2), ((uint16x4_t){12, 11, 10, 9}));
v_v2 = vmlal_u16(v_v2, vget_low_u16(v_col3), ((uint16x4_t){8, 7, 6, 5}));
v_v2 = vmlal_u16(v_v2, vget_high_u16(v_col3), ((uint16x4_t){4, 3, 2, 1}));
v_sum1 = vpadd_u32(vget_low_u32(v_v1), vget_high_u32(v_v1));
v_sum2 = vpadd_u32(vget_low_u32(v_v2), vget_high_u32(v_v2));
v_sum12 = vpadd_u32(v_sum1, v_sum2);
v_s1 += vget_lane_u32(v_sum12, 0);
v_s2 += vget_lane_u32(v_sum12, 1);
v_tail_index = (((uint64_t)(a_x.len)) & 18446744073709551584u);
if (v_tail_index < ((uint64_t)(a_x.len))) {
{
@@ -31732,13 +31732,13 @@ wuffs_png__decoder__filter_4_distance_3_arm_neon(
uint8x8_t v_fb = {0};
uint8x8_t v_fc = {0};
uint8x8_t v_fx = {0};
uint8x16_t v_fafb = {0};
uint8x16_t v_fcfc = {0};
uint8x16_t v_pa = {0};
uint8x16_t v_pb = {0};
uint8x16_t v_pc = {0};
uint8x16_t v_cmpab = {0};
uint8x16_t v_cmpac = {0};
uint16x8_t v_fafb = {0};
uint16x8_t v_fcfc = {0};
uint16x8_t v_pa = {0};
uint16x8_t v_pb = {0};
uint16x8_t v_pc = {0};
uint16x8_t v_cmpab = {0};
uint16x8_t v_cmpac = {0};
uint8x8_t v_picka = {0};
uint8x8_t v_pickb = {0};
@@ -31754,15 +31754,15 @@ wuffs_png__decoder__filter_4_distance_3_arm_neon(
while (v_curr.ptr < i_end0_curr) {
v_fb = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u32le__no_bounds_check(v_prev.ptr)));
v_fx = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u32le__no_bounds_check(v_curr.ptr)));
v_fafb = vreinterpretq_u8_u16(vaddl_u8(v_fa, v_fb));
v_fcfc = vreinterpretq_u8_u16(vaddl_u8(v_fc, v_fc));
v_pa = vreinterpretq_u8_u16(vabdl_u8(v_fb, v_fc));
v_pb = vreinterpretq_u8_u16(vabdl_u8(v_fa, v_fc));
v_pc = vreinterpretq_u8_u16(vabdq_u16(vreinterpretq_u16_u8(v_fafb), vreinterpretq_u16_u8(v_fcfc)));
v_cmpab = vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pa), vreinterpretq_u16_u8(v_pb)));
v_cmpac = vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pa), vreinterpretq_u16_u8(v_pc)));
v_picka = vmovn_u16(vreinterpretq_u16_u8(vandq_u8(v_cmpab, v_cmpac)));
v_pickb = vmovn_u16(vreinterpretq_u16_u8(vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pb), vreinterpretq_u16_u8(v_pc)))));
v_fafb = vaddl_u8(v_fa, v_fb);
v_fcfc = vaddl_u8(v_fc, v_fc);
v_pa = vabdl_u8(v_fb, v_fc);
v_pb = vabdl_u8(v_fa, v_fc);
v_pc = vabdq_u16(v_fafb, v_fcfc);
v_cmpab = vcleq_u16(v_pa, v_pb);
v_cmpac = vcleq_u16(v_pa, v_pc);
v_picka = vmovn_u16(vandq_u16(v_cmpab, v_cmpac));
v_pickb = vmovn_u16(vcleq_u16(v_pb, v_pc));
v_fx = vadd_u8(v_fx, vbsl_u8(v_picka, v_fa, vbsl_u8(v_pickb, v_fb, v_fc)));
wuffs_base__poke_u24le__no_bounds_check(v_curr.ptr, vget_lane_u32(vreinterpret_u32_u8(v_fx), 0));
v_fc = v_fb;
@@ -31771,15 +31771,15 @@ wuffs_png__decoder__filter_4_distance_3_arm_neon(
v_prev.ptr += 3;
v_fb = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u32le__no_bounds_check(v_prev.ptr)));
v_fx = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u32le__no_bounds_check(v_curr.ptr)));
v_fafb = vreinterpretq_u8_u16(vaddl_u8(v_fa, v_fb));
v_fcfc = vreinterpretq_u8_u16(vaddl_u8(v_fc, v_fc));
v_pa = vreinterpretq_u8_u16(vabdl_u8(v_fb, v_fc));
v_pb = vreinterpretq_u8_u16(vabdl_u8(v_fa, v_fc));
v_pc = vreinterpretq_u8_u16(vabdq_u16(vreinterpretq_u16_u8(v_fafb), vreinterpretq_u16_u8(v_fcfc)));
v_cmpab = vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pa), vreinterpretq_u16_u8(v_pb)));
v_cmpac = vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pa), vreinterpretq_u16_u8(v_pc)));
v_picka = vmovn_u16(vreinterpretq_u16_u8(vandq_u8(v_cmpab, v_cmpac)));
v_pickb = vmovn_u16(vreinterpretq_u16_u8(vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pb), vreinterpretq_u16_u8(v_pc)))));
v_fafb = vaddl_u8(v_fa, v_fb);
v_fcfc = vaddl_u8(v_fc, v_fc);
v_pa = vabdl_u8(v_fb, v_fc);
v_pb = vabdl_u8(v_fa, v_fc);
v_pc = vabdq_u16(v_fafb, v_fcfc);
v_cmpab = vcleq_u16(v_pa, v_pb);
v_cmpac = vcleq_u16(v_pa, v_pc);
v_picka = vmovn_u16(vandq_u16(v_cmpab, v_cmpac));
v_pickb = vmovn_u16(vcleq_u16(v_pb, v_pc));
v_fx = vadd_u8(v_fx, vbsl_u8(v_picka, v_fa, vbsl_u8(v_pickb, v_fb, v_fc)));
wuffs_base__poke_u24le__no_bounds_check(v_curr.ptr, vget_lane_u32(vreinterpret_u32_u8(v_fx), 0));
v_fc = v_fb;
@@ -31793,15 +31793,15 @@ wuffs_png__decoder__filter_4_distance_3_arm_neon(
while (v_curr.ptr < i_end1_curr) {
v_fb = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u32le__no_bounds_check(v_prev.ptr)));
v_fx = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u32le__no_bounds_check(v_curr.ptr)));
v_fafb = vreinterpretq_u8_u16(vaddl_u8(v_fa, v_fb));
v_fcfc = vreinterpretq_u8_u16(vaddl_u8(v_fc, v_fc));
v_pa = vreinterpretq_u8_u16(vabdl_u8(v_fb, v_fc));
v_pb = vreinterpretq_u8_u16(vabdl_u8(v_fa, v_fc));
v_pc = vreinterpretq_u8_u16(vabdq_u16(vreinterpretq_u16_u8(v_fafb), vreinterpretq_u16_u8(v_fcfc)));
v_cmpab = vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pa), vreinterpretq_u16_u8(v_pb)));
v_cmpac = vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pa), vreinterpretq_u16_u8(v_pc)));
v_picka = vmovn_u16(vreinterpretq_u16_u8(vandq_u8(v_cmpab, v_cmpac)));
v_pickb = vmovn_u16(vreinterpretq_u16_u8(vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pb), vreinterpretq_u16_u8(v_pc)))));
v_fafb = vaddl_u8(v_fa, v_fb);
v_fcfc = vaddl_u8(v_fc, v_fc);
v_pa = vabdl_u8(v_fb, v_fc);
v_pb = vabdl_u8(v_fa, v_fc);
v_pc = vabdq_u16(v_fafb, v_fcfc);
v_cmpab = vcleq_u16(v_pa, v_pb);
v_cmpac = vcleq_u16(v_pa, v_pc);
v_picka = vmovn_u16(vandq_u16(v_cmpab, v_cmpac));
v_pickb = vmovn_u16(vcleq_u16(v_pb, v_pc));
v_fx = vadd_u8(v_fx, vbsl_u8(v_picka, v_fa, vbsl_u8(v_pickb, v_fb, v_fc)));
wuffs_base__poke_u24le__no_bounds_check(v_curr.ptr, vget_lane_u32(vreinterpret_u32_u8(v_fx), 0));
v_fc = v_fb;
@@ -31815,15 +31815,15 @@ wuffs_png__decoder__filter_4_distance_3_arm_neon(
while (v_curr.ptr < i_end2_curr) {
v_fb = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u24le__no_bounds_check(v_prev.ptr)));
v_fx = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u24le__no_bounds_check(v_curr.ptr)));
v_fafb = vreinterpretq_u8_u16(vaddl_u8(v_fa, v_fb));
v_fcfc = vreinterpretq_u8_u16(vaddl_u8(v_fc, v_fc));
v_pa = vreinterpretq_u8_u16(vabdl_u8(v_fb, v_fc));
v_pb = vreinterpretq_u8_u16(vabdl_u8(v_fa, v_fc));
v_pc = vreinterpretq_u8_u16(vabdq_u16(vreinterpretq_u16_u8(v_fafb), vreinterpretq_u16_u8(v_fcfc)));
v_cmpab = vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pa), vreinterpretq_u16_u8(v_pb)));
v_cmpac = vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pa), vreinterpretq_u16_u8(v_pc)));
v_picka = vmovn_u16(vreinterpretq_u16_u8(vandq_u8(v_cmpab, v_cmpac)));
v_pickb = vmovn_u16(vreinterpretq_u16_u8(vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pb), vreinterpretq_u16_u8(v_pc)))));
v_fafb = vaddl_u8(v_fa, v_fb);
v_fcfc = vaddl_u8(v_fc, v_fc);
v_pa = vabdl_u8(v_fb, v_fc);
v_pb = vabdl_u8(v_fa, v_fc);
v_pc = vabdq_u16(v_fafb, v_fcfc);
v_cmpab = vcleq_u16(v_pa, v_pb);
v_cmpac = vcleq_u16(v_pa, v_pc);
v_picka = vmovn_u16(vandq_u16(v_cmpab, v_cmpac));
v_pickb = vmovn_u16(vcleq_u16(v_pb, v_pc));
v_fx = vadd_u8(v_fx, vbsl_u8(v_picka, v_fa, vbsl_u8(v_pickb, v_fb, v_fc)));
wuffs_base__poke_u24le__no_bounds_check(v_curr.ptr, vget_lane_u32(vreinterpret_u32_u8(v_fx), 0));
v_curr.ptr += 3;
@@ -31850,13 +31850,13 @@ wuffs_png__decoder__filter_4_distance_4_arm_neon(
uint8x8_t v_fb = {0};
uint8x8_t v_fc = {0};
uint8x8_t v_fx = {0};
uint8x16_t v_fafb = {0};
uint8x16_t v_fcfc = {0};
uint8x16_t v_pa = {0};
uint8x16_t v_pb = {0};
uint8x16_t v_pc = {0};
uint8x16_t v_cmpab = {0};
uint8x16_t v_cmpac = {0};
uint16x8_t v_fafb = {0};
uint16x8_t v_fcfc = {0};
uint16x8_t v_pa = {0};
uint16x8_t v_pb = {0};
uint16x8_t v_pc = {0};
uint16x8_t v_cmpab = {0};
uint16x8_t v_cmpac = {0};
uint8x8_t v_picka = {0};
uint8x8_t v_pickb = {0};
@@ -31872,15 +31872,15 @@ wuffs_png__decoder__filter_4_distance_4_arm_neon(
while (v_curr.ptr < i_end0_curr) {
v_fb = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u32le__no_bounds_check(v_prev.ptr)));
v_fx = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u32le__no_bounds_check(v_curr.ptr)));
v_fafb = vreinterpretq_u8_u16(vaddl_u8(v_fa, v_fb));
v_fcfc = vreinterpretq_u8_u16(vaddl_u8(v_fc, v_fc));
v_pa = vreinterpretq_u8_u16(vabdl_u8(v_fb, v_fc));
v_pb = vreinterpretq_u8_u16(vabdl_u8(v_fa, v_fc));
v_pc = vreinterpretq_u8_u16(vabdq_u16(vreinterpretq_u16_u8(v_fafb), vreinterpretq_u16_u8(v_fcfc)));
v_cmpab = vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pa), vreinterpretq_u16_u8(v_pb)));
v_cmpac = vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pa), vreinterpretq_u16_u8(v_pc)));
v_picka = vmovn_u16(vreinterpretq_u16_u8(vandq_u8(v_cmpab, v_cmpac)));
v_pickb = vmovn_u16(vreinterpretq_u16_u8(vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pb), vreinterpretq_u16_u8(v_pc)))));
v_fafb = vaddl_u8(v_fa, v_fb);
v_fcfc = vaddl_u8(v_fc, v_fc);
v_pa = vabdl_u8(v_fb, v_fc);
v_pb = vabdl_u8(v_fa, v_fc);
v_pc = vabdq_u16(v_fafb, v_fcfc);
v_cmpab = vcleq_u16(v_pa, v_pb);
v_cmpac = vcleq_u16(v_pa, v_pc);
v_picka = vmovn_u16(vandq_u16(v_cmpab, v_cmpac));
v_pickb = vmovn_u16(vcleq_u16(v_pb, v_pc));
v_fx = vadd_u8(v_fx, vbsl_u8(v_picka, v_fa, vbsl_u8(v_pickb, v_fb, v_fc)));
wuffs_base__poke_u32le__no_bounds_check(v_curr.ptr, vget_lane_u32(vreinterpret_u32_u8(v_fx), 0));
v_fc = v_fb;
@@ -31889,15 +31889,15 @@ wuffs_png__decoder__filter_4_distance_4_arm_neon(
v_prev.ptr += 4;
v_fb = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u32le__no_bounds_check(v_prev.ptr)));
v_fx = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u32le__no_bounds_check(v_curr.ptr)));
v_fafb = vreinterpretq_u8_u16(vaddl_u8(v_fa, v_fb));
v_fcfc = vreinterpretq_u8_u16(vaddl_u8(v_fc, v_fc));
v_pa = vreinterpretq_u8_u16(vabdl_u8(v_fb, v_fc));
v_pb = vreinterpretq_u8_u16(vabdl_u8(v_fa, v_fc));
v_pc = vreinterpretq_u8_u16(vabdq_u16(vreinterpretq_u16_u8(v_fafb), vreinterpretq_u16_u8(v_fcfc)));
v_cmpab = vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pa), vreinterpretq_u16_u8(v_pb)));
v_cmpac = vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pa), vreinterpretq_u16_u8(v_pc)));
v_picka = vmovn_u16(vreinterpretq_u16_u8(vandq_u8(v_cmpab, v_cmpac)));
v_pickb = vmovn_u16(vreinterpretq_u16_u8(vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pb), vreinterpretq_u16_u8(v_pc)))));
v_fafb = vaddl_u8(v_fa, v_fb);
v_fcfc = vaddl_u8(v_fc, v_fc);
v_pa = vabdl_u8(v_fb, v_fc);
v_pb = vabdl_u8(v_fa, v_fc);
v_pc = vabdq_u16(v_fafb, v_fcfc);
v_cmpab = vcleq_u16(v_pa, v_pb);
v_cmpac = vcleq_u16(v_pa, v_pc);
v_picka = vmovn_u16(vandq_u16(v_cmpab, v_cmpac));
v_pickb = vmovn_u16(vcleq_u16(v_pb, v_pc));
v_fx = vadd_u8(v_fx, vbsl_u8(v_picka, v_fa, vbsl_u8(v_pickb, v_fb, v_fc)));
wuffs_base__poke_u32le__no_bounds_check(v_curr.ptr, vget_lane_u32(vreinterpret_u32_u8(v_fx), 0));
v_fc = v_fb;
@@ -31911,15 +31911,15 @@ wuffs_png__decoder__filter_4_distance_4_arm_neon(
while (v_curr.ptr < i_end1_curr) {
v_fb = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u32le__no_bounds_check(v_prev.ptr)));
v_fx = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u32le__no_bounds_check(v_curr.ptr)));
v_fafb = vreinterpretq_u8_u16(vaddl_u8(v_fa, v_fb));
v_fcfc = vreinterpretq_u8_u16(vaddl_u8(v_fc, v_fc));
v_pa = vreinterpretq_u8_u16(vabdl_u8(v_fb, v_fc));
v_pb = vreinterpretq_u8_u16(vabdl_u8(v_fa, v_fc));
v_pc = vreinterpretq_u8_u16(vabdq_u16(vreinterpretq_u16_u8(v_fafb), vreinterpretq_u16_u8(v_fcfc)));
v_cmpab = vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pa), vreinterpretq_u16_u8(v_pb)));
v_cmpac = vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pa), vreinterpretq_u16_u8(v_pc)));
v_picka = vmovn_u16(vreinterpretq_u16_u8(vandq_u8(v_cmpab, v_cmpac)));
v_pickb = vmovn_u16(vreinterpretq_u16_u8(vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pb), vreinterpretq_u16_u8(v_pc)))));
v_fafb = vaddl_u8(v_fa, v_fb);
v_fcfc = vaddl_u8(v_fc, v_fc);
v_pa = vabdl_u8(v_fb, v_fc);
v_pb = vabdl_u8(v_fa, v_fc);
v_pc = vabdq_u16(v_fafb, v_fcfc);
v_cmpab = vcleq_u16(v_pa, v_pb);
v_cmpac = vcleq_u16(v_pa, v_pc);
v_picka = vmovn_u16(vandq_u16(v_cmpab, v_cmpac));
v_pickb = vmovn_u16(vcleq_u16(v_pb, v_pc));
v_fx = vadd_u8(v_fx, vbsl_u8(v_picka, v_fa, vbsl_u8(v_pickb, v_fb, v_fc)));
wuffs_base__poke_u32le__no_bounds_check(v_curr.ptr, vget_lane_u32(vreinterpret_u32_u8(v_fx), 0));
v_fc = v_fb;
+28 -27
View File
@@ -23,17 +23,18 @@ pri func hasher.up_arm_neon!(x: slice base.u8),
// The remaining variables are specific to the SIMD version.
var p__left : base.arm_neon_128
var p_right : base.arm_neon_128
var v1 : base.arm_neon_128
var v2 : base.arm_neon_128
var col0 : base.arm_neon_128
var col1 : base.arm_neon_128
var col2 : base.arm_neon_128
var col3 : base.arm_neon_128
var sum1 : base.arm_neon_64
var sum2 : base.arm_neon_64
var sum12 : base.arm_neon_64
var util : base.arm_neon_utility
var p__left : base.arm_neon_u8x16
var p_right : base.arm_neon_u8x16
var v1 : base.arm_neon_u32x4
var v2 : base.arm_neon_u32x4
var col0 : base.arm_neon_u16x8
var col1 : base.arm_neon_u16x8
var col2 : base.arm_neon_u16x8
var col3 : base.arm_neon_u16x8
var sum1 : base.arm_neon_u32x2
var sum2 : base.arm_neon_u32x2
var sum12 : base.arm_neon_u32x2
var num_iterate_bytes : base.u32
var tail_index : base.u64
@@ -79,12 +80,12 @@ pri func hasher.up_arm_neon!(x: slice base.u8),
// variables s1 and s2. The iterate loop accumulates four parallel u32
// sums in each vector. A post-iterate step merges the four u32 sums
// into a single u32 sum.
v1 = v1.create_vdupq_n_u32(a: 0)
v2 = v2.create_vdupq_n_u32(a: 0)
col0 = col0.create_vdupq_n_u16(a: 0)
col1 = col1.create_vdupq_n_u16(a: 0)
col2 = col2.create_vdupq_n_u16(a: 0)
col3 = col3.create_vdupq_n_u16(a: 0)
v1 = util.make_u32x4_repeat(a: 0)
v2 = util.make_u32x4_repeat(a: 0)
col0 = util.make_u16x8_repeat(a: 0)
col1 = util.make_u16x8_repeat(a: 0)
col2 = util.make_u16x8_repeat(a: 0)
col3 = util.make_u16x8_repeat(a: 0)
// The inner loop.
iterate (p = args.x)(length: 32, advance: 32, unroll: 1) {
@@ -93,8 +94,8 @@ pri func hasher.up_arm_neon!(x: slice base.u8),
//
// Let p__left = [u8×16: p00, p01, p02, ..., p15]
// Let p_right = [u8×16: p16, p17, p18, ..., p31]
p__left = p__left.create_slice128(a: p[.. 16])
p_right = p_right.create_slice128(a: p[16 .. 32])
p__left = util.make_u8x16_slice128(a: p[.. 16])
p_right = util.make_u8x16_slice128(a: p[16 .. 32])
// For v2j, we need to calculate the sums of the s1j terms for each
// of p's 32 elements. This is simply 32 times the same number,
@@ -141,28 +142,28 @@ pri func hasher.up_arm_neon!(x: slice base.u8),
// [u32×4: 0x04 * (Σ p28), 0x03 * (Σ p29), ..., 0x01 * (Σ p31)]
v2 = v2.vmlal_u16(
b: col0.vget_low_u16(),
c: sum2.create_literal_u16x4(a0: 0x20, a1: 0x1F, a2: 0x1E, a3: 0x1D))
c: util.make_u16x4_multiple(a00: 0x20, a01: 0x1F, a02: 0x1E, a03: 0x1D))
v2 = v2.vmlal_u16(
b: col0.vget_high_u16(),
c: sum2.create_literal_u16x4(a0: 0x1C, a1: 0x1B, a2: 0x1A, a3: 0x19))
c: util.make_u16x4_multiple(a00: 0x1C, a01: 0x1B, a02: 0x1A, a03: 0x19))
v2 = v2.vmlal_u16(
b: col1.vget_low_u16(),
c: sum2.create_literal_u16x4(a0: 0x18, a1: 0x17, a2: 0x16, a3: 0x15))
c: util.make_u16x4_multiple(a00: 0x18, a01: 0x17, a02: 0x16, a03: 0x15))
v2 = v2.vmlal_u16(
b: col1.vget_high_u16(),
c: sum2.create_literal_u16x4(a0: 0x14, a1: 0x13, a2: 0x12, a3: 0x11))
c: util.make_u16x4_multiple(a00: 0x14, a01: 0x13, a02: 0x12, a03: 0x11))
v2 = v2.vmlal_u16(
b: col2.vget_low_u16(),
c: sum2.create_literal_u16x4(a0: 0x10, a1: 0x0F, a2: 0x0E, a3: 0x0D))
c: util.make_u16x4_multiple(a00: 0x10, a01: 0x0F, a02: 0x0E, a03: 0x0D))
v2 = v2.vmlal_u16(
b: col2.vget_high_u16(),
c: sum2.create_literal_u16x4(a0: 0x0C, a1: 0x0B, a2: 0x0A, a3: 0x09))
c: util.make_u16x4_multiple(a00: 0x0C, a01: 0x0B, a02: 0x0A, a03: 0x09))
v2 = v2.vmlal_u16(
b: col3.vget_low_u16(),
c: sum2.create_literal_u16x4(a0: 0x08, a1: 0x07, a2: 0x06, a3: 0x05))
c: util.make_u16x4_multiple(a00: 0x08, a01: 0x07, a02: 0x06, a03: 0x05))
v2 = v2.vmlal_u16(
b: col3.vget_high_u16(),
c: sum2.create_literal_u16x4(a0: 0x04, a1: 0x03, a2: 0x02, a3: 0x01))
c: util.make_u16x4_multiple(a00: 0x04, a01: 0x03, a02: 0x02, a03: 0x01))
// Merge the four parallel u32 sums (v1) into the single u32 sum (s1)
// and ditto for v2. Starting with [u32×4: xx_0, xx_1, xx_2, xx_3]:
+50 -47
View File
@@ -55,24 +55,25 @@ pri func decoder.filter_3_distance_4_arm_neon!(curr: slice base.u8, prev: slice
var curr : slice base.u8
var prev : slice base.u8
var fa : base.arm_neon_64
var fb : base.arm_neon_64
var fx : base.arm_neon_64
var util : base.arm_neon_utility
var fa : base.arm_neon_u8x8
var fb : base.arm_neon_u8x8
var fx : base.arm_neon_u8x8
if args.prev.length() == 0 {
iterate (curr = args.curr)(length: 4, advance: 4, unroll: 2) {
fx = fx.create_vdup_n_u32(a: curr.peek_u32le())
fx = util.make_u32x2_repeat(a: curr.peek_u32le()).as_u8x8()
fx = fx.vadd_u8(b: fa.vhadd_u8(b: fb))
curr.poke_u32le!(a: fx.vget_lane_u32(b: 0))
curr.poke_u32le!(a: fx.as_u32x2().vget_lane_u32(b: 0))
fa = fx
}
} else {
iterate (curr = args.curr, prev = args.prev)(length: 4, advance: 4, unroll: 2) {
fb = fb.create_vdup_n_u32(a: prev.peek_u32le())
fx = fx.create_vdup_n_u32(a: curr.peek_u32le())
fb = util.make_u32x2_repeat(a: prev.peek_u32le()).as_u8x8()
fx = util.make_u32x2_repeat(a: curr.peek_u32le()).as_u8x8()
fx = fx.vadd_u8(b: fa.vhadd_u8(b: fb))
curr.poke_u32le!(a: fx.vget_lane_u32(b: 0))
curr.poke_u32le!(a: fx.as_u32x2().vget_lane_u32(b: 0))
fa = fx
}
}
@@ -95,24 +96,25 @@ pri func decoder.filter_4_distance_3_arm_neon!(curr: slice base.u8, prev: slice
var curr : slice base.u8
var prev : slice base.u8
var fa : base.arm_neon_64
var fb : base.arm_neon_64
var fc : base.arm_neon_64
var fx : base.arm_neon_64
var fafb : base.arm_neon_128
var fcfc : base.arm_neon_128
var pa : base.arm_neon_128
var pb : base.arm_neon_128
var pc : base.arm_neon_128
var cmpab : base.arm_neon_128
var cmpac : base.arm_neon_128
var picka : base.arm_neon_64
var pickb : base.arm_neon_64
var util : base.arm_neon_utility
var fa : base.arm_neon_u8x8
var fb : base.arm_neon_u8x8
var fc : base.arm_neon_u8x8
var fx : base.arm_neon_u8x8
var fafb : base.arm_neon_u16x8
var fcfc : base.arm_neon_u16x8
var pa : base.arm_neon_u16x8
var pb : base.arm_neon_u16x8
var pc : base.arm_neon_u16x8
var cmpab : base.arm_neon_u16x8
var cmpac : base.arm_neon_u16x8
var picka : base.arm_neon_u8x8
var pickb : base.arm_neon_u8x8
// § The advance is 3, not 4.
iterate (curr = args.curr, prev = args.prev)(length: 4, advance: 3, unroll: 2) {
fb = fb.create_vdup_n_u32(a: prev.peek_u32le())
fx = fx.create_vdup_n_u32(a: curr.peek_u32le())
fb = util.make_u32x2_repeat(a: prev.peek_u32le()).as_u8x8()
fx = util.make_u32x2_repeat(a: curr.peek_u32le()).as_u8x8()
fafb = fa.vaddl_u8(b: fb)
fcfc = fc.vaddl_u8(b: fc)
pa = fb.vabdl_u8(b: fc)
@@ -120,20 +122,20 @@ pri func decoder.filter_4_distance_3_arm_neon!(curr: slice base.u8, prev: slice
pc = fafb.vabdq_u16(b: fcfc)
cmpab = pa.vcleq_u16(b: pb)
cmpac = pa.vcleq_u16(b: pc)
picka = cmpab.vandq_u8(b: cmpac).vmovn_u16()
picka = cmpab.vandq_u16(b: cmpac).vmovn_u16()
pickb = pb.vcleq_u16(b: pc).vmovn_u16()
fx = fx.vadd_u8(
b: picka.vbsl_u8(b: fa,
c: pickb.vbsl_u8(b: fb, c: fc)))
// § poke_u24le replaces poke_u32le.
curr.poke_u24le!(a: fx.vget_lane_u32(b: 0))
curr.poke_u24le!(a: fx.as_u32x2().vget_lane_u32(b: 0))
fc = fb
fa = fx
} else (length: 3, advance: 3, unroll: 1) {
// § peek_u24le_as_u32 replaces peek_u32le.
fb = fb.create_vdup_n_u32(a: prev.peek_u24le_as_u32())
fb = util.make_u32x2_repeat(a: prev.peek_u24le_as_u32()).as_u8x8()
// § peek_u24le_as_u32 replaces peek_u32le.
fx = fx.create_vdup_n_u32(a: curr.peek_u24le_as_u32())
fx = util.make_u32x2_repeat(a: curr.peek_u24le_as_u32()).as_u8x8()
fafb = fa.vaddl_u8(b: fb)
fcfc = fc.vaddl_u8(b: fc)
pa = fb.vabdl_u8(b: fc)
@@ -141,13 +143,13 @@ pri func decoder.filter_4_distance_3_arm_neon!(curr: slice base.u8, prev: slice
pc = fafb.vabdq_u16(b: fcfc)
cmpab = pa.vcleq_u16(b: pb)
cmpac = pa.vcleq_u16(b: pc)
picka = cmpab.vandq_u8(b: cmpac).vmovn_u16()
picka = cmpab.vandq_u16(b: cmpac).vmovn_u16()
pickb = pb.vcleq_u16(b: pc).vmovn_u16()
fx = fx.vadd_u8(
b: picka.vbsl_u8(b: fa,
c: pickb.vbsl_u8(b: fb, c: fc)))
// § poke_u24le replaces poke_u32le.
curr.poke_u24le!(a: fx.vget_lane_u32(b: 0))
curr.poke_u24le!(a: fx.as_u32x2().vget_lane_u32(b: 0))
// § These assignments are unnecessary; this is the last iteration.
// fc = fb
// fa = fx
@@ -160,23 +162,24 @@ pri func decoder.filter_4_distance_4_arm_neon!(curr: slice base.u8, prev: slice
var curr : slice base.u8
var prev : slice base.u8
var fa : base.arm_neon_64
var fb : base.arm_neon_64
var fc : base.arm_neon_64
var fx : base.arm_neon_64
var fafb : base.arm_neon_128
var fcfc : base.arm_neon_128
var pa : base.arm_neon_128
var pb : base.arm_neon_128
var pc : base.arm_neon_128
var cmpab : base.arm_neon_128
var cmpac : base.arm_neon_128
var picka : base.arm_neon_64
var pickb : base.arm_neon_64
var util : base.arm_neon_utility
var fa : base.arm_neon_u8x8
var fb : base.arm_neon_u8x8
var fc : base.arm_neon_u8x8
var fx : base.arm_neon_u8x8
var fafb : base.arm_neon_u16x8
var fcfc : base.arm_neon_u16x8
var pa : base.arm_neon_u16x8
var pb : base.arm_neon_u16x8
var pc : base.arm_neon_u16x8
var cmpab : base.arm_neon_u16x8
var cmpac : base.arm_neon_u16x8
var picka : base.arm_neon_u8x8
var pickb : base.arm_neon_u8x8
iterate (curr = args.curr, prev = args.prev)(length: 4, advance: 4, unroll: 2) {
fb = fb.create_vdup_n_u32(a: prev.peek_u32le())
fx = fx.create_vdup_n_u32(a: curr.peek_u32le())
fb = util.make_u32x2_repeat(a: prev.peek_u32le()).as_u8x8()
fx = util.make_u32x2_repeat(a: curr.peek_u32le()).as_u8x8()
fafb = fa.vaddl_u8(b: fb) // fafb = (fa + fb)
fcfc = fc.vaddl_u8(b: fc) // fcfc = (fc + fc)
@@ -188,15 +191,15 @@ pri func decoder.filter_4_distance_4_arm_neon!(curr: slice base.u8, prev: slice
cmpab = pa.vcleq_u16(b: pb) // cmpab = (pa <= pb)
cmpac = pa.vcleq_u16(b: pc) // cmpac = (pa <= pc)
picka = cmpab.vandq_u8(b: cmpac).vmovn_u16() // picka = ((pa <= pb) && (pa <= pc))
pickb = pb.vcleq_u16(b: pc).vmovn_u16() // pickb = (pb <= pc)
picka = cmpab.vandq_u16(b: cmpac).vmovn_u16() // picka = ((pa <= pb) && (pa <= pc))
pickb = pb.vcleq_u16(b: pc).vmovn_u16() // pickb = (pb <= pc)
// Add the predictor to the residual.
fx = fx.vadd_u8(
b: picka.vbsl_u8(b: fa,
c: pickb.vbsl_u8(b: fb, c: fc)))
curr.poke_u32le!(a: fx.vget_lane_u32(b: 0))
curr.poke_u32le!(a: fx.as_u32x2().vget_lane_u32(b: 0))
fc = fb
fa = fx
}