mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-18 08:32:11 +01:00
Use arm_neon_uAxB types and methods
This commit is contained in:
1 parent
1f9b6ea535
commit
3b5d8fdfba
5 files changed
+279
-204
No files matched your search
+56
-16
@@ -775,29 +775,69 @@ func (g *gen) writeBuiltinCPUArch(b *buffer, recv *a.Expr, method t.ID, args []*
|
||||
func (g *gen) writeBuiltinCPUArchARMNeon(b *buffer, recv *a.Expr, method t.ID, args []*a.Node, sideEffectsOnly bool, depth uint32) error {
|
||||
methodStr := method.Str(g.tm)
|
||||
if strings.HasPrefix(methodStr, "make_") {
|
||||
fName := ""
|
||||
switch methodStr {
|
||||
case "make_u8x8_repeat":
|
||||
fName = "vdup_n_u8"
|
||||
case "make_u16x4_repeat":
|
||||
fName = "vdup_n_u16"
|
||||
case "make_u32x2_repeat":
|
||||
fName = "vdup_n_u32"
|
||||
case "make_u64x1_repeat":
|
||||
fName = "vdup_n_u64"
|
||||
default:
|
||||
return fmt.Errorf("internal error: unsupported cpu_arch method %q", methodStr)
|
||||
before, after, ptr := "", ")", false
|
||||
if strings.HasSuffix(methodStr, "_multiple") {
|
||||
after = "})"
|
||||
switch methodStr {
|
||||
case "make_u8x8_multiple":
|
||||
before = "((uint8x8_t){"
|
||||
case "make_u16x4_multiple":
|
||||
before = "((uint16x4_t){"
|
||||
case "make_u32x2_multiple":
|
||||
before = "((uint32x2_t){"
|
||||
case "make_u64x1_multiple":
|
||||
before = "((uint64x1_t){"
|
||||
case "make_u8x16_multiple":
|
||||
before = "((uint8x16_t){"
|
||||
case "make_u16x8_multiple":
|
||||
before = "((uint16x8_t){"
|
||||
case "make_u32x4_multiple":
|
||||
before = "((uint32x4_t){"
|
||||
case "make_u64x2_multiple":
|
||||
before = "((uint64x2_t){"
|
||||
}
|
||||
} else {
|
||||
switch methodStr {
|
||||
case "make_u8x8_repeat":
|
||||
before = "vdup_n_u8("
|
||||
case "make_u16x4_repeat":
|
||||
before = "vdup_n_u16("
|
||||
case "make_u32x2_repeat":
|
||||
before = "vdup_n_u32("
|
||||
case "make_u64x1_repeat":
|
||||
before = "vdup_n_u64("
|
||||
case "make_u8x16_repeat":
|
||||
before = "vdupq_n_u8("
|
||||
case "make_u16x8_repeat":
|
||||
before = "vdupq_n_u16("
|
||||
case "make_u32x4_repeat":
|
||||
before = "vdupq_n_u32("
|
||||
case "make_u64x2_repeat":
|
||||
before = "vdupq_n_u64("
|
||||
case "make_u8x8_slice64":
|
||||
before, ptr = "vld1_u8(", true
|
||||
case "make_u8x16_slice128":
|
||||
before, ptr = "vld1q_u8(", true
|
||||
default:
|
||||
return fmt.Errorf("internal error: unsupported cpu_arch method %q", methodStr)
|
||||
}
|
||||
}
|
||||
b.printf("%s(", fName)
|
||||
b.writes(before)
|
||||
for i, o := range args {
|
||||
if i > 0 {
|
||||
b.writes(", ")
|
||||
}
|
||||
if err := g.writeExpr(b, o.AsArg().Value(), false, depth); err != nil {
|
||||
return err
|
||||
if ptr {
|
||||
if err := g.writeExprDotPtr(b, o.AsArg().Value(), false, depth); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if err := g.writeExpr(b, o.AsArg().Value(), false, depth); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
b.writes(")")
|
||||
b.writes(after)
|
||||
return nil
|
||||
|
||||
} else if strings.HasPrefix(methodStr, "as_") {
|
||||
|
||||
@@ -568,11 +568,42 @@ var funcsOther = [...]string{
|
||||
|
||||
// ---- arm_neon_utility
|
||||
|
||||
"arm_neon_utility.make_u8x8_multiple(" +
|
||||
"a00: u8, a01: u8, a02: u8, a03: u8," +
|
||||
"a04: u8, a05: u8, a06: u8, a07: u8) arm_neon_u8x8",
|
||||
"arm_neon_utility.make_u16x4_multiple(" +
|
||||
"a00: u16, a01: u16, a02: u16, a03: u16) arm_neon_u16x4",
|
||||
"arm_neon_utility.make_u32x2_multiple(" +
|
||||
"a00: u32, a01: u32) arm_neon_u32x2",
|
||||
"arm_neon_utility.make_u64x1_multiple(" +
|
||||
"a00: u64) arm_neon_u64x1",
|
||||
|
||||
"arm_neon_utility.make_u8x16_multiple(" +
|
||||
"a00: u8, a01: u8, a02: u8, a03: u8," +
|
||||
"a04: u8, a05: u8, a06: u8, a07: u8," +
|
||||
"a08: u8, a09: u8, a10: u8, a11: u8," +
|
||||
"a12: u8, a13: u8, a14: u8, a15: u8) arm_neon_u8x16",
|
||||
"arm_neon_utility.make_u16x8_multiple(" +
|
||||
"a00: u16, a01: u16, a02: u16, a03: u16," +
|
||||
"a04: u16, a05: u16, a06: u16, a07: u16) arm_neon_u16x8",
|
||||
"arm_neon_utility.make_u32x4_multiple(" +
|
||||
"a00: u32, a01: u32, a02: u32, a03: u32) arm_neon_u32x4",
|
||||
"arm_neon_utility.make_u64x2_multiple(" +
|
||||
"a00: u64, a01: u64) arm_neon_u64x2",
|
||||
|
||||
"arm_neon_utility.make_u8x8_repeat(a: u8) arm_neon_u8x8",
|
||||
"arm_neon_utility.make_u16x4_repeat(a: u16) arm_neon_u16x4",
|
||||
"arm_neon_utility.make_u32x2_repeat(a: u32) arm_neon_u32x2",
|
||||
"arm_neon_utility.make_u64x1_repeat(a: u64) arm_neon_u64x1",
|
||||
|
||||
"arm_neon_utility.make_u8x16_repeat(a: u8) arm_neon_u8x16",
|
||||
"arm_neon_utility.make_u16x8_repeat(a: u16) arm_neon_u16x8",
|
||||
"arm_neon_utility.make_u32x4_repeat(a: u32) arm_neon_u32x4",
|
||||
"arm_neon_utility.make_u64x2_repeat(a: u64) arm_neon_u64x2",
|
||||
|
||||
"arm_neon_utility.make_u8x8_slice64(a: slice base.u8) arm_neon_u8x8",
|
||||
"arm_neon_utility.make_u8x16_slice128(a: slice base.u8) arm_neon_u8x16",
|
||||
|
||||
// ---- arm_neon_uAxB.as_uCxD
|
||||
|
||||
"arm_neon_u8x8.as_u16x4() arm_neon_u16x4",
|
||||
|
||||
@@ -17835,15 +17835,15 @@ wuffs_adler32__hasher__up_arm_neon(
|
||||
wuffs_base__slice_u8 v_p = {0};
|
||||
uint8x16_t v_p__left = {0};
|
||||
uint8x16_t v_p_right = {0};
|
||||
uint8x16_t v_v1 = {0};
|
||||
uint8x16_t v_v2 = {0};
|
||||
uint8x16_t v_col0 = {0};
|
||||
uint8x16_t v_col1 = {0};
|
||||
uint8x16_t v_col2 = {0};
|
||||
uint8x16_t v_col3 = {0};
|
||||
uint8x8_t v_sum1 = {0};
|
||||
uint8x8_t v_sum2 = {0};
|
||||
uint8x8_t v_sum12 = {0};
|
||||
uint32x4_t v_v1 = {0};
|
||||
uint32x4_t v_v2 = {0};
|
||||
uint16x8_t v_col0 = {0};
|
||||
uint16x8_t v_col1 = {0};
|
||||
uint16x8_t v_col2 = {0};
|
||||
uint16x8_t v_col3 = {0};
|
||||
uint32x2_t v_sum1 = {0};
|
||||
uint32x2_t v_sum2 = {0};
|
||||
uint32x2_t v_sum12 = {0};
|
||||
uint32_t v_num_iterate_bytes = 0;
|
||||
uint64_t v_tail_index = 0;
|
||||
|
||||
@@ -17864,44 +17864,44 @@ wuffs_adler32__hasher__up_arm_neon(
|
||||
}
|
||||
v_num_iterate_bytes = ((uint32_t)((((uint64_t)(a_x.len)) & 4294967264)));
|
||||
v_s2 += ((uint32_t)(v_s1 * v_num_iterate_bytes));
|
||||
v_v1 = vreinterpretq_u8_u32(vdupq_n_u32(0));
|
||||
v_v2 = vreinterpretq_u8_u32(vdupq_n_u32(0));
|
||||
v_col0 = vreinterpretq_u8_u16(vdupq_n_u16(0));
|
||||
v_col1 = vreinterpretq_u8_u16(vdupq_n_u16(0));
|
||||
v_col2 = vreinterpretq_u8_u16(vdupq_n_u16(0));
|
||||
v_col3 = vreinterpretq_u8_u16(vdupq_n_u16(0));
|
||||
v_v1 = vdupq_n_u32(0);
|
||||
v_v2 = vdupq_n_u32(0);
|
||||
v_col0 = vdupq_n_u16(0);
|
||||
v_col1 = vdupq_n_u16(0);
|
||||
v_col2 = vdupq_n_u16(0);
|
||||
v_col3 = vdupq_n_u16(0);
|
||||
{
|
||||
wuffs_base__slice_u8 i_slice_p = a_x;
|
||||
v_p.ptr = i_slice_p.ptr;
|
||||
v_p.len = 32;
|
||||
uint8_t* i_end0_p = v_p.ptr + (((i_slice_p.len - (size_t)(v_p.ptr - i_slice_p.ptr)) / 32) * 32);
|
||||
while (v_p.ptr < i_end0_p) {
|
||||
v_p__left = vld1q_u8(wuffs_base__slice_u8__subslice_j(v_p, 16).ptr);
|
||||
v_p_right = vld1q_u8(wuffs_base__slice_u8__subslice_ij(v_p, 16, 32).ptr);
|
||||
v_v2 = vreinterpretq_u8_u32(vaddq_u32(vreinterpretq_u32_u8(v_v2), vreinterpretq_u32_u8(v_v1)));
|
||||
v_v1 = vreinterpretq_u8_u32(vpadalq_u16(vreinterpretq_u32_u8(v_v1), vreinterpretq_u16_u8(vreinterpretq_u8_u16(vpadalq_u8(vreinterpretq_u16_u8(vreinterpretq_u8_u16(vpaddlq_u8(v_p__left))), v_p_right)))));
|
||||
v_col0 = vreinterpretq_u8_u16(vaddw_u8(vreinterpretq_u16_u8(v_col0), vget_low_u8(v_p__left)));
|
||||
v_col1 = vreinterpretq_u8_u16(vaddw_u8(vreinterpretq_u16_u8(v_col1), vget_high_u8(v_p__left)));
|
||||
v_col2 = vreinterpretq_u8_u16(vaddw_u8(vreinterpretq_u16_u8(v_col2), vget_low_u8(v_p_right)));
|
||||
v_col3 = vreinterpretq_u8_u16(vaddw_u8(vreinterpretq_u16_u8(v_col3), vget_high_u8(v_p_right)));
|
||||
v_p__left = vld1q_u8(v_p.ptr);
|
||||
v_p_right = vld1q_u8(v_p.ptr + 16);
|
||||
v_v2 = vaddq_u32(v_v2, v_v1);
|
||||
v_v1 = vpadalq_u16(v_v1, vpadalq_u8(vpaddlq_u8(v_p__left), v_p_right));
|
||||
v_col0 = vaddw_u8(v_col0, vget_low_u8(v_p__left));
|
||||
v_col1 = vaddw_u8(v_col1, vget_high_u8(v_p__left));
|
||||
v_col2 = vaddw_u8(v_col2, vget_low_u8(v_p_right));
|
||||
v_col3 = vaddw_u8(v_col3, vget_high_u8(v_p_right));
|
||||
v_p.ptr += 32;
|
||||
}
|
||||
v_p.len = 0;
|
||||
}
|
||||
v_v2 = vreinterpretq_u8_u32(vshlq_n_u32(vreinterpretq_u32_u8(v_v2), 5));
|
||||
v_v2 = vreinterpretq_u8_u32(vmlal_u16(vreinterpretq_u32_u8(v_v2), vreinterpret_u16_u8(vreinterpret_u8_u16(vget_low_u16(vreinterpretq_u16_u8(v_col0)))), vreinterpret_u16_u8(vreinterpret_u8_u16((uint16x4_t){32, 31, 30, 29}))));
|
||||
v_v2 = vreinterpretq_u8_u32(vmlal_u16(vreinterpretq_u32_u8(v_v2), vreinterpret_u16_u8(vreinterpret_u8_u16(vget_high_u16(vreinterpretq_u16_u8(v_col0)))), vreinterpret_u16_u8(vreinterpret_u8_u16((uint16x4_t){28, 27, 26, 25}))));
|
||||
v_v2 = vreinterpretq_u8_u32(vmlal_u16(vreinterpretq_u32_u8(v_v2), vreinterpret_u16_u8(vreinterpret_u8_u16(vget_low_u16(vreinterpretq_u16_u8(v_col1)))), vreinterpret_u16_u8(vreinterpret_u8_u16((uint16x4_t){24, 23, 22, 21}))));
|
||||
v_v2 = vreinterpretq_u8_u32(vmlal_u16(vreinterpretq_u32_u8(v_v2), vreinterpret_u16_u8(vreinterpret_u8_u16(vget_high_u16(vreinterpretq_u16_u8(v_col1)))), vreinterpret_u16_u8(vreinterpret_u8_u16((uint16x4_t){20, 19, 18, 17}))));
|
||||
v_v2 = vreinterpretq_u8_u32(vmlal_u16(vreinterpretq_u32_u8(v_v2), vreinterpret_u16_u8(vreinterpret_u8_u16(vget_low_u16(vreinterpretq_u16_u8(v_col2)))), vreinterpret_u16_u8(vreinterpret_u8_u16((uint16x4_t){16, 15, 14, 13}))));
|
||||
v_v2 = vreinterpretq_u8_u32(vmlal_u16(vreinterpretq_u32_u8(v_v2), vreinterpret_u16_u8(vreinterpret_u8_u16(vget_high_u16(vreinterpretq_u16_u8(v_col2)))), vreinterpret_u16_u8(vreinterpret_u8_u16((uint16x4_t){12, 11, 10, 9}))));
|
||||
v_v2 = vreinterpretq_u8_u32(vmlal_u16(vreinterpretq_u32_u8(v_v2), vreinterpret_u16_u8(vreinterpret_u8_u16(vget_low_u16(vreinterpretq_u16_u8(v_col3)))), vreinterpret_u16_u8(vreinterpret_u8_u16((uint16x4_t){8, 7, 6, 5}))));
|
||||
v_v2 = vreinterpretq_u8_u32(vmlal_u16(vreinterpretq_u32_u8(v_v2), vreinterpret_u16_u8(vreinterpret_u8_u16(vget_high_u16(vreinterpretq_u16_u8(v_col3)))), vreinterpret_u16_u8(vreinterpret_u8_u16((uint16x4_t){4, 3, 2, 1}))));
|
||||
v_sum1 = vreinterpret_u8_u32(vpadd_u32(vreinterpret_u32_u8(vreinterpret_u8_u32(vget_low_u32(vreinterpretq_u32_u8(v_v1)))), vreinterpret_u32_u8(vreinterpret_u8_u32(vget_high_u32(vreinterpretq_u32_u8(v_v1))))));
|
||||
v_sum2 = vreinterpret_u8_u32(vpadd_u32(vreinterpret_u32_u8(vreinterpret_u8_u32(vget_low_u32(vreinterpretq_u32_u8(v_v2)))), vreinterpret_u32_u8(vreinterpret_u8_u32(vget_high_u32(vreinterpretq_u32_u8(v_v2))))));
|
||||
v_sum12 = vreinterpret_u8_u32(vpadd_u32(vreinterpret_u32_u8(v_sum1), vreinterpret_u32_u8(v_sum2)));
|
||||
v_s1 += vget_lane_u32(vreinterpret_u32_u8(v_sum12), 0);
|
||||
v_s2 += vget_lane_u32(vreinterpret_u32_u8(v_sum12), 1);
|
||||
v_v2 = vshlq_n_u32(v_v2, 5);
|
||||
v_v2 = vmlal_u16(v_v2, vget_low_u16(v_col0), ((uint16x4_t){32, 31, 30, 29}));
|
||||
v_v2 = vmlal_u16(v_v2, vget_high_u16(v_col0), ((uint16x4_t){28, 27, 26, 25}));
|
||||
v_v2 = vmlal_u16(v_v2, vget_low_u16(v_col1), ((uint16x4_t){24, 23, 22, 21}));
|
||||
v_v2 = vmlal_u16(v_v2, vget_high_u16(v_col1), ((uint16x4_t){20, 19, 18, 17}));
|
||||
v_v2 = vmlal_u16(v_v2, vget_low_u16(v_col2), ((uint16x4_t){16, 15, 14, 13}));
|
||||
v_v2 = vmlal_u16(v_v2, vget_high_u16(v_col2), ((uint16x4_t){12, 11, 10, 9}));
|
||||
v_v2 = vmlal_u16(v_v2, vget_low_u16(v_col3), ((uint16x4_t){8, 7, 6, 5}));
|
||||
v_v2 = vmlal_u16(v_v2, vget_high_u16(v_col3), ((uint16x4_t){4, 3, 2, 1}));
|
||||
v_sum1 = vpadd_u32(vget_low_u32(v_v1), vget_high_u32(v_v1));
|
||||
v_sum2 = vpadd_u32(vget_low_u32(v_v2), vget_high_u32(v_v2));
|
||||
v_sum12 = vpadd_u32(v_sum1, v_sum2);
|
||||
v_s1 += vget_lane_u32(v_sum12, 0);
|
||||
v_s2 += vget_lane_u32(v_sum12, 1);
|
||||
v_tail_index = (((uint64_t)(a_x.len)) & 18446744073709551584u);
|
||||
if (v_tail_index < ((uint64_t)(a_x.len))) {
|
||||
{
|
||||
@@ -31732,13 +31732,13 @@ wuffs_png__decoder__filter_4_distance_3_arm_neon(
|
||||
uint8x8_t v_fb = {0};
|
||||
uint8x8_t v_fc = {0};
|
||||
uint8x8_t v_fx = {0};
|
||||
uint8x16_t v_fafb = {0};
|
||||
uint8x16_t v_fcfc = {0};
|
||||
uint8x16_t v_pa = {0};
|
||||
uint8x16_t v_pb = {0};
|
||||
uint8x16_t v_pc = {0};
|
||||
uint8x16_t v_cmpab = {0};
|
||||
uint8x16_t v_cmpac = {0};
|
||||
uint16x8_t v_fafb = {0};
|
||||
uint16x8_t v_fcfc = {0};
|
||||
uint16x8_t v_pa = {0};
|
||||
uint16x8_t v_pb = {0};
|
||||
uint16x8_t v_pc = {0};
|
||||
uint16x8_t v_cmpab = {0};
|
||||
uint16x8_t v_cmpac = {0};
|
||||
uint8x8_t v_picka = {0};
|
||||
uint8x8_t v_pickb = {0};
|
||||
|
||||
@@ -31754,15 +31754,15 @@ wuffs_png__decoder__filter_4_distance_3_arm_neon(
|
||||
while (v_curr.ptr < i_end0_curr) {
|
||||
v_fb = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u32le__no_bounds_check(v_prev.ptr)));
|
||||
v_fx = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u32le__no_bounds_check(v_curr.ptr)));
|
||||
v_fafb = vreinterpretq_u8_u16(vaddl_u8(v_fa, v_fb));
|
||||
v_fcfc = vreinterpretq_u8_u16(vaddl_u8(v_fc, v_fc));
|
||||
v_pa = vreinterpretq_u8_u16(vabdl_u8(v_fb, v_fc));
|
||||
v_pb = vreinterpretq_u8_u16(vabdl_u8(v_fa, v_fc));
|
||||
v_pc = vreinterpretq_u8_u16(vabdq_u16(vreinterpretq_u16_u8(v_fafb), vreinterpretq_u16_u8(v_fcfc)));
|
||||
v_cmpab = vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pa), vreinterpretq_u16_u8(v_pb)));
|
||||
v_cmpac = vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pa), vreinterpretq_u16_u8(v_pc)));
|
||||
v_picka = vmovn_u16(vreinterpretq_u16_u8(vandq_u8(v_cmpab, v_cmpac)));
|
||||
v_pickb = vmovn_u16(vreinterpretq_u16_u8(vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pb), vreinterpretq_u16_u8(v_pc)))));
|
||||
v_fafb = vaddl_u8(v_fa, v_fb);
|
||||
v_fcfc = vaddl_u8(v_fc, v_fc);
|
||||
v_pa = vabdl_u8(v_fb, v_fc);
|
||||
v_pb = vabdl_u8(v_fa, v_fc);
|
||||
v_pc = vabdq_u16(v_fafb, v_fcfc);
|
||||
v_cmpab = vcleq_u16(v_pa, v_pb);
|
||||
v_cmpac = vcleq_u16(v_pa, v_pc);
|
||||
v_picka = vmovn_u16(vandq_u16(v_cmpab, v_cmpac));
|
||||
v_pickb = vmovn_u16(vcleq_u16(v_pb, v_pc));
|
||||
v_fx = vadd_u8(v_fx, vbsl_u8(v_picka, v_fa, vbsl_u8(v_pickb, v_fb, v_fc)));
|
||||
wuffs_base__poke_u24le__no_bounds_check(v_curr.ptr, vget_lane_u32(vreinterpret_u32_u8(v_fx), 0));
|
||||
v_fc = v_fb;
|
||||
@@ -31771,15 +31771,15 @@ wuffs_png__decoder__filter_4_distance_3_arm_neon(
|
||||
v_prev.ptr += 3;
|
||||
v_fb = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u32le__no_bounds_check(v_prev.ptr)));
|
||||
v_fx = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u32le__no_bounds_check(v_curr.ptr)));
|
||||
v_fafb = vreinterpretq_u8_u16(vaddl_u8(v_fa, v_fb));
|
||||
v_fcfc = vreinterpretq_u8_u16(vaddl_u8(v_fc, v_fc));
|
||||
v_pa = vreinterpretq_u8_u16(vabdl_u8(v_fb, v_fc));
|
||||
v_pb = vreinterpretq_u8_u16(vabdl_u8(v_fa, v_fc));
|
||||
v_pc = vreinterpretq_u8_u16(vabdq_u16(vreinterpretq_u16_u8(v_fafb), vreinterpretq_u16_u8(v_fcfc)));
|
||||
v_cmpab = vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pa), vreinterpretq_u16_u8(v_pb)));
|
||||
v_cmpac = vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pa), vreinterpretq_u16_u8(v_pc)));
|
||||
v_picka = vmovn_u16(vreinterpretq_u16_u8(vandq_u8(v_cmpab, v_cmpac)));
|
||||
v_pickb = vmovn_u16(vreinterpretq_u16_u8(vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pb), vreinterpretq_u16_u8(v_pc)))));
|
||||
v_fafb = vaddl_u8(v_fa, v_fb);
|
||||
v_fcfc = vaddl_u8(v_fc, v_fc);
|
||||
v_pa = vabdl_u8(v_fb, v_fc);
|
||||
v_pb = vabdl_u8(v_fa, v_fc);
|
||||
v_pc = vabdq_u16(v_fafb, v_fcfc);
|
||||
v_cmpab = vcleq_u16(v_pa, v_pb);
|
||||
v_cmpac = vcleq_u16(v_pa, v_pc);
|
||||
v_picka = vmovn_u16(vandq_u16(v_cmpab, v_cmpac));
|
||||
v_pickb = vmovn_u16(vcleq_u16(v_pb, v_pc));
|
||||
v_fx = vadd_u8(v_fx, vbsl_u8(v_picka, v_fa, vbsl_u8(v_pickb, v_fb, v_fc)));
|
||||
wuffs_base__poke_u24le__no_bounds_check(v_curr.ptr, vget_lane_u32(vreinterpret_u32_u8(v_fx), 0));
|
||||
v_fc = v_fb;
|
||||
@@ -31793,15 +31793,15 @@ wuffs_png__decoder__filter_4_distance_3_arm_neon(
|
||||
while (v_curr.ptr < i_end1_curr) {
|
||||
v_fb = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u32le__no_bounds_check(v_prev.ptr)));
|
||||
v_fx = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u32le__no_bounds_check(v_curr.ptr)));
|
||||
v_fafb = vreinterpretq_u8_u16(vaddl_u8(v_fa, v_fb));
|
||||
v_fcfc = vreinterpretq_u8_u16(vaddl_u8(v_fc, v_fc));
|
||||
v_pa = vreinterpretq_u8_u16(vabdl_u8(v_fb, v_fc));
|
||||
v_pb = vreinterpretq_u8_u16(vabdl_u8(v_fa, v_fc));
|
||||
v_pc = vreinterpretq_u8_u16(vabdq_u16(vreinterpretq_u16_u8(v_fafb), vreinterpretq_u16_u8(v_fcfc)));
|
||||
v_cmpab = vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pa), vreinterpretq_u16_u8(v_pb)));
|
||||
v_cmpac = vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pa), vreinterpretq_u16_u8(v_pc)));
|
||||
v_picka = vmovn_u16(vreinterpretq_u16_u8(vandq_u8(v_cmpab, v_cmpac)));
|
||||
v_pickb = vmovn_u16(vreinterpretq_u16_u8(vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pb), vreinterpretq_u16_u8(v_pc)))));
|
||||
v_fafb = vaddl_u8(v_fa, v_fb);
|
||||
v_fcfc = vaddl_u8(v_fc, v_fc);
|
||||
v_pa = vabdl_u8(v_fb, v_fc);
|
||||
v_pb = vabdl_u8(v_fa, v_fc);
|
||||
v_pc = vabdq_u16(v_fafb, v_fcfc);
|
||||
v_cmpab = vcleq_u16(v_pa, v_pb);
|
||||
v_cmpac = vcleq_u16(v_pa, v_pc);
|
||||
v_picka = vmovn_u16(vandq_u16(v_cmpab, v_cmpac));
|
||||
v_pickb = vmovn_u16(vcleq_u16(v_pb, v_pc));
|
||||
v_fx = vadd_u8(v_fx, vbsl_u8(v_picka, v_fa, vbsl_u8(v_pickb, v_fb, v_fc)));
|
||||
wuffs_base__poke_u24le__no_bounds_check(v_curr.ptr, vget_lane_u32(vreinterpret_u32_u8(v_fx), 0));
|
||||
v_fc = v_fb;
|
||||
@@ -31815,15 +31815,15 @@ wuffs_png__decoder__filter_4_distance_3_arm_neon(
|
||||
while (v_curr.ptr < i_end2_curr) {
|
||||
v_fb = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u24le__no_bounds_check(v_prev.ptr)));
|
||||
v_fx = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u24le__no_bounds_check(v_curr.ptr)));
|
||||
v_fafb = vreinterpretq_u8_u16(vaddl_u8(v_fa, v_fb));
|
||||
v_fcfc = vreinterpretq_u8_u16(vaddl_u8(v_fc, v_fc));
|
||||
v_pa = vreinterpretq_u8_u16(vabdl_u8(v_fb, v_fc));
|
||||
v_pb = vreinterpretq_u8_u16(vabdl_u8(v_fa, v_fc));
|
||||
v_pc = vreinterpretq_u8_u16(vabdq_u16(vreinterpretq_u16_u8(v_fafb), vreinterpretq_u16_u8(v_fcfc)));
|
||||
v_cmpab = vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pa), vreinterpretq_u16_u8(v_pb)));
|
||||
v_cmpac = vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pa), vreinterpretq_u16_u8(v_pc)));
|
||||
v_picka = vmovn_u16(vreinterpretq_u16_u8(vandq_u8(v_cmpab, v_cmpac)));
|
||||
v_pickb = vmovn_u16(vreinterpretq_u16_u8(vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pb), vreinterpretq_u16_u8(v_pc)))));
|
||||
v_fafb = vaddl_u8(v_fa, v_fb);
|
||||
v_fcfc = vaddl_u8(v_fc, v_fc);
|
||||
v_pa = vabdl_u8(v_fb, v_fc);
|
||||
v_pb = vabdl_u8(v_fa, v_fc);
|
||||
v_pc = vabdq_u16(v_fafb, v_fcfc);
|
||||
v_cmpab = vcleq_u16(v_pa, v_pb);
|
||||
v_cmpac = vcleq_u16(v_pa, v_pc);
|
||||
v_picka = vmovn_u16(vandq_u16(v_cmpab, v_cmpac));
|
||||
v_pickb = vmovn_u16(vcleq_u16(v_pb, v_pc));
|
||||
v_fx = vadd_u8(v_fx, vbsl_u8(v_picka, v_fa, vbsl_u8(v_pickb, v_fb, v_fc)));
|
||||
wuffs_base__poke_u24le__no_bounds_check(v_curr.ptr, vget_lane_u32(vreinterpret_u32_u8(v_fx), 0));
|
||||
v_curr.ptr += 3;
|
||||
@@ -31850,13 +31850,13 @@ wuffs_png__decoder__filter_4_distance_4_arm_neon(
|
||||
uint8x8_t v_fb = {0};
|
||||
uint8x8_t v_fc = {0};
|
||||
uint8x8_t v_fx = {0};
|
||||
uint8x16_t v_fafb = {0};
|
||||
uint8x16_t v_fcfc = {0};
|
||||
uint8x16_t v_pa = {0};
|
||||
uint8x16_t v_pb = {0};
|
||||
uint8x16_t v_pc = {0};
|
||||
uint8x16_t v_cmpab = {0};
|
||||
uint8x16_t v_cmpac = {0};
|
||||
uint16x8_t v_fafb = {0};
|
||||
uint16x8_t v_fcfc = {0};
|
||||
uint16x8_t v_pa = {0};
|
||||
uint16x8_t v_pb = {0};
|
||||
uint16x8_t v_pc = {0};
|
||||
uint16x8_t v_cmpab = {0};
|
||||
uint16x8_t v_cmpac = {0};
|
||||
uint8x8_t v_picka = {0};
|
||||
uint8x8_t v_pickb = {0};
|
||||
|
||||
@@ -31872,15 +31872,15 @@ wuffs_png__decoder__filter_4_distance_4_arm_neon(
|
||||
while (v_curr.ptr < i_end0_curr) {
|
||||
v_fb = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u32le__no_bounds_check(v_prev.ptr)));
|
||||
v_fx = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u32le__no_bounds_check(v_curr.ptr)));
|
||||
v_fafb = vreinterpretq_u8_u16(vaddl_u8(v_fa, v_fb));
|
||||
v_fcfc = vreinterpretq_u8_u16(vaddl_u8(v_fc, v_fc));
|
||||
v_pa = vreinterpretq_u8_u16(vabdl_u8(v_fb, v_fc));
|
||||
v_pb = vreinterpretq_u8_u16(vabdl_u8(v_fa, v_fc));
|
||||
v_pc = vreinterpretq_u8_u16(vabdq_u16(vreinterpretq_u16_u8(v_fafb), vreinterpretq_u16_u8(v_fcfc)));
|
||||
v_cmpab = vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pa), vreinterpretq_u16_u8(v_pb)));
|
||||
v_cmpac = vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pa), vreinterpretq_u16_u8(v_pc)));
|
||||
v_picka = vmovn_u16(vreinterpretq_u16_u8(vandq_u8(v_cmpab, v_cmpac)));
|
||||
v_pickb = vmovn_u16(vreinterpretq_u16_u8(vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pb), vreinterpretq_u16_u8(v_pc)))));
|
||||
v_fafb = vaddl_u8(v_fa, v_fb);
|
||||
v_fcfc = vaddl_u8(v_fc, v_fc);
|
||||
v_pa = vabdl_u8(v_fb, v_fc);
|
||||
v_pb = vabdl_u8(v_fa, v_fc);
|
||||
v_pc = vabdq_u16(v_fafb, v_fcfc);
|
||||
v_cmpab = vcleq_u16(v_pa, v_pb);
|
||||
v_cmpac = vcleq_u16(v_pa, v_pc);
|
||||
v_picka = vmovn_u16(vandq_u16(v_cmpab, v_cmpac));
|
||||
v_pickb = vmovn_u16(vcleq_u16(v_pb, v_pc));
|
||||
v_fx = vadd_u8(v_fx, vbsl_u8(v_picka, v_fa, vbsl_u8(v_pickb, v_fb, v_fc)));
|
||||
wuffs_base__poke_u32le__no_bounds_check(v_curr.ptr, vget_lane_u32(vreinterpret_u32_u8(v_fx), 0));
|
||||
v_fc = v_fb;
|
||||
@@ -31889,15 +31889,15 @@ wuffs_png__decoder__filter_4_distance_4_arm_neon(
|
||||
v_prev.ptr += 4;
|
||||
v_fb = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u32le__no_bounds_check(v_prev.ptr)));
|
||||
v_fx = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u32le__no_bounds_check(v_curr.ptr)));
|
||||
v_fafb = vreinterpretq_u8_u16(vaddl_u8(v_fa, v_fb));
|
||||
v_fcfc = vreinterpretq_u8_u16(vaddl_u8(v_fc, v_fc));
|
||||
v_pa = vreinterpretq_u8_u16(vabdl_u8(v_fb, v_fc));
|
||||
v_pb = vreinterpretq_u8_u16(vabdl_u8(v_fa, v_fc));
|
||||
v_pc = vreinterpretq_u8_u16(vabdq_u16(vreinterpretq_u16_u8(v_fafb), vreinterpretq_u16_u8(v_fcfc)));
|
||||
v_cmpab = vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pa), vreinterpretq_u16_u8(v_pb)));
|
||||
v_cmpac = vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pa), vreinterpretq_u16_u8(v_pc)));
|
||||
v_picka = vmovn_u16(vreinterpretq_u16_u8(vandq_u8(v_cmpab, v_cmpac)));
|
||||
v_pickb = vmovn_u16(vreinterpretq_u16_u8(vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pb), vreinterpretq_u16_u8(v_pc)))));
|
||||
v_fafb = vaddl_u8(v_fa, v_fb);
|
||||
v_fcfc = vaddl_u8(v_fc, v_fc);
|
||||
v_pa = vabdl_u8(v_fb, v_fc);
|
||||
v_pb = vabdl_u8(v_fa, v_fc);
|
||||
v_pc = vabdq_u16(v_fafb, v_fcfc);
|
||||
v_cmpab = vcleq_u16(v_pa, v_pb);
|
||||
v_cmpac = vcleq_u16(v_pa, v_pc);
|
||||
v_picka = vmovn_u16(vandq_u16(v_cmpab, v_cmpac));
|
||||
v_pickb = vmovn_u16(vcleq_u16(v_pb, v_pc));
|
||||
v_fx = vadd_u8(v_fx, vbsl_u8(v_picka, v_fa, vbsl_u8(v_pickb, v_fb, v_fc)));
|
||||
wuffs_base__poke_u32le__no_bounds_check(v_curr.ptr, vget_lane_u32(vreinterpret_u32_u8(v_fx), 0));
|
||||
v_fc = v_fb;
|
||||
@@ -31911,15 +31911,15 @@ wuffs_png__decoder__filter_4_distance_4_arm_neon(
|
||||
while (v_curr.ptr < i_end1_curr) {
|
||||
v_fb = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u32le__no_bounds_check(v_prev.ptr)));
|
||||
v_fx = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u32le__no_bounds_check(v_curr.ptr)));
|
||||
v_fafb = vreinterpretq_u8_u16(vaddl_u8(v_fa, v_fb));
|
||||
v_fcfc = vreinterpretq_u8_u16(vaddl_u8(v_fc, v_fc));
|
||||
v_pa = vreinterpretq_u8_u16(vabdl_u8(v_fb, v_fc));
|
||||
v_pb = vreinterpretq_u8_u16(vabdl_u8(v_fa, v_fc));
|
||||
v_pc = vreinterpretq_u8_u16(vabdq_u16(vreinterpretq_u16_u8(v_fafb), vreinterpretq_u16_u8(v_fcfc)));
|
||||
v_cmpab = vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pa), vreinterpretq_u16_u8(v_pb)));
|
||||
v_cmpac = vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pa), vreinterpretq_u16_u8(v_pc)));
|
||||
v_picka = vmovn_u16(vreinterpretq_u16_u8(vandq_u8(v_cmpab, v_cmpac)));
|
||||
v_pickb = vmovn_u16(vreinterpretq_u16_u8(vreinterpretq_u8_u16(vcleq_u16(vreinterpretq_u16_u8(v_pb), vreinterpretq_u16_u8(v_pc)))));
|
||||
v_fafb = vaddl_u8(v_fa, v_fb);
|
||||
v_fcfc = vaddl_u8(v_fc, v_fc);
|
||||
v_pa = vabdl_u8(v_fb, v_fc);
|
||||
v_pb = vabdl_u8(v_fa, v_fc);
|
||||
v_pc = vabdq_u16(v_fafb, v_fcfc);
|
||||
v_cmpab = vcleq_u16(v_pa, v_pb);
|
||||
v_cmpac = vcleq_u16(v_pa, v_pc);
|
||||
v_picka = vmovn_u16(vandq_u16(v_cmpab, v_cmpac));
|
||||
v_pickb = vmovn_u16(vcleq_u16(v_pb, v_pc));
|
||||
v_fx = vadd_u8(v_fx, vbsl_u8(v_picka, v_fa, vbsl_u8(v_pickb, v_fb, v_fc)));
|
||||
wuffs_base__poke_u32le__no_bounds_check(v_curr.ptr, vget_lane_u32(vreinterpret_u32_u8(v_fx), 0));
|
||||
v_fc = v_fb;
|
||||
|
||||
@@ -23,17 +23,18 @@ pri func hasher.up_arm_neon!(x: slice base.u8),
|
||||
|
||||
// The remaining variables are specific to the SIMD version.
|
||||
|
||||
var p__left : base.arm_neon_128
|
||||
var p_right : base.arm_neon_128
|
||||
var v1 : base.arm_neon_128
|
||||
var v2 : base.arm_neon_128
|
||||
var col0 : base.arm_neon_128
|
||||
var col1 : base.arm_neon_128
|
||||
var col2 : base.arm_neon_128
|
||||
var col3 : base.arm_neon_128
|
||||
var sum1 : base.arm_neon_64
|
||||
var sum2 : base.arm_neon_64
|
||||
var sum12 : base.arm_neon_64
|
||||
var util : base.arm_neon_utility
|
||||
var p__left : base.arm_neon_u8x16
|
||||
var p_right : base.arm_neon_u8x16
|
||||
var v1 : base.arm_neon_u32x4
|
||||
var v2 : base.arm_neon_u32x4
|
||||
var col0 : base.arm_neon_u16x8
|
||||
var col1 : base.arm_neon_u16x8
|
||||
var col2 : base.arm_neon_u16x8
|
||||
var col3 : base.arm_neon_u16x8
|
||||
var sum1 : base.arm_neon_u32x2
|
||||
var sum2 : base.arm_neon_u32x2
|
||||
var sum12 : base.arm_neon_u32x2
|
||||
|
||||
var num_iterate_bytes : base.u32
|
||||
var tail_index : base.u64
|
||||
@@ -79,12 +80,12 @@ pri func hasher.up_arm_neon!(x: slice base.u8),
|
||||
// variables s1 and s2. The iterate loop accumulates four parallel u32
|
||||
// sums in each vector. A post-iterate step merges the four u32 sums
|
||||
// into a single u32 sum.
|
||||
v1 = v1.create_vdupq_n_u32(a: 0)
|
||||
v2 = v2.create_vdupq_n_u32(a: 0)
|
||||
col0 = col0.create_vdupq_n_u16(a: 0)
|
||||
col1 = col1.create_vdupq_n_u16(a: 0)
|
||||
col2 = col2.create_vdupq_n_u16(a: 0)
|
||||
col3 = col3.create_vdupq_n_u16(a: 0)
|
||||
v1 = util.make_u32x4_repeat(a: 0)
|
||||
v2 = util.make_u32x4_repeat(a: 0)
|
||||
col0 = util.make_u16x8_repeat(a: 0)
|
||||
col1 = util.make_u16x8_repeat(a: 0)
|
||||
col2 = util.make_u16x8_repeat(a: 0)
|
||||
col3 = util.make_u16x8_repeat(a: 0)
|
||||
|
||||
// The inner loop.
|
||||
iterate (p = args.x)(length: 32, advance: 32, unroll: 1) {
|
||||
@@ -93,8 +94,8 @@ pri func hasher.up_arm_neon!(x: slice base.u8),
|
||||
//
|
||||
// Let p__left = [u8×16: p00, p01, p02, ..., p15]
|
||||
// Let p_right = [u8×16: p16, p17, p18, ..., p31]
|
||||
p__left = p__left.create_slice128(a: p[.. 16])
|
||||
p_right = p_right.create_slice128(a: p[16 .. 32])
|
||||
p__left = util.make_u8x16_slice128(a: p[.. 16])
|
||||
p_right = util.make_u8x16_slice128(a: p[16 .. 32])
|
||||
|
||||
// For v2j, we need to calculate the sums of the s1j terms for each
|
||||
// of p's 32 elements. This is simply 32 times the same number,
|
||||
@@ -141,28 +142,28 @@ pri func hasher.up_arm_neon!(x: slice base.u8),
|
||||
// [u32×4: 0x04 * (Σ p28), 0x03 * (Σ p29), ..., 0x01 * (Σ p31)]
|
||||
v2 = v2.vmlal_u16(
|
||||
b: col0.vget_low_u16(),
|
||||
c: sum2.create_literal_u16x4(a0: 0x20, a1: 0x1F, a2: 0x1E, a3: 0x1D))
|
||||
c: util.make_u16x4_multiple(a00: 0x20, a01: 0x1F, a02: 0x1E, a03: 0x1D))
|
||||
v2 = v2.vmlal_u16(
|
||||
b: col0.vget_high_u16(),
|
||||
c: sum2.create_literal_u16x4(a0: 0x1C, a1: 0x1B, a2: 0x1A, a3: 0x19))
|
||||
c: util.make_u16x4_multiple(a00: 0x1C, a01: 0x1B, a02: 0x1A, a03: 0x19))
|
||||
v2 = v2.vmlal_u16(
|
||||
b: col1.vget_low_u16(),
|
||||
c: sum2.create_literal_u16x4(a0: 0x18, a1: 0x17, a2: 0x16, a3: 0x15))
|
||||
c: util.make_u16x4_multiple(a00: 0x18, a01: 0x17, a02: 0x16, a03: 0x15))
|
||||
v2 = v2.vmlal_u16(
|
||||
b: col1.vget_high_u16(),
|
||||
c: sum2.create_literal_u16x4(a0: 0x14, a1: 0x13, a2: 0x12, a3: 0x11))
|
||||
c: util.make_u16x4_multiple(a00: 0x14, a01: 0x13, a02: 0x12, a03: 0x11))
|
||||
v2 = v2.vmlal_u16(
|
||||
b: col2.vget_low_u16(),
|
||||
c: sum2.create_literal_u16x4(a0: 0x10, a1: 0x0F, a2: 0x0E, a3: 0x0D))
|
||||
c: util.make_u16x4_multiple(a00: 0x10, a01: 0x0F, a02: 0x0E, a03: 0x0D))
|
||||
v2 = v2.vmlal_u16(
|
||||
b: col2.vget_high_u16(),
|
||||
c: sum2.create_literal_u16x4(a0: 0x0C, a1: 0x0B, a2: 0x0A, a3: 0x09))
|
||||
c: util.make_u16x4_multiple(a00: 0x0C, a01: 0x0B, a02: 0x0A, a03: 0x09))
|
||||
v2 = v2.vmlal_u16(
|
||||
b: col3.vget_low_u16(),
|
||||
c: sum2.create_literal_u16x4(a0: 0x08, a1: 0x07, a2: 0x06, a3: 0x05))
|
||||
c: util.make_u16x4_multiple(a00: 0x08, a01: 0x07, a02: 0x06, a03: 0x05))
|
||||
v2 = v2.vmlal_u16(
|
||||
b: col3.vget_high_u16(),
|
||||
c: sum2.create_literal_u16x4(a0: 0x04, a1: 0x03, a2: 0x02, a3: 0x01))
|
||||
c: util.make_u16x4_multiple(a00: 0x04, a01: 0x03, a02: 0x02, a03: 0x01))
|
||||
|
||||
// Merge the four parallel u32 sums (v1) into the single u32 sum (s1)
|
||||
// and ditto for v2. Starting with [u32×4: xx_0, xx_1, xx_2, xx_3]:
|
||||
|
||||
@@ -55,24 +55,25 @@ pri func decoder.filter_3_distance_4_arm_neon!(curr: slice base.u8, prev: slice
|
||||
var curr : slice base.u8
|
||||
var prev : slice base.u8
|
||||
|
||||
var fa : base.arm_neon_64
|
||||
var fb : base.arm_neon_64
|
||||
var fx : base.arm_neon_64
|
||||
var util : base.arm_neon_utility
|
||||
var fa : base.arm_neon_u8x8
|
||||
var fb : base.arm_neon_u8x8
|
||||
var fx : base.arm_neon_u8x8
|
||||
|
||||
if args.prev.length() == 0 {
|
||||
iterate (curr = args.curr)(length: 4, advance: 4, unroll: 2) {
|
||||
fx = fx.create_vdup_n_u32(a: curr.peek_u32le())
|
||||
fx = util.make_u32x2_repeat(a: curr.peek_u32le()).as_u8x8()
|
||||
fx = fx.vadd_u8(b: fa.vhadd_u8(b: fb))
|
||||
curr.poke_u32le!(a: fx.vget_lane_u32(b: 0))
|
||||
curr.poke_u32le!(a: fx.as_u32x2().vget_lane_u32(b: 0))
|
||||
fa = fx
|
||||
}
|
||||
|
||||
} else {
|
||||
iterate (curr = args.curr, prev = args.prev)(length: 4, advance: 4, unroll: 2) {
|
||||
fb = fb.create_vdup_n_u32(a: prev.peek_u32le())
|
||||
fx = fx.create_vdup_n_u32(a: curr.peek_u32le())
|
||||
fb = util.make_u32x2_repeat(a: prev.peek_u32le()).as_u8x8()
|
||||
fx = util.make_u32x2_repeat(a: curr.peek_u32le()).as_u8x8()
|
||||
fx = fx.vadd_u8(b: fa.vhadd_u8(b: fb))
|
||||
curr.poke_u32le!(a: fx.vget_lane_u32(b: 0))
|
||||
curr.poke_u32le!(a: fx.as_u32x2().vget_lane_u32(b: 0))
|
||||
fa = fx
|
||||
}
|
||||
}
|
||||
@@ -95,24 +96,25 @@ pri func decoder.filter_4_distance_3_arm_neon!(curr: slice base.u8, prev: slice
|
||||
var curr : slice base.u8
|
||||
var prev : slice base.u8
|
||||
|
||||
var fa : base.arm_neon_64
|
||||
var fb : base.arm_neon_64
|
||||
var fc : base.arm_neon_64
|
||||
var fx : base.arm_neon_64
|
||||
var fafb : base.arm_neon_128
|
||||
var fcfc : base.arm_neon_128
|
||||
var pa : base.arm_neon_128
|
||||
var pb : base.arm_neon_128
|
||||
var pc : base.arm_neon_128
|
||||
var cmpab : base.arm_neon_128
|
||||
var cmpac : base.arm_neon_128
|
||||
var picka : base.arm_neon_64
|
||||
var pickb : base.arm_neon_64
|
||||
var util : base.arm_neon_utility
|
||||
var fa : base.arm_neon_u8x8
|
||||
var fb : base.arm_neon_u8x8
|
||||
var fc : base.arm_neon_u8x8
|
||||
var fx : base.arm_neon_u8x8
|
||||
var fafb : base.arm_neon_u16x8
|
||||
var fcfc : base.arm_neon_u16x8
|
||||
var pa : base.arm_neon_u16x8
|
||||
var pb : base.arm_neon_u16x8
|
||||
var pc : base.arm_neon_u16x8
|
||||
var cmpab : base.arm_neon_u16x8
|
||||
var cmpac : base.arm_neon_u16x8
|
||||
var picka : base.arm_neon_u8x8
|
||||
var pickb : base.arm_neon_u8x8
|
||||
|
||||
// § The advance is 3, not 4.
|
||||
iterate (curr = args.curr, prev = args.prev)(length: 4, advance: 3, unroll: 2) {
|
||||
fb = fb.create_vdup_n_u32(a: prev.peek_u32le())
|
||||
fx = fx.create_vdup_n_u32(a: curr.peek_u32le())
|
||||
fb = util.make_u32x2_repeat(a: prev.peek_u32le()).as_u8x8()
|
||||
fx = util.make_u32x2_repeat(a: curr.peek_u32le()).as_u8x8()
|
||||
fafb = fa.vaddl_u8(b: fb)
|
||||
fcfc = fc.vaddl_u8(b: fc)
|
||||
pa = fb.vabdl_u8(b: fc)
|
||||
@@ -120,20 +122,20 @@ pri func decoder.filter_4_distance_3_arm_neon!(curr: slice base.u8, prev: slice
|
||||
pc = fafb.vabdq_u16(b: fcfc)
|
||||
cmpab = pa.vcleq_u16(b: pb)
|
||||
cmpac = pa.vcleq_u16(b: pc)
|
||||
picka = cmpab.vandq_u8(b: cmpac).vmovn_u16()
|
||||
picka = cmpab.vandq_u16(b: cmpac).vmovn_u16()
|
||||
pickb = pb.vcleq_u16(b: pc).vmovn_u16()
|
||||
fx = fx.vadd_u8(
|
||||
b: picka.vbsl_u8(b: fa,
|
||||
c: pickb.vbsl_u8(b: fb, c: fc)))
|
||||
// § poke_u24le replaces poke_u32le.
|
||||
curr.poke_u24le!(a: fx.vget_lane_u32(b: 0))
|
||||
curr.poke_u24le!(a: fx.as_u32x2().vget_lane_u32(b: 0))
|
||||
fc = fb
|
||||
fa = fx
|
||||
} else (length: 3, advance: 3, unroll: 1) {
|
||||
// § peek_u24le_as_u32 replaces peek_u32le.
|
||||
fb = fb.create_vdup_n_u32(a: prev.peek_u24le_as_u32())
|
||||
fb = util.make_u32x2_repeat(a: prev.peek_u24le_as_u32()).as_u8x8()
|
||||
// § peek_u24le_as_u32 replaces peek_u32le.
|
||||
fx = fx.create_vdup_n_u32(a: curr.peek_u24le_as_u32())
|
||||
fx = util.make_u32x2_repeat(a: curr.peek_u24le_as_u32()).as_u8x8()
|
||||
fafb = fa.vaddl_u8(b: fb)
|
||||
fcfc = fc.vaddl_u8(b: fc)
|
||||
pa = fb.vabdl_u8(b: fc)
|
||||
@@ -141,13 +143,13 @@ pri func decoder.filter_4_distance_3_arm_neon!(curr: slice base.u8, prev: slice
|
||||
pc = fafb.vabdq_u16(b: fcfc)
|
||||
cmpab = pa.vcleq_u16(b: pb)
|
||||
cmpac = pa.vcleq_u16(b: pc)
|
||||
picka = cmpab.vandq_u8(b: cmpac).vmovn_u16()
|
||||
picka = cmpab.vandq_u16(b: cmpac).vmovn_u16()
|
||||
pickb = pb.vcleq_u16(b: pc).vmovn_u16()
|
||||
fx = fx.vadd_u8(
|
||||
b: picka.vbsl_u8(b: fa,
|
||||
c: pickb.vbsl_u8(b: fb, c: fc)))
|
||||
// § poke_u24le replaces poke_u32le.
|
||||
curr.poke_u24le!(a: fx.vget_lane_u32(b: 0))
|
||||
curr.poke_u24le!(a: fx.as_u32x2().vget_lane_u32(b: 0))
|
||||
// § These assignments are unnecessary; this is the last iteration.
|
||||
// fc = fb
|
||||
// fa = fx
|
||||
@@ -160,23 +162,24 @@ pri func decoder.filter_4_distance_4_arm_neon!(curr: slice base.u8, prev: slice
|
||||
var curr : slice base.u8
|
||||
var prev : slice base.u8
|
||||
|
||||
var fa : base.arm_neon_64
|
||||
var fb : base.arm_neon_64
|
||||
var fc : base.arm_neon_64
|
||||
var fx : base.arm_neon_64
|
||||
var fafb : base.arm_neon_128
|
||||
var fcfc : base.arm_neon_128
|
||||
var pa : base.arm_neon_128
|
||||
var pb : base.arm_neon_128
|
||||
var pc : base.arm_neon_128
|
||||
var cmpab : base.arm_neon_128
|
||||
var cmpac : base.arm_neon_128
|
||||
var picka : base.arm_neon_64
|
||||
var pickb : base.arm_neon_64
|
||||
var util : base.arm_neon_utility
|
||||
var fa : base.arm_neon_u8x8
|
||||
var fb : base.arm_neon_u8x8
|
||||
var fc : base.arm_neon_u8x8
|
||||
var fx : base.arm_neon_u8x8
|
||||
var fafb : base.arm_neon_u16x8
|
||||
var fcfc : base.arm_neon_u16x8
|
||||
var pa : base.arm_neon_u16x8
|
||||
var pb : base.arm_neon_u16x8
|
||||
var pc : base.arm_neon_u16x8
|
||||
var cmpab : base.arm_neon_u16x8
|
||||
var cmpac : base.arm_neon_u16x8
|
||||
var picka : base.arm_neon_u8x8
|
||||
var pickb : base.arm_neon_u8x8
|
||||
|
||||
iterate (curr = args.curr, prev = args.prev)(length: 4, advance: 4, unroll: 2) {
|
||||
fb = fb.create_vdup_n_u32(a: prev.peek_u32le())
|
||||
fx = fx.create_vdup_n_u32(a: curr.peek_u32le())
|
||||
fb = util.make_u32x2_repeat(a: prev.peek_u32le()).as_u8x8()
|
||||
fx = util.make_u32x2_repeat(a: curr.peek_u32le()).as_u8x8()
|
||||
|
||||
fafb = fa.vaddl_u8(b: fb) // fafb = (fa + fb)
|
||||
fcfc = fc.vaddl_u8(b: fc) // fcfc = (fc + fc)
|
||||
@@ -188,15 +191,15 @@ pri func decoder.filter_4_distance_4_arm_neon!(curr: slice base.u8, prev: slice
|
||||
cmpab = pa.vcleq_u16(b: pb) // cmpab = (pa <= pb)
|
||||
cmpac = pa.vcleq_u16(b: pc) // cmpac = (pa <= pc)
|
||||
|
||||
picka = cmpab.vandq_u8(b: cmpac).vmovn_u16() // picka = ((pa <= pb) && (pa <= pc))
|
||||
pickb = pb.vcleq_u16(b: pc).vmovn_u16() // pickb = (pb <= pc)
|
||||
picka = cmpab.vandq_u16(b: cmpac).vmovn_u16() // picka = ((pa <= pb) && (pa <= pc))
|
||||
pickb = pb.vcleq_u16(b: pc).vmovn_u16() // pickb = (pb <= pc)
|
||||
|
||||
// Add the predictor to the residual.
|
||||
fx = fx.vadd_u8(
|
||||
b: picka.vbsl_u8(b: fa,
|
||||
c: pickb.vbsl_u8(b: fb, c: fc)))
|
||||
|
||||
curr.poke_u32le!(a: fx.vget_lane_u32(b: 0))
|
||||
curr.poke_u32le!(a: fx.as_u32x2().vget_lane_u32(b: 0))
|
||||
fc = fb
|
||||
fa = fx
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user