Use arm_neon_uAxB types and methods

This commit is contained in:
Nigel Tao committed 2021-02-24 16:48:09 +11:00
1 parent 1f9b6ea535
commit 3b5d8fdfba
5 files changed
+279 -204

No files matched your search

+28 -27
View File
@@ -23,17 +23,18 @@ pri func hasher.up_arm_neon!(x: slice base.u8),
// The remaining variables are specific to the SIMD version.
var p__left : base.arm_neon_128
var p_right : base.arm_neon_128
var v1 : base.arm_neon_128
var v2 : base.arm_neon_128
var col0 : base.arm_neon_128
var col1 : base.arm_neon_128
var col2 : base.arm_neon_128
var col3 : base.arm_neon_128
var sum1 : base.arm_neon_64
var sum2 : base.arm_neon_64
var sum12 : base.arm_neon_64
var util : base.arm_neon_utility
var p__left : base.arm_neon_u8x16
var p_right : base.arm_neon_u8x16
var v1 : base.arm_neon_u32x4
var v2 : base.arm_neon_u32x4
var col0 : base.arm_neon_u16x8
var col1 : base.arm_neon_u16x8
var col2 : base.arm_neon_u16x8
var col3 : base.arm_neon_u16x8
var sum1 : base.arm_neon_u32x2
var sum2 : base.arm_neon_u32x2
var sum12 : base.arm_neon_u32x2
var num_iterate_bytes : base.u32
var tail_index : base.u64
@@ -79,12 +80,12 @@ pri func hasher.up_arm_neon!(x: slice base.u8),
// variables s1 and s2. The iterate loop accumulates four parallel u32
// sums in each vector. A post-iterate step merges the four u32 sums
// into a single u32 sum.
v1 = v1.create_vdupq_n_u32(a: 0)
v2 = v2.create_vdupq_n_u32(a: 0)
col0 = col0.create_vdupq_n_u16(a: 0)
col1 = col1.create_vdupq_n_u16(a: 0)
col2 = col2.create_vdupq_n_u16(a: 0)
col3 = col3.create_vdupq_n_u16(a: 0)
v1 = util.make_u32x4_repeat(a: 0)
v2 = util.make_u32x4_repeat(a: 0)
col0 = util.make_u16x8_repeat(a: 0)
col1 = util.make_u16x8_repeat(a: 0)
col2 = util.make_u16x8_repeat(a: 0)
col3 = util.make_u16x8_repeat(a: 0)
// The inner loop.
iterate (p = args.x)(length: 32, advance: 32, unroll: 1) {
@@ -93,8 +94,8 @@ pri func hasher.up_arm_neon!(x: slice base.u8),
//
// Let p__left = [u8×16: p00, p01, p02, ..., p15]
// Let p_right = [u8×16: p16, p17, p18, ..., p31]
p__left = p__left.create_slice128(a: p[.. 16])
p_right = p_right.create_slice128(a: p[16 .. 32])
p__left = util.make_u8x16_slice128(a: p[.. 16])
p_right = util.make_u8x16_slice128(a: p[16 .. 32])
// For v2j, we need to calculate the sums of the s1j terms for each
// of p's 32 elements. This is simply 32 times the same number,
@@ -141,28 +142,28 @@ pri func hasher.up_arm_neon!(x: slice base.u8),
// [u32×4: 0x04 * (Σ p28), 0x03 * (Σ p29), ..., 0x01 * (Σ p31)]
v2 = v2.vmlal_u16(
b: col0.vget_low_u16(),
c: sum2.create_literal_u16x4(a0: 0x20, a1: 0x1F, a2: 0x1E, a3: 0x1D))
c: util.make_u16x4_multiple(a00: 0x20, a01: 0x1F, a02: 0x1E, a03: 0x1D))
v2 = v2.vmlal_u16(
b: col0.vget_high_u16(),
c: sum2.create_literal_u16x4(a0: 0x1C, a1: 0x1B, a2: 0x1A, a3: 0x19))
c: util.make_u16x4_multiple(a00: 0x1C, a01: 0x1B, a02: 0x1A, a03: 0x19))
v2 = v2.vmlal_u16(
b: col1.vget_low_u16(),
c: sum2.create_literal_u16x4(a0: 0x18, a1: 0x17, a2: 0x16, a3: 0x15))
c: util.make_u16x4_multiple(a00: 0x18, a01: 0x17, a02: 0x16, a03: 0x15))
v2 = v2.vmlal_u16(
b: col1.vget_high_u16(),
c: sum2.create_literal_u16x4(a0: 0x14, a1: 0x13, a2: 0x12, a3: 0x11))
c: util.make_u16x4_multiple(a00: 0x14, a01: 0x13, a02: 0x12, a03: 0x11))
v2 = v2.vmlal_u16(
b: col2.vget_low_u16(),
c: sum2.create_literal_u16x4(a0: 0x10, a1: 0x0F, a2: 0x0E, a3: 0x0D))
c: util.make_u16x4_multiple(a00: 0x10, a01: 0x0F, a02: 0x0E, a03: 0x0D))
v2 = v2.vmlal_u16(
b: col2.vget_high_u16(),
c: sum2.create_literal_u16x4(a0: 0x0C, a1: 0x0B, a2: 0x0A, a3: 0x09))
c: util.make_u16x4_multiple(a00: 0x0C, a01: 0x0B, a02: 0x0A, a03: 0x09))
v2 = v2.vmlal_u16(
b: col3.vget_low_u16(),
c: sum2.create_literal_u16x4(a0: 0x08, a1: 0x07, a2: 0x06, a3: 0x05))
c: util.make_u16x4_multiple(a00: 0x08, a01: 0x07, a02: 0x06, a03: 0x05))
v2 = v2.vmlal_u16(
b: col3.vget_high_u16(),
c: sum2.create_literal_u16x4(a0: 0x04, a1: 0x03, a2: 0x02, a3: 0x01))
c: util.make_u16x4_multiple(a00: 0x04, a01: 0x03, a02: 0x02, a03: 0x01))
// Merge the four parallel u32 sums (v1) into the single u32 sum (s1)
// and ditto for v2. Starting with [u32×4: xx_0, xx_1, xx_2, xx_3]:
+50 -47
View File
@@ -55,24 +55,25 @@ pri func decoder.filter_3_distance_4_arm_neon!(curr: slice base.u8, prev: slice
var curr : slice base.u8
var prev : slice base.u8
var fa : base.arm_neon_64
var fb : base.arm_neon_64
var fx : base.arm_neon_64
var util : base.arm_neon_utility
var fa : base.arm_neon_u8x8
var fb : base.arm_neon_u8x8
var fx : base.arm_neon_u8x8
if args.prev.length() == 0 {
iterate (curr = args.curr)(length: 4, advance: 4, unroll: 2) {
fx = fx.create_vdup_n_u32(a: curr.peek_u32le())
fx = util.make_u32x2_repeat(a: curr.peek_u32le()).as_u8x8()
fx = fx.vadd_u8(b: fa.vhadd_u8(b: fb))
curr.poke_u32le!(a: fx.vget_lane_u32(b: 0))
curr.poke_u32le!(a: fx.as_u32x2().vget_lane_u32(b: 0))
fa = fx
}
} else {
iterate (curr = args.curr, prev = args.prev)(length: 4, advance: 4, unroll: 2) {
fb = fb.create_vdup_n_u32(a: prev.peek_u32le())
fx = fx.create_vdup_n_u32(a: curr.peek_u32le())
fb = util.make_u32x2_repeat(a: prev.peek_u32le()).as_u8x8()
fx = util.make_u32x2_repeat(a: curr.peek_u32le()).as_u8x8()
fx = fx.vadd_u8(b: fa.vhadd_u8(b: fb))
curr.poke_u32le!(a: fx.vget_lane_u32(b: 0))
curr.poke_u32le!(a: fx.as_u32x2().vget_lane_u32(b: 0))
fa = fx
}
}
@@ -95,24 +96,25 @@ pri func decoder.filter_4_distance_3_arm_neon!(curr: slice base.u8, prev: slice
var curr : slice base.u8
var prev : slice base.u8
var fa : base.arm_neon_64
var fb : base.arm_neon_64
var fc : base.arm_neon_64
var fx : base.arm_neon_64
var fafb : base.arm_neon_128
var fcfc : base.arm_neon_128
var pa : base.arm_neon_128
var pb : base.arm_neon_128
var pc : base.arm_neon_128
var cmpab : base.arm_neon_128
var cmpac : base.arm_neon_128
var picka : base.arm_neon_64
var pickb : base.arm_neon_64
var util : base.arm_neon_utility
var fa : base.arm_neon_u8x8
var fb : base.arm_neon_u8x8
var fc : base.arm_neon_u8x8
var fx : base.arm_neon_u8x8
var fafb : base.arm_neon_u16x8
var fcfc : base.arm_neon_u16x8
var pa : base.arm_neon_u16x8
var pb : base.arm_neon_u16x8
var pc : base.arm_neon_u16x8
var cmpab : base.arm_neon_u16x8
var cmpac : base.arm_neon_u16x8
var picka : base.arm_neon_u8x8
var pickb : base.arm_neon_u8x8
// § The advance is 3, not 4.
iterate (curr = args.curr, prev = args.prev)(length: 4, advance: 3, unroll: 2) {
fb = fb.create_vdup_n_u32(a: prev.peek_u32le())
fx = fx.create_vdup_n_u32(a: curr.peek_u32le())
fb = util.make_u32x2_repeat(a: prev.peek_u32le()).as_u8x8()
fx = util.make_u32x2_repeat(a: curr.peek_u32le()).as_u8x8()
fafb = fa.vaddl_u8(b: fb)
fcfc = fc.vaddl_u8(b: fc)
pa = fb.vabdl_u8(b: fc)
@@ -120,20 +122,20 @@ pri func decoder.filter_4_distance_3_arm_neon!(curr: slice base.u8, prev: slice
pc = fafb.vabdq_u16(b: fcfc)
cmpab = pa.vcleq_u16(b: pb)
cmpac = pa.vcleq_u16(b: pc)
picka = cmpab.vandq_u8(b: cmpac).vmovn_u16()
picka = cmpab.vandq_u16(b: cmpac).vmovn_u16()
pickb = pb.vcleq_u16(b: pc).vmovn_u16()
fx = fx.vadd_u8(
b: picka.vbsl_u8(b: fa,
c: pickb.vbsl_u8(b: fb, c: fc)))
// § poke_u24le replaces poke_u32le.
curr.poke_u24le!(a: fx.vget_lane_u32(b: 0))
curr.poke_u24le!(a: fx.as_u32x2().vget_lane_u32(b: 0))
fc = fb
fa = fx
} else (length: 3, advance: 3, unroll: 1) {
// § peek_u24le_as_u32 replaces peek_u32le.
fb = fb.create_vdup_n_u32(a: prev.peek_u24le_as_u32())
fb = util.make_u32x2_repeat(a: prev.peek_u24le_as_u32()).as_u8x8()
// § peek_u24le_as_u32 replaces peek_u32le.
fx = fx.create_vdup_n_u32(a: curr.peek_u24le_as_u32())
fx = util.make_u32x2_repeat(a: curr.peek_u24le_as_u32()).as_u8x8()
fafb = fa.vaddl_u8(b: fb)
fcfc = fc.vaddl_u8(b: fc)
pa = fb.vabdl_u8(b: fc)
@@ -141,13 +143,13 @@ pri func decoder.filter_4_distance_3_arm_neon!(curr: slice base.u8, prev: slice
pc = fafb.vabdq_u16(b: fcfc)
cmpab = pa.vcleq_u16(b: pb)
cmpac = pa.vcleq_u16(b: pc)
picka = cmpab.vandq_u8(b: cmpac).vmovn_u16()
picka = cmpab.vandq_u16(b: cmpac).vmovn_u16()
pickb = pb.vcleq_u16(b: pc).vmovn_u16()
fx = fx.vadd_u8(
b: picka.vbsl_u8(b: fa,
c: pickb.vbsl_u8(b: fb, c: fc)))
// § poke_u24le replaces poke_u32le.
curr.poke_u24le!(a: fx.vget_lane_u32(b: 0))
curr.poke_u24le!(a: fx.as_u32x2().vget_lane_u32(b: 0))
// § These assignments are unnecessary; this is the last iteration.
// fc = fb
// fa = fx
@@ -160,23 +162,24 @@ pri func decoder.filter_4_distance_4_arm_neon!(curr: slice base.u8, prev: slice
var curr : slice base.u8
var prev : slice base.u8
var fa : base.arm_neon_64
var fb : base.arm_neon_64
var fc : base.arm_neon_64
var fx : base.arm_neon_64
var fafb : base.arm_neon_128
var fcfc : base.arm_neon_128
var pa : base.arm_neon_128
var pb : base.arm_neon_128
var pc : base.arm_neon_128
var cmpab : base.arm_neon_128
var cmpac : base.arm_neon_128
var picka : base.arm_neon_64
var pickb : base.arm_neon_64
var util : base.arm_neon_utility
var fa : base.arm_neon_u8x8
var fb : base.arm_neon_u8x8
var fc : base.arm_neon_u8x8
var fx : base.arm_neon_u8x8
var fafb : base.arm_neon_u16x8
var fcfc : base.arm_neon_u16x8
var pa : base.arm_neon_u16x8
var pb : base.arm_neon_u16x8
var pc : base.arm_neon_u16x8
var cmpab : base.arm_neon_u16x8
var cmpac : base.arm_neon_u16x8
var picka : base.arm_neon_u8x8
var pickb : base.arm_neon_u8x8
iterate (curr = args.curr, prev = args.prev)(length: 4, advance: 4, unroll: 2) {
fb = fb.create_vdup_n_u32(a: prev.peek_u32le())
fx = fx.create_vdup_n_u32(a: curr.peek_u32le())
fb = util.make_u32x2_repeat(a: prev.peek_u32le()).as_u8x8()
fx = util.make_u32x2_repeat(a: curr.peek_u32le()).as_u8x8()
fafb = fa.vaddl_u8(b: fb) // fafb = (fa + fb)
fcfc = fc.vaddl_u8(b: fc) // fcfc = (fc + fc)
@@ -188,15 +191,15 @@ pri func decoder.filter_4_distance_4_arm_neon!(curr: slice base.u8, prev: slice
cmpab = pa.vcleq_u16(b: pb) // cmpab = (pa <= pb)
cmpac = pa.vcleq_u16(b: pc) // cmpac = (pa <= pc)
picka = cmpab.vandq_u8(b: cmpac).vmovn_u16() // picka = ((pa <= pb) && (pa <= pc))
pickb = pb.vcleq_u16(b: pc).vmovn_u16() // pickb = (pb <= pc)
picka = cmpab.vandq_u16(b: cmpac).vmovn_u16() // picka = ((pa <= pb) && (pa <= pc))
pickb = pb.vcleq_u16(b: pc).vmovn_u16() // pickb = (pb <= pc)
// Add the predictor to the residual.
fx = fx.vadd_u8(
b: picka.vbsl_u8(b: fa,
c: pickb.vbsl_u8(b: fb, c: fc)))
curr.poke_u32le!(a: fx.vget_lane_u32(b: 0))
curr.poke_u32le!(a: fx.as_u32x2().vget_lane_u32(b: 0))
fc = fb
fa = fx
}