mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-20 00:47:28 +01:00
Use arm_neon_uAxB types and methods
This commit is contained in:
1 parent
1f9b6ea535
commit
3b5d8fdfba
5 files changed
+279
-204
No files matched your search
@@ -23,17 +23,18 @@ pri func hasher.up_arm_neon!(x: slice base.u8),
|
||||
|
||||
// The remaining variables are specific to the SIMD version.
|
||||
|
||||
var p__left : base.arm_neon_128
|
||||
var p_right : base.arm_neon_128
|
||||
var v1 : base.arm_neon_128
|
||||
var v2 : base.arm_neon_128
|
||||
var col0 : base.arm_neon_128
|
||||
var col1 : base.arm_neon_128
|
||||
var col2 : base.arm_neon_128
|
||||
var col3 : base.arm_neon_128
|
||||
var sum1 : base.arm_neon_64
|
||||
var sum2 : base.arm_neon_64
|
||||
var sum12 : base.arm_neon_64
|
||||
var util : base.arm_neon_utility
|
||||
var p__left : base.arm_neon_u8x16
|
||||
var p_right : base.arm_neon_u8x16
|
||||
var v1 : base.arm_neon_u32x4
|
||||
var v2 : base.arm_neon_u32x4
|
||||
var col0 : base.arm_neon_u16x8
|
||||
var col1 : base.arm_neon_u16x8
|
||||
var col2 : base.arm_neon_u16x8
|
||||
var col3 : base.arm_neon_u16x8
|
||||
var sum1 : base.arm_neon_u32x2
|
||||
var sum2 : base.arm_neon_u32x2
|
||||
var sum12 : base.arm_neon_u32x2
|
||||
|
||||
var num_iterate_bytes : base.u32
|
||||
var tail_index : base.u64
|
||||
@@ -79,12 +80,12 @@ pri func hasher.up_arm_neon!(x: slice base.u8),
|
||||
// variables s1 and s2. The iterate loop accumulates four parallel u32
|
||||
// sums in each vector. A post-iterate step merges the four u32 sums
|
||||
// into a single u32 sum.
|
||||
v1 = v1.create_vdupq_n_u32(a: 0)
|
||||
v2 = v2.create_vdupq_n_u32(a: 0)
|
||||
col0 = col0.create_vdupq_n_u16(a: 0)
|
||||
col1 = col1.create_vdupq_n_u16(a: 0)
|
||||
col2 = col2.create_vdupq_n_u16(a: 0)
|
||||
col3 = col3.create_vdupq_n_u16(a: 0)
|
||||
v1 = util.make_u32x4_repeat(a: 0)
|
||||
v2 = util.make_u32x4_repeat(a: 0)
|
||||
col0 = util.make_u16x8_repeat(a: 0)
|
||||
col1 = util.make_u16x8_repeat(a: 0)
|
||||
col2 = util.make_u16x8_repeat(a: 0)
|
||||
col3 = util.make_u16x8_repeat(a: 0)
|
||||
|
||||
// The inner loop.
|
||||
iterate (p = args.x)(length: 32, advance: 32, unroll: 1) {
|
||||
@@ -93,8 +94,8 @@ pri func hasher.up_arm_neon!(x: slice base.u8),
|
||||
//
|
||||
// Let p__left = [u8×16: p00, p01, p02, ..., p15]
|
||||
// Let p_right = [u8×16: p16, p17, p18, ..., p31]
|
||||
p__left = p__left.create_slice128(a: p[.. 16])
|
||||
p_right = p_right.create_slice128(a: p[16 .. 32])
|
||||
p__left = util.make_u8x16_slice128(a: p[.. 16])
|
||||
p_right = util.make_u8x16_slice128(a: p[16 .. 32])
|
||||
|
||||
// For v2j, we need to calculate the sums of the s1j terms for each
|
||||
// of p's 32 elements. This is simply 32 times the same number,
|
||||
@@ -141,28 +142,28 @@ pri func hasher.up_arm_neon!(x: slice base.u8),
|
||||
// [u32×4: 0x04 * (Σ p28), 0x03 * (Σ p29), ..., 0x01 * (Σ p31)]
|
||||
v2 = v2.vmlal_u16(
|
||||
b: col0.vget_low_u16(),
|
||||
c: sum2.create_literal_u16x4(a0: 0x20, a1: 0x1F, a2: 0x1E, a3: 0x1D))
|
||||
c: util.make_u16x4_multiple(a00: 0x20, a01: 0x1F, a02: 0x1E, a03: 0x1D))
|
||||
v2 = v2.vmlal_u16(
|
||||
b: col0.vget_high_u16(),
|
||||
c: sum2.create_literal_u16x4(a0: 0x1C, a1: 0x1B, a2: 0x1A, a3: 0x19))
|
||||
c: util.make_u16x4_multiple(a00: 0x1C, a01: 0x1B, a02: 0x1A, a03: 0x19))
|
||||
v2 = v2.vmlal_u16(
|
||||
b: col1.vget_low_u16(),
|
||||
c: sum2.create_literal_u16x4(a0: 0x18, a1: 0x17, a2: 0x16, a3: 0x15))
|
||||
c: util.make_u16x4_multiple(a00: 0x18, a01: 0x17, a02: 0x16, a03: 0x15))
|
||||
v2 = v2.vmlal_u16(
|
||||
b: col1.vget_high_u16(),
|
||||
c: sum2.create_literal_u16x4(a0: 0x14, a1: 0x13, a2: 0x12, a3: 0x11))
|
||||
c: util.make_u16x4_multiple(a00: 0x14, a01: 0x13, a02: 0x12, a03: 0x11))
|
||||
v2 = v2.vmlal_u16(
|
||||
b: col2.vget_low_u16(),
|
||||
c: sum2.create_literal_u16x4(a0: 0x10, a1: 0x0F, a2: 0x0E, a3: 0x0D))
|
||||
c: util.make_u16x4_multiple(a00: 0x10, a01: 0x0F, a02: 0x0E, a03: 0x0D))
|
||||
v2 = v2.vmlal_u16(
|
||||
b: col2.vget_high_u16(),
|
||||
c: sum2.create_literal_u16x4(a0: 0x0C, a1: 0x0B, a2: 0x0A, a3: 0x09))
|
||||
c: util.make_u16x4_multiple(a00: 0x0C, a01: 0x0B, a02: 0x0A, a03: 0x09))
|
||||
v2 = v2.vmlal_u16(
|
||||
b: col3.vget_low_u16(),
|
||||
c: sum2.create_literal_u16x4(a0: 0x08, a1: 0x07, a2: 0x06, a3: 0x05))
|
||||
c: util.make_u16x4_multiple(a00: 0x08, a01: 0x07, a02: 0x06, a03: 0x05))
|
||||
v2 = v2.vmlal_u16(
|
||||
b: col3.vget_high_u16(),
|
||||
c: sum2.create_literal_u16x4(a0: 0x04, a1: 0x03, a2: 0x02, a3: 0x01))
|
||||
c: util.make_u16x4_multiple(a00: 0x04, a01: 0x03, a02: 0x02, a03: 0x01))
|
||||
|
||||
// Merge the four parallel u32 sums (v1) into the single u32 sum (s1)
|
||||
// and ditto for v2. Starting with [u32×4: xx_0, xx_1, xx_2, xx_3]:
|
||||
|
||||
@@ -55,24 +55,25 @@ pri func decoder.filter_3_distance_4_arm_neon!(curr: slice base.u8, prev: slice
|
||||
var curr : slice base.u8
|
||||
var prev : slice base.u8
|
||||
|
||||
var fa : base.arm_neon_64
|
||||
var fb : base.arm_neon_64
|
||||
var fx : base.arm_neon_64
|
||||
var util : base.arm_neon_utility
|
||||
var fa : base.arm_neon_u8x8
|
||||
var fb : base.arm_neon_u8x8
|
||||
var fx : base.arm_neon_u8x8
|
||||
|
||||
if args.prev.length() == 0 {
|
||||
iterate (curr = args.curr)(length: 4, advance: 4, unroll: 2) {
|
||||
fx = fx.create_vdup_n_u32(a: curr.peek_u32le())
|
||||
fx = util.make_u32x2_repeat(a: curr.peek_u32le()).as_u8x8()
|
||||
fx = fx.vadd_u8(b: fa.vhadd_u8(b: fb))
|
||||
curr.poke_u32le!(a: fx.vget_lane_u32(b: 0))
|
||||
curr.poke_u32le!(a: fx.as_u32x2().vget_lane_u32(b: 0))
|
||||
fa = fx
|
||||
}
|
||||
|
||||
} else {
|
||||
iterate (curr = args.curr, prev = args.prev)(length: 4, advance: 4, unroll: 2) {
|
||||
fb = fb.create_vdup_n_u32(a: prev.peek_u32le())
|
||||
fx = fx.create_vdup_n_u32(a: curr.peek_u32le())
|
||||
fb = util.make_u32x2_repeat(a: prev.peek_u32le()).as_u8x8()
|
||||
fx = util.make_u32x2_repeat(a: curr.peek_u32le()).as_u8x8()
|
||||
fx = fx.vadd_u8(b: fa.vhadd_u8(b: fb))
|
||||
curr.poke_u32le!(a: fx.vget_lane_u32(b: 0))
|
||||
curr.poke_u32le!(a: fx.as_u32x2().vget_lane_u32(b: 0))
|
||||
fa = fx
|
||||
}
|
||||
}
|
||||
@@ -95,24 +96,25 @@ pri func decoder.filter_4_distance_3_arm_neon!(curr: slice base.u8, prev: slice
|
||||
var curr : slice base.u8
|
||||
var prev : slice base.u8
|
||||
|
||||
var fa : base.arm_neon_64
|
||||
var fb : base.arm_neon_64
|
||||
var fc : base.arm_neon_64
|
||||
var fx : base.arm_neon_64
|
||||
var fafb : base.arm_neon_128
|
||||
var fcfc : base.arm_neon_128
|
||||
var pa : base.arm_neon_128
|
||||
var pb : base.arm_neon_128
|
||||
var pc : base.arm_neon_128
|
||||
var cmpab : base.arm_neon_128
|
||||
var cmpac : base.arm_neon_128
|
||||
var picka : base.arm_neon_64
|
||||
var pickb : base.arm_neon_64
|
||||
var util : base.arm_neon_utility
|
||||
var fa : base.arm_neon_u8x8
|
||||
var fb : base.arm_neon_u8x8
|
||||
var fc : base.arm_neon_u8x8
|
||||
var fx : base.arm_neon_u8x8
|
||||
var fafb : base.arm_neon_u16x8
|
||||
var fcfc : base.arm_neon_u16x8
|
||||
var pa : base.arm_neon_u16x8
|
||||
var pb : base.arm_neon_u16x8
|
||||
var pc : base.arm_neon_u16x8
|
||||
var cmpab : base.arm_neon_u16x8
|
||||
var cmpac : base.arm_neon_u16x8
|
||||
var picka : base.arm_neon_u8x8
|
||||
var pickb : base.arm_neon_u8x8
|
||||
|
||||
// § The advance is 3, not 4.
|
||||
iterate (curr = args.curr, prev = args.prev)(length: 4, advance: 3, unroll: 2) {
|
||||
fb = fb.create_vdup_n_u32(a: prev.peek_u32le())
|
||||
fx = fx.create_vdup_n_u32(a: curr.peek_u32le())
|
||||
fb = util.make_u32x2_repeat(a: prev.peek_u32le()).as_u8x8()
|
||||
fx = util.make_u32x2_repeat(a: curr.peek_u32le()).as_u8x8()
|
||||
fafb = fa.vaddl_u8(b: fb)
|
||||
fcfc = fc.vaddl_u8(b: fc)
|
||||
pa = fb.vabdl_u8(b: fc)
|
||||
@@ -120,20 +122,20 @@ pri func decoder.filter_4_distance_3_arm_neon!(curr: slice base.u8, prev: slice
|
||||
pc = fafb.vabdq_u16(b: fcfc)
|
||||
cmpab = pa.vcleq_u16(b: pb)
|
||||
cmpac = pa.vcleq_u16(b: pc)
|
||||
picka = cmpab.vandq_u8(b: cmpac).vmovn_u16()
|
||||
picka = cmpab.vandq_u16(b: cmpac).vmovn_u16()
|
||||
pickb = pb.vcleq_u16(b: pc).vmovn_u16()
|
||||
fx = fx.vadd_u8(
|
||||
b: picka.vbsl_u8(b: fa,
|
||||
c: pickb.vbsl_u8(b: fb, c: fc)))
|
||||
// § poke_u24le replaces poke_u32le.
|
||||
curr.poke_u24le!(a: fx.vget_lane_u32(b: 0))
|
||||
curr.poke_u24le!(a: fx.as_u32x2().vget_lane_u32(b: 0))
|
||||
fc = fb
|
||||
fa = fx
|
||||
} else (length: 3, advance: 3, unroll: 1) {
|
||||
// § peek_u24le_as_u32 replaces peek_u32le.
|
||||
fb = fb.create_vdup_n_u32(a: prev.peek_u24le_as_u32())
|
||||
fb = util.make_u32x2_repeat(a: prev.peek_u24le_as_u32()).as_u8x8()
|
||||
// § peek_u24le_as_u32 replaces peek_u32le.
|
||||
fx = fx.create_vdup_n_u32(a: curr.peek_u24le_as_u32())
|
||||
fx = util.make_u32x2_repeat(a: curr.peek_u24le_as_u32()).as_u8x8()
|
||||
fafb = fa.vaddl_u8(b: fb)
|
||||
fcfc = fc.vaddl_u8(b: fc)
|
||||
pa = fb.vabdl_u8(b: fc)
|
||||
@@ -141,13 +143,13 @@ pri func decoder.filter_4_distance_3_arm_neon!(curr: slice base.u8, prev: slice
|
||||
pc = fafb.vabdq_u16(b: fcfc)
|
||||
cmpab = pa.vcleq_u16(b: pb)
|
||||
cmpac = pa.vcleq_u16(b: pc)
|
||||
picka = cmpab.vandq_u8(b: cmpac).vmovn_u16()
|
||||
picka = cmpab.vandq_u16(b: cmpac).vmovn_u16()
|
||||
pickb = pb.vcleq_u16(b: pc).vmovn_u16()
|
||||
fx = fx.vadd_u8(
|
||||
b: picka.vbsl_u8(b: fa,
|
||||
c: pickb.vbsl_u8(b: fb, c: fc)))
|
||||
// § poke_u24le replaces poke_u32le.
|
||||
curr.poke_u24le!(a: fx.vget_lane_u32(b: 0))
|
||||
curr.poke_u24le!(a: fx.as_u32x2().vget_lane_u32(b: 0))
|
||||
// § These assignments are unnecessary; this is the last iteration.
|
||||
// fc = fb
|
||||
// fa = fx
|
||||
@@ -160,23 +162,24 @@ pri func decoder.filter_4_distance_4_arm_neon!(curr: slice base.u8, prev: slice
|
||||
var curr : slice base.u8
|
||||
var prev : slice base.u8
|
||||
|
||||
var fa : base.arm_neon_64
|
||||
var fb : base.arm_neon_64
|
||||
var fc : base.arm_neon_64
|
||||
var fx : base.arm_neon_64
|
||||
var fafb : base.arm_neon_128
|
||||
var fcfc : base.arm_neon_128
|
||||
var pa : base.arm_neon_128
|
||||
var pb : base.arm_neon_128
|
||||
var pc : base.arm_neon_128
|
||||
var cmpab : base.arm_neon_128
|
||||
var cmpac : base.arm_neon_128
|
||||
var picka : base.arm_neon_64
|
||||
var pickb : base.arm_neon_64
|
||||
var util : base.arm_neon_utility
|
||||
var fa : base.arm_neon_u8x8
|
||||
var fb : base.arm_neon_u8x8
|
||||
var fc : base.arm_neon_u8x8
|
||||
var fx : base.arm_neon_u8x8
|
||||
var fafb : base.arm_neon_u16x8
|
||||
var fcfc : base.arm_neon_u16x8
|
||||
var pa : base.arm_neon_u16x8
|
||||
var pb : base.arm_neon_u16x8
|
||||
var pc : base.arm_neon_u16x8
|
||||
var cmpab : base.arm_neon_u16x8
|
||||
var cmpac : base.arm_neon_u16x8
|
||||
var picka : base.arm_neon_u8x8
|
||||
var pickb : base.arm_neon_u8x8
|
||||
|
||||
iterate (curr = args.curr, prev = args.prev)(length: 4, advance: 4, unroll: 2) {
|
||||
fb = fb.create_vdup_n_u32(a: prev.peek_u32le())
|
||||
fx = fx.create_vdup_n_u32(a: curr.peek_u32le())
|
||||
fb = util.make_u32x2_repeat(a: prev.peek_u32le()).as_u8x8()
|
||||
fx = util.make_u32x2_repeat(a: curr.peek_u32le()).as_u8x8()
|
||||
|
||||
fafb = fa.vaddl_u8(b: fb) // fafb = (fa + fb)
|
||||
fcfc = fc.vaddl_u8(b: fc) // fcfc = (fc + fc)
|
||||
@@ -188,15 +191,15 @@ pri func decoder.filter_4_distance_4_arm_neon!(curr: slice base.u8, prev: slice
|
||||
cmpab = pa.vcleq_u16(b: pb) // cmpab = (pa <= pb)
|
||||
cmpac = pa.vcleq_u16(b: pc) // cmpac = (pa <= pc)
|
||||
|
||||
picka = cmpab.vandq_u8(b: cmpac).vmovn_u16() // picka = ((pa <= pb) && (pa <= pc))
|
||||
pickb = pb.vcleq_u16(b: pc).vmovn_u16() // pickb = (pb <= pc)
|
||||
picka = cmpab.vandq_u16(b: cmpac).vmovn_u16() // picka = ((pa <= pb) && (pa <= pc))
|
||||
pickb = pb.vcleq_u16(b: pc).vmovn_u16() // pickb = (pb <= pc)
|
||||
|
||||
// Add the predictor to the residual.
|
||||
fx = fx.vadd_u8(
|
||||
b: picka.vbsl_u8(b: fa,
|
||||
c: pickb.vbsl_u8(b: fb, c: fc)))
|
||||
|
||||
curr.poke_u32le!(a: fx.vget_lane_u32(b: 0))
|
||||
curr.poke_u32le!(a: fx.as_u32x2().vget_lane_u32(b: 0))
|
||||
fc = fb
|
||||
fa = fx
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user