Add std/adler32 hasher.up_arm_neon

On a Raspberry Pi 4 (32-bit armv7l) with -march=native and -mfpu=neon
("native" means "armv8-a+crc+simd"):

wuffs_adler32_10k/clang9                                  1.21GB/s ± 0%   6.58GB/s ± 1%  +442.43%  (p=0.008 n=5+5)
wuffs_adler32_100k/clang9                                 1.20GB/s ± 0%   6.58GB/s ± 0%  +449.37%  (p=0.008 n=5+5)

wuffs_adler32_10k/gcc8                                    1.24GB/s ± 0%   4.31GB/s ± 0%  +248.87%  (p=0.008 n=5+5)
wuffs_adler32_100k/gcc8                                   1.22GB/s ± 0%   4.29GB/s ± 0%  +251.09%  (p=0.008 n=5+5)

mimic_adler32_10k                                         1.40GB/s ± 0%   1.40GB/s ± 0%    -0.04%  (p=0.008 n=5+5)
mimic_adler32_100k                                        1.35GB/s ± 0%   1.38GB/s ± 0%    +2.77%  (p=0.008 n=5+5)

wuffs_zlib_decode_10k/clang9                              85.3MB/s ± 0%   91.1MB/s ± 0%    +6.83%  (p=0.008 n=5+5)
wuffs_zlib_decode_100k/clang9                              107MB/s ± 0%    113MB/s ± 0%    +5.20%  (p=0.008 n=5+5)

wuffs_zlib_decode_10k/gcc8                                86.1MB/s ± 0%   91.0MB/s ± 0%    +5.69%  (p=0.008 n=5+5)
wuffs_zlib_decode_100k/gcc8                                104MB/s ± 0%    111MB/s ± 0%    +7.08%  (p=0.016 n=5+4)

mimic_zlib_decode_10k                                      108MB/s ± 0%    108MB/s ± 0%      ~     (p=0.548 n=5+5)
mimic_zlib_decode_100k                                     142MB/s ± 0%    141MB/s ± 1%      ~     (p=0.310 n=5+5)

wuffs_png_decode_image_19k_8bpp/clang9                    69.9MB/s ± 0%   74.3MB/s ± 0%    +6.37%  (p=0.016 n=5+4)
wuffs_png_decode_image_40k_24bpp/clang9                   99.4MB/s ± 0%  104.3MB/s ± 0%    +4.92%  (p=0.008 n=5+5)
wuffs_png_decode_image_77k_8bpp/clang9                     266MB/s ± 0%    280MB/s ± 0%    +5.21%  (p=0.008 n=5+5)
wuffs_png_decode_image_552k_32bpp_ignore_checksum/clang9   165MB/s ± 0%    169MB/s ± 0%    +2.45%  (p=0.008 n=5+5)
wuffs_png_decode_image_552k_32bpp_verify_checksum/clang9   146MB/s ± 0%    163MB/s ± 0%   +11.86%  (p=0.008 n=5+5)
wuffs_png_decode_image_4002k_24bpp/clang9                 93.2MB/s ± 0%   97.7MB/s ± 0%    +4.78%  (p=0.008 n=5+5)

wuffs_png_decode_image_19k_8bpp/gcc8                      70.3MB/s ± 0%   74.3MB/s ± 0%    +5.70%  (p=0.008 n=5+5)
wuffs_png_decode_image_40k_24bpp/gcc8                      101MB/s ± 0%    107MB/s ± 0%    +5.67%  (p=0.008 n=5+5)
wuffs_png_decode_image_77k_8bpp/gcc8                       247MB/s ± 0%    259MB/s ± 0%    +4.71%  (p=0.008 n=5+5)
wuffs_png_decode_image_552k_32bpp_ignore_checksum/gcc8     162MB/s ± 0%    162MB/s ± 0%    +0.09%  (p=0.032 n=5+4)
wuffs_png_decode_image_552k_32bpp_verify_checksum/gcc8     140MB/s ± 0%    155MB/s ± 0%   +10.67%  (p=0.008 n=5+5)
wuffs_png_decode_image_4002k_24bpp/gcc8                   94.1MB/s ± 0%   99.4MB/s ± 0%    +5.57%  (p=0.008 n=5+5)

mimic_png_decode_image_19k_8bpp                           43.6MB/s ± 0%   43.3MB/s ± 1%      ~     (p=0.167 n=5+5)
mimic_png_decode_image_40k_24bpp                          54.6MB/s ± 0%   54.6MB/s ± 0%      ~     (p=1.000 n=5+5)
mimic_png_decode_image_77k_8bpp                            125MB/s ± 0%    123MB/s ± 0%    -1.03%  (p=0.016 n=5+4)
mimic_png_decode_image_552k_32bpp_ignore_checksum               skipped
mimic_png_decode_image_552k_32bpp_verify_checksum          102MB/s ± 0%    101MB/s ± 0%    -0.43%  (p=0.008 n=5+5)
mimic_png_decode_image_4002k_24bpp                        81.3MB/s ± 0%   81.8MB/s ± 0%    +0.62%  (p=0.008 n=5+5)
This commit is contained in:
Nigel Tao committed 2021-02-16 22:17:46 +11:00
1 parent d13039d96e
commit 010b34ed38
7 files changed
+629 -50

No files matched your search

+34 -28
View File
@@ -25,7 +25,9 @@ pub func hasher.update_u32!(x: slice base.u8) base.u32 {
if not this.started {
this.started = true
this.state = 1
choose up = [up_sse42]
choose up = [
up_arm_neon,
up_sse42]
}
this.up!(x: args.x)
return this.state
@@ -74,20 +76,22 @@ pri func hasher.up_sse42!(x: slice base.u8),
var p : slice base.u8
// The remaining variables are specific to the SIMD version.
var zeroes : base.x86_m128i
var ones : base.x86_m128i
var weights__left : base.x86_m128i
var weights_right : base.x86_m128i
var p__left : base.x86_m128i
var p_right : base.x86_m128i
var v1 : base.x86_m128i
var v2 : base.x86_m128i
var v2j : base.x86_m128i
var v2k : base.x86_m128i
var zeroes : base.x86_m128i
var ones : base.x86_m128i
var weights__left : base.x86_m128i
var weights_right : base.x86_m128i
var p__left : base.x86_m128i
var p_right : base.x86_m128i
var v1 : base.x86_m128i
var v2 : base.x86_m128i
var v2j : base.x86_m128i
var v2k : base.x86_m128i
var num_iterate_bytes : base.u32
var tail_index : base.u64
// zeroes and ones are uniform 8×u16 vectors.
// zeroes and ones are uniform u16×8 vectors.
zeroes = zeroes.create_mm_set1_epi16(a: 0)
ones = ones.create_mm_set1_epi16(a: 1)
@@ -104,11 +108,13 @@ pri func hasher.up_sse42!(x: slice base.u8),
e7: 0x09, e6: 0x0A, e5: 0x0B, e4: 0x0C,
e3: 0x0D, e2: 0x0E, e1: 0x0F, e0: 0x10)
// Just like the non-SIMD version, split the u32 state into two u16 halves
// and loop over args.x up to almost-5552 bytes at a time. The slightly
// smaller 5536 is the largest multiple of 32 less than non-SIMD's 5552.
// Decompose this.state.
s1 = this.state.low_bits(n: 16)
s2 = this.state.high_bits(n: 16)
// Just like the non-SIMD version, loop over args.x up to almost-5552 bytes
// at a time. The slightly smaller 5536 is the largest multiple of 32 less
// than non-SIMD's 5552.
while args.x.length() > 0 {
remaining = args.x[.. 0]
if args.x.length() > 5536 {
@@ -130,8 +136,8 @@ pri func hasher.up_sse42!(x: slice base.u8),
num_iterate_bytes = (args.x.length() & 0xFFFF_FFE0) as base.u32
s2 ~mod+= (s1 ~mod* num_iterate_bytes)
// Zero-initialize some 4×u32 vectors associated with the two state
// variables s1 and s2. The iterate loop accumulates 4 parallel u32
// Zero-initialize some u32×4 vectors associated with the two state
// variables s1 and s2. The iterate loop accumulates four parallel u32
// sums in each vector. A post-iterate step merges the four u32 sums
// into a single u32 sum.
v1 = v1.create_mm_setzero_si128()
@@ -143,23 +149,23 @@ pri func hasher.up_sse42!(x: slice base.u8),
// Split the 32-byte p into left and right halves. SSE4.2 works
// with 16-byte registers.
//
// Let p__left = [16×u8: p00, p01, p02, ..., p15]
// Let p_right = [16×u8: p16, p17, p18, ..., p31]
// Let p__left = [u8×16: p00, p01, p02, ..., p15]
// Let p_right = [u8×16: p16, p17, p18, ..., p31]
p__left.load_slice128!(a: p[.. 16])
p_right.load_slice128!(a: p[16 .. 32])
// For v2j, we need to calculate the sums of the s1j terms for each
// of p's 32 elements. This is simply 32 times the same number,
// that number being the sum of v1's four u32 accumulators. We
// add v1 now and multiply by 32 later, outside the inner loop.
// that number being the sum of v1's four u32 accumulators. We add
// v1 now and multiply by 32 later, outside the inner loop.
v2j = v2j._mm_add_epi32(b: v1)
// For v1, we need to add the elements of p. Computing the sum of
// absolute differences (_mm_sad_epu8) with zero just sums the
// elements. p__left._mm_sad_epu8(b: zeroes) equals
// [2×u64: p00 + p01 + ... + p07, p08 + p09 + ... + p15]
// [u64×2: p00 + p01 + ... + p07, p08 + p09 + ... + p15]
// This is equivalent (little-endian) to:
// [4×u32: p00 + p01 + ... + p07, 0, p08 + p09 + ... + p15, 0]
// [u32×4: p00 + p01 + ... + p07, 0, p08 + p09 + ... + p15, 0]
// We accumulate those "sum of p__left elements" in v1, and ditto
// for the p_right elements.
v1 = v1._mm_add_epi32(b: p__left._mm_sad_epu8(b: zeroes))
@@ -171,7 +177,7 @@ pri func hasher.up_sse42!(x: slice base.u8),
//
// The _mm_maddubs_epi16 call (vertically multiply u8 columns and
// then horizontally sum u16 pairs) with the left half produces:
// [8×u16: ((32*p00)+(31*p01)),
// [u16×8: ((32*p00)+(31*p01)),
// ((30*p02)+(29*p03)),
// ...
// ((18*p14)+(17*p15))]
@@ -179,7 +185,7 @@ pri func hasher.up_sse42!(x: slice base.u8),
// The ones._mm_madd_epi16(b: etc) call is likewise a multiply-add
// (note that it's "madd" not "add"). Multiplying by 1 is a no-op,
// so this sums u16 pairs to produce u32 values:
// [4×u32: ((32*p00)+(31*p01)+(30*p02)+(29*p03)),
// [u32×4: ((32*p00)+(31*p01)+(30*p02)+(29*p03)),
// ((28*p04)+(27*p05)+(26*p06)+(25*p07)),
// ...
// ((20*p12)+(19*p13)+(18*p14)+(17*p15))]
@@ -191,8 +197,8 @@ pri func hasher.up_sse42!(x: slice base.u8),
p_right._mm_maddubs_epi16(b: weights_right)))
}
// Merge the four per-u32-lane sums (v1) into the single u32 sum (s1).
// Starting with a 4×u32 vector [x0, x1, x2, x3]:
// Merge the four parallel u32 sums (v1) into the single u32 sum (s1).
// Starting with a u32×4 vector [x0, x1, x2, x3]:
// - shuffling with 0b1011_0001 gives [x1, x0, x3, x2].
// - adding gives [x0+x1, x0+x1, x2+x3, x2+x3].
// - shuffling with 0b0100_1110 gives [x2+x3, x2+x3, x0+x1, x0+x1].
@@ -206,7 +212,7 @@ pri func hasher.up_sse42!(x: slice base.u8),
// multiplies v2j's four u32 elements each by 32, alluded to earlier.
v2 = v2k._mm_add_epi32(b: v2j._mm_slli_epi32(imm8: 5))
// Similarly merge v2 (a 4×u32 vector) into s2 (a scalar u32 scalar).
// Similarly merge v2 (a u32×4 vector) into s2 (a u32 scalar).
v2 = v2._mm_add_epi32(b: v2._mm_shuffle_epi32(imm8: 0b1011_0001))
v2 = v2._mm_add_epi32(b: v2._mm_shuffle_epi32(imm8: 0b0100_1110))
s2 ~mod+= v2.truncate_u32()
+194
View File
@@ -0,0 +1,194 @@
// Copyright 2021 The Wuffs Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pri func hasher.up_arm_neon!(x: slice base.u8),
choose cpu_arch >= arm_neon,
{
// These variables are the same as the non-SIMD version.
var s1 : base.u32
var s2 : base.u32
var remaining : slice base.u8
var p : slice base.u8
// The remaining variables are specific to the SIMD version.
var p__left : base.arm_neon_128
var p_right : base.arm_neon_128
var v1 : base.arm_neon_128
var v2 : base.arm_neon_128
var col0 : base.arm_neon_128
var col1 : base.arm_neon_128
var col2 : base.arm_neon_128
var col3 : base.arm_neon_128
var sum1 : base.arm_neon_64
var sum2 : base.arm_neon_64
var sum12 : base.arm_neon_64
var num_iterate_bytes : base.u32
var tail_index : base.u64
// Decompose this.state.
s1 = this.state.low_bits(n: 16)
s2 = this.state.high_bits(n: 16)
// Align to a 16-byte boundary.
while (args.x.length() > 0) and ((15 & args.x.uintptr_low_12_bits()) <> 0) {
s1 ~mod+= args.x[0] as base.u32
s2 ~mod+= s1
args.x = args.x[1 ..]
} endwhile
s1 %= 65521
s2 %= 65521
// Just like the non-SIMD version, loop over args.x up to almost-5552 bytes
// at a time. The slightly smaller 5536 is the largest multiple of 32 less
// than non-SIMD's 5552.
while args.x.length() > 0 {
remaining = args.x[.. 0]
if args.x.length() > 5536 {
remaining = args.x[5536 ..]
args.x = args.x[.. 5536]
}
// The s1 state is the sum of the input bytes and the s2 state is the
// sum of the s1 state at each 1-byte step. Inside the iterate loop
// below, but starting fresh at each outer while loop iteration, s1
// consists of three parts (called s1i, s1j and s1k):
// - s1i: the initial value, before any 32-byte iterations.
// - s1j: the total contribution from previous 32-byte iterations.
// - s1k: the contribution due to the current 32-byte iteration.
//
// The upcoming iterate loop (at 32 bytes per iteration) encompasses
// num_iterate_bytes 1-byte steps. We hoist the total s1i contribution,
// (s1i * num_iterate_bytes) out here.
num_iterate_bytes = (args.x.length() & 0xFFFF_FFE0) as base.u32
s2 ~mod+= (s1 ~mod* num_iterate_bytes)
// Zero-initialize some u32×4 vectors associated with the two state
// variables s1 and s2. The iterate loop accumulates four parallel u32
// sums in each vector. A post-iterate step merges the four u32 sums
// into a single u32 sum.
v1 = v1.create_vdupq_n_u32(a: 0)
v2 = v2.create_vdupq_n_u32(a: 0)
col0 = col0.create_vdupq_n_u16(a: 0)
col1 = col1.create_vdupq_n_u16(a: 0)
col2 = col2.create_vdupq_n_u16(a: 0)
col3 = col3.create_vdupq_n_u16(a: 0)
// The inner loop.
iterate (p = args.x)(length: 32, advance: 32, unroll: 1) {
// Split the 32-byte p into left and right halves. NEON works with
// 16-byte registers.
//
// Let p__left = [u8×16: p00, p01, p02, ..., p15]
// Let p_right = [u8×16: p16, p17, p18, ..., p31]
p__left = p__left.create_slice128(a: p[.. 16])
p_right = p_right.create_slice128(a: p[16 .. 32])
// For v2j, we need to calculate the sums of the s1j terms for each
// of p's 32 elements. This is simply 32 times the same number,
// that number being the sum of v1's four u32 accumulators. We add
// v1 now and multiply by 32 later, outside the inner loop.
v2 = v2.vaddq_u32(b: v1)
// For v1, we need to add the elements of p.
//
// p__left.vpaddlq_u8() is:
// [u16×8: p00 + p01, p02 + p03, ..., p14 + p15]
//
// Combining (vpadalq_u8) that with p_right gives:
// [u16×8: p00 + p01 + p16 + p17,
// p02 + p03 + p18 + p19,
// ...
// p14 + p15 + p30 + p31]
//
// Pair-wise summing (and widening) again sets v1 to:
// [u32×4: Σ (p00 + p01 + p02 + p03 + p16 + p17 + p18 + p19),
// Σ (p04 + p05 + p06 + p07 + p20 + p21 + p22 + p23),
// ...
// Σ (p12 + p13 + p14 + p15 + p28 + p29 + p30 + p31)]
v1 = v1.vpadalq_u16(b: p__left.vpaddlq_u8().vpadalq_u8(b: p_right))
// Accumulate column sums:
// col0 = [u16×8: Σ p00, Σ p01, ..., Σ p07]
// col1 = [u16×8: Σ p08, Σ p09, ..., Σ p15]
// col2 = [u16×8: Σ p16, Σ p17, ..., Σ p23]
// col3 = [u16×8: Σ p24, Σ p25, ..., Σ p31]
col0 = col0.vaddw_u8(b: p__left.vget_low_u8())
col1 = col1.vaddw_u8(b: p__left.vget_high_u8())
col2 = col2.vaddw_u8(b: p_right.vget_low_u8())
col3 = col3.vaddw_u8(b: p_right.vget_high_u8())
}
// Multiply v2j's four u32 elements each by 32, alluded to earlier.
v2 = v2.vshlq_n_u32(b: 5)
// Add the v2k contributions in eight u32×4 vectors:
// [u32×4: 0x20 * (Σ p00), 0x1F * (Σ p01), ..., 0x1D * (Σ p03)]
// [u32×4: 0x1C * (Σ p04), 0x1B * (Σ p05), ..., 0x19 * (Σ p07)]
// ...
// [u32×4: 0x04 * (Σ p28), 0x03 * (Σ p29), ..., 0x01 * (Σ p31)]
v2 = v2.vmlal_u16(
b: col0.vget_low_u16(),
c: sum2.create_literal_u16x4(a0: 0x20, a1: 0x1F, a2: 0x1E, a3: 0x1D))
v2 = v2.vmlal_u16(
b: col0.vget_high_u16(),
c: sum2.create_literal_u16x4(a0: 0x1C, a1: 0x1B, a2: 0x1A, a3: 0x19))
v2 = v2.vmlal_u16(
b: col1.vget_low_u16(),
c: sum2.create_literal_u16x4(a0: 0x18, a1: 0x17, a2: 0x16, a3: 0x15))
v2 = v2.vmlal_u16(
b: col1.vget_high_u16(),
c: sum2.create_literal_u16x4(a0: 0x14, a1: 0x13, a2: 0x12, a3: 0x11))
v2 = v2.vmlal_u16(
b: col2.vget_low_u16(),
c: sum2.create_literal_u16x4(a0: 0x10, a1: 0x0F, a2: 0x0E, a3: 0x0D))
v2 = v2.vmlal_u16(
b: col2.vget_high_u16(),
c: sum2.create_literal_u16x4(a0: 0x0C, a1: 0x0B, a2: 0x0A, a3: 0x09))
v2 = v2.vmlal_u16(
b: col3.vget_low_u16(),
c: sum2.create_literal_u16x4(a0: 0x08, a1: 0x07, a2: 0x06, a3: 0x05))
v2 = v2.vmlal_u16(
b: col3.vget_high_u16(),
c: sum2.create_literal_u16x4(a0: 0x04, a1: 0x03, a2: 0x02, a3: 0x01))
// Merge the four parallel u32 sums (v1) into the single u32 sum (s1)
// and ditto for v2. Starting with [u32×4: xx_0, xx_1, xx_2, xx_3]:
// sum1 = [u32×2: v1_0 + v1_1, v1_2 + v1_3]
// sum2 = [u32×2: v2_0 + v2_1, v2_2 + v2_3]
// sum12 = [u32×2: v1_0 + v1_1 + v1_2 + v1_3,
// v2_0 + v2_1 + v2_2 + v2_3]
sum1 = v1.vget_low_u32().vpadd_u32(b: v1.vget_high_u32())
sum2 = v2.vget_low_u32().vpadd_u32(b: v2.vget_high_u32())
sum12 = sum1.vpadd_u32(b: sum2)
s1 ~mod+= sum12.vget_lane_u32(b: 0)
s2 ~mod+= sum12.vget_lane_u32(b: 1)
// Handle the tail of args.x that wasn't a complete 32-byte chunk.
tail_index = args.x.length() & 0xFFFF_FFFF_FFFF_FFE0 // And-not 32.
if tail_index < args.x.length() {
iterate (p = args.x[tail_index ..])(length: 1, advance: 1, unroll: 1) {
s1 ~mod+= p[0] as base.u32
s2 ~mod+= s1
}
}
// The rest of this function is the same as the non-SIMD version.
s1 %= 65521
s2 %= 65521
args.x = remaining
} endwhile
this.state = ((s2 & 0xFFFF) << 16) | (s1 & 0xFFFF)
}