Files
wuffs/std/adler32/common_up_arm_neon.wuffs
T
Nigel Tao 010b34ed38 Add std/adler32 hasher.up_arm_neon
On a Raspberry Pi 4 (32-bit armv7l) with -march=native and -mfpu=neon
("native" means "armv8-a+crc+simd"):

wuffs_adler32_10k/clang9                                  1.21GB/s ± 0%   6.58GB/s ± 1%  +442.43%  (p=0.008 n=5+5)
wuffs_adler32_100k/clang9                                 1.20GB/s ± 0%   6.58GB/s ± 0%  +449.37%  (p=0.008 n=5+5)

wuffs_adler32_10k/gcc8                                    1.24GB/s ± 0%   4.31GB/s ± 0%  +248.87%  (p=0.008 n=5+5)
wuffs_adler32_100k/gcc8                                   1.22GB/s ± 0%   4.29GB/s ± 0%  +251.09%  (p=0.008 n=5+5)

mimic_adler32_10k                                         1.40GB/s ± 0%   1.40GB/s ± 0%    -0.04%  (p=0.008 n=5+5)
mimic_adler32_100k                                        1.35GB/s ± 0%   1.38GB/s ± 0%    +2.77%  (p=0.008 n=5+5)

wuffs_zlib_decode_10k/clang9                              85.3MB/s ± 0%   91.1MB/s ± 0%    +6.83%  (p=0.008 n=5+5)
wuffs_zlib_decode_100k/clang9                              107MB/s ± 0%    113MB/s ± 0%    +5.20%  (p=0.008 n=5+5)

wuffs_zlib_decode_10k/gcc8                                86.1MB/s ± 0%   91.0MB/s ± 0%    +5.69%  (p=0.008 n=5+5)
wuffs_zlib_decode_100k/gcc8                                104MB/s ± 0%    111MB/s ± 0%    +7.08%  (p=0.016 n=5+4)

mimic_zlib_decode_10k                                      108MB/s ± 0%    108MB/s ± 0%      ~     (p=0.548 n=5+5)
mimic_zlib_decode_100k                                     142MB/s ± 0%    141MB/s ± 1%      ~     (p=0.310 n=5+5)

wuffs_png_decode_image_19k_8bpp/clang9                    69.9MB/s ± 0%   74.3MB/s ± 0%    +6.37%  (p=0.016 n=5+4)
wuffs_png_decode_image_40k_24bpp/clang9                   99.4MB/s ± 0%  104.3MB/s ± 0%    +4.92%  (p=0.008 n=5+5)
wuffs_png_decode_image_77k_8bpp/clang9                     266MB/s ± 0%    280MB/s ± 0%    +5.21%  (p=0.008 n=5+5)
wuffs_png_decode_image_552k_32bpp_ignore_checksum/clang9   165MB/s ± 0%    169MB/s ± 0%    +2.45%  (p=0.008 n=5+5)
wuffs_png_decode_image_552k_32bpp_verify_checksum/clang9   146MB/s ± 0%    163MB/s ± 0%   +11.86%  (p=0.008 n=5+5)
wuffs_png_decode_image_4002k_24bpp/clang9                 93.2MB/s ± 0%   97.7MB/s ± 0%    +4.78%  (p=0.008 n=5+5)

wuffs_png_decode_image_19k_8bpp/gcc8                      70.3MB/s ± 0%   74.3MB/s ± 0%    +5.70%  (p=0.008 n=5+5)
wuffs_png_decode_image_40k_24bpp/gcc8                      101MB/s ± 0%    107MB/s ± 0%    +5.67%  (p=0.008 n=5+5)
wuffs_png_decode_image_77k_8bpp/gcc8                       247MB/s ± 0%    259MB/s ± 0%    +4.71%  (p=0.008 n=5+5)
wuffs_png_decode_image_552k_32bpp_ignore_checksum/gcc8     162MB/s ± 0%    162MB/s ± 0%    +0.09%  (p=0.032 n=5+4)
wuffs_png_decode_image_552k_32bpp_verify_checksum/gcc8     140MB/s ± 0%    155MB/s ± 0%   +10.67%  (p=0.008 n=5+5)
wuffs_png_decode_image_4002k_24bpp/gcc8                   94.1MB/s ± 0%   99.4MB/s ± 0%    +5.57%  (p=0.008 n=5+5)

mimic_png_decode_image_19k_8bpp                           43.6MB/s ± 0%   43.3MB/s ± 1%      ~     (p=0.167 n=5+5)
mimic_png_decode_image_40k_24bpp                          54.6MB/s ± 0%   54.6MB/s ± 0%      ~     (p=1.000 n=5+5)
mimic_png_decode_image_77k_8bpp                            125MB/s ± 0%    123MB/s ± 0%    -1.03%  (p=0.016 n=5+4)
mimic_png_decode_image_552k_32bpp_ignore_checksum               skipped
mimic_png_decode_image_552k_32bpp_verify_checksum          102MB/s ± 0%    101MB/s ± 0%    -0.43%  (p=0.008 n=5+5)
mimic_png_decode_image_4002k_24bpp                        81.3MB/s ± 0%   81.8MB/s ± 0%    +0.62%  (p=0.008 n=5+5)
2021-02-16 22:17:46 +11:00

195 lines
7.3 KiB
Plaintext
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// Copyright 2021 The Wuffs Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pri func hasher.up_arm_neon!(x: slice base.u8),
choose cpu_arch >= arm_neon,
{
// These variables are the same as the non-SIMD version.
var s1 : base.u32
var s2 : base.u32
var remaining : slice base.u8
var p : slice base.u8
// The remaining variables are specific to the SIMD version.
var p__left : base.arm_neon_128
var p_right : base.arm_neon_128
var v1 : base.arm_neon_128
var v2 : base.arm_neon_128
var col0 : base.arm_neon_128
var col1 : base.arm_neon_128
var col2 : base.arm_neon_128
var col3 : base.arm_neon_128
var sum1 : base.arm_neon_64
var sum2 : base.arm_neon_64
var sum12 : base.arm_neon_64
var num_iterate_bytes : base.u32
var tail_index : base.u64
// Decompose this.state.
s1 = this.state.low_bits(n: 16)
s2 = this.state.high_bits(n: 16)
// Align to a 16-byte boundary.
while (args.x.length() > 0) and ((15 & args.x.uintptr_low_12_bits()) <> 0) {
s1 ~mod+= args.x[0] as base.u32
s2 ~mod+= s1
args.x = args.x[1 ..]
} endwhile
s1 %= 65521
s2 %= 65521
// Just like the non-SIMD version, loop over args.x up to almost-5552 bytes
// at a time. The slightly smaller 5536 is the largest multiple of 32 less
// than non-SIMD's 5552.
while args.x.length() > 0 {
remaining = args.x[.. 0]
if args.x.length() > 5536 {
remaining = args.x[5536 ..]
args.x = args.x[.. 5536]
}
// The s1 state is the sum of the input bytes and the s2 state is the
// sum of the s1 state at each 1-byte step. Inside the iterate loop
// below, but starting fresh at each outer while loop iteration, s1
// consists of three parts (called s1i, s1j and s1k):
// - s1i: the initial value, before any 32-byte iterations.
// - s1j: the total contribution from previous 32-byte iterations.
// - s1k: the contribution due to the current 32-byte iteration.
//
// The upcoming iterate loop (at 32 bytes per iteration) encompasses
// num_iterate_bytes 1-byte steps. We hoist the total s1i contribution,
// (s1i * num_iterate_bytes) out here.
num_iterate_bytes = (args.x.length() & 0xFFFF_FFE0) as base.u32
s2 ~mod+= (s1 ~mod* num_iterate_bytes)
// Zero-initialize some u32×4 vectors associated with the two state
// variables s1 and s2. The iterate loop accumulates four parallel u32
// sums in each vector. A post-iterate step merges the four u32 sums
// into a single u32 sum.
v1 = v1.create_vdupq_n_u32(a: 0)
v2 = v2.create_vdupq_n_u32(a: 0)
col0 = col0.create_vdupq_n_u16(a: 0)
col1 = col1.create_vdupq_n_u16(a: 0)
col2 = col2.create_vdupq_n_u16(a: 0)
col3 = col3.create_vdupq_n_u16(a: 0)
// The inner loop.
iterate (p = args.x)(length: 32, advance: 32, unroll: 1) {
// Split the 32-byte p into left and right halves. NEON works with
// 16-byte registers.
//
// Let p__left = [u8×16: p00, p01, p02, ..., p15]
// Let p_right = [u8×16: p16, p17, p18, ..., p31]
p__left = p__left.create_slice128(a: p[.. 16])
p_right = p_right.create_slice128(a: p[16 .. 32])
// For v2j, we need to calculate the sums of the s1j terms for each
// of p's 32 elements. This is simply 32 times the same number,
// that number being the sum of v1's four u32 accumulators. We add
// v1 now and multiply by 32 later, outside the inner loop.
v2 = v2.vaddq_u32(b: v1)
// For v1, we need to add the elements of p.
//
// p__left.vpaddlq_u8() is:
// [u16×8: p00 + p01, p02 + p03, ..., p14 + p15]
//
// Combining (vpadalq_u8) that with p_right gives:
// [u16×8: p00 + p01 + p16 + p17,
// p02 + p03 + p18 + p19,
// ...
// p14 + p15 + p30 + p31]
//
// Pair-wise summing (and widening) again sets v1 to:
// [u32×4: Σ (p00 + p01 + p02 + p03 + p16 + p17 + p18 + p19),
// Σ (p04 + p05 + p06 + p07 + p20 + p21 + p22 + p23),
// ...
// Σ (p12 + p13 + p14 + p15 + p28 + p29 + p30 + p31)]
v1 = v1.vpadalq_u16(b: p__left.vpaddlq_u8().vpadalq_u8(b: p_right))
// Accumulate column sums:
// col0 = [u16×8: Σ p00, Σ p01, ..., Σ p07]
// col1 = [u16×8: Σ p08, Σ p09, ..., Σ p15]
// col2 = [u16×8: Σ p16, Σ p17, ..., Σ p23]
// col3 = [u16×8: Σ p24, Σ p25, ..., Σ p31]
col0 = col0.vaddw_u8(b: p__left.vget_low_u8())
col1 = col1.vaddw_u8(b: p__left.vget_high_u8())
col2 = col2.vaddw_u8(b: p_right.vget_low_u8())
col3 = col3.vaddw_u8(b: p_right.vget_high_u8())
}
// Multiply v2j's four u32 elements each by 32, alluded to earlier.
v2 = v2.vshlq_n_u32(b: 5)
// Add the v2k contributions in eight u32×4 vectors:
// [u32×4: 0x20 * (Σ p00), 0x1F * (Σ p01), ..., 0x1D * (Σ p03)]
// [u32×4: 0x1C * (Σ p04), 0x1B * (Σ p05), ..., 0x19 * (Σ p07)]
// ...
// [u32×4: 0x04 * (Σ p28), 0x03 * (Σ p29), ..., 0x01 * (Σ p31)]
v2 = v2.vmlal_u16(
b: col0.vget_low_u16(),
c: sum2.create_literal_u16x4(a0: 0x20, a1: 0x1F, a2: 0x1E, a3: 0x1D))
v2 = v2.vmlal_u16(
b: col0.vget_high_u16(),
c: sum2.create_literal_u16x4(a0: 0x1C, a1: 0x1B, a2: 0x1A, a3: 0x19))
v2 = v2.vmlal_u16(
b: col1.vget_low_u16(),
c: sum2.create_literal_u16x4(a0: 0x18, a1: 0x17, a2: 0x16, a3: 0x15))
v2 = v2.vmlal_u16(
b: col1.vget_high_u16(),
c: sum2.create_literal_u16x4(a0: 0x14, a1: 0x13, a2: 0x12, a3: 0x11))
v2 = v2.vmlal_u16(
b: col2.vget_low_u16(),
c: sum2.create_literal_u16x4(a0: 0x10, a1: 0x0F, a2: 0x0E, a3: 0x0D))
v2 = v2.vmlal_u16(
b: col2.vget_high_u16(),
c: sum2.create_literal_u16x4(a0: 0x0C, a1: 0x0B, a2: 0x0A, a3: 0x09))
v2 = v2.vmlal_u16(
b: col3.vget_low_u16(),
c: sum2.create_literal_u16x4(a0: 0x08, a1: 0x07, a2: 0x06, a3: 0x05))
v2 = v2.vmlal_u16(
b: col3.vget_high_u16(),
c: sum2.create_literal_u16x4(a0: 0x04, a1: 0x03, a2: 0x02, a3: 0x01))
// Merge the four parallel u32 sums (v1) into the single u32 sum (s1)
// and ditto for v2. Starting with [u32×4: xx_0, xx_1, xx_2, xx_3]:
// sum1 = [u32×2: v1_0 + v1_1, v1_2 + v1_3]
// sum2 = [u32×2: v2_0 + v2_1, v2_2 + v2_3]
// sum12 = [u32×2: v1_0 + v1_1 + v1_2 + v1_3,
// v2_0 + v2_1 + v2_2 + v2_3]
sum1 = v1.vget_low_u32().vpadd_u32(b: v1.vget_high_u32())
sum2 = v2.vget_low_u32().vpadd_u32(b: v2.vget_high_u32())
sum12 = sum1.vpadd_u32(b: sum2)
s1 ~mod+= sum12.vget_lane_u32(b: 0)
s2 ~mod+= sum12.vget_lane_u32(b: 1)
// Handle the tail of args.x that wasn't a complete 32-byte chunk.
tail_index = args.x.length() & 0xFFFF_FFFF_FFFF_FFE0 // And-not 32.
if tail_index < args.x.length() {
iterate (p = args.x[tail_index ..])(length: 1, advance: 1, unroll: 1) {
s1 ~mod+= p[0] as base.u32
s2 ~mod+= s1
}
}
// The rest of this function is the same as the non-SIMD version.
s1 %= 65521
s2 %= 65521
args.x = remaining
} endwhile
this.state = ((s2 & 0xFFFF) << 16) | (s1 & 0xFFFF)
}