mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-19 08:38:14 +01:00
Add crc32.ieee_hasher.up_x86_avx2
name old speed new speed delta wuffs_crc32_ieee_10k/clang9 8.42GB/s ± 1% 8.12GB/s ± 1% -3.55% (p=0.000 n=9+8) wuffs_crc32_ieee_100k/clang9 11.0GB/s ± 1% 10.9GB/s ± 1% ~ (p=0.931 n=9+9) wuffs_crc32_ieee_10k/gcc10 9.20GB/s ± 0% 9.71GB/s ± 0% +5.54% (p=0.000 n=8+10) wuffs_crc32_ieee_100k/gcc10 11.9GB/s ± 2% 13.1GB/s ± 1% +10.22% (p=0.000 n=10+8) wuffs_gzip_decode_10k/clang9 219MB/s ± 1% 218MB/s ± 2% ~ (p=0.497 n=9+10) wuffs_gzip_decode_100k/clang9 283MB/s ± 2% 281MB/s ± 1% ~ (p=0.063 n=10+10) wuffs_gzip_decode_10k/gcc10 224MB/s ± 1% 223MB/s ± 1% ~ (p=0.489 n=9+9) wuffs_gzip_decode_100k/gcc10 285MB/s ± 4% 290MB/s ± 0% +1.68% (p=0.004 n=10+8) wuffs_png_decode_image_19k_8bpp/clang9 138MB/s ± 2% 138MB/s ± 1% ~ (p=0.156 n=10+9) wuffs_png_decode_image_40k_24bpp/clang9 171MB/s ± 2% 172MB/s ± 1% ~ (p=0.971 n=10+10) wuffs_png_decode_image_77k_8bpp/clang9 495MB/s ± 4% 504MB/s ± 1% +1.79% (p=0.000 n=9+9) wuffs_png_decode_image_552k_32bpp_ignore_checksum/clang9 457MB/s ± 4% 463MB/s ± 0% +1.35% (p=0.002 n=9+9) wuffs_png_decode_image_552k_32bpp_verify_checksum/clang9 435MB/s ± 5% 445MB/s ± 1% +2.31% (p=0.000 n=10+9) wuffs_png_decode_image_4002k_24bpp/clang9 172MB/s ± 2% 174MB/s ± 1% ~ (p=0.053 n=10+9) wuffs_png_decode_image_19k_8bpp/gcc10 157MB/s ± 1% 157MB/s ± 1% ~ (p=0.549 n=9+10) wuffs_png_decode_image_40k_24bpp/gcc10 187MB/s ± 1% 186MB/s ± 1% ~ (p=0.182 n=10+9) wuffs_png_decode_image_77k_8bpp/gcc10 554MB/s ± 1% 559MB/s ± 0% +0.84% (p=0.000 n=9+9) wuffs_png_decode_image_552k_32bpp_ignore_checksum/gcc10 502MB/s ± 1% 500MB/s ± 0% -0.38% (p=0.011 n=10+10) wuffs_png_decode_image_552k_32bpp_verify_checksum/gcc10 481MB/s ± 1% 478MB/s ± 0% -0.46% (p=0.002 n=8+9) wuffs_png_decode_image_4002k_24bpp/gcc10 189MB/s ± 2% 190MB/s ± 0% +0.62% (p=0.008 n=10+9)
This commit is contained in:
1 parent
1dd7ad4413
commit
2dddd388bc
7 files changed
+333
-3
No files matched your search
@@ -27,6 +27,7 @@ pub func ieee_hasher.update_u32!(x: slice base.u8) base.u32 {
|
||||
if this.state == 0 {
|
||||
choose up = [
|
||||
up_arm_crc32,
|
||||
up_x86_avx2,
|
||||
up_x86_sse42]
|
||||
}
|
||||
this.up!(x: args.x)
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
// Copyright 2021 The Wuffs Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// https://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// --------
|
||||
|
||||
// See "SIMD Implementations" in README.md for a link to Gopal et al. "Fast CRC
|
||||
// Computation for Generic Polynomials Using PCLMULQDQ Instruction".
|
||||
|
||||
// up_x86_avx2 is exactly the same as up_x86_sse42 except for the "choose
|
||||
// cpu_arch >= x86_avx2". With AVX, PCLMULQDQ has a three-operand form, not
|
||||
// just a two-operand form: https://www.felixcloutier.com/x86/pclmulqdq
|
||||
pri func ieee_hasher.up_x86_avx2!(x: slice base.u8),
|
||||
choose cpu_arch >= x86_avx2,
|
||||
{
|
||||
var s : base.u32
|
||||
var p : slice base.u8
|
||||
|
||||
var util : base.x86_sse42_utility
|
||||
var k : base.x86_m128i
|
||||
var x0 : base.x86_m128i
|
||||
var x1 : base.x86_m128i
|
||||
var x2 : base.x86_m128i
|
||||
var x3 : base.x86_m128i
|
||||
var y0 : base.x86_m128i
|
||||
var y1 : base.x86_m128i
|
||||
var y2 : base.x86_m128i
|
||||
var y3 : base.x86_m128i
|
||||
|
||||
var tail_index : base.u64
|
||||
|
||||
s = 0xFFFF_FFFF ^ this.state
|
||||
|
||||
// Align to a 16-byte boundary.
|
||||
while (args.x.length() > 0) and ((15 & args.x.uintptr_low_12_bits()) <> 0) {
|
||||
s = IEEE_TABLE[0][((s & 0xFF) as base.u8) ^ args.x[0]] ^ (s >> 8)
|
||||
args.x = args.x[1 ..]
|
||||
} endwhile
|
||||
|
||||
// For short inputs, just do a simple loop.
|
||||
if args.x.length() < 64 {
|
||||
iterate (p = args.x)(length: 1, advance: 1, unroll: 1) {
|
||||
s = IEEE_TABLE[0][((s & 0xFF) as base.u8) ^ p[0]] ^ (s >> 8)
|
||||
}
|
||||
this.state = 0xFFFF_FFFF ^ s
|
||||
return nothing
|
||||
}
|
||||
|
||||
// Load 128×4 = 512 bits from the first 64-byte chunk.
|
||||
x0 = util.make_m128i_slice128(a: args.x[0x00 .. 0x10])
|
||||
x1 = util.make_m128i_slice128(a: args.x[0x10 .. 0x20])
|
||||
x2 = util.make_m128i_slice128(a: args.x[0x20 .. 0x30])
|
||||
x3 = util.make_m128i_slice128(a: args.x[0x30 .. 0x40])
|
||||
|
||||
// Combine with the initial state.
|
||||
x0 = x0._mm_xor_si128(b: util.make_m128i_single_u32(a: s))
|
||||
|
||||
// Process the remaining 64-byte chunks.
|
||||
k = util.make_m128i_slice128(a: IEEE_X86_SSE42_K1K2[.. 16])
|
||||
iterate (p = args.x[64 ..])(length: 64, advance: 64, unroll: 1) {
|
||||
y0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||||
y1 = x1._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||||
y2 = x2._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||||
y3 = x3._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||||
|
||||
x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x11)
|
||||
x1 = x1._mm_clmulepi64_si128(b: k, imm8: 0x11)
|
||||
x2 = x2._mm_clmulepi64_si128(b: k, imm8: 0x11)
|
||||
x3 = x3._mm_clmulepi64_si128(b: k, imm8: 0x11)
|
||||
|
||||
x0 = x0._mm_xor_si128(b: y0)._mm_xor_si128(b: util.make_m128i_slice128(a: p[0x00 .. 0x10]))
|
||||
x1 = x1._mm_xor_si128(b: y1)._mm_xor_si128(b: util.make_m128i_slice128(a: p[0x10 .. 0x20]))
|
||||
x2 = x2._mm_xor_si128(b: y2)._mm_xor_si128(b: util.make_m128i_slice128(a: p[0x20 .. 0x30]))
|
||||
x3 = x3._mm_xor_si128(b: y3)._mm_xor_si128(b: util.make_m128i_slice128(a: p[0x30 .. 0x40]))
|
||||
}
|
||||
|
||||
// Reduce 128×4 = 512 bits to 128 bits.
|
||||
k = util.make_m128i_slice128(a: IEEE_X86_SSE42_K3K4[.. 16])
|
||||
y0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||||
x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x11)
|
||||
x0 = x0._mm_xor_si128(b: x1)
|
||||
x0 = x0._mm_xor_si128(b: y0)
|
||||
y0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||||
x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x11)
|
||||
x0 = x0._mm_xor_si128(b: x2)
|
||||
x0 = x0._mm_xor_si128(b: y0)
|
||||
y0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||||
x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x11)
|
||||
x0 = x0._mm_xor_si128(b: x3)
|
||||
x0 = x0._mm_xor_si128(b: y0)
|
||||
|
||||
// Reduce 128 bits to 64 bits.
|
||||
x1 = x0._mm_clmulepi64_si128(b: k, imm8: 0x10)
|
||||
x2 = util.make_m128i_multiple_u32(
|
||||
a00: 0xFFFF_FFFF,
|
||||
a01: 0x0000_0000,
|
||||
a02: 0xFFFF_FFFF,
|
||||
a03: 0x0000_0000)
|
||||
x0 = x0._mm_srli_si128(imm8: 8)
|
||||
x0 = x0._mm_xor_si128(b: x1)
|
||||
k = util.make_m128i_slice128(a: IEEE_X86_SSE42_K5ZZ[.. 16])
|
||||
x1 = x0._mm_srli_si128(imm8: 4)
|
||||
x0 = x0._mm_and_si128(b: x2)
|
||||
x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||||
x0 = x0._mm_xor_si128(b: x1)
|
||||
|
||||
// Reduce 64 bits to 32 bits (Barrett Reduction) and extract.
|
||||
//
|
||||
// Barrett Reduction is Algorithm 1 (page 14) of Gopal et al., after
|
||||
// adjusting for bit-reflection as per Figure 12 (page 21).
|
||||
k = util.make_m128i_slice128(a: IEEE_X86_SSE42_PXMU[.. 16])
|
||||
x1 = x0._mm_and_si128(b: x2)
|
||||
x1 = x1._mm_clmulepi64_si128(b: k, imm8: 0x10)
|
||||
x1 = x1._mm_and_si128(b: x2)
|
||||
x1 = x1._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||||
x0 = x0._mm_xor_si128(b: x1)
|
||||
s = x0._mm_extract_epi32(imm8: 1)
|
||||
|
||||
// Handle the tail of args.x that wasn't a complete 64-byte chunk.
|
||||
tail_index = args.x.length() & 0xFFFF_FFFF_FFFF_FFC0 // And-not 64.
|
||||
if tail_index < args.x.length() {
|
||||
iterate (p = args.x[tail_index ..])(length: 1, advance: 1, unroll: 1) {
|
||||
s = IEEE_TABLE[0][((s & 0xFF) as base.u8) ^ p[0]] ^ (s >> 8)
|
||||
}
|
||||
}
|
||||
|
||||
this.state = 0xFFFF_FFFF ^ s
|
||||
}
|
||||
Reference in new issue
Block a user