mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-21 08:47:28 +01:00
std/crc32: re-write x86_sse42 implementation
name old speed new speed delta wuffs_crc32_ieee_10k/clang14 13.5GB/s ± 0% 21.9GB/s ± 0% +62.09% (p=0.016 n=5+4) wuffs_crc32_ieee_100k/clang14 22.4GB/s ± 9% 29.7GB/s ± 0% +32.76% (p=0.016 n=5+4) wuffs_crc32_ieee_10k/gcc12 14.2GB/s ± 2% 22.2GB/s ± 0% +56.92% (p=0.008 n=5+5) wuffs_crc32_ieee_100k/gcc12 21.3GB/s ± 3% 29.6GB/s ± 1% +39.18% (p=0.008 n=5+5) wuffs_gzip_decode_10k/clang14 366MB/s ± 0% 372MB/s ± 0% +1.41% (p=0.008 n=5+5) wuffs_gzip_decode_100k/clang14 482MB/s ± 0% 494MB/s ± 0% +2.57% (p=0.008 n=5+5) wuffs_gzip_decode_10k/gcc12 398MB/s ± 0% 418MB/s ± 0% +5.19% (p=0.008 n=5+5) wuffs_gzip_decode_100k/gcc12 510MB/s ± 0% 537MB/s ± 0% +5.21% (p=0.008 n=5+5) wuffs_png_decode_image_19k_8bpp/clang14 263MB/s ± 0% 264MB/s ± 0% +0.39% (p=0.008 n=5+5) wuffs_png_decode_image_40k_24bpp/clang14 297MB/s ± 0% 298MB/s ± 0% +0.16% (p=0.008 n=5+5) wuffs_png_decode_image_77k_8bpp/clang14 932MB/s ± 0% 945MB/s ± 0% +1.33% (p=0.008 n=5+5) wuffs_png_decode_image_552k_32bpp_ignore_checksum/clang14 831MB/s ± 0% 833MB/s ± 0% +0.25% (p=0.008 n=5+5) wuffs_png_decode_image_552k_32bpp_verify_checksum/clang14 799MB/s ± 0% 802MB/s ± 0% +0.37% (p=0.008 n=5+5) wuffs_png_decode_image_4002k_24bpp/clang14 306MB/s ± 0% 307MB/s ± 0% +0.56% (p=0.008 n=5+5) wuffs_png_decode_image_19k_8bpp/gcc12 283MB/s ± 0% 270MB/s ± 0% -4.84% (p=0.008 n=5+5) wuffs_png_decode_image_40k_24bpp/gcc12 330MB/s ± 0% 329MB/s ± 0% -0.29% (p=0.016 n=5+4) wuffs_png_decode_image_77k_8bpp/gcc12 992MB/s ± 0% 957MB/s ± 0% -3.56% (p=0.008 n=5+5) wuffs_png_decode_image_552k_32bpp_ignore_checksum/gcc12 920MB/s ± 0% 908MB/s ± 0% -1.30% (p=0.008 n=5+5) wuffs_png_decode_image_552k_32bpp_verify_checksum/gcc12 875MB/s ± 0% 867MB/s ± 0% -0.95% (p=0.008 n=5+5) wuffs_png_decode_image_4002k_24bpp/gcc12 341MB/s ± 0% 342MB/s ± 0% +0.25% (p=0.008 n=5+5)
This commit is contained in:
1 parent
e541e0cf79
commit
92ff96359a
6 files changed
+377
-512
No files matched your search
@@ -28,7 +28,6 @@ pub func ieee_hasher.update!(x: roslice base.u8) {
|
||||
if this.state == 0 {
|
||||
choose up = [
|
||||
up_arm_crc32,
|
||||
up_x86_avx2,
|
||||
up_x86_sse42]
|
||||
}
|
||||
this.up!(x: args.x)
|
||||
|
||||
@@ -1,137 +0,0 @@
|
||||
// Copyright 2021 The Wuffs Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
|
||||
// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
|
||||
// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
|
||||
// option. This file may not be copied, modified, or distributed
|
||||
// except according to those terms.
|
||||
//
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
// --------
|
||||
|
||||
// See "SIMD Implementations" in README.md for a link to Gopal et al. "Fast CRC
|
||||
// Computation for Generic Polynomials Using PCLMULQDQ Instruction".
|
||||
|
||||
// up_x86_avx2 is exactly the same as up_x86_sse42 except for the "choose
|
||||
// cpu_arch >= x86_avx2". With AVX, PCLMULQDQ has a three-operand form, not
|
||||
// just a two-operand form: https://www.felixcloutier.com/x86/pclmulqdq
|
||||
pri func ieee_hasher.up_x86_avx2!(x: roslice base.u8),
|
||||
choose cpu_arch >= x86_avx2,
|
||||
{
|
||||
var s : base.u32
|
||||
var p : roslice base.u8
|
||||
|
||||
var util : base.x86_sse42_utility
|
||||
var k1k2 : base.x86_m128i
|
||||
var k3k4 : base.x86_m128i
|
||||
var k5zz : base.x86_m128i
|
||||
var pxmu : base.x86_m128i
|
||||
var x0 : base.x86_m128i
|
||||
var x1 : base.x86_m128i
|
||||
var x2 : base.x86_m128i
|
||||
var x3 : base.x86_m128i
|
||||
var y0 : base.x86_m128i
|
||||
var y1 : base.x86_m128i
|
||||
var y2 : base.x86_m128i
|
||||
var y3 : base.x86_m128i
|
||||
|
||||
var tail_index : base.u64
|
||||
|
||||
s = 0xFFFF_FFFF ^ this.state
|
||||
|
||||
// Align to a 16-byte boundary.
|
||||
while (args.x.length() > 0) and ((15 & args.x.uintptr_low_12_bits()) <> 0) {
|
||||
s = IEEE_TABLE[0][((s & 0xFF) as base.u8) ^ args.x[0]] ^ (s >> 8)
|
||||
args.x = args.x[1 ..]
|
||||
} endwhile
|
||||
|
||||
// For short inputs, just do a simple loop.
|
||||
if args.x.length() < 64 {
|
||||
iterate (p = args.x)(length: 1, advance: 1, unroll: 1) {
|
||||
s = IEEE_TABLE[0][((s & 0xFF) as base.u8) ^ p[0]] ^ (s >> 8)
|
||||
}
|
||||
this.state = 0xFFFF_FFFF ^ s
|
||||
return nothing
|
||||
}
|
||||
|
||||
// Load 128×4 = 512 bits from the first 64-byte chunk.
|
||||
x0 = util.make_m128i_slice128(a: args.x[0x00 .. 0x10])
|
||||
x1 = util.make_m128i_slice128(a: args.x[0x10 .. 0x20])
|
||||
x2 = util.make_m128i_slice128(a: args.x[0x20 .. 0x30])
|
||||
x3 = util.make_m128i_slice128(a: args.x[0x30 .. 0x40])
|
||||
|
||||
// Combine with the initial state.
|
||||
x0 = x0._mm_xor_si128(b: util.make_m128i_single_u32(a: s))
|
||||
|
||||
// Process the remaining 64-byte chunks.
|
||||
k1k2 = util.make_m128i_slice128(a: IEEE_X86_SSE42_K1K2[.. 16])
|
||||
iterate (p = args.x[64 ..])(length: 64, advance: 64, unroll: 1) {
|
||||
y0 = x0._mm_clmulepi64_si128(b: k1k2, imm8: 0x00)
|
||||
y1 = x1._mm_clmulepi64_si128(b: k1k2, imm8: 0x00)
|
||||
y2 = x2._mm_clmulepi64_si128(b: k1k2, imm8: 0x00)
|
||||
y3 = x3._mm_clmulepi64_si128(b: k1k2, imm8: 0x00)
|
||||
|
||||
x0 = x0._mm_clmulepi64_si128(b: k1k2, imm8: 0x11)
|
||||
x1 = x1._mm_clmulepi64_si128(b: k1k2, imm8: 0x11)
|
||||
x2 = x2._mm_clmulepi64_si128(b: k1k2, imm8: 0x11)
|
||||
x3 = x3._mm_clmulepi64_si128(b: k1k2, imm8: 0x11)
|
||||
|
||||
x0 = x0._mm_xor_si128(b: y0)._mm_xor_si128(b: util.make_m128i_slice128(a: p[0x00 .. 0x10]))
|
||||
x1 = x1._mm_xor_si128(b: y1)._mm_xor_si128(b: util.make_m128i_slice128(a: p[0x10 .. 0x20]))
|
||||
x2 = x2._mm_xor_si128(b: y2)._mm_xor_si128(b: util.make_m128i_slice128(a: p[0x20 .. 0x30]))
|
||||
x3 = x3._mm_xor_si128(b: y3)._mm_xor_si128(b: util.make_m128i_slice128(a: p[0x30 .. 0x40]))
|
||||
}
|
||||
|
||||
// Reduce 128×4 = 512 bits to 128 bits.
|
||||
k3k4 = util.make_m128i_slice128(a: IEEE_X86_SSE42_K3K4[.. 16])
|
||||
y0 = x0._mm_clmulepi64_si128(b: k3k4, imm8: 0x00)
|
||||
x0 = x0._mm_clmulepi64_si128(b: k3k4, imm8: 0x11)
|
||||
x0 = x0._mm_xor_si128(b: x1)
|
||||
x0 = x0._mm_xor_si128(b: y0)
|
||||
y0 = x0._mm_clmulepi64_si128(b: k3k4, imm8: 0x00)
|
||||
x0 = x0._mm_clmulepi64_si128(b: k3k4, imm8: 0x11)
|
||||
x0 = x0._mm_xor_si128(b: x2)
|
||||
x0 = x0._mm_xor_si128(b: y0)
|
||||
y0 = x0._mm_clmulepi64_si128(b: k3k4, imm8: 0x00)
|
||||
x0 = x0._mm_clmulepi64_si128(b: k3k4, imm8: 0x11)
|
||||
x0 = x0._mm_xor_si128(b: x3)
|
||||
x0 = x0._mm_xor_si128(b: y0)
|
||||
|
||||
// Reduce 128 bits to 64 bits.
|
||||
x1 = x0._mm_clmulepi64_si128(b: k3k4, imm8: 0x10)
|
||||
x2 = util.make_m128i_multiple_u32(
|
||||
a00: 0xFFFF_FFFF,
|
||||
a01: 0x0000_0000,
|
||||
a02: 0xFFFF_FFFF,
|
||||
a03: 0x0000_0000)
|
||||
x0 = x0._mm_srli_si128(imm8: 8)
|
||||
x0 = x0._mm_xor_si128(b: x1)
|
||||
k5zz = util.make_m128i_slice128(a: IEEE_X86_SSE42_K5ZZ[.. 16])
|
||||
x1 = x0._mm_srli_si128(imm8: 4)
|
||||
x0 = x0._mm_and_si128(b: x2)
|
||||
x0 = x0._mm_clmulepi64_si128(b: k5zz, imm8: 0x00)
|
||||
x0 = x0._mm_xor_si128(b: x1)
|
||||
|
||||
// Reduce 64 bits to 32 bits (Barrett Reduction) and extract.
|
||||
//
|
||||
// Barrett Reduction is Algorithm 1 (page 14) of Gopal et al., after
|
||||
// adjusting for bit-reflection as per Figure 12 (page 21).
|
||||
pxmu = util.make_m128i_slice128(a: IEEE_X86_SSE42_PXMU[.. 16])
|
||||
x1 = x0._mm_and_si128(b: x2)
|
||||
x1 = x1._mm_clmulepi64_si128(b: pxmu, imm8: 0x10)
|
||||
x1 = x1._mm_and_si128(b: x2)
|
||||
x1 = x1._mm_clmulepi64_si128(b: pxmu, imm8: 0x00)
|
||||
x1 = x1._mm_xor_si128(b: x0)
|
||||
s = x1._mm_extract_epi32(imm8: 1)
|
||||
|
||||
// Handle the tail of args.x that wasn't a complete 64-byte chunk.
|
||||
tail_index = args.x.length() & 0xFFFF_FFFF_FFFF_FFC0 // And-not 64.
|
||||
if tail_index < args.x.length() {
|
||||
iterate (p = args.x[tail_index ..])(length: 1, advance: 1, unroll: 1) {
|
||||
s = IEEE_TABLE[0][((s & 0xFF) as base.u8) ^ p[0]] ^ (s >> 8)
|
||||
}
|
||||
}
|
||||
|
||||
this.state = 0xFFFF_FFFF ^ s
|
||||
}
|
||||
+111
-128
@@ -8,32 +8,29 @@
|
||||
//
|
||||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
// --------
|
||||
|
||||
// See "SIMD Implementations" in README.md for a link to Gopal et al. "Fast CRC
|
||||
// Computation for Generic Polynomials Using PCLMULQDQ Instruction".
|
||||
|
||||
pri func ieee_hasher.up_x86_sse42!(x: roslice base.u8),
|
||||
choose cpu_arch >= x86_sse42,
|
||||
{
|
||||
var s : base.u32
|
||||
var p : roslice base.u8
|
||||
|
||||
var util : base.x86_sse42_utility
|
||||
var k1k2 : base.x86_m128i
|
||||
var k3k4 : base.x86_m128i
|
||||
var k5zz : base.x86_m128i
|
||||
var pxmu : base.x86_m128i
|
||||
var kk : base.x86_m128i
|
||||
var x0 : base.x86_m128i
|
||||
var x1 : base.x86_m128i
|
||||
var x2 : base.x86_m128i
|
||||
var x3 : base.x86_m128i
|
||||
var x4 : base.x86_m128i
|
||||
var x5 : base.x86_m128i
|
||||
var x6 : base.x86_m128i
|
||||
var x7 : base.x86_m128i
|
||||
var y0 : base.x86_m128i
|
||||
var y1 : base.x86_m128i
|
||||
var y2 : base.x86_m128i
|
||||
var y3 : base.x86_m128i
|
||||
|
||||
var tail_index : base.u64
|
||||
var y4 : base.x86_m128i
|
||||
var y5 : base.x86_m128i
|
||||
var y6 : base.x86_m128i
|
||||
var y7 : base.x86_m128i
|
||||
|
||||
s = 0xFFFF_FFFF ^ this.state
|
||||
|
||||
@@ -43,125 +40,111 @@ pri func ieee_hasher.up_x86_sse42!(x: roslice base.u8),
|
||||
args.x = args.x[1 ..]
|
||||
} endwhile
|
||||
|
||||
// For short inputs, just do a simple loop.
|
||||
if args.x.length() < 64 {
|
||||
iterate (p = args.x)(length: 1, advance: 1, unroll: 1) {
|
||||
s = IEEE_TABLE[0][((s & 0xFF) as base.u8) ^ p[0]] ^ (s >> 8)
|
||||
}
|
||||
this.state = 0xFFFF_FFFF ^ s
|
||||
return nothing
|
||||
if args.x.length() >= 128 {
|
||||
// BEGIN script/print-crc32-x86-sse42-code.go generated code.
|
||||
x0 = util.make_m128i_slice128(a: args.x[0x00 .. 0x10])
|
||||
x1 = util.make_m128i_slice128(a: args.x[0x10 .. 0x20])
|
||||
x2 = util.make_m128i_slice128(a: args.x[0x20 .. 0x30])
|
||||
x3 = util.make_m128i_slice128(a: args.x[0x30 .. 0x40])
|
||||
x4 = util.make_m128i_slice128(a: args.x[0x40 .. 0x50])
|
||||
x5 = util.make_m128i_slice128(a: args.x[0x50 .. 0x60])
|
||||
x6 = util.make_m128i_slice128(a: args.x[0x60 .. 0x70])
|
||||
x7 = util.make_m128i_slice128(a: args.x[0x70 .. 0x80])
|
||||
kk = util.make_m128i_multiple_u32(a00: 0x33FF_F533, a01: 0, a02: 0x910E_EEC1, a03: 0)
|
||||
x0 = x0._mm_xor_si128(b: util.make_m128i_single_u32(a: s))
|
||||
args.x = args.x[128 ..]
|
||||
while args.x.length() >= 128 {
|
||||
y0 = x0._mm_clmulepi64_si128(b: kk, imm8: 0x00)
|
||||
x0 = x0._mm_clmulepi64_si128(b: kk, imm8: 0x11)
|
||||
y1 = x1._mm_clmulepi64_si128(b: kk, imm8: 0x00)
|
||||
x1 = x1._mm_clmulepi64_si128(b: kk, imm8: 0x11)
|
||||
y2 = x2._mm_clmulepi64_si128(b: kk, imm8: 0x00)
|
||||
x2 = x2._mm_clmulepi64_si128(b: kk, imm8: 0x11)
|
||||
y3 = x3._mm_clmulepi64_si128(b: kk, imm8: 0x00)
|
||||
x3 = x3._mm_clmulepi64_si128(b: kk, imm8: 0x11)
|
||||
y4 = x4._mm_clmulepi64_si128(b: kk, imm8: 0x00)
|
||||
x4 = x4._mm_clmulepi64_si128(b: kk, imm8: 0x11)
|
||||
y5 = x5._mm_clmulepi64_si128(b: kk, imm8: 0x00)
|
||||
x5 = x5._mm_clmulepi64_si128(b: kk, imm8: 0x11)
|
||||
y6 = x6._mm_clmulepi64_si128(b: kk, imm8: 0x00)
|
||||
x6 = x6._mm_clmulepi64_si128(b: kk, imm8: 0x11)
|
||||
y7 = x7._mm_clmulepi64_si128(b: kk, imm8: 0x00)
|
||||
x7 = x7._mm_clmulepi64_si128(b: kk, imm8: 0x11)
|
||||
y0 = y0._mm_xor_si128(b: util.make_m128i_slice128(a: args.x[0x00 .. 0x10]))
|
||||
x0 = x0._mm_xor_si128(b: y0)
|
||||
y1 = y1._mm_xor_si128(b: util.make_m128i_slice128(a: args.x[0x10 .. 0x20]))
|
||||
x1 = x1._mm_xor_si128(b: y1)
|
||||
y2 = y2._mm_xor_si128(b: util.make_m128i_slice128(a: args.x[0x20 .. 0x30]))
|
||||
x2 = x2._mm_xor_si128(b: y2)
|
||||
y3 = y3._mm_xor_si128(b: util.make_m128i_slice128(a: args.x[0x30 .. 0x40]))
|
||||
x3 = x3._mm_xor_si128(b: y3)
|
||||
y4 = y4._mm_xor_si128(b: util.make_m128i_slice128(a: args.x[0x40 .. 0x50]))
|
||||
x4 = x4._mm_xor_si128(b: y4)
|
||||
y5 = y5._mm_xor_si128(b: util.make_m128i_slice128(a: args.x[0x50 .. 0x60]))
|
||||
x5 = x5._mm_xor_si128(b: y5)
|
||||
y6 = y6._mm_xor_si128(b: util.make_m128i_slice128(a: args.x[0x60 .. 0x70]))
|
||||
x6 = x6._mm_xor_si128(b: y6)
|
||||
y7 = y7._mm_xor_si128(b: util.make_m128i_slice128(a: args.x[0x70 .. 0x80]))
|
||||
x7 = x7._mm_xor_si128(b: y7)
|
||||
args.x = args.x[128 ..]
|
||||
} endwhile
|
||||
kk = util.make_m128i_multiple_u32(a00: 0xAE68_9191, a01: 0, a02: 0xCCAA_009E, a03: 0)
|
||||
y0 = x0._mm_clmulepi64_si128(b: kk, imm8: 0x00)
|
||||
x0 = x0._mm_clmulepi64_si128(b: kk, imm8: 0x11)
|
||||
y2 = x2._mm_clmulepi64_si128(b: kk, imm8: 0x00)
|
||||
x2 = x2._mm_clmulepi64_si128(b: kk, imm8: 0x11)
|
||||
y4 = x4._mm_clmulepi64_si128(b: kk, imm8: 0x00)
|
||||
x4 = x4._mm_clmulepi64_si128(b: kk, imm8: 0x11)
|
||||
y6 = x6._mm_clmulepi64_si128(b: kk, imm8: 0x00)
|
||||
x6 = x6._mm_clmulepi64_si128(b: kk, imm8: 0x11)
|
||||
y0 = y0._mm_xor_si128(b: x1)
|
||||
x0 = x0._mm_xor_si128(b: y0)
|
||||
y2 = y2._mm_xor_si128(b: x3)
|
||||
x2 = x2._mm_xor_si128(b: y2)
|
||||
y4 = y4._mm_xor_si128(b: x5)
|
||||
x4 = x4._mm_xor_si128(b: y4)
|
||||
y6 = y6._mm_xor_si128(b: x7)
|
||||
x6 = x6._mm_xor_si128(b: y6)
|
||||
kk = util.make_m128i_multiple_u32(a00: 0xF1DA_05AA, a01: 0, a02: 0x8125_6527, a03: 0)
|
||||
y0 = x0._mm_clmulepi64_si128(b: kk, imm8: 0x00)
|
||||
x0 = x0._mm_clmulepi64_si128(b: kk, imm8: 0x11)
|
||||
y4 = x4._mm_clmulepi64_si128(b: kk, imm8: 0x00)
|
||||
x4 = x4._mm_clmulepi64_si128(b: kk, imm8: 0x11)
|
||||
y0 = y0._mm_xor_si128(b: x2)
|
||||
x0 = x0._mm_xor_si128(b: y0)
|
||||
y4 = y4._mm_xor_si128(b: x6)
|
||||
x4 = x4._mm_xor_si128(b: y4)
|
||||
kk = util.make_m128i_multiple_u32(a00: 0x8F35_2D95, a01: 0, a02: 0x1D95_13D7, a03: 0)
|
||||
y0 = x0._mm_clmulepi64_si128(b: kk, imm8: 0x00)
|
||||
x0 = x0._mm_clmulepi64_si128(b: kk, imm8: 0x11)
|
||||
y0 = y0._mm_xor_si128(b: x4)
|
||||
x0 = x0._mm_xor_si128(b: y0)
|
||||
kk = util.make_m128i_multiple_u32(a00: 0xF701_1641, a01: 0xB4E5_B025, a02: 0xDB71_0641, a03: 1)
|
||||
s = util.make_m128i_single_u64(a: x0._mm_extract_epi64(imm8: 0)).
|
||||
_mm_clmulepi64_si128(b: kk, imm8: 0x00).
|
||||
_mm_clmulepi64_si128(b: kk, imm8: 0x10).
|
||||
_mm_extract_epi32(imm8: 2)
|
||||
kk = util.make_m128i_multiple_u32(a00: 0xF701_1641, a01: 0xB4E5_B025, a02: 0xDB71_0641, a03: 1)
|
||||
s = util.make_m128i_single_u64(a: x0._mm_extract_epi64(imm8: 1) ^ (s as base.u64)).
|
||||
_mm_clmulepi64_si128(b: kk, imm8: 0x00).
|
||||
_mm_clmulepi64_si128(b: kk, imm8: 0x10).
|
||||
_mm_extract_epi32(imm8: 2)
|
||||
// END script/print-crc32-x86-sse42-code.go generated code.
|
||||
}
|
||||
|
||||
// Load 128×4 = 512 bits from the first 64-byte chunk.
|
||||
x0 = util.make_m128i_slice128(a: args.x[0x00 .. 0x10])
|
||||
x1 = util.make_m128i_slice128(a: args.x[0x10 .. 0x20])
|
||||
x2 = util.make_m128i_slice128(a: args.x[0x20 .. 0x30])
|
||||
x3 = util.make_m128i_slice128(a: args.x[0x30 .. 0x40])
|
||||
while args.x.length() >= 8 {
|
||||
kk = util.make_m128i_multiple_u32(a00: 0xF701_1641, a01: 0xB4E5_B025, a02: 0xDB71_0641, a03: 1)
|
||||
s = util.make_m128i_single_u64(a: args.x.peek_u64le() ^ (s as base.u64)).
|
||||
_mm_clmulepi64_si128(b: kk, imm8: 0x00).
|
||||
_mm_clmulepi64_si128(b: kk, imm8: 0x10).
|
||||
_mm_extract_epi32(imm8: 2)
|
||||
args.x = args.x[8 ..]
|
||||
} endwhile
|
||||
|
||||
// Combine with the initial state.
|
||||
x0 = x0._mm_xor_si128(b: util.make_m128i_single_u32(a: s))
|
||||
|
||||
// Process the remaining 64-byte chunks.
|
||||
k1k2 = util.make_m128i_slice128(a: IEEE_X86_SSE42_K1K2[.. 16])
|
||||
iterate (p = args.x[64 ..])(length: 64, advance: 64, unroll: 1) {
|
||||
y0 = x0._mm_clmulepi64_si128(b: k1k2, imm8: 0x00)
|
||||
y1 = x1._mm_clmulepi64_si128(b: k1k2, imm8: 0x00)
|
||||
y2 = x2._mm_clmulepi64_si128(b: k1k2, imm8: 0x00)
|
||||
y3 = x3._mm_clmulepi64_si128(b: k1k2, imm8: 0x00)
|
||||
|
||||
x0 = x0._mm_clmulepi64_si128(b: k1k2, imm8: 0x11)
|
||||
x1 = x1._mm_clmulepi64_si128(b: k1k2, imm8: 0x11)
|
||||
x2 = x2._mm_clmulepi64_si128(b: k1k2, imm8: 0x11)
|
||||
x3 = x3._mm_clmulepi64_si128(b: k1k2, imm8: 0x11)
|
||||
|
||||
x0 = x0._mm_xor_si128(b: y0)._mm_xor_si128(b: util.make_m128i_slice128(a: p[0x00 .. 0x10]))
|
||||
x1 = x1._mm_xor_si128(b: y1)._mm_xor_si128(b: util.make_m128i_slice128(a: p[0x10 .. 0x20]))
|
||||
x2 = x2._mm_xor_si128(b: y2)._mm_xor_si128(b: util.make_m128i_slice128(a: p[0x20 .. 0x30]))
|
||||
x3 = x3._mm_xor_si128(b: y3)._mm_xor_si128(b: util.make_m128i_slice128(a: p[0x30 .. 0x40]))
|
||||
}
|
||||
|
||||
// Reduce 128×4 = 512 bits to 128 bits.
|
||||
k3k4 = util.make_m128i_slice128(a: IEEE_X86_SSE42_K3K4[.. 16])
|
||||
y0 = x0._mm_clmulepi64_si128(b: k3k4, imm8: 0x00)
|
||||
x0 = x0._mm_clmulepi64_si128(b: k3k4, imm8: 0x11)
|
||||
x0 = x0._mm_xor_si128(b: x1)
|
||||
x0 = x0._mm_xor_si128(b: y0)
|
||||
y0 = x0._mm_clmulepi64_si128(b: k3k4, imm8: 0x00)
|
||||
x0 = x0._mm_clmulepi64_si128(b: k3k4, imm8: 0x11)
|
||||
x0 = x0._mm_xor_si128(b: x2)
|
||||
x0 = x0._mm_xor_si128(b: y0)
|
||||
y0 = x0._mm_clmulepi64_si128(b: k3k4, imm8: 0x00)
|
||||
x0 = x0._mm_clmulepi64_si128(b: k3k4, imm8: 0x11)
|
||||
x0 = x0._mm_xor_si128(b: x3)
|
||||
x0 = x0._mm_xor_si128(b: y0)
|
||||
|
||||
// Reduce 128 bits to 64 bits.
|
||||
x1 = x0._mm_clmulepi64_si128(b: k3k4, imm8: 0x10)
|
||||
x2 = util.make_m128i_multiple_u32(
|
||||
a00: 0xFFFF_FFFF,
|
||||
a01: 0x0000_0000,
|
||||
a02: 0xFFFF_FFFF,
|
||||
a03: 0x0000_0000)
|
||||
x0 = x0._mm_srli_si128(imm8: 8)
|
||||
x0 = x0._mm_xor_si128(b: x1)
|
||||
k5zz = util.make_m128i_slice128(a: IEEE_X86_SSE42_K5ZZ[.. 16])
|
||||
x1 = x0._mm_srli_si128(imm8: 4)
|
||||
x0 = x0._mm_and_si128(b: x2)
|
||||
x0 = x0._mm_clmulepi64_si128(b: k5zz, imm8: 0x00)
|
||||
x0 = x0._mm_xor_si128(b: x1)
|
||||
|
||||
// Reduce 64 bits to 32 bits (Barrett Reduction) and extract.
|
||||
//
|
||||
// Barrett Reduction is Algorithm 1 (page 14) of Gopal et al., after
|
||||
// adjusting for bit-reflection as per Figure 12 (page 21).
|
||||
pxmu = util.make_m128i_slice128(a: IEEE_X86_SSE42_PXMU[.. 16])
|
||||
x1 = x0._mm_and_si128(b: x2)
|
||||
x1 = x1._mm_clmulepi64_si128(b: pxmu, imm8: 0x10)
|
||||
x1 = x1._mm_and_si128(b: x2)
|
||||
x1 = x1._mm_clmulepi64_si128(b: pxmu, imm8: 0x00)
|
||||
x1 = x1._mm_xor_si128(b: x0)
|
||||
s = x1._mm_extract_epi32(imm8: 1)
|
||||
|
||||
// Handle the tail of args.x that wasn't a complete 64-byte chunk.
|
||||
tail_index = args.x.length() & 0xFFFF_FFFF_FFFF_FFC0 // And-not 64.
|
||||
if tail_index < args.x.length() {
|
||||
iterate (p = args.x[tail_index ..])(length: 1, advance: 1, unroll: 1) {
|
||||
s = IEEE_TABLE[0][((s & 0xFF) as base.u8) ^ p[0]] ^ (s >> 8)
|
||||
}
|
||||
}
|
||||
while args.x.length() > 0 {
|
||||
s = IEEE_TABLE[0][((s & 0xFF) as base.u8) ^ args.x[0]] ^ (s >> 8)
|
||||
args.x = args.x[1 ..]
|
||||
} endwhile
|
||||
|
||||
this.state = 0xFFFF_FFFF ^ s
|
||||
}
|
||||
|
||||
// These constants come from page 22 of Gopal et al. They are also reproduced
|
||||
// by script/print-crc32-x86-sse42-magic-numbers.go which is runnable online at
|
||||
// https://play.golang.org/p/wH1q6GfhKOE
|
||||
//
|
||||
// The k6' constant from the Gopal paper is unused.
|
||||
//
|
||||
// The rkN names match the numbers at
|
||||
// https://github.com/intel/isa-l/blob/7b30857e20b84e5afab1a28291189b9dc571110d/crc/crc32_gzip_refl_by16_10.asm#L475-L499
|
||||
//
|
||||
// The "+§" means a harmless off-by-one difference compared to Intel's numbers.
|
||||
// https://danlark.org/2021/03/08/how-a-bug-in-the-linux-crc-32-checksum-turned-out-not-to-be-a-bug/
|
||||
// https://github.com/google/wuffs/commit/b24e046670396d7ef22ccf499051340b9288419b
|
||||
|
||||
pri const IEEE_X86_SSE42_K1K2 : roarray[16] base.u8 = [
|
||||
0xD4, 0x2B, 0x44, 0x54, 0x01, 0x00, 0x00, 0x00, // k1' = 0x1_5444_2BD4 = rk16
|
||||
0x96, 0x15, 0xE4, 0xC6, 0x01, 0x00, 0x00, 0x00, // k2' = 0x1_C6E4_1596 = rk15
|
||||
]
|
||||
|
||||
pri const IEEE_X86_SSE42_K3K4 : roarray[16] base.u8 = [
|
||||
0xD0, 0x97, 0x19, 0x75, 0x01, 0x00, 0x00, 0x00, // k3' = 0x1_7519_97D0 = rk2
|
||||
0x9E, 0x00, 0xAA, 0xCC, 0x00, 0x00, 0x00, 0x00, // k4' = 0x0_CCAA_009E = rk1
|
||||
]
|
||||
|
||||
pri const IEEE_X86_SSE42_K5ZZ : roarray[16] base.u8 = [
|
||||
0x24, 0x61, 0xCD, 0x63, 0x01, 0x00, 0x00, 0x00, // k5' = 0x1_63CD_6124 = rk6
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // Unused
|
||||
]
|
||||
|
||||
pri const IEEE_X86_SSE42_PXMU : roarray[16] base.u8 = [
|
||||
0x41, 0x06, 0x71, 0xDB, 0x01, 0x00, 0x00, 0x00, // Px' = 0x1_DB71_0641 = rk8+§
|
||||
0x41, 0x16, 0x01, 0xF7, 0x01, 0x00, 0x00, 0x00, // μ' = 0x1_F701_1641 = rk7+§
|
||||
]
|
||||
Reference in new issue
Block a user