mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-20 16:47:28 +01:00
This is just a change of name (and a change of comment), not a change of value. From my colleague, Eric Biggers: The number is not k6' = (x^32 mod P(x) << 32)' << 1 = 0x1DB710640, but rather P(x)' = 0x1DB710641. This value represents the generator polynomial of CRC-32 as a 33-bit value, using the CRC-32 bit ordering convention (lowest order bit = coefficient of highest order polynomial term). The lowest order bit is the coefficient of x^32, which by definition is 1 for CRC-32. P(x) is used as a multiplicand in Barrett reduction, which is the step which reduces a 64-bit value to the final 32-bit remainder. See "Algorithm 1" in the Gopal paper. The reason that last bit doesn't actually matter is because the output of the Barrett reduction is supposed to be fully reduced mod P(x), so only 32 bits (the bits that represent the coefficients of x^0..x^31) are supposed to be able to be nonzero. Therefore, in practice implementations will just truncate the result to 32 bits. Incorrectly using 0x1DB710640 rather than P(x) = 0x1DB710641 only causes the result to be wrong by a factor of x^32, so it doesn't affect those 32 bits.
162 lines
5.8 KiB
Plaintext
162 lines
5.8 KiB
Plaintext
// Copyright 2021 The Wuffs Authors.
|
||
//
|
||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||
// you may not use this file except in compliance with the License.
|
||
// You may obtain a copy of the License at
|
||
//
|
||
// https://www.apache.org/licenses/LICENSE-2.0
|
||
//
|
||
// Unless required by applicable law or agreed to in writing, software
|
||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
// See the License for the specific language governing permissions and
|
||
// limitations under the License.
|
||
|
||
// --------
|
||
|
||
// See "SIMD Implementations" in README.md for a link to Gopal et al. "Fast CRC
|
||
// Computation for Generic Polynomials Using PCLMULQDQ Instruction".
|
||
|
||
pri func ieee_hasher.up_x86_sse42!(x: slice base.u8),
|
||
choose cpu_arch >= x86_sse42,
|
||
{
|
||
var s : base.u32
|
||
var p : slice base.u8
|
||
|
||
var util : base.x86_sse42_utility
|
||
var k : base.x86_m128i
|
||
var x0 : base.x86_m128i
|
||
var x1 : base.x86_m128i
|
||
var x2 : base.x86_m128i
|
||
var x3 : base.x86_m128i
|
||
var y0 : base.x86_m128i
|
||
var y1 : base.x86_m128i
|
||
var y2 : base.x86_m128i
|
||
var y3 : base.x86_m128i
|
||
|
||
var tail_index : base.u64
|
||
|
||
s = 0xFFFF_FFFF ^ this.state
|
||
|
||
// Align to a 16-byte boundary.
|
||
while (args.x.length() > 0) and ((15 & args.x.uintptr_low_12_bits()) <> 0) {
|
||
s = IEEE_TABLE[0][((s & 0xFF) as base.u8) ^ args.x[0]] ^ (s >> 8)
|
||
args.x = args.x[1 ..]
|
||
} endwhile
|
||
|
||
// For short inputs, just do a simple loop.
|
||
if args.x.length() < 64 {
|
||
iterate (p = args.x)(length: 1, advance: 1, unroll: 1) {
|
||
s = IEEE_TABLE[0][((s & 0xFF) as base.u8) ^ p[0]] ^ (s >> 8)
|
||
}
|
||
this.state = 0xFFFF_FFFF ^ s
|
||
return nothing
|
||
}
|
||
|
||
// Load 128×4 = 512 bits from the first 64-byte chunk.
|
||
x0 = util.make_m128i_slice128(a: args.x[0x00 .. 0x10])
|
||
x1 = util.make_m128i_slice128(a: args.x[0x10 .. 0x20])
|
||
x2 = util.make_m128i_slice128(a: args.x[0x20 .. 0x30])
|
||
x3 = util.make_m128i_slice128(a: args.x[0x30 .. 0x40])
|
||
|
||
// Combine with the initial state.
|
||
x0 = x0._mm_xor_si128(b: util.make_m128i_single_u32(a: s))
|
||
|
||
// Process the remaining 64-byte chunks.
|
||
k = util.make_m128i_slice128(a: IEEE_X86_SSE42_K1K2[.. 16])
|
||
iterate (p = args.x[64 ..])(length: 64, advance: 64, unroll: 1) {
|
||
y0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||
y1 = x1._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||
y2 = x2._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||
y3 = x3._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||
|
||
x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x11)
|
||
x1 = x1._mm_clmulepi64_si128(b: k, imm8: 0x11)
|
||
x2 = x2._mm_clmulepi64_si128(b: k, imm8: 0x11)
|
||
x3 = x3._mm_clmulepi64_si128(b: k, imm8: 0x11)
|
||
|
||
x0 = x0._mm_xor_si128(b: y0)._mm_xor_si128(b: util.make_m128i_slice128(a: p[0x00 .. 0x10]))
|
||
x1 = x1._mm_xor_si128(b: y1)._mm_xor_si128(b: util.make_m128i_slice128(a: p[0x10 .. 0x20]))
|
||
x2 = x2._mm_xor_si128(b: y2)._mm_xor_si128(b: util.make_m128i_slice128(a: p[0x20 .. 0x30]))
|
||
x3 = x3._mm_xor_si128(b: y3)._mm_xor_si128(b: util.make_m128i_slice128(a: p[0x30 .. 0x40]))
|
||
}
|
||
|
||
// Reduce 128×4 = 512 bits to 128 bits.
|
||
k = util.make_m128i_slice128(a: IEEE_X86_SSE42_K3K4[.. 16])
|
||
y0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||
x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x11)
|
||
x0 = x0._mm_xor_si128(b: x1)
|
||
x0 = x0._mm_xor_si128(b: y0)
|
||
y0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||
x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x11)
|
||
x0 = x0._mm_xor_si128(b: x2)
|
||
x0 = x0._mm_xor_si128(b: y0)
|
||
y0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||
x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x11)
|
||
x0 = x0._mm_xor_si128(b: x3)
|
||
x0 = x0._mm_xor_si128(b: y0)
|
||
|
||
// Reduce 128 bits to 64 bits.
|
||
x1 = x0._mm_clmulepi64_si128(b: k, imm8: 0x10)
|
||
x2 = util.make_m128i_multiple_u32(
|
||
a00: 0xFFFF_FFFF,
|
||
a01: 0x0000_0000,
|
||
a02: 0xFFFF_FFFF,
|
||
a03: 0x0000_0000)
|
||
x0 = x0._mm_srli_si128(imm8: 8)
|
||
x0 = x0._mm_xor_si128(b: x1)
|
||
k = util.make_m128i_slice128(a: IEEE_X86_SSE42_K5ZZ[.. 16])
|
||
x1 = x0._mm_srli_si128(imm8: 4)
|
||
x0 = x0._mm_and_si128(b: x2)
|
||
x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||
x0 = x0._mm_xor_si128(b: x1)
|
||
|
||
// Reduce 64 bits to 32 bits (Barrett Reduction) and extract.
|
||
//
|
||
// Barrett Reduction is Algorithm 1 (page 14) of Gopal et al., after
|
||
// adjusting for bit-reflection as per Figure 12 (page 21).
|
||
k = util.make_m128i_slice128(a: IEEE_X86_SSE42_PXMU[.. 16])
|
||
x1 = x0._mm_and_si128(b: x2)
|
||
x1 = x1._mm_clmulepi64_si128(b: k, imm8: 0x10)
|
||
x1 = x1._mm_and_si128(b: x2)
|
||
x1 = x1._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||
x0 = x0._mm_xor_si128(b: x1)
|
||
s = x0._mm_extract_epi32(imm8: 1)
|
||
|
||
// Handle the tail of args.x that wasn't a complete 64-byte chunk.
|
||
tail_index = args.x.length() & 0xFFFF_FFFF_FFFF_FFC0 // And-not 64.
|
||
if tail_index < args.x.length() {
|
||
iterate (p = args.x[tail_index ..])(length: 1, advance: 1, unroll: 1) {
|
||
s = IEEE_TABLE[0][((s & 0xFF) as base.u8) ^ p[0]] ^ (s >> 8)
|
||
}
|
||
}
|
||
|
||
this.state = 0xFFFF_FFFF ^ s
|
||
}
|
||
|
||
// These constants come from page 22 of Gopal et al. They are also reproduced
|
||
// by script/print-crc32-x86-sse42-magic-numbers.go which is runnable online at
|
||
// https://play.golang.org/p/wH1q6GfhKOE
|
||
//
|
||
// The k6' constant from the Gopal paper is unused.
|
||
|
||
pri const IEEE_X86_SSE42_K1K2 : array[16] base.u8 = [
|
||
0xD4, 0x2B, 0x44, 0x54, 0x01, 0x00, 0x00, 0x00, // k1' = 0x1_5444_2BD4
|
||
0x96, 0x15, 0xE4, 0xC6, 0x01, 0x00, 0x00, 0x00, // k2' = 0x1_C6E4_1596
|
||
]
|
||
|
||
pri const IEEE_X86_SSE42_K3K4 : array[16] base.u8 = [
|
||
0xD0, 0x97, 0x19, 0x75, 0x01, 0x00, 0x00, 0x00, // k3' = 0x1_7519_97D0
|
||
0x9E, 0x00, 0xAA, 0xCC, 0x00, 0x00, 0x00, 0x00, // k4' = 0x0_CCAA_009E
|
||
]
|
||
|
||
pri const IEEE_X86_SSE42_K5ZZ : array[16] base.u8 = [
|
||
0x24, 0x61, 0xCD, 0x63, 0x01, 0x00, 0x00, 0x00, // k5' = 0x1_63CD_6124
|
||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // Unused
|
||
]
|
||
|
||
pri const IEEE_X86_SSE42_PXMU : array[16] base.u8 = [
|
||
0x41, 0x06, 0x71, 0xDB, 0x01, 0x00, 0x00, 0x00, // Px' = 0x1_DB71_0641
|
||
0x41, 0x16, 0x01, 0xF7, 0x01, 0x00, 0x00, 0x00, // μ' = 0x1_F701_1641
|
||
]
|