mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-22 16:47:28 +01:00
198 lines
7.8 KiB
Plaintext
198 lines
7.8 KiB
Plaintext
// Copyright 2021 The Wuffs Authors.
|
||
//
|
||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||
// you may not use this file except in compliance with the License.
|
||
// You may obtain a copy of the License at
|
||
//
|
||
// https://www.apache.org/licenses/LICENSE-2.0
|
||
//
|
||
// Unless required by applicable law or agreed to in writing, software
|
||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
// See the License for the specific language governing permissions and
|
||
// limitations under the License.
|
||
|
||
// --------
|
||
|
||
// See "SIMD Implementations" in README.md for a link to Gopal et al. "Fast CRC
|
||
// Computation for Generic Polynomials Using PCLMULQDQ Instruction".
|
||
|
||
pri func ieee_hasher.up_x86_sse42!(x: slice base.u8),
|
||
choose cpu_arch >= x86_sse42,
|
||
{
|
||
var s : base.u32
|
||
var p : slice base.u8
|
||
|
||
var util : base.x86_sse42_utility
|
||
var k : base.x86_m128i
|
||
var x0 : base.x86_m128i
|
||
var x1 : base.x86_m128i
|
||
var x2 : base.x86_m128i
|
||
var x3 : base.x86_m128i
|
||
var y0 : base.x86_m128i
|
||
var y1 : base.x86_m128i
|
||
var y2 : base.x86_m128i
|
||
var y3 : base.x86_m128i
|
||
|
||
var tail_index : base.u64
|
||
|
||
s = 0xFFFF_FFFF ^ this.state
|
||
|
||
// Align to a 16-byte boundary.
|
||
while (args.x.length() > 0) and ((15 & args.x.uintptr_low_12_bits()) <> 0) {
|
||
s = IEEE_TABLE[0][((s & 0xFF) as base.u8) ^ args.x[0]] ^ (s >> 8)
|
||
args.x = args.x[1 ..]
|
||
} endwhile
|
||
|
||
// For short inputs, just do a simple loop.
|
||
if args.x.length() < 64 {
|
||
iterate (p = args.x)(length: 1, advance: 1, unroll: 1) {
|
||
s = IEEE_TABLE[0][((s & 0xFF) as base.u8) ^ p[0]] ^ (s >> 8)
|
||
}
|
||
this.state = 0xFFFF_FFFF ^ s
|
||
return nothing
|
||
}
|
||
|
||
// Load 128×4 = 512 bits from the first 64-byte chunk.
|
||
x0 = util.make_m128i_slice128(a: args.x[0x00 .. 0x10])
|
||
x1 = util.make_m128i_slice128(a: args.x[0x10 .. 0x20])
|
||
x2 = util.make_m128i_slice128(a: args.x[0x20 .. 0x30])
|
||
x3 = util.make_m128i_slice128(a: args.x[0x30 .. 0x40])
|
||
|
||
// Combine with the initial state.
|
||
x0 = x0._mm_xor_si128(b: util.make_m128i_single_u32(a: s))
|
||
|
||
// Process the remaining 64-byte chunks.
|
||
k = util.make_m128i_slice128(a: IEEE_X86_SSE42_K1K2[.. 16])
|
||
iterate (p = args.x[64 ..])(length: 64, advance: 64, unroll: 1) {
|
||
y0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||
y1 = x1._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||
y2 = x2._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||
y3 = x3._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||
|
||
x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x11)
|
||
x1 = x1._mm_clmulepi64_si128(b: k, imm8: 0x11)
|
||
x2 = x2._mm_clmulepi64_si128(b: k, imm8: 0x11)
|
||
x3 = x3._mm_clmulepi64_si128(b: k, imm8: 0x11)
|
||
|
||
x0 = x0._mm_xor_si128(b: y0)._mm_xor_si128(b: util.make_m128i_slice128(a: p[0x00 .. 0x10]))
|
||
x1 = x1._mm_xor_si128(b: y1)._mm_xor_si128(b: util.make_m128i_slice128(a: p[0x10 .. 0x20]))
|
||
x2 = x2._mm_xor_si128(b: y2)._mm_xor_si128(b: util.make_m128i_slice128(a: p[0x20 .. 0x30]))
|
||
x3 = x3._mm_xor_si128(b: y3)._mm_xor_si128(b: util.make_m128i_slice128(a: p[0x30 .. 0x40]))
|
||
}
|
||
|
||
// Reduce 128×4 = 512 bits to 128 bits.
|
||
k = util.make_m128i_slice128(a: IEEE_X86_SSE42_K3K4[.. 16])
|
||
y0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||
x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x11)
|
||
x0 = x0._mm_xor_si128(b: x1)
|
||
x0 = x0._mm_xor_si128(b: y0)
|
||
y0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||
x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x11)
|
||
x0 = x0._mm_xor_si128(b: x2)
|
||
x0 = x0._mm_xor_si128(b: y0)
|
||
y0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||
x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x11)
|
||
x0 = x0._mm_xor_si128(b: x3)
|
||
x0 = x0._mm_xor_si128(b: y0)
|
||
|
||
// Reduce 128 bits to 64 bits.
|
||
x1 = x0._mm_clmulepi64_si128(b: k, imm8: 0x10)
|
||
x2 = util.make_m128i_multiple_u32(
|
||
a00: 0xFFFF_FFFF,
|
||
a01: 0x0000_0000,
|
||
a02: 0xFFFF_FFFF,
|
||
a03: 0x0000_0000)
|
||
x0 = x0._mm_srli_si128(imm8: 8)
|
||
x0 = x0._mm_xor_si128(b: x1)
|
||
k = util.make_m128i_slice128(a: IEEE_X86_SSE42_K5ZZ[.. 16])
|
||
x1 = x0._mm_srli_si128(imm8: 4)
|
||
x0 = x0._mm_and_si128(b: x2)
|
||
x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||
x0 = x0._mm_xor_si128(b: x1)
|
||
|
||
// Reduce 64 bits to 32 bits and extract.
|
||
k = util.make_m128i_slice128(a: IEEE_X86_SSE42_K6MU[.. 16])
|
||
x1 = x0._mm_and_si128(b: x2)
|
||
x1 = x1._mm_clmulepi64_si128(b: k, imm8: 0x10)
|
||
x1 = x1._mm_and_si128(b: x2)
|
||
x1 = x1._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||
x0 = x0._mm_xor_si128(b: x1)
|
||
s = x0._mm_extract_epi32(imm8: 1)
|
||
|
||
// Handle the tail of args.x that wasn't a complete 64-byte chunk.
|
||
tail_index = args.x.length() & 0xFFFF_FFFF_FFFF_FFC0 // And-not 64.
|
||
if tail_index < args.x.length() {
|
||
iterate (p = args.x[tail_index ..])(length: 1, advance: 1, unroll: 1) {
|
||
s = IEEE_TABLE[0][((s & 0xFF) as base.u8) ^ p[0]] ^ (s >> 8)
|
||
}
|
||
}
|
||
|
||
this.state = 0xFFFF_FFFF ^ s
|
||
}
|
||
|
||
// These constants come from page 22 of Gopal et al. They are also reproduced
|
||
// by script/print-crc32-x86-sse42-magic-numbers.go which is runnable online at
|
||
// https://play.golang.org/p/wH1q6GfhKOE
|
||
//
|
||
// (†) There is a discrepancy in the k6' constant. The Gopal paper gives the
|
||
// least significant byte as 0x40 and this is confirmed by the reproduction
|
||
// script. Indeed, the formula on page 22 ends with a left-shift by 1, so the
|
||
// final hex digit must be even.
|
||
//
|
||
// Nonetheless, we use 0x41 here, the same as the Linux kernel
|
||
// (https://github.com/torvalds/linux/blob/v5.11/arch/x86/crypto/crc32-pclmul_asm.S#L72).
|
||
// It's there in 2021 unchanged since that code was introduced in 2013
|
||
// (https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id=78c37d191dd6899d8c219fee597a17d6e3c5d288).
|
||
//
|
||
// The Linux kernel's 0x41 (instead of 0x40) also turns up in e.g. Chromium
|
||
// (https://source.chromium.org/chromium/chromium/src/+/master:third_party/zlib/crc32_simd.c;l=36;drc=9d4ec9349a1bf609eedb917c44c69eb0df9ff6bb)
|
||
// and Go
|
||
// (https://github.com/golang/go/blob/414fa8c35e7c2f65e2c767d6db2f25791e53b5c1/src/hash/crc32/crc32_amd64.s#L143).
|
||
// The 0x41 has surely been copy/pasted to other software too.
|
||
//
|
||
// Code (0x41) and documentation (0x40) disagree. We went with code here, for
|
||
// interoperability. Even though it's mathematical documentation, it's very
|
||
// widely used code.
|
||
//
|
||
// For academic curiosity, there are at least two possible sources of the k6'
|
||
// constant discrepancy.
|
||
//
|
||
// One is a transcription error. The Gopal paper has two consecutive lines:
|
||
// k6' = ... = 0x1DB710640
|
||
// μ' = ... = 0x1F7011641
|
||
// The final hex digits are 640 and 641.
|
||
//
|
||
// Two is that there's also this line a little earlier on the same page:
|
||
// P(x)' = 0x1DB710641
|
||
// This P(x)' number differs from k6' only in the final bit: 0x41 vs 0x40. This
|
||
// isn't coincidental. Calculating k6' (in the Galois Field GF(2) space) means
|
||
// dividing (i.e. bitwise XOR-ing) a 33-bit polynomial (x³², with only one 'on'
|
||
// bit) by the 33-bit polynomial P(x)'.
|
||
//
|
||
// The discrepancy might not matter in practice because Barrett reduction is
|
||
// already an integer approximation to a fraction. 0x40 vs 0x41 is essentially
|
||
// rounding down vs up. The approximation diverges for large inputs but might
|
||
// not matter for 'small' inputs that fit into e.g. a uint32 or uint64. The
|
||
// tests seem to pass either way (and fail with 0x3F or 0x42).
|
||
|
||
pri const IEEE_X86_SSE42_K1K2 : array[16] base.u8 = [
|
||
0xD4, 0x2B, 0x44, 0x54, 0x01, 0x00, 0x00, 0x00, // k1' = 0x1_5444_2BD4
|
||
0x96, 0x15, 0xE4, 0xC6, 0x01, 0x00, 0x00, 0x00, // k2' = 0x1_C6E4_1596
|
||
]
|
||
|
||
pri const IEEE_X86_SSE42_K3K4 : array[16] base.u8 = [
|
||
0xD0, 0x97, 0x19, 0x75, 0x01, 0x00, 0x00, 0x00, // k3' = 0x1_7519_97D0
|
||
0x9E, 0x00, 0xAA, 0xCC, 0x00, 0x00, 0x00, 0x00, // k4' = 0x0_CCAA_009E
|
||
]
|
||
|
||
pri const IEEE_X86_SSE42_K5ZZ : array[16] base.u8 = [
|
||
0x24, 0x61, 0xCD, 0x63, 0x01, 0x00, 0x00, 0x00, // k5' = 0x1_63CD_6124
|
||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // Unused
|
||
]
|
||
|
||
pri const IEEE_X86_SSE42_K6MU : array[16] base.u8 = [
|
||
0x41, 0x06, 0x71, 0xDB, 0x01, 0x00, 0x00, 0x00, // k6' = 0x1_DB71_0640 (†)
|
||
0x41, 0x16, 0x01, 0xF7, 0x01, 0x00, 0x00, 0x00, // μ' = 0x1_F701_1641
|
||
]
|