mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-17 16:32:11 +01:00
SPDX is an ISO standard for describing licenses in a computer readable format, which is useful when applying license compliance tools and the like to source code. In addition to ensuring that the Wuffs C distribution is properly annotated to the benefit of projects using Wuffs and SPDX-supporting tools, add it everywhere else where there's a license in this repo for consistency, too.
158 lines
6.1 KiB
Plaintext
158 lines
6.1 KiB
Plaintext
// Copyright 2021 The Wuffs Authors.
|
||
//
|
||
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
|
||
// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
|
||
// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
|
||
// option. This file may not be copied, modified, or distributed
|
||
// except according to those terms.
|
||
//
|
||
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
||
|
||
// --------
|
||
|
||
// See "SIMD Implementations" in README.md for a link to Gopal et al. "Fast CRC
|
||
// Computation for Generic Polynomials Using PCLMULQDQ Instruction".
|
||
|
||
pri func ieee_hasher.up_x86_sse42!(x: roslice base.u8),
|
||
choose cpu_arch >= x86_sse42,
|
||
{
|
||
var s : base.u32
|
||
var p : roslice base.u8
|
||
|
||
var util : base.x86_sse42_utility
|
||
var k : base.x86_m128i
|
||
var x0 : base.x86_m128i
|
||
var x1 : base.x86_m128i
|
||
var x2 : base.x86_m128i
|
||
var x3 : base.x86_m128i
|
||
var y0 : base.x86_m128i
|
||
var y1 : base.x86_m128i
|
||
var y2 : base.x86_m128i
|
||
var y3 : base.x86_m128i
|
||
|
||
var tail_index : base.u64
|
||
|
||
s = 0xFFFF_FFFF ^ this.state
|
||
|
||
// Align to a 16-byte boundary.
|
||
while (args.x.length() > 0) and ((15 & args.x.uintptr_low_12_bits()) <> 0) {
|
||
s = IEEE_TABLE[0][((s & 0xFF) as base.u8) ^ args.x[0]] ^ (s >> 8)
|
||
args.x = args.x[1 ..]
|
||
} endwhile
|
||
|
||
// For short inputs, just do a simple loop.
|
||
if args.x.length() < 64 {
|
||
iterate (p = args.x)(length: 1, advance: 1, unroll: 1) {
|
||
s = IEEE_TABLE[0][((s & 0xFF) as base.u8) ^ p[0]] ^ (s >> 8)
|
||
}
|
||
this.state = 0xFFFF_FFFF ^ s
|
||
return nothing
|
||
}
|
||
|
||
// Load 128×4 = 512 bits from the first 64-byte chunk.
|
||
x0 = util.make_m128i_slice128(a: args.x[0x00 .. 0x10])
|
||
x1 = util.make_m128i_slice128(a: args.x[0x10 .. 0x20])
|
||
x2 = util.make_m128i_slice128(a: args.x[0x20 .. 0x30])
|
||
x3 = util.make_m128i_slice128(a: args.x[0x30 .. 0x40])
|
||
|
||
// Combine with the initial state.
|
||
x0 = x0._mm_xor_si128(b: util.make_m128i_single_u32(a: s))
|
||
|
||
// Process the remaining 64-byte chunks.
|
||
k = util.make_m128i_slice128(a: IEEE_X86_SSE42_K1K2[.. 16])
|
||
iterate (p = args.x[64 ..])(length: 64, advance: 64, unroll: 1) {
|
||
y0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||
y1 = x1._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||
y2 = x2._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||
y3 = x3._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||
|
||
x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x11)
|
||
x1 = x1._mm_clmulepi64_si128(b: k, imm8: 0x11)
|
||
x2 = x2._mm_clmulepi64_si128(b: k, imm8: 0x11)
|
||
x3 = x3._mm_clmulepi64_si128(b: k, imm8: 0x11)
|
||
|
||
x0 = x0._mm_xor_si128(b: y0)._mm_xor_si128(b: util.make_m128i_slice128(a: p[0x00 .. 0x10]))
|
||
x1 = x1._mm_xor_si128(b: y1)._mm_xor_si128(b: util.make_m128i_slice128(a: p[0x10 .. 0x20]))
|
||
x2 = x2._mm_xor_si128(b: y2)._mm_xor_si128(b: util.make_m128i_slice128(a: p[0x20 .. 0x30]))
|
||
x3 = x3._mm_xor_si128(b: y3)._mm_xor_si128(b: util.make_m128i_slice128(a: p[0x30 .. 0x40]))
|
||
}
|
||
|
||
// Reduce 128×4 = 512 bits to 128 bits.
|
||
k = util.make_m128i_slice128(a: IEEE_X86_SSE42_K3K4[.. 16])
|
||
y0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||
x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x11)
|
||
x0 = x0._mm_xor_si128(b: x1)
|
||
x0 = x0._mm_xor_si128(b: y0)
|
||
y0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||
x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x11)
|
||
x0 = x0._mm_xor_si128(b: x2)
|
||
x0 = x0._mm_xor_si128(b: y0)
|
||
y0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||
x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x11)
|
||
x0 = x0._mm_xor_si128(b: x3)
|
||
x0 = x0._mm_xor_si128(b: y0)
|
||
|
||
// Reduce 128 bits to 64 bits.
|
||
x1 = x0._mm_clmulepi64_si128(b: k, imm8: 0x10)
|
||
x2 = util.make_m128i_multiple_u32(
|
||
a00: 0xFFFF_FFFF,
|
||
a01: 0x0000_0000,
|
||
a02: 0xFFFF_FFFF,
|
||
a03: 0x0000_0000)
|
||
x0 = x0._mm_srli_si128(imm8: 8)
|
||
x0 = x0._mm_xor_si128(b: x1)
|
||
k = util.make_m128i_slice128(a: IEEE_X86_SSE42_K5ZZ[.. 16])
|
||
x1 = x0._mm_srli_si128(imm8: 4)
|
||
x0 = x0._mm_and_si128(b: x2)
|
||
x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||
x0 = x0._mm_xor_si128(b: x1)
|
||
|
||
// Reduce 64 bits to 32 bits (Barrett Reduction) and extract.
|
||
//
|
||
// Barrett Reduction is Algorithm 1 (page 14) of Gopal et al., after
|
||
// adjusting for bit-reflection as per Figure 12 (page 21).
|
||
k = util.make_m128i_slice128(a: IEEE_X86_SSE42_PXMU[.. 16])
|
||
x1 = x0._mm_and_si128(b: x2)
|
||
x1 = x1._mm_clmulepi64_si128(b: k, imm8: 0x10)
|
||
x1 = x1._mm_and_si128(b: x2)
|
||
x1 = x1._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||
x0 = x0._mm_xor_si128(b: x1)
|
||
s = x0._mm_extract_epi32(imm8: 1)
|
||
|
||
// Handle the tail of args.x that wasn't a complete 64-byte chunk.
|
||
tail_index = args.x.length() & 0xFFFF_FFFF_FFFF_FFC0 // And-not 64.
|
||
if tail_index < args.x.length() {
|
||
iterate (p = args.x[tail_index ..])(length: 1, advance: 1, unroll: 1) {
|
||
s = IEEE_TABLE[0][((s & 0xFF) as base.u8) ^ p[0]] ^ (s >> 8)
|
||
}
|
||
}
|
||
|
||
this.state = 0xFFFF_FFFF ^ s
|
||
}
|
||
|
||
// These constants come from page 22 of Gopal et al. They are also reproduced
|
||
// by script/print-crc32-x86-sse42-magic-numbers.go which is runnable online at
|
||
// https://play.golang.org/p/wH1q6GfhKOE
|
||
//
|
||
// The k6' constant from the Gopal paper is unused.
|
||
|
||
pri const IEEE_X86_SSE42_K1K2 : roarray[16] base.u8 = [
|
||
0xD4, 0x2B, 0x44, 0x54, 0x01, 0x00, 0x00, 0x00, // k1' = 0x1_5444_2BD4
|
||
0x96, 0x15, 0xE4, 0xC6, 0x01, 0x00, 0x00, 0x00, // k2' = 0x1_C6E4_1596
|
||
]
|
||
|
||
pri const IEEE_X86_SSE42_K3K4 : roarray[16] base.u8 = [
|
||
0xD0, 0x97, 0x19, 0x75, 0x01, 0x00, 0x00, 0x00, // k3' = 0x1_7519_97D0
|
||
0x9E, 0x00, 0xAA, 0xCC, 0x00, 0x00, 0x00, 0x00, // k4' = 0x0_CCAA_009E
|
||
]
|
||
|
||
pri const IEEE_X86_SSE42_K5ZZ : roarray[16] base.u8 = [
|
||
0x24, 0x61, 0xCD, 0x63, 0x01, 0x00, 0x00, 0x00, // k5' = 0x1_63CD_6124
|
||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // Unused
|
||
]
|
||
|
||
pri const IEEE_X86_SSE42_PXMU : roarray[16] base.u8 = [
|
||
0x41, 0x06, 0x71, 0xDB, 0x01, 0x00, 0x00, 0x00, // Px' = 0x1_DB71_0641
|
||
0x41, 0x16, 0x01, 0xF7, 0x01, 0x00, 0x00, 0x00, // μ' = 0x1_F701_1641
|
||
]
|