Files
wuffs/std/crc32/common_up_x86_avx2.wuffs
T

139 lines
5.4 KiB
Plaintext
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// Copyright 2021 The Wuffs Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// --------
// See "SIMD Implementations" in README.md for a link to Gopal et al. "Fast CRC
// Computation for Generic Polynomials Using PCLMULQDQ Instruction".
// up_x86_avx2 is exactly the same as up_x86_sse42 except for the "choose
// cpu_arch >= x86_avx2". With AVX, PCLMULQDQ has a three-operand form, not
// just a two-operand form: https://www.felixcloutier.com/x86/pclmulqdq
pri func ieee_hasher.up_x86_avx2!(x: roslice base.u8),
choose cpu_arch >= x86_avx2,
{
var s : base.u32
var p : roslice base.u8
var util : base.x86_sse42_utility
var k : base.x86_m128i
var x0 : base.x86_m128i
var x1 : base.x86_m128i
var x2 : base.x86_m128i
var x3 : base.x86_m128i
var y0 : base.x86_m128i
var y1 : base.x86_m128i
var y2 : base.x86_m128i
var y3 : base.x86_m128i
var tail_index : base.u64
s = 0xFFFF_FFFF ^ this.state
// Align to a 16-byte boundary.
while (args.x.length() > 0) and ((15 & args.x.uintptr_low_12_bits()) <> 0) {
s = IEEE_TABLE[0][((s & 0xFF) as base.u8) ^ args.x[0]] ^ (s >> 8)
args.x = args.x[1 ..]
} endwhile
// For short inputs, just do a simple loop.
if args.x.length() < 64 {
iterate (p = args.x)(length: 1, advance: 1, unroll: 1) {
s = IEEE_TABLE[0][((s & 0xFF) as base.u8) ^ p[0]] ^ (s >> 8)
}
this.state = 0xFFFF_FFFF ^ s
return nothing
}
// Load 128×4 = 512 bits from the first 64-byte chunk.
x0 = util.make_m128i_slice128(a: args.x[0x00 .. 0x10])
x1 = util.make_m128i_slice128(a: args.x[0x10 .. 0x20])
x2 = util.make_m128i_slice128(a: args.x[0x20 .. 0x30])
x3 = util.make_m128i_slice128(a: args.x[0x30 .. 0x40])
// Combine with the initial state.
x0 = x0._mm_xor_si128(b: util.make_m128i_single_u32(a: s))
// Process the remaining 64-byte chunks.
k = util.make_m128i_slice128(a: IEEE_X86_SSE42_K1K2[.. 16])
iterate (p = args.x[64 ..])(length: 64, advance: 64, unroll: 1) {
y0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00)
y1 = x1._mm_clmulepi64_si128(b: k, imm8: 0x00)
y2 = x2._mm_clmulepi64_si128(b: k, imm8: 0x00)
y3 = x3._mm_clmulepi64_si128(b: k, imm8: 0x00)
x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x11)
x1 = x1._mm_clmulepi64_si128(b: k, imm8: 0x11)
x2 = x2._mm_clmulepi64_si128(b: k, imm8: 0x11)
x3 = x3._mm_clmulepi64_si128(b: k, imm8: 0x11)
x0 = x0._mm_xor_si128(b: y0)._mm_xor_si128(b: util.make_m128i_slice128(a: p[0x00 .. 0x10]))
x1 = x1._mm_xor_si128(b: y1)._mm_xor_si128(b: util.make_m128i_slice128(a: p[0x10 .. 0x20]))
x2 = x2._mm_xor_si128(b: y2)._mm_xor_si128(b: util.make_m128i_slice128(a: p[0x20 .. 0x30]))
x3 = x3._mm_xor_si128(b: y3)._mm_xor_si128(b: util.make_m128i_slice128(a: p[0x30 .. 0x40]))
}
// Reduce 128×4 = 512 bits to 128 bits.
k = util.make_m128i_slice128(a: IEEE_X86_SSE42_K3K4[.. 16])
y0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00)
x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x11)
x0 = x0._mm_xor_si128(b: x1)
x0 = x0._mm_xor_si128(b: y0)
y0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00)
x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x11)
x0 = x0._mm_xor_si128(b: x2)
x0 = x0._mm_xor_si128(b: y0)
y0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00)
x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x11)
x0 = x0._mm_xor_si128(b: x3)
x0 = x0._mm_xor_si128(b: y0)
// Reduce 128 bits to 64 bits.
x1 = x0._mm_clmulepi64_si128(b: k, imm8: 0x10)
x2 = util.make_m128i_multiple_u32(
a00: 0xFFFF_FFFF,
a01: 0x0000_0000,
a02: 0xFFFF_FFFF,
a03: 0x0000_0000)
x0 = x0._mm_srli_si128(imm8: 8)
x0 = x0._mm_xor_si128(b: x1)
k = util.make_m128i_slice128(a: IEEE_X86_SSE42_K5ZZ[.. 16])
x1 = x0._mm_srli_si128(imm8: 4)
x0 = x0._mm_and_si128(b: x2)
x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00)
x0 = x0._mm_xor_si128(b: x1)
// Reduce 64 bits to 32 bits (Barrett Reduction) and extract.
//
// Barrett Reduction is Algorithm 1 (page 14) of Gopal et al., after
// adjusting for bit-reflection as per Figure 12 (page 21).
k = util.make_m128i_slice128(a: IEEE_X86_SSE42_PXMU[.. 16])
x1 = x0._mm_and_si128(b: x2)
x1 = x1._mm_clmulepi64_si128(b: k, imm8: 0x10)
x1 = x1._mm_and_si128(b: x2)
x1 = x1._mm_clmulepi64_si128(b: k, imm8: 0x00)
x0 = x0._mm_xor_si128(b: x1)
s = x0._mm_extract_epi32(imm8: 1)
// Handle the tail of args.x that wasn't a complete 64-byte chunk.
tail_index = args.x.length() & 0xFFFF_FFFF_FFFF_FFC0 // And-not 64.
if tail_index < args.x.length() {
iterate (p = args.x[tail_index ..])(length: 1, advance: 1, unroll: 1) {
s = IEEE_TABLE[0][((s & 0xFF) as base.u8) ^ p[0]] ^ (s >> 8)
}
}
this.state = 0xFFFF_FFFF ^ s
}