mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-18 00:32:11 +01:00
std/crc32: tweak x86 to be more like std/crc64
This commit is contained in:
1 parent
5bf4c2cb51
commit
9c704c4a6a
3 files changed
+112
-100
No files matched your search
@@ -36085,7 +36085,10 @@ wuffs_crc32__ieee_hasher__up_x86_avx2(
|
||||
wuffs_base__slice_u8 a_x) {
|
||||
uint32_t v_s = 0;
|
||||
wuffs_base__slice_u8 v_p = {0};
|
||||
__m128i v_k = {0};
|
||||
__m128i v_k1k2 = {0};
|
||||
__m128i v_k3k4 = {0};
|
||||
__m128i v_k5zz = {0};
|
||||
__m128i v_pxmu = {0};
|
||||
__m128i v_x0 = {0};
|
||||
__m128i v_x1 = {0};
|
||||
__m128i v_x2 = {0};
|
||||
@@ -36121,21 +36124,21 @@ wuffs_crc32__ieee_hasher__up_x86_avx2(
|
||||
v_x2 = _mm_lddqu_si128((const __m128i*)(const void*)(a_x.ptr + 32u));
|
||||
v_x3 = _mm_lddqu_si128((const __m128i*)(const void*)(a_x.ptr + 48u));
|
||||
v_x0 = _mm_xor_si128(v_x0, _mm_cvtsi32_si128((int32_t)(v_s)));
|
||||
v_k = _mm_lddqu_si128((const __m128i*)(const void*)(WUFFS_CRC32__IEEE_X86_SSE42_K1K2));
|
||||
v_k1k2 = _mm_lddqu_si128((const __m128i*)(const void*)(WUFFS_CRC32__IEEE_X86_SSE42_K1K2));
|
||||
{
|
||||
wuffs_base__slice_u8 i_slice_p = wuffs_base__slice_u8__subslice_i(a_x, 64u);
|
||||
v_p.ptr = i_slice_p.ptr;
|
||||
v_p.len = 64;
|
||||
const uint8_t* i_end0_p = wuffs_private_impl__ptr_u8_plus_len(v_p.ptr, (((i_slice_p.len - (size_t)(v_p.ptr - i_slice_p.ptr)) / 64) * 64));
|
||||
while (v_p.ptr < i_end0_p) {
|
||||
v_y0 = _mm_clmulepi64_si128(v_x0, v_k, (int32_t)(0u));
|
||||
v_y1 = _mm_clmulepi64_si128(v_x1, v_k, (int32_t)(0u));
|
||||
v_y2 = _mm_clmulepi64_si128(v_x2, v_k, (int32_t)(0u));
|
||||
v_y3 = _mm_clmulepi64_si128(v_x3, v_k, (int32_t)(0u));
|
||||
v_x0 = _mm_clmulepi64_si128(v_x0, v_k, (int32_t)(17u));
|
||||
v_x1 = _mm_clmulepi64_si128(v_x1, v_k, (int32_t)(17u));
|
||||
v_x2 = _mm_clmulepi64_si128(v_x2, v_k, (int32_t)(17u));
|
||||
v_x3 = _mm_clmulepi64_si128(v_x3, v_k, (int32_t)(17u));
|
||||
v_y0 = _mm_clmulepi64_si128(v_x0, v_k1k2, (int32_t)(0u));
|
||||
v_y1 = _mm_clmulepi64_si128(v_x1, v_k1k2, (int32_t)(0u));
|
||||
v_y2 = _mm_clmulepi64_si128(v_x2, v_k1k2, (int32_t)(0u));
|
||||
v_y3 = _mm_clmulepi64_si128(v_x3, v_k1k2, (int32_t)(0u));
|
||||
v_x0 = _mm_clmulepi64_si128(v_x0, v_k1k2, (int32_t)(17u));
|
||||
v_x1 = _mm_clmulepi64_si128(v_x1, v_k1k2, (int32_t)(17u));
|
||||
v_x2 = _mm_clmulepi64_si128(v_x2, v_k1k2, (int32_t)(17u));
|
||||
v_x3 = _mm_clmulepi64_si128(v_x3, v_k1k2, (int32_t)(17u));
|
||||
v_x0 = _mm_xor_si128(_mm_xor_si128(v_x0, v_y0), _mm_lddqu_si128((const __m128i*)(const void*)(v_p.ptr + 0u)));
|
||||
v_x1 = _mm_xor_si128(_mm_xor_si128(v_x1, v_y1), _mm_lddqu_si128((const __m128i*)(const void*)(v_p.ptr + 16u)));
|
||||
v_x2 = _mm_xor_si128(_mm_xor_si128(v_x2, v_y2), _mm_lddqu_si128((const __m128i*)(const void*)(v_p.ptr + 32u)));
|
||||
@@ -36144,35 +36147,35 @@ wuffs_crc32__ieee_hasher__up_x86_avx2(
|
||||
}
|
||||
v_p.len = 0;
|
||||
}
|
||||
v_k = _mm_lddqu_si128((const __m128i*)(const void*)(WUFFS_CRC32__IEEE_X86_SSE42_K3K4));
|
||||
v_y0 = _mm_clmulepi64_si128(v_x0, v_k, (int32_t)(0u));
|
||||
v_x0 = _mm_clmulepi64_si128(v_x0, v_k, (int32_t)(17u));
|
||||
v_k3k4 = _mm_lddqu_si128((const __m128i*)(const void*)(WUFFS_CRC32__IEEE_X86_SSE42_K3K4));
|
||||
v_y0 = _mm_clmulepi64_si128(v_x0, v_k3k4, (int32_t)(0u));
|
||||
v_x0 = _mm_clmulepi64_si128(v_x0, v_k3k4, (int32_t)(17u));
|
||||
v_x0 = _mm_xor_si128(v_x0, v_x1);
|
||||
v_x0 = _mm_xor_si128(v_x0, v_y0);
|
||||
v_y0 = _mm_clmulepi64_si128(v_x0, v_k, (int32_t)(0u));
|
||||
v_x0 = _mm_clmulepi64_si128(v_x0, v_k, (int32_t)(17u));
|
||||
v_y0 = _mm_clmulepi64_si128(v_x0, v_k3k4, (int32_t)(0u));
|
||||
v_x0 = _mm_clmulepi64_si128(v_x0, v_k3k4, (int32_t)(17u));
|
||||
v_x0 = _mm_xor_si128(v_x0, v_x2);
|
||||
v_x0 = _mm_xor_si128(v_x0, v_y0);
|
||||
v_y0 = _mm_clmulepi64_si128(v_x0, v_k, (int32_t)(0u));
|
||||
v_x0 = _mm_clmulepi64_si128(v_x0, v_k, (int32_t)(17u));
|
||||
v_y0 = _mm_clmulepi64_si128(v_x0, v_k3k4, (int32_t)(0u));
|
||||
v_x0 = _mm_clmulepi64_si128(v_x0, v_k3k4, (int32_t)(17u));
|
||||
v_x0 = _mm_xor_si128(v_x0, v_x3);
|
||||
v_x0 = _mm_xor_si128(v_x0, v_y0);
|
||||
v_x1 = _mm_clmulepi64_si128(v_x0, v_k, (int32_t)(16u));
|
||||
v_x1 = _mm_clmulepi64_si128(v_x0, v_k3k4, (int32_t)(16u));
|
||||
v_x2 = _mm_set_epi32((int32_t)(0u), (int32_t)(4294967295u), (int32_t)(0u), (int32_t)(4294967295u));
|
||||
v_x0 = _mm_srli_si128(v_x0, (int32_t)(8u));
|
||||
v_x0 = _mm_xor_si128(v_x0, v_x1);
|
||||
v_k = _mm_lddqu_si128((const __m128i*)(const void*)(WUFFS_CRC32__IEEE_X86_SSE42_K5ZZ));
|
||||
v_k5zz = _mm_lddqu_si128((const __m128i*)(const void*)(WUFFS_CRC32__IEEE_X86_SSE42_K5ZZ));
|
||||
v_x1 = _mm_srli_si128(v_x0, (int32_t)(4u));
|
||||
v_x0 = _mm_and_si128(v_x0, v_x2);
|
||||
v_x0 = _mm_clmulepi64_si128(v_x0, v_k, (int32_t)(0u));
|
||||
v_x0 = _mm_clmulepi64_si128(v_x0, v_k5zz, (int32_t)(0u));
|
||||
v_x0 = _mm_xor_si128(v_x0, v_x1);
|
||||
v_k = _mm_lddqu_si128((const __m128i*)(const void*)(WUFFS_CRC32__IEEE_X86_SSE42_PXMU));
|
||||
v_pxmu = _mm_lddqu_si128((const __m128i*)(const void*)(WUFFS_CRC32__IEEE_X86_SSE42_PXMU));
|
||||
v_x1 = _mm_and_si128(v_x0, v_x2);
|
||||
v_x1 = _mm_clmulepi64_si128(v_x1, v_k, (int32_t)(16u));
|
||||
v_x1 = _mm_clmulepi64_si128(v_x1, v_pxmu, (int32_t)(16u));
|
||||
v_x1 = _mm_and_si128(v_x1, v_x2);
|
||||
v_x1 = _mm_clmulepi64_si128(v_x1, v_k, (int32_t)(0u));
|
||||
v_x0 = _mm_xor_si128(v_x0, v_x1);
|
||||
v_s = ((uint32_t)(_mm_extract_epi32(v_x0, (int32_t)(1u))));
|
||||
v_x1 = _mm_clmulepi64_si128(v_x1, v_pxmu, (int32_t)(0u));
|
||||
v_x1 = _mm_xor_si128(v_x1, v_x0);
|
||||
v_s = ((uint32_t)(_mm_extract_epi32(v_x1, (int32_t)(1u))));
|
||||
v_tail_index = (((uint64_t)(a_x.len)) & 18446744073709551552u);
|
||||
if (v_tail_index < ((uint64_t)(a_x.len))) {
|
||||
{
|
||||
@@ -36205,7 +36208,10 @@ wuffs_crc32__ieee_hasher__up_x86_sse42(
|
||||
wuffs_base__slice_u8 a_x) {
|
||||
uint32_t v_s = 0;
|
||||
wuffs_base__slice_u8 v_p = {0};
|
||||
__m128i v_k = {0};
|
||||
__m128i v_k1k2 = {0};
|
||||
__m128i v_k3k4 = {0};
|
||||
__m128i v_k5zz = {0};
|
||||
__m128i v_pxmu = {0};
|
||||
__m128i v_x0 = {0};
|
||||
__m128i v_x1 = {0};
|
||||
__m128i v_x2 = {0};
|
||||
@@ -36241,21 +36247,21 @@ wuffs_crc32__ieee_hasher__up_x86_sse42(
|
||||
v_x2 = _mm_lddqu_si128((const __m128i*)(const void*)(a_x.ptr + 32u));
|
||||
v_x3 = _mm_lddqu_si128((const __m128i*)(const void*)(a_x.ptr + 48u));
|
||||
v_x0 = _mm_xor_si128(v_x0, _mm_cvtsi32_si128((int32_t)(v_s)));
|
||||
v_k = _mm_lddqu_si128((const __m128i*)(const void*)(WUFFS_CRC32__IEEE_X86_SSE42_K1K2));
|
||||
v_k1k2 = _mm_lddqu_si128((const __m128i*)(const void*)(WUFFS_CRC32__IEEE_X86_SSE42_K1K2));
|
||||
{
|
||||
wuffs_base__slice_u8 i_slice_p = wuffs_base__slice_u8__subslice_i(a_x, 64u);
|
||||
v_p.ptr = i_slice_p.ptr;
|
||||
v_p.len = 64;
|
||||
const uint8_t* i_end0_p = wuffs_private_impl__ptr_u8_plus_len(v_p.ptr, (((i_slice_p.len - (size_t)(v_p.ptr - i_slice_p.ptr)) / 64) * 64));
|
||||
while (v_p.ptr < i_end0_p) {
|
||||
v_y0 = _mm_clmulepi64_si128(v_x0, v_k, (int32_t)(0u));
|
||||
v_y1 = _mm_clmulepi64_si128(v_x1, v_k, (int32_t)(0u));
|
||||
v_y2 = _mm_clmulepi64_si128(v_x2, v_k, (int32_t)(0u));
|
||||
v_y3 = _mm_clmulepi64_si128(v_x3, v_k, (int32_t)(0u));
|
||||
v_x0 = _mm_clmulepi64_si128(v_x0, v_k, (int32_t)(17u));
|
||||
v_x1 = _mm_clmulepi64_si128(v_x1, v_k, (int32_t)(17u));
|
||||
v_x2 = _mm_clmulepi64_si128(v_x2, v_k, (int32_t)(17u));
|
||||
v_x3 = _mm_clmulepi64_si128(v_x3, v_k, (int32_t)(17u));
|
||||
v_y0 = _mm_clmulepi64_si128(v_x0, v_k1k2, (int32_t)(0u));
|
||||
v_y1 = _mm_clmulepi64_si128(v_x1, v_k1k2, (int32_t)(0u));
|
||||
v_y2 = _mm_clmulepi64_si128(v_x2, v_k1k2, (int32_t)(0u));
|
||||
v_y3 = _mm_clmulepi64_si128(v_x3, v_k1k2, (int32_t)(0u));
|
||||
v_x0 = _mm_clmulepi64_si128(v_x0, v_k1k2, (int32_t)(17u));
|
||||
v_x1 = _mm_clmulepi64_si128(v_x1, v_k1k2, (int32_t)(17u));
|
||||
v_x2 = _mm_clmulepi64_si128(v_x2, v_k1k2, (int32_t)(17u));
|
||||
v_x3 = _mm_clmulepi64_si128(v_x3, v_k1k2, (int32_t)(17u));
|
||||
v_x0 = _mm_xor_si128(_mm_xor_si128(v_x0, v_y0), _mm_lddqu_si128((const __m128i*)(const void*)(v_p.ptr + 0u)));
|
||||
v_x1 = _mm_xor_si128(_mm_xor_si128(v_x1, v_y1), _mm_lddqu_si128((const __m128i*)(const void*)(v_p.ptr + 16u)));
|
||||
v_x2 = _mm_xor_si128(_mm_xor_si128(v_x2, v_y2), _mm_lddqu_si128((const __m128i*)(const void*)(v_p.ptr + 32u)));
|
||||
@@ -36264,35 +36270,35 @@ wuffs_crc32__ieee_hasher__up_x86_sse42(
|
||||
}
|
||||
v_p.len = 0;
|
||||
}
|
||||
v_k = _mm_lddqu_si128((const __m128i*)(const void*)(WUFFS_CRC32__IEEE_X86_SSE42_K3K4));
|
||||
v_y0 = _mm_clmulepi64_si128(v_x0, v_k, (int32_t)(0u));
|
||||
v_x0 = _mm_clmulepi64_si128(v_x0, v_k, (int32_t)(17u));
|
||||
v_k3k4 = _mm_lddqu_si128((const __m128i*)(const void*)(WUFFS_CRC32__IEEE_X86_SSE42_K3K4));
|
||||
v_y0 = _mm_clmulepi64_si128(v_x0, v_k3k4, (int32_t)(0u));
|
||||
v_x0 = _mm_clmulepi64_si128(v_x0, v_k3k4, (int32_t)(17u));
|
||||
v_x0 = _mm_xor_si128(v_x0, v_x1);
|
||||
v_x0 = _mm_xor_si128(v_x0, v_y0);
|
||||
v_y0 = _mm_clmulepi64_si128(v_x0, v_k, (int32_t)(0u));
|
||||
v_x0 = _mm_clmulepi64_si128(v_x0, v_k, (int32_t)(17u));
|
||||
v_y0 = _mm_clmulepi64_si128(v_x0, v_k3k4, (int32_t)(0u));
|
||||
v_x0 = _mm_clmulepi64_si128(v_x0, v_k3k4, (int32_t)(17u));
|
||||
v_x0 = _mm_xor_si128(v_x0, v_x2);
|
||||
v_x0 = _mm_xor_si128(v_x0, v_y0);
|
||||
v_y0 = _mm_clmulepi64_si128(v_x0, v_k, (int32_t)(0u));
|
||||
v_x0 = _mm_clmulepi64_si128(v_x0, v_k, (int32_t)(17u));
|
||||
v_y0 = _mm_clmulepi64_si128(v_x0, v_k3k4, (int32_t)(0u));
|
||||
v_x0 = _mm_clmulepi64_si128(v_x0, v_k3k4, (int32_t)(17u));
|
||||
v_x0 = _mm_xor_si128(v_x0, v_x3);
|
||||
v_x0 = _mm_xor_si128(v_x0, v_y0);
|
||||
v_x1 = _mm_clmulepi64_si128(v_x0, v_k, (int32_t)(16u));
|
||||
v_x1 = _mm_clmulepi64_si128(v_x0, v_k3k4, (int32_t)(16u));
|
||||
v_x2 = _mm_set_epi32((int32_t)(0u), (int32_t)(4294967295u), (int32_t)(0u), (int32_t)(4294967295u));
|
||||
v_x0 = _mm_srli_si128(v_x0, (int32_t)(8u));
|
||||
v_x0 = _mm_xor_si128(v_x0, v_x1);
|
||||
v_k = _mm_lddqu_si128((const __m128i*)(const void*)(WUFFS_CRC32__IEEE_X86_SSE42_K5ZZ));
|
||||
v_k5zz = _mm_lddqu_si128((const __m128i*)(const void*)(WUFFS_CRC32__IEEE_X86_SSE42_K5ZZ));
|
||||
v_x1 = _mm_srli_si128(v_x0, (int32_t)(4u));
|
||||
v_x0 = _mm_and_si128(v_x0, v_x2);
|
||||
v_x0 = _mm_clmulepi64_si128(v_x0, v_k, (int32_t)(0u));
|
||||
v_x0 = _mm_clmulepi64_si128(v_x0, v_k5zz, (int32_t)(0u));
|
||||
v_x0 = _mm_xor_si128(v_x0, v_x1);
|
||||
v_k = _mm_lddqu_si128((const __m128i*)(const void*)(WUFFS_CRC32__IEEE_X86_SSE42_PXMU));
|
||||
v_pxmu = _mm_lddqu_si128((const __m128i*)(const void*)(WUFFS_CRC32__IEEE_X86_SSE42_PXMU));
|
||||
v_x1 = _mm_and_si128(v_x0, v_x2);
|
||||
v_x1 = _mm_clmulepi64_si128(v_x1, v_k, (int32_t)(16u));
|
||||
v_x1 = _mm_clmulepi64_si128(v_x1, v_pxmu, (int32_t)(16u));
|
||||
v_x1 = _mm_and_si128(v_x1, v_x2);
|
||||
v_x1 = _mm_clmulepi64_si128(v_x1, v_k, (int32_t)(0u));
|
||||
v_x0 = _mm_xor_si128(v_x0, v_x1);
|
||||
v_s = ((uint32_t)(_mm_extract_epi32(v_x0, (int32_t)(1u))));
|
||||
v_x1 = _mm_clmulepi64_si128(v_x1, v_pxmu, (int32_t)(0u));
|
||||
v_x1 = _mm_xor_si128(v_x1, v_x0);
|
||||
v_s = ((uint32_t)(_mm_extract_epi32(v_x1, (int32_t)(1u))));
|
||||
v_tail_index = (((uint64_t)(a_x.len)) & 18446744073709551552u);
|
||||
if (v_tail_index < ((uint64_t)(a_x.len))) {
|
||||
{
|
||||
|
||||
@@ -23,7 +23,10 @@ pri func ieee_hasher.up_x86_avx2!(x: roslice base.u8),
|
||||
var p : roslice base.u8
|
||||
|
||||
var util : base.x86_sse42_utility
|
||||
var k : base.x86_m128i
|
||||
var k1k2 : base.x86_m128i
|
||||
var k3k4 : base.x86_m128i
|
||||
var k5zz : base.x86_m128i
|
||||
var pxmu : base.x86_m128i
|
||||
var x0 : base.x86_m128i
|
||||
var x1 : base.x86_m128i
|
||||
var x2 : base.x86_m128i
|
||||
@@ -62,17 +65,17 @@ pri func ieee_hasher.up_x86_avx2!(x: roslice base.u8),
|
||||
x0 = x0._mm_xor_si128(b: util.make_m128i_single_u32(a: s))
|
||||
|
||||
// Process the remaining 64-byte chunks.
|
||||
k = util.make_m128i_slice128(a: IEEE_X86_SSE42_K1K2[.. 16])
|
||||
k1k2 = util.make_m128i_slice128(a: IEEE_X86_SSE42_K1K2[.. 16])
|
||||
iterate (p = args.x[64 ..])(length: 64, advance: 64, unroll: 1) {
|
||||
y0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||||
y1 = x1._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||||
y2 = x2._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||||
y3 = x3._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||||
y0 = x0._mm_clmulepi64_si128(b: k1k2, imm8: 0x00)
|
||||
y1 = x1._mm_clmulepi64_si128(b: k1k2, imm8: 0x00)
|
||||
y2 = x2._mm_clmulepi64_si128(b: k1k2, imm8: 0x00)
|
||||
y3 = x3._mm_clmulepi64_si128(b: k1k2, imm8: 0x00)
|
||||
|
||||
x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x11)
|
||||
x1 = x1._mm_clmulepi64_si128(b: k, imm8: 0x11)
|
||||
x2 = x2._mm_clmulepi64_si128(b: k, imm8: 0x11)
|
||||
x3 = x3._mm_clmulepi64_si128(b: k, imm8: 0x11)
|
||||
x0 = x0._mm_clmulepi64_si128(b: k1k2, imm8: 0x11)
|
||||
x1 = x1._mm_clmulepi64_si128(b: k1k2, imm8: 0x11)
|
||||
x2 = x2._mm_clmulepi64_si128(b: k1k2, imm8: 0x11)
|
||||
x3 = x3._mm_clmulepi64_si128(b: k1k2, imm8: 0x11)
|
||||
|
||||
x0 = x0._mm_xor_si128(b: y0)._mm_xor_si128(b: util.make_m128i_slice128(a: p[0x00 .. 0x10]))
|
||||
x1 = x1._mm_xor_si128(b: y1)._mm_xor_si128(b: util.make_m128i_slice128(a: p[0x10 .. 0x20]))
|
||||
@@ -81,22 +84,22 @@ pri func ieee_hasher.up_x86_avx2!(x: roslice base.u8),
|
||||
}
|
||||
|
||||
// Reduce 128×4 = 512 bits to 128 bits.
|
||||
k = util.make_m128i_slice128(a: IEEE_X86_SSE42_K3K4[.. 16])
|
||||
y0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||||
x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x11)
|
||||
k3k4 = util.make_m128i_slice128(a: IEEE_X86_SSE42_K3K4[.. 16])
|
||||
y0 = x0._mm_clmulepi64_si128(b: k3k4, imm8: 0x00)
|
||||
x0 = x0._mm_clmulepi64_si128(b: k3k4, imm8: 0x11)
|
||||
x0 = x0._mm_xor_si128(b: x1)
|
||||
x0 = x0._mm_xor_si128(b: y0)
|
||||
y0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||||
x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x11)
|
||||
y0 = x0._mm_clmulepi64_si128(b: k3k4, imm8: 0x00)
|
||||
x0 = x0._mm_clmulepi64_si128(b: k3k4, imm8: 0x11)
|
||||
x0 = x0._mm_xor_si128(b: x2)
|
||||
x0 = x0._mm_xor_si128(b: y0)
|
||||
y0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||||
x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x11)
|
||||
y0 = x0._mm_clmulepi64_si128(b: k3k4, imm8: 0x00)
|
||||
x0 = x0._mm_clmulepi64_si128(b: k3k4, imm8: 0x11)
|
||||
x0 = x0._mm_xor_si128(b: x3)
|
||||
x0 = x0._mm_xor_si128(b: y0)
|
||||
|
||||
// Reduce 128 bits to 64 bits.
|
||||
x1 = x0._mm_clmulepi64_si128(b: k, imm8: 0x10)
|
||||
x1 = x0._mm_clmulepi64_si128(b: k3k4, imm8: 0x10)
|
||||
x2 = util.make_m128i_multiple_u32(
|
||||
a00: 0xFFFF_FFFF,
|
||||
a01: 0x0000_0000,
|
||||
@@ -104,23 +107,23 @@ pri func ieee_hasher.up_x86_avx2!(x: roslice base.u8),
|
||||
a03: 0x0000_0000)
|
||||
x0 = x0._mm_srli_si128(imm8: 8)
|
||||
x0 = x0._mm_xor_si128(b: x1)
|
||||
k = util.make_m128i_slice128(a: IEEE_X86_SSE42_K5ZZ[.. 16])
|
||||
k5zz = util.make_m128i_slice128(a: IEEE_X86_SSE42_K5ZZ[.. 16])
|
||||
x1 = x0._mm_srli_si128(imm8: 4)
|
||||
x0 = x0._mm_and_si128(b: x2)
|
||||
x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||||
x0 = x0._mm_clmulepi64_si128(b: k5zz, imm8: 0x00)
|
||||
x0 = x0._mm_xor_si128(b: x1)
|
||||
|
||||
// Reduce 64 bits to 32 bits (Barrett Reduction) and extract.
|
||||
//
|
||||
// Barrett Reduction is Algorithm 1 (page 14) of Gopal et al., after
|
||||
// adjusting for bit-reflection as per Figure 12 (page 21).
|
||||
k = util.make_m128i_slice128(a: IEEE_X86_SSE42_PXMU[.. 16])
|
||||
pxmu = util.make_m128i_slice128(a: IEEE_X86_SSE42_PXMU[.. 16])
|
||||
x1 = x0._mm_and_si128(b: x2)
|
||||
x1 = x1._mm_clmulepi64_si128(b: k, imm8: 0x10)
|
||||
x1 = x1._mm_clmulepi64_si128(b: pxmu, imm8: 0x10)
|
||||
x1 = x1._mm_and_si128(b: x2)
|
||||
x1 = x1._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||||
x0 = x0._mm_xor_si128(b: x1)
|
||||
s = x0._mm_extract_epi32(imm8: 1)
|
||||
x1 = x1._mm_clmulepi64_si128(b: pxmu, imm8: 0x00)
|
||||
x1 = x1._mm_xor_si128(b: x0)
|
||||
s = x1._mm_extract_epi32(imm8: 1)
|
||||
|
||||
// Handle the tail of args.x that wasn't a complete 64-byte chunk.
|
||||
tail_index = args.x.length() & 0xFFFF_FFFF_FFFF_FFC0 // And-not 64.
|
||||
|
||||
@@ -20,7 +20,10 @@ pri func ieee_hasher.up_x86_sse42!(x: roslice base.u8),
|
||||
var p : roslice base.u8
|
||||
|
||||
var util : base.x86_sse42_utility
|
||||
var k : base.x86_m128i
|
||||
var k1k2 : base.x86_m128i
|
||||
var k3k4 : base.x86_m128i
|
||||
var k5zz : base.x86_m128i
|
||||
var pxmu : base.x86_m128i
|
||||
var x0 : base.x86_m128i
|
||||
var x1 : base.x86_m128i
|
||||
var x2 : base.x86_m128i
|
||||
@@ -59,17 +62,17 @@ pri func ieee_hasher.up_x86_sse42!(x: roslice base.u8),
|
||||
x0 = x0._mm_xor_si128(b: util.make_m128i_single_u32(a: s))
|
||||
|
||||
// Process the remaining 64-byte chunks.
|
||||
k = util.make_m128i_slice128(a: IEEE_X86_SSE42_K1K2[.. 16])
|
||||
k1k2 = util.make_m128i_slice128(a: IEEE_X86_SSE42_K1K2[.. 16])
|
||||
iterate (p = args.x[64 ..])(length: 64, advance: 64, unroll: 1) {
|
||||
y0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||||
y1 = x1._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||||
y2 = x2._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||||
y3 = x3._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||||
y0 = x0._mm_clmulepi64_si128(b: k1k2, imm8: 0x00)
|
||||
y1 = x1._mm_clmulepi64_si128(b: k1k2, imm8: 0x00)
|
||||
y2 = x2._mm_clmulepi64_si128(b: k1k2, imm8: 0x00)
|
||||
y3 = x3._mm_clmulepi64_si128(b: k1k2, imm8: 0x00)
|
||||
|
||||
x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x11)
|
||||
x1 = x1._mm_clmulepi64_si128(b: k, imm8: 0x11)
|
||||
x2 = x2._mm_clmulepi64_si128(b: k, imm8: 0x11)
|
||||
x3 = x3._mm_clmulepi64_si128(b: k, imm8: 0x11)
|
||||
x0 = x0._mm_clmulepi64_si128(b: k1k2, imm8: 0x11)
|
||||
x1 = x1._mm_clmulepi64_si128(b: k1k2, imm8: 0x11)
|
||||
x2 = x2._mm_clmulepi64_si128(b: k1k2, imm8: 0x11)
|
||||
x3 = x3._mm_clmulepi64_si128(b: k1k2, imm8: 0x11)
|
||||
|
||||
x0 = x0._mm_xor_si128(b: y0)._mm_xor_si128(b: util.make_m128i_slice128(a: p[0x00 .. 0x10]))
|
||||
x1 = x1._mm_xor_si128(b: y1)._mm_xor_si128(b: util.make_m128i_slice128(a: p[0x10 .. 0x20]))
|
||||
@@ -78,22 +81,22 @@ pri func ieee_hasher.up_x86_sse42!(x: roslice base.u8),
|
||||
}
|
||||
|
||||
// Reduce 128×4 = 512 bits to 128 bits.
|
||||
k = util.make_m128i_slice128(a: IEEE_X86_SSE42_K3K4[.. 16])
|
||||
y0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||||
x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x11)
|
||||
k3k4 = util.make_m128i_slice128(a: IEEE_X86_SSE42_K3K4[.. 16])
|
||||
y0 = x0._mm_clmulepi64_si128(b: k3k4, imm8: 0x00)
|
||||
x0 = x0._mm_clmulepi64_si128(b: k3k4, imm8: 0x11)
|
||||
x0 = x0._mm_xor_si128(b: x1)
|
||||
x0 = x0._mm_xor_si128(b: y0)
|
||||
y0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||||
x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x11)
|
||||
y0 = x0._mm_clmulepi64_si128(b: k3k4, imm8: 0x00)
|
||||
x0 = x0._mm_clmulepi64_si128(b: k3k4, imm8: 0x11)
|
||||
x0 = x0._mm_xor_si128(b: x2)
|
||||
x0 = x0._mm_xor_si128(b: y0)
|
||||
y0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||||
x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x11)
|
||||
y0 = x0._mm_clmulepi64_si128(b: k3k4, imm8: 0x00)
|
||||
x0 = x0._mm_clmulepi64_si128(b: k3k4, imm8: 0x11)
|
||||
x0 = x0._mm_xor_si128(b: x3)
|
||||
x0 = x0._mm_xor_si128(b: y0)
|
||||
|
||||
// Reduce 128 bits to 64 bits.
|
||||
x1 = x0._mm_clmulepi64_si128(b: k, imm8: 0x10)
|
||||
x1 = x0._mm_clmulepi64_si128(b: k3k4, imm8: 0x10)
|
||||
x2 = util.make_m128i_multiple_u32(
|
||||
a00: 0xFFFF_FFFF,
|
||||
a01: 0x0000_0000,
|
||||
@@ -101,23 +104,23 @@ pri func ieee_hasher.up_x86_sse42!(x: roslice base.u8),
|
||||
a03: 0x0000_0000)
|
||||
x0 = x0._mm_srli_si128(imm8: 8)
|
||||
x0 = x0._mm_xor_si128(b: x1)
|
||||
k = util.make_m128i_slice128(a: IEEE_X86_SSE42_K5ZZ[.. 16])
|
||||
k5zz = util.make_m128i_slice128(a: IEEE_X86_SSE42_K5ZZ[.. 16])
|
||||
x1 = x0._mm_srli_si128(imm8: 4)
|
||||
x0 = x0._mm_and_si128(b: x2)
|
||||
x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||||
x0 = x0._mm_clmulepi64_si128(b: k5zz, imm8: 0x00)
|
||||
x0 = x0._mm_xor_si128(b: x1)
|
||||
|
||||
// Reduce 64 bits to 32 bits (Barrett Reduction) and extract.
|
||||
//
|
||||
// Barrett Reduction is Algorithm 1 (page 14) of Gopal et al., after
|
||||
// adjusting for bit-reflection as per Figure 12 (page 21).
|
||||
k = util.make_m128i_slice128(a: IEEE_X86_SSE42_PXMU[.. 16])
|
||||
pxmu = util.make_m128i_slice128(a: IEEE_X86_SSE42_PXMU[.. 16])
|
||||
x1 = x0._mm_and_si128(b: x2)
|
||||
x1 = x1._mm_clmulepi64_si128(b: k, imm8: 0x10)
|
||||
x1 = x1._mm_clmulepi64_si128(b: pxmu, imm8: 0x10)
|
||||
x1 = x1._mm_and_si128(b: x2)
|
||||
x1 = x1._mm_clmulepi64_si128(b: k, imm8: 0x00)
|
||||
x0 = x0._mm_xor_si128(b: x1)
|
||||
s = x0._mm_extract_epi32(imm8: 1)
|
||||
x1 = x1._mm_clmulepi64_si128(b: pxmu, imm8: 0x00)
|
||||
x1 = x1._mm_xor_si128(b: x0)
|
||||
s = x1._mm_extract_epi32(imm8: 1)
|
||||
|
||||
// Handle the tail of args.x that wasn't a complete 64-byte chunk.
|
||||
tail_index = args.x.length() & 0xFFFF_FFFF_FFFF_FFC0 // And-not 64.
|
||||
|
||||
Reference in new issue
Block a user