From 9c704c4a6afec659130963fda34f7d3c58010fa2 Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Sun, 21 Apr 2024 12:46:05 +1000 Subject: [PATCH] std/crc32: tweak x86 to be more like std/crc64 --- release/c/wuffs-unsupported-snapshot.c | 106 +++++++++++++------------ std/crc32/common_up_x86_avx2.wuffs | 53 +++++++------ std/crc32/common_up_x86_sse42.wuffs | 53 +++++++------ 3 files changed, 112 insertions(+), 100 deletions(-) diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index a3b37c04..3d4ed28a 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -36085,7 +36085,10 @@ wuffs_crc32__ieee_hasher__up_x86_avx2( wuffs_base__slice_u8 a_x) { uint32_t v_s = 0; wuffs_base__slice_u8 v_p = {0}; - __m128i v_k = {0}; + __m128i v_k1k2 = {0}; + __m128i v_k3k4 = {0}; + __m128i v_k5zz = {0}; + __m128i v_pxmu = {0}; __m128i v_x0 = {0}; __m128i v_x1 = {0}; __m128i v_x2 = {0}; @@ -36121,21 +36124,21 @@ wuffs_crc32__ieee_hasher__up_x86_avx2( v_x2 = _mm_lddqu_si128((const __m128i*)(const void*)(a_x.ptr + 32u)); v_x3 = _mm_lddqu_si128((const __m128i*)(const void*)(a_x.ptr + 48u)); v_x0 = _mm_xor_si128(v_x0, _mm_cvtsi32_si128((int32_t)(v_s))); - v_k = _mm_lddqu_si128((const __m128i*)(const void*)(WUFFS_CRC32__IEEE_X86_SSE42_K1K2)); + v_k1k2 = _mm_lddqu_si128((const __m128i*)(const void*)(WUFFS_CRC32__IEEE_X86_SSE42_K1K2)); { wuffs_base__slice_u8 i_slice_p = wuffs_base__slice_u8__subslice_i(a_x, 64u); v_p.ptr = i_slice_p.ptr; v_p.len = 64; const uint8_t* i_end0_p = wuffs_private_impl__ptr_u8_plus_len(v_p.ptr, (((i_slice_p.len - (size_t)(v_p.ptr - i_slice_p.ptr)) / 64) * 64)); while (v_p.ptr < i_end0_p) { - v_y0 = _mm_clmulepi64_si128(v_x0, v_k, (int32_t)(0u)); - v_y1 = _mm_clmulepi64_si128(v_x1, v_k, (int32_t)(0u)); - v_y2 = _mm_clmulepi64_si128(v_x2, v_k, (int32_t)(0u)); - v_y3 = _mm_clmulepi64_si128(v_x3, v_k, (int32_t)(0u)); - v_x0 = _mm_clmulepi64_si128(v_x0, v_k, (int32_t)(17u)); - v_x1 = _mm_clmulepi64_si128(v_x1, v_k, (int32_t)(17u)); - v_x2 = _mm_clmulepi64_si128(v_x2, v_k, (int32_t)(17u)); - v_x3 = _mm_clmulepi64_si128(v_x3, v_k, (int32_t)(17u)); + v_y0 = _mm_clmulepi64_si128(v_x0, v_k1k2, (int32_t)(0u)); + v_y1 = _mm_clmulepi64_si128(v_x1, v_k1k2, (int32_t)(0u)); + v_y2 = _mm_clmulepi64_si128(v_x2, v_k1k2, (int32_t)(0u)); + v_y3 = _mm_clmulepi64_si128(v_x3, v_k1k2, (int32_t)(0u)); + v_x0 = _mm_clmulepi64_si128(v_x0, v_k1k2, (int32_t)(17u)); + v_x1 = _mm_clmulepi64_si128(v_x1, v_k1k2, (int32_t)(17u)); + v_x2 = _mm_clmulepi64_si128(v_x2, v_k1k2, (int32_t)(17u)); + v_x3 = _mm_clmulepi64_si128(v_x3, v_k1k2, (int32_t)(17u)); v_x0 = _mm_xor_si128(_mm_xor_si128(v_x0, v_y0), _mm_lddqu_si128((const __m128i*)(const void*)(v_p.ptr + 0u))); v_x1 = _mm_xor_si128(_mm_xor_si128(v_x1, v_y1), _mm_lddqu_si128((const __m128i*)(const void*)(v_p.ptr + 16u))); v_x2 = _mm_xor_si128(_mm_xor_si128(v_x2, v_y2), _mm_lddqu_si128((const __m128i*)(const void*)(v_p.ptr + 32u))); @@ -36144,35 +36147,35 @@ wuffs_crc32__ieee_hasher__up_x86_avx2( } v_p.len = 0; } - v_k = _mm_lddqu_si128((const __m128i*)(const void*)(WUFFS_CRC32__IEEE_X86_SSE42_K3K4)); - v_y0 = _mm_clmulepi64_si128(v_x0, v_k, (int32_t)(0u)); - v_x0 = _mm_clmulepi64_si128(v_x0, v_k, (int32_t)(17u)); + v_k3k4 = _mm_lddqu_si128((const __m128i*)(const void*)(WUFFS_CRC32__IEEE_X86_SSE42_K3K4)); + v_y0 = _mm_clmulepi64_si128(v_x0, v_k3k4, (int32_t)(0u)); + v_x0 = _mm_clmulepi64_si128(v_x0, v_k3k4, (int32_t)(17u)); v_x0 = _mm_xor_si128(v_x0, v_x1); v_x0 = _mm_xor_si128(v_x0, v_y0); - v_y0 = _mm_clmulepi64_si128(v_x0, v_k, (int32_t)(0u)); - v_x0 = _mm_clmulepi64_si128(v_x0, v_k, (int32_t)(17u)); + v_y0 = _mm_clmulepi64_si128(v_x0, v_k3k4, (int32_t)(0u)); + v_x0 = _mm_clmulepi64_si128(v_x0, v_k3k4, (int32_t)(17u)); v_x0 = _mm_xor_si128(v_x0, v_x2); v_x0 = _mm_xor_si128(v_x0, v_y0); - v_y0 = _mm_clmulepi64_si128(v_x0, v_k, (int32_t)(0u)); - v_x0 = _mm_clmulepi64_si128(v_x0, v_k, (int32_t)(17u)); + v_y0 = _mm_clmulepi64_si128(v_x0, v_k3k4, (int32_t)(0u)); + v_x0 = _mm_clmulepi64_si128(v_x0, v_k3k4, (int32_t)(17u)); v_x0 = _mm_xor_si128(v_x0, v_x3); v_x0 = _mm_xor_si128(v_x0, v_y0); - v_x1 = _mm_clmulepi64_si128(v_x0, v_k, (int32_t)(16u)); + v_x1 = _mm_clmulepi64_si128(v_x0, v_k3k4, (int32_t)(16u)); v_x2 = _mm_set_epi32((int32_t)(0u), (int32_t)(4294967295u), (int32_t)(0u), (int32_t)(4294967295u)); v_x0 = _mm_srli_si128(v_x0, (int32_t)(8u)); v_x0 = _mm_xor_si128(v_x0, v_x1); - v_k = _mm_lddqu_si128((const __m128i*)(const void*)(WUFFS_CRC32__IEEE_X86_SSE42_K5ZZ)); + v_k5zz = _mm_lddqu_si128((const __m128i*)(const void*)(WUFFS_CRC32__IEEE_X86_SSE42_K5ZZ)); v_x1 = _mm_srli_si128(v_x0, (int32_t)(4u)); v_x0 = _mm_and_si128(v_x0, v_x2); - v_x0 = _mm_clmulepi64_si128(v_x0, v_k, (int32_t)(0u)); + v_x0 = _mm_clmulepi64_si128(v_x0, v_k5zz, (int32_t)(0u)); v_x0 = _mm_xor_si128(v_x0, v_x1); - v_k = _mm_lddqu_si128((const __m128i*)(const void*)(WUFFS_CRC32__IEEE_X86_SSE42_PXMU)); + v_pxmu = _mm_lddqu_si128((const __m128i*)(const void*)(WUFFS_CRC32__IEEE_X86_SSE42_PXMU)); v_x1 = _mm_and_si128(v_x0, v_x2); - v_x1 = _mm_clmulepi64_si128(v_x1, v_k, (int32_t)(16u)); + v_x1 = _mm_clmulepi64_si128(v_x1, v_pxmu, (int32_t)(16u)); v_x1 = _mm_and_si128(v_x1, v_x2); - v_x1 = _mm_clmulepi64_si128(v_x1, v_k, (int32_t)(0u)); - v_x0 = _mm_xor_si128(v_x0, v_x1); - v_s = ((uint32_t)(_mm_extract_epi32(v_x0, (int32_t)(1u)))); + v_x1 = _mm_clmulepi64_si128(v_x1, v_pxmu, (int32_t)(0u)); + v_x1 = _mm_xor_si128(v_x1, v_x0); + v_s = ((uint32_t)(_mm_extract_epi32(v_x1, (int32_t)(1u)))); v_tail_index = (((uint64_t)(a_x.len)) & 18446744073709551552u); if (v_tail_index < ((uint64_t)(a_x.len))) { { @@ -36205,7 +36208,10 @@ wuffs_crc32__ieee_hasher__up_x86_sse42( wuffs_base__slice_u8 a_x) { uint32_t v_s = 0; wuffs_base__slice_u8 v_p = {0}; - __m128i v_k = {0}; + __m128i v_k1k2 = {0}; + __m128i v_k3k4 = {0}; + __m128i v_k5zz = {0}; + __m128i v_pxmu = {0}; __m128i v_x0 = {0}; __m128i v_x1 = {0}; __m128i v_x2 = {0}; @@ -36241,21 +36247,21 @@ wuffs_crc32__ieee_hasher__up_x86_sse42( v_x2 = _mm_lddqu_si128((const __m128i*)(const void*)(a_x.ptr + 32u)); v_x3 = _mm_lddqu_si128((const __m128i*)(const void*)(a_x.ptr + 48u)); v_x0 = _mm_xor_si128(v_x0, _mm_cvtsi32_si128((int32_t)(v_s))); - v_k = _mm_lddqu_si128((const __m128i*)(const void*)(WUFFS_CRC32__IEEE_X86_SSE42_K1K2)); + v_k1k2 = _mm_lddqu_si128((const __m128i*)(const void*)(WUFFS_CRC32__IEEE_X86_SSE42_K1K2)); { wuffs_base__slice_u8 i_slice_p = wuffs_base__slice_u8__subslice_i(a_x, 64u); v_p.ptr = i_slice_p.ptr; v_p.len = 64; const uint8_t* i_end0_p = wuffs_private_impl__ptr_u8_plus_len(v_p.ptr, (((i_slice_p.len - (size_t)(v_p.ptr - i_slice_p.ptr)) / 64) * 64)); while (v_p.ptr < i_end0_p) { - v_y0 = _mm_clmulepi64_si128(v_x0, v_k, (int32_t)(0u)); - v_y1 = _mm_clmulepi64_si128(v_x1, v_k, (int32_t)(0u)); - v_y2 = _mm_clmulepi64_si128(v_x2, v_k, (int32_t)(0u)); - v_y3 = _mm_clmulepi64_si128(v_x3, v_k, (int32_t)(0u)); - v_x0 = _mm_clmulepi64_si128(v_x0, v_k, (int32_t)(17u)); - v_x1 = _mm_clmulepi64_si128(v_x1, v_k, (int32_t)(17u)); - v_x2 = _mm_clmulepi64_si128(v_x2, v_k, (int32_t)(17u)); - v_x3 = _mm_clmulepi64_si128(v_x3, v_k, (int32_t)(17u)); + v_y0 = _mm_clmulepi64_si128(v_x0, v_k1k2, (int32_t)(0u)); + v_y1 = _mm_clmulepi64_si128(v_x1, v_k1k2, (int32_t)(0u)); + v_y2 = _mm_clmulepi64_si128(v_x2, v_k1k2, (int32_t)(0u)); + v_y3 = _mm_clmulepi64_si128(v_x3, v_k1k2, (int32_t)(0u)); + v_x0 = _mm_clmulepi64_si128(v_x0, v_k1k2, (int32_t)(17u)); + v_x1 = _mm_clmulepi64_si128(v_x1, v_k1k2, (int32_t)(17u)); + v_x2 = _mm_clmulepi64_si128(v_x2, v_k1k2, (int32_t)(17u)); + v_x3 = _mm_clmulepi64_si128(v_x3, v_k1k2, (int32_t)(17u)); v_x0 = _mm_xor_si128(_mm_xor_si128(v_x0, v_y0), _mm_lddqu_si128((const __m128i*)(const void*)(v_p.ptr + 0u))); v_x1 = _mm_xor_si128(_mm_xor_si128(v_x1, v_y1), _mm_lddqu_si128((const __m128i*)(const void*)(v_p.ptr + 16u))); v_x2 = _mm_xor_si128(_mm_xor_si128(v_x2, v_y2), _mm_lddqu_si128((const __m128i*)(const void*)(v_p.ptr + 32u))); @@ -36264,35 +36270,35 @@ wuffs_crc32__ieee_hasher__up_x86_sse42( } v_p.len = 0; } - v_k = _mm_lddqu_si128((const __m128i*)(const void*)(WUFFS_CRC32__IEEE_X86_SSE42_K3K4)); - v_y0 = _mm_clmulepi64_si128(v_x0, v_k, (int32_t)(0u)); - v_x0 = _mm_clmulepi64_si128(v_x0, v_k, (int32_t)(17u)); + v_k3k4 = _mm_lddqu_si128((const __m128i*)(const void*)(WUFFS_CRC32__IEEE_X86_SSE42_K3K4)); + v_y0 = _mm_clmulepi64_si128(v_x0, v_k3k4, (int32_t)(0u)); + v_x0 = _mm_clmulepi64_si128(v_x0, v_k3k4, (int32_t)(17u)); v_x0 = _mm_xor_si128(v_x0, v_x1); v_x0 = _mm_xor_si128(v_x0, v_y0); - v_y0 = _mm_clmulepi64_si128(v_x0, v_k, (int32_t)(0u)); - v_x0 = _mm_clmulepi64_si128(v_x0, v_k, (int32_t)(17u)); + v_y0 = _mm_clmulepi64_si128(v_x0, v_k3k4, (int32_t)(0u)); + v_x0 = _mm_clmulepi64_si128(v_x0, v_k3k4, (int32_t)(17u)); v_x0 = _mm_xor_si128(v_x0, v_x2); v_x0 = _mm_xor_si128(v_x0, v_y0); - v_y0 = _mm_clmulepi64_si128(v_x0, v_k, (int32_t)(0u)); - v_x0 = _mm_clmulepi64_si128(v_x0, v_k, (int32_t)(17u)); + v_y0 = _mm_clmulepi64_si128(v_x0, v_k3k4, (int32_t)(0u)); + v_x0 = _mm_clmulepi64_si128(v_x0, v_k3k4, (int32_t)(17u)); v_x0 = _mm_xor_si128(v_x0, v_x3); v_x0 = _mm_xor_si128(v_x0, v_y0); - v_x1 = _mm_clmulepi64_si128(v_x0, v_k, (int32_t)(16u)); + v_x1 = _mm_clmulepi64_si128(v_x0, v_k3k4, (int32_t)(16u)); v_x2 = _mm_set_epi32((int32_t)(0u), (int32_t)(4294967295u), (int32_t)(0u), (int32_t)(4294967295u)); v_x0 = _mm_srli_si128(v_x0, (int32_t)(8u)); v_x0 = _mm_xor_si128(v_x0, v_x1); - v_k = _mm_lddqu_si128((const __m128i*)(const void*)(WUFFS_CRC32__IEEE_X86_SSE42_K5ZZ)); + v_k5zz = _mm_lddqu_si128((const __m128i*)(const void*)(WUFFS_CRC32__IEEE_X86_SSE42_K5ZZ)); v_x1 = _mm_srli_si128(v_x0, (int32_t)(4u)); v_x0 = _mm_and_si128(v_x0, v_x2); - v_x0 = _mm_clmulepi64_si128(v_x0, v_k, (int32_t)(0u)); + v_x0 = _mm_clmulepi64_si128(v_x0, v_k5zz, (int32_t)(0u)); v_x0 = _mm_xor_si128(v_x0, v_x1); - v_k = _mm_lddqu_si128((const __m128i*)(const void*)(WUFFS_CRC32__IEEE_X86_SSE42_PXMU)); + v_pxmu = _mm_lddqu_si128((const __m128i*)(const void*)(WUFFS_CRC32__IEEE_X86_SSE42_PXMU)); v_x1 = _mm_and_si128(v_x0, v_x2); - v_x1 = _mm_clmulepi64_si128(v_x1, v_k, (int32_t)(16u)); + v_x1 = _mm_clmulepi64_si128(v_x1, v_pxmu, (int32_t)(16u)); v_x1 = _mm_and_si128(v_x1, v_x2); - v_x1 = _mm_clmulepi64_si128(v_x1, v_k, (int32_t)(0u)); - v_x0 = _mm_xor_si128(v_x0, v_x1); - v_s = ((uint32_t)(_mm_extract_epi32(v_x0, (int32_t)(1u)))); + v_x1 = _mm_clmulepi64_si128(v_x1, v_pxmu, (int32_t)(0u)); + v_x1 = _mm_xor_si128(v_x1, v_x0); + v_s = ((uint32_t)(_mm_extract_epi32(v_x1, (int32_t)(1u)))); v_tail_index = (((uint64_t)(a_x.len)) & 18446744073709551552u); if (v_tail_index < ((uint64_t)(a_x.len))) { { diff --git a/std/crc32/common_up_x86_avx2.wuffs b/std/crc32/common_up_x86_avx2.wuffs index 94ead633..108beb7e 100644 --- a/std/crc32/common_up_x86_avx2.wuffs +++ b/std/crc32/common_up_x86_avx2.wuffs @@ -23,7 +23,10 @@ pri func ieee_hasher.up_x86_avx2!(x: roslice base.u8), var p : roslice base.u8 var util : base.x86_sse42_utility - var k : base.x86_m128i + var k1k2 : base.x86_m128i + var k3k4 : base.x86_m128i + var k5zz : base.x86_m128i + var pxmu : base.x86_m128i var x0 : base.x86_m128i var x1 : base.x86_m128i var x2 : base.x86_m128i @@ -62,17 +65,17 @@ pri func ieee_hasher.up_x86_avx2!(x: roslice base.u8), x0 = x0._mm_xor_si128(b: util.make_m128i_single_u32(a: s)) // Process the remaining 64-byte chunks. - k = util.make_m128i_slice128(a: IEEE_X86_SSE42_K1K2[.. 16]) + k1k2 = util.make_m128i_slice128(a: IEEE_X86_SSE42_K1K2[.. 16]) iterate (p = args.x[64 ..])(length: 64, advance: 64, unroll: 1) { - y0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00) - y1 = x1._mm_clmulepi64_si128(b: k, imm8: 0x00) - y2 = x2._mm_clmulepi64_si128(b: k, imm8: 0x00) - y3 = x3._mm_clmulepi64_si128(b: k, imm8: 0x00) + y0 = x0._mm_clmulepi64_si128(b: k1k2, imm8: 0x00) + y1 = x1._mm_clmulepi64_si128(b: k1k2, imm8: 0x00) + y2 = x2._mm_clmulepi64_si128(b: k1k2, imm8: 0x00) + y3 = x3._mm_clmulepi64_si128(b: k1k2, imm8: 0x00) - x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x11) - x1 = x1._mm_clmulepi64_si128(b: k, imm8: 0x11) - x2 = x2._mm_clmulepi64_si128(b: k, imm8: 0x11) - x3 = x3._mm_clmulepi64_si128(b: k, imm8: 0x11) + x0 = x0._mm_clmulepi64_si128(b: k1k2, imm8: 0x11) + x1 = x1._mm_clmulepi64_si128(b: k1k2, imm8: 0x11) + x2 = x2._mm_clmulepi64_si128(b: k1k2, imm8: 0x11) + x3 = x3._mm_clmulepi64_si128(b: k1k2, imm8: 0x11) x0 = x0._mm_xor_si128(b: y0)._mm_xor_si128(b: util.make_m128i_slice128(a: p[0x00 .. 0x10])) x1 = x1._mm_xor_si128(b: y1)._mm_xor_si128(b: util.make_m128i_slice128(a: p[0x10 .. 0x20])) @@ -81,22 +84,22 @@ pri func ieee_hasher.up_x86_avx2!(x: roslice base.u8), } // Reduce 128×4 = 512 bits to 128 bits. - k = util.make_m128i_slice128(a: IEEE_X86_SSE42_K3K4[.. 16]) - y0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00) - x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x11) + k3k4 = util.make_m128i_slice128(a: IEEE_X86_SSE42_K3K4[.. 16]) + y0 = x0._mm_clmulepi64_si128(b: k3k4, imm8: 0x00) + x0 = x0._mm_clmulepi64_si128(b: k3k4, imm8: 0x11) x0 = x0._mm_xor_si128(b: x1) x0 = x0._mm_xor_si128(b: y0) - y0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00) - x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x11) + y0 = x0._mm_clmulepi64_si128(b: k3k4, imm8: 0x00) + x0 = x0._mm_clmulepi64_si128(b: k3k4, imm8: 0x11) x0 = x0._mm_xor_si128(b: x2) x0 = x0._mm_xor_si128(b: y0) - y0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00) - x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x11) + y0 = x0._mm_clmulepi64_si128(b: k3k4, imm8: 0x00) + x0 = x0._mm_clmulepi64_si128(b: k3k4, imm8: 0x11) x0 = x0._mm_xor_si128(b: x3) x0 = x0._mm_xor_si128(b: y0) // Reduce 128 bits to 64 bits. - x1 = x0._mm_clmulepi64_si128(b: k, imm8: 0x10) + x1 = x0._mm_clmulepi64_si128(b: k3k4, imm8: 0x10) x2 = util.make_m128i_multiple_u32( a00: 0xFFFF_FFFF, a01: 0x0000_0000, @@ -104,23 +107,23 @@ pri func ieee_hasher.up_x86_avx2!(x: roslice base.u8), a03: 0x0000_0000) x0 = x0._mm_srli_si128(imm8: 8) x0 = x0._mm_xor_si128(b: x1) - k = util.make_m128i_slice128(a: IEEE_X86_SSE42_K5ZZ[.. 16]) + k5zz = util.make_m128i_slice128(a: IEEE_X86_SSE42_K5ZZ[.. 16]) x1 = x0._mm_srli_si128(imm8: 4) x0 = x0._mm_and_si128(b: x2) - x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00) + x0 = x0._mm_clmulepi64_si128(b: k5zz, imm8: 0x00) x0 = x0._mm_xor_si128(b: x1) // Reduce 64 bits to 32 bits (Barrett Reduction) and extract. // // Barrett Reduction is Algorithm 1 (page 14) of Gopal et al., after // adjusting for bit-reflection as per Figure 12 (page 21). - k = util.make_m128i_slice128(a: IEEE_X86_SSE42_PXMU[.. 16]) + pxmu = util.make_m128i_slice128(a: IEEE_X86_SSE42_PXMU[.. 16]) x1 = x0._mm_and_si128(b: x2) - x1 = x1._mm_clmulepi64_si128(b: k, imm8: 0x10) + x1 = x1._mm_clmulepi64_si128(b: pxmu, imm8: 0x10) x1 = x1._mm_and_si128(b: x2) - x1 = x1._mm_clmulepi64_si128(b: k, imm8: 0x00) - x0 = x0._mm_xor_si128(b: x1) - s = x0._mm_extract_epi32(imm8: 1) + x1 = x1._mm_clmulepi64_si128(b: pxmu, imm8: 0x00) + x1 = x1._mm_xor_si128(b: x0) + s = x1._mm_extract_epi32(imm8: 1) // Handle the tail of args.x that wasn't a complete 64-byte chunk. tail_index = args.x.length() & 0xFFFF_FFFF_FFFF_FFC0 // And-not 64. diff --git a/std/crc32/common_up_x86_sse42.wuffs b/std/crc32/common_up_x86_sse42.wuffs index 9e26b906..6e6e2e80 100644 --- a/std/crc32/common_up_x86_sse42.wuffs +++ b/std/crc32/common_up_x86_sse42.wuffs @@ -20,7 +20,10 @@ pri func ieee_hasher.up_x86_sse42!(x: roslice base.u8), var p : roslice base.u8 var util : base.x86_sse42_utility - var k : base.x86_m128i + var k1k2 : base.x86_m128i + var k3k4 : base.x86_m128i + var k5zz : base.x86_m128i + var pxmu : base.x86_m128i var x0 : base.x86_m128i var x1 : base.x86_m128i var x2 : base.x86_m128i @@ -59,17 +62,17 @@ pri func ieee_hasher.up_x86_sse42!(x: roslice base.u8), x0 = x0._mm_xor_si128(b: util.make_m128i_single_u32(a: s)) // Process the remaining 64-byte chunks. - k = util.make_m128i_slice128(a: IEEE_X86_SSE42_K1K2[.. 16]) + k1k2 = util.make_m128i_slice128(a: IEEE_X86_SSE42_K1K2[.. 16]) iterate (p = args.x[64 ..])(length: 64, advance: 64, unroll: 1) { - y0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00) - y1 = x1._mm_clmulepi64_si128(b: k, imm8: 0x00) - y2 = x2._mm_clmulepi64_si128(b: k, imm8: 0x00) - y3 = x3._mm_clmulepi64_si128(b: k, imm8: 0x00) + y0 = x0._mm_clmulepi64_si128(b: k1k2, imm8: 0x00) + y1 = x1._mm_clmulepi64_si128(b: k1k2, imm8: 0x00) + y2 = x2._mm_clmulepi64_si128(b: k1k2, imm8: 0x00) + y3 = x3._mm_clmulepi64_si128(b: k1k2, imm8: 0x00) - x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x11) - x1 = x1._mm_clmulepi64_si128(b: k, imm8: 0x11) - x2 = x2._mm_clmulepi64_si128(b: k, imm8: 0x11) - x3 = x3._mm_clmulepi64_si128(b: k, imm8: 0x11) + x0 = x0._mm_clmulepi64_si128(b: k1k2, imm8: 0x11) + x1 = x1._mm_clmulepi64_si128(b: k1k2, imm8: 0x11) + x2 = x2._mm_clmulepi64_si128(b: k1k2, imm8: 0x11) + x3 = x3._mm_clmulepi64_si128(b: k1k2, imm8: 0x11) x0 = x0._mm_xor_si128(b: y0)._mm_xor_si128(b: util.make_m128i_slice128(a: p[0x00 .. 0x10])) x1 = x1._mm_xor_si128(b: y1)._mm_xor_si128(b: util.make_m128i_slice128(a: p[0x10 .. 0x20])) @@ -78,22 +81,22 @@ pri func ieee_hasher.up_x86_sse42!(x: roslice base.u8), } // Reduce 128×4 = 512 bits to 128 bits. - k = util.make_m128i_slice128(a: IEEE_X86_SSE42_K3K4[.. 16]) - y0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00) - x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x11) + k3k4 = util.make_m128i_slice128(a: IEEE_X86_SSE42_K3K4[.. 16]) + y0 = x0._mm_clmulepi64_si128(b: k3k4, imm8: 0x00) + x0 = x0._mm_clmulepi64_si128(b: k3k4, imm8: 0x11) x0 = x0._mm_xor_si128(b: x1) x0 = x0._mm_xor_si128(b: y0) - y0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00) - x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x11) + y0 = x0._mm_clmulepi64_si128(b: k3k4, imm8: 0x00) + x0 = x0._mm_clmulepi64_si128(b: k3k4, imm8: 0x11) x0 = x0._mm_xor_si128(b: x2) x0 = x0._mm_xor_si128(b: y0) - y0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00) - x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x11) + y0 = x0._mm_clmulepi64_si128(b: k3k4, imm8: 0x00) + x0 = x0._mm_clmulepi64_si128(b: k3k4, imm8: 0x11) x0 = x0._mm_xor_si128(b: x3) x0 = x0._mm_xor_si128(b: y0) // Reduce 128 bits to 64 bits. - x1 = x0._mm_clmulepi64_si128(b: k, imm8: 0x10) + x1 = x0._mm_clmulepi64_si128(b: k3k4, imm8: 0x10) x2 = util.make_m128i_multiple_u32( a00: 0xFFFF_FFFF, a01: 0x0000_0000, @@ -101,23 +104,23 @@ pri func ieee_hasher.up_x86_sse42!(x: roslice base.u8), a03: 0x0000_0000) x0 = x0._mm_srli_si128(imm8: 8) x0 = x0._mm_xor_si128(b: x1) - k = util.make_m128i_slice128(a: IEEE_X86_SSE42_K5ZZ[.. 16]) + k5zz = util.make_m128i_slice128(a: IEEE_X86_SSE42_K5ZZ[.. 16]) x1 = x0._mm_srli_si128(imm8: 4) x0 = x0._mm_and_si128(b: x2) - x0 = x0._mm_clmulepi64_si128(b: k, imm8: 0x00) + x0 = x0._mm_clmulepi64_si128(b: k5zz, imm8: 0x00) x0 = x0._mm_xor_si128(b: x1) // Reduce 64 bits to 32 bits (Barrett Reduction) and extract. // // Barrett Reduction is Algorithm 1 (page 14) of Gopal et al., after // adjusting for bit-reflection as per Figure 12 (page 21). - k = util.make_m128i_slice128(a: IEEE_X86_SSE42_PXMU[.. 16]) + pxmu = util.make_m128i_slice128(a: IEEE_X86_SSE42_PXMU[.. 16]) x1 = x0._mm_and_si128(b: x2) - x1 = x1._mm_clmulepi64_si128(b: k, imm8: 0x10) + x1 = x1._mm_clmulepi64_si128(b: pxmu, imm8: 0x10) x1 = x1._mm_and_si128(b: x2) - x1 = x1._mm_clmulepi64_si128(b: k, imm8: 0x00) - x0 = x0._mm_xor_si128(b: x1) - s = x0._mm_extract_epi32(imm8: 1) + x1 = x1._mm_clmulepi64_si128(b: pxmu, imm8: 0x00) + x1 = x1._mm_xor_si128(b: x0) + s = x1._mm_extract_epi32(imm8: 1) // Handle the tail of args.x that wasn't a complete 64-byte chunk. tail_index = args.x.length() & 0xFFFF_FFFF_FFFF_FFC0 // And-not 64.