From b9b28dbe8b1520d2fdef6e8a4256f19467bc8d52 Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Fri, 6 Oct 2023 13:59:31 +1100 Subject: [PATCH] std/xxhash32: add hasher.up_x86_sse42 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit will soon be followed by a rollback, but it is committed anyway so that we can refer to these numbers in the git log. name old speed new speed delta wuffs_xxhash32_10k/clang11 7.81GB/s ± 0% 4.46GB/s ± 1% -42.86% (p=0.008 n=5+5) wuffs_xxhash32_100k/clang11 7.96GB/s ± 0% 4.67GB/s ± 1% -41.34% (p=0.008 n=5+5) wuffs_xxhash32_10k/gcc10 7.83GB/s ± 0% 4.48GB/s ± 2% -42.81% (p=0.008 n=5+5) wuffs_xxhash32_100k/gcc10 7.95GB/s ± 0% 4.69GB/s ± 2% -41.04% (p=0.008 n=5+5) mimic_xxhash32_10k 7.70GB/s ± 1% 7.73GB/s ± 1% ~ (p=0.421 n=5+5) mimic_xxhash32_100k 7.79GB/s ± 0% 7.84GB/s ± 1% ~ (p=0.730 n=4+5) --- lang/builtin/builtin.go | 1 + release/c/wuffs-unsupported-snapshot.c | 148 +++++++++++++++++++++++++ std/xxhash32/common_up_x86_sse42.wuffs | 107 ++++++++++++++++++ std/xxhash32/common_xxhash32.wuffs | 7 +- 4 files changed, 262 insertions(+), 1 deletion(-) create mode 100644 std/xxhash32/common_up_x86_sse42.wuffs diff --git a/lang/builtin/builtin.go b/lang/builtin/builtin.go index cc4a0f35..8abb51e4 100644 --- a/lang/builtin/builtin.go +++ b/lang/builtin/builtin.go @@ -815,6 +815,7 @@ var funcsOther = [...]string{ "x86_m128i._mm_min_epu16(b: x86_m128i) x86_m128i", "x86_m128i._mm_min_epu32(b: x86_m128i) x86_m128i", "x86_m128i._mm_min_epu8(b: x86_m128i) x86_m128i", + "x86_m128i._mm_mullo_epi32(b: x86_m128i) x86_m128i", "x86_m128i._mm_or_si128(b: x86_m128i) x86_m128i", "x86_m128i._mm_packs_epi16(b: x86_m128i) x86_m128i", "x86_m128i._mm_packus_epi16(b: x86_m128i) x86_m128i", diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 57c64814..931ce3f7 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -12565,6 +12565,10 @@ struct wuffs_xxhash32__hasher__struct { uint32_t f_v1; uint32_t f_v2; uint32_t f_v3; + + wuffs_base__empty_struct (*choosy_up)( + wuffs_xxhash32__hasher* self, + wuffs_base__slice_u8 a_x); } private_impl; #ifdef __cplusplus @@ -61422,12 +61426,26 @@ wuffs_wbmp__decoder__workbuf_len( // ---------------- Private Function Prototypes +#if defined(WUFFS_BASE__CPU_ARCH__X86_FAMILY) +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__empty_struct +wuffs_xxhash32__hasher__up_x86_sse42( + wuffs_xxhash32__hasher* self, + wuffs_base__slice_u8 a_x); +#endif // defined(WUFFS_BASE__CPU_ARCH__X86_FAMILY) + WUFFS_BASE__GENERATED_C_CODE static wuffs_base__empty_struct wuffs_xxhash32__hasher__up( wuffs_xxhash32__hasher* self, wuffs_base__slice_u8 a_x); +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__empty_struct +wuffs_xxhash32__hasher__up__choosy_default( + wuffs_xxhash32__hasher* self, + wuffs_base__slice_u8 a_x); + // ---------------- VTables const wuffs_base__hasher_u32__func_ptrs @@ -61485,6 +61503,8 @@ wuffs_xxhash32__hasher__initialize( } } + self->private_impl.choosy_up = &wuffs_xxhash32__hasher__up__choosy_default; + self->private_impl.magic = WUFFS_BASE__MAGIC; self->private_impl.vtable_for__wuffs_base__hasher_u32.vtable_name = wuffs_base__hasher_u32__vtable_name; @@ -61515,6 +61535,121 @@ sizeof__wuffs_xxhash32__hasher() { // ---------------- Function Implementations +// ‼ WUFFS MULTI-FILE SECTION +x86_sse42 +// -------- func xxhash32.hasher.up_x86_sse42 + +#if defined(WUFFS_BASE__CPU_ARCH__X86_FAMILY) +WUFFS_BASE__MAYBE_ATTRIBUTE_TARGET("pclmul,popcnt,sse4.2") +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__empty_struct +wuffs_xxhash32__hasher__up_x86_sse42( + wuffs_xxhash32__hasher* self, + wuffs_base__slice_u8 a_x) { + uint32_t v_new_lmu = 0; + uint32_t v_buf_u32 = 0; + uint32_t v_buf_len = 0; + uint32_t v_v0 = 0; + uint32_t v_v1 = 0; + uint32_t v_v2 = 0; + uint32_t v_v3 = 0; + wuffs_base__slice_u8 v_p = {0}; + __m128i v_v = {0}; + __m128i v_prime1 = {0}; + __m128i v_prime2 = {0}; + uint64_t v_tail = 0; + + v_new_lmu = ((uint32_t)(self->private_impl.f_length_modulo_u32 + ((uint32_t)(((uint64_t)(a_x.len)))))); + self->private_impl.f_length_overflows_u32 = ((v_new_lmu < self->private_impl.f_length_modulo_u32) || self->private_impl.f_length_overflows_u32); + self->private_impl.f_length_modulo_u32 = v_new_lmu; + while (true) { + if (self->private_impl.f_buf_len >= 16u) { + v_buf_u32 = (((uint32_t)(self->private_impl.f_buf_data[0u])) | + (((uint32_t)(self->private_impl.f_buf_data[1u])) << 8u) | + (((uint32_t)(self->private_impl.f_buf_data[2u])) << 16u) | + (((uint32_t)(self->private_impl.f_buf_data[3u])) << 24u)); + v_v0 = ((uint32_t)(self->private_impl.f_v0 + ((uint32_t)(v_buf_u32 * 2246822519u)))); + v_v0 = (((uint32_t)(v_v0 << 13u)) | (v_v0 >> 19u)); + self->private_impl.f_v0 = ((uint32_t)(v_v0 * 2654435761u)); + v_buf_u32 = (((uint32_t)(self->private_impl.f_buf_data[4u])) | + (((uint32_t)(self->private_impl.f_buf_data[5u])) << 8u) | + (((uint32_t)(self->private_impl.f_buf_data[6u])) << 16u) | + (((uint32_t)(self->private_impl.f_buf_data[7u])) << 24u)); + v_v1 = ((uint32_t)(self->private_impl.f_v1 + ((uint32_t)(v_buf_u32 * 2246822519u)))); + v_v1 = (((uint32_t)(v_v1 << 13u)) | (v_v1 >> 19u)); + self->private_impl.f_v1 = ((uint32_t)(v_v1 * 2654435761u)); + v_buf_u32 = (((uint32_t)(self->private_impl.f_buf_data[8u])) | + (((uint32_t)(self->private_impl.f_buf_data[9u])) << 8u) | + (((uint32_t)(self->private_impl.f_buf_data[10u])) << 16u) | + (((uint32_t)(self->private_impl.f_buf_data[11u])) << 24u)); + v_v2 = ((uint32_t)(self->private_impl.f_v2 + ((uint32_t)(v_buf_u32 * 2246822519u)))); + v_v2 = (((uint32_t)(v_v2 << 13u)) | (v_v2 >> 19u)); + self->private_impl.f_v2 = ((uint32_t)(v_v2 * 2654435761u)); + v_buf_u32 = (((uint32_t)(self->private_impl.f_buf_data[12u])) | + (((uint32_t)(self->private_impl.f_buf_data[13u])) << 8u) | + (((uint32_t)(self->private_impl.f_buf_data[14u])) << 16u) | + (((uint32_t)(self->private_impl.f_buf_data[15u])) << 24u)); + v_v3 = ((uint32_t)(self->private_impl.f_v3 + ((uint32_t)(v_buf_u32 * 2246822519u)))); + v_v3 = (((uint32_t)(v_v3 << 13u)) | (v_v3 >> 19u)); + self->private_impl.f_v3 = ((uint32_t)(v_v3 * 2654435761u)); + self->private_impl.f_buf_len = 0u; + break; + } + if (((uint64_t)(a_x.len)) <= 0u) { + return wuffs_base__make_empty_struct(); + } + self->private_impl.f_buf_data[self->private_impl.f_buf_len] = a_x.ptr[0u]; +#if defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + self->private_impl.f_buf_len += 1u; +#if defined(__GNUC__) +#pragma GCC diagnostic pop +#endif + a_x = wuffs_base__slice_u8__subslice_i(a_x, 1u); + } + v_v = _mm_set_epi32((int32_t)(self->private_impl.f_v3), (int32_t)(self->private_impl.f_v2), (int32_t)(self->private_impl.f_v1), (int32_t)(self->private_impl.f_v0)); + v_prime1 = _mm_set1_epi32((int32_t)(2654435761u)); + v_prime2 = _mm_set1_epi32((int32_t)(2246822519u)); + { + wuffs_base__slice_u8 i_slice_p = a_x; + v_p.ptr = i_slice_p.ptr; + v_p.len = 16; + uint8_t* i_end0_p = v_p.ptr + (((i_slice_p.len - (size_t)(v_p.ptr - i_slice_p.ptr)) / 16) * 16); + while (v_p.ptr < i_end0_p) { + v_v = _mm_add_epi32(v_v, _mm_mullo_epi32(_mm_lddqu_si128((const __m128i*)(const void*)(v_p.ptr)), v_prime2)); + v_v = _mm_or_si128(_mm_slli_epi32(v_v, (int32_t)(13u)), _mm_srli_epi32(v_v, (int32_t)(19u))); + v_v = _mm_mullo_epi32(v_v, v_prime1); + v_p.ptr += 16; + } + v_p.len = 0; + } + self->private_impl.f_v0 = ((uint32_t)(_mm_extract_epi32(v_v, (int32_t)(0u)))); + self->private_impl.f_v1 = ((uint32_t)(_mm_extract_epi32(v_v, (int32_t)(1u)))); + self->private_impl.f_v2 = ((uint32_t)(_mm_extract_epi32(v_v, (int32_t)(2u)))); + self->private_impl.f_v3 = ((uint32_t)(_mm_extract_epi32(v_v, (int32_t)(3u)))); + v_tail = (((uint64_t)(a_x.len)) & 18446744073709551600u); + if (v_tail < ((uint64_t)(a_x.len))) { + v_buf_len = ((uint32_t)((self->private_impl.f_buf_len & 15u))); + { + wuffs_base__slice_u8 i_slice_p = wuffs_base__slice_u8__subslice_i(a_x, v_tail); + v_p.ptr = i_slice_p.ptr; + v_p.len = 1; + uint8_t* i_end0_p = i_slice_p.ptr + i_slice_p.len; + while (v_p.ptr < i_end0_p) { + self->private_impl.f_buf_data[v_buf_len] = v_p.ptr[0u]; + v_buf_len = ((v_buf_len + 1u) & 15u); + v_p.ptr += 1; + } + v_p.len = 0; + } + self->private_impl.f_buf_len = ((uint8_t)(v_buf_len)); + } + return wuffs_base__make_empty_struct(); +} +#endif // defined(WUFFS_BASE__CPU_ARCH__X86_FAMILY) +// ‼ WUFFS MULTI-FILE SECTION -x86_sse42 + // -------- func xxhash32.hasher.get_quirk WUFFS_BASE__GENERATED_C_CODE @@ -61575,6 +61710,11 @@ wuffs_xxhash32__hasher__update( self->private_impl.f_v1 = 2246822519u; self->private_impl.f_v2 = 0u; self->private_impl.f_v3 = 1640531535u; + self->private_impl.choosy_up = ( +#if defined(WUFFS_BASE__CPU_ARCH__X86_FAMILY) + wuffs_base__cpu_arch__have_x86_sse42() ? &wuffs_xxhash32__hasher__up_x86_sse42 : +#endif + self->private_impl.choosy_up); } while (((uint64_t)(a_x.len)) > 0u) { v_remaining = wuffs_base__slice_u8__subslice_j(a_x, 0u); @@ -61613,6 +61753,14 @@ static wuffs_base__empty_struct wuffs_xxhash32__hasher__up( wuffs_xxhash32__hasher* self, wuffs_base__slice_u8 a_x) { + return (*self->private_impl.choosy_up)(self, a_x); +} + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__empty_struct +wuffs_xxhash32__hasher__up__choosy_default( + wuffs_xxhash32__hasher* self, + wuffs_base__slice_u8 a_x) { uint32_t v_new_lmu = 0; uint32_t v_buf_u32 = 0; uint32_t v_buf_len = 0; diff --git a/std/xxhash32/common_up_x86_sse42.wuffs b/std/xxhash32/common_up_x86_sse42.wuffs new file mode 100644 index 00000000..5c7ff83f --- /dev/null +++ b/std/xxhash32/common_up_x86_sse42.wuffs @@ -0,0 +1,107 @@ +// Copyright 2023 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +pri func hasher.up_x86_sse42!(x: roslice base.u8), + choose cpu_arch >= x86_sse42, +{ + // These variables are the same as the non-SIMD version. + var new_lmu : base.u32 + var buf_u32 : base.u32 + var buf_len : base.u32[..= 15] + var v0 : base.u32 + var v1 : base.u32 + var v2 : base.u32 + var v3 : base.u32 + var p : roslice base.u8 + + // The remaining variables are specific to the SIMD version. + + var util : base.x86_sse42_utility + var v : base.x86_m128i + var prime1 : base.x86_m128i + var prime2 : base.x86_m128i + var tail : base.u64 + + new_lmu = this.length_modulo_u32 ~mod+ ((args.x.length() & 0xFFFF_FFFF) as base.u32) + this.length_overflows_u32 = (new_lmu < this.length_modulo_u32) or this.length_overflows_u32 + this.length_modulo_u32 = new_lmu + + while true { + if this.buf_len >= 16 { + buf_u32 = (this.buf_data[0x00] as base.u32) | + ((this.buf_data[0x01] as base.u32) << 8) | + ((this.buf_data[0x02] as base.u32) << 16) | + ((this.buf_data[0x03] as base.u32) << 24) + v0 = this.v0 ~mod+ (buf_u32 ~mod* XXH_PRIME32_2) + v0 = (v0 ~mod<< 13) | (v0 >> 19) + this.v0 = v0 ~mod* XXH_PRIME32_1 + + buf_u32 = (this.buf_data[0x04] as base.u32) | + ((this.buf_data[0x05] as base.u32) << 8) | + ((this.buf_data[0x06] as base.u32) << 16) | + ((this.buf_data[0x07] as base.u32) << 24) + v1 = this.v1 ~mod+ (buf_u32 ~mod* XXH_PRIME32_2) + v1 = (v1 ~mod<< 13) | (v1 >> 19) + this.v1 = v1 ~mod* XXH_PRIME32_1 + + buf_u32 = (this.buf_data[0x08] as base.u32) | + ((this.buf_data[0x09] as base.u32) << 8) | + ((this.buf_data[0x0A] as base.u32) << 16) | + ((this.buf_data[0x0B] as base.u32) << 24) + v2 = this.v2 ~mod+ (buf_u32 ~mod* XXH_PRIME32_2) + v2 = (v2 ~mod<< 13) | (v2 >> 19) + this.v2 = v2 ~mod* XXH_PRIME32_1 + + buf_u32 = (this.buf_data[0x0C] as base.u32) | + ((this.buf_data[0x0D] as base.u32) << 8) | + ((this.buf_data[0x0E] as base.u32) << 16) | + ((this.buf_data[0x0F] as base.u32) << 24) + v3 = this.v3 ~mod+ (buf_u32 ~mod* XXH_PRIME32_2) + v3 = (v3 ~mod<< 13) | (v3 >> 19) + this.v3 = v3 ~mod* XXH_PRIME32_1 + + this.buf_len = 0 + break + } + + if args.x.length() <= 0 { + return nothing + } + this.buf_data[this.buf_len] = args.x[0] + this.buf_len += 1 + args.x = args.x[1 ..] + } endwhile + + v = util.make_m128i_multiple_u32( + a00: this.v0, a01: this.v1, a02: this.v2, a03: this.v3) + prime1 = util.make_m128i_repeat_u32(a: XXH_PRIME32_1) + prime2 = util.make_m128i_repeat_u32(a: XXH_PRIME32_2) + + iterate (p = args.x)(length: 16, advance: 16, unroll: 1) { + v = v._mm_add_epi32(b: + util.make_m128i_slice128(a: p)._mm_mullo_epi32(b: prime2)) + v = v._mm_slli_epi32(imm8: 13)._mm_or_si128(b: + v._mm_srli_epi32(imm8: 19)) + v = v._mm_mullo_epi32(b: prime1) + } + + this.v0 = v._mm_extract_epi32(imm8: 0) + this.v1 = v._mm_extract_epi32(imm8: 1) + this.v2 = v._mm_extract_epi32(imm8: 2) + this.v3 = v._mm_extract_epi32(imm8: 3) + + tail = args.x.length() & 0xFFFF_FFFF_FFFF_FFF0 + if tail < args.x.length() { + buf_len = (this.buf_len & 15) as base.u32 + iterate (p = args.x[tail ..])(length: 1, advance: 1, unroll: 1) { + this.buf_data[buf_len] = p[0] + buf_len = (buf_len + 1) & 15 + } + this.buf_len = buf_len as base.u8 + } +} diff --git a/std/xxhash32/common_xxhash32.wuffs b/std/xxhash32/common_xxhash32.wuffs index 47398430..e592d7eb 100644 --- a/std/xxhash32/common_xxhash32.wuffs +++ b/std/xxhash32/common_xxhash32.wuffs @@ -49,6 +49,9 @@ pub func hasher.update!(x: roslice base.u8) { this.v1 = INITIAL_V1 this.v2 = INITIAL_V2 this.v3 = INITIAL_V3 + + choose up = [ + up_x86_sse42] } while args.x.length() > 0 { @@ -69,7 +72,9 @@ pub func hasher.update_u32!(x: roslice base.u8) base.u32 { return this.checksum_u32() } -pri func hasher.up!(x: roslice base.u8) { +pri func hasher.up!(x: roslice base.u8), + choosy, +{ var new_lmu : base.u32 var buf_u32 : base.u32 var buf_len : base.u32[..= 15]