std/xxhash32: add hasher.up_x86_sse42

This commit will soon be followed by a rollback, but it is committed
anyway so that we can refer to these numbers in the git log.

name                         old speed      new speed      delta

wuffs_xxhash32_10k/clang11   7.81GB/s ± 0%  4.46GB/s ± 1%  -42.86%  (p=0.008 n=5+5)
wuffs_xxhash32_100k/clang11  7.96GB/s ± 0%  4.67GB/s ± 1%  -41.34%  (p=0.008 n=5+5)

wuffs_xxhash32_10k/gcc10     7.83GB/s ± 0%  4.48GB/s ± 2%  -42.81%  (p=0.008 n=5+5)
wuffs_xxhash32_100k/gcc10    7.95GB/s ± 0%  4.69GB/s ± 2%  -41.04%  (p=0.008 n=5+5)

mimic_xxhash32_10k           7.70GB/s ± 1%  7.73GB/s ± 1%     ~     (p=0.421 n=5+5)
mimic_xxhash32_100k          7.79GB/s ± 0%  7.84GB/s ± 1%     ~     (p=0.730 n=4+5)
This commit is contained in:
Nigel Tao committed 2023-10-06 13:59:31 +11:00
1 parent 1304ee6265
commit b9b28dbe8b
4 files changed
+262 -1

No files matched your search

+1
View File
@@ -815,6 +815,7 @@ var funcsOther = [...]string{
"x86_m128i._mm_min_epu16(b: x86_m128i) x86_m128i",
"x86_m128i._mm_min_epu32(b: x86_m128i) x86_m128i",
"x86_m128i._mm_min_epu8(b: x86_m128i) x86_m128i",
"x86_m128i._mm_mullo_epi32(b: x86_m128i) x86_m128i",
"x86_m128i._mm_or_si128(b: x86_m128i) x86_m128i",
"x86_m128i._mm_packs_epi16(b: x86_m128i) x86_m128i",
"x86_m128i._mm_packus_epi16(b: x86_m128i) x86_m128i",
+148
View File
@@ -12565,6 +12565,10 @@ struct wuffs_xxhash32__hasher__struct {
uint32_t f_v1;
uint32_t f_v2;
uint32_t f_v3;
wuffs_base__empty_struct (*choosy_up)(
wuffs_xxhash32__hasher* self,
wuffs_base__slice_u8 a_x);
} private_impl;
#ifdef __cplusplus
@@ -61422,12 +61426,26 @@ wuffs_wbmp__decoder__workbuf_len(
// ---------------- Private Function Prototypes
#if defined(WUFFS_BASE__CPU_ARCH__X86_FAMILY)
WUFFS_BASE__GENERATED_C_CODE
static wuffs_base__empty_struct
wuffs_xxhash32__hasher__up_x86_sse42(
wuffs_xxhash32__hasher* self,
wuffs_base__slice_u8 a_x);
#endif // defined(WUFFS_BASE__CPU_ARCH__X86_FAMILY)
WUFFS_BASE__GENERATED_C_CODE
static wuffs_base__empty_struct
wuffs_xxhash32__hasher__up(
wuffs_xxhash32__hasher* self,
wuffs_base__slice_u8 a_x);
WUFFS_BASE__GENERATED_C_CODE
static wuffs_base__empty_struct
wuffs_xxhash32__hasher__up__choosy_default(
wuffs_xxhash32__hasher* self,
wuffs_base__slice_u8 a_x);
// ---------------- VTables
const wuffs_base__hasher_u32__func_ptrs
@@ -61485,6 +61503,8 @@ wuffs_xxhash32__hasher__initialize(
}
}
self->private_impl.choosy_up = &wuffs_xxhash32__hasher__up__choosy_default;
self->private_impl.magic = WUFFS_BASE__MAGIC;
self->private_impl.vtable_for__wuffs_base__hasher_u32.vtable_name =
wuffs_base__hasher_u32__vtable_name;
@@ -61515,6 +61535,121 @@ sizeof__wuffs_xxhash32__hasher() {
// ---------------- Function Implementations
// ‼ WUFFS MULTI-FILE SECTION +x86_sse42
// -------- func xxhash32.hasher.up_x86_sse42
#if defined(WUFFS_BASE__CPU_ARCH__X86_FAMILY)
WUFFS_BASE__MAYBE_ATTRIBUTE_TARGET("pclmul,popcnt,sse4.2")
WUFFS_BASE__GENERATED_C_CODE
static wuffs_base__empty_struct
wuffs_xxhash32__hasher__up_x86_sse42(
wuffs_xxhash32__hasher* self,
wuffs_base__slice_u8 a_x) {
uint32_t v_new_lmu = 0;
uint32_t v_buf_u32 = 0;
uint32_t v_buf_len = 0;
uint32_t v_v0 = 0;
uint32_t v_v1 = 0;
uint32_t v_v2 = 0;
uint32_t v_v3 = 0;
wuffs_base__slice_u8 v_p = {0};
__m128i v_v = {0};
__m128i v_prime1 = {0};
__m128i v_prime2 = {0};
uint64_t v_tail = 0;
v_new_lmu = ((uint32_t)(self->private_impl.f_length_modulo_u32 + ((uint32_t)(((uint64_t)(a_x.len))))));
self->private_impl.f_length_overflows_u32 = ((v_new_lmu < self->private_impl.f_length_modulo_u32) || self->private_impl.f_length_overflows_u32);
self->private_impl.f_length_modulo_u32 = v_new_lmu;
while (true) {
if (self->private_impl.f_buf_len >= 16u) {
v_buf_u32 = (((uint32_t)(self->private_impl.f_buf_data[0u])) |
(((uint32_t)(self->private_impl.f_buf_data[1u])) << 8u) |
(((uint32_t)(self->private_impl.f_buf_data[2u])) << 16u) |
(((uint32_t)(self->private_impl.f_buf_data[3u])) << 24u));
v_v0 = ((uint32_t)(self->private_impl.f_v0 + ((uint32_t)(v_buf_u32 * 2246822519u))));
v_v0 = (((uint32_t)(v_v0 << 13u)) | (v_v0 >> 19u));
self->private_impl.f_v0 = ((uint32_t)(v_v0 * 2654435761u));
v_buf_u32 = (((uint32_t)(self->private_impl.f_buf_data[4u])) |
(((uint32_t)(self->private_impl.f_buf_data[5u])) << 8u) |
(((uint32_t)(self->private_impl.f_buf_data[6u])) << 16u) |
(((uint32_t)(self->private_impl.f_buf_data[7u])) << 24u));
v_v1 = ((uint32_t)(self->private_impl.f_v1 + ((uint32_t)(v_buf_u32 * 2246822519u))));
v_v1 = (((uint32_t)(v_v1 << 13u)) | (v_v1 >> 19u));
self->private_impl.f_v1 = ((uint32_t)(v_v1 * 2654435761u));
v_buf_u32 = (((uint32_t)(self->private_impl.f_buf_data[8u])) |
(((uint32_t)(self->private_impl.f_buf_data[9u])) << 8u) |
(((uint32_t)(self->private_impl.f_buf_data[10u])) << 16u) |
(((uint32_t)(self->private_impl.f_buf_data[11u])) << 24u));
v_v2 = ((uint32_t)(self->private_impl.f_v2 + ((uint32_t)(v_buf_u32 * 2246822519u))));
v_v2 = (((uint32_t)(v_v2 << 13u)) | (v_v2 >> 19u));
self->private_impl.f_v2 = ((uint32_t)(v_v2 * 2654435761u));
v_buf_u32 = (((uint32_t)(self->private_impl.f_buf_data[12u])) |
(((uint32_t)(self->private_impl.f_buf_data[13u])) << 8u) |
(((uint32_t)(self->private_impl.f_buf_data[14u])) << 16u) |
(((uint32_t)(self->private_impl.f_buf_data[15u])) << 24u));
v_v3 = ((uint32_t)(self->private_impl.f_v3 + ((uint32_t)(v_buf_u32 * 2246822519u))));
v_v3 = (((uint32_t)(v_v3 << 13u)) | (v_v3 >> 19u));
self->private_impl.f_v3 = ((uint32_t)(v_v3 * 2654435761u));
self->private_impl.f_buf_len = 0u;
break;
}
if (((uint64_t)(a_x.len)) <= 0u) {
return wuffs_base__make_empty_struct();
}
self->private_impl.f_buf_data[self->private_impl.f_buf_len] = a_x.ptr[0u];
#if defined(__GNUC__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wconversion"
#endif
self->private_impl.f_buf_len += 1u;
#if defined(__GNUC__)
#pragma GCC diagnostic pop
#endif
a_x = wuffs_base__slice_u8__subslice_i(a_x, 1u);
}
v_v = _mm_set_epi32((int32_t)(self->private_impl.f_v3), (int32_t)(self->private_impl.f_v2), (int32_t)(self->private_impl.f_v1), (int32_t)(self->private_impl.f_v0));
v_prime1 = _mm_set1_epi32((int32_t)(2654435761u));
v_prime2 = _mm_set1_epi32((int32_t)(2246822519u));
{
wuffs_base__slice_u8 i_slice_p = a_x;
v_p.ptr = i_slice_p.ptr;
v_p.len = 16;
uint8_t* i_end0_p = v_p.ptr + (((i_slice_p.len - (size_t)(v_p.ptr - i_slice_p.ptr)) / 16) * 16);
while (v_p.ptr < i_end0_p) {
v_v = _mm_add_epi32(v_v, _mm_mullo_epi32(_mm_lddqu_si128((const __m128i*)(const void*)(v_p.ptr)), v_prime2));
v_v = _mm_or_si128(_mm_slli_epi32(v_v, (int32_t)(13u)), _mm_srli_epi32(v_v, (int32_t)(19u)));
v_v = _mm_mullo_epi32(v_v, v_prime1);
v_p.ptr += 16;
}
v_p.len = 0;
}
self->private_impl.f_v0 = ((uint32_t)(_mm_extract_epi32(v_v, (int32_t)(0u))));
self->private_impl.f_v1 = ((uint32_t)(_mm_extract_epi32(v_v, (int32_t)(1u))));
self->private_impl.f_v2 = ((uint32_t)(_mm_extract_epi32(v_v, (int32_t)(2u))));
self->private_impl.f_v3 = ((uint32_t)(_mm_extract_epi32(v_v, (int32_t)(3u))));
v_tail = (((uint64_t)(a_x.len)) & 18446744073709551600u);
if (v_tail < ((uint64_t)(a_x.len))) {
v_buf_len = ((uint32_t)((self->private_impl.f_buf_len & 15u)));
{
wuffs_base__slice_u8 i_slice_p = wuffs_base__slice_u8__subslice_i(a_x, v_tail);
v_p.ptr = i_slice_p.ptr;
v_p.len = 1;
uint8_t* i_end0_p = i_slice_p.ptr + i_slice_p.len;
while (v_p.ptr < i_end0_p) {
self->private_impl.f_buf_data[v_buf_len] = v_p.ptr[0u];
v_buf_len = ((v_buf_len + 1u) & 15u);
v_p.ptr += 1;
}
v_p.len = 0;
}
self->private_impl.f_buf_len = ((uint8_t)(v_buf_len));
}
return wuffs_base__make_empty_struct();
}
#endif // defined(WUFFS_BASE__CPU_ARCH__X86_FAMILY)
// ‼ WUFFS MULTI-FILE SECTION -x86_sse42
// -------- func xxhash32.hasher.get_quirk
WUFFS_BASE__GENERATED_C_CODE
@@ -61575,6 +61710,11 @@ wuffs_xxhash32__hasher__update(
self->private_impl.f_v1 = 2246822519u;
self->private_impl.f_v2 = 0u;
self->private_impl.f_v3 = 1640531535u;
self->private_impl.choosy_up = (
#if defined(WUFFS_BASE__CPU_ARCH__X86_FAMILY)
wuffs_base__cpu_arch__have_x86_sse42() ? &wuffs_xxhash32__hasher__up_x86_sse42 :
#endif
self->private_impl.choosy_up);
}
while (((uint64_t)(a_x.len)) > 0u) {
v_remaining = wuffs_base__slice_u8__subslice_j(a_x, 0u);
@@ -61613,6 +61753,14 @@ static wuffs_base__empty_struct
wuffs_xxhash32__hasher__up(
wuffs_xxhash32__hasher* self,
wuffs_base__slice_u8 a_x) {
return (*self->private_impl.choosy_up)(self, a_x);
}
WUFFS_BASE__GENERATED_C_CODE
static wuffs_base__empty_struct
wuffs_xxhash32__hasher__up__choosy_default(
wuffs_xxhash32__hasher* self,
wuffs_base__slice_u8 a_x) {
uint32_t v_new_lmu = 0;
uint32_t v_buf_u32 = 0;
uint32_t v_buf_len = 0;
+107
View File
@@ -0,0 +1,107 @@
// Copyright 2023 The Wuffs Authors.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
pri func hasher.up_x86_sse42!(x: roslice base.u8),
choose cpu_arch >= x86_sse42,
{
// These variables are the same as the non-SIMD version.
var new_lmu : base.u32
var buf_u32 : base.u32
var buf_len : base.u32[..= 15]
var v0 : base.u32
var v1 : base.u32
var v2 : base.u32
var v3 : base.u32
var p : roslice base.u8
// The remaining variables are specific to the SIMD version.
var util : base.x86_sse42_utility
var v : base.x86_m128i
var prime1 : base.x86_m128i
var prime2 : base.x86_m128i
var tail : base.u64
new_lmu = this.length_modulo_u32 ~mod+ ((args.x.length() & 0xFFFF_FFFF) as base.u32)
this.length_overflows_u32 = (new_lmu < this.length_modulo_u32) or this.length_overflows_u32
this.length_modulo_u32 = new_lmu
while true {
if this.buf_len >= 16 {
buf_u32 = (this.buf_data[0x00] as base.u32) |
((this.buf_data[0x01] as base.u32) << 8) |
((this.buf_data[0x02] as base.u32) << 16) |
((this.buf_data[0x03] as base.u32) << 24)
v0 = this.v0 ~mod+ (buf_u32 ~mod* XXH_PRIME32_2)
v0 = (v0 ~mod<< 13) | (v0 >> 19)
this.v0 = v0 ~mod* XXH_PRIME32_1
buf_u32 = (this.buf_data[0x04] as base.u32) |
((this.buf_data[0x05] as base.u32) << 8) |
((this.buf_data[0x06] as base.u32) << 16) |
((this.buf_data[0x07] as base.u32) << 24)
v1 = this.v1 ~mod+ (buf_u32 ~mod* XXH_PRIME32_2)
v1 = (v1 ~mod<< 13) | (v1 >> 19)
this.v1 = v1 ~mod* XXH_PRIME32_1
buf_u32 = (this.buf_data[0x08] as base.u32) |
((this.buf_data[0x09] as base.u32) << 8) |
((this.buf_data[0x0A] as base.u32) << 16) |
((this.buf_data[0x0B] as base.u32) << 24)
v2 = this.v2 ~mod+ (buf_u32 ~mod* XXH_PRIME32_2)
v2 = (v2 ~mod<< 13) | (v2 >> 19)
this.v2 = v2 ~mod* XXH_PRIME32_1
buf_u32 = (this.buf_data[0x0C] as base.u32) |
((this.buf_data[0x0D] as base.u32) << 8) |
((this.buf_data[0x0E] as base.u32) << 16) |
((this.buf_data[0x0F] as base.u32) << 24)
v3 = this.v3 ~mod+ (buf_u32 ~mod* XXH_PRIME32_2)
v3 = (v3 ~mod<< 13) | (v3 >> 19)
this.v3 = v3 ~mod* XXH_PRIME32_1
this.buf_len = 0
break
}
if args.x.length() <= 0 {
return nothing
}
this.buf_data[this.buf_len] = args.x[0]
this.buf_len += 1
args.x = args.x[1 ..]
} endwhile
v = util.make_m128i_multiple_u32(
a00: this.v0, a01: this.v1, a02: this.v2, a03: this.v3)
prime1 = util.make_m128i_repeat_u32(a: XXH_PRIME32_1)
prime2 = util.make_m128i_repeat_u32(a: XXH_PRIME32_2)
iterate (p = args.x)(length: 16, advance: 16, unroll: 1) {
v = v._mm_add_epi32(b:
util.make_m128i_slice128(a: p)._mm_mullo_epi32(b: prime2))
v = v._mm_slli_epi32(imm8: 13)._mm_or_si128(b:
v._mm_srli_epi32(imm8: 19))
v = v._mm_mullo_epi32(b: prime1)
}
this.v0 = v._mm_extract_epi32(imm8: 0)
this.v1 = v._mm_extract_epi32(imm8: 1)
this.v2 = v._mm_extract_epi32(imm8: 2)
this.v3 = v._mm_extract_epi32(imm8: 3)
tail = args.x.length() & 0xFFFF_FFFF_FFFF_FFF0
if tail < args.x.length() {
buf_len = (this.buf_len & 15) as base.u32
iterate (p = args.x[tail ..])(length: 1, advance: 1, unroll: 1) {
this.buf_data[buf_len] = p[0]
buf_len = (buf_len + 1) & 15
}
this.buf_len = buf_len as base.u8
}
}
+6 -1
View File
@@ -49,6 +49,9 @@ pub func hasher.update!(x: roslice base.u8) {
this.v1 = INITIAL_V1
this.v2 = INITIAL_V2
this.v3 = INITIAL_V3
choose up = [
up_x86_sse42]
}
while args.x.length() > 0 {
@@ -69,7 +72,9 @@ pub func hasher.update_u32!(x: roslice base.u8) base.u32 {
return this.checksum_u32()
}
pri func hasher.up!(x: roslice base.u8) {
pri func hasher.up!(x: roslice base.u8),
choosy,
{
var new_lmu : base.u32
var buf_u32 : base.u32
var buf_len : base.u32[..= 15]