mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-19 08:38:14 +01:00
On a mid-range x86_64 laptop: wuffs_adler32_10k/clang9 1.35GB/s ± 0% 10.78GB/s ± 1% +700.65% (p=0.008 n=5+5) wuffs_adler32_100k/clang9 1.35GB/s ± 0% 12.88GB/s ± 0% +856.09% (p=0.008 n=5+5) wuffs_adler32_10k/gcc10 1.85GB/s ± 0% 11.21GB/s ± 0% +506.34% (p=0.008 n=5+5) wuffs_adler32_100k/gcc10 1.84GB/s ± 0% 11.43GB/s ± 0% +521.51% (p=0.008 n=5+5) mimic_adler32_10k 1.76GB/s ± 0% 1.76GB/s ± 0% ~ (p=0.841 n=5+5) mimic_adler32_100k 1.72GB/s ± 0% 1.72GB/s ± 0% ~ (p=0.222 n=5+5) wuffs_zlib_decode_10k/clang9 124MB/s ± 0% 139MB/s ± 0% +12.29% (p=0.016 n=4+5) wuffs_zlib_decode_100k/clang9 148MB/s ± 0% 155MB/s ± 0% +4.21% (p=0.008 n=5+5) wuffs_zlib_decode_10k/gcc10 133MB/s ± 1% 141MB/s ± 0% +5.75% (p=0.008 n=5+5) wuffs_zlib_decode_100k/gcc10 159MB/s ± 0% 172MB/s ± 0% +7.77% (p=0.016 n=5+4) mimic_zlib_decode_10k 133MB/s ± 0% 133MB/s ± 0% +0.38% (p=0.008 n=5+5) mimic_zlib_decode_100k 160MB/s ± 0% 160MB/s ± 0% ~ (p=0.841 n=5+5) wuffs_png_decode_image_19k_8bpp/clang9 90.6MB/s ± 0% 94.7MB/s ± 0% +4.58% (p=0.016 n=5+4) wuffs_png_decode_image_40k_24bpp/clang9 129MB/s ± 0% 135MB/s ± 0% +4.49% (p=0.008 n=5+5) wuffs_png_decode_image_77k_8bpp/clang9 335MB/s ± 0% 351MB/s ± 0% +4.62% (p=0.008 n=5+5) wuffs_png_decode_image_552k_32bpp_ignore_checksum/clang9 322MB/s ± 0% 320MB/s ± 1% -0.71% (p=0.016 n=4+5) wuffs_png_decode_image_552k_32bpp_verify_checksum/clang9 256MB/s ± 0% 308MB/s ± 0% +20.24% (p=0.008 n=5+5) wuffs_png_decode_image_4002k_24bpp/clang9 131MB/s ± 0% 138MB/s ± 0% +5.24% (p=0.008 n=5+5) wuffs_png_decode_image_19k_8bpp/gcc10 95.1MB/s ± 0% 100.8MB/s ± 1% +6.08% (p=0.008 n=5+5) wuffs_png_decode_image_40k_24bpp/gcc10 131MB/s ± 0% 136MB/s ± 0% +3.65% (p=0.008 n=5+5) wuffs_png_decode_image_77k_8bpp/gcc10 350MB/s ± 1% 371MB/s ± 0% +6.06% (p=0.008 n=5+5) wuffs_png_decode_image_552k_32bpp_ignore_checksum/gcc10 327MB/s ± 0% 333MB/s ± 0% +1.66% (p=0.008 n=5+5) wuffs_png_decode_image_552k_32bpp_verify_checksum/gcc10 274MB/s ± 0% 318MB/s ± 0% +15.93% (p=0.008 n=5+5) wuffs_png_decode_image_4002k_24bpp/gcc10 132MB/s ± 0% 138MB/s ± 0% +4.73% (p=0.008 n=5+5) mimic_png_decode_image_19k_8bpp 58.1MB/s ± 0% 58.2MB/s ± 0% +0.27% (p=0.008 n=5+5) mimic_png_decode_image_40k_24bpp 73.0MB/s ± 1% 73.3MB/s ± 0% +0.45% (p=0.008 n=5+5) mimic_png_decode_image_77k_8bpp 177MB/s ± 0% 177MB/s ± 0% ~ (p=0.421 n=5+5) mimic_png_decode_image_552k_32bpp_ignore_checksum skipped mimic_png_decode_image_552k_32bpp_verify_checksum 146MB/s ± 0% 146MB/s ± 0% ~ (p=0.310 n=5+5) mimic_png_decode_image_4002k_24bpp 104MB/s ± 0% 104MB/s ± 0% ~ (p=0.056 n=5+5)
230 lines
8.6 KiB
Plaintext
230 lines
8.6 KiB
Plaintext
// Copyright 2017 The Wuffs Authors.
|
||
//
|
||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||
// you may not use this file except in compliance with the License.
|
||
// You may obtain a copy of the License at
|
||
//
|
||
// https://www.apache.org/licenses/LICENSE-2.0
|
||
//
|
||
// Unless required by applicable law or agreed to in writing, software
|
||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
// See the License for the specific language governing permissions and
|
||
// limitations under the License.
|
||
|
||
// TODO: drop the '?' but still generate wuffs_adler32__hasher__initialize?
|
||
pub struct hasher? implements base.hasher_u32(
|
||
state : base.u32,
|
||
started : base.bool,
|
||
)
|
||
|
||
pub func hasher.set_quirk_enabled!(quirk: base.u32, enabled: base.bool) {
|
||
}
|
||
|
||
pub func hasher.update_u32!(x: slice base.u8) base.u32 {
|
||
if not this.started {
|
||
this.started = true
|
||
this.state = 1
|
||
choose up = [up_sse42]
|
||
}
|
||
this.up!(x: args.x)
|
||
return this.state
|
||
}
|
||
|
||
pri func hasher.up!(x: slice base.u8),
|
||
choosy,
|
||
{
|
||
// The Adler-32 checksum's magic 65521 and 5552 numbers are discussed in
|
||
// this package's README.md.
|
||
|
||
var s1 : base.u32
|
||
var s2 : base.u32
|
||
var remaining : slice base.u8
|
||
var p : slice base.u8
|
||
|
||
s1 = this.state.low_bits(n: 16)
|
||
s2 = this.state.high_bits(n: 16)
|
||
while args.x.length() > 0 {
|
||
remaining = args.x[.. 0]
|
||
if args.x.length() > 5552 {
|
||
remaining = args.x[5552 ..]
|
||
args.x = args.x[.. 5552]
|
||
}
|
||
|
||
// The SIMD versions of this function replace this simple iterate loop.
|
||
iterate (p = args.x)(length: 1, advance: 1, unroll: 8) {
|
||
s1 ~mod+= p[0] as base.u32
|
||
s2 ~mod+= s1
|
||
}
|
||
|
||
s1 %= 65521
|
||
s2 %= 65521
|
||
args.x = remaining
|
||
} endwhile
|
||
this.state = ((s2 & 0xFFFF) << 16) | (s1 & 0xFFFF)
|
||
}
|
||
|
||
pri func hasher.up_sse42!(x: slice base.u8),
|
||
choose cpu_arch >= x86_sse42,
|
||
{
|
||
// These variables are the same as the non-SIMD version.
|
||
var s1 : base.u32
|
||
var s2 : base.u32
|
||
var remaining : slice base.u8
|
||
var p : slice base.u8
|
||
|
||
// The remaining variables are specific to the SIMD version.
|
||
var zeroes : base.x86_m128i
|
||
var ones : base.x86_m128i
|
||
var weights__left : base.x86_m128i
|
||
var weights_right : base.x86_m128i
|
||
var p__left : base.x86_m128i
|
||
var p_right : base.x86_m128i
|
||
var v1 : base.x86_m128i
|
||
var v2 : base.x86_m128i
|
||
var v2j : base.x86_m128i
|
||
var v2k : base.x86_m128i
|
||
var num_iterate_bytes : base.u32
|
||
var tail_index : base.u64
|
||
|
||
// zeroes and ones are uniform 8×u16 vectors.
|
||
zeroes = zeroes.create_mm_set1_epi16(a: 0)
|
||
ones = ones.create_mm_set1_epi16(a: 1)
|
||
|
||
// weights__left and weights_right, little-endian, form the sequence 32,
|
||
// 31, 30, ..., 1.
|
||
weights__left = weights__left.create_mm_set_epi8(
|
||
e15: 0x11, e14: 0x12, e13: 0x13, e12: 0x14,
|
||
e11: 0x15, e10: 0x16, e9: 0x17, e8: 0x18,
|
||
e7: 0x19, e6: 0x1A, e5: 0x1B, e4: 0x1C,
|
||
e3: 0x1D, e2: 0x1E, e1: 0x1F, e0: 0x20)
|
||
weights_right = weights_right.create_mm_set_epi8(
|
||
e15: 0x01, e14: 0x02, e13: 0x03, e12: 0x04,
|
||
e11: 0x05, e10: 0x06, e9: 0x07, e8: 0x08,
|
||
e7: 0x09, e6: 0x0A, e5: 0x0B, e4: 0x0C,
|
||
e3: 0x0D, e2: 0x0E, e1: 0x0F, e0: 0x10)
|
||
|
||
// Just like the non-SIMD version, split the u32 state into two u16 halves
|
||
// and loop over args.x up to almost-5552 bytes at a time. The slightly
|
||
// smaller 5536 is the largest multiple of 32 less than non-SIMD's 5552.
|
||
s1 = this.state.low_bits(n: 16)
|
||
s2 = this.state.high_bits(n: 16)
|
||
while args.x.length() > 0 {
|
||
remaining = args.x[.. 0]
|
||
if args.x.length() > 5536 {
|
||
remaining = args.x[5536 ..]
|
||
args.x = args.x[.. 5536]
|
||
}
|
||
|
||
// The s1 state is the sum of the input bytes and the s2 state is the
|
||
// sum of the s1 state at each 1-byte step. Inside the iterate loop
|
||
// below, but starting fresh at each outer while loop iteration, s1
|
||
// consists of three parts (called s1i, s1j and s1k):
|
||
// - s1i: the initial value, before any 32-byte iterations.
|
||
// - s1j: the total contribution from previous 32-byte iterations.
|
||
// - s1k: the contribution due to the current 32-byte iteration.
|
||
//
|
||
// The upcoming iterate loop (at 32 bytes per iteration) encompasses
|
||
// num_iterate_bytes 1-byte steps. We hoist the total s1i contribution,
|
||
// (s1i * num_iterate_bytes) out here.
|
||
num_iterate_bytes = (args.x.length() & 0xFFFF_FFE0) as base.u32
|
||
s2 ~mod+= (s1 ~mod* num_iterate_bytes)
|
||
|
||
// Zero-initialize some 4×u32 vectors associated with the two state
|
||
// variables s1 and s2. The iterate loop accumulates 4 parallel u32
|
||
// sums in each vector. A post-iterate step merges the four u32 sums
|
||
// into a single u32 sum.
|
||
v1 = v1.create_mm_setzero_si128()
|
||
v2j = v2j.create_mm_setzero_si128()
|
||
v2k = v2k.create_mm_setzero_si128()
|
||
|
||
// The inner loop.
|
||
iterate (p = args.x)(length: 32, advance: 32, unroll: 1) {
|
||
// Split the 32-byte p into left and right halves. SSE4.2 works
|
||
// with 16-byte registers.
|
||
//
|
||
// Let p__left = [16×u8: p00, p01, p02, ..., p15]
|
||
// Let p_right = [16×u8: p16, p17, p18, ..., p31]
|
||
p__left.load_slice128!(a: p[.. 16])
|
||
p_right.load_slice128!(a: p[16 .. 32])
|
||
|
||
// For v2j, we need to calculate the sums of the s1j terms for each
|
||
// of p's 32 elements. This is simply 32 times the same number,
|
||
// that number being the sum of v1's four u32 accumulators. We
|
||
// add v1 now and multiply by 32 later, outside the inner loop.
|
||
v2j = v2j._mm_add_epi32(b: v1)
|
||
|
||
// For v1, we need to add the elements of p. Computing the sum of
|
||
// absolute differences (_mm_sad_epu8) with zero just sums the
|
||
// elements. p__left._mm_sad_epu8(b: zeroes) equals
|
||
// [2×u64: p00 + p01 + ... + p07, p08 + p09 + ... + p15]
|
||
// This is equivalent (little-endian) to:
|
||
// [4×u32: p00 + p01 + ... + p07, 0, p08 + p09 + ... + p15, 0]
|
||
// We accumulate those "sum of p__left elements" in v1, and ditto
|
||
// for the p_right elements.
|
||
v1 = v1._mm_add_epi32(b: p__left._mm_sad_epu8(b: zeroes))
|
||
v1 = v1._mm_add_epi32(b: p_right._mm_sad_epu8(b: zeroes))
|
||
|
||
// For v2k, we need to calculate a weighted sum: ((32 * p00) + (31
|
||
// * p01) + (30 * p02) + ... + (1 * p31)), which splits naturally
|
||
// into weighted sums of the left half and of the right half.
|
||
//
|
||
// The _mm_maddubs_epi16 call (vertically multiply u8 columns and
|
||
// then horizontally sum u16 pairs) with the left half produces:
|
||
// [8×u16: ((32*p00)+(31*p01)),
|
||
// ((30*p02)+(29*p03)),
|
||
// ...
|
||
// ((18*p14)+(17*p15))]
|
||
//
|
||
// The ones._mm_madd_epi16(b: etc) call is likewise a multiply-add
|
||
// (note that it's "madd" not "add"). Multiplying by 1 is a no-op,
|
||
// so this sums u16 pairs to produce u32 values:
|
||
// [4×u32: ((32*p00)+(31*p01)+(30*p02)+(29*p03)),
|
||
// ((28*p04)+(27*p05)+(26*p06)+(25*p07)),
|
||
// ...
|
||
// ((20*p12)+(19*p13)+(18*p14)+(17*p15))]
|
||
//
|
||
// Ditto again for the p_right elements.
|
||
v2k = v2k._mm_add_epi32(b: ones._mm_madd_epi16(b:
|
||
p__left._mm_maddubs_epi16(b: weights__left)))
|
||
v2k = v2k._mm_add_epi32(b: ones._mm_madd_epi16(b:
|
||
p_right._mm_maddubs_epi16(b: weights_right)))
|
||
}
|
||
|
||
// Merge the four per-u32-lane sums (v1) into the single u32 sum (s1).
|
||
// Starting with a 4×u32 vector [x0, x1, x2, x3]:
|
||
// - shuffling with 0b1011_0001 gives [x1, x0, x3, x2].
|
||
// - adding gives [x0+x1, x0+x1, x2+x3, x2+x3].
|
||
// - shuffling with 0b0100_1110 gives [x2+x3, x2+x3, x0+x1, x0+x1].
|
||
// - adding gives [x0+x1+x2+x3, ditto, ditto, ditto].
|
||
// The truncate_u32 call extracts the first u32: x0+x1+x2+x3.
|
||
v1 = v1._mm_add_epi32(b: v1._mm_shuffle_epi32(imm8: 0xB1)) // 0b1011_0001
|
||
v1 = v1._mm_add_epi32(b: v1._mm_shuffle_epi32(imm8: 0x4E)) // 0b0100_1110
|
||
s1 ~mod+= v1.truncate_u32()
|
||
|
||
// Combine v2j and v2k. The slli (shift logical left immediate) by 5
|
||
// multiplies v2j's four u32 elements each by 32, alluded to earlier.
|
||
v2 = v2k._mm_add_epi32(b: v2j._mm_slli_epi32(imm8: 5))
|
||
|
||
// Similarly merge v2 (a 4×u32 vector) into s2 (a scalar u32 scalar).
|
||
v2 = v2._mm_add_epi32(b: v2._mm_shuffle_epi32(imm8: 0xB1)) // 0b1011_0001
|
||
v2 = v2._mm_add_epi32(b: v2._mm_shuffle_epi32(imm8: 0x4E)) // 0b0100_1110
|
||
s2 ~mod+= v2.truncate_u32()
|
||
|
||
// Handle the tail of args.x that wasn't a complete 32-byte chunk.
|
||
tail_index = args.x.length() & 0xFFFF_FFFF_FFFF_FFE0 // And-not 32.
|
||
if tail_index < args.x.length() {
|
||
iterate (p = args.x[tail_index ..])(length: 1, advance: 1, unroll: 1) {
|
||
s1 ~mod+= p[0] as base.u32
|
||
s2 ~mod+= s1
|
||
}
|
||
}
|
||
|
||
// The rest of this function is the same as the non-SIMD version.
|
||
s1 %= 65521
|
||
s2 %= 65521
|
||
args.x = remaining
|
||
} endwhile
|
||
this.state = ((s2 & 0xFFFF) << 16) | (s1 & 0xFFFF)
|
||
}
|