mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-17 16:32:11 +01:00
std/jpeg: "AC all zero" fast path for x86_avx2
name old speed new speed delta wuffs_jpeg_decode_19k_8bpp/clang11 370MB/s ± 0% 368MB/s ± 0% -0.47% (p=0.008 n=5+5) wuffs_jpeg_decode_30k_24bpp_progressive/clang11 231MB/s ± 0% 231MB/s ± 0% -0.26% (p=0.008 n=5+5) wuffs_jpeg_decode_30k_24bpp_sequential/clang11 470MB/s ± 0% 470MB/s ± 0% ~ (p=0.841 n=5+5) wuffs_jpeg_decode_77k_24bpp/clang11 418MB/s ± 0% 416MB/s ± 0% -0.42% (p=0.008 n=5+5) wuffs_jpeg_decode_552k_24bpp_420/clang11 435MB/s ± 0% 433MB/s ± 0% -0.33% (p=0.008 n=5+5) wuffs_jpeg_decode_552k_24bpp_444/clang11 620MB/s ± 0% 646MB/s ± 0% +4.20% (p=0.008 n=5+5) wuffs_jpeg_decode_4002k_24bpp/clang11 441MB/s ± 0% 439MB/s ± 0% -0.51% (p=0.008 n=5+5) wuffs_jpeg_decode_19k_8bpp/gcc10 352MB/s ± 0% 364MB/s ± 0% +3.41% (p=0.008 n=5+5) wuffs_jpeg_decode_30k_24bpp_progressive/gcc10 296MB/s ± 0% 294MB/s ± 0% -0.69% (p=0.008 n=5+5) wuffs_jpeg_decode_30k_24bpp_sequential/gcc10 666MB/s ± 0% 680MB/s ± 0% +2.10% (p=0.008 n=5+5) wuffs_jpeg_decode_77k_24bpp/gcc10 565MB/s ± 0% 578MB/s ± 0% +2.34% (p=0.008 n=5+5) wuffs_jpeg_decode_552k_24bpp_420/gcc10 608MB/s ± 0% 622MB/s ± 0% +2.26% (p=0.016 n=5+4) wuffs_jpeg_decode_552k_24bpp_444/gcc10 624MB/s ± 0% 659MB/s ± 0% +5.47% (p=0.008 n=5+5) wuffs_jpeg_decode_4002k_24bpp/gcc10 616MB/s ± 0% 630MB/s ± 0% +2.29% (p=0.008 n=5+5)
This commit is contained in:
1 parent
2bff962cf1
commit
ecaf25ebf0
6 files changed
+104
-5
No files matched your search
@@ -611,6 +611,9 @@ func (g *gen) writeBuiltinCPUArchX86(b *buffer, recv *a.Expr, method t.ID, args
|
||||
if strings.HasPrefix(methodStr, "make_") {
|
||||
fName, tName, ptr := "", "", false
|
||||
switch methodStr {
|
||||
case "make_u64_slice_u16lex4":
|
||||
fName, tName, ptr = "wuffs_base__peek_u64le__no_bounds_check", "const uint8_t*)(const void*", true
|
||||
|
||||
case "make_m128i_multiple_u8":
|
||||
fName, tName = "_mm_set_epi8", "int8_t"
|
||||
case "make_m128i_multiple_u16":
|
||||
@@ -631,7 +634,7 @@ func (g *gen) writeBuiltinCPUArchX86(b *buffer, recv *a.Expr, method t.ID, args
|
||||
fName, tName = "_mm_cvtsi32_si128", "int32_t"
|
||||
case "make_m128i_single_u64":
|
||||
fName, tName = "_mm_cvtsi64x_si128", "int64_t"
|
||||
case "make_m128i_slice128":
|
||||
case "make_m128i_slice128", "make_m128i_slice_u16lex8":
|
||||
fName, tName, ptr = "_mm_lddqu_si128", "const __m128i*)(const void*", true
|
||||
case "make_m128i_zeroes":
|
||||
fName, tName = "_mm_setzero_si128", ""
|
||||
|
||||
@@ -770,9 +770,12 @@ var funcsOther = [...]string{
|
||||
"x86_sse42_utility.make_m128i_single_u64(a: u64) x86_m128i",
|
||||
|
||||
"x86_sse42_utility.make_m128i_slice128(a: roslice base.u8) x86_m128i",
|
||||
"x86_sse42_utility.make_m128i_slice_u16lex8(a: roslice base.u16) x86_m128i",
|
||||
|
||||
"x86_sse42_utility.make_m128i_zeroes() x86_m128i",
|
||||
|
||||
"x86_sse42_utility.make_u64_slice_u16lex4(a: roslice base.u16) u64",
|
||||
|
||||
// ---- x86_m128i
|
||||
|
||||
"x86_m128i.store_slice64!(a: slice base.u8)",
|
||||
@@ -816,6 +819,8 @@ var funcsOther = [...]string{
|
||||
"x86_m128i._mm_min_epu16(b: x86_m128i) x86_m128i",
|
||||
"x86_m128i._mm_min_epu32(b: x86_m128i) x86_m128i",
|
||||
"x86_m128i._mm_min_epu8(b: x86_m128i) x86_m128i",
|
||||
"x86_m128i._mm_or_si128(b: x86_m128i) x86_m128i",
|
||||
"x86_m128i._mm_packs_epi16(b: x86_m128i) x86_m128i",
|
||||
"x86_m128i._mm_packus_epi16(b: x86_m128i) x86_m128i",
|
||||
"x86_m128i._mm_sad_epu8(b: x86_m128i) x86_m128i",
|
||||
"x86_m128i._mm_shuffle_epi32(imm8: u32) x86_m128i",
|
||||
@@ -874,6 +879,8 @@ var funcsOther = [...]string{
|
||||
|
||||
"x86_avx2_utility.make_m256i_zeroes() x86_m256i",
|
||||
|
||||
"x86_avx2_utility.make_u64_slice_u16lex4(a: roslice base.u16) u64",
|
||||
|
||||
// ---- x86_m256i
|
||||
|
||||
"x86_m256i.store_slice64!(a: slice base.u8)",
|
||||
@@ -886,8 +893,10 @@ var funcsOther = [...]string{
|
||||
"x86_m256i._mm256_add_epi32(b: x86_m256i) x86_m256i",
|
||||
"x86_m256i._mm256_add_epi64(b: x86_m256i) x86_m256i",
|
||||
"x86_m256i._mm256_add_epi8(b: x86_m256i) x86_m256i",
|
||||
"x86_m256i._mm256_castsi256_si128() x86_m128i",
|
||||
"x86_m256i._mm256_extract_epi64(index: u32) u64",
|
||||
"x86_m256i._mm256_extracti128_si256(imm8: u32) x86_m128i",
|
||||
"x86_m256i._mm256_inserti128_si256(b: x86_m128i, imm8: u32) x86_m256i",
|
||||
"x86_m256i._mm256_madd_epi16(b: x86_m256i) x86_m256i",
|
||||
"x86_m256i._mm256_maddubs_epi16(b: x86_m256i) x86_m256i",
|
||||
"x86_m256i._mm256_mullo_epi16(b: x86_m256i) x86_m256i",
|
||||
@@ -896,6 +905,7 @@ var funcsOther = [...]string{
|
||||
"x86_m256i._mm256_permute2x128_si256(b: x86_m256i, imm8: u32) x86_m256i",
|
||||
"x86_m256i._mm256_permute4x64_epi64(imm8: u32) x86_m256i",
|
||||
"x86_m256i._mm256_sad_epu8(b: x86_m256i) x86_m256i",
|
||||
"x86_m256i._mm256_shuffle_epi32(imm8: u32) x86_m256i",
|
||||
"x86_m256i._mm256_sign_epi16(b: x86_m256i) x86_m256i",
|
||||
"x86_m256i._mm256_slli_epi16(imm8: u32) x86_m256i",
|
||||
"x86_m256i._mm256_slli_epi32(imm8: u32) x86_m256i",
|
||||
|
||||
@@ -1282,8 +1282,14 @@ func (q *checker) bcheckExprCallSpecialCases(n *a.Expr, depth uint32) (bounds, e
|
||||
advance = thirtyTwo
|
||||
case strings.HasSuffix(s, "_slice512"): // 512 bits is 64 bytes.
|
||||
advance = sixtyFour
|
||||
case strings.HasSuffix(s, "_slice_u16lex4"): // 4 u16 values is 8 bytes.
|
||||
advance = four
|
||||
case strings.HasSuffix(s, "_slice_u16lex8"): // 8 u16 values is 16 bytes.
|
||||
advance = eight
|
||||
case strings.HasSuffix(s, "_slice_u16lex16"): // 16 u16 values is 32 bytes.
|
||||
advance = sixteen
|
||||
case strings.HasSuffix(s, "_slice_u16lex32"): // 32 u16 values is 64 bytes.
|
||||
advance = thirtyTwo
|
||||
case strings.Contains(s, "_slice"):
|
||||
return bounds{}, fmt.Errorf("check: internal error: unrecognized %s method", s)
|
||||
}
|
||||
|
||||
@@ -40063,6 +40063,11 @@ wuffs_jpeg__decoder__decode_idct_x86_avx2(
|
||||
__m256i v_k_E333_133E_ADFD_1051 = {0};
|
||||
__m256i v_k_E6DC_25A1_1925_25A1 = {0};
|
||||
__m256i v_k_ECC1_E333_EFB0_ADFD = {0};
|
||||
__m128i v_az_coeffs = {0};
|
||||
__m256i v_az_ah00 = {0};
|
||||
__m256i v_az_ad00 = {0};
|
||||
__m256i v_az_eh00 = {0};
|
||||
__m256i v_az_adeh = {0};
|
||||
__m256i v_rows01 = {0};
|
||||
__m256i v_rows23 = {0};
|
||||
__m256i v_rows45 = {0};
|
||||
@@ -40216,6 +40221,23 @@ wuffs_jpeg__decoder__decode_idct_x86_avx2(
|
||||
v_k_E6DC_25A1_1925_25A1 = _mm256_set_epi16((int16_t)(9633u), (int16_t)(6437u), (int16_t)(9633u), (int16_t)(6437u), (int16_t)(9633u), (int16_t)(6437u), (int16_t)(9633u), (int16_t)(6437u), (int16_t)(9633u), (int16_t)(59100u), (int16_t)(9633u), (int16_t)(59100u), (int16_t)(9633u), (int16_t)(59100u), (int16_t)(9633u), (int16_t)(59100u));
|
||||
v_k_ECC1_E333_EFB0_ADFD = _mm256_set_epi16((int16_t)(44541u), (int16_t)(61360u), (int16_t)(44541u), (int16_t)(61360u), (int16_t)(44541u), (int16_t)(61360u), (int16_t)(44541u), (int16_t)(61360u), (int16_t)(58163u), (int16_t)(60609u), (int16_t)(58163u), (int16_t)(60609u), (int16_t)(58163u), (int16_t)(60609u), (int16_t)(58163u), (int16_t)(60609u));
|
||||
do {
|
||||
if (0u == (wuffs_base__peek_u64le__no_bounds_check((const uint8_t*)(const void*)(self->private_data.f_mcu_blocks[0u] + 8u)) | wuffs_base__peek_u64le__no_bounds_check((const uint8_t*)(const void*)(self->private_data.f_mcu_blocks[0u] + 16u)))) {
|
||||
v_az_coeffs = _mm_or_si128(_mm_or_si128(_mm_or_si128(_mm_or_si128(_mm_or_si128(_mm_or_si128(_mm_lddqu_si128((const __m128i*)(const void*)(self->private_data.f_mcu_blocks[0u] + 8u)), _mm_lddqu_si128((const __m128i*)(const void*)(self->private_data.f_mcu_blocks[0u] + 16u))), _mm_lddqu_si128((const __m128i*)(const void*)(self->private_data.f_mcu_blocks[0u] + 24u))), _mm_lddqu_si128((const __m128i*)(const void*)(self->private_data.f_mcu_blocks[0u] + 32u))), _mm_lddqu_si128((const __m128i*)(const void*)(self->private_data.f_mcu_blocks[0u] + 40u))), _mm_lddqu_si128((const __m128i*)(const void*)(self->private_data.f_mcu_blocks[0u] + 48u))), _mm_lddqu_si128((const __m128i*)(const void*)(self->private_data.f_mcu_blocks[0u] + 56u)));
|
||||
if (0u == ((uint64_t)(_mm_cvtsi128_si64(_mm_packs_epi16(v_az_coeffs, v_az_coeffs))))) {
|
||||
v_rows01 = _mm256_lddqu_si256((const __m256i*)(const void*)(self->private_data.f_mcu_blocks[0u] + 0u));
|
||||
v_quants01 = _mm256_lddqu_si256((const __m256i*)(const void*)(self->private_impl.f_quant_tables[a_q] + 0u));
|
||||
v_rows01 = _mm256_mullo_epi16(v_rows01, v_quants01);
|
||||
v_az_ah00 = _mm256_slli_epi16(v_rows01, (int32_t)(2u));
|
||||
v_az_ad00 = _mm256_unpacklo_epi16(v_az_ah00, v_az_ah00);
|
||||
v_az_eh00 = _mm256_unpackhi_epi16(v_az_ah00, v_az_ah00);
|
||||
v_az_adeh = _mm256_inserti128_si256(v_az_ad00, _mm256_castsi256_si128(v_az_eh00), (int32_t)(1u));
|
||||
v_intermediateae = _mm256_shuffle_epi32(v_az_adeh, (int32_t)(0u));
|
||||
v_intermediatebf = _mm256_shuffle_epi32(v_az_adeh, (int32_t)(85u));
|
||||
v_intermediatecg = _mm256_shuffle_epi32(v_az_adeh, (int32_t)(170u));
|
||||
v_intermediatedh = _mm256_shuffle_epi32(v_az_adeh, (int32_t)(255u));
|
||||
break;
|
||||
}
|
||||
}
|
||||
v_rows01 = _mm256_lddqu_si256((const __m256i*)(const void*)(self->private_data.f_mcu_blocks[0u] + 0u));
|
||||
v_rows23 = _mm256_lddqu_si256((const __m256i*)(const void*)(self->private_data.f_mcu_blocks[0u] + 16u));
|
||||
v_rows45 = _mm256_lddqu_si256((const __m256i*)(const void*)(self->private_data.f_mcu_blocks[0u] + 32u));
|
||||
|
||||
@@ -16,6 +16,8 @@ pri func decoder.decode_idct!(dst_buffer: slice base.u8, dst_stride: base.u64, q
|
||||
choosy,
|
||||
{
|
||||
// This method implements the same algorithm as libjpeg-turbo's jidctint.c.
|
||||
// It defines CONST_BITS = 13 and PASS1_BITS = 2, so that right-shifting by
|
||||
// 11 is to shift by (CONST_BITS - PASS1_BITS).
|
||||
|
||||
var bq0 : base.u32
|
||||
var bq2 : base.u32
|
||||
|
||||
@@ -15,7 +15,10 @@
|
||||
pri func decoder.decode_idct_x86_avx2!(dst_buffer: slice base.u8, dst_stride: base.u64, q: base.u32[..= 3]),
|
||||
choose cpu_arch >= x86_avx2,
|
||||
{
|
||||
var util : base.x86_avx2_utility
|
||||
var sse42_util : base.x86_sse42_utility
|
||||
var util : base.x86_avx2_utility
|
||||
|
||||
// SIMD constants.
|
||||
|
||||
var k_0000 : base.x86_m256i
|
||||
var k_8080 : base.x86_m256i
|
||||
@@ -29,6 +32,16 @@ pri func decoder.decode_idct_x86_avx2!(dst_buffer: slice base.u8, dst_stride: ba
|
||||
var k_E6DC_25A1_1925_25A1 : base.x86_m256i
|
||||
var k_ECC1_E333_EFB0_ADFD : base.x86_m256i
|
||||
|
||||
// "Are the AC terms zero" variable names start with "az_".
|
||||
|
||||
var az_coeffs : base.x86_m128i
|
||||
var az_ah00 : base.x86_m256i
|
||||
var az_ad00 : base.x86_m256i
|
||||
var az_eh00 : base.x86_m256i
|
||||
var az_adeh : base.x86_m256i
|
||||
|
||||
// Set up variables.
|
||||
|
||||
var rows01 : base.x86_m256i
|
||||
var rows23 : base.x86_m256i
|
||||
var rows45 : base.x86_m256i
|
||||
@@ -261,14 +274,57 @@ pri func decoder.decode_idct_x86_avx2!(dst_buffer: slice base.u8, dst_stride: ba
|
||||
inv 8 <= args.dst_stride,
|
||||
{{
|
||||
|
||||
// TODO: faster code path (and break.goto_do_second_pass) if the AC terms
|
||||
// are zero.
|
||||
// Name the columns a ..= h and the rows 0 ..= 7.
|
||||
//
|
||||
// Check if the AC terms are zero. First, a relative cheap check of the
|
||||
// (a1, b1, c1, d1, a2, b2, c2, d2) coefficients - two half-rows.
|
||||
if 0 == (util.make_u64_slice_u16lex4(a: this.mcu_blocks[0][0x08 .. 0x0C]) |
|
||||
util.make_u64_slice_u16lex4(a: this.mcu_blocks[0][0x10 .. 0x14])) {
|
||||
|
||||
// A more thorough check of every coefficient in rows 1 ..= 7.
|
||||
az_coeffs = sse42_util.make_m128i_slice_u16lex8(a: this.mcu_blocks[0][0x08 .. 0x10])._mm_or_si128(b:
|
||||
sse42_util.make_m128i_slice_u16lex8(a: this.mcu_blocks[0][0x10 .. 0x18]))._mm_or_si128(b:
|
||||
sse42_util.make_m128i_slice_u16lex8(a: this.mcu_blocks[0][0x18 .. 0x20]))._mm_or_si128(b:
|
||||
sse42_util.make_m128i_slice_u16lex8(a: this.mcu_blocks[0][0x20 .. 0x28]))._mm_or_si128(b:
|
||||
sse42_util.make_m128i_slice_u16lex8(a: this.mcu_blocks[0][0x28 .. 0x30]))._mm_or_si128(b:
|
||||
sse42_util.make_m128i_slice_u16lex8(a: this.mcu_blocks[0][0x30 .. 0x38]))._mm_or_si128(b:
|
||||
sse42_util.make_m128i_slice_u16lex8(a: this.mcu_blocks[0][0x38 .. 0x40]))
|
||||
|
||||
// Pack u16x8 into u8x8 in the low 64 bits (using signed saturation)
|
||||
// and bulk-compare to zero. Only "zero versus non-zero" matters.
|
||||
if 0 == az_coeffs._mm_packs_epi16(b: az_coeffs).truncate_u64() {
|
||||
// Dequantize, similar to the "Set up" code path below.
|
||||
rows01 = util.make_m256i_slice_u16lex16(a: this.mcu_blocks[0][0x00 .. 0x10])
|
||||
quants01 = util.make_m256i_slice_u16lex16(a: this.quant_tables[args.q][0x00 .. 0x10])
|
||||
rows01 = rows01._mm256_mullo_epi16(b: quants01)
|
||||
|
||||
// az_ah00 = i16x16 [a b c d e f g h 0 0 0 0 0 0 0 0]
|
||||
// az_ad00 = i16x16 [a a b b c c d d 0 0 0 0 0 0 0 0]
|
||||
// az_eh00 = i16x16 [e e f f g g h h 0 0 0 0 0 0 0 0]
|
||||
// az_adeh = i16x16 [a a b b c c d d e e f f g g h h]
|
||||
az_ah00 = rows01._mm256_slli_epi16(imm8: 2) // PASS1_BITS is 2.
|
||||
az_ad00 = az_ah00._mm256_unpacklo_epi16(b: az_ah00)
|
||||
az_eh00 = az_ah00._mm256_unpackhi_epi16(b: az_ah00)
|
||||
az_adeh = az_ad00._mm256_inserti128_si256(b: az_eh00._mm256_castsi256_si128(), imm8: 1)
|
||||
|
||||
// intermediateae = i16x16 [a a a a a a a a e e e e e e e e]
|
||||
// intermediatebf = i16x16 [b b b b b b b b f f f f f f f f]
|
||||
// intermediatecg = i16x16 [c c c c c c c c g g g g g g g g]
|
||||
// intermediatedh = i16x16 [d d d d d d d d h h h h h h h h]
|
||||
intermediateae = az_adeh._mm256_shuffle_epi32(imm8: 0x00)
|
||||
intermediatebf = az_adeh._mm256_shuffle_epi32(imm8: 0x55)
|
||||
intermediatecg = az_adeh._mm256_shuffle_epi32(imm8: 0xAA)
|
||||
intermediatedh = az_adeh._mm256_shuffle_epi32(imm8: 0xFF)
|
||||
|
||||
break.goto_do_second_pass
|
||||
}
|
||||
}
|
||||
|
||||
// ----
|
||||
|
||||
// Set up.
|
||||
|
||||
// Dequantize. Name the columns a ..= h and the rows 0 ..= 7 so we get:
|
||||
// Dequantize:
|
||||
//
|
||||
// rows01 = i16x16 [a0 b0 .. g0 h0
|
||||
// a1 b1 .. g1 h1]
|
||||
|
||||
Reference in new issue
Block a user