From ecaf25ebf08401cf53a5c992fd6aad640f5a9fd0 Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Tue, 18 Jul 2023 11:02:56 +1000 Subject: [PATCH] std/jpeg: "AC all zero" fast path for x86_avx2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit name old speed new speed delta wuffs_jpeg_decode_19k_8bpp/clang11 370MB/s ± 0% 368MB/s ± 0% -0.47% (p=0.008 n=5+5) wuffs_jpeg_decode_30k_24bpp_progressive/clang11 231MB/s ± 0% 231MB/s ± 0% -0.26% (p=0.008 n=5+5) wuffs_jpeg_decode_30k_24bpp_sequential/clang11 470MB/s ± 0% 470MB/s ± 0% ~ (p=0.841 n=5+5) wuffs_jpeg_decode_77k_24bpp/clang11 418MB/s ± 0% 416MB/s ± 0% -0.42% (p=0.008 n=5+5) wuffs_jpeg_decode_552k_24bpp_420/clang11 435MB/s ± 0% 433MB/s ± 0% -0.33% (p=0.008 n=5+5) wuffs_jpeg_decode_552k_24bpp_444/clang11 620MB/s ± 0% 646MB/s ± 0% +4.20% (p=0.008 n=5+5) wuffs_jpeg_decode_4002k_24bpp/clang11 441MB/s ± 0% 439MB/s ± 0% -0.51% (p=0.008 n=5+5) wuffs_jpeg_decode_19k_8bpp/gcc10 352MB/s ± 0% 364MB/s ± 0% +3.41% (p=0.008 n=5+5) wuffs_jpeg_decode_30k_24bpp_progressive/gcc10 296MB/s ± 0% 294MB/s ± 0% -0.69% (p=0.008 n=5+5) wuffs_jpeg_decode_30k_24bpp_sequential/gcc10 666MB/s ± 0% 680MB/s ± 0% +2.10% (p=0.008 n=5+5) wuffs_jpeg_decode_77k_24bpp/gcc10 565MB/s ± 0% 578MB/s ± 0% +2.34% (p=0.008 n=5+5) wuffs_jpeg_decode_552k_24bpp_420/gcc10 608MB/s ± 0% 622MB/s ± 0% +2.26% (p=0.016 n=5+4) wuffs_jpeg_decode_552k_24bpp_444/gcc10 624MB/s ± 0% 659MB/s ± 0% +5.47% (p=0.008 n=5+5) wuffs_jpeg_decode_4002k_24bpp/gcc10 616MB/s ± 0% 630MB/s ± 0% +2.29% (p=0.008 n=5+5) --- internal/cgen/builtin.go | 5 +- lang/builtin/builtin.go | 10 ++++ lang/check/bounds.go | 6 +++ release/c/wuffs-unsupported-snapshot.c | 22 +++++++++ std/jpeg/decode_idct_default.wuffs | 2 + std/jpeg/decode_idct_x86_avx2.wuffs | 64 ++++++++++++++++++++++++-- 6 files changed, 104 insertions(+), 5 deletions(-) diff --git a/internal/cgen/builtin.go b/internal/cgen/builtin.go index f7a3fdf4..c7601094 100644 --- a/internal/cgen/builtin.go +++ b/internal/cgen/builtin.go @@ -611,6 +611,9 @@ func (g *gen) writeBuiltinCPUArchX86(b *buffer, recv *a.Expr, method t.ID, args if strings.HasPrefix(methodStr, "make_") { fName, tName, ptr := "", "", false switch methodStr { + case "make_u64_slice_u16lex4": + fName, tName, ptr = "wuffs_base__peek_u64le__no_bounds_check", "const uint8_t*)(const void*", true + case "make_m128i_multiple_u8": fName, tName = "_mm_set_epi8", "int8_t" case "make_m128i_multiple_u16": @@ -631,7 +634,7 @@ func (g *gen) writeBuiltinCPUArchX86(b *buffer, recv *a.Expr, method t.ID, args fName, tName = "_mm_cvtsi32_si128", "int32_t" case "make_m128i_single_u64": fName, tName = "_mm_cvtsi64x_si128", "int64_t" - case "make_m128i_slice128": + case "make_m128i_slice128", "make_m128i_slice_u16lex8": fName, tName, ptr = "_mm_lddqu_si128", "const __m128i*)(const void*", true case "make_m128i_zeroes": fName, tName = "_mm_setzero_si128", "" diff --git a/lang/builtin/builtin.go b/lang/builtin/builtin.go index 1d537722..cdfcd27b 100644 --- a/lang/builtin/builtin.go +++ b/lang/builtin/builtin.go @@ -770,9 +770,12 @@ var funcsOther = [...]string{ "x86_sse42_utility.make_m128i_single_u64(a: u64) x86_m128i", "x86_sse42_utility.make_m128i_slice128(a: roslice base.u8) x86_m128i", + "x86_sse42_utility.make_m128i_slice_u16lex8(a: roslice base.u16) x86_m128i", "x86_sse42_utility.make_m128i_zeroes() x86_m128i", + "x86_sse42_utility.make_u64_slice_u16lex4(a: roslice base.u16) u64", + // ---- x86_m128i "x86_m128i.store_slice64!(a: slice base.u8)", @@ -816,6 +819,8 @@ var funcsOther = [...]string{ "x86_m128i._mm_min_epu16(b: x86_m128i) x86_m128i", "x86_m128i._mm_min_epu32(b: x86_m128i) x86_m128i", "x86_m128i._mm_min_epu8(b: x86_m128i) x86_m128i", + "x86_m128i._mm_or_si128(b: x86_m128i) x86_m128i", + "x86_m128i._mm_packs_epi16(b: x86_m128i) x86_m128i", "x86_m128i._mm_packus_epi16(b: x86_m128i) x86_m128i", "x86_m128i._mm_sad_epu8(b: x86_m128i) x86_m128i", "x86_m128i._mm_shuffle_epi32(imm8: u32) x86_m128i", @@ -874,6 +879,8 @@ var funcsOther = [...]string{ "x86_avx2_utility.make_m256i_zeroes() x86_m256i", + "x86_avx2_utility.make_u64_slice_u16lex4(a: roslice base.u16) u64", + // ---- x86_m256i "x86_m256i.store_slice64!(a: slice base.u8)", @@ -886,8 +893,10 @@ var funcsOther = [...]string{ "x86_m256i._mm256_add_epi32(b: x86_m256i) x86_m256i", "x86_m256i._mm256_add_epi64(b: x86_m256i) x86_m256i", "x86_m256i._mm256_add_epi8(b: x86_m256i) x86_m256i", + "x86_m256i._mm256_castsi256_si128() x86_m128i", "x86_m256i._mm256_extract_epi64(index: u32) u64", "x86_m256i._mm256_extracti128_si256(imm8: u32) x86_m128i", + "x86_m256i._mm256_inserti128_si256(b: x86_m128i, imm8: u32) x86_m256i", "x86_m256i._mm256_madd_epi16(b: x86_m256i) x86_m256i", "x86_m256i._mm256_maddubs_epi16(b: x86_m256i) x86_m256i", "x86_m256i._mm256_mullo_epi16(b: x86_m256i) x86_m256i", @@ -896,6 +905,7 @@ var funcsOther = [...]string{ "x86_m256i._mm256_permute2x128_si256(b: x86_m256i, imm8: u32) x86_m256i", "x86_m256i._mm256_permute4x64_epi64(imm8: u32) x86_m256i", "x86_m256i._mm256_sad_epu8(b: x86_m256i) x86_m256i", + "x86_m256i._mm256_shuffle_epi32(imm8: u32) x86_m256i", "x86_m256i._mm256_sign_epi16(b: x86_m256i) x86_m256i", "x86_m256i._mm256_slli_epi16(imm8: u32) x86_m256i", "x86_m256i._mm256_slli_epi32(imm8: u32) x86_m256i", diff --git a/lang/check/bounds.go b/lang/check/bounds.go index cab64fdd..8547b852 100644 --- a/lang/check/bounds.go +++ b/lang/check/bounds.go @@ -1282,8 +1282,14 @@ func (q *checker) bcheckExprCallSpecialCases(n *a.Expr, depth uint32) (bounds, e advance = thirtyTwo case strings.HasSuffix(s, "_slice512"): // 512 bits is 64 bytes. advance = sixtyFour + case strings.HasSuffix(s, "_slice_u16lex4"): // 4 u16 values is 8 bytes. + advance = four + case strings.HasSuffix(s, "_slice_u16lex8"): // 8 u16 values is 16 bytes. + advance = eight case strings.HasSuffix(s, "_slice_u16lex16"): // 16 u16 values is 32 bytes. advance = sixteen + case strings.HasSuffix(s, "_slice_u16lex32"): // 32 u16 values is 64 bytes. + advance = thirtyTwo case strings.Contains(s, "_slice"): return bounds{}, fmt.Errorf("check: internal error: unrecognized %s method", s) } diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 3f77881c..ed5da687 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -40063,6 +40063,11 @@ wuffs_jpeg__decoder__decode_idct_x86_avx2( __m256i v_k_E333_133E_ADFD_1051 = {0}; __m256i v_k_E6DC_25A1_1925_25A1 = {0}; __m256i v_k_ECC1_E333_EFB0_ADFD = {0}; + __m128i v_az_coeffs = {0}; + __m256i v_az_ah00 = {0}; + __m256i v_az_ad00 = {0}; + __m256i v_az_eh00 = {0}; + __m256i v_az_adeh = {0}; __m256i v_rows01 = {0}; __m256i v_rows23 = {0}; __m256i v_rows45 = {0}; @@ -40216,6 +40221,23 @@ wuffs_jpeg__decoder__decode_idct_x86_avx2( v_k_E6DC_25A1_1925_25A1 = _mm256_set_epi16((int16_t)(9633u), (int16_t)(6437u), (int16_t)(9633u), (int16_t)(6437u), (int16_t)(9633u), (int16_t)(6437u), (int16_t)(9633u), (int16_t)(6437u), (int16_t)(9633u), (int16_t)(59100u), (int16_t)(9633u), (int16_t)(59100u), (int16_t)(9633u), (int16_t)(59100u), (int16_t)(9633u), (int16_t)(59100u)); v_k_ECC1_E333_EFB0_ADFD = _mm256_set_epi16((int16_t)(44541u), (int16_t)(61360u), (int16_t)(44541u), (int16_t)(61360u), (int16_t)(44541u), (int16_t)(61360u), (int16_t)(44541u), (int16_t)(61360u), (int16_t)(58163u), (int16_t)(60609u), (int16_t)(58163u), (int16_t)(60609u), (int16_t)(58163u), (int16_t)(60609u), (int16_t)(58163u), (int16_t)(60609u)); do { + if (0u == (wuffs_base__peek_u64le__no_bounds_check((const uint8_t*)(const void*)(self->private_data.f_mcu_blocks[0u] + 8u)) | wuffs_base__peek_u64le__no_bounds_check((const uint8_t*)(const void*)(self->private_data.f_mcu_blocks[0u] + 16u)))) { + v_az_coeffs = _mm_or_si128(_mm_or_si128(_mm_or_si128(_mm_or_si128(_mm_or_si128(_mm_or_si128(_mm_lddqu_si128((const __m128i*)(const void*)(self->private_data.f_mcu_blocks[0u] + 8u)), _mm_lddqu_si128((const __m128i*)(const void*)(self->private_data.f_mcu_blocks[0u] + 16u))), _mm_lddqu_si128((const __m128i*)(const void*)(self->private_data.f_mcu_blocks[0u] + 24u))), _mm_lddqu_si128((const __m128i*)(const void*)(self->private_data.f_mcu_blocks[0u] + 32u))), _mm_lddqu_si128((const __m128i*)(const void*)(self->private_data.f_mcu_blocks[0u] + 40u))), _mm_lddqu_si128((const __m128i*)(const void*)(self->private_data.f_mcu_blocks[0u] + 48u))), _mm_lddqu_si128((const __m128i*)(const void*)(self->private_data.f_mcu_blocks[0u] + 56u))); + if (0u == ((uint64_t)(_mm_cvtsi128_si64(_mm_packs_epi16(v_az_coeffs, v_az_coeffs))))) { + v_rows01 = _mm256_lddqu_si256((const __m256i*)(const void*)(self->private_data.f_mcu_blocks[0u] + 0u)); + v_quants01 = _mm256_lddqu_si256((const __m256i*)(const void*)(self->private_impl.f_quant_tables[a_q] + 0u)); + v_rows01 = _mm256_mullo_epi16(v_rows01, v_quants01); + v_az_ah00 = _mm256_slli_epi16(v_rows01, (int32_t)(2u)); + v_az_ad00 = _mm256_unpacklo_epi16(v_az_ah00, v_az_ah00); + v_az_eh00 = _mm256_unpackhi_epi16(v_az_ah00, v_az_ah00); + v_az_adeh = _mm256_inserti128_si256(v_az_ad00, _mm256_castsi256_si128(v_az_eh00), (int32_t)(1u)); + v_intermediateae = _mm256_shuffle_epi32(v_az_adeh, (int32_t)(0u)); + v_intermediatebf = _mm256_shuffle_epi32(v_az_adeh, (int32_t)(85u)); + v_intermediatecg = _mm256_shuffle_epi32(v_az_adeh, (int32_t)(170u)); + v_intermediatedh = _mm256_shuffle_epi32(v_az_adeh, (int32_t)(255u)); + break; + } + } v_rows01 = _mm256_lddqu_si256((const __m256i*)(const void*)(self->private_data.f_mcu_blocks[0u] + 0u)); v_rows23 = _mm256_lddqu_si256((const __m256i*)(const void*)(self->private_data.f_mcu_blocks[0u] + 16u)); v_rows45 = _mm256_lddqu_si256((const __m256i*)(const void*)(self->private_data.f_mcu_blocks[0u] + 32u)); diff --git a/std/jpeg/decode_idct_default.wuffs b/std/jpeg/decode_idct_default.wuffs index 39a35667..459ccb3a 100644 --- a/std/jpeg/decode_idct_default.wuffs +++ b/std/jpeg/decode_idct_default.wuffs @@ -16,6 +16,8 @@ pri func decoder.decode_idct!(dst_buffer: slice base.u8, dst_stride: base.u64, q choosy, { // This method implements the same algorithm as libjpeg-turbo's jidctint.c. + // It defines CONST_BITS = 13 and PASS1_BITS = 2, so that right-shifting by + // 11 is to shift by (CONST_BITS - PASS1_BITS). var bq0 : base.u32 var bq2 : base.u32 diff --git a/std/jpeg/decode_idct_x86_avx2.wuffs b/std/jpeg/decode_idct_x86_avx2.wuffs index 262e096f..c8ddb39b 100644 --- a/std/jpeg/decode_idct_x86_avx2.wuffs +++ b/std/jpeg/decode_idct_x86_avx2.wuffs @@ -15,7 +15,10 @@ pri func decoder.decode_idct_x86_avx2!(dst_buffer: slice base.u8, dst_stride: base.u64, q: base.u32[..= 3]), choose cpu_arch >= x86_avx2, { - var util : base.x86_avx2_utility + var sse42_util : base.x86_sse42_utility + var util : base.x86_avx2_utility + + // SIMD constants. var k_0000 : base.x86_m256i var k_8080 : base.x86_m256i @@ -29,6 +32,16 @@ pri func decoder.decode_idct_x86_avx2!(dst_buffer: slice base.u8, dst_stride: ba var k_E6DC_25A1_1925_25A1 : base.x86_m256i var k_ECC1_E333_EFB0_ADFD : base.x86_m256i + // "Are the AC terms zero" variable names start with "az_". + + var az_coeffs : base.x86_m128i + var az_ah00 : base.x86_m256i + var az_ad00 : base.x86_m256i + var az_eh00 : base.x86_m256i + var az_adeh : base.x86_m256i + + // Set up variables. + var rows01 : base.x86_m256i var rows23 : base.x86_m256i var rows45 : base.x86_m256i @@ -261,14 +274,57 @@ pri func decoder.decode_idct_x86_avx2!(dst_buffer: slice base.u8, dst_stride: ba inv 8 <= args.dst_stride, {{ - // TODO: faster code path (and break.goto_do_second_pass) if the AC terms - // are zero. + // Name the columns a ..= h and the rows 0 ..= 7. + // + // Check if the AC terms are zero. First, a relative cheap check of the + // (a1, b1, c1, d1, a2, b2, c2, d2) coefficients - two half-rows. + if 0 == (util.make_u64_slice_u16lex4(a: this.mcu_blocks[0][0x08 .. 0x0C]) | + util.make_u64_slice_u16lex4(a: this.mcu_blocks[0][0x10 .. 0x14])) { + + // A more thorough check of every coefficient in rows 1 ..= 7. + az_coeffs = sse42_util.make_m128i_slice_u16lex8(a: this.mcu_blocks[0][0x08 .. 0x10])._mm_or_si128(b: + sse42_util.make_m128i_slice_u16lex8(a: this.mcu_blocks[0][0x10 .. 0x18]))._mm_or_si128(b: + sse42_util.make_m128i_slice_u16lex8(a: this.mcu_blocks[0][0x18 .. 0x20]))._mm_or_si128(b: + sse42_util.make_m128i_slice_u16lex8(a: this.mcu_blocks[0][0x20 .. 0x28]))._mm_or_si128(b: + sse42_util.make_m128i_slice_u16lex8(a: this.mcu_blocks[0][0x28 .. 0x30]))._mm_or_si128(b: + sse42_util.make_m128i_slice_u16lex8(a: this.mcu_blocks[0][0x30 .. 0x38]))._mm_or_si128(b: + sse42_util.make_m128i_slice_u16lex8(a: this.mcu_blocks[0][0x38 .. 0x40])) + + // Pack u16x8 into u8x8 in the low 64 bits (using signed saturation) + // and bulk-compare to zero. Only "zero versus non-zero" matters. + if 0 == az_coeffs._mm_packs_epi16(b: az_coeffs).truncate_u64() { + // Dequantize, similar to the "Set up" code path below. + rows01 = util.make_m256i_slice_u16lex16(a: this.mcu_blocks[0][0x00 .. 0x10]) + quants01 = util.make_m256i_slice_u16lex16(a: this.quant_tables[args.q][0x00 .. 0x10]) + rows01 = rows01._mm256_mullo_epi16(b: quants01) + + // az_ah00 = i16x16 [a b c d e f g h 0 0 0 0 0 0 0 0] + // az_ad00 = i16x16 [a a b b c c d d 0 0 0 0 0 0 0 0] + // az_eh00 = i16x16 [e e f f g g h h 0 0 0 0 0 0 0 0] + // az_adeh = i16x16 [a a b b c c d d e e f f g g h h] + az_ah00 = rows01._mm256_slli_epi16(imm8: 2) // PASS1_BITS is 2. + az_ad00 = az_ah00._mm256_unpacklo_epi16(b: az_ah00) + az_eh00 = az_ah00._mm256_unpackhi_epi16(b: az_ah00) + az_adeh = az_ad00._mm256_inserti128_si256(b: az_eh00._mm256_castsi256_si128(), imm8: 1) + + // intermediateae = i16x16 [a a a a a a a a e e e e e e e e] + // intermediatebf = i16x16 [b b b b b b b b f f f f f f f f] + // intermediatecg = i16x16 [c c c c c c c c g g g g g g g g] + // intermediatedh = i16x16 [d d d d d d d d h h h h h h h h] + intermediateae = az_adeh._mm256_shuffle_epi32(imm8: 0x00) + intermediatebf = az_adeh._mm256_shuffle_epi32(imm8: 0x55) + intermediatecg = az_adeh._mm256_shuffle_epi32(imm8: 0xAA) + intermediatedh = az_adeh._mm256_shuffle_epi32(imm8: 0xFF) + + break.goto_do_second_pass + } + } // ---- // Set up. - // Dequantize. Name the columns a ..= h and the rows 0 ..= 7 so we get: + // Dequantize: // // rows01 = i16x16 [a0 b0 .. g0 h0 // a1 b1 .. g1 h1]