std/jpeg: "AC all zero" fast path for x86_avx2

name                                             old speed     new speed     delta

wuffs_jpeg_decode_19k_8bpp/clang11               370MB/s ± 0%  368MB/s ± 0%  -0.47%  (p=0.008 n=5+5)
wuffs_jpeg_decode_30k_24bpp_progressive/clang11  231MB/s ± 0%  231MB/s ± 0%  -0.26%  (p=0.008 n=5+5)
wuffs_jpeg_decode_30k_24bpp_sequential/clang11   470MB/s ± 0%  470MB/s ± 0%    ~     (p=0.841 n=5+5)
wuffs_jpeg_decode_77k_24bpp/clang11              418MB/s ± 0%  416MB/s ± 0%  -0.42%  (p=0.008 n=5+5)
wuffs_jpeg_decode_552k_24bpp_420/clang11         435MB/s ± 0%  433MB/s ± 0%  -0.33%  (p=0.008 n=5+5)
wuffs_jpeg_decode_552k_24bpp_444/clang11         620MB/s ± 0%  646MB/s ± 0%  +4.20%  (p=0.008 n=5+5)
wuffs_jpeg_decode_4002k_24bpp/clang11            441MB/s ± 0%  439MB/s ± 0%  -0.51%  (p=0.008 n=5+5)

wuffs_jpeg_decode_19k_8bpp/gcc10                 352MB/s ± 0%  364MB/s ± 0%  +3.41%  (p=0.008 n=5+5)
wuffs_jpeg_decode_30k_24bpp_progressive/gcc10    296MB/s ± 0%  294MB/s ± 0%  -0.69%  (p=0.008 n=5+5)
wuffs_jpeg_decode_30k_24bpp_sequential/gcc10     666MB/s ± 0%  680MB/s ± 0%  +2.10%  (p=0.008 n=5+5)
wuffs_jpeg_decode_77k_24bpp/gcc10                565MB/s ± 0%  578MB/s ± 0%  +2.34%  (p=0.008 n=5+5)
wuffs_jpeg_decode_552k_24bpp_420/gcc10           608MB/s ± 0%  622MB/s ± 0%  +2.26%  (p=0.016 n=5+4)
wuffs_jpeg_decode_552k_24bpp_444/gcc10           624MB/s ± 0%  659MB/s ± 0%  +5.47%  (p=0.008 n=5+5)
wuffs_jpeg_decode_4002k_24bpp/gcc10              616MB/s ± 0%  630MB/s ± 0%  +2.29%  (p=0.008 n=5+5)
This commit is contained in:
Nigel Tao committed 2023-07-18 12:04:19 +10:00
1 parent 2bff962cf1
commit ecaf25ebf0
6 files changed
+104 -5

No files matched your search

+4 -1
View File
@@ -611,6 +611,9 @@ func (g *gen) writeBuiltinCPUArchX86(b *buffer, recv *a.Expr, method t.ID, args
if strings.HasPrefix(methodStr, "make_") {
fName, tName, ptr := "", "", false
switch methodStr {
case "make_u64_slice_u16lex4":
fName, tName, ptr = "wuffs_base__peek_u64le__no_bounds_check", "const uint8_t*)(const void*", true
case "make_m128i_multiple_u8":
fName, tName = "_mm_set_epi8", "int8_t"
case "make_m128i_multiple_u16":
@@ -631,7 +634,7 @@ func (g *gen) writeBuiltinCPUArchX86(b *buffer, recv *a.Expr, method t.ID, args
fName, tName = "_mm_cvtsi32_si128", "int32_t"
case "make_m128i_single_u64":
fName, tName = "_mm_cvtsi64x_si128", "int64_t"
case "make_m128i_slice128":
case "make_m128i_slice128", "make_m128i_slice_u16lex8":
fName, tName, ptr = "_mm_lddqu_si128", "const __m128i*)(const void*", true
case "make_m128i_zeroes":
fName, tName = "_mm_setzero_si128", ""
+10
View File
@@ -770,9 +770,12 @@ var funcsOther = [...]string{
"x86_sse42_utility.make_m128i_single_u64(a: u64) x86_m128i",
"x86_sse42_utility.make_m128i_slice128(a: roslice base.u8) x86_m128i",
"x86_sse42_utility.make_m128i_slice_u16lex8(a: roslice base.u16) x86_m128i",
"x86_sse42_utility.make_m128i_zeroes() x86_m128i",
"x86_sse42_utility.make_u64_slice_u16lex4(a: roslice base.u16) u64",
// ---- x86_m128i
"x86_m128i.store_slice64!(a: slice base.u8)",
@@ -816,6 +819,8 @@ var funcsOther = [...]string{
"x86_m128i._mm_min_epu16(b: x86_m128i) x86_m128i",
"x86_m128i._mm_min_epu32(b: x86_m128i) x86_m128i",
"x86_m128i._mm_min_epu8(b: x86_m128i) x86_m128i",
"x86_m128i._mm_or_si128(b: x86_m128i) x86_m128i",
"x86_m128i._mm_packs_epi16(b: x86_m128i) x86_m128i",
"x86_m128i._mm_packus_epi16(b: x86_m128i) x86_m128i",
"x86_m128i._mm_sad_epu8(b: x86_m128i) x86_m128i",
"x86_m128i._mm_shuffle_epi32(imm8: u32) x86_m128i",
@@ -874,6 +879,8 @@ var funcsOther = [...]string{
"x86_avx2_utility.make_m256i_zeroes() x86_m256i",
"x86_avx2_utility.make_u64_slice_u16lex4(a: roslice base.u16) u64",
// ---- x86_m256i
"x86_m256i.store_slice64!(a: slice base.u8)",
@@ -886,8 +893,10 @@ var funcsOther = [...]string{
"x86_m256i._mm256_add_epi32(b: x86_m256i) x86_m256i",
"x86_m256i._mm256_add_epi64(b: x86_m256i) x86_m256i",
"x86_m256i._mm256_add_epi8(b: x86_m256i) x86_m256i",
"x86_m256i._mm256_castsi256_si128() x86_m128i",
"x86_m256i._mm256_extract_epi64(index: u32) u64",
"x86_m256i._mm256_extracti128_si256(imm8: u32) x86_m128i",
"x86_m256i._mm256_inserti128_si256(b: x86_m128i, imm8: u32) x86_m256i",
"x86_m256i._mm256_madd_epi16(b: x86_m256i) x86_m256i",
"x86_m256i._mm256_maddubs_epi16(b: x86_m256i) x86_m256i",
"x86_m256i._mm256_mullo_epi16(b: x86_m256i) x86_m256i",
@@ -896,6 +905,7 @@ var funcsOther = [...]string{
"x86_m256i._mm256_permute2x128_si256(b: x86_m256i, imm8: u32) x86_m256i",
"x86_m256i._mm256_permute4x64_epi64(imm8: u32) x86_m256i",
"x86_m256i._mm256_sad_epu8(b: x86_m256i) x86_m256i",
"x86_m256i._mm256_shuffle_epi32(imm8: u32) x86_m256i",
"x86_m256i._mm256_sign_epi16(b: x86_m256i) x86_m256i",
"x86_m256i._mm256_slli_epi16(imm8: u32) x86_m256i",
"x86_m256i._mm256_slli_epi32(imm8: u32) x86_m256i",
+6
View File
@@ -1282,8 +1282,14 @@ func (q *checker) bcheckExprCallSpecialCases(n *a.Expr, depth uint32) (bounds, e
advance = thirtyTwo
case strings.HasSuffix(s, "_slice512"): // 512 bits is 64 bytes.
advance = sixtyFour
case strings.HasSuffix(s, "_slice_u16lex4"): // 4 u16 values is 8 bytes.
advance = four
case strings.HasSuffix(s, "_slice_u16lex8"): // 8 u16 values is 16 bytes.
advance = eight
case strings.HasSuffix(s, "_slice_u16lex16"): // 16 u16 values is 32 bytes.
advance = sixteen
case strings.HasSuffix(s, "_slice_u16lex32"): // 32 u16 values is 64 bytes.
advance = thirtyTwo
case strings.Contains(s, "_slice"):
return bounds{}, fmt.Errorf("check: internal error: unrecognized %s method", s)
}
+22
View File
@@ -40063,6 +40063,11 @@ wuffs_jpeg__decoder__decode_idct_x86_avx2(
__m256i v_k_E333_133E_ADFD_1051 = {0};
__m256i v_k_E6DC_25A1_1925_25A1 = {0};
__m256i v_k_ECC1_E333_EFB0_ADFD = {0};
__m128i v_az_coeffs = {0};
__m256i v_az_ah00 = {0};
__m256i v_az_ad00 = {0};
__m256i v_az_eh00 = {0};
__m256i v_az_adeh = {0};
__m256i v_rows01 = {0};
__m256i v_rows23 = {0};
__m256i v_rows45 = {0};
@@ -40216,6 +40221,23 @@ wuffs_jpeg__decoder__decode_idct_x86_avx2(
v_k_E6DC_25A1_1925_25A1 = _mm256_set_epi16((int16_t)(9633u), (int16_t)(6437u), (int16_t)(9633u), (int16_t)(6437u), (int16_t)(9633u), (int16_t)(6437u), (int16_t)(9633u), (int16_t)(6437u), (int16_t)(9633u), (int16_t)(59100u), (int16_t)(9633u), (int16_t)(59100u), (int16_t)(9633u), (int16_t)(59100u), (int16_t)(9633u), (int16_t)(59100u));
v_k_ECC1_E333_EFB0_ADFD = _mm256_set_epi16((int16_t)(44541u), (int16_t)(61360u), (int16_t)(44541u), (int16_t)(61360u), (int16_t)(44541u), (int16_t)(61360u), (int16_t)(44541u), (int16_t)(61360u), (int16_t)(58163u), (int16_t)(60609u), (int16_t)(58163u), (int16_t)(60609u), (int16_t)(58163u), (int16_t)(60609u), (int16_t)(58163u), (int16_t)(60609u));
do {
if (0u == (wuffs_base__peek_u64le__no_bounds_check((const uint8_t*)(const void*)(self->private_data.f_mcu_blocks[0u] + 8u)) | wuffs_base__peek_u64le__no_bounds_check((const uint8_t*)(const void*)(self->private_data.f_mcu_blocks[0u] + 16u)))) {
v_az_coeffs = _mm_or_si128(_mm_or_si128(_mm_or_si128(_mm_or_si128(_mm_or_si128(_mm_or_si128(_mm_lddqu_si128((const __m128i*)(const void*)(self->private_data.f_mcu_blocks[0u] + 8u)), _mm_lddqu_si128((const __m128i*)(const void*)(self->private_data.f_mcu_blocks[0u] + 16u))), _mm_lddqu_si128((const __m128i*)(const void*)(self->private_data.f_mcu_blocks[0u] + 24u))), _mm_lddqu_si128((const __m128i*)(const void*)(self->private_data.f_mcu_blocks[0u] + 32u))), _mm_lddqu_si128((const __m128i*)(const void*)(self->private_data.f_mcu_blocks[0u] + 40u))), _mm_lddqu_si128((const __m128i*)(const void*)(self->private_data.f_mcu_blocks[0u] + 48u))), _mm_lddqu_si128((const __m128i*)(const void*)(self->private_data.f_mcu_blocks[0u] + 56u)));
if (0u == ((uint64_t)(_mm_cvtsi128_si64(_mm_packs_epi16(v_az_coeffs, v_az_coeffs))))) {
v_rows01 = _mm256_lddqu_si256((const __m256i*)(const void*)(self->private_data.f_mcu_blocks[0u] + 0u));
v_quants01 = _mm256_lddqu_si256((const __m256i*)(const void*)(self->private_impl.f_quant_tables[a_q] + 0u));
v_rows01 = _mm256_mullo_epi16(v_rows01, v_quants01);
v_az_ah00 = _mm256_slli_epi16(v_rows01, (int32_t)(2u));
v_az_ad00 = _mm256_unpacklo_epi16(v_az_ah00, v_az_ah00);
v_az_eh00 = _mm256_unpackhi_epi16(v_az_ah00, v_az_ah00);
v_az_adeh = _mm256_inserti128_si256(v_az_ad00, _mm256_castsi256_si128(v_az_eh00), (int32_t)(1u));
v_intermediateae = _mm256_shuffle_epi32(v_az_adeh, (int32_t)(0u));
v_intermediatebf = _mm256_shuffle_epi32(v_az_adeh, (int32_t)(85u));
v_intermediatecg = _mm256_shuffle_epi32(v_az_adeh, (int32_t)(170u));
v_intermediatedh = _mm256_shuffle_epi32(v_az_adeh, (int32_t)(255u));
break;
}
}
v_rows01 = _mm256_lddqu_si256((const __m256i*)(const void*)(self->private_data.f_mcu_blocks[0u] + 0u));
v_rows23 = _mm256_lddqu_si256((const __m256i*)(const void*)(self->private_data.f_mcu_blocks[0u] + 16u));
v_rows45 = _mm256_lddqu_si256((const __m256i*)(const void*)(self->private_data.f_mcu_blocks[0u] + 32u));
+2
View File
@@ -16,6 +16,8 @@ pri func decoder.decode_idct!(dst_buffer: slice base.u8, dst_stride: base.u64, q
choosy,
{
// This method implements the same algorithm as libjpeg-turbo's jidctint.c.
// It defines CONST_BITS = 13 and PASS1_BITS = 2, so that right-shifting by
// 11 is to shift by (CONST_BITS - PASS1_BITS).
var bq0 : base.u32
var bq2 : base.u32
+60 -4
View File
@@ -15,7 +15,10 @@
pri func decoder.decode_idct_x86_avx2!(dst_buffer: slice base.u8, dst_stride: base.u64, q: base.u32[..= 3]),
choose cpu_arch >= x86_avx2,
{
var util : base.x86_avx2_utility
var sse42_util : base.x86_sse42_utility
var util : base.x86_avx2_utility
// SIMD constants.
var k_0000 : base.x86_m256i
var k_8080 : base.x86_m256i
@@ -29,6 +32,16 @@ pri func decoder.decode_idct_x86_avx2!(dst_buffer: slice base.u8, dst_stride: ba
var k_E6DC_25A1_1925_25A1 : base.x86_m256i
var k_ECC1_E333_EFB0_ADFD : base.x86_m256i
// "Are the AC terms zero" variable names start with "az_".
var az_coeffs : base.x86_m128i
var az_ah00 : base.x86_m256i
var az_ad00 : base.x86_m256i
var az_eh00 : base.x86_m256i
var az_adeh : base.x86_m256i
// Set up variables.
var rows01 : base.x86_m256i
var rows23 : base.x86_m256i
var rows45 : base.x86_m256i
@@ -261,14 +274,57 @@ pri func decoder.decode_idct_x86_avx2!(dst_buffer: slice base.u8, dst_stride: ba
inv 8 <= args.dst_stride,
{{
// TODO: faster code path (and break.goto_do_second_pass) if the AC terms
// are zero.
// Name the columns a ..= h and the rows 0 ..= 7.
//
// Check if the AC terms are zero. First, a relative cheap check of the
// (a1, b1, c1, d1, a2, b2, c2, d2) coefficients - two half-rows.
if 0 == (util.make_u64_slice_u16lex4(a: this.mcu_blocks[0][0x08 .. 0x0C]) |
util.make_u64_slice_u16lex4(a: this.mcu_blocks[0][0x10 .. 0x14])) {
// A more thorough check of every coefficient in rows 1 ..= 7.
az_coeffs = sse42_util.make_m128i_slice_u16lex8(a: this.mcu_blocks[0][0x08 .. 0x10])._mm_or_si128(b:
sse42_util.make_m128i_slice_u16lex8(a: this.mcu_blocks[0][0x10 .. 0x18]))._mm_or_si128(b:
sse42_util.make_m128i_slice_u16lex8(a: this.mcu_blocks[0][0x18 .. 0x20]))._mm_or_si128(b:
sse42_util.make_m128i_slice_u16lex8(a: this.mcu_blocks[0][0x20 .. 0x28]))._mm_or_si128(b:
sse42_util.make_m128i_slice_u16lex8(a: this.mcu_blocks[0][0x28 .. 0x30]))._mm_or_si128(b:
sse42_util.make_m128i_slice_u16lex8(a: this.mcu_blocks[0][0x30 .. 0x38]))._mm_or_si128(b:
sse42_util.make_m128i_slice_u16lex8(a: this.mcu_blocks[0][0x38 .. 0x40]))
// Pack u16x8 into u8x8 in the low 64 bits (using signed saturation)
// and bulk-compare to zero. Only "zero versus non-zero" matters.
if 0 == az_coeffs._mm_packs_epi16(b: az_coeffs).truncate_u64() {
// Dequantize, similar to the "Set up" code path below.
rows01 = util.make_m256i_slice_u16lex16(a: this.mcu_blocks[0][0x00 .. 0x10])
quants01 = util.make_m256i_slice_u16lex16(a: this.quant_tables[args.q][0x00 .. 0x10])
rows01 = rows01._mm256_mullo_epi16(b: quants01)
// az_ah00 = i16x16 [a b c d e f g h 0 0 0 0 0 0 0 0]
// az_ad00 = i16x16 [a a b b c c d d 0 0 0 0 0 0 0 0]
// az_eh00 = i16x16 [e e f f g g h h 0 0 0 0 0 0 0 0]
// az_adeh = i16x16 [a a b b c c d d e e f f g g h h]
az_ah00 = rows01._mm256_slli_epi16(imm8: 2) // PASS1_BITS is 2.
az_ad00 = az_ah00._mm256_unpacklo_epi16(b: az_ah00)
az_eh00 = az_ah00._mm256_unpackhi_epi16(b: az_ah00)
az_adeh = az_ad00._mm256_inserti128_si256(b: az_eh00._mm256_castsi256_si128(), imm8: 1)
// intermediateae = i16x16 [a a a a a a a a e e e e e e e e]
// intermediatebf = i16x16 [b b b b b b b b f f f f f f f f]
// intermediatecg = i16x16 [c c c c c c c c g g g g g g g g]
// intermediatedh = i16x16 [d d d d d d d d h h h h h h h h]
intermediateae = az_adeh._mm256_shuffle_epi32(imm8: 0x00)
intermediatebf = az_adeh._mm256_shuffle_epi32(imm8: 0x55)
intermediatecg = az_adeh._mm256_shuffle_epi32(imm8: 0xAA)
intermediatedh = az_adeh._mm256_shuffle_epi32(imm8: 0xFF)
break.goto_do_second_pass
}
}
// ----
// Set up.
// Dequantize. Name the columns a ..= h and the rows 0 ..= 7 so we get:
// Dequantize:
//
// rows01 = i16x16 [a0 b0 .. g0 h0
// a1 b1 .. g1 h1]