std/jpeg: add decoder.decode_idct_x86_avx2

name                                             old speed     new speed     delta

wuffs_jpeg_decode_19k_8bpp/clang11               237MB/s ± 0%  370MB/s ± 0%  +55.94%  (p=0.008 n=5+5)
wuffs_jpeg_decode_30k_24bpp_progressive/clang11  205MB/s ± 0%  231MB/s ± 0%  +12.80%  (p=0.008 n=5+5)
wuffs_jpeg_decode_30k_24bpp_sequential/clang11   379MB/s ± 0%  470MB/s ± 0%  +23.92%  (p=0.008 n=5+5)
wuffs_jpeg_decode_77k_24bpp/clang11              332MB/s ± 0%  418MB/s ± 0%  +25.95%  (p=0.008 n=5+5)
wuffs_jpeg_decode_552k_24bpp_420/clang11         345MB/s ± 0%  435MB/s ± 0%  +25.99%  (p=0.008 n=5+5)
wuffs_jpeg_decode_552k_24bpp_444/clang11         483MB/s ± 0%  620MB/s ± 0%  +28.53%  (p=0.008 n=5+5)
wuffs_jpeg_decode_4002k_24bpp/clang11            349MB/s ± 0%  441MB/s ± 0%  +26.47%  (p=0.008 n=5+5)

wuffs_jpeg_decode_19k_8bpp/gcc10                 231MB/s ± 0%  352MB/s ± 0%  +52.78%  (p=0.008 n=5+5)
wuffs_jpeg_decode_30k_24bpp_progressive/gcc10    227MB/s ± 0%  296MB/s ± 0%  +30.44%  (p=0.008 n=5+5)
wuffs_jpeg_decode_30k_24bpp_sequential/gcc10     500MB/s ± 0%  666MB/s ± 0%  +33.21%  (p=0.008 n=5+5)
wuffs_jpeg_decode_77k_24bpp/gcc10                417MB/s ± 0%  565MB/s ± 0%  +35.32%  (p=0.008 n=5+5)
wuffs_jpeg_decode_552k_24bpp_420/gcc10           440MB/s ± 0%  608MB/s ± 0%  +38.37%  (p=0.008 n=5+5)
wuffs_jpeg_decode_552k_24bpp_444/gcc10           478MB/s ± 0%  624MB/s ± 0%  +30.70%  (p=0.008 n=5+5)
wuffs_jpeg_decode_4002k_24bpp/gcc10              445MB/s ± 0%  616MB/s ± 0%  +38.42%  (p=0.008 n=5+5)
This commit is contained in:
Nigel Tao committed 2023-07-18 09:17:08 +10:00
1 parent 0af626b850
commit 2bff962cf1
7 files changed
+1282 -8

No files matched your search

+356
View File
@@ -40054,6 +40054,357 @@ wuffs_jpeg__decoder__decode_idct_x86_avx2(
wuffs_base__slice_u8 a_dst_buffer,
uint64_t a_dst_stride,
uint32_t a_q) {
__m256i v_k_0000 = {0};
__m256i v_k_8080 = {0};
__m256i v_k_0000_0002 = {0};
__m256i v_k_0001_FFFF = {0};
__m256i v_k_0400_0000 = {0};
__m256i v_k_29CF_1151_D630_1151 = {0};
__m256i v_k_E333_133E_ADFD_1051 = {0};
__m256i v_k_E6DC_25A1_1925_25A1 = {0};
__m256i v_k_ECC1_E333_EFB0_ADFD = {0};
__m256i v_rows01 = {0};
__m256i v_rows23 = {0};
__m256i v_rows45 = {0};
__m256i v_rows67 = {0};
__m256i v_quants01 = {0};
__m256i v_quants23 = {0};
__m256i v_quants45 = {0};
__m256i v_quants67 = {0};
__m256i v_rows04 = {0};
__m256i v_rows31 = {0};
__m256i v_rows26 = {0};
__m256i v_rows75 = {0};
__m256i v_fp_rows62 = {0};
__m256i v_fp_bq2662ad = {0};
__m256i v_fp_bq2662eh = {0};
__m256i v_fp_cb26ad = {0};
__m256i v_fp_cb26eh = {0};
__m256i v_fp_rows40pos = {0};
__m256i v_fp_rows04neg = {0};
__m256i v_fp_rows0pm4 = {0};
__m256i v_fp_ccpmad = {0};
__m256i v_fp_ccpmeh = {0};
__m256i v_fp_cd01ad = {0};
__m256i v_fp_cd01eh = {0};
__m256i v_fp_cd32ad = {0};
__m256i v_fp_cd32eh = {0};
__m256i v_fp_sums7351 = {0};
__m256i v_fp_sums5173 = {0};
__m256i v_fp_ci73515173ad = {0};
__m256i v_fp_ci73515173eh = {0};
__m256i v_fp_cl7351ad = {0};
__m256i v_fp_cl7351eh = {0};
__m256i v_fp_rows13 = {0};
__m256i v_fp_bq7153ad = {0};
__m256i v_fp_bq7153eh = {0};
__m256i v_fp_ck75ad = {0};
__m256i v_fp_ck75eh = {0};
__m256i v_fp_cl5173ad = {0};
__m256i v_fp_cl5173eh = {0};
__m256i v_fp_ck13ad = {0};
__m256i v_fp_ck13eh = {0};
__m256i v_intermediate01ad = {0};
__m256i v_intermediate01eh = {0};
__m256i v_intermediate01 = {0};
__m256i v_intermediate32ad = {0};
__m256i v_intermediate32eh = {0};
__m256i v_intermediate32 = {0};
__m256i v_intermediate45ad = {0};
__m256i v_intermediate45eh = {0};
__m256i v_intermediate45 = {0};
__m256i v_intermediate76ad = {0};
__m256i v_intermediate76eh = {0};
__m256i v_intermediate76 = {0};
__m256i v_ita0a1e0e1 = {0};
__m256i v_ita2a3e2e3 = {0};
__m256i v_ita4a5e4e5 = {0};
__m256i v_ita6a7e6e7 = {0};
__m256i v_ita0c0e0g0 = {0};
__m256i v_ita1c1e1g1 = {0};
__m256i v_ita4c4e4g4 = {0};
__m256i v_ita5c5e5g5 = {0};
__m256i v_ita0b0e0f0 = {0};
__m256i v_ita4b4e4f4 = {0};
__m256i v_itc0d0g0h0 = {0};
__m256i v_itc4d4g4h4 = {0};
__m256i v_intermediateae = {0};
__m256i v_intermediatebf = {0};
__m256i v_intermediatecg = {0};
__m256i v_intermediatedh = {0};
__m256i v_intermediatedb = {0};
__m256i v_intermediatehf = {0};
__m256i v_sp_cols62 = {0};
__m256i v_sp_bq2662ad = {0};
__m256i v_sp_bq2662eh = {0};
__m256i v_sp_rb26ad = {0};
__m256i v_sp_rb26eh = {0};
__m256i v_sp_cols40pos = {0};
__m256i v_sp_cols04neg = {0};
__m256i v_sp_cols0pm4 = {0};
__m256i v_sp_rcpmad = {0};
__m256i v_sp_rcpmeh = {0};
__m256i v_sp_rd01ad = {0};
__m256i v_sp_rd01eh = {0};
__m256i v_sp_rd32ad = {0};
__m256i v_sp_rd32eh = {0};
__m256i v_sp_sums7351 = {0};
__m256i v_sp_sums5173 = {0};
__m256i v_sp_ri73515173ad = {0};
__m256i v_sp_ri73515173eh = {0};
__m256i v_sp_rl7351ad = {0};
__m256i v_sp_rl7351eh = {0};
__m256i v_sp_cols13 = {0};
__m256i v_sp_bq7153ad = {0};
__m256i v_sp_bq7153eh = {0};
__m256i v_sp_rk75ad = {0};
__m256i v_sp_rk75eh = {0};
__m256i v_sp_rl5173ad = {0};
__m256i v_sp_rl5173eh = {0};
__m256i v_sp_rk13ad = {0};
__m256i v_sp_rk13eh = {0};
__m256i v_final01ad = {0};
__m256i v_final01eh = {0};
__m256i v_final01 = {0};
__m256i v_final32ad = {0};
__m256i v_final32eh = {0};
__m256i v_final32 = {0};
__m256i v_final45ad = {0};
__m256i v_final45eh = {0};
__m256i v_final45 = {0};
__m256i v_final76ad = {0};
__m256i v_final76eh = {0};
__m256i v_final76 = {0};
__m256i v_fta0a1e0e1 = {0};
__m256i v_fta2a3e2e3 = {0};
__m256i v_fta4a5e4e5 = {0};
__m256i v_fta6a7e6e7 = {0};
__m256i v_fta0c0e0g0 = {0};
__m256i v_fta1c1e1g1 = {0};
__m256i v_fta4c4e4g4 = {0};
__m256i v_fta5c5e5g5 = {0};
__m256i v_fta0b0e0f0 = {0};
__m256i v_ftc0d0g0h0 = {0};
__m256i v_fta4b4e4f4 = {0};
__m256i v_ftc4d4g4h4 = {0};
__m256i v_finalae = {0};
__m256i v_finalbf = {0};
__m256i v_finalcg = {0};
__m256i v_finaldh = {0};
__m256i v_final0145 = {0};
__m256i v_final2367 = {0};
uint64_t v_final0 = 0;
uint64_t v_final1 = 0;
uint64_t v_final2 = 0;
uint64_t v_final3 = 0;
uint64_t v_final4 = 0;
uint64_t v_final5 = 0;
uint64_t v_final6 = 0;
uint64_t v_final7 = 0;
wuffs_base__slice_u8 v_remaining = {0};
if (8u > a_dst_stride) {
return wuffs_base__make_empty_struct();
}
v_k_0000 = _mm256_set_epi16((int16_t)(0u), (int16_t)(0u), (int16_t)(0u), (int16_t)(0u), (int16_t)(0u), (int16_t)(0u), (int16_t)(0u), (int16_t)(0u), (int16_t)(0u), (int16_t)(0u), (int16_t)(0u), (int16_t)(0u), (int16_t)(0u), (int16_t)(0u), (int16_t)(0u), (int16_t)(0u));
v_k_8080 = _mm256_set_epi16((int16_t)(32896u), (int16_t)(32896u), (int16_t)(32896u), (int16_t)(32896u), (int16_t)(32896u), (int16_t)(32896u), (int16_t)(32896u), (int16_t)(32896u), (int16_t)(32896u), (int16_t)(32896u), (int16_t)(32896u), (int16_t)(32896u), (int16_t)(32896u), (int16_t)(32896u), (int16_t)(32896u), (int16_t)(32896u));
v_k_0000_0002 = _mm256_set_epi16((int16_t)(2u), (int16_t)(0u), (int16_t)(2u), (int16_t)(0u), (int16_t)(2u), (int16_t)(0u), (int16_t)(2u), (int16_t)(0u), (int16_t)(2u), (int16_t)(0u), (int16_t)(2u), (int16_t)(0u), (int16_t)(2u), (int16_t)(0u), (int16_t)(2u), (int16_t)(0u));
v_k_0001_FFFF = _mm256_set_epi16((int16_t)(65535u), (int16_t)(65535u), (int16_t)(65535u), (int16_t)(65535u), (int16_t)(65535u), (int16_t)(65535u), (int16_t)(65535u), (int16_t)(65535u), (int16_t)(1u), (int16_t)(1u), (int16_t)(1u), (int16_t)(1u), (int16_t)(1u), (int16_t)(1u), (int16_t)(1u), (int16_t)(1u));
v_k_0400_0000 = _mm256_set_epi16((int16_t)(0u), (int16_t)(1024u), (int16_t)(0u), (int16_t)(1024u), (int16_t)(0u), (int16_t)(1024u), (int16_t)(0u), (int16_t)(1024u), (int16_t)(0u), (int16_t)(1024u), (int16_t)(0u), (int16_t)(1024u), (int16_t)(0u), (int16_t)(1024u), (int16_t)(0u), (int16_t)(1024u));
v_k_29CF_1151_D630_1151 = _mm256_set_epi16((int16_t)(4433u), (int16_t)(54832u), (int16_t)(4433u), (int16_t)(54832u), (int16_t)(4433u), (int16_t)(54832u), (int16_t)(4433u), (int16_t)(54832u), (int16_t)(4433u), (int16_t)(10703u), (int16_t)(4433u), (int16_t)(10703u), (int16_t)(4433u), (int16_t)(10703u), (int16_t)(4433u), (int16_t)(10703u));
v_k_E333_133E_ADFD_1051 = _mm256_set_epi16((int16_t)(4177u), (int16_t)(44541u), (int16_t)(4177u), (int16_t)(44541u), (int16_t)(4177u), (int16_t)(44541u), (int16_t)(4177u), (int16_t)(44541u), (int16_t)(4926u), (int16_t)(58163u), (int16_t)(4926u), (int16_t)(58163u), (int16_t)(4926u), (int16_t)(58163u), (int16_t)(4926u), (int16_t)(58163u));
v_k_E6DC_25A1_1925_25A1 = _mm256_set_epi16((int16_t)(9633u), (int16_t)(6437u), (int16_t)(9633u), (int16_t)(6437u), (int16_t)(9633u), (int16_t)(6437u), (int16_t)(9633u), (int16_t)(6437u), (int16_t)(9633u), (int16_t)(59100u), (int16_t)(9633u), (int16_t)(59100u), (int16_t)(9633u), (int16_t)(59100u), (int16_t)(9633u), (int16_t)(59100u));
v_k_ECC1_E333_EFB0_ADFD = _mm256_set_epi16((int16_t)(44541u), (int16_t)(61360u), (int16_t)(44541u), (int16_t)(61360u), (int16_t)(44541u), (int16_t)(61360u), (int16_t)(44541u), (int16_t)(61360u), (int16_t)(58163u), (int16_t)(60609u), (int16_t)(58163u), (int16_t)(60609u), (int16_t)(58163u), (int16_t)(60609u), (int16_t)(58163u), (int16_t)(60609u));
do {
v_rows01 = _mm256_lddqu_si256((const __m256i*)(const void*)(self->private_data.f_mcu_blocks[0u] + 0u));
v_rows23 = _mm256_lddqu_si256((const __m256i*)(const void*)(self->private_data.f_mcu_blocks[0u] + 16u));
v_rows45 = _mm256_lddqu_si256((const __m256i*)(const void*)(self->private_data.f_mcu_blocks[0u] + 32u));
v_rows67 = _mm256_lddqu_si256((const __m256i*)(const void*)(self->private_data.f_mcu_blocks[0u] + 48u));
v_quants01 = _mm256_lddqu_si256((const __m256i*)(const void*)(self->private_impl.f_quant_tables[a_q] + 0u));
v_quants23 = _mm256_lddqu_si256((const __m256i*)(const void*)(self->private_impl.f_quant_tables[a_q] + 16u));
v_quants45 = _mm256_lddqu_si256((const __m256i*)(const void*)(self->private_impl.f_quant_tables[a_q] + 32u));
v_quants67 = _mm256_lddqu_si256((const __m256i*)(const void*)(self->private_impl.f_quant_tables[a_q] + 48u));
v_rows01 = _mm256_mullo_epi16(v_rows01, v_quants01);
v_rows23 = _mm256_mullo_epi16(v_rows23, v_quants23);
v_rows45 = _mm256_mullo_epi16(v_rows45, v_quants45);
v_rows67 = _mm256_mullo_epi16(v_rows67, v_quants67);
v_rows04 = _mm256_permute2x128_si256(v_rows01, v_rows45, (int32_t)(32u));
v_rows31 = _mm256_permute2x128_si256(v_rows23, v_rows01, (int32_t)(49u));
v_rows26 = _mm256_permute2x128_si256(v_rows23, v_rows67, (int32_t)(32u));
v_rows75 = _mm256_permute2x128_si256(v_rows67, v_rows45, (int32_t)(49u));
v_fp_rows62 = _mm256_permute2x128_si256(v_rows26, v_rows26, (int32_t)(1u));
v_fp_bq2662ad = _mm256_unpacklo_epi16(v_rows26, v_fp_rows62);
v_fp_bq2662eh = _mm256_unpackhi_epi16(v_rows26, v_fp_rows62);
v_fp_cb26ad = _mm256_madd_epi16(v_fp_bq2662ad, v_k_29CF_1151_D630_1151);
v_fp_cb26eh = _mm256_madd_epi16(v_fp_bq2662eh, v_k_29CF_1151_D630_1151);
v_fp_rows40pos = _mm256_permute2x128_si256(v_rows04, v_rows04, (int32_t)(1u));
v_fp_rows04neg = _mm256_sign_epi16(v_rows04, v_k_0001_FFFF);
v_fp_rows0pm4 = _mm256_add_epi16(v_fp_rows40pos, v_fp_rows04neg);
v_fp_ccpmad = _mm256_srai_epi32(_mm256_unpacklo_epi16(v_k_0000, v_fp_rows0pm4), (int32_t)(3u));
v_fp_ccpmeh = _mm256_srai_epi32(_mm256_unpackhi_epi16(v_k_0000, v_fp_rows0pm4), (int32_t)(3u));
v_fp_cd01ad = _mm256_add_epi32(v_fp_ccpmad, v_fp_cb26ad);
v_fp_cd01eh = _mm256_add_epi32(v_fp_ccpmeh, v_fp_cb26eh);
v_fp_cd32ad = _mm256_sub_epi32(v_fp_ccpmad, v_fp_cb26ad);
v_fp_cd32eh = _mm256_sub_epi32(v_fp_ccpmeh, v_fp_cb26eh);
v_fp_sums7351 = _mm256_add_epi16(v_rows75, v_rows31);
v_fp_sums5173 = _mm256_permute2x128_si256(v_fp_sums7351, v_fp_sums7351, (int32_t)(1u));
v_fp_ci73515173ad = _mm256_unpacklo_epi16(v_fp_sums7351, v_fp_sums5173);
v_fp_ci73515173eh = _mm256_unpackhi_epi16(v_fp_sums7351, v_fp_sums5173);
v_fp_cl7351ad = _mm256_madd_epi16(v_fp_ci73515173ad, v_k_E6DC_25A1_1925_25A1);
v_fp_cl7351eh = _mm256_madd_epi16(v_fp_ci73515173eh, v_k_E6DC_25A1_1925_25A1);
v_fp_rows13 = _mm256_permute2x128_si256(v_rows31, v_rows31, (int32_t)(1u));
v_fp_bq7153ad = _mm256_unpacklo_epi16(v_rows75, v_fp_rows13);
v_fp_bq7153eh = _mm256_unpackhi_epi16(v_rows75, v_fp_rows13);
v_fp_ck75ad = _mm256_add_epi32(_mm256_madd_epi16(v_fp_bq7153ad, v_k_ECC1_E333_EFB0_ADFD), v_fp_cl7351ad);
v_fp_ck75eh = _mm256_add_epi32(_mm256_madd_epi16(v_fp_bq7153eh, v_k_ECC1_E333_EFB0_ADFD), v_fp_cl7351eh);
v_fp_cl5173ad = _mm256_permute2x128_si256(v_fp_cl7351ad, v_fp_cl7351ad, (int32_t)(1u));
v_fp_cl5173eh = _mm256_permute2x128_si256(v_fp_cl7351eh, v_fp_cl7351eh, (int32_t)(1u));
v_fp_ck13ad = _mm256_add_epi32(v_fp_cl5173ad, _mm256_madd_epi16(v_fp_bq7153ad, v_k_E333_133E_ADFD_1051));
v_fp_ck13eh = _mm256_add_epi32(v_fp_cl5173eh, _mm256_madd_epi16(v_fp_bq7153eh, v_k_E333_133E_ADFD_1051));
v_intermediate01ad = _mm256_srai_epi32(_mm256_add_epi32(_mm256_add_epi32(v_fp_cd01ad, v_fp_ck13ad), v_k_0400_0000), (int32_t)(11u));
v_intermediate01eh = _mm256_srai_epi32(_mm256_add_epi32(_mm256_add_epi32(v_fp_cd01eh, v_fp_ck13eh), v_k_0400_0000), (int32_t)(11u));
v_intermediate01 = _mm256_packs_epi32(v_intermediate01ad, v_intermediate01eh);
v_intermediate32ad = _mm256_srai_epi32(_mm256_add_epi32(_mm256_add_epi32(v_fp_cd32ad, v_fp_ck75ad), v_k_0400_0000), (int32_t)(11u));
v_intermediate32eh = _mm256_srai_epi32(_mm256_add_epi32(_mm256_add_epi32(v_fp_cd32eh, v_fp_ck75eh), v_k_0400_0000), (int32_t)(11u));
v_intermediate32 = _mm256_packs_epi32(v_intermediate32ad, v_intermediate32eh);
v_intermediate45ad = _mm256_srai_epi32(_mm256_add_epi32(_mm256_sub_epi32(v_fp_cd32ad, v_fp_ck75ad), v_k_0400_0000), (int32_t)(11u));
v_intermediate45eh = _mm256_srai_epi32(_mm256_add_epi32(_mm256_sub_epi32(v_fp_cd32eh, v_fp_ck75eh), v_k_0400_0000), (int32_t)(11u));
v_intermediate45 = _mm256_packs_epi32(v_intermediate45ad, v_intermediate45eh);
v_intermediate76ad = _mm256_srai_epi32(_mm256_add_epi32(_mm256_sub_epi32(v_fp_cd01ad, v_fp_ck13ad), v_k_0400_0000), (int32_t)(11u));
v_intermediate76eh = _mm256_srai_epi32(_mm256_add_epi32(_mm256_sub_epi32(v_fp_cd01eh, v_fp_ck13eh), v_k_0400_0000), (int32_t)(11u));
v_intermediate76 = _mm256_packs_epi32(v_intermediate76ad, v_intermediate76eh);
v_ita0a1e0e1 = _mm256_permute4x64_epi64(v_intermediate01, (int32_t)(216u));
v_ita2a3e2e3 = _mm256_permute4x64_epi64(v_intermediate32, (int32_t)(114u));
v_ita4a5e4e5 = _mm256_permute4x64_epi64(v_intermediate45, (int32_t)(216u));
v_ita6a7e6e7 = _mm256_permute4x64_epi64(v_intermediate76, (int32_t)(114u));
v_ita0c0e0g0 = _mm256_unpacklo_epi16(v_ita0a1e0e1, v_ita2a3e2e3);
v_ita1c1e1g1 = _mm256_unpackhi_epi16(v_ita0a1e0e1, v_ita2a3e2e3);
v_ita4c4e4g4 = _mm256_unpacklo_epi16(v_ita4a5e4e5, v_ita6a7e6e7);
v_ita5c5e5g5 = _mm256_unpackhi_epi16(v_ita4a5e4e5, v_ita6a7e6e7);
v_ita0b0e0f0 = _mm256_unpacklo_epi16(v_ita0c0e0g0, v_ita1c1e1g1);
v_itc0d0g0h0 = _mm256_unpackhi_epi16(v_ita0c0e0g0, v_ita1c1e1g1);
v_ita4b4e4f4 = _mm256_unpacklo_epi16(v_ita4c4e4g4, v_ita5c5e5g5);
v_itc4d4g4h4 = _mm256_unpackhi_epi16(v_ita4c4e4g4, v_ita5c5e5g5);
v_intermediateae = _mm256_unpacklo_epi64(v_ita0b0e0f0, v_ita4b4e4f4);
v_intermediatebf = _mm256_unpackhi_epi64(v_ita0b0e0f0, v_ita4b4e4f4);
v_intermediatecg = _mm256_unpacklo_epi64(v_itc0d0g0h0, v_itc4d4g4h4);
v_intermediatedh = _mm256_unpackhi_epi64(v_itc0d0g0h0, v_itc4d4g4h4);
} while (0);
v_intermediatedb = _mm256_permute2x128_si256(v_intermediatedh, v_intermediatebf, (int32_t)(32u));
v_intermediatehf = _mm256_permute2x128_si256(v_intermediatedh, v_intermediatebf, (int32_t)(49u));
v_sp_cols62 = _mm256_permute2x128_si256(v_intermediatecg, v_intermediatecg, (int32_t)(1u));
v_sp_bq2662ad = _mm256_unpacklo_epi16(v_intermediatecg, v_sp_cols62);
v_sp_bq2662eh = _mm256_unpackhi_epi16(v_intermediatecg, v_sp_cols62);
v_sp_rb26ad = _mm256_madd_epi16(v_sp_bq2662ad, v_k_29CF_1151_D630_1151);
v_sp_rb26eh = _mm256_madd_epi16(v_sp_bq2662eh, v_k_29CF_1151_D630_1151);
v_sp_cols40pos = _mm256_permute2x128_si256(v_intermediateae, v_intermediateae, (int32_t)(1u));
v_sp_cols04neg = _mm256_sign_epi16(v_intermediateae, v_k_0001_FFFF);
v_sp_cols0pm4 = _mm256_add_epi16(v_sp_cols40pos, v_sp_cols04neg);
v_sp_rcpmad = _mm256_srai_epi32(_mm256_unpacklo_epi16(v_k_0000, v_sp_cols0pm4), (int32_t)(3u));
v_sp_rcpmeh = _mm256_srai_epi32(_mm256_unpackhi_epi16(v_k_0000, v_sp_cols0pm4), (int32_t)(3u));
v_sp_rd01ad = _mm256_add_epi32(v_sp_rcpmad, v_sp_rb26ad);
v_sp_rd01eh = _mm256_add_epi32(v_sp_rcpmeh, v_sp_rb26eh);
v_sp_rd32ad = _mm256_sub_epi32(v_sp_rcpmad, v_sp_rb26ad);
v_sp_rd32eh = _mm256_sub_epi32(v_sp_rcpmeh, v_sp_rb26eh);
v_sp_sums7351 = _mm256_add_epi16(v_intermediatehf, v_intermediatedb);
v_sp_sums5173 = _mm256_permute2x128_si256(v_sp_sums7351, v_sp_sums7351, (int32_t)(1u));
v_sp_ri73515173ad = _mm256_unpacklo_epi16(v_sp_sums7351, v_sp_sums5173);
v_sp_ri73515173eh = _mm256_unpackhi_epi16(v_sp_sums7351, v_sp_sums5173);
v_sp_rl7351ad = _mm256_madd_epi16(v_sp_ri73515173ad, v_k_E6DC_25A1_1925_25A1);
v_sp_rl7351eh = _mm256_madd_epi16(v_sp_ri73515173eh, v_k_E6DC_25A1_1925_25A1);
v_sp_cols13 = _mm256_permute2x128_si256(v_intermediatedb, v_intermediatedb, (int32_t)(1u));
v_sp_bq7153ad = _mm256_unpacklo_epi16(v_intermediatehf, v_sp_cols13);
v_sp_bq7153eh = _mm256_unpackhi_epi16(v_intermediatehf, v_sp_cols13);
v_sp_rk75ad = _mm256_add_epi32(_mm256_madd_epi16(v_sp_bq7153ad, v_k_ECC1_E333_EFB0_ADFD), v_sp_rl7351ad);
v_sp_rk75eh = _mm256_add_epi32(_mm256_madd_epi16(v_sp_bq7153eh, v_k_ECC1_E333_EFB0_ADFD), v_sp_rl7351eh);
v_sp_rl5173ad = _mm256_permute2x128_si256(v_sp_rl7351ad, v_sp_rl7351ad, (int32_t)(1u));
v_sp_rl5173eh = _mm256_permute2x128_si256(v_sp_rl7351eh, v_sp_rl7351eh, (int32_t)(1u));
v_sp_rk13ad = _mm256_add_epi32(v_sp_rl5173ad, _mm256_madd_epi16(v_sp_bq7153ad, v_k_E333_133E_ADFD_1051));
v_sp_rk13eh = _mm256_add_epi32(v_sp_rl5173eh, _mm256_madd_epi16(v_sp_bq7153eh, v_k_E333_133E_ADFD_1051));
v_final01ad = _mm256_srai_epi32(_mm256_add_epi32(_mm256_add_epi32(v_sp_rd01ad, v_sp_rk13ad), v_k_0000_0002), (int32_t)(18u));
v_final01eh = _mm256_srai_epi32(_mm256_add_epi32(_mm256_add_epi32(v_sp_rd01eh, v_sp_rk13eh), v_k_0000_0002), (int32_t)(18u));
v_final01 = _mm256_packs_epi32(v_final01ad, v_final01eh);
v_final32ad = _mm256_srai_epi32(_mm256_add_epi32(_mm256_add_epi32(v_sp_rd32ad, v_sp_rk75ad), v_k_0000_0002), (int32_t)(18u));
v_final32eh = _mm256_srai_epi32(_mm256_add_epi32(_mm256_add_epi32(v_sp_rd32eh, v_sp_rk75eh), v_k_0000_0002), (int32_t)(18u));
v_final32 = _mm256_packs_epi32(v_final32ad, v_final32eh);
v_final45ad = _mm256_srai_epi32(_mm256_add_epi32(_mm256_sub_epi32(v_sp_rd32ad, v_sp_rk75ad), v_k_0000_0002), (int32_t)(18u));
v_final45eh = _mm256_srai_epi32(_mm256_add_epi32(_mm256_sub_epi32(v_sp_rd32eh, v_sp_rk75eh), v_k_0000_0002), (int32_t)(18u));
v_final45 = _mm256_packs_epi32(v_final45ad, v_final45eh);
v_final76ad = _mm256_srai_epi32(_mm256_add_epi32(_mm256_sub_epi32(v_sp_rd01ad, v_sp_rk13ad), v_k_0000_0002), (int32_t)(18u));
v_final76eh = _mm256_srai_epi32(_mm256_add_epi32(_mm256_sub_epi32(v_sp_rd01eh, v_sp_rk13eh), v_k_0000_0002), (int32_t)(18u));
v_final76 = _mm256_packs_epi32(v_final76ad, v_final76eh);
v_fta0a1e0e1 = _mm256_permute4x64_epi64(v_final01, (int32_t)(216u));
v_fta2a3e2e3 = _mm256_permute4x64_epi64(v_final32, (int32_t)(114u));
v_fta4a5e4e5 = _mm256_permute4x64_epi64(v_final45, (int32_t)(216u));
v_fta6a7e6e7 = _mm256_permute4x64_epi64(v_final76, (int32_t)(114u));
v_fta0c0e0g0 = _mm256_unpacklo_epi16(v_fta0a1e0e1, v_fta2a3e2e3);
v_fta1c1e1g1 = _mm256_unpackhi_epi16(v_fta0a1e0e1, v_fta2a3e2e3);
v_fta4c4e4g4 = _mm256_unpacklo_epi16(v_fta4a5e4e5, v_fta6a7e6e7);
v_fta5c5e5g5 = _mm256_unpackhi_epi16(v_fta4a5e4e5, v_fta6a7e6e7);
v_fta0b0e0f0 = _mm256_unpacklo_epi16(v_fta0c0e0g0, v_fta1c1e1g1);
v_ftc0d0g0h0 = _mm256_unpackhi_epi16(v_fta0c0e0g0, v_fta1c1e1g1);
v_fta4b4e4f4 = _mm256_unpacklo_epi16(v_fta4c4e4g4, v_fta5c5e5g5);
v_ftc4d4g4h4 = _mm256_unpackhi_epi16(v_fta4c4e4g4, v_fta5c5e5g5);
v_finalae = _mm256_unpacklo_epi64(v_fta0b0e0f0, v_fta4b4e4f4);
v_finalbf = _mm256_unpackhi_epi64(v_fta0b0e0f0, v_fta4b4e4f4);
v_finalcg = _mm256_unpacklo_epi64(v_ftc0d0g0h0, v_ftc4d4g4h4);
v_finaldh = _mm256_unpackhi_epi64(v_ftc0d0g0h0, v_ftc4d4g4h4);
v_final0145 = _mm256_add_epi8(_mm256_packs_epi16(v_finalae, v_finalbf), v_k_8080);
v_final2367 = _mm256_add_epi8(_mm256_packs_epi16(v_finalcg, v_finaldh), v_k_8080);
v_final0 = _mm256_extract_epi64(v_final0145, (int32_t)(0u));
v_final1 = _mm256_extract_epi64(v_final0145, (int32_t)(1u));
v_final2 = _mm256_extract_epi64(v_final2367, (int32_t)(0u));
v_final3 = _mm256_extract_epi64(v_final2367, (int32_t)(1u));
v_final4 = _mm256_extract_epi64(v_final0145, (int32_t)(2u));
v_final5 = _mm256_extract_epi64(v_final0145, (int32_t)(3u));
v_final6 = _mm256_extract_epi64(v_final2367, (int32_t)(2u));
v_final7 = _mm256_extract_epi64(v_final2367, (int32_t)(3u));
if (a_dst_stride > ((uint64_t)(a_dst_buffer.len))) {
return wuffs_base__make_empty_struct();
}
v_remaining = wuffs_base__slice_u8__subslice_i(a_dst_buffer, a_dst_stride);
wuffs_base__poke_u64le__no_bounds_check(a_dst_buffer.ptr, v_final0);
a_dst_buffer = v_remaining;
if (a_dst_stride > ((uint64_t)(a_dst_buffer.len))) {
return wuffs_base__make_empty_struct();
}
v_remaining = wuffs_base__slice_u8__subslice_i(a_dst_buffer, a_dst_stride);
wuffs_base__poke_u64le__no_bounds_check(a_dst_buffer.ptr, v_final1);
a_dst_buffer = v_remaining;
if (a_dst_stride > ((uint64_t)(a_dst_buffer.len))) {
return wuffs_base__make_empty_struct();
}
v_remaining = wuffs_base__slice_u8__subslice_i(a_dst_buffer, a_dst_stride);
wuffs_base__poke_u64le__no_bounds_check(a_dst_buffer.ptr, v_final2);
a_dst_buffer = v_remaining;
if (a_dst_stride > ((uint64_t)(a_dst_buffer.len))) {
return wuffs_base__make_empty_struct();
}
v_remaining = wuffs_base__slice_u8__subslice_i(a_dst_buffer, a_dst_stride);
wuffs_base__poke_u64le__no_bounds_check(a_dst_buffer.ptr, v_final3);
a_dst_buffer = v_remaining;
if (a_dst_stride > ((uint64_t)(a_dst_buffer.len))) {
return wuffs_base__make_empty_struct();
}
v_remaining = wuffs_base__slice_u8__subslice_i(a_dst_buffer, a_dst_stride);
wuffs_base__poke_u64le__no_bounds_check(a_dst_buffer.ptr, v_final4);
a_dst_buffer = v_remaining;
if (a_dst_stride > ((uint64_t)(a_dst_buffer.len))) {
return wuffs_base__make_empty_struct();
}
v_remaining = wuffs_base__slice_u8__subslice_i(a_dst_buffer, a_dst_stride);
wuffs_base__poke_u64le__no_bounds_check(a_dst_buffer.ptr, v_final5);
a_dst_buffer = v_remaining;
if (a_dst_stride > ((uint64_t)(a_dst_buffer.len))) {
return wuffs_base__make_empty_struct();
}
v_remaining = wuffs_base__slice_u8__subslice_i(a_dst_buffer, a_dst_stride);
wuffs_base__poke_u64le__no_bounds_check(a_dst_buffer.ptr, v_final6);
a_dst_buffer = v_remaining;
if (8u > ((uint64_t)(a_dst_buffer.len))) {
return wuffs_base__make_empty_struct();
}
wuffs_base__poke_u64le__no_bounds_check(a_dst_buffer.ptr, v_final7);
return wuffs_base__make_empty_struct();
}
#endif // defined(WUFFS_BASE__CPU_ARCH__X86_FAMILY)
@@ -40392,6 +40743,11 @@ wuffs_jpeg__decoder__do_decode_image_config(
iop_a_src += self->private_data.s_do_decode_image_config[0].scratch;
self->private_impl.f_payload_length = 0u;
}
self->private_impl.choosy_decode_idct = (
#if defined(WUFFS_BASE__CPU_ARCH__X86_FAMILY)
wuffs_base__cpu_arch__have_x86_avx2() ? &wuffs_jpeg__decoder__decode_idct_x86_avx2 :
#endif
self->private_impl.choosy_decode_idct);
self->private_impl.f_frame_config_io_position = wuffs_base__u64__sat_add((a_src ? a_src->meta.pos : 0), ((uint64_t)(iop_a_src - io0_a_src)));
if (a_dst != NULL) {
v_pixfmt = 536870920u;