From 2bff962cf1bcb263046d1f07239d1b27e8b8c726 Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Mon, 17 Jul 2023 15:08:07 +1000 Subject: [PATCH] std/jpeg: add decoder.decode_idct_x86_avx2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit name old speed new speed delta wuffs_jpeg_decode_19k_8bpp/clang11 237MB/s ± 0% 370MB/s ± 0% +55.94% (p=0.008 n=5+5) wuffs_jpeg_decode_30k_24bpp_progressive/clang11 205MB/s ± 0% 231MB/s ± 0% +12.80% (p=0.008 n=5+5) wuffs_jpeg_decode_30k_24bpp_sequential/clang11 379MB/s ± 0% 470MB/s ± 0% +23.92% (p=0.008 n=5+5) wuffs_jpeg_decode_77k_24bpp/clang11 332MB/s ± 0% 418MB/s ± 0% +25.95% (p=0.008 n=5+5) wuffs_jpeg_decode_552k_24bpp_420/clang11 345MB/s ± 0% 435MB/s ± 0% +25.99% (p=0.008 n=5+5) wuffs_jpeg_decode_552k_24bpp_444/clang11 483MB/s ± 0% 620MB/s ± 0% +28.53% (p=0.008 n=5+5) wuffs_jpeg_decode_4002k_24bpp/clang11 349MB/s ± 0% 441MB/s ± 0% +26.47% (p=0.008 n=5+5) wuffs_jpeg_decode_19k_8bpp/gcc10 231MB/s ± 0% 352MB/s ± 0% +52.78% (p=0.008 n=5+5) wuffs_jpeg_decode_30k_24bpp_progressive/gcc10 227MB/s ± 0% 296MB/s ± 0% +30.44% (p=0.008 n=5+5) wuffs_jpeg_decode_30k_24bpp_sequential/gcc10 500MB/s ± 0% 666MB/s ± 0% +33.21% (p=0.008 n=5+5) wuffs_jpeg_decode_77k_24bpp/gcc10 417MB/s ± 0% 565MB/s ± 0% +35.32% (p=0.008 n=5+5) wuffs_jpeg_decode_552k_24bpp_420/gcc10 440MB/s ± 0% 608MB/s ± 0% +38.37% (p=0.008 n=5+5) wuffs_jpeg_decode_552k_24bpp_444/gcc10 478MB/s ± 0% 624MB/s ± 0% +30.70% (p=0.008 n=5+5) wuffs_jpeg_decode_4002k_24bpp/gcc10 445MB/s ± 0% 616MB/s ± 0% +38.42% (p=0.008 n=5+5) --- internal/cgen/builtin.go | 35 +- lang/builtin/builtin.go | 28 + lang/check/bounds.go | 4 + release/c/wuffs-unsupported-snapshot.c | 356 ++++++++++ std/jpeg/decode_idct_x86_avx2.wuffs | 861 ++++++++++++++++++++++++- std/jpeg/decode_jpeg.wuffs | 4 + test/c/std/jpeg.c | 2 +- 7 files changed, 1282 insertions(+), 8 deletions(-) diff --git a/internal/cgen/builtin.go b/internal/cgen/builtin.go index 5f7a6c47..f7a3fdf4 100644 --- a/internal/cgen/builtin.go +++ b/internal/cgen/builtin.go @@ -652,7 +652,7 @@ func (g *gen) writeBuiltinCPUArchX86(b *buffer, recv *a.Expr, method t.ID, args fName, tName = "_mm256_set1_epi32", "int32_t" case "make_m256i_repeat_u64": fName, tName = "_mm256_set1_epi64x", "int64_t" - case "make_m256i_slice256": + case "make_m256i_slice256", "make_m256i_slice_u16lex16": fName, tName, ptr = "_mm256_lddqu_si256", "const __m256i*)(const void*", true case "make_m256i_zeroes": fName, tName = "_mm256_setzero_si256", "" @@ -690,19 +690,42 @@ func (g *gen) writeBuiltinCPUArchX86(b *buffer, recv *a.Expr, method t.ID, args // avoid a "expression result unused" compiler error. b.writes("(") } - switch methodStr { - case "store_slice64": - b.writes("_mm_storeu_si64((void*)(") - case "store_slice128": - b.writes("_mm_storeu_si128((__m128i*)(void*)(") + prefix, cast := "", "" + switch recv.MType().QID() { + case [2]t.ID{t.IDBase, t.IDX86M128I}: + switch methodStr { + case "store_slice64": + prefix = "_mm_storeu_si64((void*)(" + case "store_slice128": + prefix = "_mm_storeu_si128((__m128i*)(void*)(" + } + case [2]t.ID{t.IDBase, t.IDX86M256I}: + switch methodStr { + case "store_slice64": + prefix = "_mm_storeu_si64((void*)(" + cast = "_mm256_castsi256_si128(" + case "store_slice128": + prefix = "_mm_storeu_si128((__m128i*)(void*)(" + cast = "_mm256_castsi256_si128(" + case "store_slice256": + prefix = "_mm256_storeu_si256((__m256i*)(void*)(" + } } + if prefix == "" { + return fmt.Errorf("internal error: unsupported cpu_arch method %q", methodStr) + } + b.writes(prefix) if err := g.writeExprDotPtr(b, args[0].AsArg().Value(), false, depth); err != nil { return err } b.writes("), ") + b.writes(cast) if err := g.writeExpr(b, recv, false, depth); err != nil { return err } + if cast != "" { + b.writes(")") + } b.writes(")") if !sideEffectsOnly { b.writes(", wuffs_base__make_empty_struct())") diff --git a/lang/builtin/builtin.go b/lang/builtin/builtin.go index 59a44b35..1d537722 100644 --- a/lang/builtin/builtin.go +++ b/lang/builtin/builtin.go @@ -870,29 +870,57 @@ var funcsOther = [...]string{ "x86_avx2_utility.make_m256i_repeat_u64(a: u64) x86_m256i", "x86_avx2_utility.make_m256i_slice256(a: roslice base.u8) x86_m256i", + "x86_avx2_utility.make_m256i_slice_u16lex16(a: roslice base.u16) x86_m256i", "x86_avx2_utility.make_m256i_zeroes() x86_m256i", // ---- x86_m256i + "x86_m256i.store_slice64!(a: slice base.u8)", + "x86_m256i.store_slice128!(a: slice base.u8)", + "x86_m256i.store_slice256!(a: slice base.u8)", + // TODO: generate these methods automatically? "x86_m256i._mm256_add_epi16(b: x86_m256i) x86_m256i", "x86_m256i._mm256_add_epi32(b: x86_m256i) x86_m256i", "x86_m256i._mm256_add_epi64(b: x86_m256i) x86_m256i", "x86_m256i._mm256_add_epi8(b: x86_m256i) x86_m256i", + "x86_m256i._mm256_extract_epi64(index: u32) u64", "x86_m256i._mm256_extracti128_si256(imm8: u32) x86_m128i", "x86_m256i._mm256_madd_epi16(b: x86_m256i) x86_m256i", "x86_m256i._mm256_maddubs_epi16(b: x86_m256i) x86_m256i", + "x86_m256i._mm256_mullo_epi16(b: x86_m256i) x86_m256i", + "x86_m256i._mm256_packs_epi16(b: x86_m256i) x86_m256i", + "x86_m256i._mm256_packs_epi32(b: x86_m256i) x86_m256i", + "x86_m256i._mm256_permute2x128_si256(b: x86_m256i, imm8: u32) x86_m256i", + "x86_m256i._mm256_permute4x64_epi64(imm8: u32) x86_m256i", "x86_m256i._mm256_sad_epu8(b: x86_m256i) x86_m256i", + "x86_m256i._mm256_sign_epi16(b: x86_m256i) x86_m256i", "x86_m256i._mm256_slli_epi16(imm8: u32) x86_m256i", "x86_m256i._mm256_slli_epi32(imm8: u32) x86_m256i", "x86_m256i._mm256_slli_epi64(imm8: u32) x86_m256i", "x86_m256i._mm256_slli_si256(imm8: u32) x86_m256i", + "x86_m256i._mm256_srai_epi16(imm8: u32) x86_m256i", + "x86_m256i._mm256_srai_epi32(imm8: u32) x86_m256i", + "x86_m256i._mm256_srai_epi64(imm8: u32) x86_m256i", + "x86_m256i._mm256_srai_si256(imm8: u32) x86_m256i", "x86_m256i._mm256_srli_epi16(imm8: u32) x86_m256i", "x86_m256i._mm256_srli_epi32(imm8: u32) x86_m256i", "x86_m256i._mm256_srli_epi64(imm8: u32) x86_m256i", "x86_m256i._mm256_srli_si256(imm8: u32) x86_m256i", + "x86_m256i._mm256_sub_epi16(b: x86_m256i) x86_m256i", + "x86_m256i._mm256_sub_epi32(b: x86_m256i) x86_m256i", + "x86_m256i._mm256_sub_epi64(b: x86_m256i) x86_m256i", + "x86_m256i._mm256_sub_epi8(b: x86_m256i) x86_m256i", + "x86_m256i._mm256_unpackhi_epi16(b: x86_m256i) x86_m256i", + "x86_m256i._mm256_unpackhi_epi32(b: x86_m256i) x86_m256i", + "x86_m256i._mm256_unpackhi_epi64(b: x86_m256i) x86_m256i", + "x86_m256i._mm256_unpackhi_epi8(b: x86_m256i) x86_m256i", + "x86_m256i._mm256_unpacklo_epi16(b: x86_m256i) x86_m256i", + "x86_m256i._mm256_unpacklo_epi32(b: x86_m256i) x86_m256i", + "x86_m256i._mm256_unpacklo_epi64(b: x86_m256i) x86_m256i", + "x86_m256i._mm256_unpacklo_epi8(b: x86_m256i) x86_m256i", } var Interfaces = []string{ diff --git a/lang/check/bounds.go b/lang/check/bounds.go index 9a0983a3..cab64fdd 100644 --- a/lang/check/bounds.go +++ b/lang/check/bounds.go @@ -1282,6 +1282,10 @@ func (q *checker) bcheckExprCallSpecialCases(n *a.Expr, depth uint32) (bounds, e advance = thirtyTwo case strings.HasSuffix(s, "_slice512"): // 512 bits is 64 bytes. advance = sixtyFour + case strings.HasSuffix(s, "_slice_u16lex16"): // 16 u16 values is 32 bytes. + advance = sixteen + case strings.Contains(s, "_slice"): + return bounds{}, fmt.Errorf("check: internal error: unrecognized %s method", s) } } } diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index fdde6687..3f77881c 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -40054,6 +40054,357 @@ wuffs_jpeg__decoder__decode_idct_x86_avx2( wuffs_base__slice_u8 a_dst_buffer, uint64_t a_dst_stride, uint32_t a_q) { + __m256i v_k_0000 = {0}; + __m256i v_k_8080 = {0}; + __m256i v_k_0000_0002 = {0}; + __m256i v_k_0001_FFFF = {0}; + __m256i v_k_0400_0000 = {0}; + __m256i v_k_29CF_1151_D630_1151 = {0}; + __m256i v_k_E333_133E_ADFD_1051 = {0}; + __m256i v_k_E6DC_25A1_1925_25A1 = {0}; + __m256i v_k_ECC1_E333_EFB0_ADFD = {0}; + __m256i v_rows01 = {0}; + __m256i v_rows23 = {0}; + __m256i v_rows45 = {0}; + __m256i v_rows67 = {0}; + __m256i v_quants01 = {0}; + __m256i v_quants23 = {0}; + __m256i v_quants45 = {0}; + __m256i v_quants67 = {0}; + __m256i v_rows04 = {0}; + __m256i v_rows31 = {0}; + __m256i v_rows26 = {0}; + __m256i v_rows75 = {0}; + __m256i v_fp_rows62 = {0}; + __m256i v_fp_bq2662ad = {0}; + __m256i v_fp_bq2662eh = {0}; + __m256i v_fp_cb26ad = {0}; + __m256i v_fp_cb26eh = {0}; + __m256i v_fp_rows40pos = {0}; + __m256i v_fp_rows04neg = {0}; + __m256i v_fp_rows0pm4 = {0}; + __m256i v_fp_ccpmad = {0}; + __m256i v_fp_ccpmeh = {0}; + __m256i v_fp_cd01ad = {0}; + __m256i v_fp_cd01eh = {0}; + __m256i v_fp_cd32ad = {0}; + __m256i v_fp_cd32eh = {0}; + __m256i v_fp_sums7351 = {0}; + __m256i v_fp_sums5173 = {0}; + __m256i v_fp_ci73515173ad = {0}; + __m256i v_fp_ci73515173eh = {0}; + __m256i v_fp_cl7351ad = {0}; + __m256i v_fp_cl7351eh = {0}; + __m256i v_fp_rows13 = {0}; + __m256i v_fp_bq7153ad = {0}; + __m256i v_fp_bq7153eh = {0}; + __m256i v_fp_ck75ad = {0}; + __m256i v_fp_ck75eh = {0}; + __m256i v_fp_cl5173ad = {0}; + __m256i v_fp_cl5173eh = {0}; + __m256i v_fp_ck13ad = {0}; + __m256i v_fp_ck13eh = {0}; + __m256i v_intermediate01ad = {0}; + __m256i v_intermediate01eh = {0}; + __m256i v_intermediate01 = {0}; + __m256i v_intermediate32ad = {0}; + __m256i v_intermediate32eh = {0}; + __m256i v_intermediate32 = {0}; + __m256i v_intermediate45ad = {0}; + __m256i v_intermediate45eh = {0}; + __m256i v_intermediate45 = {0}; + __m256i v_intermediate76ad = {0}; + __m256i v_intermediate76eh = {0}; + __m256i v_intermediate76 = {0}; + __m256i v_ita0a1e0e1 = {0}; + __m256i v_ita2a3e2e3 = {0}; + __m256i v_ita4a5e4e5 = {0}; + __m256i v_ita6a7e6e7 = {0}; + __m256i v_ita0c0e0g0 = {0}; + __m256i v_ita1c1e1g1 = {0}; + __m256i v_ita4c4e4g4 = {0}; + __m256i v_ita5c5e5g5 = {0}; + __m256i v_ita0b0e0f0 = {0}; + __m256i v_ita4b4e4f4 = {0}; + __m256i v_itc0d0g0h0 = {0}; + __m256i v_itc4d4g4h4 = {0}; + __m256i v_intermediateae = {0}; + __m256i v_intermediatebf = {0}; + __m256i v_intermediatecg = {0}; + __m256i v_intermediatedh = {0}; + __m256i v_intermediatedb = {0}; + __m256i v_intermediatehf = {0}; + __m256i v_sp_cols62 = {0}; + __m256i v_sp_bq2662ad = {0}; + __m256i v_sp_bq2662eh = {0}; + __m256i v_sp_rb26ad = {0}; + __m256i v_sp_rb26eh = {0}; + __m256i v_sp_cols40pos = {0}; + __m256i v_sp_cols04neg = {0}; + __m256i v_sp_cols0pm4 = {0}; + __m256i v_sp_rcpmad = {0}; + __m256i v_sp_rcpmeh = {0}; + __m256i v_sp_rd01ad = {0}; + __m256i v_sp_rd01eh = {0}; + __m256i v_sp_rd32ad = {0}; + __m256i v_sp_rd32eh = {0}; + __m256i v_sp_sums7351 = {0}; + __m256i v_sp_sums5173 = {0}; + __m256i v_sp_ri73515173ad = {0}; + __m256i v_sp_ri73515173eh = {0}; + __m256i v_sp_rl7351ad = {0}; + __m256i v_sp_rl7351eh = {0}; + __m256i v_sp_cols13 = {0}; + __m256i v_sp_bq7153ad = {0}; + __m256i v_sp_bq7153eh = {0}; + __m256i v_sp_rk75ad = {0}; + __m256i v_sp_rk75eh = {0}; + __m256i v_sp_rl5173ad = {0}; + __m256i v_sp_rl5173eh = {0}; + __m256i v_sp_rk13ad = {0}; + __m256i v_sp_rk13eh = {0}; + __m256i v_final01ad = {0}; + __m256i v_final01eh = {0}; + __m256i v_final01 = {0}; + __m256i v_final32ad = {0}; + __m256i v_final32eh = {0}; + __m256i v_final32 = {0}; + __m256i v_final45ad = {0}; + __m256i v_final45eh = {0}; + __m256i v_final45 = {0}; + __m256i v_final76ad = {0}; + __m256i v_final76eh = {0}; + __m256i v_final76 = {0}; + __m256i v_fta0a1e0e1 = {0}; + __m256i v_fta2a3e2e3 = {0}; + __m256i v_fta4a5e4e5 = {0}; + __m256i v_fta6a7e6e7 = {0}; + __m256i v_fta0c0e0g0 = {0}; + __m256i v_fta1c1e1g1 = {0}; + __m256i v_fta4c4e4g4 = {0}; + __m256i v_fta5c5e5g5 = {0}; + __m256i v_fta0b0e0f0 = {0}; + __m256i v_ftc0d0g0h0 = {0}; + __m256i v_fta4b4e4f4 = {0}; + __m256i v_ftc4d4g4h4 = {0}; + __m256i v_finalae = {0}; + __m256i v_finalbf = {0}; + __m256i v_finalcg = {0}; + __m256i v_finaldh = {0}; + __m256i v_final0145 = {0}; + __m256i v_final2367 = {0}; + uint64_t v_final0 = 0; + uint64_t v_final1 = 0; + uint64_t v_final2 = 0; + uint64_t v_final3 = 0; + uint64_t v_final4 = 0; + uint64_t v_final5 = 0; + uint64_t v_final6 = 0; + uint64_t v_final7 = 0; + wuffs_base__slice_u8 v_remaining = {0}; + + if (8u > a_dst_stride) { + return wuffs_base__make_empty_struct(); + } + v_k_0000 = _mm256_set_epi16((int16_t)(0u), (int16_t)(0u), (int16_t)(0u), (int16_t)(0u), (int16_t)(0u), (int16_t)(0u), (int16_t)(0u), (int16_t)(0u), (int16_t)(0u), (int16_t)(0u), (int16_t)(0u), (int16_t)(0u), (int16_t)(0u), (int16_t)(0u), (int16_t)(0u), (int16_t)(0u)); + v_k_8080 = _mm256_set_epi16((int16_t)(32896u), (int16_t)(32896u), (int16_t)(32896u), (int16_t)(32896u), (int16_t)(32896u), (int16_t)(32896u), (int16_t)(32896u), (int16_t)(32896u), (int16_t)(32896u), (int16_t)(32896u), (int16_t)(32896u), (int16_t)(32896u), (int16_t)(32896u), (int16_t)(32896u), (int16_t)(32896u), (int16_t)(32896u)); + v_k_0000_0002 = _mm256_set_epi16((int16_t)(2u), (int16_t)(0u), (int16_t)(2u), (int16_t)(0u), (int16_t)(2u), (int16_t)(0u), (int16_t)(2u), (int16_t)(0u), (int16_t)(2u), (int16_t)(0u), (int16_t)(2u), (int16_t)(0u), (int16_t)(2u), (int16_t)(0u), (int16_t)(2u), (int16_t)(0u)); + v_k_0001_FFFF = _mm256_set_epi16((int16_t)(65535u), (int16_t)(65535u), (int16_t)(65535u), (int16_t)(65535u), (int16_t)(65535u), (int16_t)(65535u), (int16_t)(65535u), (int16_t)(65535u), (int16_t)(1u), (int16_t)(1u), (int16_t)(1u), (int16_t)(1u), (int16_t)(1u), (int16_t)(1u), (int16_t)(1u), (int16_t)(1u)); + v_k_0400_0000 = _mm256_set_epi16((int16_t)(0u), (int16_t)(1024u), (int16_t)(0u), (int16_t)(1024u), (int16_t)(0u), (int16_t)(1024u), (int16_t)(0u), (int16_t)(1024u), (int16_t)(0u), (int16_t)(1024u), (int16_t)(0u), (int16_t)(1024u), (int16_t)(0u), (int16_t)(1024u), (int16_t)(0u), (int16_t)(1024u)); + v_k_29CF_1151_D630_1151 = _mm256_set_epi16((int16_t)(4433u), (int16_t)(54832u), (int16_t)(4433u), (int16_t)(54832u), (int16_t)(4433u), (int16_t)(54832u), (int16_t)(4433u), (int16_t)(54832u), (int16_t)(4433u), (int16_t)(10703u), (int16_t)(4433u), (int16_t)(10703u), (int16_t)(4433u), (int16_t)(10703u), (int16_t)(4433u), (int16_t)(10703u)); + v_k_E333_133E_ADFD_1051 = _mm256_set_epi16((int16_t)(4177u), (int16_t)(44541u), (int16_t)(4177u), (int16_t)(44541u), (int16_t)(4177u), (int16_t)(44541u), (int16_t)(4177u), (int16_t)(44541u), (int16_t)(4926u), (int16_t)(58163u), (int16_t)(4926u), (int16_t)(58163u), (int16_t)(4926u), (int16_t)(58163u), (int16_t)(4926u), (int16_t)(58163u)); + v_k_E6DC_25A1_1925_25A1 = _mm256_set_epi16((int16_t)(9633u), (int16_t)(6437u), (int16_t)(9633u), (int16_t)(6437u), (int16_t)(9633u), (int16_t)(6437u), (int16_t)(9633u), (int16_t)(6437u), (int16_t)(9633u), (int16_t)(59100u), (int16_t)(9633u), (int16_t)(59100u), (int16_t)(9633u), (int16_t)(59100u), (int16_t)(9633u), (int16_t)(59100u)); + v_k_ECC1_E333_EFB0_ADFD = _mm256_set_epi16((int16_t)(44541u), (int16_t)(61360u), (int16_t)(44541u), (int16_t)(61360u), (int16_t)(44541u), (int16_t)(61360u), (int16_t)(44541u), (int16_t)(61360u), (int16_t)(58163u), (int16_t)(60609u), (int16_t)(58163u), (int16_t)(60609u), (int16_t)(58163u), (int16_t)(60609u), (int16_t)(58163u), (int16_t)(60609u)); + do { + v_rows01 = _mm256_lddqu_si256((const __m256i*)(const void*)(self->private_data.f_mcu_blocks[0u] + 0u)); + v_rows23 = _mm256_lddqu_si256((const __m256i*)(const void*)(self->private_data.f_mcu_blocks[0u] + 16u)); + v_rows45 = _mm256_lddqu_si256((const __m256i*)(const void*)(self->private_data.f_mcu_blocks[0u] + 32u)); + v_rows67 = _mm256_lddqu_si256((const __m256i*)(const void*)(self->private_data.f_mcu_blocks[0u] + 48u)); + v_quants01 = _mm256_lddqu_si256((const __m256i*)(const void*)(self->private_impl.f_quant_tables[a_q] + 0u)); + v_quants23 = _mm256_lddqu_si256((const __m256i*)(const void*)(self->private_impl.f_quant_tables[a_q] + 16u)); + v_quants45 = _mm256_lddqu_si256((const __m256i*)(const void*)(self->private_impl.f_quant_tables[a_q] + 32u)); + v_quants67 = _mm256_lddqu_si256((const __m256i*)(const void*)(self->private_impl.f_quant_tables[a_q] + 48u)); + v_rows01 = _mm256_mullo_epi16(v_rows01, v_quants01); + v_rows23 = _mm256_mullo_epi16(v_rows23, v_quants23); + v_rows45 = _mm256_mullo_epi16(v_rows45, v_quants45); + v_rows67 = _mm256_mullo_epi16(v_rows67, v_quants67); + v_rows04 = _mm256_permute2x128_si256(v_rows01, v_rows45, (int32_t)(32u)); + v_rows31 = _mm256_permute2x128_si256(v_rows23, v_rows01, (int32_t)(49u)); + v_rows26 = _mm256_permute2x128_si256(v_rows23, v_rows67, (int32_t)(32u)); + v_rows75 = _mm256_permute2x128_si256(v_rows67, v_rows45, (int32_t)(49u)); + v_fp_rows62 = _mm256_permute2x128_si256(v_rows26, v_rows26, (int32_t)(1u)); + v_fp_bq2662ad = _mm256_unpacklo_epi16(v_rows26, v_fp_rows62); + v_fp_bq2662eh = _mm256_unpackhi_epi16(v_rows26, v_fp_rows62); + v_fp_cb26ad = _mm256_madd_epi16(v_fp_bq2662ad, v_k_29CF_1151_D630_1151); + v_fp_cb26eh = _mm256_madd_epi16(v_fp_bq2662eh, v_k_29CF_1151_D630_1151); + v_fp_rows40pos = _mm256_permute2x128_si256(v_rows04, v_rows04, (int32_t)(1u)); + v_fp_rows04neg = _mm256_sign_epi16(v_rows04, v_k_0001_FFFF); + v_fp_rows0pm4 = _mm256_add_epi16(v_fp_rows40pos, v_fp_rows04neg); + v_fp_ccpmad = _mm256_srai_epi32(_mm256_unpacklo_epi16(v_k_0000, v_fp_rows0pm4), (int32_t)(3u)); + v_fp_ccpmeh = _mm256_srai_epi32(_mm256_unpackhi_epi16(v_k_0000, v_fp_rows0pm4), (int32_t)(3u)); + v_fp_cd01ad = _mm256_add_epi32(v_fp_ccpmad, v_fp_cb26ad); + v_fp_cd01eh = _mm256_add_epi32(v_fp_ccpmeh, v_fp_cb26eh); + v_fp_cd32ad = _mm256_sub_epi32(v_fp_ccpmad, v_fp_cb26ad); + v_fp_cd32eh = _mm256_sub_epi32(v_fp_ccpmeh, v_fp_cb26eh); + v_fp_sums7351 = _mm256_add_epi16(v_rows75, v_rows31); + v_fp_sums5173 = _mm256_permute2x128_si256(v_fp_sums7351, v_fp_sums7351, (int32_t)(1u)); + v_fp_ci73515173ad = _mm256_unpacklo_epi16(v_fp_sums7351, v_fp_sums5173); + v_fp_ci73515173eh = _mm256_unpackhi_epi16(v_fp_sums7351, v_fp_sums5173); + v_fp_cl7351ad = _mm256_madd_epi16(v_fp_ci73515173ad, v_k_E6DC_25A1_1925_25A1); + v_fp_cl7351eh = _mm256_madd_epi16(v_fp_ci73515173eh, v_k_E6DC_25A1_1925_25A1); + v_fp_rows13 = _mm256_permute2x128_si256(v_rows31, v_rows31, (int32_t)(1u)); + v_fp_bq7153ad = _mm256_unpacklo_epi16(v_rows75, v_fp_rows13); + v_fp_bq7153eh = _mm256_unpackhi_epi16(v_rows75, v_fp_rows13); + v_fp_ck75ad = _mm256_add_epi32(_mm256_madd_epi16(v_fp_bq7153ad, v_k_ECC1_E333_EFB0_ADFD), v_fp_cl7351ad); + v_fp_ck75eh = _mm256_add_epi32(_mm256_madd_epi16(v_fp_bq7153eh, v_k_ECC1_E333_EFB0_ADFD), v_fp_cl7351eh); + v_fp_cl5173ad = _mm256_permute2x128_si256(v_fp_cl7351ad, v_fp_cl7351ad, (int32_t)(1u)); + v_fp_cl5173eh = _mm256_permute2x128_si256(v_fp_cl7351eh, v_fp_cl7351eh, (int32_t)(1u)); + v_fp_ck13ad = _mm256_add_epi32(v_fp_cl5173ad, _mm256_madd_epi16(v_fp_bq7153ad, v_k_E333_133E_ADFD_1051)); + v_fp_ck13eh = _mm256_add_epi32(v_fp_cl5173eh, _mm256_madd_epi16(v_fp_bq7153eh, v_k_E333_133E_ADFD_1051)); + v_intermediate01ad = _mm256_srai_epi32(_mm256_add_epi32(_mm256_add_epi32(v_fp_cd01ad, v_fp_ck13ad), v_k_0400_0000), (int32_t)(11u)); + v_intermediate01eh = _mm256_srai_epi32(_mm256_add_epi32(_mm256_add_epi32(v_fp_cd01eh, v_fp_ck13eh), v_k_0400_0000), (int32_t)(11u)); + v_intermediate01 = _mm256_packs_epi32(v_intermediate01ad, v_intermediate01eh); + v_intermediate32ad = _mm256_srai_epi32(_mm256_add_epi32(_mm256_add_epi32(v_fp_cd32ad, v_fp_ck75ad), v_k_0400_0000), (int32_t)(11u)); + v_intermediate32eh = _mm256_srai_epi32(_mm256_add_epi32(_mm256_add_epi32(v_fp_cd32eh, v_fp_ck75eh), v_k_0400_0000), (int32_t)(11u)); + v_intermediate32 = _mm256_packs_epi32(v_intermediate32ad, v_intermediate32eh); + v_intermediate45ad = _mm256_srai_epi32(_mm256_add_epi32(_mm256_sub_epi32(v_fp_cd32ad, v_fp_ck75ad), v_k_0400_0000), (int32_t)(11u)); + v_intermediate45eh = _mm256_srai_epi32(_mm256_add_epi32(_mm256_sub_epi32(v_fp_cd32eh, v_fp_ck75eh), v_k_0400_0000), (int32_t)(11u)); + v_intermediate45 = _mm256_packs_epi32(v_intermediate45ad, v_intermediate45eh); + v_intermediate76ad = _mm256_srai_epi32(_mm256_add_epi32(_mm256_sub_epi32(v_fp_cd01ad, v_fp_ck13ad), v_k_0400_0000), (int32_t)(11u)); + v_intermediate76eh = _mm256_srai_epi32(_mm256_add_epi32(_mm256_sub_epi32(v_fp_cd01eh, v_fp_ck13eh), v_k_0400_0000), (int32_t)(11u)); + v_intermediate76 = _mm256_packs_epi32(v_intermediate76ad, v_intermediate76eh); + v_ita0a1e0e1 = _mm256_permute4x64_epi64(v_intermediate01, (int32_t)(216u)); + v_ita2a3e2e3 = _mm256_permute4x64_epi64(v_intermediate32, (int32_t)(114u)); + v_ita4a5e4e5 = _mm256_permute4x64_epi64(v_intermediate45, (int32_t)(216u)); + v_ita6a7e6e7 = _mm256_permute4x64_epi64(v_intermediate76, (int32_t)(114u)); + v_ita0c0e0g0 = _mm256_unpacklo_epi16(v_ita0a1e0e1, v_ita2a3e2e3); + v_ita1c1e1g1 = _mm256_unpackhi_epi16(v_ita0a1e0e1, v_ita2a3e2e3); + v_ita4c4e4g4 = _mm256_unpacklo_epi16(v_ita4a5e4e5, v_ita6a7e6e7); + v_ita5c5e5g5 = _mm256_unpackhi_epi16(v_ita4a5e4e5, v_ita6a7e6e7); + v_ita0b0e0f0 = _mm256_unpacklo_epi16(v_ita0c0e0g0, v_ita1c1e1g1); + v_itc0d0g0h0 = _mm256_unpackhi_epi16(v_ita0c0e0g0, v_ita1c1e1g1); + v_ita4b4e4f4 = _mm256_unpacklo_epi16(v_ita4c4e4g4, v_ita5c5e5g5); + v_itc4d4g4h4 = _mm256_unpackhi_epi16(v_ita4c4e4g4, v_ita5c5e5g5); + v_intermediateae = _mm256_unpacklo_epi64(v_ita0b0e0f0, v_ita4b4e4f4); + v_intermediatebf = _mm256_unpackhi_epi64(v_ita0b0e0f0, v_ita4b4e4f4); + v_intermediatecg = _mm256_unpacklo_epi64(v_itc0d0g0h0, v_itc4d4g4h4); + v_intermediatedh = _mm256_unpackhi_epi64(v_itc0d0g0h0, v_itc4d4g4h4); + } while (0); + v_intermediatedb = _mm256_permute2x128_si256(v_intermediatedh, v_intermediatebf, (int32_t)(32u)); + v_intermediatehf = _mm256_permute2x128_si256(v_intermediatedh, v_intermediatebf, (int32_t)(49u)); + v_sp_cols62 = _mm256_permute2x128_si256(v_intermediatecg, v_intermediatecg, (int32_t)(1u)); + v_sp_bq2662ad = _mm256_unpacklo_epi16(v_intermediatecg, v_sp_cols62); + v_sp_bq2662eh = _mm256_unpackhi_epi16(v_intermediatecg, v_sp_cols62); + v_sp_rb26ad = _mm256_madd_epi16(v_sp_bq2662ad, v_k_29CF_1151_D630_1151); + v_sp_rb26eh = _mm256_madd_epi16(v_sp_bq2662eh, v_k_29CF_1151_D630_1151); + v_sp_cols40pos = _mm256_permute2x128_si256(v_intermediateae, v_intermediateae, (int32_t)(1u)); + v_sp_cols04neg = _mm256_sign_epi16(v_intermediateae, v_k_0001_FFFF); + v_sp_cols0pm4 = _mm256_add_epi16(v_sp_cols40pos, v_sp_cols04neg); + v_sp_rcpmad = _mm256_srai_epi32(_mm256_unpacklo_epi16(v_k_0000, v_sp_cols0pm4), (int32_t)(3u)); + v_sp_rcpmeh = _mm256_srai_epi32(_mm256_unpackhi_epi16(v_k_0000, v_sp_cols0pm4), (int32_t)(3u)); + v_sp_rd01ad = _mm256_add_epi32(v_sp_rcpmad, v_sp_rb26ad); + v_sp_rd01eh = _mm256_add_epi32(v_sp_rcpmeh, v_sp_rb26eh); + v_sp_rd32ad = _mm256_sub_epi32(v_sp_rcpmad, v_sp_rb26ad); + v_sp_rd32eh = _mm256_sub_epi32(v_sp_rcpmeh, v_sp_rb26eh); + v_sp_sums7351 = _mm256_add_epi16(v_intermediatehf, v_intermediatedb); + v_sp_sums5173 = _mm256_permute2x128_si256(v_sp_sums7351, v_sp_sums7351, (int32_t)(1u)); + v_sp_ri73515173ad = _mm256_unpacklo_epi16(v_sp_sums7351, v_sp_sums5173); + v_sp_ri73515173eh = _mm256_unpackhi_epi16(v_sp_sums7351, v_sp_sums5173); + v_sp_rl7351ad = _mm256_madd_epi16(v_sp_ri73515173ad, v_k_E6DC_25A1_1925_25A1); + v_sp_rl7351eh = _mm256_madd_epi16(v_sp_ri73515173eh, v_k_E6DC_25A1_1925_25A1); + v_sp_cols13 = _mm256_permute2x128_si256(v_intermediatedb, v_intermediatedb, (int32_t)(1u)); + v_sp_bq7153ad = _mm256_unpacklo_epi16(v_intermediatehf, v_sp_cols13); + v_sp_bq7153eh = _mm256_unpackhi_epi16(v_intermediatehf, v_sp_cols13); + v_sp_rk75ad = _mm256_add_epi32(_mm256_madd_epi16(v_sp_bq7153ad, v_k_ECC1_E333_EFB0_ADFD), v_sp_rl7351ad); + v_sp_rk75eh = _mm256_add_epi32(_mm256_madd_epi16(v_sp_bq7153eh, v_k_ECC1_E333_EFB0_ADFD), v_sp_rl7351eh); + v_sp_rl5173ad = _mm256_permute2x128_si256(v_sp_rl7351ad, v_sp_rl7351ad, (int32_t)(1u)); + v_sp_rl5173eh = _mm256_permute2x128_si256(v_sp_rl7351eh, v_sp_rl7351eh, (int32_t)(1u)); + v_sp_rk13ad = _mm256_add_epi32(v_sp_rl5173ad, _mm256_madd_epi16(v_sp_bq7153ad, v_k_E333_133E_ADFD_1051)); + v_sp_rk13eh = _mm256_add_epi32(v_sp_rl5173eh, _mm256_madd_epi16(v_sp_bq7153eh, v_k_E333_133E_ADFD_1051)); + v_final01ad = _mm256_srai_epi32(_mm256_add_epi32(_mm256_add_epi32(v_sp_rd01ad, v_sp_rk13ad), v_k_0000_0002), (int32_t)(18u)); + v_final01eh = _mm256_srai_epi32(_mm256_add_epi32(_mm256_add_epi32(v_sp_rd01eh, v_sp_rk13eh), v_k_0000_0002), (int32_t)(18u)); + v_final01 = _mm256_packs_epi32(v_final01ad, v_final01eh); + v_final32ad = _mm256_srai_epi32(_mm256_add_epi32(_mm256_add_epi32(v_sp_rd32ad, v_sp_rk75ad), v_k_0000_0002), (int32_t)(18u)); + v_final32eh = _mm256_srai_epi32(_mm256_add_epi32(_mm256_add_epi32(v_sp_rd32eh, v_sp_rk75eh), v_k_0000_0002), (int32_t)(18u)); + v_final32 = _mm256_packs_epi32(v_final32ad, v_final32eh); + v_final45ad = _mm256_srai_epi32(_mm256_add_epi32(_mm256_sub_epi32(v_sp_rd32ad, v_sp_rk75ad), v_k_0000_0002), (int32_t)(18u)); + v_final45eh = _mm256_srai_epi32(_mm256_add_epi32(_mm256_sub_epi32(v_sp_rd32eh, v_sp_rk75eh), v_k_0000_0002), (int32_t)(18u)); + v_final45 = _mm256_packs_epi32(v_final45ad, v_final45eh); + v_final76ad = _mm256_srai_epi32(_mm256_add_epi32(_mm256_sub_epi32(v_sp_rd01ad, v_sp_rk13ad), v_k_0000_0002), (int32_t)(18u)); + v_final76eh = _mm256_srai_epi32(_mm256_add_epi32(_mm256_sub_epi32(v_sp_rd01eh, v_sp_rk13eh), v_k_0000_0002), (int32_t)(18u)); + v_final76 = _mm256_packs_epi32(v_final76ad, v_final76eh); + v_fta0a1e0e1 = _mm256_permute4x64_epi64(v_final01, (int32_t)(216u)); + v_fta2a3e2e3 = _mm256_permute4x64_epi64(v_final32, (int32_t)(114u)); + v_fta4a5e4e5 = _mm256_permute4x64_epi64(v_final45, (int32_t)(216u)); + v_fta6a7e6e7 = _mm256_permute4x64_epi64(v_final76, (int32_t)(114u)); + v_fta0c0e0g0 = _mm256_unpacklo_epi16(v_fta0a1e0e1, v_fta2a3e2e3); + v_fta1c1e1g1 = _mm256_unpackhi_epi16(v_fta0a1e0e1, v_fta2a3e2e3); + v_fta4c4e4g4 = _mm256_unpacklo_epi16(v_fta4a5e4e5, v_fta6a7e6e7); + v_fta5c5e5g5 = _mm256_unpackhi_epi16(v_fta4a5e4e5, v_fta6a7e6e7); + v_fta0b0e0f0 = _mm256_unpacklo_epi16(v_fta0c0e0g0, v_fta1c1e1g1); + v_ftc0d0g0h0 = _mm256_unpackhi_epi16(v_fta0c0e0g0, v_fta1c1e1g1); + v_fta4b4e4f4 = _mm256_unpacklo_epi16(v_fta4c4e4g4, v_fta5c5e5g5); + v_ftc4d4g4h4 = _mm256_unpackhi_epi16(v_fta4c4e4g4, v_fta5c5e5g5); + v_finalae = _mm256_unpacklo_epi64(v_fta0b0e0f0, v_fta4b4e4f4); + v_finalbf = _mm256_unpackhi_epi64(v_fta0b0e0f0, v_fta4b4e4f4); + v_finalcg = _mm256_unpacklo_epi64(v_ftc0d0g0h0, v_ftc4d4g4h4); + v_finaldh = _mm256_unpackhi_epi64(v_ftc0d0g0h0, v_ftc4d4g4h4); + v_final0145 = _mm256_add_epi8(_mm256_packs_epi16(v_finalae, v_finalbf), v_k_8080); + v_final2367 = _mm256_add_epi8(_mm256_packs_epi16(v_finalcg, v_finaldh), v_k_8080); + v_final0 = _mm256_extract_epi64(v_final0145, (int32_t)(0u)); + v_final1 = _mm256_extract_epi64(v_final0145, (int32_t)(1u)); + v_final2 = _mm256_extract_epi64(v_final2367, (int32_t)(0u)); + v_final3 = _mm256_extract_epi64(v_final2367, (int32_t)(1u)); + v_final4 = _mm256_extract_epi64(v_final0145, (int32_t)(2u)); + v_final5 = _mm256_extract_epi64(v_final0145, (int32_t)(3u)); + v_final6 = _mm256_extract_epi64(v_final2367, (int32_t)(2u)); + v_final7 = _mm256_extract_epi64(v_final2367, (int32_t)(3u)); + if (a_dst_stride > ((uint64_t)(a_dst_buffer.len))) { + return wuffs_base__make_empty_struct(); + } + v_remaining = wuffs_base__slice_u8__subslice_i(a_dst_buffer, a_dst_stride); + wuffs_base__poke_u64le__no_bounds_check(a_dst_buffer.ptr, v_final0); + a_dst_buffer = v_remaining; + if (a_dst_stride > ((uint64_t)(a_dst_buffer.len))) { + return wuffs_base__make_empty_struct(); + } + v_remaining = wuffs_base__slice_u8__subslice_i(a_dst_buffer, a_dst_stride); + wuffs_base__poke_u64le__no_bounds_check(a_dst_buffer.ptr, v_final1); + a_dst_buffer = v_remaining; + if (a_dst_stride > ((uint64_t)(a_dst_buffer.len))) { + return wuffs_base__make_empty_struct(); + } + v_remaining = wuffs_base__slice_u8__subslice_i(a_dst_buffer, a_dst_stride); + wuffs_base__poke_u64le__no_bounds_check(a_dst_buffer.ptr, v_final2); + a_dst_buffer = v_remaining; + if (a_dst_stride > ((uint64_t)(a_dst_buffer.len))) { + return wuffs_base__make_empty_struct(); + } + v_remaining = wuffs_base__slice_u8__subslice_i(a_dst_buffer, a_dst_stride); + wuffs_base__poke_u64le__no_bounds_check(a_dst_buffer.ptr, v_final3); + a_dst_buffer = v_remaining; + if (a_dst_stride > ((uint64_t)(a_dst_buffer.len))) { + return wuffs_base__make_empty_struct(); + } + v_remaining = wuffs_base__slice_u8__subslice_i(a_dst_buffer, a_dst_stride); + wuffs_base__poke_u64le__no_bounds_check(a_dst_buffer.ptr, v_final4); + a_dst_buffer = v_remaining; + if (a_dst_stride > ((uint64_t)(a_dst_buffer.len))) { + return wuffs_base__make_empty_struct(); + } + v_remaining = wuffs_base__slice_u8__subslice_i(a_dst_buffer, a_dst_stride); + wuffs_base__poke_u64le__no_bounds_check(a_dst_buffer.ptr, v_final5); + a_dst_buffer = v_remaining; + if (a_dst_stride > ((uint64_t)(a_dst_buffer.len))) { + return wuffs_base__make_empty_struct(); + } + v_remaining = wuffs_base__slice_u8__subslice_i(a_dst_buffer, a_dst_stride); + wuffs_base__poke_u64le__no_bounds_check(a_dst_buffer.ptr, v_final6); + a_dst_buffer = v_remaining; + if (8u > ((uint64_t)(a_dst_buffer.len))) { + return wuffs_base__make_empty_struct(); + } + wuffs_base__poke_u64le__no_bounds_check(a_dst_buffer.ptr, v_final7); return wuffs_base__make_empty_struct(); } #endif // defined(WUFFS_BASE__CPU_ARCH__X86_FAMILY) @@ -40392,6 +40743,11 @@ wuffs_jpeg__decoder__do_decode_image_config( iop_a_src += self->private_data.s_do_decode_image_config[0].scratch; self->private_impl.f_payload_length = 0u; } + self->private_impl.choosy_decode_idct = ( +#if defined(WUFFS_BASE__CPU_ARCH__X86_FAMILY) + wuffs_base__cpu_arch__have_x86_avx2() ? &wuffs_jpeg__decoder__decode_idct_x86_avx2 : +#endif + self->private_impl.choosy_decode_idct); self->private_impl.f_frame_config_io_position = wuffs_base__u64__sat_add((a_src ? a_src->meta.pos : 0), ((uint64_t)(iop_a_src - io0_a_src))); if (a_dst != NULL) { v_pixfmt = 536870920u; diff --git a/std/jpeg/decode_idct_x86_avx2.wuffs b/std/jpeg/decode_idct_x86_avx2.wuffs index 573d41e6..262e096f 100644 --- a/std/jpeg/decode_idct_x86_avx2.wuffs +++ b/std/jpeg/decode_idct_x86_avx2.wuffs @@ -15,5 +15,864 @@ pri func decoder.decode_idct_x86_avx2!(dst_buffer: slice base.u8, dst_stride: base.u64, q: base.u32[..= 3]), choose cpu_arch >= x86_avx2, { - // TODO. + var util : base.x86_avx2_utility + + var k_0000 : base.x86_m256i + var k_8080 : base.x86_m256i + + var k_0000_0002 : base.x86_m256i + var k_0001_FFFF : base.x86_m256i + var k_0400_0000 : base.x86_m256i + + var k_29CF_1151_D630_1151 : base.x86_m256i + var k_E333_133E_ADFD_1051 : base.x86_m256i + var k_E6DC_25A1_1925_25A1 : base.x86_m256i + var k_ECC1_E333_EFB0_ADFD : base.x86_m256i + + var rows01 : base.x86_m256i + var rows23 : base.x86_m256i + var rows45 : base.x86_m256i + var rows67 : base.x86_m256i + + var quants01 : base.x86_m256i + var quants23 : base.x86_m256i + var quants45 : base.x86_m256i + var quants67 : base.x86_m256i + + var rows04 : base.x86_m256i + var rows31 : base.x86_m256i + var rows26 : base.x86_m256i + var rows75 : base.x86_m256i + + // First pass variable names start with "fp_". + + var fp_rows62 : base.x86_m256i + var fp_bq2662ad : base.x86_m256i + var fp_bq2662eh : base.x86_m256i + var fp_cb26ad : base.x86_m256i + var fp_cb26eh : base.x86_m256i + + var fp_rows40pos : base.x86_m256i + var fp_rows04neg : base.x86_m256i + var fp_rows0pm4 : base.x86_m256i + var fp_ccpmad : base.x86_m256i + var fp_ccpmeh : base.x86_m256i + + var fp_cd01ad : base.x86_m256i + var fp_cd01eh : base.x86_m256i + var fp_cd32ad : base.x86_m256i + var fp_cd32eh : base.x86_m256i + + var fp_sums7351 : base.x86_m256i + var fp_sums5173 : base.x86_m256i + var fp_ci73515173ad : base.x86_m256i + var fp_ci73515173eh : base.x86_m256i + var fp_cl7351ad : base.x86_m256i + var fp_cl7351eh : base.x86_m256i + + var fp_rows13 : base.x86_m256i + var fp_bq7153ad : base.x86_m256i + var fp_bq7153eh : base.x86_m256i + var fp_ck75ad : base.x86_m256i + var fp_ck75eh : base.x86_m256i + + var fp_cl5173ad : base.x86_m256i + var fp_cl5173eh : base.x86_m256i + var fp_ck13ad : base.x86_m256i + var fp_ck13eh : base.x86_m256i + + // Intermediate variables between first and second pass. + + var intermediate01ad : base.x86_m256i + var intermediate01eh : base.x86_m256i + var intermediate01 : base.x86_m256i + var intermediate32ad : base.x86_m256i + var intermediate32eh : base.x86_m256i + var intermediate32 : base.x86_m256i + var intermediate45ad : base.x86_m256i + var intermediate45eh : base.x86_m256i + var intermediate45 : base.x86_m256i + var intermediate76ad : base.x86_m256i + var intermediate76eh : base.x86_m256i + var intermediate76 : base.x86_m256i + + var ita0a1e0e1 : base.x86_m256i + var ita2a3e2e3 : base.x86_m256i + var ita4a5e4e5 : base.x86_m256i + var ita6a7e6e7 : base.x86_m256i + + var ita0c0e0g0 : base.x86_m256i + var ita1c1e1g1 : base.x86_m256i + var ita4c4e4g4 : base.x86_m256i + var ita5c5e5g5 : base.x86_m256i + + var ita0b0e0f0 : base.x86_m256i + var ita4b4e4f4 : base.x86_m256i + var itc0d0g0h0 : base.x86_m256i + var itc4d4g4h4 : base.x86_m256i + + var intermediateae : base.x86_m256i + var intermediatebf : base.x86_m256i + var intermediatecg : base.x86_m256i + var intermediatedh : base.x86_m256i + + var intermediatedb : base.x86_m256i + var intermediatehf : base.x86_m256i + + // Second pass variable names start with "sp_". + + var sp_cols62 : base.x86_m256i + var sp_bq2662ad : base.x86_m256i + var sp_bq2662eh : base.x86_m256i + var sp_rb26ad : base.x86_m256i + var sp_rb26eh : base.x86_m256i + + var sp_cols40pos : base.x86_m256i + var sp_cols04neg : base.x86_m256i + var sp_cols0pm4 : base.x86_m256i + var sp_rcpmad : base.x86_m256i + var sp_rcpmeh : base.x86_m256i + + var sp_rd01ad : base.x86_m256i + var sp_rd01eh : base.x86_m256i + var sp_rd32ad : base.x86_m256i + var sp_rd32eh : base.x86_m256i + + var sp_sums7351 : base.x86_m256i + var sp_sums5173 : base.x86_m256i + var sp_ri73515173ad : base.x86_m256i + var sp_ri73515173eh : base.x86_m256i + var sp_rl7351ad : base.x86_m256i + var sp_rl7351eh : base.x86_m256i + + var sp_cols13 : base.x86_m256i + var sp_bq7153ad : base.x86_m256i + var sp_bq7153eh : base.x86_m256i + var sp_rk75ad : base.x86_m256i + var sp_rk75eh : base.x86_m256i + + var sp_rl5173ad : base.x86_m256i + var sp_rl5173eh : base.x86_m256i + var sp_rk13ad : base.x86_m256i + var sp_rk13eh : base.x86_m256i + + // Final variables. + + var final01ad : base.x86_m256i + var final01eh : base.x86_m256i + var final01 : base.x86_m256i + var final32ad : base.x86_m256i + var final32eh : base.x86_m256i + var final32 : base.x86_m256i + var final45ad : base.x86_m256i + var final45eh : base.x86_m256i + var final45 : base.x86_m256i + var final76ad : base.x86_m256i + var final76eh : base.x86_m256i + var final76 : base.x86_m256i + + var fta0a1e0e1 : base.x86_m256i + var fta2a3e2e3 : base.x86_m256i + var fta4a5e4e5 : base.x86_m256i + var fta6a7e6e7 : base.x86_m256i + var fta0c0e0g0 : base.x86_m256i + var fta1c1e1g1 : base.x86_m256i + var fta4c4e4g4 : base.x86_m256i + var fta5c5e5g5 : base.x86_m256i + var fta0b0e0f0 : base.x86_m256i + var ftc0d0g0h0 : base.x86_m256i + var fta4b4e4f4 : base.x86_m256i + var ftc4d4g4h4 : base.x86_m256i + var finalae : base.x86_m256i + var finalbf : base.x86_m256i + var finalcg : base.x86_m256i + var finaldh : base.x86_m256i + + var final0145 : base.x86_m256i + var final2367 : base.x86_m256i + var final0 : base.u64 + var final1 : base.u64 + var final2 : base.u64 + var final3 : base.u64 + var final4 : base.u64 + var final5 : base.u64 + var final6 : base.u64 + var final7 : base.u64 + + var remaining : slice base.u8 + + // ---- + + if 8 > args.dst_stride { + return nothing + } + + k_0000 = util.make_m256i_multiple_u16( + a00: 0x0000, a01: 0x0000, a02: 0x0000, a03: 0x0000, + a04: 0x0000, a05: 0x0000, a06: 0x0000, a07: 0x0000, + a08: 0x0000, a09: 0x0000, a10: 0x0000, a11: 0x0000, + a12: 0x0000, a13: 0x0000, a14: 0x0000, a15: 0x0000) + k_8080 = util.make_m256i_multiple_u16( + a00: 0x8080, a01: 0x8080, a02: 0x8080, a03: 0x8080, + a04: 0x8080, a05: 0x8080, a06: 0x8080, a07: 0x8080, + a08: 0x8080, a09: 0x8080, a10: 0x8080, a11: 0x8080, + a12: 0x8080, a13: 0x8080, a14: 0x8080, a15: 0x8080) + + k_0000_0002 = util.make_m256i_multiple_u16( + a00: 0x0000, a01: 0x0002, a02: 0x0000, a03: 0x0002, + a04: 0x0000, a05: 0x0002, a06: 0x0000, a07: 0x0002, + a08: 0x0000, a09: 0x0002, a10: 0x0000, a11: 0x0002, + a12: 0x0000, a13: 0x0002, a14: 0x0000, a15: 0x0002) + k_0001_FFFF = util.make_m256i_multiple_u16( + a00: 0x0001, a01: 0x0001, a02: 0x0001, a03: 0x0001, + a04: 0x0001, a05: 0x0001, a06: 0x0001, a07: 0x0001, + a08: 0xFFFF, a09: 0xFFFF, a10: 0xFFFF, a11: 0xFFFF, + a12: 0xFFFF, a13: 0xFFFF, a14: 0xFFFF, a15: 0xFFFF) + k_0400_0000 = util.make_m256i_multiple_u16( + a00: 0x0400, a01: 0x0000, a02: 0x0400, a03: 0x0000, + a04: 0x0400, a05: 0x0000, a06: 0x0400, a07: 0x0000, + a08: 0x0400, a09: 0x0000, a10: 0x0400, a11: 0x0000, + a12: 0x0400, a13: 0x0000, a14: 0x0400, a15: 0x0000) + + k_29CF_1151_D630_1151 = util.make_m256i_multiple_u16( + a00: 0x29CF, a01: 0x1151, a02: 0x29CF, a03: 0x1151, + a04: 0x29CF, a05: 0x1151, a06: 0x29CF, a07: 0x1151, + a08: 0xD630, a09: 0x1151, a10: 0xD630, a11: 0x1151, + a12: 0xD630, a13: 0x1151, a14: 0xD630, a15: 0x1151) + k_E333_133E_ADFD_1051 = util.make_m256i_multiple_u16( + a00: 0xE333, a01: 0x133E, a02: 0xE333, a03: 0x133E, + a04: 0xE333, a05: 0x133E, a06: 0xE333, a07: 0x133E, + a08: 0xADFD, a09: 0x1051, a10: 0xADFD, a11: 0x1051, + a12: 0xADFD, a13: 0x1051, a14: 0xADFD, a15: 0x1051) + k_E6DC_25A1_1925_25A1 = util.make_m256i_multiple_u16( + a00: 0xE6DC, a01: 0x25A1, a02: 0xE6DC, a03: 0x25A1, + a04: 0xE6DC, a05: 0x25A1, a06: 0xE6DC, a07: 0x25A1, + a08: 0x1925, a09: 0x25A1, a10: 0x1925, a11: 0x25A1, + a12: 0x1925, a13: 0x25A1, a14: 0x1925, a15: 0x25A1) + k_ECC1_E333_EFB0_ADFD = util.make_m256i_multiple_u16( + a00: 0xECC1, a01: 0xE333, a02: 0xECC1, a03: 0xE333, + a04: 0xECC1, a05: 0xE333, a06: 0xECC1, a07: 0xE333, + a08: 0xEFB0, a09: 0xADFD, a10: 0xEFB0, a11: 0xADFD, + a12: 0xEFB0, a13: 0xADFD, a14: 0xEFB0, a15: 0xADFD) + + // ---- + + while.goto_do_second_pass true, + inv 8 <= args.dst_stride, + {{ + + // TODO: faster code path (and break.goto_do_second_pass) if the AC terms + // are zero. + + // ---- + + // Set up. + + // Dequantize. Name the columns a ..= h and the rows 0 ..= 7 so we get: + // + // rows01 = i16x16 [a0 b0 .. g0 h0 + // a1 b1 .. g1 h1] + // rows23 = i16x16 [a2 b2 .. g2 e2 + // a3 b3 .. g3 h3] + // rows45 = i16x16 [a4 b4 .. g4 h4 + // a5 b5 .. g5 h5] + // rows67 = i16x16 [a6 b6 .. g6 h6 + // a7 b7 .. g7 h7] + rows01 = util.make_m256i_slice_u16lex16(a: this.mcu_blocks[0][0x00 .. 0x10]) + rows23 = util.make_m256i_slice_u16lex16(a: this.mcu_blocks[0][0x10 .. 0x20]) + rows45 = util.make_m256i_slice_u16lex16(a: this.mcu_blocks[0][0x20 .. 0x30]) + rows67 = util.make_m256i_slice_u16lex16(a: this.mcu_blocks[0][0x30 .. 0x40]) + quants01 = util.make_m256i_slice_u16lex16(a: this.quant_tables[args.q][0x00 .. 0x10]) + quants23 = util.make_m256i_slice_u16lex16(a: this.quant_tables[args.q][0x10 .. 0x20]) + quants45 = util.make_m256i_slice_u16lex16(a: this.quant_tables[args.q][0x20 .. 0x30]) + quants67 = util.make_m256i_slice_u16lex16(a: this.quant_tables[args.q][0x30 .. 0x40]) + rows01 = rows01._mm256_mullo_epi16(b: quants01) + rows23 = rows23._mm256_mullo_epi16(b: quants23) + rows45 = rows45._mm256_mullo_epi16(b: quants45) + rows67 = rows67._mm256_mullo_epi16(b: quants67) + + // Permute: + // + // rows04 = i16x16 [a0 b0 .. g0 h0 + // a4 b4 .. g4 h4] + // rows31 = i16x16 [a3 b3 .. g3 e3 + // a1 b1 .. g1 h1] + // rows26 = i16x16 [a2 b2 .. g2 h2 + // a6 b6 .. g6 h6] + // rows75 = i16x16 [a7 b7 .. g7 h7 + // a5 b5 .. g5 h5] + rows04 = rows01._mm256_permute2x128_si256(b: rows45, imm8: 0x20) + rows31 = rows23._mm256_permute2x128_si256(b: rows01, imm8: 0x31) + rows26 = rows23._mm256_permute2x128_si256(b: rows67, imm8: 0x20) + rows75 = rows67._mm256_permute2x128_si256(b: rows45, imm8: 0x31) + + // ---- + + // First pass (even rows). + + // This non-SIMD code: + // + // cb2 = (bq2 ~mod* 0x0000_29CF) ~mod+ (bq6 ~mod* 0x0000_1151) + // cb6 = (bq2 ~mod* 0x0000_1151) ~mod+ (bq6 ~mod* 0xFFFF_D630) + // + // becomes: + // + // fp_rows62 = i16x16 [a6 b6 .. g6 h6 + // a2 b2 .. g2 h2] + // fp_bq2662ad = i16x16 [a2 a6 .. d2 d6 + // a6 a2 .. d6 d2] + // fp_bq2662eh = i16x16 [e2 e6 .. h2 h6 + // e6 e2 .. h6 h2] + // fp_cb26ad = i32x8 [cb2.a .. cb2.d + // cb6.a .. cb6.d] + // = i32x8 [a2*0x29CF+a6*0x1151 .. d2*0x29CF+d6*0x1151 + // a6*0xD630+a2*0x1151 .. d6*0xD630+d2*0x1151] + // fp_cb26eh = i32x8 [cb2.e .. cb2.h + // cb6.e .. cb6.h] + // = i32x8 [e2*0x29CF+e6*0x1151 .. h2*0x29CF+h6*0x1151 + // e6*0xD630+e2*0x1151 .. h6*0xD630+h2*0x1151] + fp_rows62 = rows26._mm256_permute2x128_si256(b: rows26, imm8: 0x01) + fp_bq2662ad = rows26._mm256_unpacklo_epi16(b: fp_rows62) + fp_bq2662eh = rows26._mm256_unpackhi_epi16(b: fp_rows62) + fp_cb26ad = fp_bq2662ad._mm256_madd_epi16(b: k_29CF_1151_D630_1151) + fp_cb26eh = fp_bq2662eh._mm256_madd_epi16(b: k_29CF_1151_D630_1151) + + // This non-SIMD code: + // + // ccp = (bq0 ~mod+ bq4) ~mod<< 13 + // ccm = (bq0 ~mod- bq4) ~mod<< 13 + // + // becomes: + // + // fp_rows40pos = i16x16 [+a4 +b4 .. +g4 +h4 + // +a0 +b0 .. +g0 +h0] + // fp_rows04neg = i16x16 [+a0 +b0 .. +g0 +h0 + // -a4 -b4 .. -g4 -h4] + // fp_rows0pm4 = i16x16 [a0+a4 b0+b4 .. g0+g4 h0+h4 + // a0-a4 b0-b4 .. g0-g4 h0-h4] + // fp_ccpmad = i32x8 [ccp.a .. ccp.d + // ccm.a .. ccm.d] + // = i32x8 [(a0+a4)<<13 .. (d0+d4)<<13 + // (a0-a4)<<13 .. (d0-d4)<<13] + // fp_ccpmeh = i32x8 [ccp.e .. ccp.h + // ccm.e .. ccm.h] + // = i32x8 [(e0+e4)<<13 .. (h0+h4)<<13 + // (e0-e4)<<13 .. (h0-h4)<<13] + fp_rows40pos = rows04._mm256_permute2x128_si256(b: rows04, imm8: 0x01) + fp_rows04neg = rows04._mm256_sign_epi16(b: k_0001_FFFF) + fp_rows0pm4 = fp_rows40pos._mm256_add_epi16(b: fp_rows04neg) + fp_ccpmad = k_0000._mm256_unpacklo_epi16(b: fp_rows0pm4)._mm256_srai_epi32(imm8: 16 - 13) + fp_ccpmeh = k_0000._mm256_unpackhi_epi16(b: fp_rows0pm4)._mm256_srai_epi32(imm8: 16 - 13) + + // This non-SIMD code: + // + // cd0 = ccp ~mod+ cb2 + // cd1 = ccm ~mod+ cb6 + // cd2 = ccm ~mod- cb6 + // cd3 = ccp ~mod- cb2 + // + // becomes: + // + // fp_cd01ad = i32x8 [cd0.a .. cd0.d + // cd1.a .. cd1.d] + // = i32x8 [ccp.a+cb2.a .. ccp.d+cb2.d + // ccm.a+cb6.a .. ccm.d+cb6.d] + // fp_cd01eh = i32x8 [cd0.e .. cd0.h + // cd1.e .. cd1.h] + // = i32x8 [ccp.e+cb2.e .. ccp.h+cb2.h + // ccm.e+cb6.e .. ccm.h+cb6.h] + // fp_cd32ad = i32x8 [cd3.a .. cd3.d + // cd2.a .. cd2.d] + // = i32x8 [ccp.a-cb2.a .. ccp.d-cb2.d + // ccm.a-cb6.a .. ccm.d-cb6.d] + // fp_cd32eh = i32x8 [cd3.e .. cd3.h + // cd2.e .. cd2.h] + // = i32x8 [ccp.e-cb2.e .. ccp.h-cb2.h + // ccm.e-cb6.e .. ccm.h-cb6.h] + fp_cd01ad = fp_ccpmad._mm256_add_epi32(b: fp_cb26ad) + fp_cd01eh = fp_ccpmeh._mm256_add_epi32(b: fp_cb26eh) + fp_cd32ad = fp_ccpmad._mm256_sub_epi32(b: fp_cb26ad) + fp_cd32eh = fp_ccpmeh._mm256_sub_epi32(b: fp_cb26eh) + + // ---- + + // First pass (odd rows). + + // This non-SIMD code: + // + // ci73 = bq7 ~mod+ bq3 + // ci51 = bq5 ~mod+ bq1 + // cl73 = (ci73 ~mod* 0xFFFF_E6DC) ~mod+ (ci51 ~mod* 0x0000_25A1) + // cl51 = (ci73 ~mod* 0x0000_25A1) ~mod+ (ci51 ~mod* 0x0000_1925) + // + // becomes: + // + // fp_sums7351 = i16x16 [a7+a3 b7+b3 .. g7+g3 h7+h3 + // a5+a1 b5+b1 .. g5+g1 h5+h1] + // fp_sums5173 = i16x16 [a5+a1 b5+b1 .. g5+g1 h5+h1 + // a7+a3 b7+b3 .. g7+g3 h7+h3] + // fp_ci73515173ad = i16x16 [a7+a3 a5+a1 .. d7+d3 d5+d1 + // a5+a1 a7+a3 .. d5+d1 d7+d3] + // fp_ci73515173eh = i16x16 [e7+e3 e5+e1 .. h7+h3 h5+d1 + // e5+e1 e7+e3 .. h5+h1 h7+d3] + // fp_cl7351ad = i32x8 [cl73.a .. cl73.d + // cl51.a .. cl51.d] + // = i32x8 [(a7+a3)*0xE6DC+(a5+a1)*0x25A1 .. (d7+d3)*0xE6DC+(d5+d1)*0x25A1 + // (a5+a1)*0x1925+(a7+a3)*0x25A1 .. (d5+d1)*0x1925+(d7+d3)*0x25A1] + // fp_cl7351eh = i32x8 [cl73.e .. cl73.h + // cl51.e .. cl51.h] + // = i32x8 [(e7+e3)*0xE6DC+(e5+e1)*0x25A1 .. (h7+h3)*0xE6DC+(h5+h1)*0x25A1 + // (e5+e1)*0x1925+(e7+e3)*0x25A1 .. (h5+h1)*0x1925+(h7+h3)*0x25A1] + fp_sums7351 = rows75._mm256_add_epi16(b: rows31) + fp_sums5173 = fp_sums7351._mm256_permute2x128_si256(b: fp_sums7351, imm8: 0x01) + fp_ci73515173ad = fp_sums7351._mm256_unpacklo_epi16(b: fp_sums5173) + fp_ci73515173eh = fp_sums7351._mm256_unpackhi_epi16(b: fp_sums5173) + fp_cl7351ad = fp_ci73515173ad._mm256_madd_epi16(b: k_E6DC_25A1_1925_25A1) + fp_cl7351eh = fp_ci73515173eh._mm256_madd_epi16(b: k_E6DC_25A1_1925_25A1) + + // This non-SIMD code: + // + // ck5 = cl51 ~mod+ ((bq3 ~mod* 0xFFFF_ADFD) ~mod+ (bq5 ~mod* 0xFFFF_EFB0)) + // ck7 = cl73 ~mod+ ((bq1 ~mod* 0xFFFF_E333) ~mod+ (bq7 ~mod* 0xFFFF_ECC1)) + // + // becomes: + // + // fp_rows13 = i16x16 [a1 b1 .. g1 e1 a3 b3 .. g3 h3] + // fp_bq7153ad = i16x16 [a7 a1 .. d7 d1 a5 a3 .. d5 d3] + // fp_bq7153eh = i16x16 [e7 e1 .. h7 h1 e5 e3 .. h5 h3] + // fp_ck75ad = i32x8 [ck7.a .. ck7.d + // ck5.a .. ck5.d] + // = i32x8 [cl73.a+a7*0xECC1+a1*0xE333 .. cl73.d+d7*0xECC1+d1*0xE333 + // cl51.a+a5*0xEFB0+a3*0xADFD .. cl51.d+d5*0xEFB0+d3*0xADFD] + // fp_ck75eh = i32x8 [ck7.e .. ck7.h + // ck5.e .. ck5.h] + // = i32x8 [cl73.e+e7*0xECC1+e1*0xE333 .. cl73.h+h7*0xECC1+h1*0xE333 + // cl51.e+e5*0xEFB0+e3*0xADFD .. cl51.h+h5*0xEFB0+h3*0xADFD] + fp_rows13 = rows31._mm256_permute2x128_si256(b: rows31, imm8: 0x01) + fp_bq7153ad = rows75._mm256_unpacklo_epi16(b: fp_rows13) + fp_bq7153eh = rows75._mm256_unpackhi_epi16(b: fp_rows13) + fp_ck75ad = fp_bq7153ad._mm256_madd_epi16(b: k_ECC1_E333_EFB0_ADFD)._mm256_add_epi32(b: fp_cl7351ad) + fp_ck75eh = fp_bq7153eh._mm256_madd_epi16(b: k_ECC1_E333_EFB0_ADFD)._mm256_add_epi32(b: fp_cl7351eh) + + // This non-SIMD code: + // + // ck1 = cl51 ~mod+ ((bq1 ~mod* 0x0000_133E) ~mod+ (bq7 ~mod* 0xFFFF_E333)) + // ck3 = cl73 ~mod+ ((bq3 ~mod* 0x0000_1051) ~mod+ (bq5 ~mod* 0xFFFF_ADFD)) + // + // becomes: + // + // fp_cl5173ad = i32x8 [cl51.a .. cl51.d + // cl73.a .. cl73.d] + // fp_cl5173eh = i32x8 [cl51.e .. cl51.h + // cl73.e .. cl73.h] + // fp_ck13ad = i32x8 [ck1.a .. ck1.d + // ck3.a .. ck3.d] + // = i32x8 [cl51.a+a7*0xE333+a1*0x133E .. cl51.d+d7*0xE333+d1*0x133E + // cl73.a+a5*0xADFD+a3*0x1051 .. cl73.d+d5*0xADFD+d3*0x1051] + // fp_ck13eh = i32x8 [ck1.e .. ck3.h + // ck3.e .. ck1.h] + // = i32x8 [cl51.e+e7*0xE333+e1*0x133E .. cl51.h+h7*0xE333+h1*0x133E + // cl73.e+e5*0xADFD+e3*0x1051 .. cl73.h+h5*0xADFD+h3*0x1051] + fp_cl5173ad = fp_cl7351ad._mm256_permute2x128_si256(b: fp_cl7351ad, imm8: 0x01) + fp_cl5173eh = fp_cl7351eh._mm256_permute2x128_si256(b: fp_cl7351eh, imm8: 0x01) + fp_ck13ad = fp_cl5173ad._mm256_add_epi32(b: fp_bq7153ad._mm256_madd_epi16(b: k_E333_133E_ADFD_1051)) + fp_ck13eh = fp_cl5173eh._mm256_add_epi32(b: fp_bq7153eh._mm256_madd_epi16(b: k_E333_133E_ADFD_1051)) + + // ---- + + // First pass (combine rows). + + // We have now calculated: + // + // fp_ck13ad = i32x8 [ck1.a .. ck1.d ck3.a .. ck3.d] + // fp_ck13eh = i32x8 [ck1.e .. ck1.h ck3.e .. ck3.h] + // fp_ck75ad = i32x8 [ck7.a .. ck7.d ck5.a .. ck5.d] + // fp_ck75eh = i32x8 [ck7.e .. ck7.h ck5.e .. ck5.h] + // fp_cd01ad = i32x8 [cd0.a .. cd0.d cd1.a .. cd1.d] + // fp_cd01eh = i32x8 [cd0.e .. cd0.h cd1.e .. cd1.h] + // fp_cd32ad = i32x8 [cd3.a .. cd3.d cd2.a .. cd2.d] + // fp_cd32eh = i32x8 [cd3.e .. cd3.h cd2.e .. cd2.h] + // + // This non-SIMD code: + // + // intermediate0 = this.util.sign_extend_rshift_u32(a: (cd0 ~mod+ ck1) ~mod+ (1 << 10), n: 11) + // intermediate1 = this.util.sign_extend_rshift_u32(a: (cd1 ~mod+ ck3) ~mod+ (1 << 10), n: 11) + // intermediate2 = this.util.sign_extend_rshift_u32(a: (cd2 ~mod+ ck5) ~mod+ (1 << 10), n: 11) + // intermediate3 = this.util.sign_extend_rshift_u32(a: (cd3 ~mod+ ck7) ~mod+ (1 << 10), n: 11) + // intermediate4 = this.util.sign_extend_rshift_u32(a: (cd3 ~mod- ck7) ~mod+ (1 << 10), n: 11) + // intermediate5 = this.util.sign_extend_rshift_u32(a: (cd2 ~mod- ck5) ~mod+ (1 << 10), n: 11) + // intermediate6 = this.util.sign_extend_rshift_u32(a: (cd1 ~mod- ck3) ~mod+ (1 << 10), n: 11) + // intermediate7 = this.util.sign_extend_rshift_u32(a: (cd0 ~mod- ck1) ~mod+ (1 << 10), n: 11) + // + // becomes a mix of adds, subtracts, shifts and packs. + intermediate01ad = fp_cd01ad._mm256_add_epi32(b: fp_ck13ad)._mm256_add_epi32(b: k_0400_0000)._mm256_srai_epi32(imm8: 11) + intermediate01eh = fp_cd01eh._mm256_add_epi32(b: fp_ck13eh)._mm256_add_epi32(b: k_0400_0000)._mm256_srai_epi32(imm8: 11) + intermediate01 = intermediate01ad._mm256_packs_epi32(b: intermediate01eh) + intermediate32ad = fp_cd32ad._mm256_add_epi32(b: fp_ck75ad)._mm256_add_epi32(b: k_0400_0000)._mm256_srai_epi32(imm8: 11) + intermediate32eh = fp_cd32eh._mm256_add_epi32(b: fp_ck75eh)._mm256_add_epi32(b: k_0400_0000)._mm256_srai_epi32(imm8: 11) + intermediate32 = intermediate32ad._mm256_packs_epi32(b: intermediate32eh) + intermediate45ad = fp_cd32ad._mm256_sub_epi32(b: fp_ck75ad)._mm256_add_epi32(b: k_0400_0000)._mm256_srai_epi32(imm8: 11) + intermediate45eh = fp_cd32eh._mm256_sub_epi32(b: fp_ck75eh)._mm256_add_epi32(b: k_0400_0000)._mm256_srai_epi32(imm8: 11) + intermediate45 = intermediate45ad._mm256_packs_epi32(b: intermediate45eh) + intermediate76ad = fp_cd01ad._mm256_sub_epi32(b: fp_ck13ad)._mm256_add_epi32(b: k_0400_0000)._mm256_srai_epi32(imm8: 11) + intermediate76eh = fp_cd01eh._mm256_sub_epi32(b: fp_ck13eh)._mm256_add_epi32(b: k_0400_0000)._mm256_srai_epi32(imm8: 11) + intermediate76 = intermediate76ad._mm256_packs_epi32(b: intermediate76eh) + + // ---- + + // Transpose/re-order the intermediate rows and columns. We start with: + // + // intermediate01 = i16x16 [in.a0 .. in.d0 in.e0 .. in.h0 in.a1 .. in.d1 in.e1 .. in.h1] + // intermediate32 = i16x16 [in.a3 .. in.d3 in.e3 .. in.h3 in.a2 .. in.d2 in.e2 .. in.h2] + // intermediate45 = i16x16 [in.a4 .. in.d4 in.e4 .. in.h4 in.a5 .. in.d5 in.e5 .. in.h5] + // intermediate76 = i16x16 [in.a7 .. in.d7 in.e7 .. in.h7 in.a6 .. in.d6 in.e6 .. in.h6] + + ita0a1e0e1 = intermediate01._mm256_permute4x64_epi64(imm8: 0xD8) + ita2a3e2e3 = intermediate32._mm256_permute4x64_epi64(imm8: 0x72) + ita4a5e4e5 = intermediate45._mm256_permute4x64_epi64(imm8: 0xD8) + ita6a7e6e7 = intermediate76._mm256_permute4x64_epi64(imm8: 0x72) + + // We now have: + // + // ita0a1e0e1 = i16x16 [in.a0 .. in.d0 in.a1 .. in.d1 in.e0 .. in.h0 in.e1 .. in.h1] + // ita2a3e2e3 = i16x16 [in.a2 .. in.d2 in.a3 .. in.d3 in.e2 .. in.h2 in.e3 .. in.h3] + // ita4a5e4e5 = i16x16 [in.a4 .. in.d4 in.a5 .. in.d5 in.e4 .. in.h4 in.e5 .. in.h5] + // ita6a7e6e7 = i16x16 [in.a6 .. in.d6 in.a7 .. in.d7 in.e6 .. in.h6 in.e7 .. in.h7] + + ita0c0e0g0 = ita0a1e0e1._mm256_unpacklo_epi16(b: ita2a3e2e3) + ita1c1e1g1 = ita0a1e0e1._mm256_unpackhi_epi16(b: ita2a3e2e3) + ita4c4e4g4 = ita4a5e4e5._mm256_unpacklo_epi16(b: ita6a7e6e7) + ita5c5e5g5 = ita4a5e4e5._mm256_unpackhi_epi16(b: ita6a7e6e7) + + // We now have: + // + // ita0c0e0g0 = i16x16 [in.a0 in.a2 .. in.d0 in.d2 in.e0 in.e2 .. in.h0 in.h2] + // ita1c1e1g1 = i16x16 [in.a1 in.a3 .. in.d1 in.d3 in.e1 in.e3 .. in.h1 in.h3] + // ita4c4e4g4 = i16x16 [in.a4 in.a6 .. in.d4 in.d6 in.e4 in.e6 .. in.h4 in.h6] + // ita5c5e5g5 = i16x16 [in.a5 in.a7 .. in.d5 in.d7 in.e5 in.e7 .. in.h5 in.h7] + + ita0b0e0f0 = ita0c0e0g0._mm256_unpacklo_epi16(b: ita1c1e1g1) + itc0d0g0h0 = ita0c0e0g0._mm256_unpackhi_epi16(b: ita1c1e1g1) + ita4b4e4f4 = ita4c4e4g4._mm256_unpacklo_epi16(b: ita5c5e5g5) + itc4d4g4h4 = ita4c4e4g4._mm256_unpackhi_epi16(b: ita5c5e5g5) + + // We now have: + // + // ita0b0e0f0 = i16x16 [in.a0 .. in.a3 in.b0 .. in.b3 in.e0 .. in.e3 in.f0 .. in.f3] + // itc0d0g0h0 = i16x16 [in.c0 .. in.c3 in.d0 .. in.d3 in.g0 .. in.g3 in.h0 .. in.h3] + // ita4b4e4f4 = i16x16 [in.a4 .. in.a7 in.b4 .. in.b7 in.e4 .. in.e7 in.f4 .. in.f7] + // itc4d4g4h4 = i16x16 [in.c4 .. in.c7 in.d4 .. in.d7 in.g4 .. in.g7 in.h4 .. in.h7] + + intermediateae = ita0b0e0f0._mm256_unpacklo_epi64(b: ita4b4e4f4) + intermediatebf = ita0b0e0f0._mm256_unpackhi_epi64(b: ita4b4e4f4) + intermediatecg = itc0d0g0h0._mm256_unpacklo_epi64(b: itc4d4g4h4) + intermediatedh = itc0d0g0h0._mm256_unpackhi_epi64(b: itc4d4g4h4) + + // We now have: + // + // intermediateae = i16x16 [in.a0 .. in.a3 in.a4 .. in.a7 in.e0 .. in.e3 in.e4 .. in.e7] + // intermediatebf = i16x16 [in.b0 .. in.b3 in.b4 .. in.b7 in.f0 .. in.f4 in.f5 .. in.f7] + // intermediatecg = i16x16 [in.c0 .. in.c3 in.c4 .. in.c7 in.g0 .. in.g3 in.g4 .. in.g7] + // intermediatedh = i16x16 [in.d0 .. in.d3 in.d4 .. in.d7 in.h0 .. in.h4 in.h5 .. in.h7] + + break.goto_do_second_pass + }} endwhile.goto_do_second_pass + + // ---- + + // To recap, we have: + // + // intermediateae = i16x16 [in.a0 in.a1 .. in.a6 in.a7 + // in.e0 in.e1 .. in.e6 in.e7] + // intermediatebf = i16x16 [in.b0 in.b1 .. in.b6 in.b7 + // in.f0 in.f1 .. in.f6 in.f7] + // intermediatecg = i16x16 [in.c0 in.c1 .. in.c6 in.c7 + // in.g0 in.g1 .. in.g6 in.g7] + // intermediatedh = i16x16 [in.d0 in.d1 .. in.d6 in.d7 + // in.h0 in.h1 .. in.h6 in.h7] + // + // Which is similar to this, but note the transposition and re-ordering: + // + // rows01 = i16x16 [a0 b0 .. g0 h0 + // a1 b1 .. g1 h1] + // rows23 = i16x16 [a2 b2 .. g2 e2 + // a3 b3 .. g3 h3] + // rows45 = i16x16 [a4 b4 .. g4 h4 + // a5 b5 .. g5 h5] + // rows67 = i16x16 [a6 b6 .. g6 h6 + // a7 b7 .. g7 h7] + + // Permute: + // + // intermediatedb = i16x16 [in.d0 in.d1 .. in.d6 in.d7 + // in.b0 in.b1 .. in.b6 in.b7] + // intermediatehf = i16x16 [in.h0 in.h1 .. in.h6 in.h7 + // in.f0 in.f1 .. in.f6 in.f7] + // + // The row/column analogies are: + // + // [a0 .. h0 a4 .. h4] = rows04 <=> intermediateae = [in.a0 .. in.a7 in.e0 .. in.e7] + // [a3 .. h3 a1 .. h1] = rows31 <=> intermediatedb = [in.d0 .. in.d7 in.b0 .. in.b7] + // [a2 .. h2 a6 .. h6] = rows26 <=> intermediatecg = [in.c0 .. in.c7 in.g0 .. in.g7] + // [a7 .. h7 a5 .. h5] = rows75 <=> intermediatehf = [in.h0 .. in.h7 in.f0 .. in.f7] + intermediatedb = intermediatedh._mm256_permute2x128_si256(b: intermediatebf, imm8: 0x20) + intermediatehf = intermediatedh._mm256_permute2x128_si256(b: intermediatebf, imm8: 0x31) + + // ---- + + // Second pass (even columns). + + // This non-SIMD code: + // + // rb2 = (in2 ~mod* 0x0000_29CF) ~mod+ (in6 ~mod* 0x0000_1151) + // rb6 = (in2 ~mod* 0x0000_1151) ~mod+ (in6 ~mod* 0xFFFF_D630) + // + // becomes: + // + // sp_rb26ad = i32x8 [rb2.a .. rb2.d rb6.a .. rb6.d] transposed + // sp_rb26eh = i32x8 [rb2.e .. rb2.h rb6.e .. rb6.h] transposed + // + // 'Tranposed' means that the rows and column names should be swapped. + // rb2.a, rb6.h, etc should really be rbc.0, rbg.7, etc but the former + // notation lines up closer with the first pass' comments. + sp_cols62 = intermediatecg._mm256_permute2x128_si256(b: intermediatecg, imm8: 0x01) + sp_bq2662ad = intermediatecg._mm256_unpacklo_epi16(b: sp_cols62) + sp_bq2662eh = intermediatecg._mm256_unpackhi_epi16(b: sp_cols62) + sp_rb26ad = sp_bq2662ad._mm256_madd_epi16(b: k_29CF_1151_D630_1151) + sp_rb26eh = sp_bq2662eh._mm256_madd_epi16(b: k_29CF_1151_D630_1151) + + // This non-SIMD code: + // + // rcp = (in0 ~mod+ in4) ~mod<< 13 + // rcm = (in0 ~mod- in4) ~mod<< 13 + // + // becomes: + // + // sp_rcpmad = i32x8 [rcp.a .. rcp.d rcm.a .. rcm.d] transposed + // sp_rcpmeh = i32x8 [rcp.e .. rcp.h rcm.e .. rcm.h] transposed + sp_cols40pos = intermediateae._mm256_permute2x128_si256(b: intermediateae, imm8: 0x01) + sp_cols04neg = intermediateae._mm256_sign_epi16(b: k_0001_FFFF) + sp_cols0pm4 = sp_cols40pos._mm256_add_epi16(b: sp_cols04neg) + sp_rcpmad = k_0000._mm256_unpacklo_epi16(b: sp_cols0pm4)._mm256_srai_epi32(imm8: 16 - 13) + sp_rcpmeh = k_0000._mm256_unpackhi_epi16(b: sp_cols0pm4)._mm256_srai_epi32(imm8: 16 - 13) + + // This non-SIMD code: + // + // rd0 = rcp ~mod+ rb2 + // rd1 = rcm ~mod+ rb6 + // rd2 = rcm ~mod- rb6 + // rd3 = rcp ~mod- rb2 + // + // becomes: + // + // sp_rd01ad = i32x8 [rd0.a .. rd0.d rd1.a .. rd1.d] transposed + // sp_rd01eh = i32x8 [rd0.e .. rd0.h rd1.e .. rd1.h] transposed + // sp_rd32ad = i32x8 [rd3.a .. rd3.d rd2.a .. rd2.d] transposed + // sp_rd32eh = i32x8 [rd3.e .. rd3.h rd2.e .. rd2.h] transposed + sp_rd01ad = sp_rcpmad._mm256_add_epi32(b: sp_rb26ad) + sp_rd01eh = sp_rcpmeh._mm256_add_epi32(b: sp_rb26eh) + sp_rd32ad = sp_rcpmad._mm256_sub_epi32(b: sp_rb26ad) + sp_rd32eh = sp_rcpmeh._mm256_sub_epi32(b: sp_rb26eh) + + // ---- + + // Second pass (odd columns). + + // This non-SIMD code: + // + // ri73 = in7 ~mod+ in3 + // ri51 = in5 ~mod+ in1 + // rl73 = (ri73 ~mod* 0xFFFF_E6DC) ~mod+ (ri51 ~mod* 0x0000_25A1) + // rl51 = (ri73 ~mod* 0x0000_25A1) ~mod+ (ri51 ~mod* 0x0000_1925) + // + // becomes: + // + // sp_rl7351ad = i32x8 [rl73.a .. rl73.d rl51.a .. rl51.d] transposed + // sp_rl7351eh = i32x8 [rl73.e .. rl73.h rl51.e .. rl51.h] transposed + sp_sums7351 = intermediatehf._mm256_add_epi16(b: intermediatedb) + sp_sums5173 = sp_sums7351._mm256_permute2x128_si256(b: sp_sums7351, imm8: 0x01) + sp_ri73515173ad = sp_sums7351._mm256_unpacklo_epi16(b: sp_sums5173) + sp_ri73515173eh = sp_sums7351._mm256_unpackhi_epi16(b: sp_sums5173) + sp_rl7351ad = sp_ri73515173ad._mm256_madd_epi16(b: k_E6DC_25A1_1925_25A1) + sp_rl7351eh = sp_ri73515173eh._mm256_madd_epi16(b: k_E6DC_25A1_1925_25A1) + + // This non-SIMD code: + // + // rk5 = rl51 ~mod+ ((in3 ~mod* 0xFFFF_ADFD) ~mod+ (in5 ~mod* 0xFFFF_EFB0)) + // rk7 = rl73 ~mod+ ((in1 ~mod* 0xFFFF_E333) ~mod+ (in7 ~mod* 0xFFFF_ECC1)) + // + // becomes: + // + // sp_rk75ad = i32x8 [rk7.a .. rk7.d rk5.a .. rk5.d] transposed + // sp_rk75eh = i32x8 [rk7.e .. rk7.h rk5.e .. rk5.h] transposed + sp_cols13 = intermediatedb._mm256_permute2x128_si256(b: intermediatedb, imm8: 0x01) + sp_bq7153ad = intermediatehf._mm256_unpacklo_epi16(b: sp_cols13) + sp_bq7153eh = intermediatehf._mm256_unpackhi_epi16(b: sp_cols13) + sp_rk75ad = sp_bq7153ad._mm256_madd_epi16(b: k_ECC1_E333_EFB0_ADFD)._mm256_add_epi32(b: sp_rl7351ad) + sp_rk75eh = sp_bq7153eh._mm256_madd_epi16(b: k_ECC1_E333_EFB0_ADFD)._mm256_add_epi32(b: sp_rl7351eh) + + // This non-SIMD code: + // + // rk1 = rl51 ~mod+ ((in1 ~mod* 0x0000_133E) ~mod+ (in7 ~mod* 0xFFFF_E333)) + // rk3 = rl73 ~mod+ ((in3 ~mod* 0x0000_1051) ~mod+ (in5 ~mod* 0xFFFF_ADFD)) + // + // becomes: + // + // sp_rk13ad = i32x8 [rk1.a .. rk1.d rk3.a .. rk3.d] transposed + // sp_rk13eh = i32x8 [rk1.e .. rk3.h rk3.e .. rk1.h] transposed + sp_rl5173ad = sp_rl7351ad._mm256_permute2x128_si256(b: sp_rl7351ad, imm8: 0x01) + sp_rl5173eh = sp_rl7351eh._mm256_permute2x128_si256(b: sp_rl7351eh, imm8: 0x01) + sp_rk13ad = sp_rl5173ad._mm256_add_epi32(b: sp_bq7153ad._mm256_madd_epi16(b: k_E333_133E_ADFD_1051)) + sp_rk13eh = sp_rl5173eh._mm256_add_epi32(b: sp_bq7153eh._mm256_madd_epi16(b: k_E333_133E_ADFD_1051)) + + // ---- + + // Second pass (combine columns). + + // We have now calculated: + // + // sp_rk13ad = i32x8 [rk1.a .. rk1.d rk3.a .. rk3.d] transposed + // sp_rk13eh = i32x8 [rk1.e .. rk1.h rk3.e .. rk3.h] transposed + // sp_rk75ad = i32x8 [rk7.a .. rk7.d rk5.a .. rk5.d] transposed + // sp_rk75eh = i32x8 [rk7.e .. rk7.h rk5.e .. rk5.h] transposed + // sp_rd01ad = i32x8 [rd0.a .. rd0.d rd1.a .. rd1.d] transposed + // sp_rd01eh = i32x8 [rd0.e .. rd0.h rd1.e .. rd1.h] transposed + // sp_rd32ad = i32x8 [rd3.a .. rd3.d rd2.a .. rd2.d] transposed + // sp_rd32eh = i32x8 [rd3.e .. rd3.h rd2.e .. rd2.h] transposed + // + // This non-SIMD code: + // + // final0 = this.util.sign_extend_rshift_u32(a: (rd0 ~mod+ rk1) ~mod+ (1 << 17), n: 18) + // final1 = this.util.sign_extend_rshift_u32(a: (rd1 ~mod+ rk3) ~mod+ (1 << 17), n: 18) + // final2 = this.util.sign_extend_rshift_u32(a: (rd2 ~mod+ rk5) ~mod+ (1 << 17), n: 18) + // final3 = this.util.sign_extend_rshift_u32(a: (rd3 ~mod+ rk7) ~mod+ (1 << 17), n: 18) + // final4 = this.util.sign_extend_rshift_u32(a: (rd3 ~mod- rk7) ~mod+ (1 << 17), n: 18) + // final5 = this.util.sign_extend_rshift_u32(a: (rd2 ~mod- rk5) ~mod+ (1 << 17), n: 18) + // final6 = this.util.sign_extend_rshift_u32(a: (rd1 ~mod- rk3) ~mod+ (1 << 17), n: 18) + // final7 = this.util.sign_extend_rshift_u32(a: (rd0 ~mod- rk1) ~mod+ (1 << 17), n: 18) + // + // becomes a mix of adds, subtracts, shifts and packs. + final01ad = sp_rd01ad._mm256_add_epi32(b: sp_rk13ad)._mm256_add_epi32(b: k_0000_0002)._mm256_srai_epi32(imm8: 18) + final01eh = sp_rd01eh._mm256_add_epi32(b: sp_rk13eh)._mm256_add_epi32(b: k_0000_0002)._mm256_srai_epi32(imm8: 18) + final01 = final01ad._mm256_packs_epi32(b: final01eh) + final32ad = sp_rd32ad._mm256_add_epi32(b: sp_rk75ad)._mm256_add_epi32(b: k_0000_0002)._mm256_srai_epi32(imm8: 18) + final32eh = sp_rd32eh._mm256_add_epi32(b: sp_rk75eh)._mm256_add_epi32(b: k_0000_0002)._mm256_srai_epi32(imm8: 18) + final32 = final32ad._mm256_packs_epi32(b: final32eh) + final45ad = sp_rd32ad._mm256_sub_epi32(b: sp_rk75ad)._mm256_add_epi32(b: k_0000_0002)._mm256_srai_epi32(imm8: 18) + final45eh = sp_rd32eh._mm256_sub_epi32(b: sp_rk75eh)._mm256_add_epi32(b: k_0000_0002)._mm256_srai_epi32(imm8: 18) + final45 = final45ad._mm256_packs_epi32(b: final45eh) + final76ad = sp_rd01ad._mm256_sub_epi32(b: sp_rk13ad)._mm256_add_epi32(b: k_0000_0002)._mm256_srai_epi32(imm8: 18) + final76eh = sp_rd01eh._mm256_sub_epi32(b: sp_rk13eh)._mm256_add_epi32(b: k_0000_0002)._mm256_srai_epi32(imm8: 18) + final76 = final76ad._mm256_packs_epi32(b: final76eh) + + // ---- + + // Transpose/re-order the final rows and columns. + fta0a1e0e1 = final01._mm256_permute4x64_epi64(imm8: 0xD8) + fta2a3e2e3 = final32._mm256_permute4x64_epi64(imm8: 0x72) + fta4a5e4e5 = final45._mm256_permute4x64_epi64(imm8: 0xD8) + fta6a7e6e7 = final76._mm256_permute4x64_epi64(imm8: 0x72) + fta0c0e0g0 = fta0a1e0e1._mm256_unpacklo_epi16(b: fta2a3e2e3) + fta1c1e1g1 = fta0a1e0e1._mm256_unpackhi_epi16(b: fta2a3e2e3) + fta4c4e4g4 = fta4a5e4e5._mm256_unpacklo_epi16(b: fta6a7e6e7) + fta5c5e5g5 = fta4a5e4e5._mm256_unpackhi_epi16(b: fta6a7e6e7) + fta0b0e0f0 = fta0c0e0g0._mm256_unpacklo_epi16(b: fta1c1e1g1) + ftc0d0g0h0 = fta0c0e0g0._mm256_unpackhi_epi16(b: fta1c1e1g1) + fta4b4e4f4 = fta4c4e4g4._mm256_unpacklo_epi16(b: fta5c5e5g5) + ftc4d4g4h4 = fta4c4e4g4._mm256_unpackhi_epi16(b: fta5c5e5g5) + finalae = fta0b0e0f0._mm256_unpacklo_epi64(b: fta4b4e4f4) + finalbf = fta0b0e0f0._mm256_unpackhi_epi64(b: fta4b4e4f4) + finalcg = ftc0d0g0h0._mm256_unpacklo_epi64(b: ftc4d4g4h4) + finaldh = ftc0d0g0h0._mm256_unpackhi_epi64(b: ftc4d4g4h4) + + // ---- + + // Pack i16x16 into i8x32 (using signed saturation; i8 ranges from -128 ..= + // +127), bias by +0x80 (u8 ranges from 0 ..= 255) and extract the rows. + // + // We also drop the "'Tranposed' means that the rows and column names + // should be swapped". Rows are back to 0 ..= 7 and columns are a ..= h. + final0145 = finalae._mm256_packs_epi16(b: finalbf)._mm256_add_epi8(b: k_8080) + final2367 = finalcg._mm256_packs_epi16(b: finaldh)._mm256_add_epi8(b: k_8080) + final0 = final0145._mm256_extract_epi64(index: 0) + final1 = final0145._mm256_extract_epi64(index: 1) + final2 = final2367._mm256_extract_epi64(index: 0) + final3 = final2367._mm256_extract_epi64(index: 1) + final4 = final0145._mm256_extract_epi64(index: 2) + final5 = final0145._mm256_extract_epi64(index: 3) + final6 = final2367._mm256_extract_epi64(index: 2) + final7 = final2367._mm256_extract_epi64(index: 3) + + // ---- + + // Write to the args.dst_buffer. + + if args.dst_stride > args.dst_buffer.length() { + return nothing + } + assert 8 <= args.dst_buffer.length() via "a <= b: a <= c; c <= b"(c: args.dst_stride) + assert args.dst_buffer.length() >= 8 via "a >= b: b <= a"() + remaining = args.dst_buffer[args.dst_stride ..] + args.dst_buffer.poke_u64le!(a: final0) + args.dst_buffer = remaining + + if args.dst_stride > args.dst_buffer.length() { + return nothing + } + assert 8 <= args.dst_buffer.length() via "a <= b: a <= c; c <= b"(c: args.dst_stride) + assert args.dst_buffer.length() >= 8 via "a >= b: b <= a"() + remaining = args.dst_buffer[args.dst_stride ..] + args.dst_buffer.poke_u64le!(a: final1) + args.dst_buffer = remaining + + if args.dst_stride > args.dst_buffer.length() { + return nothing + } + assert 8 <= args.dst_buffer.length() via "a <= b: a <= c; c <= b"(c: args.dst_stride) + assert args.dst_buffer.length() >= 8 via "a >= b: b <= a"() + remaining = args.dst_buffer[args.dst_stride ..] + args.dst_buffer.poke_u64le!(a: final2) + args.dst_buffer = remaining + + if args.dst_stride > args.dst_buffer.length() { + return nothing + } + assert 8 <= args.dst_buffer.length() via "a <= b: a <= c; c <= b"(c: args.dst_stride) + assert args.dst_buffer.length() >= 8 via "a >= b: b <= a"() + remaining = args.dst_buffer[args.dst_stride ..] + args.dst_buffer.poke_u64le!(a: final3) + args.dst_buffer = remaining + + if args.dst_stride > args.dst_buffer.length() { + return nothing + } + assert 8 <= args.dst_buffer.length() via "a <= b: a <= c; c <= b"(c: args.dst_stride) + assert args.dst_buffer.length() >= 8 via "a >= b: b <= a"() + remaining = args.dst_buffer[args.dst_stride ..] + args.dst_buffer.poke_u64le!(a: final4) + args.dst_buffer = remaining + + if args.dst_stride > args.dst_buffer.length() { + return nothing + } + assert 8 <= args.dst_buffer.length() via "a <= b: a <= c; c <= b"(c: args.dst_stride) + assert args.dst_buffer.length() >= 8 via "a >= b: b <= a"() + remaining = args.dst_buffer[args.dst_stride ..] + args.dst_buffer.poke_u64le!(a: final5) + args.dst_buffer = remaining + + if args.dst_stride > args.dst_buffer.length() { + return nothing + } + assert 8 <= args.dst_buffer.length() via "a <= b: a <= c; c <= b"(c: args.dst_stride) + assert args.dst_buffer.length() >= 8 via "a >= b: b <= a"() + remaining = args.dst_buffer[args.dst_stride ..] + args.dst_buffer.poke_u64le!(a: final6) + args.dst_buffer = remaining + + if 8 > args.dst_buffer.length() { + return nothing + } + assert args.dst_buffer.length() >= 8 via "a >= b: b <= a"() + args.dst_buffer.poke_u64le!(a: final7) } diff --git a/std/jpeg/decode_jpeg.wuffs b/std/jpeg/decode_jpeg.wuffs index 64b11f77..3c60b17c 100644 --- a/std/jpeg/decode_jpeg.wuffs +++ b/std/jpeg/decode_jpeg.wuffs @@ -466,6 +466,10 @@ pri func decoder.do_decode_image_config?(dst: nptr base.image_config, src: base. this.payload_length = 0 } endwhile + choose decode_idct = [ + // TODO: decode_idct_arm_neon, + decode_idct_x86_avx2] + this.frame_config_io_position = args.src.position() if args.dst <> nullptr { diff --git a/test/c/std/jpeg.c b/test/c/std/jpeg.c index b7c068e5..83d9cee1 100644 --- a/test/c/std/jpeg.c +++ b/test/c/std/jpeg.c @@ -597,7 +597,7 @@ test_wuffs_jpeg_decode_idct() { #if defined(WUFFS_BASE__CPU_ARCH__X86_FAMILY) } else if (wuffs_base__cpu_arch__have_x86_avx2()) { func_name = "x86_avx2"; - // TODO: func = &wuffs_jpeg__decoder__decode_idct_x86_avx2; + func = &wuffs_jpeg__decoder__decode_idct_x86_avx2; #endif }