From cd769a0cdf1b5affee13f6089b995f3d39569cb4 Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Sat, 22 Jul 2023 16:32:59 +1000 Subject: [PATCH] pixconv: add upsample_inv_h2v2_triangle_x86_avx2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit name old speed new speed delta wuffs_jpeg_decode_19k_8bpp/clang11 370MB/s ± 0% 376MB/s ± 0% +1.84% (p=0.008 n=5+5) wuffs_jpeg_decode_30k_24bpp_progressive/clang11 295MB/s ± 0% 311MB/s ± 0% +5.56% (p=0.008 n=5+5) wuffs_jpeg_decode_30k_24bpp_sequential/clang11 726MB/s ± 0% 1021MB/s ± 0% +40.62% (p=0.008 n=5+5) wuffs_jpeg_decode_77k_24bpp/clang11 622MB/s ± 0% 842MB/s ± 0% +35.49% (p=0.008 n=5+5) wuffs_jpeg_decode_552k_24bpp_420/clang11 659MB/s ± 0% 917MB/s ± 0% +39.15% (p=0.008 n=5+5) wuffs_jpeg_decode_552k_24bpp_444/clang11 649MB/s ± 0% 649MB/s ± 0% -0.09% (p=0.016 n=5+5) wuffs_jpeg_decode_4002k_24bpp/clang11 675MB/s ± 0% 921MB/s ± 0% +36.49% (p=0.008 n=5+5) wuffs_jpeg_decode_19k_8bpp/gcc10 351MB/s ± 0% 326MB/s ± 0% -7.21% (p=0.008 n=5+5) wuffs_jpeg_decode_30k_24bpp_progressive/gcc10 334MB/s ± 0% 292MB/s ± 0% -12.67% (p=0.008 n=5+5) wuffs_jpeg_decode_30k_24bpp_sequential/gcc10 924MB/s ± 0% 920MB/s ± 0% -0.41% (p=0.008 n=5+5) wuffs_jpeg_decode_77k_24bpp/gcc10 740MB/s ± 0% 742MB/s ± 0% +0.27% (p=0.008 n=5+5) wuffs_jpeg_decode_552k_24bpp_420/gcc10 812MB/s ± 0% 792MB/s ± 0% -2.38% (p=0.008 n=5+5) wuffs_jpeg_decode_552k_24bpp_444/gcc10 647MB/s ± 0% 627MB/s ± 0% -3.16% (p=0.016 n=5+4) wuffs_jpeg_decode_4002k_24bpp/gcc10 817MB/s ± 0% 792MB/s ± 0% -3.01% (p=0.016 n=4+5) --- .../cgen/base/pixconv-submodule-x86-avx2.c | 211 +++++++++++++- internal/cgen/base/pixconv-submodule-ycck.c | 40 ++- release/c/wuffs-unsupported-snapshot.c | 257 +++++++++++++++++- 3 files changed, 482 insertions(+), 26 deletions(-) diff --git a/internal/cgen/base/pixconv-submodule-x86-avx2.c b/internal/cgen/base/pixconv-submodule-x86-avx2.c index c2d325ce..4abbb410 100644 --- a/internal/cgen/base/pixconv-submodule-x86-avx2.c +++ b/internal/cgen/base/pixconv-submodule-x86-avx2.c @@ -34,7 +34,7 @@ wuffs_base__pixel_swizzler__swizzle_ycc__convert_bgrx_x86_avx2( size_t dst_stride = dst->private_impl.planes[0].stride; uint8_t* dst_iter = dst->private_impl.planes[0].ptr + - (dst_stride * ((size_t)y)) + (4 * ((size_t)x)); + (dst_stride * ((size_t)y)) + (4u * ((size_t)x)); // u0001 = u16x16 [0x0001 .. 0x0001] // u00FF = u16x16 [0x00FF .. 0x00FF] @@ -276,8 +276,8 @@ wuffs_base__pixel_swizzler__swizzle_ycc__convert_bgrx_x86_avx2( // Advance by up to 32 pixels. The first iteration might be smaller than 32 // so that all of the remaining steps are exactly 32. - uint32_t n = 32 - (31 & (x - x_end)); - dst_iter += 4 * n; + uint32_t n = 32u - (31u & (x - x_end)); + dst_iter += 4u * n; up0 += n; up1 += n; up2 += n; @@ -305,7 +305,7 @@ wuffs_base__pixel_swizzler__swizzle_ycc__convert_rgbx_x86_avx2( size_t dst_stride = dst->private_impl.planes[0].stride; uint8_t* dst_iter = dst->private_impl.planes[0].ptr + - (dst_stride * ((size_t)y)) + (4 * ((size_t)x)); + (dst_stride * ((size_t)y)) + (4u * ((size_t)x)); const __m256i u0001 = _mm256_set1_epi16(0x0001); const __m256i u00FF = _mm256_set1_epi16(0x00FF); @@ -424,8 +424,8 @@ wuffs_base__pixel_swizzler__swizzle_ycc__convert_rgbx_x86_avx2( _mm256_storeu_si256((__m256i*)(void*)(dst_iter + 0x40), mix32); _mm256_storeu_si256((__m256i*)(void*)(dst_iter + 0x60), mix33); - uint32_t n = 32 - (31 & (x - x_end)); - dst_iter += 4 * n; + uint32_t n = 32u - (31u & (x - x_end)); + dst_iter += 4u * n; up0 += n; up1 += n; up2 += n; @@ -433,5 +433,204 @@ wuffs_base__pixel_swizzler__swizzle_ycc__convert_rgbx_x86_avx2( } } +WUFFS_BASE__MAYBE_ATTRIBUTE_TARGET("pclmul,popcnt,sse4.2,avx2") +static const uint8_t* // +wuffs_base__pixel_swizzler__swizzle_ycc__upsample_inv_h2v2_triangle_x86_avx2( + uint8_t* dst_ptr, + const uint8_t* src_ptr_major, + const uint8_t* src_ptr_minor, + size_t src_len, + uint32_t h1v2_bias_ignored, + bool first_column, + bool last_column) { + uint8_t* dp = dst_ptr; + const uint8_t* sp_major = src_ptr_major; + const uint8_t* sp_minor = src_ptr_minor; + + if (first_column) { + src_len--; + if ((src_len <= 0u) && last_column) { + uint32_t sv = (12u * ((uint32_t)(*sp_major++))) + // + (4u * ((uint32_t)(*sp_minor++))); + *dp++ = (uint8_t)((sv + 8u) >> 4u); + *dp++ = (uint8_t)((sv + 7u) >> 4u); + return dst_ptr; + } + + uint32_t sv_major_m1 = sp_major[-0]; // Clamp offset to zero. + uint32_t sv_minor_m1 = sp_minor[-0]; // Clamp offset to zero. + uint32_t sv_major_p1 = sp_major[+1]; + uint32_t sv_minor_p1 = sp_minor[+1]; + + uint32_t sv = (9u * ((uint32_t)(*sp_major++))) + // + (3u * ((uint32_t)(*sp_minor++))); + *dp++ = (uint8_t)((sv + (3u * sv_major_m1) + (sv_minor_m1) + 8u) >> 4u); + *dp++ = (uint8_t)((sv + (3u * sv_major_p1) + (sv_minor_p1) + 7u) >> 4u); + if (src_len <= 0u) { + return dst_ptr; + } + } + + if (last_column) { + src_len--; + } + + if (src_len < 32) { + // This fallback is the same as the non-SIMD-capable code path. + for (; src_len > 0u; src_len--) { + uint32_t sv_major_m1 = sp_major[-1]; + uint32_t sv_minor_m1 = sp_minor[-1]; + uint32_t sv_major_p1 = sp_major[+1]; + uint32_t sv_minor_p1 = sp_minor[+1]; + + uint32_t sv = (9u * ((uint32_t)(*sp_major++))) + // + (3u * ((uint32_t)(*sp_minor++))); + *dp++ = (uint8_t)((sv + (3u * sv_major_m1) + (sv_minor_m1) + 8u) >> 4u); + *dp++ = (uint8_t)((sv + (3u * sv_major_p1) + (sv_minor_p1) + 7u) >> 4u); + } + + } else { + while (src_len > 0u) { + // Load 1+32+1 samples (six u8x32 vectors) from the major (jxx) and minor + // (nxx) rows. + // + // major_p0 = [j00 j01 j02 j03 .. j28 j29 j30 j31] // p0 = "plus 0" + // minor_p0 = [n00 n01 n02 n03 .. n28 n29 n30 n31] // p0 = "plus 0" + // major_m1 = [jm1 j00 j01 j02 .. j27 j28 j29 j30] // m1 = "minus 1" + // minor_m1 = [nm1 n00 n01 n02 .. n27 n28 n29 n30] // m1 = "minus 1" + // major_p1 = [j01 j02 j03 j04 .. j29 j30 j31 j32] // p1 = "plus 1" + // minor_p1 = [n01 n02 n03 n04 .. n29 n30 n31 n32] // p1 = "plus 1" + __m256i major_p0 = + _mm256_lddqu_si256((const __m256i*)(const void*)(sp_major + 0)); + __m256i minor_p0 = + _mm256_lddqu_si256((const __m256i*)(const void*)(sp_minor + 0)); + __m256i major_m1 = + _mm256_lddqu_si256((const __m256i*)(const void*)(sp_major - 1)); + __m256i minor_m1 = + _mm256_lddqu_si256((const __m256i*)(const void*)(sp_minor - 1)); + __m256i major_p1 = + _mm256_lddqu_si256((const __m256i*)(const void*)(sp_major + 1)); + __m256i minor_p1 = + _mm256_lddqu_si256((const __m256i*)(const void*)(sp_minor + 1)); + + // Unpack, staying with u8x32 vectors. + // + // step1_p0_lo = [j00 n00 j01 n01 .. j07 n07 j16 n16 j17 n17 .. j23 n23] + // step1_p0_hi = [j08 n08 j09 n09 .. j15 n15 j24 n24 j25 n25 .. j31 n31] + // step1_m1_lo = [jm1 nm1 j00 n00 .. j06 n06 j15 n15 j16 n16 .. j22 n22] + // step1_m1_hi = [j07 n07 j08 n08 .. j14 n14 j23 n23 j24 n24 .. j30 n30] + // step1_p1_lo = [j01 n01 j02 n02 .. j08 n08 j17 n17 j18 n18 .. j24 n24] + // step1_p1_hi = [j09 n09 j10 n10 .. j16 n16 j25 n25 j26 n26 .. j32 n32] + __m256i step1_p0_lo = _mm256_unpacklo_epi8(major_p0, minor_p0); + __m256i step1_p0_hi = _mm256_unpackhi_epi8(major_p0, minor_p0); + __m256i step1_m1_lo = _mm256_unpacklo_epi8(major_m1, minor_m1); + __m256i step1_m1_hi = _mm256_unpackhi_epi8(major_m1, minor_m1); + __m256i step1_p1_lo = _mm256_unpacklo_epi8(major_p1, minor_p1); + __m256i step1_p1_hi = _mm256_unpackhi_epi8(major_p1, minor_p1); + + // Multiply-add to get u16x16 vectors. + // + // step2_p0_lo = [9*j00+3*n00 9*j01+3*n01 .. 9*j23+3*n23] + // step2_p0_hi = [9*j08+3*n08 9*j09+3*n09 .. 9*j31+3*n31] + // step2_m1_lo = [3*jm1+1*nm1 3*j00+1*n00 .. 3*j22+1*n22] + // step2_m1_hi = [3*j07+1*n07 3*j08+1*n08 .. 3*j30+1*n30] + // step2_p1_lo = [3*j01+1*n01 3*j02+1*n02 .. 3*j24+1*n24] + // step2_p1_hi = [3*j09+1*n09 3*j10+1*n10 .. 3*j32+1*n32] + const __m256i k0309 = _mm256_set1_epi16(0x0309); + const __m256i k0103 = _mm256_set1_epi16(0x0103); + __m256i step2_p0_lo = _mm256_maddubs_epi16(step1_p0_lo, k0309); + __m256i step2_p0_hi = _mm256_maddubs_epi16(step1_p0_hi, k0309); + __m256i step2_m1_lo = _mm256_maddubs_epi16(step1_m1_lo, k0103); + __m256i step2_m1_hi = _mm256_maddubs_epi16(step1_m1_hi, k0103); + __m256i step2_p1_lo = _mm256_maddubs_epi16(step1_p1_lo, k0103); + __m256i step2_p1_hi = _mm256_maddubs_epi16(step1_p1_hi, k0103); + + // Compute the weighted sums of (p0, m1) and (p0, p1). For example: + // + // step3_m1_lo[00] = ((9*j00) + (3*n00) + (3*jm1) + (1*nm1)) as u16 + // step3_p1_hi[15] = ((9*j31) + (3*n31) + (3*j32) + (1*n32)) as u16 + __m256i step3_m1_lo = _mm256_add_epi16(step2_p0_lo, step2_m1_lo); + __m256i step3_m1_hi = _mm256_add_epi16(step2_p0_hi, step2_m1_hi); + __m256i step3_p1_lo = _mm256_add_epi16(step2_p0_lo, step2_p1_lo); + __m256i step3_p1_hi = _mm256_add_epi16(step2_p0_hi, step2_p1_hi); + + // Bias by 8 (on the left) or 7 (on the right) and then divide by 16 + // (which is 9+3+3+1) to get a weighted average. On the left (m1), shift + // the u16 right value by 4. On the right (p1), shift right by 4 and then + // shift left by 8 so that, when still in the u16x16 little-endian + // interpretation, we have: + // - m1_element = (etcetera + 8) >> 4 + // - p1_element = ((etcetera + 7) >> 4) << 8 + // + // step4_m1_lo = [0x00?? 0x00?? ... 0x00?? 0x00??] + // step4_p1_lo = [0x??00 0x??00 ... 0x??00 0x??00] + // step4_m1_hi = [0x00?? 0x00?? ... 0x00?? 0x00??] + // step4_p1_hi = [0x??00 0x??00 ... 0x??00 0x??00] + __m256i step4_m1_lo = _mm256_srli_epi16( + _mm256_add_epi16(step3_m1_lo, _mm256_set1_epi16(8)), 4); + __m256i step4_p1_lo = _mm256_slli_epi16( + _mm256_srli_epi16(_mm256_add_epi16(step3_p1_lo, _mm256_set1_epi16(7)), + 4), + 8); + __m256i step4_m1_hi = _mm256_srli_epi16( + _mm256_add_epi16(step3_m1_hi, _mm256_set1_epi16(8)), 4); + __m256i step4_p1_hi = _mm256_slli_epi16( + _mm256_srli_epi16(_mm256_add_epi16(step3_p1_hi, _mm256_set1_epi16(7)), + 4), + 8); + + // Bitwise-or two "0x00"-rich u16x16 vectors to get a u8x32 vector. Do + // that twice. Once for the low columns and once for the high columns. + // + // In terms of jxx (major row) or nxx (minor row) source samples: + // - low columns means ( 0 .. 8; 16 .. 24). + // - high columns means ( 8 .. 16; 24 .. 32). + // + // In terms of dxx destination samples (there are twice as many): + // - low columns means ( 0 .. 16; 32 .. 48). + // - high columns means (16 .. 32; 48 .. 64). + // + // step5_lo = [d00 d01 .. d14 d15 d32 d33 .. d46 d47] + // step5_hi = [d16 d17 .. d30 d31 d48 d49 .. d62 d63] + // + // The d00, d02 ... d62 even elements come from (p0, m1) weighted sums. + // The d01, d03 ... d63 odd elements come from (p0, p1) weighted sums. + __m256i step5_lo = _mm256_or_si256(step4_m1_lo, step4_p1_lo); + __m256i step5_hi = _mm256_or_si256(step4_m1_hi, step4_p1_hi); + + // Permute and store. + // + // step6_00_31 = [d00 d01 .. d14 d15 d16 d17 .. d30 d31] + // step6_32_63 = [d32 d33 .. d46 d47 d48 d49 .. d62 d63] + __m256i step6_00_31 = _mm256_permute2x128_si256(step5_lo, step5_hi, 0x20); + __m256i step6_32_63 = _mm256_permute2x128_si256(step5_lo, step5_hi, 0x31); + _mm256_storeu_si256((__m256i*)(void*)(dp + 0x00), step6_00_31); + _mm256_storeu_si256((__m256i*)(void*)(dp + 0x20), step6_32_63); + + // Advance by up to 32 source samples (64 destination samples). The first + // iteration might be smaller than 32 so that all of the remaining steps + // are exactly 32. + size_t n = 32u - (31u & (0u - src_len)); + dp += 2u * n; + sp_major += n; + sp_minor += n; + src_len -= n; + } + } + + if (last_column) { + uint32_t sv_major_m1 = sp_major[-1]; + uint32_t sv_minor_m1 = sp_minor[-1]; + uint32_t sv_major_p1 = sp_major[+0]; // Clamp offset to zero. + uint32_t sv_minor_p1 = sp_minor[+0]; // Clamp offset to zero. + + uint32_t sv = (9u * ((uint32_t)(*sp_major++))) + // + (3u * ((uint32_t)(*sp_minor++))); + *dp++ = (uint8_t)((sv + (3u * sv_major_m1) + (sv_minor_m1) + 8u) >> 4u); + *dp++ = (uint8_t)((sv + (3u * sv_major_p1) + (sv_minor_p1) + 7u) >> 4u); + } + + return dst_ptr; +} #endif // defined(WUFFS_BASE__CPU_ARCH__X86_FAMILY) // ‼ WUFFS MULTI-FILE SECTION -x86_avx2 diff --git a/internal/cgen/base/pixconv-submodule-ycck.c b/internal/cgen/base/pixconv-submodule-ycck.c index 8c1a1240..52250676 100644 --- a/internal/cgen/base/pixconv-submodule-ycck.c +++ b/internal/cgen/base/pixconv-submodule-ycck.c @@ -36,6 +36,17 @@ wuffs_base__pixel_swizzler__swizzle_ycc__convert_rgbx_x86_avx2( const uint8_t* up0, const uint8_t* up1, const uint8_t* up2); + +WUFFS_BASE__MAYBE_ATTRIBUTE_TARGET("pclmul,popcnt,sse4.2,avx2") +static const uint8_t* // +wuffs_base__pixel_swizzler__swizzle_ycc__upsample_inv_h2v2_triangle_x86_avx2( + uint8_t* dst_ptr, + const uint8_t* src_ptr_major, + const uint8_t* src_ptr_minor, + size_t src_len, + uint32_t h1v2_bias_ignored, + bool first_column, + bool last_column); #endif // -------- @@ -529,16 +540,15 @@ wuffs_base__pixel_swizzler__swizzle_ycc__general__triangle_filter( uint32_t half_width_for_2to1, uint32_t half_height_for_2to1, uint8_t* scratch_buffer_2k_ptr, + const wuffs_base__pixel_swizzler__swizzle_ycc__upsample_func ( + *upfuncs)[4][4], wuffs_base__pixel_swizzler__swizzle_ycc__convert_func convfunc) { wuffs_base__pixel_swizzler__swizzle_ycc__upsample_func upfunc0 = - wuffs_base__pixel_swizzler__swizzle_ycc__upsample_funcs - [(inv_h0 - 1u) & 3u][(inv_v0 - 1u) & 3u]; + (*upfuncs)[(inv_h0 - 1u) & 3u][(inv_v0 - 1u) & 3u]; wuffs_base__pixel_swizzler__swizzle_ycc__upsample_func upfunc1 = - wuffs_base__pixel_swizzler__swizzle_ycc__upsample_funcs - [(inv_h1 - 1u) & 3u][(inv_v1 - 1u) & 3u]; + (*upfuncs)[(inv_h1 - 1u) & 3u][(inv_v1 - 1u) & 3u]; wuffs_base__pixel_swizzler__swizzle_ycc__upsample_func upfunc2 = - wuffs_base__pixel_swizzler__swizzle_ycc__upsample_funcs - [(inv_h2 - 1u) & 3u][(inv_v2 - 1u) & 3u]; + (*upfuncs)[(inv_h2 - 1u) & 3u][(inv_v2 - 1u) & 3u]; // First row. uint32_t h1v2_bias = 1u; @@ -662,6 +672,8 @@ wuffs_base__pixel_swizzler__swizzle_ycc__general__box_filter( uint32_t half_width_for_2to1, uint32_t half_height_for_2to1, uint8_t* scratch_buffer_2k_ptr, + const wuffs_base__pixel_swizzler__swizzle_ycc__upsample_func ( + *upfuncs)[4][4], wuffs_base__pixel_swizzler__swizzle_ycc__convert_func convfunc) { // Convert an inv_h or inv_v value from {1, 2, 3, 4} to {12, 6, 4, 3}. uint32_t h0_out_of_12 = 12u / inv_h0; @@ -763,6 +775,8 @@ wuffs_base__pixel_swizzler__swizzle_ycc__bgrx__hv11( uint32_t half_width_for_2to1, uint32_t half_height_for_2to1, uint8_t* scratch_buffer_2k_ptr, + const wuffs_base__pixel_swizzler__swizzle_ycc__upsample_func ( + *upfuncs)[4][4], wuffs_base__pixel_swizzler__swizzle_ycc__convert_func convfunc) { uint32_t y = 0u; for (; y < height; y++) { @@ -1020,14 +1034,26 @@ wuffs_base__pixel_swizzler__swizzle_ycck( uint32_t half_width_for_2to1, // uint32_t half_height_for_2to1, // uint8_t* scratch_buffer_2k_ptr, // + const wuffs_base__pixel_swizzler__swizzle_ycc__upsample_func( + *upfuncs)[4][4], wuffs_base__pixel_swizzler__swizzle_ycc__convert_func convfunc) = &wuffs_base__pixel_swizzler__swizzle_ycc__general__box_filter; + wuffs_base__pixel_swizzler__swizzle_ycc__upsample_func upfuncs[4][4]; + memcpy(&upfuncs, &wuffs_base__pixel_swizzler__swizzle_ycc__upsample_funcs, + sizeof upfuncs); + if (triangle_filter_for_2to1 && (wuffs_base__pixel_swizzler__has_triangle_upsampler(inv_h0, inv_v0) || wuffs_base__pixel_swizzler__has_triangle_upsampler(inv_h1, inv_v1) || wuffs_base__pixel_swizzler__has_triangle_upsampler(inv_h2, inv_v2))) { func = &wuffs_base__pixel_swizzler__swizzle_ycc__general__triangle_filter; +#if defined(WUFFS_BASE__CPU_ARCH__X86_FAMILY) + if (wuffs_base__cpu_arch__have_x86_sse42()) { + upfuncs[1][1] = + wuffs_base__pixel_swizzler__swizzle_ycc__upsample_inv_h2v2_triangle_x86_avx2; + } +#endif } else { switch (dst->pixcfg.private_impl.pixfmt.repr) { @@ -1058,7 +1084,7 @@ wuffs_base__pixel_swizzler__swizzle_ycck( inv_h0, inv_h1, inv_h2, // inv_v0, inv_v1, inv_v2, // half_width_for_2to1, half_height_for_2to1, // - scratch_buffer_2k.ptr, convfunc); + scratch_buffer_2k.ptr, &upfuncs, convfunc); return wuffs_base__make_status(NULL); } diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 35cf844a..f654652c 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -23568,6 +23568,17 @@ wuffs_base__pixel_swizzler__swizzle_ycc__convert_rgbx_x86_avx2( const uint8_t* up0, const uint8_t* up1, const uint8_t* up2); + +WUFFS_BASE__MAYBE_ATTRIBUTE_TARGET("pclmul,popcnt,sse4.2,avx2") +static const uint8_t* // +wuffs_base__pixel_swizzler__swizzle_ycc__upsample_inv_h2v2_triangle_x86_avx2( + uint8_t* dst_ptr, + const uint8_t* src_ptr_major, + const uint8_t* src_ptr_minor, + size_t src_len, + uint32_t h1v2_bias_ignored, + bool first_column, + bool last_column); #endif // -------- @@ -24061,16 +24072,15 @@ wuffs_base__pixel_swizzler__swizzle_ycc__general__triangle_filter( uint32_t half_width_for_2to1, uint32_t half_height_for_2to1, uint8_t* scratch_buffer_2k_ptr, + const wuffs_base__pixel_swizzler__swizzle_ycc__upsample_func ( + *upfuncs)[4][4], wuffs_base__pixel_swizzler__swizzle_ycc__convert_func convfunc) { wuffs_base__pixel_swizzler__swizzle_ycc__upsample_func upfunc0 = - wuffs_base__pixel_swizzler__swizzle_ycc__upsample_funcs - [(inv_h0 - 1u) & 3u][(inv_v0 - 1u) & 3u]; + (*upfuncs)[(inv_h0 - 1u) & 3u][(inv_v0 - 1u) & 3u]; wuffs_base__pixel_swizzler__swizzle_ycc__upsample_func upfunc1 = - wuffs_base__pixel_swizzler__swizzle_ycc__upsample_funcs - [(inv_h1 - 1u) & 3u][(inv_v1 - 1u) & 3u]; + (*upfuncs)[(inv_h1 - 1u) & 3u][(inv_v1 - 1u) & 3u]; wuffs_base__pixel_swizzler__swizzle_ycc__upsample_func upfunc2 = - wuffs_base__pixel_swizzler__swizzle_ycc__upsample_funcs - [(inv_h2 - 1u) & 3u][(inv_v2 - 1u) & 3u]; + (*upfuncs)[(inv_h2 - 1u) & 3u][(inv_v2 - 1u) & 3u]; // First row. uint32_t h1v2_bias = 1u; @@ -24194,6 +24204,8 @@ wuffs_base__pixel_swizzler__swizzle_ycc__general__box_filter( uint32_t half_width_for_2to1, uint32_t half_height_for_2to1, uint8_t* scratch_buffer_2k_ptr, + const wuffs_base__pixel_swizzler__swizzle_ycc__upsample_func ( + *upfuncs)[4][4], wuffs_base__pixel_swizzler__swizzle_ycc__convert_func convfunc) { // Convert an inv_h or inv_v value from {1, 2, 3, 4} to {12, 6, 4, 3}. uint32_t h0_out_of_12 = 12u / inv_h0; @@ -24293,6 +24305,8 @@ wuffs_base__pixel_swizzler__swizzle_ycc__bgrx__hv11( uint32_t half_width_for_2to1, uint32_t half_height_for_2to1, uint8_t* scratch_buffer_2k_ptr, + const wuffs_base__pixel_swizzler__swizzle_ycc__upsample_func ( + *upfuncs)[4][4], wuffs_base__pixel_swizzler__swizzle_ycc__convert_func convfunc) { uint32_t y = 0u; for (; y < height; y++) { @@ -24370,6 +24384,8 @@ wuffs_base__pixel_swizzler__swizzle_ycc__rgbx__hv11( uint32_t half_width_for_2to1, uint32_t half_height_for_2to1, uint8_t* scratch_buffer_2k_ptr, + const wuffs_base__pixel_swizzler__swizzle_ycc__upsample_func ( + *upfuncs)[4][4], wuffs_base__pixel_swizzler__swizzle_ycc__convert_func convfunc) { uint32_t y = 0u; for (; y < height; y++) { @@ -24446,6 +24462,8 @@ wuffs_base__pixel_swizzler__swizzle_ycc__bgrx__box_filter( uint32_t half_width_for_2to1, uint32_t half_height_for_2to1, uint8_t* scratch_buffer_2k_ptr, + const wuffs_base__pixel_swizzler__swizzle_ycc__upsample_func ( + *upfuncs)[4][4], wuffs_base__pixel_swizzler__swizzle_ycc__convert_func convfunc) { uint32_t h0_out_of_12 = 12u / inv_h0; uint32_t h1_out_of_12 = 12u / inv_h1; @@ -24542,6 +24560,8 @@ wuffs_base__pixel_swizzler__swizzle_ycc__rgbx__box_filter( uint32_t half_width_for_2to1, uint32_t half_height_for_2to1, uint8_t* scratch_buffer_2k_ptr, + const wuffs_base__pixel_swizzler__swizzle_ycc__upsample_func ( + *upfuncs)[4][4], wuffs_base__pixel_swizzler__swizzle_ycc__convert_func convfunc) { uint32_t h0_out_of_12 = 12u / inv_h0; uint32_t h1_out_of_12 = 12u / inv_h1; @@ -24816,14 +24836,26 @@ wuffs_base__pixel_swizzler__swizzle_ycck( uint32_t half_width_for_2to1, // uint32_t half_height_for_2to1, // uint8_t* scratch_buffer_2k_ptr, // + const wuffs_base__pixel_swizzler__swizzle_ycc__upsample_func( + *upfuncs)[4][4], wuffs_base__pixel_swizzler__swizzle_ycc__convert_func convfunc) = &wuffs_base__pixel_swizzler__swizzle_ycc__general__box_filter; + wuffs_base__pixel_swizzler__swizzle_ycc__upsample_func upfuncs[4][4]; + memcpy(&upfuncs, &wuffs_base__pixel_swizzler__swizzle_ycc__upsample_funcs, + sizeof upfuncs); + if (triangle_filter_for_2to1 && (wuffs_base__pixel_swizzler__has_triangle_upsampler(inv_h0, inv_v0) || wuffs_base__pixel_swizzler__has_triangle_upsampler(inv_h1, inv_v1) || wuffs_base__pixel_swizzler__has_triangle_upsampler(inv_h2, inv_v2))) { func = &wuffs_base__pixel_swizzler__swizzle_ycc__general__triangle_filter; +#if defined(WUFFS_BASE__CPU_ARCH__X86_FAMILY) + if (wuffs_base__cpu_arch__have_x86_sse42()) { + upfuncs[1][1] = + wuffs_base__pixel_swizzler__swizzle_ycc__upsample_inv_h2v2_triangle_x86_avx2; + } +#endif } else { switch (dst->pixcfg.private_impl.pixfmt.repr) { @@ -24854,7 +24886,7 @@ wuffs_base__pixel_swizzler__swizzle_ycck( inv_h0, inv_h1, inv_h2, // inv_v0, inv_v1, inv_v2, // half_width_for_2to1, half_height_for_2to1, // - scratch_buffer_2k.ptr, convfunc); + scratch_buffer_2k.ptr, &upfuncs, convfunc); return wuffs_base__make_status(NULL); } @@ -24881,7 +24913,7 @@ wuffs_base__pixel_swizzler__swizzle_ycc__convert_bgrx_x86_avx2( size_t dst_stride = dst->private_impl.planes[0].stride; uint8_t* dst_iter = dst->private_impl.planes[0].ptr + - (dst_stride * ((size_t)y)) + (4 * ((size_t)x)); + (dst_stride * ((size_t)y)) + (4u * ((size_t)x)); // u0001 = u16x16 [0x0001 .. 0x0001] // u00FF = u16x16 [0x00FF .. 0x00FF] @@ -25123,8 +25155,8 @@ wuffs_base__pixel_swizzler__swizzle_ycc__convert_bgrx_x86_avx2( // Advance by up to 32 pixels. The first iteration might be smaller than 32 // so that all of the remaining steps are exactly 32. - uint32_t n = 32 - (31 & (x - x_end)); - dst_iter += 4 * n; + uint32_t n = 32u - (31u & (x - x_end)); + dst_iter += 4u * n; up0 += n; up1 += n; up2 += n; @@ -25152,7 +25184,7 @@ wuffs_base__pixel_swizzler__swizzle_ycc__convert_rgbx_x86_avx2( size_t dst_stride = dst->private_impl.planes[0].stride; uint8_t* dst_iter = dst->private_impl.planes[0].ptr + - (dst_stride * ((size_t)y)) + (4 * ((size_t)x)); + (dst_stride * ((size_t)y)) + (4u * ((size_t)x)); const __m256i u0001 = _mm256_set1_epi16(0x0001); const __m256i u00FF = _mm256_set1_epi16(0x00FF); @@ -25271,8 +25303,8 @@ wuffs_base__pixel_swizzler__swizzle_ycc__convert_rgbx_x86_avx2( _mm256_storeu_si256((__m256i*)(void*)(dst_iter + 0x40), mix32); _mm256_storeu_si256((__m256i*)(void*)(dst_iter + 0x60), mix33); - uint32_t n = 32 - (31 & (x - x_end)); - dst_iter += 4 * n; + uint32_t n = 32u - (31u & (x - x_end)); + dst_iter += 4u * n; up0 += n; up1 += n; up2 += n; @@ -25280,6 +25312,205 @@ wuffs_base__pixel_swizzler__swizzle_ycc__convert_rgbx_x86_avx2( } } +WUFFS_BASE__MAYBE_ATTRIBUTE_TARGET("pclmul,popcnt,sse4.2,avx2") +static const uint8_t* // +wuffs_base__pixel_swizzler__swizzle_ycc__upsample_inv_h2v2_triangle_x86_avx2( + uint8_t* dst_ptr, + const uint8_t* src_ptr_major, + const uint8_t* src_ptr_minor, + size_t src_len, + uint32_t h1v2_bias_ignored, + bool first_column, + bool last_column) { + uint8_t* dp = dst_ptr; + const uint8_t* sp_major = src_ptr_major; + const uint8_t* sp_minor = src_ptr_minor; + + if (first_column) { + src_len--; + if ((src_len <= 0u) && last_column) { + uint32_t sv = (12u * ((uint32_t)(*sp_major++))) + // + (4u * ((uint32_t)(*sp_minor++))); + *dp++ = (uint8_t)((sv + 8u) >> 4u); + *dp++ = (uint8_t)((sv + 7u) >> 4u); + return dst_ptr; + } + + uint32_t sv_major_m1 = sp_major[-0]; // Clamp offset to zero. + uint32_t sv_minor_m1 = sp_minor[-0]; // Clamp offset to zero. + uint32_t sv_major_p1 = sp_major[+1]; + uint32_t sv_minor_p1 = sp_minor[+1]; + + uint32_t sv = (9u * ((uint32_t)(*sp_major++))) + // + (3u * ((uint32_t)(*sp_minor++))); + *dp++ = (uint8_t)((sv + (3u * sv_major_m1) + (sv_minor_m1) + 8u) >> 4u); + *dp++ = (uint8_t)((sv + (3u * sv_major_p1) + (sv_minor_p1) + 7u) >> 4u); + if (src_len <= 0u) { + return dst_ptr; + } + } + + if (last_column) { + src_len--; + } + + if (src_len < 32) { + // This fallback is the same as the non-SIMD-capable code path. + for (; src_len > 0u; src_len--) { + uint32_t sv_major_m1 = sp_major[-1]; + uint32_t sv_minor_m1 = sp_minor[-1]; + uint32_t sv_major_p1 = sp_major[+1]; + uint32_t sv_minor_p1 = sp_minor[+1]; + + uint32_t sv = (9u * ((uint32_t)(*sp_major++))) + // + (3u * ((uint32_t)(*sp_minor++))); + *dp++ = (uint8_t)((sv + (3u * sv_major_m1) + (sv_minor_m1) + 8u) >> 4u); + *dp++ = (uint8_t)((sv + (3u * sv_major_p1) + (sv_minor_p1) + 7u) >> 4u); + } + + } else { + while (src_len > 0u) { + // Load 1+32+1 samples (six u8x32 vectors) from the major (jxx) and minor + // (nxx) rows. + // + // major_p0 = [j00 j01 j02 j03 .. j28 j29 j30 j31] // p0 = "plus 0" + // minor_p0 = [n00 n01 n02 n03 .. n28 n29 n30 n31] // p0 = "plus 0" + // major_m1 = [jm1 j00 j01 j02 .. j27 j28 j29 j30] // m1 = "minus 1" + // minor_m1 = [nm1 n00 n01 n02 .. n27 n28 n29 n30] // m1 = "minus 1" + // major_p1 = [j01 j02 j03 j04 .. j29 j30 j31 j32] // p1 = "plus 1" + // minor_p1 = [n01 n02 n03 n04 .. n29 n30 n31 n32] // p1 = "plus 1" + __m256i major_p0 = + _mm256_lddqu_si256((const __m256i*)(const void*)(sp_major + 0)); + __m256i minor_p0 = + _mm256_lddqu_si256((const __m256i*)(const void*)(sp_minor + 0)); + __m256i major_m1 = + _mm256_lddqu_si256((const __m256i*)(const void*)(sp_major - 1)); + __m256i minor_m1 = + _mm256_lddqu_si256((const __m256i*)(const void*)(sp_minor - 1)); + __m256i major_p1 = + _mm256_lddqu_si256((const __m256i*)(const void*)(sp_major + 1)); + __m256i minor_p1 = + _mm256_lddqu_si256((const __m256i*)(const void*)(sp_minor + 1)); + + // Unpack, staying with u8x32 vectors. + // + // step1_p0_lo = [j00 n00 j01 n01 .. j07 n07 j16 n16 j17 n17 .. j23 n23] + // step1_p0_hi = [j08 n08 j09 n09 .. j15 n15 j24 n24 j25 n25 .. j31 n31] + // step1_m1_lo = [jm1 nm1 j00 n00 .. j06 n06 j15 n15 j16 n16 .. j22 n22] + // step1_m1_hi = [j07 n07 j08 n08 .. j14 n14 j23 n23 j24 n24 .. j30 n30] + // step1_p1_lo = [j01 n01 j02 n02 .. j08 n08 j17 n17 j18 n18 .. j24 n24] + // step1_p1_hi = [j09 n09 j10 n10 .. j16 n16 j25 n25 j26 n26 .. j32 n32] + __m256i step1_p0_lo = _mm256_unpacklo_epi8(major_p0, minor_p0); + __m256i step1_p0_hi = _mm256_unpackhi_epi8(major_p0, minor_p0); + __m256i step1_m1_lo = _mm256_unpacklo_epi8(major_m1, minor_m1); + __m256i step1_m1_hi = _mm256_unpackhi_epi8(major_m1, minor_m1); + __m256i step1_p1_lo = _mm256_unpacklo_epi8(major_p1, minor_p1); + __m256i step1_p1_hi = _mm256_unpackhi_epi8(major_p1, minor_p1); + + // Multiply-add to get u16x16 vectors. + // + // step2_p0_lo = [9*j00+3*n00 9*j01+3*n01 .. 9*j23+3*n23] + // step2_p0_hi = [9*j08+3*n08 9*j09+3*n09 .. 9*j31+3*n31] + // step2_m1_lo = [3*jm1+1*nm1 3*j00+1*n00 .. 3*j22+1*n22] + // step2_m1_hi = [3*j07+1*n07 3*j08+1*n08 .. 3*j30+1*n30] + // step2_p1_lo = [3*j01+1*n01 3*j02+1*n02 .. 3*j24+1*n24] + // step2_p1_hi = [3*j09+1*n09 3*j10+1*n10 .. 3*j32+1*n32] + const __m256i k0309 = _mm256_set1_epi16(0x0309); + const __m256i k0103 = _mm256_set1_epi16(0x0103); + __m256i step2_p0_lo = _mm256_maddubs_epi16(step1_p0_lo, k0309); + __m256i step2_p0_hi = _mm256_maddubs_epi16(step1_p0_hi, k0309); + __m256i step2_m1_lo = _mm256_maddubs_epi16(step1_m1_lo, k0103); + __m256i step2_m1_hi = _mm256_maddubs_epi16(step1_m1_hi, k0103); + __m256i step2_p1_lo = _mm256_maddubs_epi16(step1_p1_lo, k0103); + __m256i step2_p1_hi = _mm256_maddubs_epi16(step1_p1_hi, k0103); + + // Compute the weighted sums of (p0, m1) and (p0, p1). For example: + // + // step3_m1_lo[00] = ((9*j00) + (3*n00) + (3*jm1) + (1*nm1)) as u16 + // step3_p1_hi[15] = ((9*j31) + (3*n31) + (3*j32) + (1*n32)) as u16 + __m256i step3_m1_lo = _mm256_add_epi16(step2_p0_lo, step2_m1_lo); + __m256i step3_m1_hi = _mm256_add_epi16(step2_p0_hi, step2_m1_hi); + __m256i step3_p1_lo = _mm256_add_epi16(step2_p0_lo, step2_p1_lo); + __m256i step3_p1_hi = _mm256_add_epi16(step2_p0_hi, step2_p1_hi); + + // Bias by 8 (on the left) or 7 (on the right) and then divide by 16 + // (which is 9+3+3+1) to get a weighted average. On the left (m1), shift + // the u16 right value by 4. On the right (p1), shift right by 4 and then + // shift left by 8 so that, when still in the u16x16 little-endian + // interpretation, we have: + // - m1_element = (etcetera + 8) >> 4 + // - p1_element = ((etcetera + 7) >> 4) << 8 + // + // step4_m1_lo = [0x00?? 0x00?? ... 0x00?? 0x00??] + // step4_p1_lo = [0x??00 0x??00 ... 0x??00 0x??00] + // step4_m1_hi = [0x00?? 0x00?? ... 0x00?? 0x00??] + // step4_p1_hi = [0x??00 0x??00 ... 0x??00 0x??00] + __m256i step4_m1_lo = _mm256_srli_epi16( + _mm256_add_epi16(step3_m1_lo, _mm256_set1_epi16(8)), 4); + __m256i step4_p1_lo = _mm256_slli_epi16( + _mm256_srli_epi16(_mm256_add_epi16(step3_p1_lo, _mm256_set1_epi16(7)), + 4), + 8); + __m256i step4_m1_hi = _mm256_srli_epi16( + _mm256_add_epi16(step3_m1_hi, _mm256_set1_epi16(8)), 4); + __m256i step4_p1_hi = _mm256_slli_epi16( + _mm256_srli_epi16(_mm256_add_epi16(step3_p1_hi, _mm256_set1_epi16(7)), + 4), + 8); + + // Bitwise-or two "0x00"-rich u16x16 vectors to get a u8x32 vector. Do + // that twice. Once for the low columns and once for the high columns. + // + // In terms of jxx (major row) or nxx (minor row) source samples: + // - low columns means ( 0 .. 8; 16 .. 24). + // - high columns means ( 8 .. 16; 24 .. 32). + // + // In terms of dxx destination samples (there are twice as many): + // - low columns means ( 0 .. 16; 32 .. 48). + // - high columns means (16 .. 32; 48 .. 64). + // + // step5_lo = [d00 d01 .. d14 d15 d32 d33 .. d46 d47] + // step5_hi = [d16 d17 .. d30 d31 d48 d49 .. d62 d63] + // + // The d00, d02 ... d62 even elements come from (p0, m1) weighted sums. + // The d01, d03 ... d63 odd elements come from (p0, p1) weighted sums. + __m256i step5_lo = _mm256_or_si256(step4_m1_lo, step4_p1_lo); + __m256i step5_hi = _mm256_or_si256(step4_m1_hi, step4_p1_hi); + + // Permute and store. + // + // step6_00_31 = [d00 d01 .. d14 d15 d16 d17 .. d30 d31] + // step6_32_63 = [d32 d33 .. d46 d47 d48 d49 .. d62 d63] + __m256i step6_00_31 = _mm256_permute2x128_si256(step5_lo, step5_hi, 0x20); + __m256i step6_32_63 = _mm256_permute2x128_si256(step5_lo, step5_hi, 0x31); + _mm256_storeu_si256((__m256i*)(void*)(dp + 0x00), step6_00_31); + _mm256_storeu_si256((__m256i*)(void*)(dp + 0x20), step6_32_63); + + // Advance by up to 32 source samples (64 destination samples). The first + // iteration might be smaller than 32 so that all of the remaining steps + // are exactly 32. + size_t n = 32u - (31u & (0u - src_len)); + dp += 2u * n; + sp_major += n; + sp_minor += n; + src_len -= n; + } + } + + if (last_column) { + uint32_t sv_major_m1 = sp_major[-1]; + uint32_t sv_minor_m1 = sp_minor[-1]; + uint32_t sv_major_p1 = sp_major[+0]; // Clamp offset to zero. + uint32_t sv_minor_p1 = sp_minor[+0]; // Clamp offset to zero. + + uint32_t sv = (9u * ((uint32_t)(*sp_major++))) + // + (3u * ((uint32_t)(*sp_minor++))); + *dp++ = (uint8_t)((sv + (3u * sv_major_m1) + (sv_minor_m1) + 8u) >> 4u); + *dp++ = (uint8_t)((sv + (3u * sv_major_p1) + (sv_minor_p1) + 7u) >> 4u); + } + + return dst_ptr; +} #endif // defined(WUFFS_BASE__CPU_ARCH__X86_FAMILY) // ‼ WUFFS MULTI-FILE SECTION -x86_avx2