mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-18 00:32:11 +01:00
pixconv: add swizzle_ycc__convert_rgbx_x86_avx2
Some of the benchmark numbers move around even though they don't use the new etc__convert_rgbx_x86_avx2 code paths. It's possibly just a code alignment coincidence per https://news.ycombinator.com/item?id=23633583 name old speed new speed delta wuffs_jpeg_decode_19k_8bpp/clang11 372MB/s ± 0% 370MB/s ± 0% -0.64% (p=0.008 n=5+5) wuffs_jpeg_decode_30k_24bpp_progressive/clang11 278MB/s ± 0% 295MB/s ± 0% +5.89% (p=0.008 n=5+5) wuffs_jpeg_decode_30k_24bpp_sequential/clang11 710MB/s ± 0% 726MB/s ± 0% +2.30% (p=0.008 n=5+5) wuffs_jpeg_decode_77k_24bpp/clang11 603MB/s ± 0% 622MB/s ± 0% +3.12% (p=0.008 n=5+5) wuffs_jpeg_decode_552k_24bpp_420/clang11 635MB/s ± 0% 659MB/s ± 0% +3.82% (p=0.008 n=5+5) wuffs_jpeg_decode_552k_24bpp_444/clang11 650MB/s ± 0% 649MB/s ± 0% -0.17% (p=0.008 n=5+5) wuffs_jpeg_decode_4002k_24bpp/clang11 629MB/s ± 0% 675MB/s ± 0% +7.20% (p=0.008 n=5+5) wuffs_jpeg_decode_19k_8bpp/gcc10 364MB/s ± 0% 351MB/s ± 0% -3.62% (p=0.008 n=5+5) wuffs_jpeg_decode_30k_24bpp_progressive/gcc10 334MB/s ± 0% 334MB/s ± 0% -0.06% (p=0.032 n=5+5) wuffs_jpeg_decode_30k_24bpp_sequential/gcc10 944MB/s ± 0% 924MB/s ± 0% -2.19% (p=0.008 n=5+5) wuffs_jpeg_decode_77k_24bpp/gcc10 769MB/s ± 0% 740MB/s ± 0% -3.75% (p=0.008 n=5+5) wuffs_jpeg_decode_552k_24bpp_420/gcc10 842MB/s ± 0% 812MB/s ± 0% -3.60% (p=0.008 n=5+5) wuffs_jpeg_decode_552k_24bpp_444/gcc10 661MB/s ± 0% 647MB/s ± 0% -2.13% (p=0.008 n=5+5) wuffs_jpeg_decode_4002k_24bpp/gcc10 849MB/s ± 0% 817MB/s ± 0% -3.77% (p=0.016 n=5+4)
This commit is contained in:
1 parent
4facef4e01
commit
35c1f76f86
3 files changed
+338
No files matched your search
@@ -222,6 +222,8 @@ wuffs_base__pixel_swizzler__swizzle_ycc__convert_bgrx_x86_avx2(
|
||||
// mix01 = [b01 g01 b03 g03 .. b0F g0F b11 g11 .. b1D g1D b1F g1F]
|
||||
// mix02 = [r00 % r02 % .. r0E % r10 % .. r1C % r1E %]
|
||||
// mix03 = [r01 % r03 % .. r0F % r11 % .. r1D % r1F %]
|
||||
//
|
||||
// See also § below.
|
||||
__m256i mix00 = _mm256_unpacklo_epi8(packed_b_eve, packed_g_eve);
|
||||
__m256i mix01 = _mm256_unpacklo_epi8(packed_b_odd, packed_g_odd);
|
||||
__m256i mix02 = _mm256_unpacklo_epi8(packed_r_eve, uFFFF);
|
||||
@@ -282,5 +284,154 @@ wuffs_base__pixel_swizzler__swizzle_ycc__convert_bgrx_x86_avx2(
|
||||
x += n;
|
||||
}
|
||||
}
|
||||
|
||||
// The rgbx flavor (below) is exactly the same as the bgrx flavor (above)
|
||||
// except for the lines marked with a § and that comments were stripped.
|
||||
WUFFS_BASE__MAYBE_ATTRIBUTE_TARGET("pclmul,popcnt,sse4.2,avx2")
|
||||
static void //
|
||||
wuffs_base__pixel_swizzler__swizzle_ycc__convert_rgbx_x86_avx2(
|
||||
wuffs_base__pixel_buffer* dst,
|
||||
uint32_t x,
|
||||
uint32_t x_end,
|
||||
uint32_t y,
|
||||
const uint8_t* up0,
|
||||
const uint8_t* up1,
|
||||
const uint8_t* up2) {
|
||||
if ((x + 32u) > x_end) {
|
||||
wuffs_base__pixel_swizzler__swizzle_ycc__convert_bgrx( //
|
||||
dst, x, x_end, y, up0, up1, up2);
|
||||
return;
|
||||
}
|
||||
|
||||
size_t dst_stride = dst->private_impl.planes[0].stride;
|
||||
uint8_t* dst_iter = dst->private_impl.planes[0].ptr +
|
||||
(dst_stride * ((size_t)y)) + (4 * ((size_t)x));
|
||||
|
||||
const __m256i u0001 = _mm256_set1_epi16(0x0001);
|
||||
const __m256i u00FF = _mm256_set1_epi16(0x00FF);
|
||||
const __m256i uFF80 = _mm256_set1_epi16(0xFF80);
|
||||
const __m256i uFFFF = _mm256_set1_epi16(0xFFFF);
|
||||
|
||||
const __m256i p8000_p0000 = _mm256_set_epi16( //
|
||||
0x0000, 0x8000, 0x0000, 0x8000, 0x0000, 0x8000, 0x0000, 0x8000, //
|
||||
0x0000, 0x8000, 0x0000, 0x8000, 0x0000, 0x8000, 0x0000, 0x8000);
|
||||
|
||||
const __m256i m3A5E = _mm256_set1_epi16(-0x3A5E);
|
||||
const __m256i p66E9 = _mm256_set1_epi16(+0x66E9);
|
||||
const __m256i m581A_p492E = _mm256_set_epi16( //
|
||||
+0x492E, -0x581A, +0x492E, -0x581A, //
|
||||
+0x492E, -0x581A, +0x492E, -0x581A, //
|
||||
+0x492E, -0x581A, +0x492E, -0x581A, //
|
||||
+0x492E, -0x581A, +0x492E, -0x581A);
|
||||
|
||||
while (x < x_end) {
|
||||
__m256i cb_all = _mm256_lddqu_si256((const __m256i*)(const void*)up1);
|
||||
__m256i cr_all = _mm256_lddqu_si256((const __m256i*)(const void*)up2);
|
||||
__m256i cb_eve = _mm256_add_epi16(uFF80, _mm256_and_si256(cb_all, u00FF));
|
||||
__m256i cr_eve = _mm256_add_epi16(uFF80, _mm256_and_si256(cr_all, u00FF));
|
||||
__m256i cb_odd = _mm256_add_epi16(uFF80, _mm256_srli_epi16(cb_all, 8));
|
||||
__m256i cr_odd = _mm256_add_epi16(uFF80, _mm256_srli_epi16(cr_all, 8));
|
||||
|
||||
__m256i tmp_by_eve = _mm256_srai_epi16(
|
||||
_mm256_add_epi16(
|
||||
_mm256_mulhi_epi16(_mm256_add_epi16(cb_eve, cb_eve), m3A5E), u0001),
|
||||
1);
|
||||
__m256i tmp_by_odd = _mm256_srai_epi16(
|
||||
_mm256_add_epi16(
|
||||
_mm256_mulhi_epi16(_mm256_add_epi16(cb_odd, cb_odd), m3A5E), u0001),
|
||||
1);
|
||||
__m256i tmp_ry_eve = _mm256_srai_epi16(
|
||||
_mm256_add_epi16(
|
||||
_mm256_mulhi_epi16(_mm256_add_epi16(cr_eve, cr_eve), p66E9), u0001),
|
||||
1);
|
||||
__m256i tmp_ry_odd = _mm256_srai_epi16(
|
||||
_mm256_add_epi16(
|
||||
_mm256_mulhi_epi16(_mm256_add_epi16(cr_odd, cr_odd), p66E9), u0001),
|
||||
1);
|
||||
|
||||
__m256i by_eve =
|
||||
_mm256_add_epi16(tmp_by_eve, _mm256_add_epi16(cb_eve, cb_eve));
|
||||
__m256i by_odd =
|
||||
_mm256_add_epi16(tmp_by_odd, _mm256_add_epi16(cb_odd, cb_odd));
|
||||
__m256i ry_eve = _mm256_add_epi16(tmp_ry_eve, cr_eve);
|
||||
__m256i ry_odd = _mm256_add_epi16(tmp_ry_odd, cr_odd);
|
||||
|
||||
__m256i tmp0_gy_eve_lo = _mm256_madd_epi16( //
|
||||
_mm256_unpacklo_epi16(cb_eve, cr_eve), m581A_p492E);
|
||||
__m256i tmp0_gy_eve_hi = _mm256_madd_epi16( //
|
||||
_mm256_unpackhi_epi16(cb_eve, cr_eve), m581A_p492E);
|
||||
__m256i tmp0_gy_odd_lo = _mm256_madd_epi16( //
|
||||
_mm256_unpacklo_epi16(cb_odd, cr_odd), m581A_p492E);
|
||||
__m256i tmp0_gy_odd_hi = _mm256_madd_epi16( //
|
||||
_mm256_unpackhi_epi16(cb_odd, cr_odd), m581A_p492E);
|
||||
|
||||
__m256i tmp1_gy_eve_lo =
|
||||
_mm256_srai_epi32(_mm256_add_epi32(tmp0_gy_eve_lo, p8000_p0000), 16);
|
||||
__m256i tmp1_gy_eve_hi =
|
||||
_mm256_srai_epi32(_mm256_add_epi32(tmp0_gy_eve_hi, p8000_p0000), 16);
|
||||
__m256i tmp1_gy_odd_lo =
|
||||
_mm256_srai_epi32(_mm256_add_epi32(tmp0_gy_odd_lo, p8000_p0000), 16);
|
||||
__m256i tmp1_gy_odd_hi =
|
||||
_mm256_srai_epi32(_mm256_add_epi32(tmp0_gy_odd_hi, p8000_p0000), 16);
|
||||
|
||||
__m256i gy_eve = _mm256_sub_epi16(
|
||||
_mm256_packs_epi32(tmp1_gy_eve_lo, tmp1_gy_eve_hi), cr_eve);
|
||||
__m256i gy_odd = _mm256_sub_epi16(
|
||||
_mm256_packs_epi32(tmp1_gy_odd_lo, tmp1_gy_odd_hi), cr_odd);
|
||||
|
||||
__m256i yy_all = _mm256_lddqu_si256((const __m256i*)(const void*)up0);
|
||||
__m256i yy_eve = _mm256_and_si256(yy_all, u00FF);
|
||||
__m256i yy_odd = _mm256_srli_epi16(yy_all, 8);
|
||||
|
||||
__m256i loose_b_eve = _mm256_add_epi16(by_eve, yy_eve);
|
||||
__m256i loose_b_odd = _mm256_add_epi16(by_odd, yy_odd);
|
||||
__m256i packed_b_eve = _mm256_packus_epi16(loose_b_eve, loose_b_eve);
|
||||
__m256i packed_b_odd = _mm256_packus_epi16(loose_b_odd, loose_b_odd);
|
||||
|
||||
__m256i loose_g_eve = _mm256_add_epi16(gy_eve, yy_eve);
|
||||
__m256i loose_g_odd = _mm256_add_epi16(gy_odd, yy_odd);
|
||||
__m256i packed_g_eve = _mm256_packus_epi16(loose_g_eve, loose_g_eve);
|
||||
__m256i packed_g_odd = _mm256_packus_epi16(loose_g_odd, loose_g_odd);
|
||||
|
||||
__m256i loose_r_eve = _mm256_add_epi16(ry_eve, yy_eve);
|
||||
__m256i loose_r_odd = _mm256_add_epi16(ry_odd, yy_odd);
|
||||
__m256i packed_r_eve = _mm256_packus_epi16(loose_r_eve, loose_r_eve);
|
||||
__m256i packed_r_odd = _mm256_packus_epi16(loose_r_odd, loose_r_odd);
|
||||
|
||||
// § Note the swapped B and R channels.
|
||||
__m256i mix00 = _mm256_unpacklo_epi8(packed_r_eve, packed_g_eve);
|
||||
__m256i mix01 = _mm256_unpacklo_epi8(packed_r_odd, packed_g_odd);
|
||||
__m256i mix02 = _mm256_unpacklo_epi8(packed_b_eve, uFFFF);
|
||||
__m256i mix03 = _mm256_unpacklo_epi8(packed_b_odd, uFFFF);
|
||||
|
||||
__m256i mix10 = _mm256_unpacklo_epi16(mix00, mix02);
|
||||
__m256i mix11 = _mm256_unpacklo_epi16(mix01, mix03);
|
||||
__m256i mix12 = _mm256_unpackhi_epi16(mix00, mix02);
|
||||
__m256i mix13 = _mm256_unpackhi_epi16(mix01, mix03);
|
||||
|
||||
__m256i mix20 = _mm256_unpacklo_epi32(mix10, mix11);
|
||||
__m256i mix21 = _mm256_unpackhi_epi32(mix10, mix11);
|
||||
__m256i mix22 = _mm256_unpacklo_epi32(mix12, mix13);
|
||||
__m256i mix23 = _mm256_unpackhi_epi32(mix12, mix13);
|
||||
|
||||
__m256i mix30 = _mm256_permute2x128_si256(mix20, mix21, 0x20);
|
||||
__m256i mix31 = _mm256_permute2x128_si256(mix22, mix23, 0x20);
|
||||
__m256i mix32 = _mm256_permute2x128_si256(mix20, mix21, 0x31);
|
||||
__m256i mix33 = _mm256_permute2x128_si256(mix22, mix23, 0x31);
|
||||
|
||||
_mm256_storeu_si256((__m256i*)(void*)(dst_iter + 0x00), mix30);
|
||||
_mm256_storeu_si256((__m256i*)(void*)(dst_iter + 0x20), mix31);
|
||||
_mm256_storeu_si256((__m256i*)(void*)(dst_iter + 0x40), mix32);
|
||||
_mm256_storeu_si256((__m256i*)(void*)(dst_iter + 0x60), mix33);
|
||||
|
||||
uint32_t n = 32 - (31 & (x - x_end));
|
||||
dst_iter += 4 * n;
|
||||
up0 += n;
|
||||
up1 += n;
|
||||
up2 += n;
|
||||
x += n;
|
||||
}
|
||||
}
|
||||
|
||||
#endif // defined(WUFFS_BASE__CPU_ARCH__X86_FAMILY)
|
||||
// ‼ WUFFS MULTI-FILE SECTION -x86_avx2
|
||||
@@ -25,6 +25,17 @@ wuffs_base__pixel_swizzler__swizzle_ycc__convert_bgrx_x86_avx2(
|
||||
const uint8_t* up0,
|
||||
const uint8_t* up1,
|
||||
const uint8_t* up2);
|
||||
|
||||
WUFFS_BASE__MAYBE_ATTRIBUTE_TARGET("pclmul,popcnt,sse4.2,avx2")
|
||||
static void //
|
||||
wuffs_base__pixel_swizzler__swizzle_ycc__convert_rgbx_x86_avx2(
|
||||
wuffs_base__pixel_buffer* dst,
|
||||
uint32_t x,
|
||||
uint32_t x_end,
|
||||
uint32_t y,
|
||||
const uint8_t* up0,
|
||||
const uint8_t* up1,
|
||||
const uint8_t* up2);
|
||||
#endif
|
||||
|
||||
// --------
|
||||
@@ -977,6 +988,13 @@ wuffs_base__pixel_swizzler__swizzle_ycck(
|
||||
case WUFFS_BASE__PIXEL_FORMAT__RGBA_NONPREMUL:
|
||||
case WUFFS_BASE__PIXEL_FORMAT__RGBA_PREMUL:
|
||||
case WUFFS_BASE__PIXEL_FORMAT__RGBX:
|
||||
#if defined(WUFFS_BASE__CPU_ARCH__X86_FAMILY)
|
||||
if (wuffs_base__cpu_arch__have_x86_sse42()) {
|
||||
convfunc =
|
||||
&wuffs_base__pixel_swizzler__swizzle_ycc__convert_rgbx_x86_avx2;
|
||||
break;
|
||||
}
|
||||
#endif
|
||||
convfunc = &wuffs_base__pixel_swizzler__swizzle_ycc__convert_rgbx;
|
||||
break;
|
||||
default:
|
||||
|
||||
@@ -23526,6 +23526,17 @@ wuffs_base__pixel_swizzler__swizzle_ycc__convert_bgrx_x86_avx2(
|
||||
const uint8_t* up0,
|
||||
const uint8_t* up1,
|
||||
const uint8_t* up2);
|
||||
|
||||
WUFFS_BASE__MAYBE_ATTRIBUTE_TARGET("pclmul,popcnt,sse4.2,avx2")
|
||||
static void //
|
||||
wuffs_base__pixel_swizzler__swizzle_ycc__convert_rgbx_x86_avx2(
|
||||
wuffs_base__pixel_buffer* dst,
|
||||
uint32_t x,
|
||||
uint32_t x_end,
|
||||
uint32_t y,
|
||||
const uint8_t* up0,
|
||||
const uint8_t* up1,
|
||||
const uint8_t* up2);
|
||||
#endif
|
||||
|
||||
// --------
|
||||
@@ -24742,6 +24753,13 @@ wuffs_base__pixel_swizzler__swizzle_ycck(
|
||||
case WUFFS_BASE__PIXEL_FORMAT__RGBA_NONPREMUL:
|
||||
case WUFFS_BASE__PIXEL_FORMAT__RGBA_PREMUL:
|
||||
case WUFFS_BASE__PIXEL_FORMAT__RGBX:
|
||||
#if defined(WUFFS_BASE__CPU_ARCH__X86_FAMILY)
|
||||
if (wuffs_base__cpu_arch__have_x86_sse42()) {
|
||||
convfunc =
|
||||
&wuffs_base__pixel_swizzler__swizzle_ycc__convert_rgbx_x86_avx2;
|
||||
break;
|
||||
}
|
||||
#endif
|
||||
convfunc = &wuffs_base__pixel_swizzler__swizzle_ycc__convert_rgbx;
|
||||
break;
|
||||
default:
|
||||
@@ -25020,6 +25038,8 @@ wuffs_base__pixel_swizzler__swizzle_ycc__convert_bgrx_x86_avx2(
|
||||
// mix01 = [b01 g01 b03 g03 .. b0F g0F b11 g11 .. b1D g1D b1F g1F]
|
||||
// mix02 = [r00 % r02 % .. r0E % r10 % .. r1C % r1E %]
|
||||
// mix03 = [r01 % r03 % .. r0F % r11 % .. r1D % r1F %]
|
||||
//
|
||||
// See also § below.
|
||||
__m256i mix00 = _mm256_unpacklo_epi8(packed_b_eve, packed_g_eve);
|
||||
__m256i mix01 = _mm256_unpacklo_epi8(packed_b_odd, packed_g_odd);
|
||||
__m256i mix02 = _mm256_unpacklo_epi8(packed_r_eve, uFFFF);
|
||||
@@ -25080,6 +25100,155 @@ wuffs_base__pixel_swizzler__swizzle_ycc__convert_bgrx_x86_avx2(
|
||||
x += n;
|
||||
}
|
||||
}
|
||||
|
||||
// The rgbx flavor (below) is exactly the same as the bgrx flavor (above)
|
||||
// except for the lines marked with a § and that comments were stripped.
|
||||
WUFFS_BASE__MAYBE_ATTRIBUTE_TARGET("pclmul,popcnt,sse4.2,avx2")
|
||||
static void //
|
||||
wuffs_base__pixel_swizzler__swizzle_ycc__convert_rgbx_x86_avx2(
|
||||
wuffs_base__pixel_buffer* dst,
|
||||
uint32_t x,
|
||||
uint32_t x_end,
|
||||
uint32_t y,
|
||||
const uint8_t* up0,
|
||||
const uint8_t* up1,
|
||||
const uint8_t* up2) {
|
||||
if ((x + 32u) > x_end) {
|
||||
wuffs_base__pixel_swizzler__swizzle_ycc__convert_bgrx( //
|
||||
dst, x, x_end, y, up0, up1, up2);
|
||||
return;
|
||||
}
|
||||
|
||||
size_t dst_stride = dst->private_impl.planes[0].stride;
|
||||
uint8_t* dst_iter = dst->private_impl.planes[0].ptr +
|
||||
(dst_stride * ((size_t)y)) + (4 * ((size_t)x));
|
||||
|
||||
const __m256i u0001 = _mm256_set1_epi16(0x0001);
|
||||
const __m256i u00FF = _mm256_set1_epi16(0x00FF);
|
||||
const __m256i uFF80 = _mm256_set1_epi16(0xFF80);
|
||||
const __m256i uFFFF = _mm256_set1_epi16(0xFFFF);
|
||||
|
||||
const __m256i p8000_p0000 = _mm256_set_epi16( //
|
||||
0x0000, 0x8000, 0x0000, 0x8000, 0x0000, 0x8000, 0x0000, 0x8000, //
|
||||
0x0000, 0x8000, 0x0000, 0x8000, 0x0000, 0x8000, 0x0000, 0x8000);
|
||||
|
||||
const __m256i m3A5E = _mm256_set1_epi16(-0x3A5E);
|
||||
const __m256i p66E9 = _mm256_set1_epi16(+0x66E9);
|
||||
const __m256i m581A_p492E = _mm256_set_epi16( //
|
||||
+0x492E, -0x581A, +0x492E, -0x581A, //
|
||||
+0x492E, -0x581A, +0x492E, -0x581A, //
|
||||
+0x492E, -0x581A, +0x492E, -0x581A, //
|
||||
+0x492E, -0x581A, +0x492E, -0x581A);
|
||||
|
||||
while (x < x_end) {
|
||||
__m256i cb_all = _mm256_lddqu_si256((const __m256i*)(const void*)up1);
|
||||
__m256i cr_all = _mm256_lddqu_si256((const __m256i*)(const void*)up2);
|
||||
__m256i cb_eve = _mm256_add_epi16(uFF80, _mm256_and_si256(cb_all, u00FF));
|
||||
__m256i cr_eve = _mm256_add_epi16(uFF80, _mm256_and_si256(cr_all, u00FF));
|
||||
__m256i cb_odd = _mm256_add_epi16(uFF80, _mm256_srli_epi16(cb_all, 8));
|
||||
__m256i cr_odd = _mm256_add_epi16(uFF80, _mm256_srli_epi16(cr_all, 8));
|
||||
|
||||
__m256i tmp_by_eve = _mm256_srai_epi16(
|
||||
_mm256_add_epi16(
|
||||
_mm256_mulhi_epi16(_mm256_add_epi16(cb_eve, cb_eve), m3A5E), u0001),
|
||||
1);
|
||||
__m256i tmp_by_odd = _mm256_srai_epi16(
|
||||
_mm256_add_epi16(
|
||||
_mm256_mulhi_epi16(_mm256_add_epi16(cb_odd, cb_odd), m3A5E), u0001),
|
||||
1);
|
||||
__m256i tmp_ry_eve = _mm256_srai_epi16(
|
||||
_mm256_add_epi16(
|
||||
_mm256_mulhi_epi16(_mm256_add_epi16(cr_eve, cr_eve), p66E9), u0001),
|
||||
1);
|
||||
__m256i tmp_ry_odd = _mm256_srai_epi16(
|
||||
_mm256_add_epi16(
|
||||
_mm256_mulhi_epi16(_mm256_add_epi16(cr_odd, cr_odd), p66E9), u0001),
|
||||
1);
|
||||
|
||||
__m256i by_eve =
|
||||
_mm256_add_epi16(tmp_by_eve, _mm256_add_epi16(cb_eve, cb_eve));
|
||||
__m256i by_odd =
|
||||
_mm256_add_epi16(tmp_by_odd, _mm256_add_epi16(cb_odd, cb_odd));
|
||||
__m256i ry_eve = _mm256_add_epi16(tmp_ry_eve, cr_eve);
|
||||
__m256i ry_odd = _mm256_add_epi16(tmp_ry_odd, cr_odd);
|
||||
|
||||
__m256i tmp0_gy_eve_lo = _mm256_madd_epi16( //
|
||||
_mm256_unpacklo_epi16(cb_eve, cr_eve), m581A_p492E);
|
||||
__m256i tmp0_gy_eve_hi = _mm256_madd_epi16( //
|
||||
_mm256_unpackhi_epi16(cb_eve, cr_eve), m581A_p492E);
|
||||
__m256i tmp0_gy_odd_lo = _mm256_madd_epi16( //
|
||||
_mm256_unpacklo_epi16(cb_odd, cr_odd), m581A_p492E);
|
||||
__m256i tmp0_gy_odd_hi = _mm256_madd_epi16( //
|
||||
_mm256_unpackhi_epi16(cb_odd, cr_odd), m581A_p492E);
|
||||
|
||||
__m256i tmp1_gy_eve_lo =
|
||||
_mm256_srai_epi32(_mm256_add_epi32(tmp0_gy_eve_lo, p8000_p0000), 16);
|
||||
__m256i tmp1_gy_eve_hi =
|
||||
_mm256_srai_epi32(_mm256_add_epi32(tmp0_gy_eve_hi, p8000_p0000), 16);
|
||||
__m256i tmp1_gy_odd_lo =
|
||||
_mm256_srai_epi32(_mm256_add_epi32(tmp0_gy_odd_lo, p8000_p0000), 16);
|
||||
__m256i tmp1_gy_odd_hi =
|
||||
_mm256_srai_epi32(_mm256_add_epi32(tmp0_gy_odd_hi, p8000_p0000), 16);
|
||||
|
||||
__m256i gy_eve = _mm256_sub_epi16(
|
||||
_mm256_packs_epi32(tmp1_gy_eve_lo, tmp1_gy_eve_hi), cr_eve);
|
||||
__m256i gy_odd = _mm256_sub_epi16(
|
||||
_mm256_packs_epi32(tmp1_gy_odd_lo, tmp1_gy_odd_hi), cr_odd);
|
||||
|
||||
__m256i yy_all = _mm256_lddqu_si256((const __m256i*)(const void*)up0);
|
||||
__m256i yy_eve = _mm256_and_si256(yy_all, u00FF);
|
||||
__m256i yy_odd = _mm256_srli_epi16(yy_all, 8);
|
||||
|
||||
__m256i loose_b_eve = _mm256_add_epi16(by_eve, yy_eve);
|
||||
__m256i loose_b_odd = _mm256_add_epi16(by_odd, yy_odd);
|
||||
__m256i packed_b_eve = _mm256_packus_epi16(loose_b_eve, loose_b_eve);
|
||||
__m256i packed_b_odd = _mm256_packus_epi16(loose_b_odd, loose_b_odd);
|
||||
|
||||
__m256i loose_g_eve = _mm256_add_epi16(gy_eve, yy_eve);
|
||||
__m256i loose_g_odd = _mm256_add_epi16(gy_odd, yy_odd);
|
||||
__m256i packed_g_eve = _mm256_packus_epi16(loose_g_eve, loose_g_eve);
|
||||
__m256i packed_g_odd = _mm256_packus_epi16(loose_g_odd, loose_g_odd);
|
||||
|
||||
__m256i loose_r_eve = _mm256_add_epi16(ry_eve, yy_eve);
|
||||
__m256i loose_r_odd = _mm256_add_epi16(ry_odd, yy_odd);
|
||||
__m256i packed_r_eve = _mm256_packus_epi16(loose_r_eve, loose_r_eve);
|
||||
__m256i packed_r_odd = _mm256_packus_epi16(loose_r_odd, loose_r_odd);
|
||||
|
||||
// § Note the swapped B and R channels.
|
||||
__m256i mix00 = _mm256_unpacklo_epi8(packed_r_eve, packed_g_eve);
|
||||
__m256i mix01 = _mm256_unpacklo_epi8(packed_r_odd, packed_g_odd);
|
||||
__m256i mix02 = _mm256_unpacklo_epi8(packed_b_eve, uFFFF);
|
||||
__m256i mix03 = _mm256_unpacklo_epi8(packed_b_odd, uFFFF);
|
||||
|
||||
__m256i mix10 = _mm256_unpacklo_epi16(mix00, mix02);
|
||||
__m256i mix11 = _mm256_unpacklo_epi16(mix01, mix03);
|
||||
__m256i mix12 = _mm256_unpackhi_epi16(mix00, mix02);
|
||||
__m256i mix13 = _mm256_unpackhi_epi16(mix01, mix03);
|
||||
|
||||
__m256i mix20 = _mm256_unpacklo_epi32(mix10, mix11);
|
||||
__m256i mix21 = _mm256_unpackhi_epi32(mix10, mix11);
|
||||
__m256i mix22 = _mm256_unpacklo_epi32(mix12, mix13);
|
||||
__m256i mix23 = _mm256_unpackhi_epi32(mix12, mix13);
|
||||
|
||||
__m256i mix30 = _mm256_permute2x128_si256(mix20, mix21, 0x20);
|
||||
__m256i mix31 = _mm256_permute2x128_si256(mix22, mix23, 0x20);
|
||||
__m256i mix32 = _mm256_permute2x128_si256(mix20, mix21, 0x31);
|
||||
__m256i mix33 = _mm256_permute2x128_si256(mix22, mix23, 0x31);
|
||||
|
||||
_mm256_storeu_si256((__m256i*)(void*)(dst_iter + 0x00), mix30);
|
||||
_mm256_storeu_si256((__m256i*)(void*)(dst_iter + 0x20), mix31);
|
||||
_mm256_storeu_si256((__m256i*)(void*)(dst_iter + 0x40), mix32);
|
||||
_mm256_storeu_si256((__m256i*)(void*)(dst_iter + 0x60), mix33);
|
||||
|
||||
uint32_t n = 32 - (31 & (x - x_end));
|
||||
dst_iter += 4 * n;
|
||||
up0 += n;
|
||||
up1 += n;
|
||||
up2 += n;
|
||||
x += n;
|
||||
}
|
||||
}
|
||||
|
||||
#endif // defined(WUFFS_BASE__CPU_ARCH__X86_FAMILY)
|
||||
// ‼ WUFFS MULTI-FILE SECTION -x86_avx2
|
||||
|
||||
|
||||
Reference in new issue
Block a user