Files
wuffs/internal/cgen/base/pixconv-submodule-x86-avx2.c
T
Nigel Tao 74a6ea2fcd all: relicense from Apache2 to dual Apache2/MIT
The LICENSE has changed from a single license (Apache2) to a dual
license (Apache2 or MIT, at your option).

If you've previously used Wuffs under the Apache2 license, you are able
to keep doing so. No further action required by you.

The two files listed below (previous releases) are exceptions, as they
are older snapshots, frozen in time. Their license has not changed:
release/c/wuffs-v0.2.c
release/c/wuffs-v0.3.c

Every name in the AUTHORS file, other than Google Inc., has signed
Google's CLA (Contributor License Agreement).
2023-10-06 10:00:59 +11:00

639 lines
28 KiB
C

// Copyright 2023 The Wuffs Authors.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
// --------
// ‼ WUFFS MULTI-FILE SECTION +x86_avx2
#if defined(WUFFS_BASE__CPU_ARCH__X86_64)
WUFFS_BASE__MAYBE_ATTRIBUTE_TARGET("pclmul,popcnt,sse4.2,avx2")
static void //
wuffs_base__pixel_swizzler__swizzle_ycc__convert_3_bgrx_x86_avx2(
wuffs_base__pixel_buffer* dst,
uint32_t x,
uint32_t x_end,
uint32_t y,
const uint8_t* up0,
const uint8_t* up1,
const uint8_t* up2) {
if ((x + 32u) > x_end) {
wuffs_base__pixel_swizzler__swizzle_ycc__convert_3_bgrx( //
dst, x, x_end, y, up0, up1, up2);
return;
}
size_t dst_stride = dst->private_impl.planes[0].stride;
uint8_t* dst_iter = dst->private_impl.planes[0].ptr +
(dst_stride * ((size_t)y)) + (4u * ((size_t)x));
// u0001 = u16x16 [0x0001 .. 0x0001]
// u00FF = u16x16 [0x00FF .. 0x00FF]
// uFF80 = u16x16 [0xFF80 .. 0xFF80]
// uFFFF = u16x16 [0xFFFF .. 0xFFFF]
const __m256i u0001 = _mm256_set1_epi16(+0x0001);
const __m256i u00FF = _mm256_set1_epi16(+0x00FF);
const __m256i uFF80 = _mm256_set1_epi16(-0x0080);
const __m256i uFFFF = _mm256_set1_epi16(-0x0001);
// p8000_p0000 = u16x16 [0x8000 0x0000 .. 0x8000 0x0000]
const __m256i p8000_p0000 = _mm256_set_epi16( //
+0x0000, -0x8000, +0x0000, -0x8000, //
+0x0000, -0x8000, +0x0000, -0x8000, //
+0x0000, -0x8000, +0x0000, -0x8000, //
+0x0000, -0x8000, +0x0000, -0x8000);
// Per wuffs_base__color_ycc__as__color_u32, the formulae:
//
// R = Y + 1.40200 * Cr
// G = Y - 0.34414 * Cb - 0.71414 * Cr
// B = Y + 1.77200 * Cb
//
// When scaled by 1<<16:
//
// 0.34414 becomes 0x0581A = 22554.
// 0.71414 becomes 0x0B6D2 = 46802.
// 1.40200 becomes 0x166E9 = 91881.
// 1.77200 becomes 0x1C5A2 = 116130.
//
// Separate the integer and fractional parts, since we work with signed
// 16-bit SIMD lanes. The fractional parts range from -0.5 .. +0.5 (as
// floating-point) which is from -0x8000 .. +0x8000 (as fixed-point).
//
// -0x3A5E = -0x20000 + 0x1C5A2 The B:Cb factor.
// +0x66E9 = -0x10000 + 0x166E9 The R:Cr factor.
// -0x581A = +0x00000 - 0x0581A The G:Cb factor.
// +0x492E = +0x10000 - 0x0B6D2 The G:Cr factor.
const __m256i m3A5E = _mm256_set1_epi16(-0x3A5E);
const __m256i p66E9 = _mm256_set1_epi16(+0x66E9);
const __m256i m581A_p492E = _mm256_set_epi16( //
+0x492E, -0x581A, +0x492E, -0x581A, //
+0x492E, -0x581A, +0x492E, -0x581A, //
+0x492E, -0x581A, +0x492E, -0x581A, //
+0x492E, -0x581A, +0x492E, -0x581A);
while (x < x_end) {
// Load chroma values in even and odd columns (the high 8 bits of each
// u16x16 element are zero) and then subtract 0x0080.
//
// cb_all = u8x32 [cb.00 cb.01 cb.02 cb.03 .. cb.1C cb.1D cb.1E cb.1F]
// cb_eve = i16x16 [cb.00-0x80 cb.02-0x80 .. cb.1C-0x80 cb.1E-0x80 ]
// cb_odd = i16x16 [cb.01-0x80 cb.03-0x80 .. cb.1D-0x80 cb.1F-0x80 ]
//
// Ditto for the cr_xxx Chroma-Red values.
__m256i cb_all = _mm256_lddqu_si256((const __m256i*)(const void*)up1);
__m256i cr_all = _mm256_lddqu_si256((const __m256i*)(const void*)up2);
__m256i cb_eve = _mm256_add_epi16(uFF80, _mm256_and_si256(cb_all, u00FF));
__m256i cr_eve = _mm256_add_epi16(uFF80, _mm256_and_si256(cr_all, u00FF));
__m256i cb_odd = _mm256_add_epi16(uFF80, _mm256_srli_epi16(cb_all, 8));
__m256i cr_odd = _mm256_add_epi16(uFF80, _mm256_srli_epi16(cr_all, 8));
// ----
// Calculate:
//
// B-Y = (+1.77200 * Cb) as floating-point
// R-Y = (+1.40200 * Cr) as floating-point
//
// B-Y = ((0x2_0000 - 0x3A5E) * Cb) as fixed-point
// R-Y = ((0x1_0000 + 0x66E9) * Cr) as fixed-point
//
// B-Y = ((-0x3A5E * Cb) + ("2.0" * Cb))
// R-Y = ((+0x66E9 * Cr) + ("1.0" * Cr))
// Multiply by m3A5E or p66E9, taking the high 16 bits. There's also a
// doubling (add x to itself), adding-of-1 and halving (shift right by 1).
// That makes multiply-and-take-high round to nearest (instead of down).
__m256i tmp_by_eve = _mm256_srai_epi16(
_mm256_add_epi16(
_mm256_mulhi_epi16(_mm256_add_epi16(cb_eve, cb_eve), m3A5E), u0001),
1);
__m256i tmp_by_odd = _mm256_srai_epi16(
_mm256_add_epi16(
_mm256_mulhi_epi16(_mm256_add_epi16(cb_odd, cb_odd), m3A5E), u0001),
1);
__m256i tmp_ry_eve = _mm256_srai_epi16(
_mm256_add_epi16(
_mm256_mulhi_epi16(_mm256_add_epi16(cr_eve, cr_eve), p66E9), u0001),
1);
__m256i tmp_ry_odd = _mm256_srai_epi16(
_mm256_add_epi16(
_mm256_mulhi_epi16(_mm256_add_epi16(cr_odd, cr_odd), p66E9), u0001),
1);
// Add (2 * Cb) and (1 * Cr).
__m256i by_eve =
_mm256_add_epi16(tmp_by_eve, _mm256_add_epi16(cb_eve, cb_eve));
__m256i by_odd =
_mm256_add_epi16(tmp_by_odd, _mm256_add_epi16(cb_odd, cb_odd));
__m256i ry_eve = _mm256_add_epi16(tmp_ry_eve, cr_eve);
__m256i ry_odd = _mm256_add_epi16(tmp_ry_odd, cr_odd);
// ----
// Calculate:
//
// G-Y = (-0.34414 * Cb) +
// (-0.71414 * Cr) as floating-point
//
// G-Y = ((+0x0_0000 - 0x581A) * Cb) +
// ((-0x1_0000 + 0x492E) * Cr) as fixed-point
//
// G-Y = (-0x581A * Cb) +
// (+0x492E * Cr) - ("1.0" * Cr)
// Multiply-add to get ((-0x581A * Cb) + (+0x492E * Cr)).
__m256i tmp0_gy_eve_lo = _mm256_madd_epi16( //
_mm256_unpacklo_epi16(cb_eve, cr_eve), m581A_p492E);
__m256i tmp0_gy_eve_hi = _mm256_madd_epi16( //
_mm256_unpackhi_epi16(cb_eve, cr_eve), m581A_p492E);
__m256i tmp0_gy_odd_lo = _mm256_madd_epi16( //
_mm256_unpacklo_epi16(cb_odd, cr_odd), m581A_p492E);
__m256i tmp0_gy_odd_hi = _mm256_madd_epi16( //
_mm256_unpackhi_epi16(cb_odd, cr_odd), m581A_p492E);
// Divide the i32x8 vectors by (1 << 16), rounding to nearest.
__m256i tmp1_gy_eve_lo =
_mm256_srai_epi32(_mm256_add_epi32(tmp0_gy_eve_lo, p8000_p0000), 16);
__m256i tmp1_gy_eve_hi =
_mm256_srai_epi32(_mm256_add_epi32(tmp0_gy_eve_hi, p8000_p0000), 16);
__m256i tmp1_gy_odd_lo =
_mm256_srai_epi32(_mm256_add_epi32(tmp0_gy_odd_lo, p8000_p0000), 16);
__m256i tmp1_gy_odd_hi =
_mm256_srai_epi32(_mm256_add_epi32(tmp0_gy_odd_hi, p8000_p0000), 16);
// Pack the ((-0x581A * Cb) + (+0x492E * Cr)) as i16x16 and subtract Cr.
__m256i gy_eve = _mm256_sub_epi16(
_mm256_packs_epi32(tmp1_gy_eve_lo, tmp1_gy_eve_hi), cr_eve);
__m256i gy_odd = _mm256_sub_epi16(
_mm256_packs_epi32(tmp1_gy_odd_lo, tmp1_gy_odd_hi), cr_odd);
// ----
// Add Y to (B-Y), (G-Y) and (R-Y) to produce B, G and R.
//
// For the resultant packed_x_xxx vectors, only elements 0 ..= 7 and 16 ..=
// 23 of the 32-element vectors matter (since we'll unpacklo but not
// unpackhi them). Let … denote 8 ignored consecutive u8 values and let %
// denote 0xFF. We'll end this section with:
//
// packed_b_eve = u8x32 [b00 b02 .. b0C b0E … b10 b12 .. b1C b1E …]
// packed_b_odd = u8x32 [b01 b03 .. b0D b0F … b11 b13 .. b1D b1F …]
// packed_g_eve = u8x32 [g00 g02 .. g0C g0E … g10 g12 .. g1C g1E …]
// packed_g_odd = u8x32 [g01 g03 .. g0D g0F … g11 g13 .. g1D g1F …]
// packed_r_eve = u8x32 [r00 r02 .. r0C r0E … r10 r12 .. r1C r1E …]
// packed_r_odd = u8x32 [r01 r03 .. r0D r0F … r11 r13 .. r1D r1F …]
// uFFFF = u8x32 [ % % .. % % … % % .. % % …]
__m256i yy_all = _mm256_lddqu_si256((const __m256i*)(const void*)up0);
__m256i yy_eve = _mm256_and_si256(yy_all, u00FF);
__m256i yy_odd = _mm256_srli_epi16(yy_all, 8);
__m256i loose_b_eve = _mm256_add_epi16(by_eve, yy_eve);
__m256i loose_b_odd = _mm256_add_epi16(by_odd, yy_odd);
__m256i packed_b_eve = _mm256_packus_epi16(loose_b_eve, loose_b_eve);
__m256i packed_b_odd = _mm256_packus_epi16(loose_b_odd, loose_b_odd);
__m256i loose_g_eve = _mm256_add_epi16(gy_eve, yy_eve);
__m256i loose_g_odd = _mm256_add_epi16(gy_odd, yy_odd);
__m256i packed_g_eve = _mm256_packus_epi16(loose_g_eve, loose_g_eve);
__m256i packed_g_odd = _mm256_packus_epi16(loose_g_odd, loose_g_odd);
__m256i loose_r_eve = _mm256_add_epi16(ry_eve, yy_eve);
__m256i loose_r_odd = _mm256_add_epi16(ry_odd, yy_odd);
__m256i packed_r_eve = _mm256_packus_epi16(loose_r_eve, loose_r_eve);
__m256i packed_r_odd = _mm256_packus_epi16(loose_r_odd, loose_r_odd);
// ----
// Mix those values (unpacking in 8, 16 and then 32 bit units) to get the
// desired BGRX/RGBX order.
//
// From here onwards, all of our __m256i registers are u8x32.
// mix00 = [b00 g00 b02 g02 .. b0E g0E b10 g10 .. b1C g1C b1E g1E]
// mix01 = [b01 g01 b03 g03 .. b0F g0F b11 g11 .. b1D g1D b1F g1F]
// mix02 = [r00 % r02 % .. r0E % r10 % .. r1C % r1E %]
// mix03 = [r01 % r03 % .. r0F % r11 % .. r1D % r1F %]
//
// See also § below.
__m256i mix00 = _mm256_unpacklo_epi8(packed_b_eve, packed_g_eve);
__m256i mix01 = _mm256_unpacklo_epi8(packed_b_odd, packed_g_odd);
__m256i mix02 = _mm256_unpacklo_epi8(packed_r_eve, uFFFF);
__m256i mix03 = _mm256_unpacklo_epi8(packed_r_odd, uFFFF);
// mix10 = [b00 g00 r00 % b02 g02 r02 % b04 g04 r04 % b06 g06 r06 %
// b10 g10 r10 % b12 g12 r12 % b14 g14 r14 % b16 g16 r16 %]
// mix11 = [b01 g01 r01 % b03 g03 r03 % b05 g05 r05 % b07 g07 r07 %
// b11 g11 r11 % b13 g13 r13 % b15 g15 r15 % b17 g17 r17 %]
// mix12 = [b08 g08 r08 % b0A g0A r0A % b0C g0C r0C % b0E g0E r0E %
// b18 g18 r18 % b1A g1A r1A % b1C g1C r1C % b1E g1E r1E %]
// mix13 = [b09 g09 r09 % b0B g0B r0B % b0D g0D r0D % b0F g0F r0F %
// b19 g19 r19 % b1B g1B r1B % b1D g1D r1D % b1F g1F r1F %]
__m256i mix10 = _mm256_unpacklo_epi16(mix00, mix02);
__m256i mix11 = _mm256_unpacklo_epi16(mix01, mix03);
__m256i mix12 = _mm256_unpackhi_epi16(mix00, mix02);
__m256i mix13 = _mm256_unpackhi_epi16(mix01, mix03);
// mix20 = [b00 g00 r00 % b01 g01 r01 % b02 g02 r02 % b03 g03 r03 %
// b10 g10 r10 % b11 g11 r11 % b12 g12 r12 % b13 g13 r13 %]
// mix21 = [b04 g04 r04 % b05 g05 r05 % b06 g06 r06 % b07 g07 r07 %
// b14 g14 r14 % b15 g15 r15 % b16 g16 r16 % b17 g17 r17 %]
// mix22 = [b08 g08 r08 % b09 g09 r09 % b0A g0A r0A % b0B g0B r0B %
// b18 g18 r18 % b19 g19 r19 % b1A g1A r1A % b1B g1B r1B %]
// mix23 = [b0C g0C r0C % b0D g0D r0D % b0E g0E r0E % b0F g0F r0F %
// b1C g1C r1C % b1D g1D r1D % b1E g1E r1E % b1F g1F r1F %]
__m256i mix20 = _mm256_unpacklo_epi32(mix10, mix11);
__m256i mix21 = _mm256_unpackhi_epi32(mix10, mix11);
__m256i mix22 = _mm256_unpacklo_epi32(mix12, mix13);
__m256i mix23 = _mm256_unpackhi_epi32(mix12, mix13);
// mix30 = [b00 g00 r00 % b01 g01 r01 % b02 g02 r02 % b03 g03 r03 %
// b04 g04 r04 % b05 g05 r05 % b06 g06 r06 % b07 g07 r07 %]
// mix31 = [b08 g08 r08 % b09 g09 r09 % b0A g0A r0A % b0B g0B r0B %
// b0C g0C r0C % b0D g0D r0D % b0E g0E r0E % b0F g0F r0F %]
// mix32 = [b10 g10 r10 % b11 g11 r11 % b12 g12 r12 % b13 g13 r13 %
// b14 g14 r14 % b15 g15 r15 % b16 g16 r16 % b17 g17 r17 %]
// mix33 = [b18 g18 r18 % b19 g19 r19 % b1A g1A r1A % b1B g1B r1B %
// b1C g1C r1C % b1D g1D r1D % b1E g1E r1E % b1F g1F r1F %]
__m256i mix30 = _mm256_permute2x128_si256(mix20, mix21, 0x20);
__m256i mix31 = _mm256_permute2x128_si256(mix22, mix23, 0x20);
__m256i mix32 = _mm256_permute2x128_si256(mix20, mix21, 0x31);
__m256i mix33 = _mm256_permute2x128_si256(mix22, mix23, 0x31);
// Write out four u8x32 SIMD registers (128 bytes, 32 BGRX/RGBX pixels).
_mm256_storeu_si256((__m256i*)(void*)(dst_iter + 0x00), mix30);
_mm256_storeu_si256((__m256i*)(void*)(dst_iter + 0x20), mix31);
_mm256_storeu_si256((__m256i*)(void*)(dst_iter + 0x40), mix32);
_mm256_storeu_si256((__m256i*)(void*)(dst_iter + 0x60), mix33);
// Advance by up to 32 pixels. The first iteration might be smaller than 32
// so that all of the remaining steps are exactly 32.
uint32_t n = 32u - (31u & (x - x_end));
dst_iter += 4u * n;
up0 += n;
up1 += n;
up2 += n;
x += n;
}
}
// The rgbx flavor (below) is exactly the same as the bgrx flavor (above)
// except for the lines marked with a § and that comments were stripped.
WUFFS_BASE__MAYBE_ATTRIBUTE_TARGET("pclmul,popcnt,sse4.2,avx2")
static void //
wuffs_base__pixel_swizzler__swizzle_ycc__convert_3_rgbx_x86_avx2(
wuffs_base__pixel_buffer* dst,
uint32_t x,
uint32_t x_end,
uint32_t y,
const uint8_t* up0,
const uint8_t* up1,
const uint8_t* up2) {
if ((x + 32u) > x_end) {
wuffs_base__pixel_swizzler__swizzle_ycc__convert_3_bgrx( //
dst, x, x_end, y, up0, up1, up2);
return;
}
size_t dst_stride = dst->private_impl.planes[0].stride;
uint8_t* dst_iter = dst->private_impl.planes[0].ptr +
(dst_stride * ((size_t)y)) + (4u * ((size_t)x));
const __m256i u0001 = _mm256_set1_epi16(+0x0001);
const __m256i u00FF = _mm256_set1_epi16(+0x00FF);
const __m256i uFF80 = _mm256_set1_epi16(-0x0080);
const __m256i uFFFF = _mm256_set1_epi16(-0x0001);
const __m256i p8000_p0000 = _mm256_set_epi16( //
+0x0000, -0x8000, +0x0000, -0x8000, //
+0x0000, -0x8000, +0x0000, -0x8000, //
+0x0000, -0x8000, +0x0000, -0x8000, //
+0x0000, -0x8000, +0x0000, -0x8000);
const __m256i m3A5E = _mm256_set1_epi16(-0x3A5E);
const __m256i p66E9 = _mm256_set1_epi16(+0x66E9);
const __m256i m581A_p492E = _mm256_set_epi16( //
+0x492E, -0x581A, +0x492E, -0x581A, //
+0x492E, -0x581A, +0x492E, -0x581A, //
+0x492E, -0x581A, +0x492E, -0x581A, //
+0x492E, -0x581A, +0x492E, -0x581A);
while (x < x_end) {
__m256i cb_all = _mm256_lddqu_si256((const __m256i*)(const void*)up1);
__m256i cr_all = _mm256_lddqu_si256((const __m256i*)(const void*)up2);
__m256i cb_eve = _mm256_add_epi16(uFF80, _mm256_and_si256(cb_all, u00FF));
__m256i cr_eve = _mm256_add_epi16(uFF80, _mm256_and_si256(cr_all, u00FF));
__m256i cb_odd = _mm256_add_epi16(uFF80, _mm256_srli_epi16(cb_all, 8));
__m256i cr_odd = _mm256_add_epi16(uFF80, _mm256_srli_epi16(cr_all, 8));
__m256i tmp_by_eve = _mm256_srai_epi16(
_mm256_add_epi16(
_mm256_mulhi_epi16(_mm256_add_epi16(cb_eve, cb_eve), m3A5E), u0001),
1);
__m256i tmp_by_odd = _mm256_srai_epi16(
_mm256_add_epi16(
_mm256_mulhi_epi16(_mm256_add_epi16(cb_odd, cb_odd), m3A5E), u0001),
1);
__m256i tmp_ry_eve = _mm256_srai_epi16(
_mm256_add_epi16(
_mm256_mulhi_epi16(_mm256_add_epi16(cr_eve, cr_eve), p66E9), u0001),
1);
__m256i tmp_ry_odd = _mm256_srai_epi16(
_mm256_add_epi16(
_mm256_mulhi_epi16(_mm256_add_epi16(cr_odd, cr_odd), p66E9), u0001),
1);
__m256i by_eve =
_mm256_add_epi16(tmp_by_eve, _mm256_add_epi16(cb_eve, cb_eve));
__m256i by_odd =
_mm256_add_epi16(tmp_by_odd, _mm256_add_epi16(cb_odd, cb_odd));
__m256i ry_eve = _mm256_add_epi16(tmp_ry_eve, cr_eve);
__m256i ry_odd = _mm256_add_epi16(tmp_ry_odd, cr_odd);
__m256i tmp0_gy_eve_lo = _mm256_madd_epi16( //
_mm256_unpacklo_epi16(cb_eve, cr_eve), m581A_p492E);
__m256i tmp0_gy_eve_hi = _mm256_madd_epi16( //
_mm256_unpackhi_epi16(cb_eve, cr_eve), m581A_p492E);
__m256i tmp0_gy_odd_lo = _mm256_madd_epi16( //
_mm256_unpacklo_epi16(cb_odd, cr_odd), m581A_p492E);
__m256i tmp0_gy_odd_hi = _mm256_madd_epi16( //
_mm256_unpackhi_epi16(cb_odd, cr_odd), m581A_p492E);
__m256i tmp1_gy_eve_lo =
_mm256_srai_epi32(_mm256_add_epi32(tmp0_gy_eve_lo, p8000_p0000), 16);
__m256i tmp1_gy_eve_hi =
_mm256_srai_epi32(_mm256_add_epi32(tmp0_gy_eve_hi, p8000_p0000), 16);
__m256i tmp1_gy_odd_lo =
_mm256_srai_epi32(_mm256_add_epi32(tmp0_gy_odd_lo, p8000_p0000), 16);
__m256i tmp1_gy_odd_hi =
_mm256_srai_epi32(_mm256_add_epi32(tmp0_gy_odd_hi, p8000_p0000), 16);
__m256i gy_eve = _mm256_sub_epi16(
_mm256_packs_epi32(tmp1_gy_eve_lo, tmp1_gy_eve_hi), cr_eve);
__m256i gy_odd = _mm256_sub_epi16(
_mm256_packs_epi32(tmp1_gy_odd_lo, tmp1_gy_odd_hi), cr_odd);
__m256i yy_all = _mm256_lddqu_si256((const __m256i*)(const void*)up0);
__m256i yy_eve = _mm256_and_si256(yy_all, u00FF);
__m256i yy_odd = _mm256_srli_epi16(yy_all, 8);
__m256i loose_b_eve = _mm256_add_epi16(by_eve, yy_eve);
__m256i loose_b_odd = _mm256_add_epi16(by_odd, yy_odd);
__m256i packed_b_eve = _mm256_packus_epi16(loose_b_eve, loose_b_eve);
__m256i packed_b_odd = _mm256_packus_epi16(loose_b_odd, loose_b_odd);
__m256i loose_g_eve = _mm256_add_epi16(gy_eve, yy_eve);
__m256i loose_g_odd = _mm256_add_epi16(gy_odd, yy_odd);
__m256i packed_g_eve = _mm256_packus_epi16(loose_g_eve, loose_g_eve);
__m256i packed_g_odd = _mm256_packus_epi16(loose_g_odd, loose_g_odd);
__m256i loose_r_eve = _mm256_add_epi16(ry_eve, yy_eve);
__m256i loose_r_odd = _mm256_add_epi16(ry_odd, yy_odd);
__m256i packed_r_eve = _mm256_packus_epi16(loose_r_eve, loose_r_eve);
__m256i packed_r_odd = _mm256_packus_epi16(loose_r_odd, loose_r_odd);
// § Note the swapped B and R channels.
__m256i mix00 = _mm256_unpacklo_epi8(packed_r_eve, packed_g_eve);
__m256i mix01 = _mm256_unpacklo_epi8(packed_r_odd, packed_g_odd);
__m256i mix02 = _mm256_unpacklo_epi8(packed_b_eve, uFFFF);
__m256i mix03 = _mm256_unpacklo_epi8(packed_b_odd, uFFFF);
__m256i mix10 = _mm256_unpacklo_epi16(mix00, mix02);
__m256i mix11 = _mm256_unpacklo_epi16(mix01, mix03);
__m256i mix12 = _mm256_unpackhi_epi16(mix00, mix02);
__m256i mix13 = _mm256_unpackhi_epi16(mix01, mix03);
__m256i mix20 = _mm256_unpacklo_epi32(mix10, mix11);
__m256i mix21 = _mm256_unpackhi_epi32(mix10, mix11);
__m256i mix22 = _mm256_unpacklo_epi32(mix12, mix13);
__m256i mix23 = _mm256_unpackhi_epi32(mix12, mix13);
__m256i mix30 = _mm256_permute2x128_si256(mix20, mix21, 0x20);
__m256i mix31 = _mm256_permute2x128_si256(mix22, mix23, 0x20);
__m256i mix32 = _mm256_permute2x128_si256(mix20, mix21, 0x31);
__m256i mix33 = _mm256_permute2x128_si256(mix22, mix23, 0x31);
_mm256_storeu_si256((__m256i*)(void*)(dst_iter + 0x00), mix30);
_mm256_storeu_si256((__m256i*)(void*)(dst_iter + 0x20), mix31);
_mm256_storeu_si256((__m256i*)(void*)(dst_iter + 0x40), mix32);
_mm256_storeu_si256((__m256i*)(void*)(dst_iter + 0x60), mix33);
uint32_t n = 32u - (31u & (x - x_end));
dst_iter += 4u * n;
up0 += n;
up1 += n;
up2 += n;
x += n;
}
}
#if defined(__GNUC__) && !defined(__clang__)
// No-op.
#else
WUFFS_BASE__MAYBE_ATTRIBUTE_TARGET("pclmul,popcnt,sse4.2,avx2")
static const uint8_t* //
wuffs_base__pixel_swizzler__swizzle_ycc__upsample_inv_h2v2_triangle_x86_avx2(
uint8_t* dst_ptr,
const uint8_t* src_ptr_major,
const uint8_t* src_ptr_minor,
size_t src_len,
uint32_t h1v2_bias_ignored,
bool first_column,
bool last_column) {
uint8_t* dp = dst_ptr;
const uint8_t* sp_major = src_ptr_major;
const uint8_t* sp_minor = src_ptr_minor;
if (first_column) {
src_len--;
if ((src_len <= 0u) && last_column) {
uint32_t sv = (12u * ((uint32_t)(*sp_major++))) + //
(4u * ((uint32_t)(*sp_minor++)));
*dp++ = (uint8_t)((sv + 8u) >> 4u);
*dp++ = (uint8_t)((sv + 7u) >> 4u);
return dst_ptr;
}
uint32_t sv_major_m1 = sp_major[-0]; // Clamp offset to zero.
uint32_t sv_minor_m1 = sp_minor[-0]; // Clamp offset to zero.
uint32_t sv_major_p1 = sp_major[+1];
uint32_t sv_minor_p1 = sp_minor[+1];
uint32_t sv = (9u * ((uint32_t)(*sp_major++))) + //
(3u * ((uint32_t)(*sp_minor++)));
*dp++ = (uint8_t)((sv + (3u * sv_major_m1) + (sv_minor_m1) + 8u) >> 4u);
*dp++ = (uint8_t)((sv + (3u * sv_major_p1) + (sv_minor_p1) + 7u) >> 4u);
if (src_len <= 0u) {
return dst_ptr;
}
}
if (last_column) {
src_len--;
}
if (src_len < 32) {
// This fallback is the same as the non-SIMD-capable code path.
for (; src_len > 0u; src_len--) {
uint32_t sv_major_m1 = sp_major[-1];
uint32_t sv_minor_m1 = sp_minor[-1];
uint32_t sv_major_p1 = sp_major[+1];
uint32_t sv_minor_p1 = sp_minor[+1];
uint32_t sv = (9u * ((uint32_t)(*sp_major++))) + //
(3u * ((uint32_t)(*sp_minor++)));
*dp++ = (uint8_t)((sv + (3u * sv_major_m1) + (sv_minor_m1) + 8u) >> 4u);
*dp++ = (uint8_t)((sv + (3u * sv_major_p1) + (sv_minor_p1) + 7u) >> 4u);
}
} else {
while (src_len > 0u) {
// Load 1+32+1 samples (six u8x32 vectors) from the major (jxx) and minor
// (nxx) rows.
//
// major_p0 = [j00 j01 j02 j03 .. j28 j29 j30 j31] // p0 = "plus 0"
// minor_p0 = [n00 n01 n02 n03 .. n28 n29 n30 n31] // p0 = "plus 0"
// major_m1 = [jm1 j00 j01 j02 .. j27 j28 j29 j30] // m1 = "minus 1"
// minor_m1 = [nm1 n00 n01 n02 .. n27 n28 n29 n30] // m1 = "minus 1"
// major_p1 = [j01 j02 j03 j04 .. j29 j30 j31 j32] // p1 = "plus 1"
// minor_p1 = [n01 n02 n03 n04 .. n29 n30 n31 n32] // p1 = "plus 1"
__m256i major_p0 =
_mm256_lddqu_si256((const __m256i*)(const void*)(sp_major + 0));
__m256i minor_p0 =
_mm256_lddqu_si256((const __m256i*)(const void*)(sp_minor + 0));
__m256i major_m1 =
_mm256_lddqu_si256((const __m256i*)(const void*)(sp_major - 1));
__m256i minor_m1 =
_mm256_lddqu_si256((const __m256i*)(const void*)(sp_minor - 1));
__m256i major_p1 =
_mm256_lddqu_si256((const __m256i*)(const void*)(sp_major + 1));
__m256i minor_p1 =
_mm256_lddqu_si256((const __m256i*)(const void*)(sp_minor + 1));
// Unpack, staying with u8x32 vectors.
//
// step1_p0_lo = [j00 n00 j01 n01 .. j07 n07 j16 n16 j17 n17 .. j23 n23]
// step1_p0_hi = [j08 n08 j09 n09 .. j15 n15 j24 n24 j25 n25 .. j31 n31]
// step1_m1_lo = [jm1 nm1 j00 n00 .. j06 n06 j15 n15 j16 n16 .. j22 n22]
// step1_m1_hi = [j07 n07 j08 n08 .. j14 n14 j23 n23 j24 n24 .. j30 n30]
// step1_p1_lo = [j01 n01 j02 n02 .. j08 n08 j17 n17 j18 n18 .. j24 n24]
// step1_p1_hi = [j09 n09 j10 n10 .. j16 n16 j25 n25 j26 n26 .. j32 n32]
__m256i step1_p0_lo = _mm256_unpacklo_epi8(major_p0, minor_p0);
__m256i step1_p0_hi = _mm256_unpackhi_epi8(major_p0, minor_p0);
__m256i step1_m1_lo = _mm256_unpacklo_epi8(major_m1, minor_m1);
__m256i step1_m1_hi = _mm256_unpackhi_epi8(major_m1, minor_m1);
__m256i step1_p1_lo = _mm256_unpacklo_epi8(major_p1, minor_p1);
__m256i step1_p1_hi = _mm256_unpackhi_epi8(major_p1, minor_p1);
// Multiply-add to get u16x16 vectors.
//
// step2_p0_lo = [9*j00+3*n00 9*j01+3*n01 .. 9*j23+3*n23]
// step2_p0_hi = [9*j08+3*n08 9*j09+3*n09 .. 9*j31+3*n31]
// step2_m1_lo = [3*jm1+1*nm1 3*j00+1*n00 .. 3*j22+1*n22]
// step2_m1_hi = [3*j07+1*n07 3*j08+1*n08 .. 3*j30+1*n30]
// step2_p1_lo = [3*j01+1*n01 3*j02+1*n02 .. 3*j24+1*n24]
// step2_p1_hi = [3*j09+1*n09 3*j10+1*n10 .. 3*j32+1*n32]
const __m256i k0309 = _mm256_set1_epi16(0x0309);
const __m256i k0103 = _mm256_set1_epi16(0x0103);
__m256i step2_p0_lo = _mm256_maddubs_epi16(step1_p0_lo, k0309);
__m256i step2_p0_hi = _mm256_maddubs_epi16(step1_p0_hi, k0309);
__m256i step2_m1_lo = _mm256_maddubs_epi16(step1_m1_lo, k0103);
__m256i step2_m1_hi = _mm256_maddubs_epi16(step1_m1_hi, k0103);
__m256i step2_p1_lo = _mm256_maddubs_epi16(step1_p1_lo, k0103);
__m256i step2_p1_hi = _mm256_maddubs_epi16(step1_p1_hi, k0103);
// Compute the weighted sums of (p0, m1) and (p0, p1). For example:
//
// step3_m1_lo[00] = ((9*j00) + (3*n00) + (3*jm1) + (1*nm1)) as u16
// step3_p1_hi[15] = ((9*j31) + (3*n31) + (3*j32) + (1*n32)) as u16
__m256i step3_m1_lo = _mm256_add_epi16(step2_p0_lo, step2_m1_lo);
__m256i step3_m1_hi = _mm256_add_epi16(step2_p0_hi, step2_m1_hi);
__m256i step3_p1_lo = _mm256_add_epi16(step2_p0_lo, step2_p1_lo);
__m256i step3_p1_hi = _mm256_add_epi16(step2_p0_hi, step2_p1_hi);
// Bias by 8 (on the left) or 7 (on the right) and then divide by 16
// (which is 9+3+3+1) to get a weighted average. On the left (m1), shift
// the u16 right value by 4. On the right (p1), shift right by 4 and then
// shift left by 8 so that, when still in the u16x16 little-endian
// interpretation, we have:
// - m1_element = (etcetera + 8) >> 4
// - p1_element = ((etcetera + 7) >> 4) << 8
//
// step4_m1_lo = [0x00?? 0x00?? ... 0x00?? 0x00??]
// step4_p1_lo = [0x??00 0x??00 ... 0x??00 0x??00]
// step4_m1_hi = [0x00?? 0x00?? ... 0x00?? 0x00??]
// step4_p1_hi = [0x??00 0x??00 ... 0x??00 0x??00]
__m256i step4_m1_lo = _mm256_srli_epi16(
_mm256_add_epi16(step3_m1_lo, _mm256_set1_epi16(8)), 4);
__m256i step4_p1_lo = _mm256_slli_epi16(
_mm256_srli_epi16(_mm256_add_epi16(step3_p1_lo, _mm256_set1_epi16(7)),
4),
8);
__m256i step4_m1_hi = _mm256_srli_epi16(
_mm256_add_epi16(step3_m1_hi, _mm256_set1_epi16(8)), 4);
__m256i step4_p1_hi = _mm256_slli_epi16(
_mm256_srli_epi16(_mm256_add_epi16(step3_p1_hi, _mm256_set1_epi16(7)),
4),
8);
// Bitwise-or two "0x00"-rich u16x16 vectors to get a u8x32 vector. Do
// that twice. Once for the low columns and once for the high columns.
//
// In terms of jxx (major row) or nxx (minor row) source samples:
// - low columns means ( 0 .. 8; 16 .. 24).
// - high columns means ( 8 .. 16; 24 .. 32).
//
// In terms of dxx destination samples (there are twice as many):
// - low columns means ( 0 .. 16; 32 .. 48).
// - high columns means (16 .. 32; 48 .. 64).
//
// step5_lo = [d00 d01 .. d14 d15 d32 d33 .. d46 d47]
// step5_hi = [d16 d17 .. d30 d31 d48 d49 .. d62 d63]
//
// The d00, d02 ... d62 even elements come from (p0, m1) weighted sums.
// The d01, d03 ... d63 odd elements come from (p0, p1) weighted sums.
__m256i step5_lo = _mm256_or_si256(step4_m1_lo, step4_p1_lo);
__m256i step5_hi = _mm256_or_si256(step4_m1_hi, step4_p1_hi);
// Permute and store.
//
// step6_00_31 = [d00 d01 .. d14 d15 d16 d17 .. d30 d31]
// step6_32_63 = [d32 d33 .. d46 d47 d48 d49 .. d62 d63]
__m256i step6_00_31 = _mm256_permute2x128_si256(step5_lo, step5_hi, 0x20);
__m256i step6_32_63 = _mm256_permute2x128_si256(step5_lo, step5_hi, 0x31);
_mm256_storeu_si256((__m256i*)(void*)(dp + 0x00), step6_00_31);
_mm256_storeu_si256((__m256i*)(void*)(dp + 0x20), step6_32_63);
// Advance by up to 32 source samples (64 destination samples). The first
// iteration might be smaller than 32 so that all of the remaining steps
// are exactly 32.
size_t n = 32u - (31u & (0u - src_len));
dp += 2u * n;
sp_major += n;
sp_minor += n;
src_len -= n;
}
}
if (last_column) {
uint32_t sv_major_m1 = sp_major[-1];
uint32_t sv_minor_m1 = sp_minor[-1];
uint32_t sv_major_p1 = sp_major[+0]; // Clamp offset to zero.
uint32_t sv_minor_p1 = sp_minor[+0]; // Clamp offset to zero.
uint32_t sv = (9u * ((uint32_t)(*sp_major++))) + //
(3u * ((uint32_t)(*sp_minor++)));
*dp++ = (uint8_t)((sv + (3u * sv_major_m1) + (sv_minor_m1) + 8u) >> 4u);
*dp++ = (uint8_t)((sv + (3u * sv_major_p1) + (sv_minor_p1) + 7u) >> 4u);
}
return dst_ptr;
}
#endif
#endif // defined(WUFFS_BASE__CPU_ARCH__X86_64)
// ‼ WUFFS MULTI-FILE SECTION -x86_avx2