Add std/png decoder.filter_1_distance_4_arm_neon

On a Raspberry Pi 4 (32-bit armv7l) with -march=native and -mfpu=neon
("native" means "armv8-a+crc+simd"):

wuffs_png_decode_filt_1_dist_4/clang9                     1.09GB/s ± 1%  1.42GB/s ± 0%  +30.50%  (p=0.008 n=5+5)
wuffs_png_decode_filt_1_dist_4/gcc8                        859MB/s ± 0%   891MB/s ± 0%   +3.66%  (p=0.008 n=5+5)
This commit is contained in:
Nigel Tao committed 2021-02-14 21:43:56 +11:00
1 parent 84f3592dbe
commit 435330494e
10 files changed
+187 -3

No files matched your search

+48 -3
View File
@@ -523,13 +523,43 @@ func (g *gen) writeBuiltinCPUArch(b *buffer, recv *a.Expr, method t.ID, args []*
return nil
}
armCRC32U32 := recv.MType().Eq(typeExprARMCRC32U32)
armNeon := recv.MType().Eq(typeExprARMNeon64) || recv.MType().Eq(typeExprARMNeon128)
methodStr := method.Str(g.tm)
vreinterpretU8Uxx, vreinterpretUxxU8, vreinterpretClose := "", "", ""
if armNeon {
switch {
case strings.HasSuffix(methodStr, "_u16"):
vreinterpretU8Uxx = "vreinterpret_u8_u16("
vreinterpretUxxU8 = "vreinterpret_u16_u8("
vreinterpretClose = ")"
case strings.HasSuffix(methodStr, "_u32"):
vreinterpretU8Uxx = "vreinterpret_u8_u32("
vreinterpretUxxU8 = "vreinterpret_u32_u8("
vreinterpretClose = ")"
case strings.HasSuffix(methodStr, "_u64"):
vreinterpretU8Uxx = "vreinterpret_u8_u64("
vreinterpretUxxU8 = "vreinterpret_u64_u8("
vreinterpretClose = ")"
}
}
const create = "create"
if methodStr := method.Str(g.tm); methodStr == "value" {
if methodStr == "value" {
return g.writeExpr(b, recv, false, depth)
} else if methodStr == create {
return g.writeExpr(b, args[0].AsArg().Value(), false, depth)
} else if strings.HasPrefix(methodStr, create) {
b.printf("%s(", methodStr[len(create):])
methodStr = methodStr[len(create):]
if armNeon && (methodStr != "") && (methodStr[0] == '_') {
methodStr = methodStr[1:]
}
b.writes(vreinterpretU8Uxx)
b.printf("%s(", methodStr)
for i, o := range args {
if i > 0 {
b.writes(", ")
@@ -554,21 +584,35 @@ func (g *gen) writeBuiltinCPUArch(b *buffer, recv *a.Expr, method t.ID, args []*
}
b.writes(after)
}
b.writes(vreinterpretClose)
} else {
armCRC32U32 := recv.MType().Eq(typeExprARMCRC32U32)
if armCRC32U32 {
b.writeb('_')
}
b.printf("%s(", methodStr)
if armNeon && recv.MType().IsCPUArchType() {
b.writes(vreinterpretUxxU8)
}
if err := g.writeExpr(b, recv, false, depth); err != nil {
return err
}
if armNeon && recv.MType().IsCPUArchType() {
b.writes(vreinterpretClose)
}
for _, o := range args {
b.writes(", ")
after := ""
v := o.AsArg().Value()
if armCRC32U32 {
// No-op.
} else if armNeon {
if v.MType().IsCPUArchType() {
b.writes(vreinterpretUxxU8)
after = vreinterpretClose
}
} else if !v.MType().IsCPUArchType() {
b.writes("(int32_t)(")
after = ")"
@@ -579,6 +623,7 @@ func (g *gen) writeBuiltinCPUArch(b *buffer, recv *a.Expr, method t.ID, args []*
b.writes(after)
}
}
b.writes(")")
return nil
}
+2
View File
@@ -44,6 +44,8 @@ var (
maxInt64 = big.NewInt((1 << 63) - 1)
typeExprARMCRC32U32 = a.NewTypeExpr(0, t.IDBase, t.IDARMCRC32U32, nil, nil, nil)
typeExprARMNeon64 = a.NewTypeExpr(0, t.IDBase, t.IDARMNeon64, nil, nil, nil)
typeExprARMNeon128 = a.NewTypeExpr(0, t.IDBase, t.IDARMNeon128, nil, nil, nil)
typeExprPixelSwizzler = a.NewTypeExpr(0, t.IDBase, t.IDPixelSwizzler, nil, nil, nil)
typeExprUtility = a.NewTypeExpr(0, t.IDBase, t.IDUtility, nil, nil, nil)
)
+2
View File
@@ -578,6 +578,8 @@ var cTypeNames = [...]string{
t.IDTokenWriter: "wuffs_base__token_buffer*",
t.IDARMCRC32U32: "uint32_t",
t.IDARMNeon64: "uint8x8_t",
t.IDARMNeon128: "uint8x16_t",
t.IDX86M128I: "__m128i",
}
+22
View File
@@ -246,6 +246,8 @@ var Types = []string{
// ----
"arm_crc32_u32",
"arm_neon_64",
"arm_neon_128",
"x86_m128i",
}
@@ -542,6 +544,26 @@ var Funcs = []string{
"arm_crc32_u32._crc32w(b: u32) arm_crc32_u32",
"arm_crc32_u32._crc32d(b: u64) arm_crc32_u32",
// ---- arm_neon_64
// TODO: generate these methods automatically?
"arm_neon_64.create_vdup_n_u16(a: u16) arm_neon_64",
"arm_neon_64.create_vdup_n_u32(a: u32) arm_neon_64",
"arm_neon_64.create_vdup_n_u64(a: u64) arm_neon_64",
"arm_neon_64.create_vdup_n_u8(a: u8) arm_neon_64",
"arm_neon_64.vadd_u16(b: arm_neon_64) arm_neon_64",
"arm_neon_64.vadd_u32(b: arm_neon_64) arm_neon_64",
"arm_neon_64.vadd_u64(b: arm_neon_64) arm_neon_64",
"arm_neon_64.vadd_u8(b: arm_neon_64) arm_neon_64",
"arm_neon_64.vget_lane_u16(b: u32[..= 3]) u16",
"arm_neon_64.vget_lane_u32(b: u32[..= 1]) u32",
"arm_neon_64.vget_lane_u64(b: u32[..= 0]) u64",
"arm_neon_64.vget_lane_u8(b: u32[..= 7]) u8",
// ---- arm_neon_128
// ---- x86_m128i
"x86_m128i.load_u32!(a: u32)",
+4
View File
@@ -76,6 +76,8 @@ var (
typeExprDecodeFrameOptions = a.NewTypeExpr(0, t.IDBase, t.IDDecodeFrameOptions, nil, nil, nil)
typeExprARMCRC32U32 = a.NewTypeExpr(0, t.IDBase, t.IDARMCRC32U32, nil, nil, nil)
typeExprARMNeon64 = a.NewTypeExpr(0, t.IDBase, t.IDARMNeon64, nil, nil, nil)
typeExprARMNeon128 = a.NewTypeExpr(0, t.IDBase, t.IDARMNeon128, nil, nil, nil)
typeExprX86M128I = a.NewTypeExpr(0, t.IDBase, t.IDX86M128I, nil, nil, nil)
typeExprSliceU8 = a.NewTypeExpr(t.IDSlice, 0, 0, nil, nil, typeExprU8)
@@ -127,6 +129,8 @@ var builtInTypeMap = typeMap{
t.IDDecodeFrameOptions: typeExprDecodeFrameOptions,
t.IDARMCRC32U32: typeExprARMCRC32U32,
t.IDARMNeon64: typeExprARMNeon64,
t.IDARMNeon128: typeExprARMNeon128,
t.IDX86M128I: typeExprX86M128I,
}
+2
View File
@@ -54,6 +54,8 @@ func (q *checker) tcheckCPUArchBits(cab cpuArchBits, typ *a.TypeExpr) error {
switch qid[1] {
case t.IDARMCRC32U32:
need = cpuArchBitsARMCRC32
case t.IDARMNeon64, t.IDARMNeon128:
need = cpuArchBitsARMNeon
case t.IDX86M128I:
need = cpuArchBitsX86SSE42
}
+6
View File
@@ -670,6 +670,9 @@ const (
IDARMCRC32U32 = ID(0x308)
IDARMNeon64 = ID(0x310) // 64-bit D (double-word) register
IDARMNeon128 = ID(0x311) // 128-bit Q ( quad-word) register
IDX86SSE42 = ID(0x320)
IDX86AVX2 = ID(0x321)
@@ -1091,6 +1094,9 @@ var builtInsByID = [nBuiltInIDs]string{
IDARMCRC32U32: "arm_crc32_u32",
IDARMNeon64: "arm_neon_64",
IDARMNeon128: "arm_neon_128",
IDX86SSE42: "x86_sse42",
IDX86AVX2: "x86_avx2",
+54
View File
@@ -31200,6 +31200,13 @@ WUFFS_PNG__NUM_CHANNELS[8] WUFFS_BASE__POTENTIALLY_UNUSED = {
// ---------------- Private Function Prototypes
#if defined(WUFFS_BASE__CPU_ARCH__ARM_NEON)
static wuffs_base__empty_struct
wuffs_png__decoder__filter_1_distance_4_arm_neon(
wuffs_png__decoder* self,
wuffs_base__slice_u8 a_curr);
#endif // defined(WUFFS_BASE__CPU_ARCH__ARM_NEON)
static wuffs_base__empty_struct
wuffs_png__decoder__filter_1(
wuffs_png__decoder* self,
@@ -31495,6 +31502,50 @@ sizeof__wuffs_png__decoder() {
// ---------------- Function Implementations
// -------- func png.decoder.filter_1_distance_4_arm_neon
#if defined(WUFFS_BASE__CPU_ARCH__ARM_NEON)
static wuffs_base__empty_struct
wuffs_png__decoder__filter_1_distance_4_arm_neon(
wuffs_png__decoder* self,
wuffs_base__slice_u8 a_curr) {
wuffs_base__slice_u8 v_c = {0};
uint8x8_t v_fa = {0};
uint8x8_t v_fx = {0};
v_fa = vreinterpret_u8_u32(vdup_n_u32(0));
{
wuffs_base__slice_u8 i_slice_c = a_curr;
v_c.ptr = i_slice_c.ptr;
v_c.len = 4;
uint8_t* i_end0_c = v_c.ptr + (((i_slice_c.len - (size_t)(v_c.ptr - i_slice_c.ptr)) / 8) * 8);
while (v_c.ptr < i_end0_c) {
v_fx = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u32le__no_bounds_check(v_c.ptr)));
v_fx = vadd_u8(v_fx, v_fa);
wuffs_base__poke_u32le__no_bounds_check(v_c.ptr, vget_lane_u32(vreinterpret_u32_u8(v_fx), 0));
v_fa = v_fx;
v_c.ptr += 4;
v_fx = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u32le__no_bounds_check(v_c.ptr)));
v_fx = vadd_u8(v_fx, v_fa);
wuffs_base__poke_u32le__no_bounds_check(v_c.ptr, vget_lane_u32(vreinterpret_u32_u8(v_fx), 0));
v_fa = v_fx;
v_c.ptr += 4;
}
v_c.len = 4;
uint8_t* i_end1_c = v_c.ptr + (((i_slice_c.len - (size_t)(v_c.ptr - i_slice_c.ptr)) / 4) * 4);
while (v_c.ptr < i_end1_c) {
v_fx = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u32le__no_bounds_check(v_c.ptr)));
v_fx = vadd_u8(v_fx, v_fa);
wuffs_base__poke_u32le__no_bounds_check(v_c.ptr, vget_lane_u32(vreinterpret_u32_u8(v_fx), 0));
v_fa = v_fx;
v_c.ptr += 4;
}
v_c.len = 0;
}
return wuffs_base__make_empty_struct();
}
#endif // defined(WUFFS_BASE__CPU_ARCH__ARM_NEON)
// -------- func png.decoder.filter_1
static wuffs_base__empty_struct
@@ -33233,6 +33284,9 @@ wuffs_png__decoder__choose_filter_implementations(
&wuffs_png__decoder__filter_4_distance_3_fallback);
} else if (self->private_impl.f_filter_distance == 4) {
self->private_impl.choosy_filter_1 = (
#if defined(WUFFS_BASE__CPU_ARCH__ARM_NEON)
wuffs_base__cpu_arch__have_arm_neon() ? &wuffs_png__decoder__filter_1_distance_4_arm_neon :
#endif
#if defined(WUFFS_BASE__CPU_ARCH__X86_64)
wuffs_base__cpu_arch__have_sse42() ? &wuffs_png__decoder__filter_1_distance_4_sse42 :
#endif
+46
View File
@@ -0,0 +1,46 @@
// Copyright 2021 The Wuffs Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// --------
// Compared to libpng's ARM NEON implementation, this implementation cannot
// assume that the row pointers are 16-byte aligned. libpng allocates its own
// buffers, Wuffs code cannot call malloc and instead uses whatever buffers it
// is given. Wuffs also uncompresses the entire zlib stream into a continuous
// array. Rows are therefore not necessarily padded to alignment boundaries,
// especially considering the one additional filter byte per row.
//
// Wuffs' "cpu_arch >= arm_neon" requires __ARM_FEATURE_UNALIGNED but still, we
// can't e.g. cast a uint8_t* to a uint32_t* without proving alignment.
// --------
// Filter 1: Sub.
pri func decoder.filter_1_distance_4_arm_neon!(curr: slice base.u8),
choose cpu_arch >= arm_neon,
{
var c : slice base.u8
var fa : base.arm_neon_64
var fx : base.arm_neon_64
fa = fa.create_vdup_n_u32(a: 0)
iterate (c = args.curr)(length: 4, advance: 4, unroll: 2) {
fx = fx.create_vdup_n_u32(a: c.peek_u32le())
fx = fx.vadd_u8(b: fa)
c.poke_u32le!(a: fx.vget_lane_u32(b: 0))
fa = fx
}
}
+1
View File
@@ -380,6 +380,7 @@ pri func decoder.choose_filter_implementations!() {
filter_4_distance_3_fallback]
} else if this.filter_distance == 4 {
choose filter_1 = [
filter_1_distance_4_arm_neon,
filter_1_distance_4_sse42,
filter_1_distance_4_fallback]
choose filter_3 = [