From 435330494e7f5c7978f4417b7b53f8c8ceba0cd7 Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Sun, 14 Feb 2021 21:42:24 +1100 Subject: [PATCH] Add std/png decoder.filter_1_distance_4_arm_neon MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On a Raspberry Pi 4 (32-bit armv7l) with -march=native and -mfpu=neon ("native" means "armv8-a+crc+simd"): wuffs_png_decode_filt_1_dist_4/clang9 1.09GB/s ± 1% 1.42GB/s ± 0% +30.50% (p=0.008 n=5+5) wuffs_png_decode_filt_1_dist_4/gcc8 859MB/s ± 0% 891MB/s ± 0% +3.66% (p=0.008 n=5+5) --- internal/cgen/builtin.go | 51 ++++++++++++++++++++++-- internal/cgen/cgen.go | 2 + internal/cgen/expr.go | 2 + lang/builtin/builtin.go | 22 +++++++++++ lang/check/resolve.go | 4 ++ lang/check/type.go | 2 + lang/token/list.go | 6 +++ release/c/wuffs-unsupported-snapshot.c | 54 ++++++++++++++++++++++++++ std/png/decode_filter_arm_neon.wuffs | 46 ++++++++++++++++++++++ std/png/decode_png.wuffs | 1 + 10 files changed, 187 insertions(+), 3 deletions(-) create mode 100644 std/png/decode_filter_arm_neon.wuffs diff --git a/internal/cgen/builtin.go b/internal/cgen/builtin.go index 45dbefe7..3bb90eab 100644 --- a/internal/cgen/builtin.go +++ b/internal/cgen/builtin.go @@ -523,13 +523,43 @@ func (g *gen) writeBuiltinCPUArch(b *buffer, recv *a.Expr, method t.ID, args []* return nil } + armCRC32U32 := recv.MType().Eq(typeExprARMCRC32U32) + armNeon := recv.MType().Eq(typeExprARMNeon64) || recv.MType().Eq(typeExprARMNeon128) + + methodStr := method.Str(g.tm) + vreinterpretU8Uxx, vreinterpretUxxU8, vreinterpretClose := "", "", "" + if armNeon { + switch { + case strings.HasSuffix(methodStr, "_u16"): + vreinterpretU8Uxx = "vreinterpret_u8_u16(" + vreinterpretUxxU8 = "vreinterpret_u16_u8(" + vreinterpretClose = ")" + case strings.HasSuffix(methodStr, "_u32"): + vreinterpretU8Uxx = "vreinterpret_u8_u32(" + vreinterpretUxxU8 = "vreinterpret_u32_u8(" + vreinterpretClose = ")" + case strings.HasSuffix(methodStr, "_u64"): + vreinterpretU8Uxx = "vreinterpret_u8_u64(" + vreinterpretUxxU8 = "vreinterpret_u64_u8(" + vreinterpretClose = ")" + } + } + const create = "create" - if methodStr := method.Str(g.tm); methodStr == "value" { + if methodStr == "value" { return g.writeExpr(b, recv, false, depth) + } else if methodStr == create { return g.writeExpr(b, args[0].AsArg().Value(), false, depth) + } else if strings.HasPrefix(methodStr, create) { - b.printf("%s(", methodStr[len(create):]) + methodStr = methodStr[len(create):] + if armNeon && (methodStr != "") && (methodStr[0] == '_') { + methodStr = methodStr[1:] + } + + b.writes(vreinterpretU8Uxx) + b.printf("%s(", methodStr) for i, o := range args { if i > 0 { b.writes(", ") @@ -554,21 +584,35 @@ func (g *gen) writeBuiltinCPUArch(b *buffer, recv *a.Expr, method t.ID, args []* } b.writes(after) } + b.writes(vreinterpretClose) + } else { - armCRC32U32 := recv.MType().Eq(typeExprARMCRC32U32) if armCRC32U32 { b.writeb('_') } b.printf("%s(", methodStr) + + if armNeon && recv.MType().IsCPUArchType() { + b.writes(vreinterpretUxxU8) + } if err := g.writeExpr(b, recv, false, depth); err != nil { return err } + if armNeon && recv.MType().IsCPUArchType() { + b.writes(vreinterpretClose) + } + for _, o := range args { b.writes(", ") after := "" v := o.AsArg().Value() if armCRC32U32 { // No-op. + } else if armNeon { + if v.MType().IsCPUArchType() { + b.writes(vreinterpretUxxU8) + after = vreinterpretClose + } } else if !v.MType().IsCPUArchType() { b.writes("(int32_t)(") after = ")" @@ -579,6 +623,7 @@ func (g *gen) writeBuiltinCPUArch(b *buffer, recv *a.Expr, method t.ID, args []* b.writes(after) } } + b.writes(")") return nil } diff --git a/internal/cgen/cgen.go b/internal/cgen/cgen.go index 492c517a..f15fd178 100644 --- a/internal/cgen/cgen.go +++ b/internal/cgen/cgen.go @@ -44,6 +44,8 @@ var ( maxInt64 = big.NewInt((1 << 63) - 1) typeExprARMCRC32U32 = a.NewTypeExpr(0, t.IDBase, t.IDARMCRC32U32, nil, nil, nil) + typeExprARMNeon64 = a.NewTypeExpr(0, t.IDBase, t.IDARMNeon64, nil, nil, nil) + typeExprARMNeon128 = a.NewTypeExpr(0, t.IDBase, t.IDARMNeon128, nil, nil, nil) typeExprPixelSwizzler = a.NewTypeExpr(0, t.IDBase, t.IDPixelSwizzler, nil, nil, nil) typeExprUtility = a.NewTypeExpr(0, t.IDBase, t.IDUtility, nil, nil, nil) ) diff --git a/internal/cgen/expr.go b/internal/cgen/expr.go index c280ced0..ddf21044 100644 --- a/internal/cgen/expr.go +++ b/internal/cgen/expr.go @@ -578,6 +578,8 @@ var cTypeNames = [...]string{ t.IDTokenWriter: "wuffs_base__token_buffer*", t.IDARMCRC32U32: "uint32_t", + t.IDARMNeon64: "uint8x8_t", + t.IDARMNeon128: "uint8x16_t", t.IDX86M128I: "__m128i", } diff --git a/lang/builtin/builtin.go b/lang/builtin/builtin.go index 25d7cf77..589aaa33 100644 --- a/lang/builtin/builtin.go +++ b/lang/builtin/builtin.go @@ -246,6 +246,8 @@ var Types = []string{ // ---- "arm_crc32_u32", + "arm_neon_64", + "arm_neon_128", "x86_m128i", } @@ -542,6 +544,26 @@ var Funcs = []string{ "arm_crc32_u32._crc32w(b: u32) arm_crc32_u32", "arm_crc32_u32._crc32d(b: u64) arm_crc32_u32", + // ---- arm_neon_64 + + // TODO: generate these methods automatically? + + "arm_neon_64.create_vdup_n_u16(a: u16) arm_neon_64", + "arm_neon_64.create_vdup_n_u32(a: u32) arm_neon_64", + "arm_neon_64.create_vdup_n_u64(a: u64) arm_neon_64", + "arm_neon_64.create_vdup_n_u8(a: u8) arm_neon_64", + + "arm_neon_64.vadd_u16(b: arm_neon_64) arm_neon_64", + "arm_neon_64.vadd_u32(b: arm_neon_64) arm_neon_64", + "arm_neon_64.vadd_u64(b: arm_neon_64) arm_neon_64", + "arm_neon_64.vadd_u8(b: arm_neon_64) arm_neon_64", + "arm_neon_64.vget_lane_u16(b: u32[..= 3]) u16", + "arm_neon_64.vget_lane_u32(b: u32[..= 1]) u32", + "arm_neon_64.vget_lane_u64(b: u32[..= 0]) u64", + "arm_neon_64.vget_lane_u8(b: u32[..= 7]) u8", + + // ---- arm_neon_128 + // ---- x86_m128i "x86_m128i.load_u32!(a: u32)", diff --git a/lang/check/resolve.go b/lang/check/resolve.go index 611321bc..ae672292 100644 --- a/lang/check/resolve.go +++ b/lang/check/resolve.go @@ -76,6 +76,8 @@ var ( typeExprDecodeFrameOptions = a.NewTypeExpr(0, t.IDBase, t.IDDecodeFrameOptions, nil, nil, nil) typeExprARMCRC32U32 = a.NewTypeExpr(0, t.IDBase, t.IDARMCRC32U32, nil, nil, nil) + typeExprARMNeon64 = a.NewTypeExpr(0, t.IDBase, t.IDARMNeon64, nil, nil, nil) + typeExprARMNeon128 = a.NewTypeExpr(0, t.IDBase, t.IDARMNeon128, nil, nil, nil) typeExprX86M128I = a.NewTypeExpr(0, t.IDBase, t.IDX86M128I, nil, nil, nil) typeExprSliceU8 = a.NewTypeExpr(t.IDSlice, 0, 0, nil, nil, typeExprU8) @@ -127,6 +129,8 @@ var builtInTypeMap = typeMap{ t.IDDecodeFrameOptions: typeExprDecodeFrameOptions, t.IDARMCRC32U32: typeExprARMCRC32U32, + t.IDARMNeon64: typeExprARMNeon64, + t.IDARMNeon128: typeExprARMNeon128, t.IDX86M128I: typeExprX86M128I, } diff --git a/lang/check/type.go b/lang/check/type.go index f444972b..ff596f63 100644 --- a/lang/check/type.go +++ b/lang/check/type.go @@ -54,6 +54,8 @@ func (q *checker) tcheckCPUArchBits(cab cpuArchBits, typ *a.TypeExpr) error { switch qid[1] { case t.IDARMCRC32U32: need = cpuArchBitsARMCRC32 + case t.IDARMNeon64, t.IDARMNeon128: + need = cpuArchBitsARMNeon case t.IDX86M128I: need = cpuArchBitsX86SSE42 } diff --git a/lang/token/list.go b/lang/token/list.go index 22664d01..dcedf8ef 100644 --- a/lang/token/list.go +++ b/lang/token/list.go @@ -670,6 +670,9 @@ const ( IDARMCRC32U32 = ID(0x308) + IDARMNeon64 = ID(0x310) // 64-bit D (double-word) register + IDARMNeon128 = ID(0x311) // 128-bit Q ( quad-word) register + IDX86SSE42 = ID(0x320) IDX86AVX2 = ID(0x321) @@ -1091,6 +1094,9 @@ var builtInsByID = [nBuiltInIDs]string{ IDARMCRC32U32: "arm_crc32_u32", + IDARMNeon64: "arm_neon_64", + IDARMNeon128: "arm_neon_128", + IDX86SSE42: "x86_sse42", IDX86AVX2: "x86_avx2", diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 09a23792..04a07a74 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -31200,6 +31200,13 @@ WUFFS_PNG__NUM_CHANNELS[8] WUFFS_BASE__POTENTIALLY_UNUSED = { // ---------------- Private Function Prototypes +#if defined(WUFFS_BASE__CPU_ARCH__ARM_NEON) +static wuffs_base__empty_struct +wuffs_png__decoder__filter_1_distance_4_arm_neon( + wuffs_png__decoder* self, + wuffs_base__slice_u8 a_curr); +#endif // defined(WUFFS_BASE__CPU_ARCH__ARM_NEON) + static wuffs_base__empty_struct wuffs_png__decoder__filter_1( wuffs_png__decoder* self, @@ -31495,6 +31502,50 @@ sizeof__wuffs_png__decoder() { // ---------------- Function Implementations +// -------- func png.decoder.filter_1_distance_4_arm_neon + +#if defined(WUFFS_BASE__CPU_ARCH__ARM_NEON) +static wuffs_base__empty_struct +wuffs_png__decoder__filter_1_distance_4_arm_neon( + wuffs_png__decoder* self, + wuffs_base__slice_u8 a_curr) { + wuffs_base__slice_u8 v_c = {0}; + uint8x8_t v_fa = {0}; + uint8x8_t v_fx = {0}; + + v_fa = vreinterpret_u8_u32(vdup_n_u32(0)); + { + wuffs_base__slice_u8 i_slice_c = a_curr; + v_c.ptr = i_slice_c.ptr; + v_c.len = 4; + uint8_t* i_end0_c = v_c.ptr + (((i_slice_c.len - (size_t)(v_c.ptr - i_slice_c.ptr)) / 8) * 8); + while (v_c.ptr < i_end0_c) { + v_fx = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u32le__no_bounds_check(v_c.ptr))); + v_fx = vadd_u8(v_fx, v_fa); + wuffs_base__poke_u32le__no_bounds_check(v_c.ptr, vget_lane_u32(vreinterpret_u32_u8(v_fx), 0)); + v_fa = v_fx; + v_c.ptr += 4; + v_fx = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u32le__no_bounds_check(v_c.ptr))); + v_fx = vadd_u8(v_fx, v_fa); + wuffs_base__poke_u32le__no_bounds_check(v_c.ptr, vget_lane_u32(vreinterpret_u32_u8(v_fx), 0)); + v_fa = v_fx; + v_c.ptr += 4; + } + v_c.len = 4; + uint8_t* i_end1_c = v_c.ptr + (((i_slice_c.len - (size_t)(v_c.ptr - i_slice_c.ptr)) / 4) * 4); + while (v_c.ptr < i_end1_c) { + v_fx = vreinterpret_u8_u32(vdup_n_u32(wuffs_base__peek_u32le__no_bounds_check(v_c.ptr))); + v_fx = vadd_u8(v_fx, v_fa); + wuffs_base__poke_u32le__no_bounds_check(v_c.ptr, vget_lane_u32(vreinterpret_u32_u8(v_fx), 0)); + v_fa = v_fx; + v_c.ptr += 4; + } + v_c.len = 0; + } + return wuffs_base__make_empty_struct(); +} +#endif // defined(WUFFS_BASE__CPU_ARCH__ARM_NEON) + // -------- func png.decoder.filter_1 static wuffs_base__empty_struct @@ -33233,6 +33284,9 @@ wuffs_png__decoder__choose_filter_implementations( &wuffs_png__decoder__filter_4_distance_3_fallback); } else if (self->private_impl.f_filter_distance == 4) { self->private_impl.choosy_filter_1 = ( +#if defined(WUFFS_BASE__CPU_ARCH__ARM_NEON) + wuffs_base__cpu_arch__have_arm_neon() ? &wuffs_png__decoder__filter_1_distance_4_arm_neon : +#endif #if defined(WUFFS_BASE__CPU_ARCH__X86_64) wuffs_base__cpu_arch__have_sse42() ? &wuffs_png__decoder__filter_1_distance_4_sse42 : #endif diff --git a/std/png/decode_filter_arm_neon.wuffs b/std/png/decode_filter_arm_neon.wuffs new file mode 100644 index 00000000..a5a1c0c8 --- /dev/null +++ b/std/png/decode_filter_arm_neon.wuffs @@ -0,0 +1,46 @@ +// Copyright 2021 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// -------- + +// Compared to libpng's ARM NEON implementation, this implementation cannot +// assume that the row pointers are 16-byte aligned. libpng allocates its own +// buffers, Wuffs code cannot call malloc and instead uses whatever buffers it +// is given. Wuffs also uncompresses the entire zlib stream into a continuous +// array. Rows are therefore not necessarily padded to alignment boundaries, +// especially considering the one additional filter byte per row. +// +// Wuffs' "cpu_arch >= arm_neon" requires __ARM_FEATURE_UNALIGNED but still, we +// can't e.g. cast a uint8_t* to a uint32_t* without proving alignment. + +// -------- + +// Filter 1: Sub. + +pri func decoder.filter_1_distance_4_arm_neon!(curr: slice base.u8), + choose cpu_arch >= arm_neon, +{ + var c : slice base.u8 + var fa : base.arm_neon_64 + var fx : base.arm_neon_64 + + fa = fa.create_vdup_n_u32(a: 0) + + iterate (c = args.curr)(length: 4, advance: 4, unroll: 2) { + fx = fx.create_vdup_n_u32(a: c.peek_u32le()) + fx = fx.vadd_u8(b: fa) + c.poke_u32le!(a: fx.vget_lane_u32(b: 0)) + fa = fx + } +} diff --git a/std/png/decode_png.wuffs b/std/png/decode_png.wuffs index e7d841f2..7157fa23 100644 --- a/std/png/decode_png.wuffs +++ b/std/png/decode_png.wuffs @@ -380,6 +380,7 @@ pri func decoder.choose_filter_implementations!() { filter_4_distance_3_fallback] } else if this.filter_distance == 4 { choose filter_1 = [ + filter_1_distance_4_arm_neon, filter_1_distance_4_sse42, filter_1_distance_4_fallback] choose filter_3 = [