From 4e3a005d288f655e5f979d6e061b2cd5a0115a10 Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Mon, 8 Mar 2021 15:41:48 +1100 Subject: [PATCH] Use 8-byte-chunk copy for decode_huffman_fast32 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On a Raspberry Pi 4 (32-bit armv7l) with -march=native and -mfpu=neon ("native" means "armv8-a+crc+simd"): wuffs_deflate_decode_1k_full_init/clang9 45.1MB/s ± 5% 47.2MB/s ± 6% ~ (p=0.151 n=5+5) wuffs_deflate_decode_1k_part_init/clang9 60.6MB/s ± 0% 64.2MB/s ± 0% +5.94% (p=0.008 n=5+5) wuffs_deflate_decode_10k_full_init/clang9 82.0MB/s ± 1% 104.8MB/s ± 1% +27.75% (p=0.008 n=5+5) wuffs_deflate_decode_10k_part_init/clang9 87.9MB/s ± 0% 114.4MB/s ± 0% +30.13% (p=0.008 n=5+5) wuffs_deflate_decode_100k_just_one_read/clang9 111MB/s ± 0% 139MB/s ± 0% +26.03% (p=0.008 n=5+5) wuffs_deflate_decode_100k_many_big_reads/clang9 60.2MB/s ± 0% 63.3MB/s ± 0% +5.28% (p=0.008 n=5+5) wuffs_deflate_decode_1k_full_init/gcc8 45.8MB/s ± 3% 47.4MB/s ± 1% +3.38% (p=0.032 n=5+5) wuffs_deflate_decode_1k_part_init/gcc8 63.0MB/s ± 1% 62.8MB/s ± 1% ~ (p=0.151 n=5+5) wuffs_deflate_decode_10k_full_init/gcc8 87.9MB/s ± 0% 102.9MB/s ± 1% +17.08% (p=0.008 n=5+5) wuffs_deflate_decode_10k_part_init/gcc8 95.3MB/s ± 0% 111.5MB/s ± 0% +17.03% (p=0.008 n=5+5) wuffs_deflate_decode_100k_just_one_read/gcc8 115MB/s ± 0% 130MB/s ± 0% +12.38% (p=0.008 n=5+5) wuffs_deflate_decode_100k_many_big_reads/gcc8 58.9MB/s ± 0% 60.1MB/s ± 0% +1.94% (p=0.008 n=5+5) mimic_deflate_decode_1k 77.7MB/s ± 1% 77.2MB/s ± 1% ~ (p=0.222 n=5+5) mimic_deflate_decode_10k 114MB/s ± 0% 114MB/s ± 1% ~ (p=0.548 n=5+5) mimic_deflate_decode_100k_just_one_read 158MB/s ± 0% 158MB/s ± 0% ~ (p=0.056 n=5+5) mimic_deflate_decode_100k_many_big_reads 113MB/s ± 0% 113MB/s ± 0% ~ (p=0.690 n=5+5) ---- wuffs_gzip_decode_10k/clang9 91.4MB/s ± 0% 106.9MB/s ± 0% +16.93% (p=0.008 n=5+5) wuffs_gzip_decode_100k/clang9 113MB/s ± 0% 128MB/s ± 0% +13.01% (p=0.008 n=5+5) wuffs_gzip_decode_10k/gcc8 92.2MB/s ± 0% 109.7MB/s ± 0% +19.00% (p=0.008 n=5+5) wuffs_gzip_decode_100k/gcc8 111MB/s ± 0% 119MB/s ± 0% +7.43% (p=0.008 n=5+5) mimic_gzip_decode_10k 98.2MB/s ± 0% 98.1MB/s ± 0% ~ (p=0.222 n=5+5) mimic_gzip_decode_100k 128MB/s ± 0% 128MB/s ± 0% +0.11% (p=0.008 n=5+5) ---- wuffs_zlib_decode_10k/clang9 90.8MB/s ± 0% 111.5MB/s ± 0% +22.86% (p=0.008 n=5+5) wuffs_zlib_decode_100k/clang9 108MB/s ± 0% 134MB/s ± 0% +23.90% (p=0.008 n=5+5) wuffs_zlib_decode_10k/gcc8 92.3MB/s ± 0% 108.2MB/s ± 0% +17.15% (p=0.008 n=5+5) wuffs_zlib_decode_100k/gcc8 112MB/s ± 0% 119MB/s ± 0% +6.90% (p=0.008 n=5+5) mimic_zlib_decode_10k 108MB/s ± 0% 108MB/s ± 0% +0.39% (p=0.008 n=5+5) mimic_zlib_decode_100k 143MB/s ± 0% 141MB/s ± 0% -1.28% (p=0.008 n=5+5) ---- wuffs_png_decode_image_19k_8bpp/clang9 72.4MB/s ± 0% 81.9MB/s ± 0% +13.10% (p=0.008 n=5+5) wuffs_png_decode_image_40k_24bpp/clang9 104MB/s ± 0% 104MB/s ± 0% -0.40% (p=0.008 n=5+5) wuffs_png_decode_image_77k_8bpp/clang9 274MB/s ± 0% 301MB/s ± 0% +9.80% (p=0.008 n=5+5) wuffs_png_decode_image_552k_32bpp_ignore_checksum/clang9 165MB/s ± 0% 183MB/s ± 0% +10.96% (p=0.016 n=4+5) wuffs_png_decode_image_552k_32bpp_verify_checksum/clang9 160MB/s ± 1% 178MB/s ± 0% +11.28% (p=0.008 n=5+5) wuffs_png_decode_image_4002k_24bpp/clang9 98.1MB/s ± 0% 98.9MB/s ± 0% +0.82% (p=0.016 n=5+4) wuffs_png_decode_image_19k_8bpp/gcc8 73.5MB/s ± 0% 80.4MB/s ± 0% +9.33% (p=0.008 n=5+5) wuffs_png_decode_image_40k_24bpp/gcc8 104MB/s ± 0% 106MB/s ± 0% +1.73% (p=0.008 n=5+5) wuffs_png_decode_image_77k_8bpp/gcc8 248MB/s ± 0% 272MB/s ± 0% +9.67% (p=0.008 n=5+5) wuffs_png_decode_image_552k_32bpp_ignore_checksum/gcc8 163MB/s ± 0% 177MB/s ± 0% +8.89% (p=0.008 n=5+5) wuffs_png_decode_image_552k_32bpp_verify_checksum/gcc8 157MB/s ± 0% 170MB/s ± 0% +8.42% (p=0.008 n=5+5) wuffs_png_decode_image_4002k_24bpp/gcc8 98.6MB/s ± 0% 100.9MB/s ± 0% +2.29% (p=0.008 n=5+5) mimic_png_decode_image_19k_8bpp 43.7MB/s ± 0% 43.7MB/s ± 0% ~ (p=0.690 n=5+5) mimic_png_decode_image_40k_24bpp 54.6MB/s ± 0% 54.6MB/s ± 0% ~ (p=0.151 n=5+5) mimic_png_decode_image_77k_8bpp 125MB/s ± 0% 124MB/s ± 0% -0.28% (p=0.008 n=5+5) mimic_png_decode_image_552k_32bpp_ignore_checksum skipped mimic_png_decode_image_552k_32bpp_verify_checksum 102MB/s ± 0% 102MB/s ± 0% ~ (p=0.238 n=5+5) mimic_png_decode_image_4002k_24bpp 81.7MB/s ± 0% 81.8MB/s ± 0% +0.09% (p=0.008 n=5+5) --- release/c/wuffs-unsupported-snapshot.c | 14 ++++++--- std/deflate/decode_huffman_fast32.wuffs | 42 ++++++++++++++++++------- 2 files changed, 40 insertions(+), 16 deletions(-) diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 7733c6ec..dadba2b3 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -23953,7 +23953,7 @@ wuffs_deflate__decoder__decode_huffman_fast32( v_lmask = ((((uint32_t)(1)) << self->private_impl.f_n_huffs_bits[0]) - 1); v_dmask = ((((uint32_t)(1)) << self->private_impl.f_n_huffs_bits[1]) - 1); label__loop__continue:; - while ((((uint64_t)(io2_a_dst - iop_a_dst)) >= 258) && (((uint64_t)(io2_a_src - iop_a_src)) >= 12)) { + while ((((uint64_t)(io2_a_dst - iop_a_dst)) >= 266) && (((uint64_t)(io2_a_src - iop_a_src)) >= 12)) { if (v_n_bits < 15) { v_bits |= (((uint32_t)(wuffs_base__peek_u8be__no_bounds_check(iop_a_src))) << v_n_bits); iop_a_src += 1; @@ -24104,14 +24104,18 @@ wuffs_deflate__decoder__decode_huffman_fast32( if (v_length == 0) { goto label__loop__continue; } - if ((((uint64_t)((v_dist_minus_1 + 1))) > ((uint64_t)(iop_a_dst - io0_a_dst))) || (((uint64_t)(v_length)) > ((uint64_t)(io2_a_dst - iop_a_dst)))) { + if ((((uint64_t)((v_dist_minus_1 + 1))) > ((uint64_t)(iop_a_dst - io0_a_dst))) || (((uint64_t)(v_length)) > ((uint64_t)(io2_a_dst - iop_a_dst))) || (((uint64_t)((v_length + 8))) > ((uint64_t)(io2_a_dst - iop_a_dst)))) { status = wuffs_base__make_status(wuffs_deflate__error__internal_error_inconsistent_distance); goto exit; } - } else { } - wuffs_base__io_writer__limited_copy_u32_from_history_fast( - &iop_a_dst, io0_a_dst, io2_a_dst, v_length, (v_dist_minus_1 + 1)); + if ((v_dist_minus_1 + 1) >= 8) { + wuffs_base__io_writer__limited_copy_u32_from_history_8_byte_chunks_fast( + &iop_a_dst, io0_a_dst, io2_a_dst, v_length, (v_dist_minus_1 + 1)); + } else { + wuffs_base__io_writer__limited_copy_u32_from_history_fast( + &iop_a_dst, io0_a_dst, io2_a_dst, v_length, (v_dist_minus_1 + 1)); + } goto label__0__break; } label__0__break:; diff --git a/std/deflate/decode_huffman_fast32.wuffs b/std/deflate/decode_huffman_fast32.wuffs index d75ee370..75c229ee 100644 --- a/std/deflate/decode_huffman_fast32.wuffs +++ b/std/deflate/decode_huffman_fast32.wuffs @@ -44,7 +44,7 @@ pri func decoder.decode_huffman_fast32!(dst: base.io_writer, src: base.io_reader dmask = ((1 as base.u32) << this.n_huffs_bits[1]) - 1 // Check up front, on each iteration, that we have enough buffer space to - // both read (12 bytes) and write (258 bytes) as much as we need to. Doing + // both read (12 bytes) and write (266 bytes) as much as we need to. Doing // this check once (per iteration), up front, removes the need to check // multiple times inside the loop body, so it's faster overall. // @@ -52,6 +52,12 @@ pri func decoder.decode_huffman_fast32!(dst: base.io_writer, src: base.io_reader // 258 is the maximum length in a length-distance pair, as specified in the // RFC section 3.2.5. Compressed blocks (length and distance codes). // + // We adjust 258 up to (258 + 8) = 266 since it can be faster to overshoot + // a little and make multiple-of-8-byte copies even when the length in the + // length-distance pair isn't an exact multiple-of-8. Strictly speaking, + // 264 (an exact multiple-of-8) is the tight bound but (258 + 8) is easier + // for the Wuffs proof system, as length's type is refined to [..= 258], + // // For reading, strictly speaking, we only need 6 bytes (48 bits) of // available input, because the H-L Literal/Length code is up to 15 bits // plus up to 5 extra bits, the H-D Distance code is up to 15 bits plus up @@ -69,7 +75,7 @@ pri func decoder.decode_huffman_fast32!(dst: base.io_writer, src: base.io_reader // for each. There are also up to 13 extra bits, or 2 bytes. Thus, each // code requires up to 2 + 2 + 2 == 6 bytes, and there are two codes (one // from H-L and one from H-D). Therefore, we check for at least 12 bytes. - while.loop(args.dst.length() >= 258) and (args.src.length() >= 12) { + while.loop(args.dst.length() >= 266) and (args.src.length() >= 12) { // Ensure that we have at least 15 bits of input. if n_bits < 15 { assert n_bits >= 0 // TODO: this shouldn't be necessary. @@ -286,8 +292,12 @@ pri func decoder.decode_huffman_fast32!(dst: base.io_writer, src: base.io_reader // its presence minimizes the diff between decode_huffman_fastxx and // decode_huffman_slow. while true, - pre args.dst.length() >= 258, + pre args.dst.length() >= 266, { + // We can therefore prove: + assert (length as base.u64) <= args.dst.length() via "a <= b: a <= c; c <= b"(c: 266) + assert ((length + 8) as base.u64) <= args.dst.length() via "a <= b: a <= c; c <= b"(c: 266) + // Copy from this.history. if ((dist_minus_1 + 1) as base.u64) > args.dst.history_length() { // Set (hlen, hdist) to be the length-distance pair to copy @@ -325,23 +335,33 @@ pri func decoder.decode_huffman_fast32!(dst: base.io_writer, src: base.io_reader } if (((dist_minus_1 + 1) as base.u64) > args.dst.history_length()) or - ((length as base.u64) > args.dst.length()) { + ((length as base.u64) > args.dst.length()) or + (((length + 8) as base.u64) > args.dst.length()) { return "#internal error: inconsistent distance" } - } else { - assert (length as base.u64) <= 258 - assert args.dst.length() >= 258 - // We can therefore prove: - assert (length as base.u64) <= args.dst.length() via "a <= b: a <= c; c <= b"(c: 258) } // Once again, redundant but explicit assertions. assert (dist_minus_1 + 1) > 0 assert ((dist_minus_1 + 1) as base.u64) <= args.dst.history_length() assert (length as base.u64) <= args.dst.length() + assert ((length + 8) as base.u64) <= args.dst.length() // Copy from args.dst. - args.dst.limited_copy_u32_from_history_fast!( - up_to: length, distance: (dist_minus_1 + 1)) + // + // For short distances, less than 8 bytes, copying atomic 8-byte + // chunks can result in incorrect output, so we fall back to a + // slower 1-byte-at-a-time copy. Deflate's copy-from-history can + // pick up freshly written bytes. A length = 5, distance = 2 copy + // starting with "abc" should give "abcbcbcb" but the 8-byte chunk + // technique would give "abcbc???", the exact output depending on + // what was previously in the writer buffer. + if (dist_minus_1 + 1) >= 8 { + args.dst.limited_copy_u32_from_history_8_byte_chunks_fast!( + up_to: length, distance: (dist_minus_1 + 1)) + } else { + args.dst.limited_copy_u32_from_history_fast!( + up_to: length, distance: (dist_minus_1 + 1)) + } break } endwhile } endwhile.loop