From e65c861207d778928b66764a736ecfcba1c988f8 Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Thu, 5 Mar 2020 12:03:57 +1100 Subject: [PATCH] Extend std/deflate ringbuffer to simplify copies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit name old speed new speed delta wuffs_deflate_decode_1k_full_init/clang5 155MB/s ± 1% 154MB/s ± 0% -0.98% (p=0.000 n=10+9) wuffs_deflate_decode_1k_part_init/clang5 190MB/s ± 0% 193MB/s ± 1% +1.49% (p=0.000 n=10+9) wuffs_deflate_decode_10k_full_init/clang5 240MB/s ± 0% 249MB/s ± 0% +3.68% (p=0.000 n=9+10) wuffs_deflate_decode_10k_part_init/clang5 247MB/s ± 0% 257MB/s ± 1% +4.03% (p=0.000 n=10+10) wuffs_deflate_decode_100k_just_one_read/clang5 292MB/s ± 1% 291MB/s ± 0% -0.34% (p=0.043 n=10+9) wuffs_deflate_decode_100k_many_big_reads/clang5 246MB/s ± 0% 250MB/s ± 0% +1.77% (p=0.000 n=9+10) wuffs_deflate_decode_1k_full_init/gcc9 166MB/s ± 1% 165MB/s ± 1% -0.48% (p=0.009 n=10+10) wuffs_deflate_decode_1k_part_init/gcc9 208MB/s ± 1% 212MB/s ± 0% +1.85% (p=0.000 n=10+9) wuffs_deflate_decode_10k_full_init/gcc9 248MB/s ± 1% 281MB/s ± 0% +13.05% (p=0.000 n=9+10) wuffs_deflate_decode_10k_part_init/gcc9 256MB/s ± 0% 290MB/s ± 0% +13.34% (p=0.000 n=10+10) wuffs_deflate_decode_100k_just_one_read/gcc9 336MB/s ± 0% 333MB/s ± 0% -0.93% (p=0.000 n=9+8) wuffs_deflate_decode_100k_many_big_reads/gcc9 262MB/s ± 0% 280MB/s ± 1% +6.99% (p=0.000 n=9+10) --- release/c/wuffs-unsupported-snapshot.c | 63 ++++++++++++-------------- std/deflate/decode_deflate.wuffs | 32 ++++++++----- std/deflate/decode_huffman_fast.wuffs | 26 ++--------- std/deflate/decode_huffman_slow.wuffs | 4 +- test/c/std/deflate.c | 24 ++++++++++ 5 files changed, 80 insertions(+), 69 deletions(-) diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 3ee50672..4b84d8a7 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -4541,7 +4541,7 @@ struct wuffs_deflate__decoder__struct { struct { uint32_t f_huffs[2][1024]; - uint8_t f_history[32768]; + uint8_t f_history[33025]; uint8_t f_code_lengths[320]; struct { @@ -11617,18 +11617,24 @@ wuffs_deflate__decoder__add_history(wuffs_deflate__decoder* self, if (((uint64_t)(v_s.len)) >= 32768) { v_s = wuffs_base__slice_u8__suffix(v_s, 32768); wuffs_base__slice_u8__copy_from_slice( - wuffs_base__make_slice_u8(self->private_data.f_history, 32768), v_s); + wuffs_base__slice_u8__subslice_j( + wuffs_base__make_slice_u8(self->private_data.f_history, 33025), + 32768), + v_s); self->private_impl.f_history_index = 32768; } else { v_n_copied = wuffs_base__slice_u8__copy_from_slice( - wuffs_base__slice_u8__subslice_i( - wuffs_base__make_slice_u8(self->private_data.f_history, 32768), - (self->private_impl.f_history_index & 32767)), + wuffs_base__slice_u8__subslice_ij( + wuffs_base__make_slice_u8(self->private_data.f_history, 33025), + (self->private_impl.f_history_index & 32767), 32768), v_s); if (v_n_copied < ((uint64_t)(v_s.len))) { v_s = wuffs_base__slice_u8__subslice_i(v_s, v_n_copied); v_n_copied = wuffs_base__slice_u8__copy_from_slice( - wuffs_base__make_slice_u8(self->private_data.f_history, 32768), v_s); + wuffs_base__slice_u8__subslice_j( + wuffs_base__make_slice_u8(self->private_data.f_history, 33025), + 32768), + v_s); self->private_impl.f_history_index = (((uint32_t)((v_n_copied & 32767))) + 32768); } else { @@ -11641,6 +11647,11 @@ wuffs_deflate__decoder__add_history(wuffs_deflate__decoder* self, ((uint32_t)((v_n_copied & 32767))) + v_already_full); } } + wuffs_base__slice_u8__copy_from_slice( + wuffs_base__slice_u8__subslice_i( + wuffs_base__make_slice_u8(self->private_data.f_history, 33025), + 32768), + wuffs_base__make_slice_u8(self->private_data.f_history, 33025)); return wuffs_base__make_empty_struct(); } @@ -12649,7 +12660,6 @@ wuffs_deflate__decoder__decode_huffman_fast(wuffs_deflate__decoder* self, uint32_t v_redir_mask = 0; uint32_t v_length = 0; uint32_t v_dist_minus_1 = 0; - uint32_t v_n_copied = 0; uint32_t v_hlen = 0; uint32_t v_hdist = 0; @@ -12858,7 +12868,6 @@ label__loop__continue:; 32767); v_bits >>= v_table_entry_n_bits; v_n_bits -= v_table_entry_n_bits; - v_n_copied = 0; while (true) { if (((uint64_t)((v_dist_minus_1 + 1))) > ((uint64_t)(iop_a_dst - io0_a_dst))) { @@ -12877,23 +12886,11 @@ label__loop__continue:; goto exit; } v_hdist = (self->private_impl.f_history_index - v_hdist); - while (true) { - v_n_copied = wuffs_base__io_writer__copy_n32_from_slice( - &iop_a_dst, io2_a_dst, v_hlen, - wuffs_base__slice_u8__subslice_i( - wuffs_base__make_slice_u8(self->private_data.f_history, - 32768), - (v_hdist & 32767))); - if (v_hlen <= v_n_copied) { - goto label__0__break; - } - v_hlen -= v_n_copied; - wuffs_base__io_writer__copy_n32_from_slice( - &iop_a_dst, io2_a_dst, v_hlen, - wuffs_base__make_slice_u8(self->private_data.f_history, 32768)); - goto label__0__break; - } - label__0__break:; + wuffs_base__io_writer__copy_n32_from_slice( + &iop_a_dst, io2_a_dst, v_hlen, + wuffs_base__slice_u8__subslice_i( + wuffs_base__make_slice_u8(self->private_data.f_history, 33025), + (v_hdist & 32767))); if (v_length == 0) { goto label__loop__continue; } @@ -12906,9 +12903,9 @@ label__loop__continue:; } wuffs_base__io_writer__copy_n32_from_history_fast( &iop_a_dst, io0_a_dst, io2_a_dst, v_length, (v_dist_minus_1 + 1)); - goto label__1__break; + goto label__0__break; } - label__1__break:; + label__0__break:; } label__loop__break:; while (v_n_bits >= 8) { @@ -13257,10 +13254,10 @@ wuffs_deflate__decoder__decode_huffman_slow(wuffs_deflate__decoder* self, while (true) { v_n_copied = wuffs_base__io_writer__copy_n32_from_slice( &iop_a_dst, io2_a_dst, v_hlen, - wuffs_base__slice_u8__subslice_i( + wuffs_base__slice_u8__subslice_ij( wuffs_base__make_slice_u8(self->private_data.f_history, - 32768), - (v_hdist & 32767))); + 33025), + (v_hdist & 32767), 32768)); if (v_hlen <= v_n_copied) { v_hlen = 0; goto label__4__break; @@ -13281,10 +13278,10 @@ wuffs_deflate__decoder__decode_huffman_slow(wuffs_deflate__decoder* self, while (true) { v_n_copied = wuffs_base__io_writer__copy_n32_from_slice( &iop_a_dst, io2_a_dst, v_hlen, - wuffs_base__slice_u8__subslice_i( + wuffs_base__slice_u8__subslice_ij( wuffs_base__make_slice_u8(self->private_data.f_history, - 32768), - (v_hdist & 32767))); + 33025), + (v_hdist & 32767), 32768)); if (v_hlen <= v_n_copied) { v_hlen = 0; goto label__5__break; diff --git a/std/deflate/decode_deflate.wuffs b/std/deflate/decode_deflate.wuffs index a293e28b..97ffe6df 100644 --- a/std/deflate/decode_deflate.wuffs +++ b/std/deflate/decode_deflate.wuffs @@ -117,18 +117,24 @@ pub struct decoder? implements base.io_transformer( // Exactly one of the eight bits [24 ..= 31] should be set. huffs : array[2] array[huffs_table_size] base.u32, - // history holds up to the last 32KiB of decoded output, if the decoding - // was incomplete (e.g. due to a short read or write). RFC 1951 (DEFLATE) - // gives the maximum distance in a length-distance back-reference as 32768, - // or 0x8000. + // history[.. 0x8000] holds up to the last 32KiB of decoded output, if the + // decoding was incomplete (e.g. due to a short read or write). RFC 1951 + // (DEFLATE) gives the maximum distance in a length-distance back-reference + // as 32768, or 0x8000. Similarly, the RFC gives the maximum length as 258. // - // It is a ringbuffer, so that the most distant byte in the decoding isn't - // necessarily history[0]. The ringbuffer is full (i.e. it holds 32KiB of - // history) if and only if history_index >= 0x8000. + // history[.. 0x8000] is a ringbuffer, so that the most distant byte in + // the decoding isn't necessarily history[0]. The ringbuffer is full (i.e. + // it holds 32KiB of history) if and only if history_index >= 0x8000. // // history[history_index & 0x7FFF] is where the next byte of decoded output // will be written. - history : array[0x8000] base.u8, // 32 KiB. + // + // When suspended in decoder.transform_io, or after an add_history call, + // history[0x8000 .. 0x8000 + (ML - 1)] duplicates history[.. (ML - 1)], + // where ML is the maximum length (258 as stated above). This simplifies + // copying up to ML bytes from the ringbuffer, as there is no need to split + // the copy around the 0x8000 index. + history : array[0x8000 + (258 - 1)] base.u8, // 32 KiB + (ML - 1) bytes. // code_lengths is used to pass out-of-band data to init_huff. // @@ -147,12 +153,12 @@ pub func decoder.add_history!(hist: slice base.u8) { // If s is longer than the ringbuffer, we can ignore the previous value // of history_index, as we will overwrite the whole ringbuffer. s = s.suffix(up_to: 0x8000) - this.history[..].copy_from_slice!(s: s) + this.history[.. 0x8000].copy_from_slice!(s: s) this.history_index = 0x8000 } else { // Otherwise, append s to the history ringbuffer starting at the // previous history_index (modulo 0x8000). - n_copied = this.history[this.history_index & 0x7FFF ..].copy_from_slice!(s: s) + n_copied = this.history[this.history_index & 0x7FFF .. 0x8000].copy_from_slice!(s: s) if n_copied < s.length() { // a_slice.copy_from(s:b_slice) returns the minimum of the two // slice lengths. If that value is less than b_slice.length(), then @@ -162,7 +168,7 @@ pub func decoder.add_history!(hist: slice base.u8) { // wrap around and copy the remainder of s over the start of the // history ringbuffer. s = s[n_copied ..] - n_copied = this.history[..].copy_from_slice!(s: s) + n_copied = this.history[.. 0x8000].copy_from_slice!(s: s) // Set history_index (modulo 0x8000) to the length of this // remainder. The &0x7FFF is redundant, but proves to the compiler // that the conversion to u32 will not overflow. The +0x8000 is to @@ -178,6 +184,10 @@ pub func decoder.add_history!(hist: slice base.u8) { this.history_index = (this.history_index & 0x7FFF) + ((n_copied & 0x7FFF) as base.u32) + already_full } } + + // Have the tail of this.history duplicate the head. Look for "ML" in the + // comments for the history field for more discussion. + this.history[0x8000 ..].copy_from_slice!(s: this.history[..]) } pub func decoder.workbuf_len() base.range_ii_u64 { diff --git a/std/deflate/decode_huffman_fast.wuffs b/std/deflate/decode_huffman_fast.wuffs index 7f7bd0f3..a803456f 100644 --- a/std/deflate/decode_huffman_fast.wuffs +++ b/std/deflate/decode_huffman_fast.wuffs @@ -30,7 +30,6 @@ pri func decoder.decode_huffman_fast!(dst: base.io_writer, src: base.io_reader) var redir_mask : base.u32[..= 0x7FFF] var length : base.u32[..= 258] var dist_minus_1 : base.u32[..= 0x7FFF] - var n_copied : base.u32 var hlen : base.u32[..= 0x7FFF] var hdist : base.u32 @@ -284,7 +283,6 @@ pri func decoder.decode_huffman_fast!(dst: base.io_writer, src: base.io_reader) bits >>= table_entry_n_bits n_bits -= table_entry_n_bits - n_copied = 0 // The "while true { etc; break }" is a redundant version of "etc", but // its presence minimizes the diff between decode_huffman_fast and // decode_huffman_slow. @@ -314,31 +312,13 @@ pri func decoder.decode_huffman_fast!(dst: base.io_writer, src: base.io_reader) // start copying from. hdist = this.history_index - hdist - // Copy from hdist to the end of this.history, and then - // possibly wrap around the this.history ringbuffer. + // Copy from hdist to the end of this.history. // // This copying is simpler than the decode_huffman_slow version // because it cannot yield. We have already checked that // args.dst.available() is large enough. - // - // The "while true { etc; break }" is a redundant version of - // "etc", but its presence minimizes the diff between - // decode_huffman_fast and decode_huffman_slow. - while true, - inv args.dst.available() >= 258, - { - // TODO: copy_from_slice32 should probably update the - // "args.dst.available() >= 258" fact. It should certainly - // invalidate any fact that was "<=" instead of ">=". - n_copied = args.dst.copy_n32_from_slice!( - n: hlen, s: this.history[hdist & 0x7FFF ..]) - if hlen <= n_copied { - break - } - hlen -= n_copied - args.dst.copy_n32_from_slice!(n: hlen, s: this.history[..]) - break - } + args.dst.copy_n32_from_slice!( + n: hlen, s: this.history[hdist & 0x7FFF ..]) if length == 0 { // No need to copy from args.dst. diff --git a/std/deflate/decode_huffman_slow.wuffs b/std/deflate/decode_huffman_slow.wuffs index a6b07d23..934964ee 100644 --- a/std/deflate/decode_huffman_slow.wuffs +++ b/std/deflate/decode_huffman_slow.wuffs @@ -227,7 +227,7 @@ pri func decoder.decode_huffman_slow?(dst: base.io_writer, src: base.io_reader) // Copy from hdist to the end of this.history. while true { n_copied = args.dst.copy_n32_from_slice!( - n: hlen, s: this.history[hdist & 0x7FFF ..]) + n: hlen, s: this.history[hdist & 0x7FFF .. 0x8000]) if hlen <= n_copied { hlen = 0 break @@ -246,7 +246,7 @@ pri func decoder.decode_huffman_slow?(dst: base.io_writer, src: base.io_reader) if hlen > 0 { while true { n_copied = args.dst.copy_n32_from_slice!( - n: hlen, s: this.history[hdist & 0x7FFF ..]) + n: hlen, s: this.history[hdist & 0x7FFF .. 0x8000]) if hlen <= n_copied { hlen = 0 break diff --git a/test/c/std/deflate.c b/test/c/std/deflate.c index 9883ba88..2aaa430b 100644 --- a/test/c/std/deflate.c +++ b/test/c/std/deflate.c @@ -424,6 +424,30 @@ do_test_wuffs_deflate_history(int i, ": decode: have \"%s\", want \"%s\"", i, starting_history_index, have_z.repr, want_z); } + + // Check that head and the tail of the ringbuffer match. + if (wuffs_base__status__is_suspension(&have_z)) { + const size_t max_length_minus_1 = 257; + + wuffs_base__io_buffer head = ((wuffs_base__io_buffer){ + .data = ((wuffs_base__slice_u8){ + .ptr = dec->private_data.f_history + 0, + .len = max_length_minus_1, + }), + }); + head.meta.wi = max_length_minus_1; + + wuffs_base__io_buffer tail = ((wuffs_base__io_buffer){ + .data = ((wuffs_base__slice_u8){ + .ptr = dec->private_data.f_history + 0x8000, + .len = max_length_minus_1, + }), + }); + tail.meta.wi = max_length_minus_1; + + CHECK_STRING(check_io_buffers_equal("head vs tail ", &head, &tail)); + } + return NULL; }