Extend std/deflate ringbuffer to simplify copies

name                                             old speed     new speed     delta

wuffs_deflate_decode_1k_full_init/clang5         155MB/s ± 1%  154MB/s ± 0%   -0.98%  (p=0.000 n=10+9)
wuffs_deflate_decode_1k_part_init/clang5         190MB/s ± 0%  193MB/s ± 1%   +1.49%  (p=0.000 n=10+9)
wuffs_deflate_decode_10k_full_init/clang5        240MB/s ± 0%  249MB/s ± 0%   +3.68%  (p=0.000 n=9+10)
wuffs_deflate_decode_10k_part_init/clang5        247MB/s ± 0%  257MB/s ± 1%   +4.03%  (p=0.000 n=10+10)
wuffs_deflate_decode_100k_just_one_read/clang5   292MB/s ± 1%  291MB/s ± 0%   -0.34%  (p=0.043 n=10+9)
wuffs_deflate_decode_100k_many_big_reads/clang5  246MB/s ± 0%  250MB/s ± 0%   +1.77%  (p=0.000 n=9+10)

wuffs_deflate_decode_1k_full_init/gcc9           166MB/s ± 1%  165MB/s ± 1%   -0.48%  (p=0.009 n=10+10)
wuffs_deflate_decode_1k_part_init/gcc9           208MB/s ± 1%  212MB/s ± 0%   +1.85%  (p=0.000 n=10+9)
wuffs_deflate_decode_10k_full_init/gcc9          248MB/s ± 1%  281MB/s ± 0%  +13.05%  (p=0.000 n=9+10)
wuffs_deflate_decode_10k_part_init/gcc9          256MB/s ± 0%  290MB/s ± 0%  +13.34%  (p=0.000 n=10+10)
wuffs_deflate_decode_100k_just_one_read/gcc9     336MB/s ± 0%  333MB/s ± 0%   -0.93%  (p=0.000 n=9+8)
wuffs_deflate_decode_100k_many_big_reads/gcc9    262MB/s ± 0%  280MB/s ± 1%   +6.99%  (p=0.000 n=9+10)
This commit is contained in:
Nigel Tao committed 2020-03-05 12:35:03 +11:00
1 parent 771d4f9d98
commit e65c861207
5 files changed
+80 -69

No files matched your search

+30 -33
View File
@@ -4541,7 +4541,7 @@ struct wuffs_deflate__decoder__struct {
struct {
uint32_t f_huffs[2][1024];
uint8_t f_history[32768];
uint8_t f_history[33025];
uint8_t f_code_lengths[320];
struct {
@@ -11617,18 +11617,24 @@ wuffs_deflate__decoder__add_history(wuffs_deflate__decoder* self,
if (((uint64_t)(v_s.len)) >= 32768) {
v_s = wuffs_base__slice_u8__suffix(v_s, 32768);
wuffs_base__slice_u8__copy_from_slice(
wuffs_base__make_slice_u8(self->private_data.f_history, 32768), v_s);
wuffs_base__slice_u8__subslice_j(
wuffs_base__make_slice_u8(self->private_data.f_history, 33025),
32768),
v_s);
self->private_impl.f_history_index = 32768;
} else {
v_n_copied = wuffs_base__slice_u8__copy_from_slice(
wuffs_base__slice_u8__subslice_i(
wuffs_base__make_slice_u8(self->private_data.f_history, 32768),
(self->private_impl.f_history_index & 32767)),
wuffs_base__slice_u8__subslice_ij(
wuffs_base__make_slice_u8(self->private_data.f_history, 33025),
(self->private_impl.f_history_index & 32767), 32768),
v_s);
if (v_n_copied < ((uint64_t)(v_s.len))) {
v_s = wuffs_base__slice_u8__subslice_i(v_s, v_n_copied);
v_n_copied = wuffs_base__slice_u8__copy_from_slice(
wuffs_base__make_slice_u8(self->private_data.f_history, 32768), v_s);
wuffs_base__slice_u8__subslice_j(
wuffs_base__make_slice_u8(self->private_data.f_history, 33025),
32768),
v_s);
self->private_impl.f_history_index =
(((uint32_t)((v_n_copied & 32767))) + 32768);
} else {
@@ -11641,6 +11647,11 @@ wuffs_deflate__decoder__add_history(wuffs_deflate__decoder* self,
((uint32_t)((v_n_copied & 32767))) + v_already_full);
}
}
wuffs_base__slice_u8__copy_from_slice(
wuffs_base__slice_u8__subslice_i(
wuffs_base__make_slice_u8(self->private_data.f_history, 33025),
32768),
wuffs_base__make_slice_u8(self->private_data.f_history, 33025));
return wuffs_base__make_empty_struct();
}
@@ -12649,7 +12660,6 @@ wuffs_deflate__decoder__decode_huffman_fast(wuffs_deflate__decoder* self,
uint32_t v_redir_mask = 0;
uint32_t v_length = 0;
uint32_t v_dist_minus_1 = 0;
uint32_t v_n_copied = 0;
uint32_t v_hlen = 0;
uint32_t v_hdist = 0;
@@ -12858,7 +12868,6 @@ label__loop__continue:;
32767);
v_bits >>= v_table_entry_n_bits;
v_n_bits -= v_table_entry_n_bits;
v_n_copied = 0;
while (true) {
if (((uint64_t)((v_dist_minus_1 + 1))) >
((uint64_t)(iop_a_dst - io0_a_dst))) {
@@ -12877,23 +12886,11 @@ label__loop__continue:;
goto exit;
}
v_hdist = (self->private_impl.f_history_index - v_hdist);
while (true) {
v_n_copied = wuffs_base__io_writer__copy_n32_from_slice(
&iop_a_dst, io2_a_dst, v_hlen,
wuffs_base__slice_u8__subslice_i(
wuffs_base__make_slice_u8(self->private_data.f_history,
32768),
(v_hdist & 32767)));
if (v_hlen <= v_n_copied) {
goto label__0__break;
}
v_hlen -= v_n_copied;
wuffs_base__io_writer__copy_n32_from_slice(
&iop_a_dst, io2_a_dst, v_hlen,
wuffs_base__make_slice_u8(self->private_data.f_history, 32768));
goto label__0__break;
}
label__0__break:;
wuffs_base__io_writer__copy_n32_from_slice(
&iop_a_dst, io2_a_dst, v_hlen,
wuffs_base__slice_u8__subslice_i(
wuffs_base__make_slice_u8(self->private_data.f_history, 33025),
(v_hdist & 32767)));
if (v_length == 0) {
goto label__loop__continue;
}
@@ -12906,9 +12903,9 @@ label__loop__continue:;
}
wuffs_base__io_writer__copy_n32_from_history_fast(
&iop_a_dst, io0_a_dst, io2_a_dst, v_length, (v_dist_minus_1 + 1));
goto label__1__break;
goto label__0__break;
}
label__1__break:;
label__0__break:;
}
label__loop__break:;
while (v_n_bits >= 8) {
@@ -13257,10 +13254,10 @@ wuffs_deflate__decoder__decode_huffman_slow(wuffs_deflate__decoder* self,
while (true) {
v_n_copied = wuffs_base__io_writer__copy_n32_from_slice(
&iop_a_dst, io2_a_dst, v_hlen,
wuffs_base__slice_u8__subslice_i(
wuffs_base__slice_u8__subslice_ij(
wuffs_base__make_slice_u8(self->private_data.f_history,
32768),
(v_hdist & 32767)));
33025),
(v_hdist & 32767), 32768));
if (v_hlen <= v_n_copied) {
v_hlen = 0;
goto label__4__break;
@@ -13281,10 +13278,10 @@ wuffs_deflate__decoder__decode_huffman_slow(wuffs_deflate__decoder* self,
while (true) {
v_n_copied = wuffs_base__io_writer__copy_n32_from_slice(
&iop_a_dst, io2_a_dst, v_hlen,
wuffs_base__slice_u8__subslice_i(
wuffs_base__slice_u8__subslice_ij(
wuffs_base__make_slice_u8(self->private_data.f_history,
32768),
(v_hdist & 32767)));
33025),
(v_hdist & 32767), 32768));
if (v_hlen <= v_n_copied) {
v_hlen = 0;
goto label__5__break;
+21 -11
View File
@@ -117,18 +117,24 @@ pub struct decoder? implements base.io_transformer(
// Exactly one of the eight bits [24 ..= 31] should be set.
huffs : array[2] array[huffs_table_size] base.u32,
// history holds up to the last 32KiB of decoded output, if the decoding
// was incomplete (e.g. due to a short read or write). RFC 1951 (DEFLATE)
// gives the maximum distance in a length-distance back-reference as 32768,
// or 0x8000.
// history[.. 0x8000] holds up to the last 32KiB of decoded output, if the
// decoding was incomplete (e.g. due to a short read or write). RFC 1951
// (DEFLATE) gives the maximum distance in a length-distance back-reference
// as 32768, or 0x8000. Similarly, the RFC gives the maximum length as 258.
//
// It is a ringbuffer, so that the most distant byte in the decoding isn't
// necessarily history[0]. The ringbuffer is full (i.e. it holds 32KiB of
// history) if and only if history_index >= 0x8000.
// history[.. 0x8000] is a ringbuffer, so that the most distant byte in
// the decoding isn't necessarily history[0]. The ringbuffer is full (i.e.
// it holds 32KiB of history) if and only if history_index >= 0x8000.
//
// history[history_index & 0x7FFF] is where the next byte of decoded output
// will be written.
history : array[0x8000] base.u8, // 32 KiB.
//
// When suspended in decoder.transform_io, or after an add_history call,
// history[0x8000 .. 0x8000 + (ML - 1)] duplicates history[.. (ML - 1)],
// where ML is the maximum length (258 as stated above). This simplifies
// copying up to ML bytes from the ringbuffer, as there is no need to split
// the copy around the 0x8000 index.
history : array[0x8000 + (258 - 1)] base.u8, // 32 KiB + (ML - 1) bytes.
// code_lengths is used to pass out-of-band data to init_huff.
//
@@ -147,12 +153,12 @@ pub func decoder.add_history!(hist: slice base.u8) {
// If s is longer than the ringbuffer, we can ignore the previous value
// of history_index, as we will overwrite the whole ringbuffer.
s = s.suffix(up_to: 0x8000)
this.history[..].copy_from_slice!(s: s)
this.history[.. 0x8000].copy_from_slice!(s: s)
this.history_index = 0x8000
} else {
// Otherwise, append s to the history ringbuffer starting at the
// previous history_index (modulo 0x8000).
n_copied = this.history[this.history_index & 0x7FFF ..].copy_from_slice!(s: s)
n_copied = this.history[this.history_index & 0x7FFF .. 0x8000].copy_from_slice!(s: s)
if n_copied < s.length() {
// a_slice.copy_from(s:b_slice) returns the minimum of the two
// slice lengths. If that value is less than b_slice.length(), then
@@ -162,7 +168,7 @@ pub func decoder.add_history!(hist: slice base.u8) {
// wrap around and copy the remainder of s over the start of the
// history ringbuffer.
s = s[n_copied ..]
n_copied = this.history[..].copy_from_slice!(s: s)
n_copied = this.history[.. 0x8000].copy_from_slice!(s: s)
// Set history_index (modulo 0x8000) to the length of this
// remainder. The &0x7FFF is redundant, but proves to the compiler
// that the conversion to u32 will not overflow. The +0x8000 is to
@@ -178,6 +184,10 @@ pub func decoder.add_history!(hist: slice base.u8) {
this.history_index = (this.history_index & 0x7FFF) + ((n_copied & 0x7FFF) as base.u32) + already_full
}
}
// Have the tail of this.history duplicate the head. Look for "ML" in the
// comments for the history field for more discussion.
this.history[0x8000 ..].copy_from_slice!(s: this.history[..])
}
pub func decoder.workbuf_len() base.range_ii_u64 {
+3 -23
View File
@@ -30,7 +30,6 @@ pri func decoder.decode_huffman_fast!(dst: base.io_writer, src: base.io_reader)
var redir_mask : base.u32[..= 0x7FFF]
var length : base.u32[..= 258]
var dist_minus_1 : base.u32[..= 0x7FFF]
var n_copied : base.u32
var hlen : base.u32[..= 0x7FFF]
var hdist : base.u32
@@ -284,7 +283,6 @@ pri func decoder.decode_huffman_fast!(dst: base.io_writer, src: base.io_reader)
bits >>= table_entry_n_bits
n_bits -= table_entry_n_bits
n_copied = 0
// The "while true { etc; break }" is a redundant version of "etc", but
// its presence minimizes the diff between decode_huffman_fast and
// decode_huffman_slow.
@@ -314,31 +312,13 @@ pri func decoder.decode_huffman_fast!(dst: base.io_writer, src: base.io_reader)
// start copying from.
hdist = this.history_index - hdist
// Copy from hdist to the end of this.history, and then
// possibly wrap around the this.history ringbuffer.
// Copy from hdist to the end of this.history.
//
// This copying is simpler than the decode_huffman_slow version
// because it cannot yield. We have already checked that
// args.dst.available() is large enough.
//
// The "while true { etc; break }" is a redundant version of
// "etc", but its presence minimizes the diff between
// decode_huffman_fast and decode_huffman_slow.
while true,
inv args.dst.available() >= 258,
{
// TODO: copy_from_slice32 should probably update the
// "args.dst.available() >= 258" fact. It should certainly
// invalidate any fact that was "<=" instead of ">=".
n_copied = args.dst.copy_n32_from_slice!(
n: hlen, s: this.history[hdist & 0x7FFF ..])
if hlen <= n_copied {
break
}
hlen -= n_copied
args.dst.copy_n32_from_slice!(n: hlen, s: this.history[..])
break
}
args.dst.copy_n32_from_slice!(
n: hlen, s: this.history[hdist & 0x7FFF ..])
if length == 0 {
// No need to copy from args.dst.
+2 -2
View File
@@ -227,7 +227,7 @@ pri func decoder.decode_huffman_slow?(dst: base.io_writer, src: base.io_reader)
// Copy from hdist to the end of this.history.
while true {
n_copied = args.dst.copy_n32_from_slice!(
n: hlen, s: this.history[hdist & 0x7FFF ..])
n: hlen, s: this.history[hdist & 0x7FFF .. 0x8000])
if hlen <= n_copied {
hlen = 0
break
@@ -246,7 +246,7 @@ pri func decoder.decode_huffman_slow?(dst: base.io_writer, src: base.io_reader)
if hlen > 0 {
while true {
n_copied = args.dst.copy_n32_from_slice!(
n: hlen, s: this.history[hdist & 0x7FFF ..])
n: hlen, s: this.history[hdist & 0x7FFF .. 0x8000])
if hlen <= n_copied {
hlen = 0
break
+24
View File
@@ -424,6 +424,30 @@ do_test_wuffs_deflate_history(int i,
": decode: have \"%s\", want \"%s\"",
i, starting_history_index, have_z.repr, want_z);
}
// Check that head and the tail of the ringbuffer match.
if (wuffs_base__status__is_suspension(&have_z)) {
const size_t max_length_minus_1 = 257;
wuffs_base__io_buffer head = ((wuffs_base__io_buffer){
.data = ((wuffs_base__slice_u8){
.ptr = dec->private_data.f_history + 0,
.len = max_length_minus_1,
}),
});
head.meta.wi = max_length_minus_1;
wuffs_base__io_buffer tail = ((wuffs_base__io_buffer){
.data = ((wuffs_base__slice_u8){
.ptr = dec->private_data.f_history + 0x8000,
.len = max_length_minus_1,
}),
});
tail.meta.wi = max_length_minus_1;
CHECK_STRING(check_io_buffers_equal("head vs tail ", &head, &tail));
}
return NULL;
}