Files
wuffs/std/deflate/decode_huffman_fast32.wuffs
T
Nigel Tao 592bc14ed1 Have std/deflate allow non-compact dst history
Fixes #52

On a mid-range x86_64 laptop (2016, Skylake):

name                                                      old speed      new speed      delta

wuffs_deflate_decode_1k_full_init/clang9                   101MB/s ± 1%   100MB/s ± 1%   -1.22%  (p=0.000 n=9+9)
wuffs_deflate_decode_1k_part_init/clang9                   119MB/s ± 1%   119MB/s ± 1%     ~     (p=0.139 n=8+9)
wuffs_deflate_decode_10k_full_init/clang9                  221MB/s ± 1%   210MB/s ± 1%   -4.94%  (p=0.000 n=9+9)
wuffs_deflate_decode_10k_part_init/clang9                  225MB/s ± 3%   215MB/s ± 1%   -4.33%  (p=0.000 n=10+8)
wuffs_deflate_decode_100k_just_one_read/clang9             290MB/s ± 0%   284MB/s ± 1%   -2.26%  (p=0.000 n=8+9)
wuffs_deflate_decode_100k_many_big_reads/clang9            188MB/s ± 3%   177MB/s ± 1%   -5.78%  (p=0.000 n=9+10)

wuffs_deflate_decode_1k_full_init/gcc10                    101MB/s ± 0%    97MB/s ± 1%   -3.65%  (p=0.000 n=8+9)
wuffs_deflate_decode_1k_part_init/gcc10                    119MB/s ± 1%   113MB/s ± 2%   -4.65%  (p=0.000 n=9+9)
wuffs_deflate_decode_10k_full_init/gcc10                   214MB/s ± 3%   224MB/s ± 1%   +4.83%  (p=0.000 n=10+10)
wuffs_deflate_decode_10k_part_init/gcc10                   222MB/s ± 1%   231MB/s ± 1%   +4.03%  (p=0.000 n=9+10)
wuffs_deflate_decode_100k_just_one_read/gcc10              281MB/s ± 2%   297MB/s ± 0%   +5.78%  (p=0.000 n=9+9)
wuffs_deflate_decode_100k_many_big_reads/gcc10             184MB/s ± 0%   194MB/s ± 0%   +5.85%  (p=0.000 n=10+8)

mimic_deflate_decode_1k_full_init                          138MB/s ± 1%   138MB/s ± 0%     ~     (p=0.546 n=9+9)
mimic_deflate_decode_10k_full_init                         156MB/s ± 2%   156MB/s ± 0%     ~     (p=0.387 n=9+9)
mimic_deflate_decode_100k_just_one_read                    191MB/s ± 1%   190MB/s ± 2%   -0.94%  (p=0.004 n=9+10)
mimic_deflate_decode_100k_many_big_reads                   147MB/s ± 1%   147MB/s ± 1%     ~     (p=0.356 n=9+10)

wuffs_gzip_decode_10k/clang9                               220MB/s ± 1%   209MB/s ± 1%   -5.29%  (p=0.000 n=10+9)
wuffs_gzip_decode_100k/clang9                              283MB/s ± 2%   279MB/s ± 1%   -1.47%  (p=0.002 n=10+10)

wuffs_gzip_decode_10k/gcc10                                218MB/s ± 1%   214MB/s ± 2%   -1.76%  (p=0.000 n=9+10)
wuffs_gzip_decode_100k/gcc10                               274MB/s ± 0%   264MB/s ± 2%   -3.61%  (p=0.000 n=7+10)

mimic_gzip_decode_10k                                      131MB/s ± 2%   132MB/s ± 0%   +0.81%  (p=0.000 n=10+8)
mimic_gzip_decode_100k                                     156MB/s ± 0%   156MB/s ± 1%     ~     (p=0.423 n=8+9)

wuffs_zlib_decode_10k/clang9                               223MB/s ± 2%   209MB/s ± 2%   -6.41%  (p=0.000 n=10+10)
wuffs_zlib_decode_100k/clang9                              285MB/s ± 0%   280MB/s ± 0%   -1.87%  (p=0.000 n=8+9)

wuffs_zlib_decode_10k/gcc10                                235MB/s ± 0%   215MB/s ± 1%   -8.22%  (p=0.000 n=10+9)
wuffs_zlib_decode_100k/gcc10                               293MB/s ± 0%   268MB/s ± 1%   -8.38%  (p=0.000 n=8+10)

mimic_zlib_decode_10k                                      145MB/s ± 1%   145MB/s ± 1%   -0.45%  (p=0.004 n=9+9)
mimic_zlib_decode_100k                                     175MB/s ± 1%   174MB/s ± 0%     ~     (p=0.055 n=10+8)

wuffs_png_decode_image_19k_8bpp/clang9                     142MB/s ± 1%   152MB/s ± 1%   +6.98%  (p=0.000 n=9+9)
wuffs_png_decode_image_40k_24bpp/clang9                    172MB/s ± 0%   170MB/s ± 0%   -1.63%  (p=0.000 n=8+9)
wuffs_png_decode_image_77k_8bpp/clang9                     504MB/s ± 2%   540MB/s ± 1%   +7.28%  (p=0.000 n=9+10)
wuffs_png_decode_image_552k_32bpp_ignore_checksum/clang9   462MB/s ± 1%   470MB/s ± 0%   +1.57%  (p=0.000 n=9+10)
wuffs_png_decode_image_552k_32bpp_verify_checksum/clang9   445MB/s ± 1%   451MB/s ± 0%   +1.49%  (p=0.000 n=8+10)
wuffs_png_decode_image_4002k_24bpp/clang9                  175MB/s ± 3%   175MB/s ± 1%     ~     (p=0.356 n=9+10)

wuffs_png_decode_image_19k_8bpp/gcc10                      150MB/s ± 0%   153MB/s ± 1%   +2.03%  (p=0.000 n=10+10)
wuffs_png_decode_image_40k_24bpp/gcc10                     186MB/s ± 0%   186MB/s ± 0%     ~     (p=0.408 n=10+8)
wuffs_png_decode_image_77k_8bpp/gcc10                      543MB/s ± 0%   542MB/s ± 0%   -0.23%  (p=0.019 n=9+9)
wuffs_png_decode_image_552k_32bpp_ignore_checksum/gcc10    478MB/s ± 1%   498MB/s ± 0%   +4.12%  (p=0.000 n=10+9)
wuffs_png_decode_image_552k_32bpp_verify_checksum/gcc10    451MB/s ± 4%   476MB/s ± 0%   +5.37%  (p=0.000 n=10+9)
wuffs_png_decode_image_4002k_24bpp/gcc10                   185MB/s ± 2%   190MB/s ± 1%   +2.78%  (p=0.000 n=10+9)

mimic_png_decode_image_19k_8bpp                           63.0MB/s ± 0%  62.9MB/s ± 1%     ~     (p=0.673 n=8+9)
mimic_png_decode_image_40k_24bpp                          78.7MB/s ± 0%  78.9MB/s ± 0%     ~     (p=0.059 n=9+8)
mimic_png_decode_image_77k_8bpp                            192MB/s ± 1%   193MB/s ± 3%   +0.44%  (p=0.028 n=10+9)
mimic_png_decode_image_552k_32bpp_verify_checksum          157MB/s ± 0%   158MB/s ± 1%     ~     (p=0.053 n=10+9)
mimic_png_decode_image_4002k_24bpp                         113MB/s ± 1%   113MB/s ± 1%     ~     (p=0.053 n=10+9)
2021-08-15 10:50:41 +10:00

404 lines
15 KiB
Plaintext

// Copyright 2017 The Wuffs Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// TODO: describe how the xxx_fastxx version differs from the xxx_slow one, the
// assumptions that xxx_fastxx makes, and how that makes it fast.
pri func decoder.decode_huffman_fast32!(dst: base.io_writer, src: base.io_reader) base.status {
// When editing this function, consider making the equivalent change to the
// decode_huffman_slow function. Keep the diff between the two
// decode_huffman_*.wuffs files as small as possible, while retaining both
// correctness and performance.
var bits : base.u32
var n_bits : base.u32
var table_entry : base.u32
var table_entry_n_bits : base.u32[..= 15]
var lmask : base.u32[..= 511]
var dmask : base.u32[..= 511]
var redir_top : base.u32[..= 0xFFFF]
var redir_mask : base.u32[..= 0x7FFF]
var length : base.u32[..= 258]
var dist_minus_1 : base.u32[..= 0x7FFF]
var hlen : base.u32[..= 0x7FFF]
var hdist : base.u32
var hdist_adjustment : base.u32
if (this.n_bits >= 8) or ((this.bits >> (this.n_bits & 7)) <> 0) {
return "#internal error: inconsistent n_bits"
}
bits = this.bits
n_bits = this.n_bits
lmask = ((1 as base.u32) << this.n_huffs_bits[0]) - 1
dmask = ((1 as base.u32) << this.n_huffs_bits[1]) - 1
if this.transformed_history_count < args.dst.history_position() {
return base."#bad I/O position"
}
hdist_adjustment = ((this.transformed_history_count - args.dst.history_position()) & 0xFFFF_FFFF) as base.u32
// Check up front, on each iteration, that we have enough buffer space to
// both read (12 bytes) and write (266 bytes) as much as we need to. Doing
// this check once (per iteration), up front, removes the need to check
// multiple times inside the loop body, so it's faster overall.
//
// For writing, a literal code obviously corresponds to writing 1 byte, and
// 258 is the maximum length in a length-distance pair, as specified in the
// RFC section 3.2.5. Compressed blocks (length and distance codes).
//
// We adjust 258 up to (258 + 8) = 266 since it can be faster to overshoot
// a little and make multiple-of-8-byte copies even when the length in the
// length-distance pair isn't an exact multiple-of-8. Strictly speaking,
// 264 (an exact multiple-of-8) is the tight bound but (258 + 8) is easier
// for the Wuffs proof system, as length's type is refined to [..= 258],
//
// For reading, strictly speaking, we only need 6 bytes (48 bits) of
// available input, because the H-L Literal/Length code is up to 15 bits
// plus up to 5 extra bits, the H-D Distance code is up to 15 bits plus up
// to 13 extra bits and 15 + 5 + 15 + 13 == 48. However,
// args.src.length() assertions are made in terms of bytes, not bits.
//
// In theory, we could write assertions that mixed bytes and bits, such as:
//
// assert ((args.src.length() >= 6) and (n_bits >= 5)) or
// ((args.src.length() >= 4) and (n_bits >= 21))
//
// but that looks clumsy and makes the proofs more complicated. Instead, we
// are conservative and work in terms of bytes only. In bytes, each code
// requires either one (direct) or two (redirect) lookups, up to 2 bytes
// for each. There are also up to 13 extra bits, or 2 bytes. Thus, each
// code requires up to 2 + 2 + 2 == 6 bytes, and there are two codes (one
// from H-L and one from H-D). Therefore, we check for at least 12 bytes.
while.loop(args.dst.length() >= 266) and (args.src.length() >= 12) {
// Ensure that we have at least 15 bits of input.
if n_bits < 15 {
assert n_bits >= 0 // TODO: this shouldn't be necessary.
bits |= args.src.peek_u8_as_u32() << n_bits
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
n_bits += 8
bits |= args.src.peek_u8_as_u32() << n_bits
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
n_bits += 8
assert n_bits >= 15
} else {
assert args.src.length() >= 10
}
// These assertions are redundant, but are listed explicitly for
// clarity. They must hold regardless of which branch was taken for the
// if statement above.
assert args.src.length() >= 10
assert n_bits >= 15
// Decode an lcode symbol from H-L.
table_entry = this.huffs[0][bits & lmask]
table_entry_n_bits = table_entry & 0x0F
bits >>= table_entry_n_bits
n_bits -= table_entry_n_bits
if (table_entry >> 31) <> 0 {
// Literal.
args.dst.write_u8_fast!(a: ((table_entry >> 8) & 0xFF) as base.u8)
continue.loop
} else if (table_entry >> 30) <> 0 {
// No-op; code continues past the if-else chain.
assert args.src.length() >= 8
} else if (table_entry >> 29) <> 0 {
// End of block.
this.end_of_block = true
break.loop
} else if (table_entry >> 28) <> 0 {
// Redirect.
// Ensure that we have at least 15 bits of input.
if n_bits < 15 {
assert n_bits >= 0 // TODO: this shouldn't be necessary.
bits |= args.src.peek_u8_as_u32() << n_bits
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
n_bits += 8
bits |= args.src.peek_u8_as_u32() << n_bits
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
n_bits += 8
assert n_bits >= 15
} else {
assert args.src.length() >= 8
}
// Once again, redundant but explicit assertions.
assert args.src.length() >= 8
assert n_bits >= 15
redir_top = (table_entry >> 8) & 0xFFFF
redir_mask = ((1 as base.u32) << ((table_entry >> 4) & 0x0F)) - 1
table_entry = this.huffs[0][(redir_top + (bits & redir_mask)) & HUFFS_TABLE_MASK]
table_entry_n_bits = table_entry & 0x0F
bits >>= table_entry_n_bits
n_bits -= table_entry_n_bits
if (table_entry >> 31) <> 0 {
// Literal.
args.dst.write_u8_fast!(a: ((table_entry >> 8) & 0xFF) as base.u8)
continue.loop
} else if (table_entry >> 30) <> 0 {
// No-op; code continues past the if-else chain.
} else if (table_entry >> 29) <> 0 {
// End of block.
this.end_of_block = true
break.loop
} else if (table_entry >> 28) <> 0 {
return "#internal error: inconsistent Huffman decoder state"
} else if (table_entry >> 27) <> 0 {
return "#bad Huffman code"
} else {
return "#internal error: inconsistent Huffman decoder state"
}
// Once again, redundant but explicit assertions.
assert args.src.length() >= 8
} else if (table_entry >> 27) <> 0 {
return "#bad Huffman code"
} else {
return "#internal error: inconsistent Huffman decoder state"
}
// length = base_number_minus_3 + 3 + extra_bits.
//
// The -3 is from the bias in script/print-deflate-magic-numbers.go.
// That bias makes the "& 0xFF" 1 and 15-ish lines below correct.
length = ((table_entry >> 8) & 0xFF) + 3
table_entry_n_bits = (table_entry >> 4) & 0x0F
if table_entry_n_bits > 0 {
// Ensure that we have at least 15 bits of input.
if n_bits < 15 {
assert n_bits >= 0 // TODO: this shouldn't be necessary.
bits |= args.src.peek_u8_as_u32() << n_bits
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
n_bits += 8
bits |= args.src.peek_u8_as_u32() << n_bits
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
n_bits += 8
assert n_bits >= 15
} else {
assert args.src.length() >= 6
}
// Once again, redundant but explicit assertions.
assert args.src.length() >= 6
assert n_bits >= 15
// The "+ 253" is the same as "- 3", after the "& 0xFF", but the
// plus form won't require an underflow check.
length = ((length + 253 + bits.low_bits(n: table_entry_n_bits)) & 0xFF) + 3
bits >>= table_entry_n_bits
n_bits -= table_entry_n_bits
} else {
assert args.src.length() >= 6
}
// Ensure that we have at least 15 bits of input.
if n_bits < 15 {
assert n_bits >= 0 // TODO: this shouldn't be necessary.
bits |= args.src.peek_u8_as_u32() << n_bits
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
n_bits += 8
bits |= args.src.peek_u8_as_u32() << n_bits
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
n_bits += 8
assert n_bits >= 15
} else {
assert args.src.length() >= 4
}
// Once again, redundant but explicit assertions.
assert args.src.length() >= 4
assert n_bits >= 15
// Decode a dcode symbol from H-D.
table_entry = this.huffs[1][bits & dmask]
table_entry_n_bits = table_entry & 15
bits >>= table_entry_n_bits
n_bits -= table_entry_n_bits
// Check for a redirect.
if (table_entry >> 28) == 1 {
// Ensure that we have at least 15 bits of input.
if n_bits < 15 {
assert n_bits >= 0 // TODO: this shouldn't be necessary.
bits |= args.src.peek_u8_as_u32() << n_bits
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
n_bits += 8
bits |= args.src.peek_u8_as_u32() << n_bits
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
n_bits += 8
assert n_bits >= 15
} else {
assert args.src.length() >= 2
}
// Once again, redundant but explicit assertions.
assert args.src.length() >= 2
assert n_bits >= 15
redir_top = (table_entry >> 8) & 0xFFFF
redir_mask = ((1 as base.u32) << ((table_entry >> 4) & 0x0F)) - 1
table_entry = this.huffs[1][(redir_top + (bits & redir_mask)) & HUFFS_TABLE_MASK]
table_entry_n_bits = table_entry & 0x0F
bits >>= table_entry_n_bits
n_bits -= table_entry_n_bits
} else {
assert args.src.length() >= 2
}
// For H-D, all symbols should be base_number + extra_bits.
if (table_entry >> 24) <> 0x40 {
if (table_entry >> 24) == 0x08 {
return "#bad Huffman code"
}
return "#internal error: inconsistent Huffman decoder state"
}
// dist_minus_1 = base_number_minus_1 + extra_bits.
// distance = dist_minus_1 + 1.
//
// The -1 is from the bias in script/print-deflate-magic-numbers.go.
// That bias makes the "& 0x7FFF" 2 and 15-ish lines below correct and
// undoing that bias makes proving (dist_minus_1 + 1) > 0 trivial.
dist_minus_1 = (table_entry >> 8) & 0x7FFF
table_entry_n_bits = (table_entry >> 4) & 0x0F
// Ensure that we have at least table_entry_n_bits bits of input.
if n_bits < table_entry_n_bits {
assert n_bits < 15 via "a < b: a < c; c <= b"(c: table_entry_n_bits)
assert n_bits >= 0 // TODO: this shouldn't be necessary.
bits |= args.src.peek_u8_as_u32() << n_bits
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
n_bits += 8
bits |= args.src.peek_u8_as_u32() << n_bits
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
n_bits += 8
assert table_entry_n_bits <= n_bits via "a <= b: a <= c; c <= b"(c: 16)
assert n_bits >= table_entry_n_bits via "a >= b: b <= a"()
}
// Once again, redundant but explicit assertions.
assert n_bits >= table_entry_n_bits
dist_minus_1 = (dist_minus_1 + bits.low_bits(n: table_entry_n_bits)) & 0x7FFF
bits >>= table_entry_n_bits
n_bits -= table_entry_n_bits
// The "while true { etc; break }" is a redundant version of "etc", but
// its presence minimizes the diff between decode_huffman_fastxx and
// decode_huffman_slow.
while true,
pre args.dst.length() >= 266,
{
// We can therefore prove:
assert (length as base.u64) <= args.dst.length() via "a <= b: a <= c; c <= b"(c: 266)
assert ((length + 8) as base.u64) <= args.dst.length() via "a <= b: a <= c; c <= b"(c: 266)
// Copy from this.history.
if ((dist_minus_1 + 1) as base.u64) > args.dst.history_length() {
// Set (hlen, hdist) to be the length-distance pair to copy
// from this.history, and (length, distance) to be the
// remaining length-distance pair to copy from args.dst.
hlen = 0
hdist = (((dist_minus_1 + 1) as base.u64) - args.dst.history_length()) as base.u32
if length > hdist {
assert hdist < length via "a < b: b > a"()
assert hdist < 0x8000 via "a < b: a < c; c <= b"(c: length)
length -= hdist
hlen = hdist
} else {
hlen = length
length = 0
}
hdist ~mod+= hdist_adjustment
if this.history_index < hdist {
return "#bad distance"
}
// Re-purpose the hdist variable as the this.history index to
// start copying from.
hdist = this.history_index - hdist
// Copy from hdist to the end of this.history.
//
// This copying is simpler than the decode_huffman_slow version
// because it cannot yield. We have already checked that
// args.dst.length() is large enough.
args.dst.limited_copy_u32_from_slice!(
up_to: hlen, s: this.history[hdist & 0x7FFF ..])
if length == 0 {
// No need to copy from args.dst.
continue.loop
}
if (((dist_minus_1 + 1) as base.u64) > args.dst.history_length()) or
((length as base.u64) > args.dst.length()) or
(((length + 8) as base.u64) > args.dst.length()) {
return "#internal error: inconsistent distance"
}
}
// Once again, redundant but explicit assertions.
assert (dist_minus_1 + 1) >= 1
assert ((dist_minus_1 + 1) as base.u64) <= args.dst.history_length()
assert (length as base.u64) <= args.dst.length()
assert ((length + 8) as base.u64) <= args.dst.length()
// Copy from args.dst.
//
// For short distances, less than 8 bytes, copying atomic 8-byte
// chunks can result in incorrect output, so we fall back to a
// slower 1-byte-at-a-time copy. Deflate's copy-from-history can
// pick up freshly written bytes. A length = 5, distance = 2 copy
// starting with "abc" should give "abcbcbcb" but the 8-byte chunk
// technique would give "abcbc???", the exact output depending on
// what was previously in the writer buffer.
if (dist_minus_1 + 1) >= 8 {
args.dst.limited_copy_u32_from_history_8_byte_chunks_fast!(
up_to: length, distance: (dist_minus_1 + 1))
} else {
args.dst.limited_copy_u32_from_history_fast!(
up_to: length, distance: (dist_minus_1 + 1))
}
break
} endwhile
} endwhile.loop
// Ensure n_bits < 8 by rewindng args.src, if we loaded too many of its
// bytes into the bits variable.
//
// Note that we can unconditionally call undo_read (without resulting in an
// "invalid I/O operation" error code) only because this whole function can
// never suspend, as all of its I/O operations were checked beforehand for
// sufficient buffer space. Otherwise, resuming from the suspension could
// mean that the (possibly different) args.src is no longer rewindable,
// even if conceptually, this function was responsible for reading the
// bytes we want to rewind.
while n_bits >= 8,
post n_bits < 8,
{
n_bits -= 8
if args.src.can_undo_byte() {
args.src.undo_byte!()
} else {
return "#internal error: inconsistent I/O"
}
} endwhile
this.bits = bits & (((1 as base.u32) << n_bits) - 1)
this.n_bits = n_bits
if (this.n_bits >= 8) or ((this.bits >> this.n_bits) <> 0) {
return "#internal error: inconsistent n_bits"
}
}