diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index c08442c1..1ef07d92 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -21968,6 +21968,12 @@ wuffs_deflate__decoder__init_huff( uint32_t a_n_codes1, uint32_t a_base_symbol); +static wuffs_base__status +wuffs_deflate__decoder__decode_huffman_fast32( + wuffs_deflate__decoder* self, + wuffs_base__io_buffer* a_dst, + wuffs_base__io_buffer* a_src); + static wuffs_base__status wuffs_deflate__decoder__decode_huffman_fast64( wuffs_deflate__decoder* self, @@ -22329,7 +22335,7 @@ wuffs_deflate__decoder__decode_blocks( if (a_src) { a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); } - v_status = wuffs_deflate__decoder__decode_huffman_fast64(self, a_dst, a_src); + v_status = wuffs_deflate__decoder__decode_huffman_fast32(self, a_dst, a_src); if (a_src) { iop_a_src = a_src->data.ptr + a_src->meta.ri; } @@ -23069,6 +23075,204 @@ wuffs_deflate__decoder__init_huff( return wuffs_base__make_status(NULL); } +// -------- func deflate.decoder.decode_huffman_fast32 + +static wuffs_base__status +wuffs_deflate__decoder__decode_huffman_fast32( + wuffs_deflate__decoder* self, + wuffs_base__io_buffer* a_dst, + wuffs_base__io_buffer* a_src) { + wuffs_base__status status = wuffs_base__make_status(NULL); + + uint64_t v_bits = 0; + uint32_t v_n_bits = 0; + uint32_t v_table_entry = 0; + uint32_t v_table_entry_n_bits = 0; + uint64_t v_lmask = 0; + uint64_t v_dmask = 0; + uint32_t v_redir_top = 0; + uint32_t v_redir_mask = 0; + uint32_t v_length = 0; + uint32_t v_dist_minus_1 = 0; + uint32_t v_hlen = 0; + uint32_t v_hdist = 0; + + uint8_t* iop_a_dst = NULL; + uint8_t* io0_a_dst WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + uint8_t* io1_a_dst WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + uint8_t* io2_a_dst WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + if (a_dst) { + io0_a_dst = a_dst->data.ptr; + io1_a_dst = io0_a_dst + a_dst->meta.wi; + iop_a_dst = io1_a_dst; + io2_a_dst = io0_a_dst + a_dst->data.len; + if (a_dst->meta.closed) { + io2_a_dst = iop_a_dst; + } + } + const uint8_t* iop_a_src = NULL; + const uint8_t* io0_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + const uint8_t* io1_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + const uint8_t* io2_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + if (a_src) { + io0_a_src = a_src->data.ptr; + io1_a_src = io0_a_src + a_src->meta.ri; + iop_a_src = io1_a_src; + io2_a_src = io0_a_src + a_src->meta.wi; + } + + if ((self->private_impl.f_n_bits >= 8) || ((self->private_impl.f_bits >> (self->private_impl.f_n_bits & 7)) != 0)) { + status = wuffs_base__make_status(wuffs_deflate__error__internal_error_inconsistent_n_bits); + goto exit; + } + v_bits = ((uint64_t)(self->private_impl.f_bits)); + v_n_bits = self->private_impl.f_n_bits; + v_lmask = ((((uint64_t)(1)) << self->private_impl.f_n_huffs_bits[0]) - 1); + v_dmask = ((((uint64_t)(1)) << self->private_impl.f_n_huffs_bits[1]) - 1); + label__loop__continue:; + while ((((uint64_t)(io2_a_dst - iop_a_dst)) >= 258) && (((uint64_t)(io2_a_src - iop_a_src)) >= 8)) { + v_bits |= ((uint64_t)(wuffs_base__peek_u64le__no_bounds_check(iop_a_src) << (v_n_bits & 63))); + iop_a_src += ((63 - (v_n_bits & 63)) >> 3); + v_n_bits |= 56; + v_table_entry = self->private_data.f_huffs[0][(v_bits & v_lmask)]; + v_table_entry_n_bits = (v_table_entry & 15); + v_bits >>= v_table_entry_n_bits; + v_n_bits -= v_table_entry_n_bits; + if ((v_table_entry >> 31) != 0) { + (wuffs_base__poke_u8be__no_bounds_check(iop_a_dst, ((uint8_t)(((v_table_entry >> 8) & 255)))), iop_a_dst += 1); + goto label__loop__continue; + } else if ((v_table_entry >> 30) != 0) { + } else if ((v_table_entry >> 29) != 0) { + self->private_impl.f_end_of_block = true; + goto label__loop__break; + } else if ((v_table_entry >> 28) != 0) { + v_redir_top = ((v_table_entry >> 8) & 65535); + v_redir_mask = ((((uint32_t)(1)) << ((v_table_entry >> 4) & 15)) - 1); + v_table_entry = self->private_data.f_huffs[0][((v_redir_top + (((uint32_t)((v_bits & 4294967295))) & v_redir_mask)) & 1023)]; + v_table_entry_n_bits = (v_table_entry & 15); + v_bits >>= v_table_entry_n_bits; + v_n_bits -= v_table_entry_n_bits; + if ((v_table_entry >> 31) != 0) { + (wuffs_base__poke_u8be__no_bounds_check(iop_a_dst, ((uint8_t)(((v_table_entry >> 8) & 255)))), iop_a_dst += 1); + goto label__loop__continue; + } else if ((v_table_entry >> 30) != 0) { + } else if ((v_table_entry >> 29) != 0) { + self->private_impl.f_end_of_block = true; + goto label__loop__break; + } else if ((v_table_entry >> 28) != 0) { + status = wuffs_base__make_status(wuffs_deflate__error__internal_error_inconsistent_huffman_decoder_state); + goto exit; + } else if ((v_table_entry >> 27) != 0) { + status = wuffs_base__make_status(wuffs_deflate__error__bad_huffman_code); + goto exit; + } else { + status = wuffs_base__make_status(wuffs_deflate__error__internal_error_inconsistent_huffman_decoder_state); + goto exit; + } + } else if ((v_table_entry >> 27) != 0) { + status = wuffs_base__make_status(wuffs_deflate__error__bad_huffman_code); + goto exit; + } else { + status = wuffs_base__make_status(wuffs_deflate__error__internal_error_inconsistent_huffman_decoder_state); + goto exit; + } + v_length = (((v_table_entry >> 8) & 255) + 3); + v_table_entry_n_bits = ((v_table_entry >> 4) & 15); + if (v_table_entry_n_bits > 0) { + v_length = (((v_length + 253 + ((uint32_t)(((v_bits) & WUFFS_BASE__LOW_BITS_MASK__U64(v_table_entry_n_bits))))) & 255) + 3); + v_bits >>= v_table_entry_n_bits; + v_n_bits -= v_table_entry_n_bits; + } + v_table_entry = self->private_data.f_huffs[1][(v_bits & v_dmask)]; + v_table_entry_n_bits = (v_table_entry & 15); + v_bits >>= v_table_entry_n_bits; + v_n_bits -= v_table_entry_n_bits; + if ((v_table_entry >> 28) == 1) { + v_redir_top = ((v_table_entry >> 8) & 65535); + v_redir_mask = ((((uint32_t)(1)) << ((v_table_entry >> 4) & 15)) - 1); + v_table_entry = self->private_data.f_huffs[1][((v_redir_top + (((uint32_t)((v_bits & 4294967295))) & v_redir_mask)) & 1023)]; + v_table_entry_n_bits = (v_table_entry & 15); + v_bits >>= v_table_entry_n_bits; + v_n_bits -= v_table_entry_n_bits; + } + if ((v_table_entry >> 24) != 64) { + if ((v_table_entry >> 24) == 8) { + status = wuffs_base__make_status(wuffs_deflate__error__bad_huffman_code); + goto exit; + } + status = wuffs_base__make_status(wuffs_deflate__error__internal_error_inconsistent_huffman_decoder_state); + goto exit; + } + v_dist_minus_1 = ((v_table_entry >> 8) & 32767); + v_table_entry_n_bits = ((v_table_entry >> 4) & 15); + v_dist_minus_1 = ((v_dist_minus_1 + ((uint32_t)(((v_bits) & WUFFS_BASE__LOW_BITS_MASK__U64(v_table_entry_n_bits))))) & 32767); + v_bits >>= v_table_entry_n_bits; + v_n_bits -= v_table_entry_n_bits; + while (true) { + if (((uint64_t)((v_dist_minus_1 + 1))) > ((uint64_t)(iop_a_dst - io0_a_dst))) { + v_hlen = 0; + v_hdist = ((uint32_t)((((uint64_t)((v_dist_minus_1 + 1))) - ((uint64_t)(iop_a_dst - io0_a_dst))))); + if (v_length > v_hdist) { + v_length -= v_hdist; + v_hlen = v_hdist; + } else { + v_hlen = v_length; + v_length = 0; + } + if (self->private_impl.f_history_index < v_hdist) { + status = wuffs_base__make_status(wuffs_deflate__error__bad_distance); + goto exit; + } + v_hdist = (self->private_impl.f_history_index - v_hdist); + wuffs_base__io_writer__limited_copy_u32_from_slice( + &iop_a_dst, io2_a_dst,v_hlen, wuffs_base__slice_u8__subslice_i(wuffs_base__make_slice_u8(self->private_data.f_history, 33025), (v_hdist & 32767))); + if (v_length == 0) { + goto label__loop__continue; + } + if ((((uint64_t)((v_dist_minus_1 + 1))) > ((uint64_t)(iop_a_dst - io0_a_dst))) || (((uint64_t)(v_length)) > ((uint64_t)(io2_a_dst - iop_a_dst)))) { + status = wuffs_base__make_status(wuffs_deflate__error__internal_error_inconsistent_distance); + goto exit; + } + } else { + } + wuffs_base__io_writer__limited_copy_u32_from_history_fast( + &iop_a_dst, io0_a_dst, io2_a_dst, v_length, (v_dist_minus_1 + 1)); + goto label__0__break; + } + label__0__break:; + } + label__loop__break:; + if (v_n_bits > 63) { + status = wuffs_base__make_status(wuffs_deflate__error__internal_error_inconsistent_n_bits); + goto exit; + } + while (v_n_bits >= 8) { + v_n_bits -= 8; + if (iop_a_src > io1_a_src) { + iop_a_src--; + } else { + status = wuffs_base__make_status(wuffs_deflate__error__internal_error_inconsistent_i_o); + goto exit; + } + } + self->private_impl.f_bits = ((uint32_t)((v_bits & ((((uint64_t)(1)) << v_n_bits) - 1)))); + self->private_impl.f_n_bits = v_n_bits; + if ((self->private_impl.f_n_bits >= 8) || ((self->private_impl.f_bits >> self->private_impl.f_n_bits) != 0)) { + status = wuffs_base__make_status(wuffs_deflate__error__internal_error_inconsistent_n_bits); + goto exit; + } + goto exit; + exit: + if (a_dst) { + a_dst->meta.wi = ((size_t)(iop_a_dst - a_dst->data.ptr)); + } + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + + return status; +} + // -------- func deflate.decoder.decode_huffman_fast64 static wuffs_base__status diff --git a/std/deflate/decode_deflate.wuffs b/std/deflate/decode_deflate.wuffs index c4e51c6d..9a98acc5 100644 --- a/std/deflate/decode_deflate.wuffs +++ b/std/deflate/decode_deflate.wuffs @@ -259,7 +259,7 @@ pri func decoder.decode_blocks?(dst: base.io_writer, src: base.io_reader) { this.end_of_block = false while true { if this.util.cpu_arch_is_32_bit() { - status = this.decode_huffman_fast64!(dst: args.dst, src: args.src) + status = this.decode_huffman_fast32!(dst: args.dst, src: args.src) } else { status = this.decode_huffman_fast64!(dst: args.dst, src: args.src) } diff --git a/std/deflate/decode_huffman_fast32.wuffs b/std/deflate/decode_huffman_fast32.wuffs new file mode 100644 index 00000000..cbeb6913 --- /dev/null +++ b/std/deflate/decode_huffman_fast32.wuffs @@ -0,0 +1,271 @@ +// Copyright 2017 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// TODO: describe how the xxx_fastxx version differs from the xxx_slow one, the +// assumptions that xxx_fastxx makes, and how that makes it fast. +pri func decoder.decode_huffman_fast32!(dst: base.io_writer, src: base.io_reader) base.status { + // When editing this function, consider making the equivalent change to the + // decode_huffman_slow function. Keep the diff between the two + // decode_huffman_*.wuffs files as small as possible, while retaining both + // correctness and performance. + + var bits : base.u64 + var n_bits : base.u32 + var table_entry : base.u32 + var table_entry_n_bits : base.u32[..= 15] + var lmask : base.u64[..= 511] + var dmask : base.u64[..= 511] + var redir_top : base.u32[..= 0xFFFF] + var redir_mask : base.u32[..= 0x7FFF] + var length : base.u32[..= 258] + var dist_minus_1 : base.u32[..= 0x7FFF] + var hlen : base.u32[..= 0x7FFF] + var hdist : base.u32 + + if (this.n_bits >= 8) or ((this.bits >> (this.n_bits & 7)) <> 0) { + return "#internal error: inconsistent n_bits" + } + + bits = this.bits as base.u64 + n_bits = this.n_bits + + lmask = ((1 as base.u64) << this.n_huffs_bits[0]) - 1 + dmask = ((1 as base.u64) << this.n_huffs_bits[1]) - 1 + + // Check up front, on each iteration, that we have enough buffer space to + // both read (8 bytes) and write (258 bytes) as much as we need to. Doing + // this check once (per iteration), up front, removes the need to check + // multiple times inside the loop body, so it's faster overall. + // + // For writing, a literal code obviously corresponds to writing 1 byte, and + // 258 is the maximum length in a length-distance pair, as specified in the + // RFC section 3.2.5. Compressed blocks (length and distance codes). + // + // For reading, strictly speaking, we only need 6 bytes (48 bits) of + // available input, because the H-L Literal/Length code is up to 15 bits + // plus up to 5 extra bits, the H-D Distance code is up to 15 bits plus up + // to 13 extra bits and 15 + 5 + 15 + 13 == 48. However, it's faster to + // read 64 bits than 48 bits or (48 - n_bits) bits. + while.loop(args.dst.length() >= 258) and (args.src.length() >= 8) { + // Ensure that we have at least 56 bits of input. + // + // This is "Variant 4" of + // https://fgiesen.wordpress.com/2018/02/20/reading-bits-in-far-too-many-ways-part-2/ + // + // 56, the number of bits in 7 bytes, is a property of that "Variant 4" + // bit-reading technique, and not related to the DEFLATE format per se. + // + // The "& 63" is a no-op, and not part of the original "Variant 4" + // technique, but satisfies Wuffs' overflow/underflow checks. + bits |= args.src.peek_u64le() ~mod<< (n_bits & 63) + args.src.skip_u32_fast!(actual: (63 - (n_bits & 63)) >> 3, worst_case: 8) + n_bits |= 56 + + // Decode an lcode symbol from H-L. + table_entry = this.huffs[0][bits & lmask] + table_entry_n_bits = table_entry & 0x0F + bits >>= table_entry_n_bits + n_bits ~mod-= table_entry_n_bits + + if (table_entry >> 31) <> 0 { + // Literal. + args.dst.write_u8_fast!(a: ((table_entry >> 8) & 0xFF) as base.u8) + continue.loop + } else if (table_entry >> 30) <> 0 { + // No-op; code continues past the if-else chain. + } else if (table_entry >> 29) <> 0 { + // End of block. + this.end_of_block = true + break.loop + } else if (table_entry >> 28) <> 0 { + // Redirect. + redir_top = (table_entry >> 8) & 0xFFFF + redir_mask = ((1 as base.u32) << ((table_entry >> 4) & 0x0F)) - 1 + table_entry = this.huffs[0][ + (redir_top + (((bits & 0xFFFF_FFFF) as base.u32) & redir_mask)) & HUFFS_TABLE_MASK] + table_entry_n_bits = table_entry & 0x0F + bits >>= table_entry_n_bits + n_bits ~mod-= table_entry_n_bits + + if (table_entry >> 31) <> 0 { + // Literal. + args.dst.write_u8_fast!(a: ((table_entry >> 8) & 0xFF) as base.u8) + continue.loop + } else if (table_entry >> 30) <> 0 { + // No-op; code continues past the if-else chain. + } else if (table_entry >> 29) <> 0 { + // End of block. + this.end_of_block = true + break.loop + } else if (table_entry >> 28) <> 0 { + return "#internal error: inconsistent Huffman decoder state" + } else if (table_entry >> 27) <> 0 { + return "#bad Huffman code" + } else { + return "#internal error: inconsistent Huffman decoder state" + } + + } else if (table_entry >> 27) <> 0 { + return "#bad Huffman code" + } else { + return "#internal error: inconsistent Huffman decoder state" + } + + // length = base_number_minus_3 + 3 + extra_bits. + // + // The -3 is from the bias in script/print-deflate-magic-numbers.go. + // That bias makes the "& 0xFF" 1 and 15-ish lines below correct. + length = ((table_entry >> 8) & 0xFF) + 3 + table_entry_n_bits = (table_entry >> 4) & 0x0F + if table_entry_n_bits > 0 { + // The "+ 253" is the same as "- 3", after the "& 0xFF", but the + // plus form won't require an underflow check. + length = ((length + 253 + (bits.low_bits(n: table_entry_n_bits) as base.u32)) & 0xFF) + 3 + bits >>= table_entry_n_bits + n_bits ~mod-= table_entry_n_bits + } + + // Decode a dcode symbol from H-D. + table_entry = this.huffs[1][bits & dmask] + table_entry_n_bits = table_entry & 15 + bits >>= table_entry_n_bits + n_bits ~mod-= table_entry_n_bits + + // Check for a redirect. + if (table_entry >> 28) == 1 { + redir_top = (table_entry >> 8) & 0xFFFF + redir_mask = ((1 as base.u32) << ((table_entry >> 4) & 0x0F)) - 1 + table_entry = this.huffs[1][ + (redir_top + (((bits & 0xFFFF_FFFF) as base.u32) & redir_mask)) & HUFFS_TABLE_MASK] + table_entry_n_bits = table_entry & 0x0F + bits >>= table_entry_n_bits + n_bits ~mod-= table_entry_n_bits + } + + // For H-D, all symbols should be base_number + extra_bits. + if (table_entry >> 24) <> 0x40 { + if (table_entry >> 24) == 0x08 { + return "#bad Huffman code" + } + return "#internal error: inconsistent Huffman decoder state" + } + + // dist_minus_1 = base_number_minus_1 + extra_bits. + // distance = dist_minus_1 + 1. + // + // The -1 is from the bias in script/print-deflate-magic-numbers.go. + // That bias makes the "& 0x7FFF" 2 and 15-ish lines below correct and + // undoing that bias makes proving (dist_minus_1 + 1) > 0 trivial. + dist_minus_1 = (table_entry >> 8) & 0x7FFF + table_entry_n_bits = (table_entry >> 4) & 0x0F + + dist_minus_1 = (dist_minus_1 + (bits.low_bits(n: table_entry_n_bits) as base.u32)) & 0x7FFF + bits >>= table_entry_n_bits + n_bits ~mod-= table_entry_n_bits + + // The "while true { etc; break }" is a redundant version of "etc", but + // its presence minimizes the diff between decode_huffman_fastxx and + // decode_huffman_slow. + while true, + pre args.dst.length() >= 258, + { + // Copy from this.history. + if ((dist_minus_1 + 1) as base.u64) > args.dst.history_length() { + // Set (hlen, hdist) to be the length-distance pair to copy + // from this.history, and (length, distance) to be the + // remaining length-distance pair to copy from args.dst. + hlen = 0 + hdist = (((dist_minus_1 + 1) as base.u64) - args.dst.history_length()) as base.u32 + if length > hdist { + assert hdist < length via "a < b: b > a"() + assert hdist < 0x8000 via "a < b: a < c; c <= b"(c: length) + length -= hdist + hlen = hdist + } else { + hlen = length + length = 0 + } + if this.history_index < hdist { + return "#bad distance" + } + // Re-purpose the hdist variable as the this.history index to + // start copying from. + hdist = this.history_index - hdist + + // Copy from hdist to the end of this.history. + // + // This copying is simpler than the decode_huffman_slow version + // because it cannot yield. We have already checked that + // args.dst.length() is large enough. + args.dst.limited_copy_u32_from_slice!( + up_to: hlen, s: this.history[hdist & 0x7FFF ..]) + + if length == 0 { + // No need to copy from args.dst. + continue.loop + } + + if (((dist_minus_1 + 1) as base.u64) > args.dst.history_length()) or + ((length as base.u64) > args.dst.length()) { + return "#internal error: inconsistent distance" + } + } else { + assert (length as base.u64) <= 258 + assert args.dst.length() >= 258 + // We can therefore prove: + assert (length as base.u64) <= args.dst.length() via "a <= b: a <= c; c <= b"(c: 258) + } + // Once again, redundant but explicit assertions. + assert (dist_minus_1 + 1) > 0 + assert ((dist_minus_1 + 1) as base.u64) <= args.dst.history_length() + assert (length as base.u64) <= args.dst.length() + + // Copy from args.dst. + args.dst.limited_copy_u32_from_history_fast!( + up_to: length, distance: (dist_minus_1 + 1)) + break + } endwhile + } endwhile.loop + + // Ensure n_bits < 8 by rewindng args.src, if we loaded too many of its + // bytes into the bits variable. + // + // Note that we can unconditionally call undo_read (without resulting in an + // "invalid I/O operation" error code) only because this whole function can + // never suspend, as all of its I/O operations were checked beforehand for + // sufficient buffer space. Otherwise, resuming from the suspension could + // mean that the (possibly different) args.src is no longer rewindable, + // even if conceptually, this function was responsible for reading the + // bytes we want to rewind. + if n_bits > 63 { + return "#internal error: inconsistent n_bits" + } + while n_bits >= 8, + post n_bits < 8, + { + n_bits -= 8 + if args.src.can_undo_byte() { + args.src.undo_byte!() + } else { + return "#internal error: inconsistent I/O" + } + } endwhile + + this.bits = (bits & (((1 as base.u64) << n_bits) - 1)) as base.u32 + this.n_bits = n_bits + + if (this.n_bits >= 8) or ((this.bits >> this.n_bits) <> 0) { + return "#internal error: inconsistent n_bits" + } +}