From 9958d6c52031e4c7e2e1f0c56563cb4c8188d1f6 Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Sat, 30 Mar 2024 21:24:59 +1100 Subject: [PATCH] std/lzma: add decoder.decode_bitstream_fast MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit name old speed new speed delta wuffs_lzma_decode_100k/clang14 52.7MB/s ± 0% 54.8MB/s ± 1% +4.00% (p=0.008 n=5+5) wuffs_lzma_decode_100k/gcc12 50.3MB/s ± 1% 56.8MB/s ± 0% +12.93% (p=0.008 n=5+5) $ time example-mzcat < linux-6.8.2.tar.xz > /dev/null Before: user 0m9.725s After: user 0m8.409s --- release/c/wuffs-unsupported-snapshot.c | 907 +++++++++++++++++++- std/lzma/decode_bitstream_fast.wuffs | 1070 ++++++++++++++++++++++++ std/lzma/decode_bitstream_slow.wuffs | 8 +- std/lzma/decode_lzma.wuffs | 20 +- 4 files changed, 1998 insertions(+), 7 deletions(-) create mode 100644 std/lzma/decode_bitstream_fast.wuffs diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index ce15746b..b8643469 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -10518,6 +10518,7 @@ struct wuffs_lzma__decoder__struct { bool f_lzma2_need_dict_reset; bool f_prev_lzma2_chunk_was_uncompressed; bool f_allow_non_zero_initial_byte; + bool f_end_of_chunk; uint8_t f_stashed_bytes[2]; uint32_t f_stashed_bits; uint32_t f_stashed_range; @@ -51820,6 +51821,7 @@ const char wuffs_lzma__error__bad_header[] = "#lzma: bad header"; const char wuffs_lzma__error__truncated_input[] = "#lzma: truncated input"; const char wuffs_lzma__error__unsupported_decoded_length[] = "#lzma: unsupported decoded length"; const char wuffs_lzma__error__unsupported_properties[] = "#lzma: unsupported properties"; +const char wuffs_lzma__error__internal_error_inconsistent_i_o[] = "#lzma: internal error: inconsistent I/O"; const char wuffs_lzma__error__internal_error_inconsistent_dictionary_state[] = "#lzma: internal error: inconsistent dictionary state"; // ---------------- Private Consts @@ -51859,6 +51861,14 @@ WUFFS_LZMA__CLAMP_NO_MORE_THAN_3[8] WUFFS_BASE__POTENTIALLY_UNUSED = { // ---------------- Private Function Prototypes +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__status +wuffs_lzma__decoder__decode_bitstream_fast( + wuffs_lzma__decoder* self, + wuffs_base__io_buffer* a_dst, + wuffs_base__io_buffer* a_src, + wuffs_base__slice_u8 a_workbuf); + WUFFS_BASE__GENERATED_C_CODE static wuffs_base__status wuffs_lzma__decoder__decode_bitstream_slow( @@ -52002,6 +52012,872 @@ sizeof__wuffs_lzma__decoder(void) { // ---------------- Function Implementations +// -------- func lzma.decoder.decode_bitstream_fast + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__status +wuffs_lzma__decoder__decode_bitstream_fast( + wuffs_lzma__decoder* self, + wuffs_base__io_buffer* a_dst, + wuffs_base__io_buffer* a_src, + wuffs_base__slice_u8 a_workbuf) { + wuffs_base__status status = wuffs_base__make_status(NULL); + + uint8_t v_c8 = 0; + uint32_t v_bits = 0; + uint32_t v_range = 0; + uint32_t v_state = 0; + uint32_t v_rep0 = 0; + uint32_t v_rep1 = 0; + uint32_t v_rep2 = 0; + uint32_t v_rep3 = 0; + uint32_t v_reptmp = 0; + uint32_t v_rep = 0; + uint64_t v_pos = 0; + uint64_t v_pos_end = 0; + uint32_t v_lc = 0; + uint64_t v_lp_mask = 0; + uint64_t v_pb_mask = 0; + uint32_t v_prob = 0; + uint32_t v_threshold = 0; + uint32_t v_tree_node = 0; + uint8_t v_prev_byte = 0; + uint32_t v_match_byte = 0; + uint32_t v_match_cusp = 0; + uint32_t v_len_state = 0; + uint32_t v_slot = 0; + uint32_t v_len = 0; + uint32_t v_lanl_offset = 0; + uint32_t v_lanl_old_offset = 0; + uint32_t v_lanl_index = 0; + uint32_t v_num_extra_bits = 0; + uint32_t v_dist_extra_bits = 0; + uint32_t v_high_bit_was_on = 0; + uint32_t v_i = 0; + uint32_t v_index_ao00 = 0; + uint32_t v_index_ao41 = 0; + uint32_t v_index_lit = 0; + uint32_t v_index_len = 0; + uint32_t v_index_small_dist_base = 0; + uint32_t v_index_small_dist_extra = 0; + uint32_t v_index_small_dist = 0; + uint32_t v_index_large_dist = 0; + uint32_t v_dist = 0; + uint32_t v_adj_dist = 0; + uint64_t v_wb_index = 0; + + uint8_t* iop_a_dst = NULL; + uint8_t* io0_a_dst WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + uint8_t* io1_a_dst WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + uint8_t* io2_a_dst WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + if (a_dst && a_dst->data.ptr) { + io0_a_dst = a_dst->data.ptr; + io1_a_dst = io0_a_dst + a_dst->meta.wi; + iop_a_dst = io1_a_dst; + io2_a_dst = io0_a_dst + a_dst->data.len; + if (a_dst->meta.closed) { + io2_a_dst = iop_a_dst; + } + } + const uint8_t* iop_a_src = NULL; + const uint8_t* io0_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + const uint8_t* io1_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + const uint8_t* io2_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + if (a_src && a_src->data.ptr) { + io0_a_src = a_src->data.ptr; + io1_a_src = io0_a_src + a_src->meta.ri; + iop_a_src = io1_a_src; + io2_a_src = io0_a_src + a_src->meta.wi; + } + + v_prev_byte = self->private_impl.f_stashed_bytes[0u]; + v_match_byte = ((uint32_t)(self->private_impl.f_stashed_bytes[1u])); + v_bits = self->private_impl.f_stashed_bits; + v_range = self->private_impl.f_stashed_range; + v_state = self->private_impl.f_stashed_state; + v_rep0 = self->private_impl.f_stashed_rep0; + v_rep1 = self->private_impl.f_stashed_rep1; + v_rep2 = self->private_impl.f_stashed_rep2; + v_rep3 = self->private_impl.f_stashed_rep3; + v_pos = self->private_impl.f_stashed_pos; + v_pos_end = self->private_impl.f_stashed_pos_end; + v_lc = self->private_impl.f_lc; + v_lp_mask = ((((uint64_t)(1u)) << self->private_impl.f_lp) - 1u); + v_pb_mask = ((((uint64_t)(1u)) << self->private_impl.f_pb) - 1u); + while ((((uint64_t)(io2_a_dst - iop_a_dst)) >= 282u) && (((uint64_t)(io2_a_src - iop_a_src)) >= 48u)) { + if (v_pos >= v_pos_end) { + self->private_impl.f_end_of_chunk = true; + break; + } + v_index_ao00 = ((v_state << 4u) | ((uint32_t)((v_pos & v_pb_mask)))); + v_prob = ((uint32_t)(self->private_data.f_probs_ao00[v_index_ao00])); + v_threshold = ((uint32_t)((v_range >> 11u) * v_prob)); + if (v_bits < v_threshold) { + v_range = v_threshold; + v_prob += (((uint32_t)(2048u - v_prob)) >> 5u); + self->private_data.f_probs_ao00[v_index_ao00] = ((uint16_t)(v_prob)); + if ((v_range >> 24u) == 0u) { + v_c8 = wuffs_base__peek_u8be__no_bounds_check(iop_a_src); + iop_a_src += 1u; + v_bits = (((uint32_t)(v_bits << 8u)) | ((uint32_t)(v_c8))); + v_range <<= 8u; + } + v_index_lit = (15u & ((((uint32_t)((v_pos & v_lp_mask))) << v_lc) | (((uint32_t)(v_prev_byte)) >> (8u - v_lc)))); + if (v_state >= 7u) { + v_lanl_offset = 256u; + v_tree_node = 1u; + while (v_tree_node < 256u) { + v_match_byte <<= 1u; + v_lanl_old_offset = v_lanl_offset; + v_lanl_offset &= v_match_byte; + v_lanl_index = (v_lanl_offset + v_lanl_old_offset + v_tree_node); + v_prob = ((uint32_t)(self->private_data.f_probs_lit[v_index_lit][v_lanl_index])); + v_threshold = ((uint32_t)((v_range >> 11u) * v_prob)); + if (v_bits < v_threshold) { + v_lanl_offset = ((v_lanl_offset ^ v_lanl_old_offset) & 256u); + v_range = v_threshold; + v_prob += (((uint32_t)(2048u - v_prob)) >> 5u); + self->private_data.f_probs_lit[v_index_lit][v_lanl_index] = ((uint16_t)(v_prob)); + v_tree_node = (v_tree_node << 1u); + } else { + v_bits -= v_threshold; + v_range -= v_threshold; + v_prob -= (v_prob >> 5u); + self->private_data.f_probs_lit[v_index_lit][v_lanl_index] = ((uint16_t)(v_prob)); + v_tree_node = ((v_tree_node << 1u) | 1u); + } + if ((v_range >> 24u) == 0u) { + if (((uint64_t)(io2_a_src - iop_a_src)) <= 0u) { + status = wuffs_base__make_status(wuffs_lzma__error__internal_error_inconsistent_i_o); + goto exit; + } + v_c8 = wuffs_base__peek_u8be__no_bounds_check(iop_a_src); + iop_a_src += 1u; + v_bits = (((uint32_t)(v_bits << 8u)) | ((uint32_t)(v_c8))); + v_range <<= 8u; + } + } + } else { + v_tree_node = 1u; + while (v_tree_node < 256u) { + v_prob = ((uint32_t)(self->private_data.f_probs_lit[v_index_lit][v_tree_node])); + v_threshold = ((uint32_t)((v_range >> 11u) * v_prob)); + if (v_bits < v_threshold) { + v_range = v_threshold; + v_prob += (((uint32_t)(2048u - v_prob)) >> 5u); + self->private_data.f_probs_lit[v_index_lit][v_tree_node] = ((uint16_t)(v_prob)); + v_tree_node = (v_tree_node << 1u); + } else { + v_bits -= v_threshold; + v_range -= v_threshold; + v_prob -= (v_prob >> 5u); + self->private_data.f_probs_lit[v_index_lit][v_tree_node] = ((uint16_t)(v_prob)); + v_tree_node = ((v_tree_node << 1u) | 1u); + } + if ((v_range >> 24u) == 0u) { + if (((uint64_t)(io2_a_src - iop_a_src)) <= 0u) { + status = wuffs_base__make_status(wuffs_lzma__error__internal_error_inconsistent_i_o); + goto exit; + } + v_c8 = wuffs_base__peek_u8be__no_bounds_check(iop_a_src); + iop_a_src += 1u; + v_bits = (((uint32_t)(v_bits << 8u)) | ((uint32_t)(v_c8))); + v_range <<= 8u; + } + } + } + v_prev_byte = ((uint8_t)(v_tree_node)); + (wuffs_base__poke_u8be__no_bounds_check(iop_a_dst, v_prev_byte), iop_a_dst += 1); + v_pos += 1u; + v_state = ((uint32_t)(WUFFS_LZMA__STATE_TRANSITION_LITERAL[v_state])); + continue; + } + v_bits -= v_threshold; + v_range -= v_threshold; + v_prob -= (v_prob >> 5u); + self->private_data.f_probs_ao00[v_index_ao00] = ((uint16_t)(v_prob)); + if ((v_range >> 24u) == 0u) { + v_c8 = wuffs_base__peek_u8be__no_bounds_check(iop_a_src); + iop_a_src += 1u; + v_bits = (((uint32_t)(v_bits << 8u)) | ((uint32_t)(v_c8))); + v_range <<= 8u; + } else { + } + do { + v_prob = ((uint32_t)(self->private_data.f_probs_ao20[v_state])); + v_threshold = ((uint32_t)((v_range >> 11u) * v_prob)); + if (v_bits < v_threshold) { + v_range = v_threshold; + v_prob += (((uint32_t)(2048u - v_prob)) >> 5u); + self->private_data.f_probs_ao20[v_state] = ((uint16_t)(v_prob)); + if ((v_range >> 24u) == 0u) { + v_c8 = wuffs_base__peek_u8be__no_bounds_check(iop_a_src); + iop_a_src += 1u; + v_bits = (((uint32_t)(v_bits << 8u)) | ((uint32_t)(v_c8))); + v_range <<= 8u; + } else { + } + do { + v_prob = ((uint32_t)(self->private_data.f_probs_match_len_low[0u][0u])); + v_threshold = ((uint32_t)((v_range >> 11u) * v_prob)); + if (v_bits < v_threshold) { + v_range = v_threshold; + v_prob += (((uint32_t)(2048u - v_prob)) >> 5u); + self->private_data.f_probs_match_len_low[0u][0u] = ((uint16_t)(v_prob)); + if ((v_range >> 24u) == 0u) { + v_c8 = wuffs_base__peek_u8be__no_bounds_check(iop_a_src); + iop_a_src += 1u; + v_bits = (((uint32_t)(v_bits << 8u)) | ((uint32_t)(v_c8))); + v_range <<= 8u; + } + v_index_len = ((uint32_t)((v_pos & v_pb_mask))); + v_tree_node = 1u; + while (v_tree_node < 8u) { + v_prob = ((uint32_t)(self->private_data.f_probs_match_len_low[v_index_len][v_tree_node])); + v_threshold = ((uint32_t)((v_range >> 11u) * v_prob)); + if (v_bits < v_threshold) { + v_range = v_threshold; + v_prob += (((uint32_t)(2048u - v_prob)) >> 5u); + self->private_data.f_probs_match_len_low[v_index_len][v_tree_node] = ((uint16_t)(v_prob)); + v_tree_node = (v_tree_node << 1u); + } else { + v_bits -= v_threshold; + v_range -= v_threshold; + v_prob -= (v_prob >> 5u); + self->private_data.f_probs_match_len_low[v_index_len][v_tree_node] = ((uint16_t)(v_prob)); + v_tree_node = ((v_tree_node << 1u) | 1u); + } + if ((v_range >> 24u) == 0u) { + if (((uint64_t)(io2_a_src - iop_a_src)) <= 0u) { + status = wuffs_base__make_status(wuffs_lzma__error__internal_error_inconsistent_i_o); + goto exit; + } + v_c8 = wuffs_base__peek_u8be__no_bounds_check(iop_a_src); + iop_a_src += 1u; + v_bits = (((uint32_t)(v_bits << 8u)) | ((uint32_t)(v_c8))); + v_range <<= 8u; + } + } + v_len_state = ((uint32_t)(WUFFS_LZMA__CLAMP_NO_MORE_THAN_3[(v_tree_node & 7u)])); + v_len = ((v_tree_node & 7u) + 2u); + break; + } + v_bits -= v_threshold; + v_range -= v_threshold; + v_prob -= (v_prob >> 5u); + self->private_data.f_probs_match_len_low[0u][0u] = ((uint16_t)(v_prob)); + if ((v_range >> 24u) == 0u) { + if (((uint64_t)(io2_a_src - iop_a_src)) <= 0u) { + status = wuffs_base__make_status(wuffs_lzma__error__internal_error_inconsistent_i_o); + goto exit; + } + v_c8 = wuffs_base__peek_u8be__no_bounds_check(iop_a_src); + iop_a_src += 1u; + v_bits = (((uint32_t)(v_bits << 8u)) | ((uint32_t)(v_c8))); + v_range <<= 8u; + } + v_prob = ((uint32_t)(self->private_data.f_probs_match_len_mid[0u][0u])); + v_threshold = ((uint32_t)((v_range >> 11u) * v_prob)); + if (v_bits < v_threshold) { + v_range = v_threshold; + v_prob += (((uint32_t)(2048u - v_prob)) >> 5u); + self->private_data.f_probs_match_len_mid[0u][0u] = ((uint16_t)(v_prob)); + if ((v_range >> 24u) == 0u) { + if (((uint64_t)(io2_a_src - iop_a_src)) <= 0u) { + status = wuffs_base__make_status(wuffs_lzma__error__internal_error_inconsistent_i_o); + goto exit; + } + v_c8 = wuffs_base__peek_u8be__no_bounds_check(iop_a_src); + iop_a_src += 1u; + v_bits = (((uint32_t)(v_bits << 8u)) | ((uint32_t)(v_c8))); + v_range <<= 8u; + } + v_index_len = ((uint32_t)((v_pos & v_pb_mask))); + v_tree_node = 1u; + while (v_tree_node < 8u) { + v_prob = ((uint32_t)(self->private_data.f_probs_match_len_mid[v_index_len][v_tree_node])); + v_threshold = ((uint32_t)((v_range >> 11u) * v_prob)); + if (v_bits < v_threshold) { + v_range = v_threshold; + v_prob += (((uint32_t)(2048u - v_prob)) >> 5u); + self->private_data.f_probs_match_len_mid[v_index_len][v_tree_node] = ((uint16_t)(v_prob)); + v_tree_node = (v_tree_node << 1u); + } else { + v_bits -= v_threshold; + v_range -= v_threshold; + v_prob -= (v_prob >> 5u); + self->private_data.f_probs_match_len_mid[v_index_len][v_tree_node] = ((uint16_t)(v_prob)); + v_tree_node = ((v_tree_node << 1u) | 1u); + } + if ((v_range >> 24u) == 0u) { + if (((uint64_t)(io2_a_src - iop_a_src)) <= 0u) { + status = wuffs_base__make_status(wuffs_lzma__error__internal_error_inconsistent_i_o); + goto exit; + } + v_c8 = wuffs_base__peek_u8be__no_bounds_check(iop_a_src); + iop_a_src += 1u; + v_bits = (((uint32_t)(v_bits << 8u)) | ((uint32_t)(v_c8))); + v_range <<= 8u; + } + } + v_len = ((v_tree_node & 7u) + 10u); + v_len_state = 3u; + break; + } + v_bits -= v_threshold; + v_range -= v_threshold; + v_prob -= (v_prob >> 5u); + self->private_data.f_probs_match_len_mid[0u][0u] = ((uint16_t)(v_prob)); + if ((v_range >> 24u) == 0u) { + if (((uint64_t)(io2_a_src - iop_a_src)) <= 0u) { + status = wuffs_base__make_status(wuffs_lzma__error__internal_error_inconsistent_i_o); + goto exit; + } + v_c8 = wuffs_base__peek_u8be__no_bounds_check(iop_a_src); + iop_a_src += 1u; + v_bits = (((uint32_t)(v_bits << 8u)) | ((uint32_t)(v_c8))); + v_range <<= 8u; + } + v_tree_node = 1u; + while (v_tree_node < 256u) { + v_prob = ((uint32_t)(self->private_data.f_probs_match_len_high[0u][v_tree_node])); + v_threshold = ((uint32_t)((v_range >> 11u) * v_prob)); + if (v_bits < v_threshold) { + v_range = v_threshold; + v_prob += (((uint32_t)(2048u - v_prob)) >> 5u); + self->private_data.f_probs_match_len_high[0u][v_tree_node] = ((uint16_t)(v_prob)); + v_tree_node = (v_tree_node << 1u); + } else { + v_bits -= v_threshold; + v_range -= v_threshold; + v_prob -= (v_prob >> 5u); + self->private_data.f_probs_match_len_high[0u][v_tree_node] = ((uint16_t)(v_prob)); + v_tree_node = ((v_tree_node << 1u) | 1u); + } + if ((v_range >> 24u) == 0u) { + if (((uint64_t)(io2_a_src - iop_a_src)) <= 0u) { + status = wuffs_base__make_status(wuffs_lzma__error__internal_error_inconsistent_i_o); + goto exit; + } + v_c8 = wuffs_base__peek_u8be__no_bounds_check(iop_a_src); + iop_a_src += 1u; + v_bits = (((uint32_t)(v_bits << 8u)) | ((uint32_t)(v_c8))); + v_range <<= 8u; + } + } + v_len = ((v_tree_node & 255u) + 18u); + v_len_state = 3u; + } while (0); + v_slot = 1u; + while (v_slot < 64u) { + v_prob = ((uint32_t)(self->private_data.f_probs_slot[v_len_state][v_slot])); + v_threshold = ((uint32_t)((v_range >> 11u) * v_prob)); + if (v_bits < v_threshold) { + v_range = v_threshold; + v_prob += (((uint32_t)(2048u - v_prob)) >> 5u); + self->private_data.f_probs_slot[v_len_state][v_slot] = ((uint16_t)(v_prob)); + v_slot = (v_slot << 1u); + } else { + v_bits -= v_threshold; + v_range -= v_threshold; + v_prob -= (v_prob >> 5u); + self->private_data.f_probs_slot[v_len_state][v_slot] = ((uint16_t)(v_prob)); + v_slot = ((v_slot << 1u) | 1u); + } + if ((v_range >> 24u) == 0u) { + if (((uint64_t)(io2_a_src - iop_a_src)) <= 0u) { + status = wuffs_base__make_status(wuffs_lzma__error__internal_error_inconsistent_i_o); + goto exit; + } + v_c8 = wuffs_base__peek_u8be__no_bounds_check(iop_a_src); + iop_a_src += 1u; + v_bits = (((uint32_t)(v_bits << 8u)) | ((uint32_t)(v_c8))); + v_range <<= 8u; + } + } + v_slot &= 63u; + v_rep = v_slot; + if (v_slot < 4u) { + } else if (v_slot < 14u) { + v_num_extra_bits = ((v_slot >> 1u) - 1u); + v_rep = ((2u | (v_slot & 1u)) << v_num_extra_bits); + v_index_small_dist_base = ((uint32_t)(v_rep - v_slot)); + v_index_small_dist_extra = 1u; + v_dist_extra_bits = 0u; + v_i = 0u; + while (v_i < v_num_extra_bits) { + v_index_small_dist = (((uint32_t)(v_index_small_dist_base + v_index_small_dist_extra)) & 127u); + v_prob = ((uint32_t)(self->private_data.f_probs_small_dist[v_index_small_dist])); + v_threshold = ((uint32_t)((v_range >> 11u) * v_prob)); + if (v_bits < v_threshold) { + v_range = v_threshold; + v_prob += (((uint32_t)(2048u - v_prob)) >> 5u); + self->private_data.f_probs_small_dist[v_index_small_dist] = ((uint16_t)(v_prob)); + v_index_small_dist_extra = ((uint32_t)(v_index_small_dist_extra << 1u)); + v_i += 1u; + } else { + v_bits -= v_threshold; + v_range -= v_threshold; + v_prob -= (v_prob >> 5u); + self->private_data.f_probs_small_dist[v_index_small_dist] = ((uint16_t)(v_prob)); + v_index_small_dist_extra = (((uint32_t)(v_index_small_dist_extra << 1u)) | 1u); + v_dist_extra_bits |= (((uint32_t)(1u)) << v_i); + v_i += 1u; + } + if ((v_range >> 24u) == 0u) { + if (((uint64_t)(io2_a_src - iop_a_src)) <= 0u) { + status = wuffs_base__make_status(wuffs_lzma__error__internal_error_inconsistent_i_o); + goto exit; + } + v_c8 = wuffs_base__peek_u8be__no_bounds_check(iop_a_src); + iop_a_src += 1u; + v_bits = (((uint32_t)(v_bits << 8u)) | ((uint32_t)(v_c8))); + v_range <<= 8u; + } + } + v_rep += v_dist_extra_bits; + } else { + v_num_extra_bits = ((v_slot >> 1u) - 1u); + v_rep = ((2u | (v_slot & 1u)) << v_num_extra_bits); + v_dist_extra_bits = 0u; + while (true) { + v_range >>= 1u; + v_bits -= v_range; + v_high_bit_was_on = ((uint32_t)(0u - (v_bits >> 31u))); + v_bits += (v_range & v_high_bit_was_on); + v_dist_extra_bits = (((uint32_t)(v_dist_extra_bits << 1u)) | (((uint32_t)(v_high_bit_was_on + 1u)) & 1u)); + if ((v_range >> 24u) == 0u) { + if (((uint64_t)(io2_a_src - iop_a_src)) <= 0u) { + status = wuffs_base__make_status(wuffs_lzma__error__internal_error_inconsistent_i_o); + goto exit; + } + v_c8 = wuffs_base__peek_u8be__no_bounds_check(iop_a_src); + iop_a_src += 1u; + v_bits = (((uint32_t)(v_bits << 8u)) | ((uint32_t)(v_c8))); + v_range <<= 8u; + } + v_num_extra_bits -= 1u; + if (v_num_extra_bits <= 4u) { + break; + } + } + v_dist_extra_bits <<= 4u; + v_index_large_dist = 1u; + while (true) { + v_prob = ((uint32_t)(self->private_data.f_probs_large_dist[v_index_large_dist])); + v_threshold = ((uint32_t)((v_range >> 11u) * v_prob)); + if (v_bits < v_threshold) { + v_range = v_threshold; + v_prob += (((uint32_t)(2048u - v_prob)) >> 5u); + self->private_data.f_probs_large_dist[v_index_large_dist] = ((uint16_t)(v_prob)); + v_index_large_dist = (15u & ((uint32_t)(v_index_large_dist << 1u))); + } else { + v_bits -= v_threshold; + v_range -= v_threshold; + v_prob -= (v_prob >> 5u); + self->private_data.f_probs_large_dist[v_index_large_dist] = ((uint16_t)(v_prob)); + v_index_large_dist = (15u & (((uint32_t)(v_index_large_dist << 1u)) | 1u)); + v_dist_extra_bits |= (((uint32_t)(1u)) << (4u - v_num_extra_bits)); + } + if ((v_range >> 24u) == 0u) { + if (((uint64_t)(io2_a_src - iop_a_src)) <= 0u) { + status = wuffs_base__make_status(wuffs_lzma__error__internal_error_inconsistent_i_o); + goto exit; + } + v_c8 = wuffs_base__peek_u8be__no_bounds_check(iop_a_src); + iop_a_src += 1u; + v_bits = (((uint32_t)(v_bits << 8u)) | ((uint32_t)(v_c8))); + v_range <<= 8u; + } + v_num_extra_bits -= 1u; + if (v_num_extra_bits <= 0u) { + break; + } + } + v_rep += v_dist_extra_bits; + } + if (v_rep >= 4294967295u) { + self->private_impl.f_end_of_chunk = true; + goto label__outer__break; + } + v_rep3 = v_rep2; + v_rep2 = v_rep1; + v_rep1 = v_rep0; + v_rep0 = v_rep; + v_state = ((uint32_t)(WUFFS_LZMA__STATE_TRANSITION_MATCH[v_state])); + break; + } + v_bits -= v_threshold; + v_range -= v_threshold; + v_prob -= (v_prob >> 5u); + self->private_data.f_probs_ao20[v_state] = ((uint16_t)(v_prob)); + if ((v_range >> 24u) == 0u) { + v_c8 = wuffs_base__peek_u8be__no_bounds_check(iop_a_src); + iop_a_src += 1u; + v_bits = (((uint32_t)(v_bits << 8u)) | ((uint32_t)(v_c8))); + v_range <<= 8u; + } else { + } + v_prob = ((uint32_t)(self->private_data.f_probs_ao40[v_state])); + v_threshold = ((uint32_t)((v_range >> 11u) * v_prob)); + if (v_bits < v_threshold) { + v_range = v_threshold; + v_prob += (((uint32_t)(2048u - v_prob)) >> 5u); + self->private_data.f_probs_ao40[v_state] = ((uint16_t)(v_prob)); + if ((v_range >> 24u) == 0u) { + v_c8 = wuffs_base__peek_u8be__no_bounds_check(iop_a_src); + iop_a_src += 1u; + v_bits = (((uint32_t)(v_bits << 8u)) | ((uint32_t)(v_c8))); + v_range <<= 8u; + } else { + } + v_index_ao41 = ((v_state << 4u) | ((uint32_t)((v_pos & v_pb_mask)))); + v_prob = ((uint32_t)(self->private_data.f_probs_ao41[v_index_ao41])); + v_threshold = ((uint32_t)((v_range >> 11u) * v_prob)); + if (v_bits < v_threshold) { + v_range = v_threshold; + v_prob += (((uint32_t)(2048u - v_prob)) >> 5u); + self->private_data.f_probs_ao41[v_index_ao41] = ((uint16_t)(v_prob)); + if ((v_range >> 24u) == 0u) { + v_c8 = wuffs_base__peek_u8be__no_bounds_check(iop_a_src); + iop_a_src += 1u; + v_bits = (((uint32_t)(v_bits << 8u)) | ((uint32_t)(v_c8))); + v_range <<= 8u; + } + v_len = 1u; + v_state = ((uint32_t)(WUFFS_LZMA__STATE_TRANSITION_SHORTREP[v_state])); + break; + } + v_bits -= v_threshold; + v_range -= v_threshold; + v_prob -= (v_prob >> 5u); + self->private_data.f_probs_ao41[v_index_ao41] = ((uint16_t)(v_prob)); + if ((v_range >> 24u) == 0u) { + v_c8 = wuffs_base__peek_u8be__no_bounds_check(iop_a_src); + iop_a_src += 1u; + v_bits = (((uint32_t)(v_bits << 8u)) | ((uint32_t)(v_c8))); + v_range <<= 8u; + } + } else { + v_bits -= v_threshold; + v_range -= v_threshold; + v_prob -= (v_prob >> 5u); + self->private_data.f_probs_ao40[v_state] = ((uint16_t)(v_prob)); + if ((v_range >> 24u) == 0u) { + v_c8 = wuffs_base__peek_u8be__no_bounds_check(iop_a_src); + iop_a_src += 1u; + v_bits = (((uint32_t)(v_bits << 8u)) | ((uint32_t)(v_c8))); + v_range <<= 8u; + } else { + } + v_prob = ((uint32_t)(self->private_data.f_probs_ao60[v_state])); + v_threshold = ((uint32_t)((v_range >> 11u) * v_prob)); + if (v_bits < v_threshold) { + v_range = v_threshold; + v_prob += (((uint32_t)(2048u - v_prob)) >> 5u); + self->private_data.f_probs_ao60[v_state] = ((uint16_t)(v_prob)); + if ((v_range >> 24u) == 0u) { + v_c8 = wuffs_base__peek_u8be__no_bounds_check(iop_a_src); + iop_a_src += 1u; + v_bits = (((uint32_t)(v_bits << 8u)) | ((uint32_t)(v_c8))); + v_range <<= 8u; + } + v_reptmp = v_rep1; + v_rep1 = v_rep0; + v_rep0 = v_reptmp; + } else { + v_bits -= v_threshold; + v_range -= v_threshold; + v_prob -= (v_prob >> 5u); + self->private_data.f_probs_ao60[v_state] = ((uint16_t)(v_prob)); + if ((v_range >> 24u) == 0u) { + v_c8 = wuffs_base__peek_u8be__no_bounds_check(iop_a_src); + iop_a_src += 1u; + v_bits = (((uint32_t)(v_bits << 8u)) | ((uint32_t)(v_c8))); + v_range <<= 8u; + } else { + } + v_prob = ((uint32_t)(self->private_data.f_probs_ao63[v_state])); + v_threshold = ((uint32_t)((v_range >> 11u) * v_prob)); + if (v_bits < v_threshold) { + v_range = v_threshold; + v_prob += (((uint32_t)(2048u - v_prob)) >> 5u); + self->private_data.f_probs_ao63[v_state] = ((uint16_t)(v_prob)); + if ((v_range >> 24u) == 0u) { + v_c8 = wuffs_base__peek_u8be__no_bounds_check(iop_a_src); + iop_a_src += 1u; + v_bits = (((uint32_t)(v_bits << 8u)) | ((uint32_t)(v_c8))); + v_range <<= 8u; + } + v_reptmp = v_rep2; + v_rep2 = v_rep1; + v_rep1 = v_rep0; + v_rep0 = v_reptmp; + } else { + v_bits -= v_threshold; + v_range -= v_threshold; + v_prob -= (v_prob >> 5u); + self->private_data.f_probs_ao63[v_state] = ((uint16_t)(v_prob)); + if ((v_range >> 24u) == 0u) { + v_c8 = wuffs_base__peek_u8be__no_bounds_check(iop_a_src); + iop_a_src += 1u; + v_bits = (((uint32_t)(v_bits << 8u)) | ((uint32_t)(v_c8))); + v_range <<= 8u; + } + v_reptmp = v_rep3; + v_rep3 = v_rep2; + v_rep2 = v_rep1; + v_rep1 = v_rep0; + v_rep0 = v_reptmp; + } + } + } + do { + v_prob = ((uint32_t)(self->private_data.f_probs_longrep_len_low[0u][0u])); + v_threshold = ((uint32_t)((v_range >> 11u) * v_prob)); + if (v_bits < v_threshold) { + v_range = v_threshold; + v_prob += (((uint32_t)(2048u - v_prob)) >> 5u); + self->private_data.f_probs_longrep_len_low[0u][0u] = ((uint16_t)(v_prob)); + if ((v_range >> 24u) == 0u) { + if (((uint64_t)(io2_a_src - iop_a_src)) <= 0u) { + status = wuffs_base__make_status(wuffs_lzma__error__internal_error_inconsistent_i_o); + goto exit; + } + v_c8 = wuffs_base__peek_u8be__no_bounds_check(iop_a_src); + iop_a_src += 1u; + v_bits = (((uint32_t)(v_bits << 8u)) | ((uint32_t)(v_c8))); + v_range <<= 8u; + } + v_index_len = ((uint32_t)((v_pos & v_pb_mask))); + v_tree_node = 1u; + while (v_tree_node < 8u) { + v_prob = ((uint32_t)(self->private_data.f_probs_longrep_len_low[v_index_len][v_tree_node])); + v_threshold = ((uint32_t)((v_range >> 11u) * v_prob)); + if (v_bits < v_threshold) { + v_range = v_threshold; + v_prob += (((uint32_t)(2048u - v_prob)) >> 5u); + self->private_data.f_probs_longrep_len_low[v_index_len][v_tree_node] = ((uint16_t)(v_prob)); + v_tree_node = (v_tree_node << 1u); + } else { + v_bits -= v_threshold; + v_range -= v_threshold; + v_prob -= (v_prob >> 5u); + self->private_data.f_probs_longrep_len_low[v_index_len][v_tree_node] = ((uint16_t)(v_prob)); + v_tree_node = ((v_tree_node << 1u) | 1u); + } + if ((v_range >> 24u) == 0u) { + if (((uint64_t)(io2_a_src - iop_a_src)) <= 0u) { + status = wuffs_base__make_status(wuffs_lzma__error__internal_error_inconsistent_i_o); + goto exit; + } + v_c8 = wuffs_base__peek_u8be__no_bounds_check(iop_a_src); + iop_a_src += 1u; + v_bits = (((uint32_t)(v_bits << 8u)) | ((uint32_t)(v_c8))); + v_range <<= 8u; + } + } + v_len = ((v_tree_node & 7u) + 2u); + v_state = ((uint32_t)(WUFFS_LZMA__STATE_TRANSITION_LONGREP[v_state])); + break; + } + v_bits -= v_threshold; + v_range -= v_threshold; + v_prob -= (v_prob >> 5u); + self->private_data.f_probs_longrep_len_low[0u][0u] = ((uint16_t)(v_prob)); + if ((v_range >> 24u) == 0u) { + if (((uint64_t)(io2_a_src - iop_a_src)) <= 0u) { + status = wuffs_base__make_status(wuffs_lzma__error__internal_error_inconsistent_i_o); + goto exit; + } + v_c8 = wuffs_base__peek_u8be__no_bounds_check(iop_a_src); + iop_a_src += 1u; + v_bits = (((uint32_t)(v_bits << 8u)) | ((uint32_t)(v_c8))); + v_range <<= 8u; + } + v_prob = ((uint32_t)(self->private_data.f_probs_longrep_len_mid[0u][0u])); + v_threshold = ((uint32_t)((v_range >> 11u) * v_prob)); + if (v_bits < v_threshold) { + v_range = v_threshold; + v_prob += (((uint32_t)(2048u - v_prob)) >> 5u); + self->private_data.f_probs_longrep_len_mid[0u][0u] = ((uint16_t)(v_prob)); + if ((v_range >> 24u) == 0u) { + if (((uint64_t)(io2_a_src - iop_a_src)) <= 0u) { + status = wuffs_base__make_status(wuffs_lzma__error__internal_error_inconsistent_i_o); + goto exit; + } + v_c8 = wuffs_base__peek_u8be__no_bounds_check(iop_a_src); + iop_a_src += 1u; + v_bits = (((uint32_t)(v_bits << 8u)) | ((uint32_t)(v_c8))); + v_range <<= 8u; + } + v_index_len = ((uint32_t)((v_pos & v_pb_mask))); + v_tree_node = 1u; + while (v_tree_node < 8u) { + v_prob = ((uint32_t)(self->private_data.f_probs_longrep_len_mid[v_index_len][v_tree_node])); + v_threshold = ((uint32_t)((v_range >> 11u) * v_prob)); + if (v_bits < v_threshold) { + v_range = v_threshold; + v_prob += (((uint32_t)(2048u - v_prob)) >> 5u); + self->private_data.f_probs_longrep_len_mid[v_index_len][v_tree_node] = ((uint16_t)(v_prob)); + v_tree_node = (v_tree_node << 1u); + } else { + v_bits -= v_threshold; + v_range -= v_threshold; + v_prob -= (v_prob >> 5u); + self->private_data.f_probs_longrep_len_mid[v_index_len][v_tree_node] = ((uint16_t)(v_prob)); + v_tree_node = ((v_tree_node << 1u) | 1u); + } + if ((v_range >> 24u) == 0u) { + if (((uint64_t)(io2_a_src - iop_a_src)) <= 0u) { + status = wuffs_base__make_status(wuffs_lzma__error__internal_error_inconsistent_i_o); + goto exit; + } + v_c8 = wuffs_base__peek_u8be__no_bounds_check(iop_a_src); + iop_a_src += 1u; + v_bits = (((uint32_t)(v_bits << 8u)) | ((uint32_t)(v_c8))); + v_range <<= 8u; + } + } + v_len = ((v_tree_node & 7u) + 10u); + v_state = ((uint32_t)(WUFFS_LZMA__STATE_TRANSITION_LONGREP[v_state])); + break; + } + v_bits -= v_threshold; + v_range -= v_threshold; + v_prob -= (v_prob >> 5u); + self->private_data.f_probs_longrep_len_mid[0u][0u] = ((uint16_t)(v_prob)); + if ((v_range >> 24u) == 0u) { + if (((uint64_t)(io2_a_src - iop_a_src)) <= 0u) { + status = wuffs_base__make_status(wuffs_lzma__error__internal_error_inconsistent_i_o); + goto exit; + } + v_c8 = wuffs_base__peek_u8be__no_bounds_check(iop_a_src); + iop_a_src += 1u; + v_bits = (((uint32_t)(v_bits << 8u)) | ((uint32_t)(v_c8))); + v_range <<= 8u; + } + v_tree_node = 1u; + while (v_tree_node < 256u) { + v_prob = ((uint32_t)(self->private_data.f_probs_longrep_len_high[0u][v_tree_node])); + v_threshold = ((uint32_t)((v_range >> 11u) * v_prob)); + if (v_bits < v_threshold) { + v_range = v_threshold; + v_prob += (((uint32_t)(2048u - v_prob)) >> 5u); + self->private_data.f_probs_longrep_len_high[0u][v_tree_node] = ((uint16_t)(v_prob)); + v_tree_node = (v_tree_node << 1u); + } else { + v_bits -= v_threshold; + v_range -= v_threshold; + v_prob -= (v_prob >> 5u); + self->private_data.f_probs_longrep_len_high[0u][v_tree_node] = ((uint16_t)(v_prob)); + v_tree_node = ((v_tree_node << 1u) | 1u); + } + if ((v_range >> 24u) == 0u) { + if (((uint64_t)(io2_a_src - iop_a_src)) <= 0u) { + status = wuffs_base__make_status(wuffs_lzma__error__internal_error_inconsistent_i_o); + goto exit; + } + v_c8 = wuffs_base__peek_u8be__no_bounds_check(iop_a_src); + iop_a_src += 1u; + v_bits = (((uint32_t)(v_bits << 8u)) | ((uint32_t)(v_c8))); + v_range <<= 8u; + } + } + v_len = ((v_tree_node & 255u) + 18u); + v_state = ((uint32_t)(WUFFS_LZMA__STATE_TRANSITION_LONGREP[v_state])); + } while (0); + } while (0); + v_dist = (v_rep0 + 1u); + if ((((uint64_t)(v_dist)) > v_pos) || (((uint64_t)(v_dist)) > ((uint64_t)(self->private_impl.f_dict_size)))) { + status = wuffs_base__make_status(wuffs_lzma__error__bad_distance); + goto exit; + } + v_pos += ((uint64_t)(v_len)); + if (((uint64_t)(v_dist)) > ((uint64_t)(iop_a_dst - io0_a_dst))) { + v_adj_dist = ((uint32_t)((((uint64_t)(v_dist)) - ((uint64_t)(iop_a_dst - io0_a_dst))))); + if (v_adj_dist > self->private_impl.f_dict_seen) { + status = wuffs_base__make_status(wuffs_lzma__error__bad_distance); + goto exit; + } + v_wb_index = ((uint64_t)(((uint64_t)(self->private_impl.f_dict_workbuf_index)) - ((uint64_t)(v_adj_dist)))); + while (v_wb_index >= 9223372036854775808u) { + v_wb_index += ((uint64_t)(self->private_impl.f_dict_size)); + } + if (v_wb_index >= ((uint64_t)(a_workbuf.len))) { + status = wuffs_base__make_status(wuffs_lzma__error__internal_error_inconsistent_dictionary_state); + goto exit; + } + if (v_len < v_adj_dist) { + wuffs_private_impl__io_writer__limited_copy_u32_from_slice( + &iop_a_dst, io2_a_dst,(v_len + 1u), wuffs_base__slice_u8__subslice_i(a_workbuf, v_wb_index)); + if ( ! (iop_a_dst > io1_a_dst)) { + status = wuffs_base__make_status(wuffs_lzma__error__internal_error_inconsistent_dictionary_state); + goto exit; + } + v_match_byte = ((uint32_t)(iop_a_dst[-1])); + iop_a_dst--; + if ( ! (iop_a_dst > io1_a_dst)) { + status = wuffs_base__make_status(wuffs_lzma__error__internal_error_inconsistent_dictionary_state); + goto exit; + } + v_prev_byte = iop_a_dst[-1]; + continue; + } else if (v_len == v_adj_dist) { + wuffs_private_impl__io_writer__limited_copy_u32_from_slice( + &iop_a_dst, io2_a_dst,v_len, wuffs_base__slice_u8__subslice_i(a_workbuf, v_wb_index)); + wuffs_private_impl__io_writer__limited_copy_u32_from_history( + &iop_a_dst, io0_a_dst, io2_a_dst, 1u, v_dist); + if ( ! (iop_a_dst > io1_a_dst)) { + status = wuffs_base__make_status(wuffs_lzma__error__internal_error_inconsistent_dictionary_state); + goto exit; + } + v_match_byte = ((uint32_t)(iop_a_dst[-1])); + iop_a_dst--; + if ( ! (iop_a_dst > io1_a_dst)) { + status = wuffs_base__make_status(wuffs_lzma__error__internal_error_inconsistent_dictionary_state); + goto exit; + } + v_prev_byte = iop_a_dst[-1]; + continue; + } + wuffs_private_impl__io_writer__limited_copy_u32_from_slice( + &iop_a_dst, io2_a_dst,v_adj_dist, wuffs_base__slice_u8__subslice_i(a_workbuf, v_wb_index)); + v_len -= v_adj_dist; + if ((((uint64_t)(v_len)) > ((uint64_t)(io2_a_dst - iop_a_dst))) || (((uint64_t)(v_dist)) > ((uint64_t)(iop_a_dst - io0_a_dst)))) { + status = wuffs_base__make_status(wuffs_lzma__error__internal_error_inconsistent_dictionary_state); + goto exit; + } + } + v_match_cusp = wuffs_private_impl__io_writer__limited_copy_u32_from_history_fast_return_cusp( + &iop_a_dst, io0_a_dst, io2_a_dst, v_len, v_dist); + v_match_byte = (v_match_cusp >> 8u); + v_prev_byte = ((uint8_t)(v_match_cusp)); + } + label__outer__break:; + self->private_impl.f_stashed_bytes[0u] = v_prev_byte; + self->private_impl.f_stashed_bytes[1u] = ((uint8_t)(v_match_byte)); + self->private_impl.f_stashed_bits = v_bits; + self->private_impl.f_stashed_range = v_range; + self->private_impl.f_stashed_state = v_state; + self->private_impl.f_stashed_rep0 = v_rep0; + self->private_impl.f_stashed_rep1 = v_rep1; + self->private_impl.f_stashed_rep2 = v_rep2; + self->private_impl.f_stashed_rep3 = v_rep3; + self->private_impl.f_stashed_pos = v_pos; + self->private_impl.f_stashed_pos_end = v_pos_end; + goto exit; + exit: + if (a_dst && a_dst->data.ptr) { + a_dst->meta.wi = ((size_t)(iop_a_dst - a_dst->data.ptr)); + } + if (a_src && a_src->data.ptr) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + + return status; +} + // -------- func lzma.decoder.decode_bitstream_slow WUFFS_BASE__GENERATED_C_CODE @@ -52129,7 +53005,11 @@ wuffs_lzma__decoder__decode_bitstream_slow( v_lc = self->private_impl.f_lc; v_lp_mask = ((((uint64_t)(1u)) << self->private_impl.f_lp) - 1u); v_pb_mask = ((((uint64_t)(1u)) << self->private_impl.f_pb) - 1u); - while (v_pos < v_pos_end) { + while ( ! (self->private_impl.p_decode_bitstream_slow != 0)) { + if (v_pos >= v_pos_end) { + self->private_impl.f_end_of_chunk = true; + break; + } v_index_ao00 = ((v_state << 4u) | ((uint32_t)((v_pos & v_pb_mask)))); v_prob = ((uint32_t)(self->private_data.f_probs_ao00[v_index_ao00])); v_threshold = ((uint32_t)((v_range >> 11u) * v_prob)); @@ -52586,6 +53466,7 @@ wuffs_lzma__decoder__decode_bitstream_slow( v_rep += v_dist_extra_bits; } if (v_rep >= 4294967295u) { + self->private_impl.f_end_of_chunk = true; goto label__outer__break; } v_rep3 = v_rep2; @@ -53883,14 +54764,30 @@ wuffs_lzma__decoder__decode_bitstream( wuffs_base__slice_u8 a_workbuf) { wuffs_base__status status = wuffs_base__make_status(NULL); + wuffs_base__status v_status = wuffs_base__make_status(NULL); + uint32_t coro_susp_point = self->private_impl.p_decode_bitstream; switch (coro_susp_point) { WUFFS_BASE__COROUTINE_SUSPENSION_POINT_0; - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(1); - status = wuffs_lzma__decoder__decode_bitstream_slow(self, a_dst, a_src, a_workbuf); - if (status.repr) { - goto suspend; + self->private_impl.f_end_of_chunk = false; + while (true) { + v_status = wuffs_lzma__decoder__decode_bitstream_fast(self, a_dst, a_src, a_workbuf); + if (wuffs_base__status__is_error(&v_status)) { + status = v_status; + goto exit; + } + if (self->private_impl.f_end_of_chunk) { + break; + } + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(1); + status = wuffs_lzma__decoder__decode_bitstream_slow(self, a_dst, a_src, a_workbuf); + if (status.repr) { + goto suspend; + } + if (self->private_impl.f_end_of_chunk) { + break; + } } goto ok; diff --git a/std/lzma/decode_bitstream_fast.wuffs b/std/lzma/decode_bitstream_fast.wuffs new file mode 100644 index 00000000..05053daf --- /dev/null +++ b/std/lzma/decode_bitstream_fast.wuffs @@ -0,0 +1,1070 @@ +// Copyright 2023 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. +// +// SPDX-License-Identifier: Apache-2.0 OR MIT + +// -------- + +// The AlgOveNN line numbers refer to the ./README.md file. +// +// Most LZMA C/C++ implementations use macros for the "decodeTheNextBym()" code +// fragment in that "Algorithm Overview" section, repeated multiple times in +// the decoding algorithm. Wuffs does not have macros (or inline functions), so +// its LZMA implementation looks relatively verbose and repetitive (even though +// it's equivalent to the C/C++ code, at the machine code level). + +pri func decoder.decode_bitstream_fast!(dst: base.io_writer, src: base.io_reader, workbuf: roslice base.u8) base.status { + var c8 : base.u8 + + var bits : base.u32 + var range : base.u32 + + var state : base.u32[..= 11] + + // repN is what the "Algorithm Overview" calls (mrud[N] - 1). The minimum + // distance is 1. + var rep0 : base.u32[..= 0xFFFF_FFFE] + var rep1 : base.u32[..= 0xFFFF_FFFE] + var rep2 : base.u32[..= 0xFFFF_FFFE] + var rep3 : base.u32[..= 0xFFFF_FFFE] + var reptmp : base.u32[..= 0xFFFF_FFFE] + var rep : base.u32 + + var pos : base.u64 + var pos_end : base.u64 + + var lc : base.u32[..= 8] + var lp_mask : base.u64[..= 15] + var pb_mask : base.u64[..= 15] + + var prob : base.u32 + var threshold : base.u32 + var tree_node : base.u32[..= 0x1FF] + var prev_byte : base.u8 + var match_byte : base.u32 + var match_cusp : base.u32[..= 0xFFFF] + var len_state : base.u32[..= 3] // Equals min(len-2, 3) when in a MATCH. + var slot : base.u32[..= 0x7F] + var len : base.u32[..= 273] + + // LANL stands for Literal After Non-Literal. These variables are used when + // decoding a LITERAL and state >= 7. + var lanl_offset : base.u32[..= 0x100] + var lanl_old_offset : base.u32[..= 0x100] + var lanl_index : base.u32[..= 0x2FF] + + var num_extra_bits : base.u32[..= 30] + var dist_extra_bits : base.u32 + var high_bit_was_on : base.u32 + var i : base.u32 + + var index_ao00 : base.u32[..= (12 << 4) - 1] + var index_ao41 : base.u32[..= (12 << 4) - 1] + var index_lit : base.u32[..= (1 << 4) - 1] + var index_len : base.u32[..= (1 << 4) - 1] + + var index_small_dist_base : base.u32 + var index_small_dist_extra : base.u32 + var index_small_dist : base.u32[..= 127] + + var index_large_dist : base.u32[..= 15] + + // dist is part of the Lempel-Ziv (length, distance) pair. When dist is at + // or below the args.dst history length then the LZ copy can be resolved + // just from args.dst (and adj_dist and wb_index are unused). Otherwise, if + // e.g. dist was 100 and args.dst.history_length() was 40 then adj_dist + // would be 60: the LZ copy starts 60 bytes before the end of the + // args.workbuf ringbuffer. wb_index is that start index, equal to + // ((this.dict_workbuf_index ~mod- adj_dist) modulo this.dict_size). + var dist : base.u32 + var adj_dist : base.u32 + var wb_index : base.u64 + + prev_byte = this.stashed_bytes[0] + match_byte = this.stashed_bytes[1] as base.u32 + bits = this.stashed_bits + range = this.stashed_range + state = this.stashed_state + rep0 = this.stashed_rep0 + rep1 = this.stashed_rep1 + rep2 = this.stashed_rep2 + rep3 = this.stashed_rep3 + pos = this.stashed_pos + pos_end = this.stashed_pos_end + + lc = this.lc + lp_mask = ((1 as base.u64) << this.lp) - 1 + pb_mask = ((1 as base.u64) << this.pb) - 1 + + // "outer" is the core loop discussed in README.md's "Algorithm Overview". + while.outer(args.dst.length() >= 282) and (args.src.length() >= 48) { + if pos >= pos_end { + this.end_of_chunk = true + break.outer + } + + // AlgOve00 if decodeTheNextBym() == 0 + index_ao00 = (state << 4) | ((pos & pb_mask) as base.u32) + prob = this.probs_ao00[index_ao00] as base.u32 + threshold = (range >> 11) ~mod* prob + if bits < threshold { + range = threshold + prob ~mod+= (2048 ~mod- prob) >> 5 + this.probs_ao00[index_ao00] = (prob & 0xFFFF) as base.u16 + if (range >> 24) == 0 { + c8 = args.src.peek_u8() + args.src.skip_u32_fast!(actual: 1, worst_case: 1) + bits = (bits ~mod<< 8) | (c8 as base.u32) + range ~mod<<= 8 + } + + // AlgOve01 // Decode a LITERAL. + // AlgOve02 literal = decodeLiteral() + + index_lit = 15 & ( + (((pos & lp_mask) as base.u32) << lc) | + ((prev_byte as base.u32) >> (8 - lc))) + + if state >= 7 { + // We are decoding an 8-bit LITERAL, and (state >= 7) means + // that we most recently decoded a NON-LITERAL. The + // this.probs_lit[index_lit] array has 0x300 elements: three + // banks of 0x100. + // + // For (state < 7), we only use the first bank (and we run the + // while.low_state loop instead). This is equivalent to running + // the while.high_state loop where lanl_offset starts at and + // stays at zero for the entire loop. match_byte is ignored. + // + // For (state >= 7), we start by using the two later banks, but + // as we are decoding those 8 bits (from MSB to LSB order), we + // drop down to the first bank (and stay there) if and when the + // decoded bit doesn't equal the corresponding bit (in the same + // MSB to LSB order) in the match_byte variable. + // + // lanl_offset stays at 0x100 until that "if and when" point + // (which typically happens; if all 8 bits were equal then the + // previous NON-LITERAL copy could often just have been + // longer), when lanl_offset then drops down to zero. We then + // effectively run the remaining (8 - I) iterations in the + // while.low_state loop, where I is the iteration count. + // + // Whether we use the 0x100 or 0x200 bank, out of the two later + // banks, depends on the match_byte's I'th bit (MSB to LSB). + // match_byte is conceptually a base.u8 (it is <= 0xFF outside + // of this code fragment) but the intermediate computation is + // simpler as a base.u32 (ignoring the high 23 bits). + lanl_offset = 0x100 + + tree_node = 1 + while.high_state tree_node < 0x100, + inv args.dst.length() >= 282, + { + match_byte ~mod<<= 1 + lanl_old_offset = lanl_offset + lanl_offset &= match_byte + lanl_index = lanl_offset + lanl_old_offset + tree_node + + prob = this.probs_lit[index_lit][lanl_index] as base.u32 + threshold = (range >> 11) ~mod* prob + if bits < threshold { + lanl_offset = (lanl_offset ^ lanl_old_offset) & 0x100 + range = threshold + prob ~mod+= (2048 ~mod- prob) >> 5 + this.probs_lit[index_lit][lanl_index] = (prob & 0xFFFF) as base.u16 + tree_node = (tree_node << 1) + } else { + bits ~mod-= threshold + range ~mod-= threshold + prob ~mod-= prob >> 5 + this.probs_lit[index_lit][lanl_index] = (prob & 0xFFFF) as base.u16 + tree_node = (tree_node << 1) | 1 + } + if (range >> 24) == 0 { + if args.src.length() <= 0 { + return "#internal error: inconsistent I/O" + } + c8 = args.src.peek_u8() + args.src.skip_u32_fast!(actual: 1, worst_case: 1) + bits = (bits ~mod<< 8) | (c8 as base.u32) + range ~mod<<= 8 + } + } endwhile.high_state + + } else { + tree_node = 1 + while.low_state tree_node < 0x100, + inv args.dst.length() >= 282, + { + prob = this.probs_lit[index_lit][tree_node] as base.u32 + threshold = (range >> 11) ~mod* prob + if bits < threshold { + range = threshold + prob ~mod+= (2048 ~mod- prob) >> 5 + this.probs_lit[index_lit][tree_node] = (prob & 0xFFFF) as base.u16 + tree_node = (tree_node << 1) + } else { + bits ~mod-= threshold + range ~mod-= threshold + prob ~mod-= prob >> 5 + this.probs_lit[index_lit][tree_node] = (prob & 0xFFFF) as base.u16 + tree_node = (tree_node << 1) | 1 + } + if (range >> 24) == 0 { + if args.src.length() <= 0 { + return "#internal error: inconsistent I/O" + } + c8 = args.src.peek_u8() + args.src.skip_u32_fast!(actual: 1, worst_case: 1) + bits = (bits ~mod<< 8) | (c8 as base.u32) + range ~mod<<= 8 + } + } endwhile.low_state + } + assert args.dst.length() >= 282 + + // AlgOve03 emitLiteral(literal) + // AlgOve04 continue + prev_byte = (tree_node & 0xFF) as base.u8 + args.dst.write_u8_fast!(a: prev_byte) + pos ~mod+= 1 + state = STATE_TRANSITION_LITERAL[state] as base.u32 + continue.outer + } + + // AlgOve00a if decodeTheNextBym() == 0 // If-false branch. + bits ~mod-= threshold + range ~mod-= threshold + prob ~mod-= prob >> 5 + this.probs_ao00[index_ao00] = (prob & 0xFFFF) as base.u16 + if (range >> 24) == 0 { + c8 = args.src.peek_u8() + args.src.skip_u32_fast!(actual: 1, worst_case: 1) + bits = (bits ~mod<< 8) | (c8 as base.u32) + range ~mod<<= 8 + } else { + assert args.src.length() >= 47 + } + + while.goto_do_the_lz_copy true, + pre args.src.length() >= 47, + inv args.dst.length() >= 282, + post len >= 1, + {{ + // AlgOve20 if decodeTheNextBym() == 0 + prob = this.probs_ao20[state] as base.u32 + threshold = (range >> 11) ~mod* prob + if bits < threshold { + range = threshold + prob ~mod+= (2048 ~mod- prob) >> 5 + this.probs_ao20[state] = (prob & 0xFFFF) as base.u16 + if (range >> 24) == 0 { + c8 = args.src.peek_u8() + args.src.skip_u32_fast!(actual: 1, worst_case: 1) + bits = (bits ~mod<< 8) | (c8 as base.u32) + range ~mod<<= 8 + } else { + assert args.src.length() >= 46 + } + + // AlgOve21 // Decode a MATCH. + + while.goto_have_len true, + pre args.src.length() >= 46, + inv args.dst.length() >= 282, + post len >= 1, + {{ + // AlgOve22 len = decodeLen() + + // AlgOve22.0 if decodeTheNextBym() == 0 + prob = this.probs_match_len_low[0][0] as base.u32 + threshold = (range >> 11) ~mod* prob + if bits < threshold { + range = threshold + prob ~mod+= (2048 ~mod- prob) >> 5 + this.probs_match_len_low[0][0] = (prob & 0xFFFF) as base.u16 + if (range >> 24) == 0 { + c8 = args.src.peek_u8() + args.src.skip_u32_fast!(actual: 1, worst_case: 1) + bits = (bits ~mod<< 8) | (c8 as base.u32) + range ~mod<<= 8 + } + + // AlgOve22.1 // Decode a low length. + // AlgOve22.2 len = decodeMultipleByms(3) + 2 + index_len = (pos & pb_mask) as base.u32 + tree_node = 1 + while tree_node < 0x08, + inv args.dst.length() >= 282, + { + prob = this.probs_match_len_low[index_len][tree_node] as base.u32 + threshold = (range >> 11) ~mod* prob + if bits < threshold { + range = threshold + prob ~mod+= (2048 ~mod- prob) >> 5 + this.probs_match_len_low[index_len][tree_node] = (prob & 0xFFFF) as base.u16 + tree_node = (tree_node << 1) + } else { + bits ~mod-= threshold + range ~mod-= threshold + prob ~mod-= prob >> 5 + this.probs_match_len_low[index_len][tree_node] = (prob & 0xFFFF) as base.u16 + tree_node = (tree_node << 1) | 1 + } + if (range >> 24) == 0 { + if args.src.length() <= 0 { + return "#internal error: inconsistent I/O" + } + c8 = args.src.peek_u8() + args.src.skip_u32_fast!(actual: 1, worst_case: 1) + bits = (bits ~mod<< 8) | (c8 as base.u32) + range ~mod<<= 8 + } + } endwhile + len_state = CLAMP_NO_MORE_THAN_3[tree_node & 0x07] as base.u32 + len = (tree_node & 0x07) + 2 + assert len >= 1 + break.goto_have_len // goto AlgOve23. + } + + // AlgOve22.0a if decodeTheNextBym() == 0 // If-false branch. + bits ~mod-= threshold + range ~mod-= threshold + prob ~mod-= prob >> 5 + this.probs_match_len_low[0][0] = (prob & 0xFFFF) as base.u16 + if (range >> 24) == 0 { + if args.src.length() <= 0 { + return "#internal error: inconsistent I/O" + } + c8 = args.src.peek_u8() + args.src.skip_u32_fast!(actual: 1, worst_case: 1) + bits = (bits ~mod<< 8) | (c8 as base.u32) + range ~mod<<= 8 + } + + // AlgOve22.3 if decodeTheNextBym() == 0 + prob = this.probs_match_len_mid[0][0] as base.u32 + threshold = (range >> 11) ~mod* prob + if bits < threshold { + range = threshold + prob ~mod+= (2048 ~mod- prob) >> 5 + this.probs_match_len_mid[0][0] = (prob & 0xFFFF) as base.u16 + if (range >> 24) == 0 { + if args.src.length() <= 0 { + return "#internal error: inconsistent I/O" + } + c8 = args.src.peek_u8() + args.src.skip_u32_fast!(actual: 1, worst_case: 1) + bits = (bits ~mod<< 8) | (c8 as base.u32) + range ~mod<<= 8 + } + + // AlgOve22.4 // Decode a middle length. + // AlgOve22.5 len = decodeMultipleByms(3) + 10 + index_len = (pos & pb_mask) as base.u32 + tree_node = 1 + while tree_node < 0x08, + inv args.dst.length() >= 282, + { + prob = this.probs_match_len_mid[index_len][tree_node] as base.u32 + threshold = (range >> 11) ~mod* prob + if bits < threshold { + range = threshold + prob ~mod+= (2048 ~mod- prob) >> 5 + this.probs_match_len_mid[index_len][tree_node] = (prob & 0xFFFF) as base.u16 + tree_node = (tree_node << 1) + } else { + bits ~mod-= threshold + range ~mod-= threshold + prob ~mod-= prob >> 5 + this.probs_match_len_mid[index_len][tree_node] = (prob & 0xFFFF) as base.u16 + tree_node = (tree_node << 1) | 1 + } + if (range >> 24) == 0 { + if args.src.length() <= 0 { + return "#internal error: inconsistent I/O" + } + c8 = args.src.peek_u8() + args.src.skip_u32_fast!(actual: 1, worst_case: 1) + bits = (bits ~mod<< 8) | (c8 as base.u32) + range ~mod<<= 8 + } + } endwhile + len = (tree_node & 0x07) + 10 + assert len >= 1 + len_state = 3 + break.goto_have_len // goto AlgOve23. + } + + // AlgOve22.3a if decodeTheNextBym() == 0 // If-false branch. + bits ~mod-= threshold + range ~mod-= threshold + prob ~mod-= prob >> 5 + this.probs_match_len_mid[0][0] = (prob & 0xFFFF) as base.u16 + if (range >> 24) == 0 { + if args.src.length() <= 0 { + return "#internal error: inconsistent I/O" + } + c8 = args.src.peek_u8() + args.src.skip_u32_fast!(actual: 1, worst_case: 1) + bits = (bits ~mod<< 8) | (c8 as base.u32) + range ~mod<<= 8 + } + + // AlgOve22.7 // Decode a high length. + // AlgOve22.8 len = decodeMultipleByms(8) + 18 + tree_node = 1 + while tree_node < 0x100, + inv args.dst.length() >= 282, + { + prob = this.probs_match_len_high[0][tree_node] as base.u32 + threshold = (range >> 11) ~mod* prob + if bits < threshold { + range = threshold + prob ~mod+= (2048 ~mod- prob) >> 5 + this.probs_match_len_high[0][tree_node] = (prob & 0xFFFF) as base.u16 + tree_node = (tree_node << 1) + } else { + bits ~mod-= threshold + range ~mod-= threshold + prob ~mod-= prob >> 5 + this.probs_match_len_high[0][tree_node] = (prob & 0xFFFF) as base.u16 + tree_node = (tree_node << 1) | 1 + } + if (range >> 24) == 0 { + if args.src.length() <= 0 { + return "#internal error: inconsistent I/O" + } + c8 = args.src.peek_u8() + args.src.skip_u32_fast!(actual: 1, worst_case: 1) + bits = (bits ~mod<< 8) | (c8 as base.u32) + range ~mod<<= 8 + } + } endwhile + len = (tree_node & 0xFF) + 18 + assert len >= 1 + len_state = 3 + break.goto_have_len + }} endwhile.goto_have_len + + // AlgOve23 slot = decodeSlot(min(len-2, 3)) + slot = 1 + while slot < 0x40, + inv args.dst.length() >= 282, + inv len >= 1, + { + prob = this.probs_slot[len_state][slot] as base.u32 + threshold = (range >> 11) ~mod* prob + if bits < threshold { + range = threshold + prob ~mod+= (2048 ~mod- prob) >> 5 + this.probs_slot[len_state][slot] = (prob & 0xFFFF) as base.u16 + slot = (slot << 1) + } else { + bits ~mod-= threshold + range ~mod-= threshold + prob ~mod-= prob >> 5 + this.probs_slot[len_state][slot] = (prob & 0xFFFF) as base.u16 + slot = (slot << 1) | 1 + } + if (range >> 24) == 0 { + if args.src.length() <= 0 { + return "#internal error: inconsistent I/O" + } + c8 = args.src.peek_u8() + args.src.skip_u32_fast!(actual: 1, worst_case: 1) + bits = (bits ~mod<< 8) | (c8 as base.u32) + range ~mod<<= 8 + } + } endwhile + slot &= 0x3F + + // AlgOve24 distBiasedBy1 = decodeDistBiasedBy1(slot) + rep = slot + if slot < 4 { + // No-op. + + } else if slot < 14 { + num_extra_bits = (slot >> 1) - 1 + rep = (2 | (slot & 1)) << num_extra_bits + + // Read num_extra_bits "xxxx" bits. + index_small_dist_base = rep ~mod- slot + index_small_dist_extra = 1 + dist_extra_bits = 0 + i = 0 + while i < num_extra_bits, + inv args.dst.length() >= 282, + inv len >= 1, + { + assert i < 30 via "a < b: a < c; c <= b"(c: num_extra_bits) + index_small_dist = (index_small_dist_base ~mod+ index_small_dist_extra) & 127 + prob = this.probs_small_dist[index_small_dist] as base.u32 + threshold = (range >> 11) ~mod* prob + if bits < threshold { + range = threshold + prob ~mod+= (2048 ~mod- prob) >> 5 + this.probs_small_dist[index_small_dist] = (prob & 0xFFFF) as base.u16 + index_small_dist_extra = (index_small_dist_extra ~mod<< 1) + i += 1 + } else { + bits ~mod-= threshold + range ~mod-= threshold + prob ~mod-= prob >> 5 + this.probs_small_dist[index_small_dist] = (prob & 0xFFFF) as base.u16 + index_small_dist_extra = (index_small_dist_extra ~mod<< 1) | 1 + dist_extra_bits |= (1 as base.u32) << i + i += 1 + } + if (range >> 24) == 0 { + if args.src.length() <= 0 { + return "#internal error: inconsistent I/O" + } + c8 = args.src.peek_u8() + args.src.skip_u32_fast!(actual: 1, worst_case: 1) + bits = (bits ~mod<< 8) | (c8 as base.u32) + range ~mod<<= 8 + } + } endwhile + rep ~mod+= dist_extra_bits + + } else { + num_extra_bits = (slot >> 1) - 1 + assert num_extra_bits >= 6 + assert num_extra_bits <= 30 + rep = (2 | (slot & 1)) << num_extra_bits + + // Read (num_extra_bits - 4) "yyyy" bits. + dist_extra_bits = 0 + while true, + pre num_extra_bits > 4, + inv args.dst.length() >= 282, + inv len >= 1, + post num_extra_bits > 0, + post num_extra_bits <= 4, + { + range >>= 1 + bits ~mod-= range + high_bit_was_on = 0 ~mod- (bits >> 31) + bits ~mod+= range & high_bit_was_on + dist_extra_bits = (dist_extra_bits ~mod<< 1) | ((high_bit_was_on ~mod+ 1) & 1) + if (range >> 24) == 0 { + if args.src.length() <= 0 { + return "#internal error: inconsistent I/O" + } + c8 = args.src.peek_u8() + args.src.skip_u32_fast!(actual: 1, worst_case: 1) + bits = (bits ~mod<< 8) | (c8 as base.u32) + range ~mod<<= 8 + } + + num_extra_bits -= 1 + if num_extra_bits <= 4 { + break + } + } endwhile + dist_extra_bits ~mod<<= 4 + + // Read 4 "zzzz" bits. + index_large_dist = 1 + while true, + pre num_extra_bits > 0, + pre num_extra_bits <= 4, + inv args.dst.length() >= 282, + inv len >= 1, + { + prob = this.probs_large_dist[index_large_dist] as base.u32 + threshold = (range >> 11) ~mod* prob + if bits < threshold { + range = threshold + prob ~mod+= (2048 ~mod- prob) >> 5 + this.probs_large_dist[index_large_dist] = (prob & 0xFFFF) as base.u16 + index_large_dist = 15 & ((index_large_dist ~mod<< 1)) + } else { + bits ~mod-= threshold + range ~mod-= threshold + prob ~mod-= prob >> 5 + this.probs_large_dist[index_large_dist] = (prob & 0xFFFF) as base.u16 + index_large_dist = 15 & ((index_large_dist ~mod<< 1) | 1) + dist_extra_bits |= (1 as base.u32) << (4 - num_extra_bits) + } + if (range >> 24) == 0 { + if args.src.length() <= 0 { + return "#internal error: inconsistent I/O" + } + c8 = args.src.peek_u8() + args.src.skip_u32_fast!(actual: 1, worst_case: 1) + bits = (bits ~mod<< 8) | (c8 as base.u32) + range ~mod<<= 8 + } + + num_extra_bits -= 1 + if num_extra_bits <= 0 { + break + } + } endwhile + rep ~mod+= dist_extra_bits + } + + // AlgOve25 if distBiasedBy1 == 0xFFFF_FFFF + if rep >= 0xFFFF_FFFF { + // AlgOve26 break + this.end_of_chunk = true + break.outer + } + + // AlgOve28 mrud = (1 + distBiasedBy1, mrud[0], mrud[1], mrud[2]) + rep3 = rep2 + rep2 = rep1 + rep1 = rep0 + rep0 = rep + + assert len >= 1 + state = STATE_TRANSITION_MATCH[state] as base.u32 + break.goto_do_the_lz_copy // goto AlgOve90. + } + assert args.src.length() >= 47 + + // AlgOve20a if decodeTheNextBym() == 0 // If-false branch. + bits ~mod-= threshold + range ~mod-= threshold + prob ~mod-= prob >> 5 + this.probs_ao20[state] = (prob & 0xFFFF) as base.u16 + if (range >> 24) == 0 { + c8 = args.src.peek_u8() + args.src.skip_u32_fast!(actual: 1, worst_case: 1) + bits = (bits ~mod<< 8) | (c8 as base.u32) + range ~mod<<= 8 + } else { + assert args.src.length() >= 46 + } + + // AlgOve40 if decodeTheNextBym() == 0 + prob = this.probs_ao40[state] as base.u32 + threshold = (range >> 11) ~mod* prob + if bits < threshold { + range = threshold + prob ~mod+= (2048 ~mod- prob) >> 5 + this.probs_ao40[state] = (prob & 0xFFFF) as base.u16 + if (range >> 24) == 0 { + c8 = args.src.peek_u8() + args.src.skip_u32_fast!(actual: 1, worst_case: 1) + bits = (bits ~mod<< 8) | (c8 as base.u32) + range ~mod<<= 8 + } else { + assert args.src.length() >= 45 + } + + // AlgOve41 if decodeTheNextBym() == 0 + index_ao41 = (state << 4) | ((pos & pb_mask) as base.u32) + prob = this.probs_ao41[index_ao41] as base.u32 + threshold = (range >> 11) ~mod* prob + if bits < threshold { + range = threshold + prob ~mod+= (2048 ~mod- prob) >> 5 + this.probs_ao41[index_ao41] = (prob & 0xFFFF) as base.u16 + if (range >> 24) == 0 { + c8 = args.src.peek_u8() + args.src.skip_u32_fast!(actual: 1, worst_case: 1) + bits = (bits ~mod<< 8) | (c8 as base.u32) + range ~mod<<= 8 + } + + // AlgOve42 // Decode a SHORTREP. + // AlgOve43 len = 1 + len = 1 + + assert len >= 1 + state = STATE_TRANSITION_SHORTREP[state] as base.u32 + break.goto_do_the_lz_copy // goto AlgOve90. + } + + // AlgOve41a if decodeTheNextBym() == 0 // If-false branch. + bits ~mod-= threshold + range ~mod-= threshold + prob ~mod-= prob >> 5 + this.probs_ao41[index_ao41] = (prob & 0xFFFF) as base.u16 + if (range >> 24) == 0 { + c8 = args.src.peek_u8() + args.src.skip_u32_fast!(actual: 1, worst_case: 1) + bits = (bits ~mod<< 8) | (c8 as base.u32) + range ~mod<<= 8 + } + + // AlgOve46 // Decode a LONGREP[0]. + + } else { + // AlgOve40a if decodeTheNextBym() == 0 // If-false branch. + bits ~mod-= threshold + range ~mod-= threshold + prob ~mod-= prob >> 5 + this.probs_ao40[state] = (prob & 0xFFFF) as base.u16 + if (range >> 24) == 0 { + c8 = args.src.peek_u8() + args.src.skip_u32_fast!(actual: 1, worst_case: 1) + bits = (bits ~mod<< 8) | (c8 as base.u32) + range ~mod<<= 8 + } else { + assert args.src.length() >= 45 + } + + // AlgOve60 if decodeTheNextBym() == 0 + prob = this.probs_ao60[state] as base.u32 + threshold = (range >> 11) ~mod* prob + if bits < threshold { + range = threshold + prob ~mod+= (2048 ~mod- prob) >> 5 + this.probs_ao60[state] = (prob & 0xFFFF) as base.u16 + if (range >> 24) == 0 { + c8 = args.src.peek_u8() + args.src.skip_u32_fast!(actual: 1, worst_case: 1) + bits = (bits ~mod<< 8) | (c8 as base.u32) + range ~mod<<= 8 + } + + // AlgOve61 // Decode a LONGREP[1]. + // AlgOve62 mrud = (mrud[1], mrud[0], mrud[2], mrud[3]) + reptmp = rep1 + rep1 = rep0 + rep0 = reptmp + + } else { + // AlgOve60a if decodeTheNextBym() == 0 // If-false branch. + bits ~mod-= threshold + range ~mod-= threshold + prob ~mod-= prob >> 5 + this.probs_ao60[state] = (prob & 0xFFFF) as base.u16 + if (range >> 24) == 0 { + c8 = args.src.peek_u8() + args.src.skip_u32_fast!(actual: 1, worst_case: 1) + bits = (bits ~mod<< 8) | (c8 as base.u32) + range ~mod<<= 8 + } else { + assert args.src.length() >= 44 + } + + // AlgOve63 if decodeTheNextBym() == 0 + prob = this.probs_ao63[state] as base.u32 + threshold = (range >> 11) ~mod* prob + if bits < threshold { + range = threshold + prob ~mod+= (2048 ~mod- prob) >> 5 + this.probs_ao63[state] = (prob & 0xFFFF) as base.u16 + if (range >> 24) == 0 { + c8 = args.src.peek_u8() + args.src.skip_u32_fast!(actual: 1, worst_case: 1) + bits = (bits ~mod<< 8) | (c8 as base.u32) + range ~mod<<= 8 + } + + // AlgOve64 // Decode a LONGREP[2]. + // AlgOve65 mrud = (mrud[2], mrud[0], mrud[1], mrud[3]) + reptmp = rep2 + rep2 = rep1 + rep1 = rep0 + rep0 = reptmp + + } else { + // AlgOve63a if decodeTheNextBym() == 0 // If-false branch. + bits ~mod-= threshold + range ~mod-= threshold + prob ~mod-= prob >> 5 + this.probs_ao63[state] = (prob & 0xFFFF) as base.u16 + if (range >> 24) == 0 { + c8 = args.src.peek_u8() + args.src.skip_u32_fast!(actual: 1, worst_case: 1) + bits = (bits ~mod<< 8) | (c8 as base.u32) + range ~mod<<= 8 + } + + // AlgOve67 // Decode a LONGREP[3]. + // AlgOve68 mrud = (mrud[3], mrud[0], mrud[1], mrud[2]) + reptmp = rep3 + rep3 = rep2 + rep2 = rep1 + rep1 = rep0 + rep0 = reptmp + } + } + } + + // AlgOve80 len = decodeLen() + while.goto_after_decode_len true, + inv args.dst.length() >= 282, + post len >= 1, + {{ + // AlgOve80.0 if decodeTheNextBym() == 0 + prob = this.probs_longrep_len_low[0][0] as base.u32 + threshold = (range >> 11) ~mod* prob + if bits < threshold { + range = threshold + prob ~mod+= (2048 ~mod- prob) >> 5 + this.probs_longrep_len_low[0][0] = (prob & 0xFFFF) as base.u16 + if (range >> 24) == 0 { + if args.src.length() <= 0 { + return "#internal error: inconsistent I/O" + } + c8 = args.src.peek_u8() + args.src.skip_u32_fast!(actual: 1, worst_case: 1) + bits = (bits ~mod<< 8) | (c8 as base.u32) + range ~mod<<= 8 + } + + // AlgOve80.1 // Decode a low length. + // AlgOve80.2 len = decodeMultipleByms(3) + 2 + index_len = (pos & pb_mask) as base.u32 + tree_node = 1 + while tree_node < 0x08, + inv args.dst.length() >= 282, + { + prob = this.probs_longrep_len_low[index_len][tree_node] as base.u32 + threshold = (range >> 11) ~mod* prob + if bits < threshold { + range = threshold + prob ~mod+= (2048 ~mod- prob) >> 5 + this.probs_longrep_len_low[index_len][tree_node] = (prob & 0xFFFF) as base.u16 + tree_node = (tree_node << 1) + } else { + bits ~mod-= threshold + range ~mod-= threshold + prob ~mod-= prob >> 5 + this.probs_longrep_len_low[index_len][tree_node] = (prob & 0xFFFF) as base.u16 + tree_node = (tree_node << 1) | 1 + } + if (range >> 24) == 0 { + if args.src.length() <= 0 { + return "#internal error: inconsistent I/O" + } + c8 = args.src.peek_u8() + args.src.skip_u32_fast!(actual: 1, worst_case: 1) + bits = (bits ~mod<< 8) | (c8 as base.u32) + range ~mod<<= 8 + } + } endwhile + len = (tree_node & 0x07) + 2 + assert len >= 1 + state = STATE_TRANSITION_LONGREP[state] as base.u32 + break.goto_after_decode_len + } + + // AlgOve80.0a if decodeTheNextBym() == 0 // If-false branch. + bits ~mod-= threshold + range ~mod-= threshold + prob ~mod-= prob >> 5 + this.probs_longrep_len_low[0][0] = (prob & 0xFFFF) as base.u16 + if (range >> 24) == 0 { + if args.src.length() <= 0 { + return "#internal error: inconsistent I/O" + } + c8 = args.src.peek_u8() + args.src.skip_u32_fast!(actual: 1, worst_case: 1) + bits = (bits ~mod<< 8) | (c8 as base.u32) + range ~mod<<= 8 + } + + // AlgOve80.3 if decodeTheNextBym() == 0 + prob = this.probs_longrep_len_mid[0][0] as base.u32 + threshold = (range >> 11) ~mod* prob + if bits < threshold { + range = threshold + prob ~mod+= (2048 ~mod- prob) >> 5 + this.probs_longrep_len_mid[0][0] = (prob & 0xFFFF) as base.u16 + if (range >> 24) == 0 { + if args.src.length() <= 0 { + return "#internal error: inconsistent I/O" + } + c8 = args.src.peek_u8() + args.src.skip_u32_fast!(actual: 1, worst_case: 1) + bits = (bits ~mod<< 8) | (c8 as base.u32) + range ~mod<<= 8 + } + + // AlgOve80.4 // Decode a middle length. + // AlgOve80.5 len = decodeMultipleByms(3) + 10 + index_len = (pos & pb_mask) as base.u32 + tree_node = 1 + while tree_node < 0x08, + inv args.dst.length() >= 282, + { + prob = this.probs_longrep_len_mid[index_len][tree_node] as base.u32 + threshold = (range >> 11) ~mod* prob + if bits < threshold { + range = threshold + prob ~mod+= (2048 ~mod- prob) >> 5 + this.probs_longrep_len_mid[index_len][tree_node] = (prob & 0xFFFF) as base.u16 + tree_node = (tree_node << 1) + } else { + bits ~mod-= threshold + range ~mod-= threshold + prob ~mod-= prob >> 5 + this.probs_longrep_len_mid[index_len][tree_node] = (prob & 0xFFFF) as base.u16 + tree_node = (tree_node << 1) | 1 + } + if (range >> 24) == 0 { + if args.src.length() <= 0 { + return "#internal error: inconsistent I/O" + } + c8 = args.src.peek_u8() + args.src.skip_u32_fast!(actual: 1, worst_case: 1) + bits = (bits ~mod<< 8) | (c8 as base.u32) + range ~mod<<= 8 + } + } endwhile + len = (tree_node & 0x07) + 10 + assert len >= 1 + state = STATE_TRANSITION_LONGREP[state] as base.u32 + break.goto_after_decode_len + } + + // AlgOve80.3a if decodeTheNextBym() == 0 // If-false branch. + bits ~mod-= threshold + range ~mod-= threshold + prob ~mod-= prob >> 5 + this.probs_longrep_len_mid[0][0] = (prob & 0xFFFF) as base.u16 + if (range >> 24) == 0 { + if args.src.length() <= 0 { + return "#internal error: inconsistent I/O" + } + c8 = args.src.peek_u8() + args.src.skip_u32_fast!(actual: 1, worst_case: 1) + bits = (bits ~mod<< 8) | (c8 as base.u32) + range ~mod<<= 8 + } + + // AlgOve80.7 // Decode a high length. + // AlgOve80.8 len = decodeMultipleByms(8) + 18 + tree_node = 1 + while tree_node < 0x100, + inv args.dst.length() >= 282, + { + prob = this.probs_longrep_len_high[0][tree_node] as base.u32 + threshold = (range >> 11) ~mod* prob + if bits < threshold { + range = threshold + prob ~mod+= (2048 ~mod- prob) >> 5 + this.probs_longrep_len_high[0][tree_node] = (prob & 0xFFFF) as base.u16 + tree_node = (tree_node << 1) + } else { + bits ~mod-= threshold + range ~mod-= threshold + prob ~mod-= prob >> 5 + this.probs_longrep_len_high[0][tree_node] = (prob & 0xFFFF) as base.u16 + tree_node = (tree_node << 1) | 1 + } + if (range >> 24) == 0 { + if args.src.length() <= 0 { + return "#internal error: inconsistent I/O" + } + c8 = args.src.peek_u8() + args.src.skip_u32_fast!(actual: 1, worst_case: 1) + bits = (bits ~mod<< 8) | (c8 as base.u32) + range ~mod<<= 8 + } + } endwhile + len = (tree_node & 0xFF) + 18 + assert len >= 1 + state = STATE_TRANSITION_LONGREP[state] as base.u32 + + break.goto_after_decode_len + }} endwhile.goto_after_decode_len + break.goto_do_the_lz_copy + }} endwhile.goto_do_the_lz_copy + assert args.dst.length() >= 282 + + // AlgOve90 labelDoTheLZCopy: + // AlgOve94 emitCopy(len, mrud[0]) + + assert len >= 1 + dist = rep0 + 1 + assert dist >= 1 + if ((dist as base.u64) > pos) or + ((dist as base.u64) > (this.dict_size as base.u64)) { + return "#bad distance" + } + pos ~mod+= len as base.u64 + assert (len as base.u64) <= args.dst.length() via "a <= b: a <= c; c <= b"(c: 282) + + // Copy from the args.workbuf history, if necessary. + if (dist as base.u64) > args.dst.history_length() { + adj_dist = (((dist as base.u64) - args.dst.history_length()) & 0xFFFF_FFFF) as base.u32 + if adj_dist > this.dict_seen { + return "#bad distance" + } + wb_index = (this.dict_workbuf_index as base.u64) ~mod- (adj_dist as base.u64) + while wb_index >= 0x8000_0000_0000_0000, + inv dist >= 1, + { + wb_index ~mod+= this.dict_size as base.u64 + } endwhile + if wb_index >= args.workbuf.length() { + return "#internal error: inconsistent dictionary state" + } + + if len < adj_dist { + // Copy len bytes of history and one more byte for the later + // half of the two-byte cusp, all from args.workbuf. + args.dst.limited_copy_u32_from_slice!(up_to: len + 1, s: args.workbuf[wb_index ..]) + if not args.dst.can_undo_byte() { + return "#internal error: inconsistent dictionary state" + } + match_byte = args.dst.peek_undo_byte() as base.u32 + args.dst.undo_byte!() + if not args.dst.can_undo_byte() { + return "#internal error: inconsistent dictionary state" + } + prev_byte = args.dst.peek_undo_byte() + continue.outer + } else if len == adj_dist { + // Like above, copy len bytes of history and one more byte for + // the later half of the two-byte cusp, but the copy-source + // crosses over from args.workbuf to args.dst. + args.dst.limited_copy_u32_from_slice!(up_to: len, s: args.workbuf[wb_index ..]) + args.dst.limited_copy_u32_from_history!(up_to: 1, distance: dist) + if not args.dst.can_undo_byte() { + return "#internal error: inconsistent dictionary state" + } + match_byte = args.dst.peek_undo_byte() as base.u32 + args.dst.undo_byte!() + if not args.dst.can_undo_byte() { + return "#internal error: inconsistent dictionary state" + } + prev_byte = args.dst.peek_undo_byte() + continue.outer + } + args.dst.limited_copy_u32_from_slice!(up_to: adj_dist, s: args.workbuf[wb_index ..]) + len -= adj_dist + assert len >= 1 + if ((len as base.u64) > args.dst.length()) or + ((dist as base.u64) > args.dst.history_length()) { + return "#internal error: inconsistent dictionary state" + } + } + + // Copy from the args.dst history. + assert len >= 1 + assert dist >= 1 + assert (len as base.u64) <= args.dst.length() + assert (dist as base.u64) <= args.dst.history_length() + match_cusp = args.dst.limited_copy_u32_from_history_fast_return_cusp!( + up_to: len, distance: dist) + match_byte = match_cusp >> 8 + prev_byte = (match_cusp & 0xFF) as base.u8 + } endwhile.outer + + this.stashed_bytes[0] = prev_byte + this.stashed_bytes[1] = (match_byte & 0xFF) as base.u8 + this.stashed_bits = bits + this.stashed_range = range + this.stashed_state = state + this.stashed_rep0 = rep0 + this.stashed_rep1 = rep1 + this.stashed_rep2 = rep2 + this.stashed_rep3 = rep3 + this.stashed_pos = pos + this.stashed_pos_end = pos_end +} diff --git a/std/lzma/decode_bitstream_slow.wuffs b/std/lzma/decode_bitstream_slow.wuffs index 80252320..23300ccc 100644 --- a/std/lzma/decode_bitstream_slow.wuffs +++ b/std/lzma/decode_bitstream_slow.wuffs @@ -102,7 +102,12 @@ pri func decoder.decode_bitstream_slow?(dst: base.io_writer, src: base.io_reader pb_mask = ((1 as base.u64) << this.pb) - 1 // "outer" is the core loop discussed in README.md's "Algorithm Overview". - while.outer pos < pos_end { + while.outer not coroutine_resumed { + if pos >= pos_end { + this.end_of_chunk = true + break.outer + } + // AlgOve00 if decodeTheNextBym() == 0 index_ao00 = (state << 4) | ((pos & pb_mask) as base.u32) prob = this.probs_ao00[index_ao00] as base.u32 @@ -534,6 +539,7 @@ pri func decoder.decode_bitstream_slow?(dst: base.io_writer, src: base.io_reader // AlgOve25 if distBiasedBy1 == 0xFFFF_FFFF if rep >= 0xFFFF_FFFF { // AlgOve26 break + this.end_of_chunk = true break.outer } diff --git a/std/lzma/decode_lzma.wuffs b/std/lzma/decode_lzma.wuffs index 4b346842..8a3d5747 100644 --- a/std/lzma/decode_lzma.wuffs +++ b/std/lzma/decode_lzma.wuffs @@ -18,6 +18,7 @@ pub status "#truncated input" pub status "#unsupported decoded length" pub status "#unsupported properties" +pri status "#internal error: inconsistent I/O" pri status "#internal error: inconsistent dictionary state" pub const DECODER_DST_HISTORY_RETAIN_LENGTH_MAX_INCL_WORST_CASE : base.u64 = 0 @@ -62,6 +63,8 @@ pub struct decoder? implements base.io_transformer( allow_non_zero_initial_byte : base.bool, + end_of_chunk : base.bool, + stashed_bytes : array[2] base.u8, // 0 is prev_byte, 1 is match_byte. stashed_bits : base.u32, stashed_range : base.u32, @@ -558,7 +561,22 @@ pri func decoder.do_transform_io?(dst: base.io_writer, src: base.io_reader, work } pri func decoder.decode_bitstream?(dst: base.io_writer, src: base.io_reader, workbuf: roslice base.u8) { - this.decode_bitstream_slow?(dst: args.dst, src: args.src, workbuf: args.workbuf) + var status : base.status + + this.end_of_chunk = false + while true { + status = this.decode_bitstream_fast!(dst: args.dst, src: args.src, workbuf: args.workbuf) + if status.is_error() { + return status + } + if this.end_of_chunk { + break + } + this.decode_bitstream_slow?(dst: args.dst, src: args.src, workbuf: args.workbuf) + if this.end_of_chunk { + break + } + } endwhile } pri func decoder.update_stashed_bytes?(dst: base.io_writer, workbuf: roslice base.u8) {