diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 3fd992ac..c60c3ee6 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -9610,6 +9610,7 @@ struct wuffs_brotli__decoder__struct { bool f_implicit_zero_distance; uint32_t f_insert_length; uint32_t f_copy_length; + uint32_t f_cmd_state; bool f_quirk_large_window; bool f_large_window; bool f_started; @@ -9763,7 +9764,6 @@ struct wuffs_brotli__decoder__struct { } s_decode_block_switch; struct { uint32_t v_cmd_code; - uint32_t v_insert_length; uint32_t v_tree; uint32_t v_distance; uint32_t v_i; @@ -9774,6 +9774,7 @@ struct wuffs_brotli__decoder__struct { } s_process_commands; struct { uint32_t v_code; + uint32_t v_nbits; } s_read_distance; } private_data; @@ -77416,6 +77417,20 @@ wuffs_brotli__decoder__read_symbol( wuffs_base__io_buffer* a_src, uint32_t a_hoff); +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__decoder__read_bits_fast( + wuffs_brotli__decoder* self, + wuffs_base__io_buffer* a_src, + uint32_t a_n); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__decoder__read_symbol_fast( + wuffs_brotli__decoder* self, + wuffs_base__io_buffer* a_src, + uint32_t a_hoff); + WUFFS_BASE__GENERATED_C_CODE static bool wuffs_brotli__decoder__jump_to_byte_boundary( @@ -77695,6 +77710,13 @@ wuffs_brotli__decoder__process_commands( wuffs_base__slice_u8 a_workbuf, wuffs_base__slice_u8 a_dictbuf); +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__status +wuffs_brotli__decoder__process_commands_fast( + wuffs_brotli__decoder* self, + wuffs_base__io_buffer* a_src, + wuffs_base__slice_u8 a_workbuf); + WUFFS_BASE__GENERATED_C_CODE static wuffs_base__empty_struct wuffs_brotli__decoder__update_dist_htree_index( @@ -81730,6 +81752,132 @@ wuffs_brotli__decoder__read_symbol( return status; } +// -------- func brotli.decoder.read_bits_fast + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__decoder__read_bits_fast( + wuffs_brotli__decoder* self, + wuffs_base__io_buffer* a_src, + uint32_t a_n) { + uint32_t v_n = 0; + uint32_t v_bits = 0; + uint32_t v_n_bits = 0; + + const uint8_t* iop_a_src = NULL; + const uint8_t* io0_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + const uint8_t* io1_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + const uint8_t* io2_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + if (a_src && a_src->data.ptr) { + io0_a_src = a_src->data.ptr; + io1_a_src = io0_a_src + a_src->meta.ri; + iop_a_src = io1_a_src; + io2_a_src = io0_a_src + a_src->meta.wi; + } + + v_n = a_n; + v_bits = self->private_impl.f_bits; + v_n_bits = self->private_impl.f_n_bits; + while (v_n_bits < v_n) { + if (((uint64_t)(io2_a_src - iop_a_src)) <= 0u) { + self->private_impl.f_bits = v_bits; + self->private_impl.f_n_bits = v_n_bits; + if (a_src && a_src->data.ptr) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + return 1u; + } + v_bits |= (((uint32_t)(wuffs_base__peek_u8be__no_bounds_check(iop_a_src))) << v_n_bits); + iop_a_src += 1u; + v_n_bits += 8u; + } + self->private_impl.f_bits_value = (v_bits & ((((uint32_t)(1u)) << v_n) - 1u)); + self->private_impl.f_bits = (v_bits >> v_n); + self->private_impl.f_n_bits = (v_n_bits - v_n); + if (a_src && a_src->data.ptr) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + return 0u; +} + +// -------- func brotli.decoder.read_symbol_fast + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__decoder__read_symbol_fast( + wuffs_brotli__decoder* self, + wuffs_base__io_buffer* a_src, + uint32_t a_hoff) { + uint32_t v_hoff = 0; + uint32_t v_bits = 0; + uint32_t v_n_bits = 0; + uint32_t v_i = 0; + uint32_t v_e = 0; + uint32_t v_e_bits = 0; + + const uint8_t* iop_a_src = NULL; + const uint8_t* io0_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + const uint8_t* io1_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + const uint8_t* io2_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + if (a_src && a_src->data.ptr) { + io0_a_src = a_src->data.ptr; + io1_a_src = io0_a_src + a_src->meta.ri; + iop_a_src = io1_a_src; + io2_a_src = io0_a_src + a_src->meta.wi; + } + + v_hoff = a_hoff; + v_bits = self->private_impl.f_bits; + v_n_bits = self->private_impl.f_n_bits; + while (true) { + v_i = ((uint32_t)(v_hoff + (v_bits & 255u))); + if (v_i >= 678034u) { + break; + } + v_e = ((uint32_t)(self->private_data.f_huffs[v_i])); + v_e_bits = (v_e & 15u); + if (v_e_bits <= 8u) { + if (v_n_bits >= v_e_bits) { + self->private_impl.f_symbol = (v_e >> 4u); + self->private_impl.f_bits = (v_bits >> v_e_bits); + self->private_impl.f_n_bits = (v_n_bits - v_e_bits); + if (a_src && a_src->data.ptr) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + return 0u; + } + } else if (v_n_bits > 8u) { + v_i = ((uint32_t)(((uint32_t)(v_hoff + (v_e >> 4u))) + ((v_bits >> 8u) & ((((uint32_t)(1u)) << (v_e_bits - 8u)) - 1u)))); + if (v_i >= 678034u) { + break; + } + v_e = ((uint32_t)(self->private_data.f_huffs[v_i])); + v_e_bits = ((v_e & 15u) + 8u); + if (v_n_bits >= v_e_bits) { + self->private_impl.f_symbol = (v_e >> 4u); + self->private_impl.f_bits = (v_bits >> v_e_bits); + self->private_impl.f_n_bits = (v_n_bits - v_e_bits); + if (a_src && a_src->data.ptr) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + return 0u; + } + } + if ((v_n_bits >= 24u) || (((uint64_t)(io2_a_src - iop_a_src)) <= 0u)) { + break; + } + v_bits |= (((uint32_t)(wuffs_base__peek_u8be__no_bounds_check(iop_a_src))) << v_n_bits); + iop_a_src += 1u; + v_n_bits += 8u; + } + self->private_impl.f_bits = v_bits; + self->private_impl.f_n_bits = v_n_bits; + if (a_src && a_src->data.ptr) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + return 1u; +} + // -------- func brotli.decoder.jump_to_byte_boundary WUFFS_BASE__GENERATED_C_CODE @@ -84518,7 +84666,6 @@ wuffs_brotli__decoder__process_commands( wuffs_base__status status = wuffs_base__make_status(NULL); uint32_t v_cmd_code = 0; - uint32_t v_insert_length = 0; uint32_t v_p1 = 0; uint32_t v_p2 = 0; uint32_t v_mode_offset = 0; @@ -84534,11 +84681,31 @@ wuffs_brotli__decoder__process_commands( uint32_t v_address = 0; uint32_t v_chunk = 0; uint64_t v_j = 0; + uint32_t v_fr = 0; + uint64_t v_dst_i = 0; + uint64_t v_src_i = 0; + uint64_t v_k64 = 0; + uint64_t v_n64 = 0; + uint64_t v_dlo = 0; + uint64_t v_dhi = 0; + uint64_t v_slo = 0; + uint64_t v_shi = 0; + wuffs_base__status v_fs = wuffs_base__make_status(NULL); + + const uint8_t* iop_a_src = NULL; + const uint8_t* io0_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + const uint8_t* io1_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + const uint8_t* io2_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + if (a_src && a_src->data.ptr) { + io0_a_src = a_src->data.ptr; + io1_a_src = io0_a_src + a_src->meta.ri; + iop_a_src = io1_a_src; + io2_a_src = io0_a_src + a_src->meta.wi; + } uint32_t coro_susp_point = self->private_impl.p_process_commands; if (coro_susp_point) { v_cmd_code = self->private_data.s_process_commands.v_cmd_code; - v_insert_length = self->private_data.s_process_commands.v_insert_length; v_tree = self->private_data.s_process_commands.v_tree; v_distance = self->private_data.s_process_commands.v_distance; v_i = self->private_data.s_process_commands.v_i; @@ -84551,47 +84718,134 @@ wuffs_brotli__decoder__process_commands( WUFFS_BASE__COROUTINE_SUSPENSION_POINT_0; while (true) { - if (self->private_impl.f_block_length[1u] == 0u) { - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(1); - status = wuffs_brotli__decoder__decode_block_switch(self, a_src, 1u); - if (status.repr) { - goto suspend; + if ((self->private_impl.f_cmd_state == 0u) && (((uint64_t)(io2_a_src - iop_a_src)) >= 16u)) { + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + v_fs = wuffs_brotli__decoder__process_commands_fast(self, a_src, a_workbuf); + if (a_src) { + iop_a_src = a_src->data.ptr + a_src->meta.ri; + } + if ( ! wuffs_base__status__is_ok(&v_fs)) { + status = v_fs; + if (wuffs_base__status__is_error(&status)) { + goto exit; + } else if (wuffs_base__status__is_suspension(&status)) { + status = wuffs_base__make_status(wuffs_base__error__cannot_return_a_suspension); + goto exit; + } + goto ok; + } else if (self->private_impl.f_cmd_state == 4u) { + self->private_impl.f_cmd_state = 0u; + status = wuffs_base__make_status(NULL); + goto ok; } - self->private_impl.f_htree_command = (self->private_impl.f_block_type_rb[3u] & 255u); - continue; } - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(2); - status = wuffs_brotli__decoder__read_symbol(self, a_src, (161792u + (self->private_impl.f_htree_command * 1080u))); - if (status.repr) { - goto suspend; + if (self->private_impl.f_cmd_state == 0u) { + if (self->private_impl.f_block_length[1u] == 0u) { + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(1); + status = wuffs_brotli__decoder__decode_block_switch(self, a_src, 1u); + if (a_src) { + iop_a_src = a_src->data.ptr + a_src->meta.ri; + } + if (status.repr) { + goto suspend; + } + self->private_impl.f_htree_command = (self->private_impl.f_block_type_rb[3u] & 255u); + continue; + } + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + v_fr = wuffs_brotli__decoder__read_symbol_fast(self, a_src, (161792u + (self->private_impl.f_htree_command * 1080u))); + if (a_src) { + iop_a_src = a_src->data.ptr + a_src->meta.ri; + } + if (v_fr != 0u) { + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(2); + status = wuffs_brotli__decoder__read_symbol(self, a_src, (161792u + (self->private_impl.f_htree_command * 1080u))); + if (a_src) { + iop_a_src = a_src->data.ptr + a_src->meta.ri; + } + if (status.repr) { + goto suspend; + } + } + if (self->private_impl.f_symbol >= 704u) { + status = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); + goto exit; + } + v_cmd_code = wuffs_base__u32__min(self->private_impl.f_symbol, 703u); + self->private_impl.f_implicit_zero_distance = (WUFFS_BROTLI__CMD_IMPLICIT_ZERO_DISTANCE[v_cmd_code] != 0u); + self->private_impl.f_distance_context = ((uint32_t)(WUFFS_BROTLI__CMD_DISTANCE_CONTEXT[v_cmd_code])); + wuffs_brotli__decoder__update_dist_htree_index(self); + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + v_fr = wuffs_brotli__decoder__read_bits_fast(self, a_src, ((uint32_t)(WUFFS_BROTLI__CMD_INSERT_LEN_EXTRA_BITS[v_cmd_code]))); + if (a_src) { + iop_a_src = a_src->data.ptr + a_src->meta.ri; + } + if (v_fr != 0u) { + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(3); + status = wuffs_brotli__decoder__read_bits(self, a_src, ((uint32_t)(WUFFS_BROTLI__CMD_INSERT_LEN_EXTRA_BITS[v_cmd_code]))); + if (a_src) { + iop_a_src = a_src->data.ptr + a_src->meta.ri; + } + if (status.repr) { + goto suspend; + } + } + self->private_impl.f_insert_length = ((uint32_t)(WUFFS_BROTLI__CMD_INSERT_LEN_OFFSET[v_cmd_code] + self->private_impl.f_bits_value)); + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + v_fr = wuffs_brotli__decoder__read_bits_fast(self, a_src, ((uint32_t)(WUFFS_BROTLI__CMD_COPY_LEN_EXTRA_BITS[v_cmd_code]))); + if (a_src) { + iop_a_src = a_src->data.ptr + a_src->meta.ri; + } + if (v_fr != 0u) { + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(4); + status = wuffs_brotli__decoder__read_bits(self, a_src, ((uint32_t)(WUFFS_BROTLI__CMD_COPY_LEN_EXTRA_BITS[v_cmd_code]))); + if (a_src) { + iop_a_src = a_src->data.ptr + a_src->meta.ri; + } + if (status.repr) { + goto suspend; + } + } + self->private_impl.f_copy_length = ((uint32_t)(WUFFS_BROTLI__CMD_COPY_LEN_OFFSET[v_cmd_code] + self->private_impl.f_bits_value)); + self->private_impl.f_block_length[1u] -= 1u; + if (self->private_impl.f_insert_length > 0u) { + self->private_impl.f_mb_remaining -= self->private_impl.f_insert_length; + self->private_impl.f_cmd_state = 1u; + } else { + self->private_impl.f_cmd_state = 2u; + } } - if (self->private_impl.f_symbol >= 704u) { - status = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); - goto exit; - } - v_cmd_code = wuffs_base__u32__min(self->private_impl.f_symbol, 703u); - self->private_impl.f_implicit_zero_distance = (WUFFS_BROTLI__CMD_IMPLICIT_ZERO_DISTANCE[v_cmd_code] != 0u); - self->private_impl.f_distance_context = ((uint32_t)(WUFFS_BROTLI__CMD_DISTANCE_CONTEXT[v_cmd_code])); - wuffs_brotli__decoder__update_dist_htree_index(self); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(3); - status = wuffs_brotli__decoder__read_bits(self, a_src, ((uint32_t)(WUFFS_BROTLI__CMD_INSERT_LEN_EXTRA_BITS[v_cmd_code]))); - if (status.repr) { - goto suspend; - } - v_insert_length = ((uint32_t)(WUFFS_BROTLI__CMD_INSERT_LEN_OFFSET[v_cmd_code] + self->private_impl.f_bits_value)); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(4); - status = wuffs_brotli__decoder__read_bits(self, a_src, ((uint32_t)(WUFFS_BROTLI__CMD_COPY_LEN_EXTRA_BITS[v_cmd_code]))); - if (status.repr) { - goto suspend; - } - self->private_impl.f_copy_length = ((uint32_t)(WUFFS_BROTLI__CMD_COPY_LEN_OFFSET[v_cmd_code] + self->private_impl.f_bits_value)); - self->private_impl.f_block_length[1u] -= 1u; - if (v_insert_length > 0u) { - self->private_impl.f_mb_remaining -= v_insert_length; - while (v_insert_length > 0u) { + if (self->private_impl.f_cmd_state == 1u) { + while (self->private_impl.f_insert_length > 0u) { if (self->private_impl.f_block_length[0u] == 0u) { + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } WUFFS_BASE__COROUTINE_SUSPENSION_POINT(5); status = wuffs_brotli__decoder__decode_block_switch(self, a_src, 0u); + if (a_src) { + iop_a_src = a_src->data.ptr + a_src->meta.ri; + } if (status.repr) { goto suspend; } @@ -84603,14 +84857,29 @@ wuffs_brotli__decoder__process_commands( v_mode_offset = (self->private_impl.f_context_mode << 9u); v_ctx = ((uint32_t)(((uint8_t)(WUFFS_BROTLI__CONTEXT_LOOKUP[(v_mode_offset | v_p1)] | WUFFS_BROTLI__CONTEXT_LOOKUP[(v_mode_offset | 256u | v_p2)])))); v_tree = ((uint32_t)(self->private_data.f_context_maps[(self->private_impl.f_context_map_slice + v_ctx)])); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(6); - status = wuffs_brotli__decoder__read_symbol(self, a_src, (v_tree * 632u)); - if (status.repr) { - goto suspend; + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + v_fr = wuffs_brotli__decoder__read_symbol_fast(self, a_src, (v_tree * 632u)); + if (a_src) { + iop_a_src = a_src->data.ptr + a_src->meta.ri; + } + if (v_fr != 0u) { + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(6); + status = wuffs_brotli__decoder__read_symbol(self, a_src, (v_tree * 632u)); + if (a_src) { + iop_a_src = a_src->data.ptr + a_src->meta.ri; + } + if (status.repr) { + goto suspend; + } } wuffs_brotli__decoder__put_byte(self, a_workbuf, ((uint8_t)(self->private_impl.f_symbol))); self->private_impl.f_block_length[0u] -= 1u; - v_insert_length -= 1u; + self->private_impl.f_insert_length -= 1u; if (self->private_impl.f_pos == self->private_impl.f_rb_size) { WUFFS_BASE__COROUTINE_SUSPENSION_POINT(7); status = wuffs_brotli__decoder__ring_event(self); @@ -84619,30 +84888,47 @@ wuffs_brotli__decoder__process_commands( } } } + self->private_impl.f_cmd_state = 2u; if ((self->private_impl.f_mb_remaining == 0u) || (self->private_impl.f_mb_remaining >= 2147483648u)) { + self->private_impl.f_cmd_state = 0u; status = wuffs_base__make_status(NULL); goto ok; } } - if (self->private_impl.f_implicit_zero_distance) { - self->private_impl.f_distance_context = 1u; - self->private_impl.f_dist_rb_idx -= 1u; - self->private_impl.f_distance_code = self->private_impl.f_dist_rb[(self->private_impl.f_dist_rb_idx & 3u)]; - } else { - if (self->private_impl.f_block_length[2u] == 0u) { - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(8); - status = wuffs_brotli__decoder__decode_block_switch(self, a_src, 2u); + if (self->private_impl.f_cmd_state == 2u) { + if (self->private_impl.f_implicit_zero_distance) { + self->private_impl.f_distance_context = 1u; + self->private_impl.f_dist_rb_idx -= 1u; + self->private_impl.f_distance_code = self->private_impl.f_dist_rb[(self->private_impl.f_dist_rb_idx & 3u)]; + } else { + if (self->private_impl.f_block_length[2u] == 0u) { + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(8); + status = wuffs_brotli__decoder__decode_block_switch(self, a_src, 2u); + if (a_src) { + iop_a_src = a_src->data.ptr + a_src->meta.ri; + } + if (status.repr) { + goto suspend; + } + self->private_impl.f_dist_context_map_slice = ((self->private_impl.f_block_type_rb[5u] & 255u) << 2u); + wuffs_brotli__decoder__update_dist_htree_index(self); + } + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(9); + status = wuffs_brotli__decoder__read_distance(self, a_src); + if (a_src) { + iop_a_src = a_src->data.ptr + a_src->meta.ri; + } if (status.repr) { goto suspend; } - self->private_impl.f_dist_context_map_slice = ((self->private_impl.f_block_type_rb[5u] & 255u) << 2u); - wuffs_brotli__decoder__update_dist_htree_index(self); - } - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(9); - status = wuffs_brotli__decoder__read_distance(self, a_src); - if (status.repr) { - goto suspend; } + self->private_impl.f_cmd_state = 3u; } if (self->private_impl.f_max_distance != self->private_impl.f_max_backward_distance) { self->private_impl.f_max_distance = wuffs_base__u32__min(self->private_impl.f_pos, self->private_impl.f_max_backward_distance); @@ -84732,6 +85018,33 @@ wuffs_brotli__decoder__process_commands( self->private_impl.f_dist_rb[(self->private_impl.f_dist_rb_idx & 3u)] = v_distance; self->private_impl.f_dist_rb_idx += 1u; self->private_impl.f_mb_remaining -= v_i; + v_dst_i = ((uint64_t)(self->private_impl.f_pos)); + v_src_i = ((uint64_t)((((uint32_t)(self->private_impl.f_pos - v_distance)) & ((uint32_t)(self->private_impl.f_window_size - 1u))))); + if (((v_dst_i + ((uint64_t)(v_i))) < ((uint64_t)(self->private_impl.f_rb_size))) && + ((v_src_i + ((uint64_t)(v_i))) <= ((uint64_t)(self->private_impl.f_window_size))) && + (((uint64_t)(self->private_impl.f_window_size)) <= ((uint64_t)(a_workbuf.len))) && + (v_distance > 0u)) { + v_k64 = 0u; + while (v_k64 < ((uint64_t)(v_i))) { + v_n64 = ((uint64_t)(((uint64_t)(v_i)) - v_k64)); + v_n64 = wuffs_base__u64__min(v_n64, ((uint64_t)(v_distance))); + v_dlo = ((uint64_t)(v_dst_i + v_k64)); + v_dhi = ((uint64_t)(v_dlo + v_n64)); + v_slo = ((uint64_t)(v_src_i + v_k64)); + v_shi = ((uint64_t)(v_slo + v_n64)); + if ((v_n64 == 0u) || + (v_dlo > v_dhi) || + (v_dhi > ((uint64_t)(a_workbuf.len))) || + (v_slo > v_shi) || + (v_shi > ((uint64_t)(a_workbuf.len)))) { + break; + } + wuffs_private_impl__slice_u8__copy_from_slice(wuffs_base__slice_u8__subslice_ij(a_workbuf, v_dlo, v_dhi), wuffs_base__slice_u8__subslice_ij(a_workbuf, v_slo, v_shi)); + v_k64 += v_n64; + } + self->private_impl.f_pos += ((uint32_t)(v_k64)); + v_i -= ((uint32_t)(v_k64)); + } while (v_i > 0u) { v_b = wuffs_brotli__decoder__get_byte(self, a_workbuf, v_distance); wuffs_brotli__decoder__put_byte(self, a_workbuf, v_b); @@ -84745,6 +85058,7 @@ wuffs_brotli__decoder__process_commands( } } } + self->private_impl.f_cmd_state = 0u; if ((self->private_impl.f_mb_remaining == 0u) || (self->private_impl.f_mb_remaining >= 2147483648u)) { status = wuffs_base__make_status(NULL); goto ok; @@ -84760,7 +85074,6 @@ wuffs_brotli__decoder__process_commands( suspend: self->private_impl.p_process_commands = wuffs_base__status__is_suspension(&status) ? coro_susp_point : 0; self->private_data.s_process_commands.v_cmd_code = v_cmd_code; - self->private_data.s_process_commands.v_insert_length = v_insert_length; self->private_data.s_process_commands.v_tree = v_tree; self->private_data.s_process_commands.v_distance = v_distance; self->private_data.s_process_commands.v_i = v_i; @@ -84771,6 +85084,330 @@ wuffs_brotli__decoder__process_commands( goto exit; exit: + if (a_src && a_src->data.ptr) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + + return status; +} + +// -------- func brotli.decoder.process_commands_fast + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__status +wuffs_brotli__decoder__process_commands_fast( + wuffs_brotli__decoder* self, + wuffs_base__io_buffer* a_src, + wuffs_base__slice_u8 a_workbuf) { + wuffs_base__status status = wuffs_base__make_status(NULL); + + wuffs_base__status v_ret = wuffs_base__make_status(NULL); + uint64_t v_bits = 0; + uint32_t v_n_bits = 0; + uint32_t v_i = 0; + uint32_t v_e = 0; + uint32_t v_e_bits = 0; + uint32_t v_cmd_code = 0; + uint32_t v_nb = 0; + uint32_t v_p1 = 0; + uint32_t v_p2 = 0; + uint32_t v_mode_offset = 0; + uint32_t v_ctx = 0; + uint32_t v_tree = 0; + uint32_t v_code = 0; + uint32_t v_nbits = 0; + uint32_t v_lo = 0; + uint32_t v_max_distance = 0; + uint32_t v_distance = 0; + uint32_t v_n = 0; + uint64_t v_dst_i = 0; + uint64_t v_src_i = 0; + uint64_t v_k64 = 0; + uint64_t v_n64 = 0; + uint64_t v_dlo = 0; + uint64_t v_dhi = 0; + uint64_t v_slo = 0; + uint64_t v_shi = 0; + + const uint8_t* iop_a_src = NULL; + const uint8_t* io0_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + const uint8_t* io1_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + const uint8_t* io2_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + if (a_src && a_src->data.ptr) { + io0_a_src = a_src->data.ptr; + io1_a_src = io0_a_src + a_src->meta.ri; + iop_a_src = io1_a_src; + io2_a_src = io0_a_src + a_src->meta.wi; + } + + if (self->private_impl.f_n_bits >= 8u) { + status = wuffs_base__make_status(NULL); + goto ok; + } + v_n_bits = (self->private_impl.f_n_bits & 7u); + v_bits = (((uint64_t)(self->private_impl.f_bits)) & ((((uint64_t)(1u)) << v_n_bits) - 1u)); + while (true) { + if (self->private_impl.f_cmd_state == 0u) { + if ((self->private_impl.f_block_length[1u] == 0u) || (((uint64_t)(io2_a_src - iop_a_src)) < 16u)) { + break; + } + v_bits |= ((uint64_t)(wuffs_base__peek_u64le__no_bounds_check(iop_a_src) << v_n_bits)); + iop_a_src += ((63u - v_n_bits) >> 3u); + v_n_bits |= 56u; + v_i = ((uint32_t)((161792u + (self->private_impl.f_htree_command * 1080u)) + ((uint32_t)((v_bits & 255u))))); + if (v_i >= 678034u) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); + break; + } + v_e = ((uint32_t)(self->private_data.f_huffs[v_i])); + v_e_bits = (v_e & 15u); + if (v_e_bits > 8u) { + v_i = ((uint32_t)(((uint32_t)((161792u + (self->private_impl.f_htree_command * 1080u)) + (v_e >> 4u))) + (((uint32_t)(((v_bits >> 8u) & 32767u))) & ((((uint32_t)(1u)) << (v_e_bits - 8u)) - 1u)))); + if (v_i >= 678034u) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); + break; + } + v_e = ((uint32_t)(self->private_data.f_huffs[v_i])); + v_e_bits = ((v_e & 15u) + 8u); + } + if (v_n_bits < v_e_bits) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_bit_reader); + break; + } + v_bits >>= v_e_bits; + v_n_bits -= v_e_bits; + if ((v_e >> 4u) >= 704u) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); + break; + } + self->private_impl.f_symbol = (v_e >> 4u); + v_cmd_code = wuffs_base__u32__min(self->private_impl.f_symbol, 703u); + self->private_impl.f_implicit_zero_distance = (WUFFS_BROTLI__CMD_IMPLICIT_ZERO_DISTANCE[v_cmd_code] != 0u); + self->private_impl.f_distance_context = ((uint32_t)(WUFFS_BROTLI__CMD_DISTANCE_CONTEXT[v_cmd_code])); + wuffs_brotli__decoder__update_dist_htree_index(self); + v_nb = ((uint32_t)(WUFFS_BROTLI__CMD_INSERT_LEN_EXTRA_BITS[v_cmd_code])); + if (v_n_bits < v_nb) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_bit_reader); + break; + } + self->private_impl.f_bits_value = ((uint32_t)((v_bits & ((((uint64_t)(1u)) << v_nb) - 1u)))); + v_bits >>= v_nb; + v_n_bits -= v_nb; + self->private_impl.f_insert_length = ((uint32_t)(WUFFS_BROTLI__CMD_INSERT_LEN_OFFSET[v_cmd_code] + self->private_impl.f_bits_value)); + if (((uint64_t)(io2_a_src - iop_a_src)) < 8u) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_bit_reader); + break; + } + v_bits |= ((uint64_t)(wuffs_base__peek_u64le__no_bounds_check(iop_a_src) << v_n_bits)); + iop_a_src += ((63u - v_n_bits) >> 3u); + v_n_bits |= 56u; + v_nb = ((uint32_t)(WUFFS_BROTLI__CMD_COPY_LEN_EXTRA_BITS[v_cmd_code])); + if (v_n_bits < v_nb) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_bit_reader); + break; + } + self->private_impl.f_bits_value = ((uint32_t)((v_bits & ((((uint64_t)(1u)) << v_nb) - 1u)))); + v_bits >>= v_nb; + v_n_bits -= v_nb; + self->private_impl.f_copy_length = ((uint32_t)(WUFFS_BROTLI__CMD_COPY_LEN_OFFSET[v_cmd_code] + self->private_impl.f_bits_value)); + self->private_impl.f_block_length[1u] -= 1u; + if (self->private_impl.f_insert_length > 0u) { + self->private_impl.f_mb_remaining -= self->private_impl.f_insert_length; + self->private_impl.f_cmd_state = 1u; + } else { + self->private_impl.f_cmd_state = 2u; + } + } + if (self->private_impl.f_cmd_state == 1u) { + if (((uint64_t)(self->private_impl.f_window_size)) > ((uint64_t)(a_workbuf.len))) { + break; + } + v_p1 = ((uint32_t)(wuffs_brotli__decoder__get_byte(self, a_workbuf, 1u))); + v_p2 = ((uint32_t)(wuffs_brotli__decoder__get_byte(self, a_workbuf, 2u))); + while (self->private_impl.f_insert_length > 0u) { + if ((self->private_impl.f_block_length[0u] == 0u) || (((uint32_t)(self->private_impl.f_pos + 1u)) >= self->private_impl.f_rb_size)) { + goto label__outer__break; + } + if (v_n_bits < 24u) { + if (((uint64_t)(io2_a_src - iop_a_src)) < 8u) { + goto label__outer__break; + } + v_bits |= ((uint64_t)(wuffs_base__peek_u64le__no_bounds_check(iop_a_src) << v_n_bits)); + iop_a_src += ((63u - v_n_bits) >> 3u); + v_n_bits |= 56u; + } + v_mode_offset = (self->private_impl.f_context_mode << 9u); + v_ctx = ((uint32_t)(((uint8_t)(WUFFS_BROTLI__CONTEXT_LOOKUP[(v_mode_offset | v_p1)] | WUFFS_BROTLI__CONTEXT_LOOKUP[(v_mode_offset | 256u | v_p2)])))); + v_tree = ((uint32_t)(self->private_data.f_context_maps[(self->private_impl.f_context_map_slice + v_ctx)])); + v_i = ((uint32_t)((v_tree * 632u) + ((uint32_t)((v_bits & 255u))))); + if (v_i >= 678034u) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); + goto label__outer__break; + } + v_e = ((uint32_t)(self->private_data.f_huffs[v_i])); + v_e_bits = (v_e & 15u); + if (v_e_bits > 8u) { + v_i = ((uint32_t)(((uint32_t)((v_tree * 632u) + (v_e >> 4u))) + (((uint32_t)(((v_bits >> 8u) & 32767u))) & ((((uint32_t)(1u)) << (v_e_bits - 8u)) - 1u)))); + if (v_i >= 678034u) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); + goto label__outer__break; + } + v_e = ((uint32_t)(self->private_data.f_huffs[v_i])); + v_e_bits = ((v_e & 15u) + 8u); + } + if (v_n_bits < v_e_bits) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_bit_reader); + goto label__outer__break; + } + v_bits >>= v_e_bits; + v_n_bits -= v_e_bits; + self->private_impl.f_symbol = (v_e >> 4u); + wuffs_brotli__decoder__put_byte(self, a_workbuf, ((uint8_t)(self->private_impl.f_symbol))); + v_p2 = v_p1; + v_p1 = (self->private_impl.f_symbol & 255u); + self->private_impl.f_block_length[0u] -= 1u; + self->private_impl.f_insert_length -= 1u; + } + self->private_impl.f_cmd_state = 2u; + if ((self->private_impl.f_mb_remaining == 0u) || (self->private_impl.f_mb_remaining >= 2147483648u)) { + self->private_impl.f_cmd_state = 4u; + break; + } + } + if (self->private_impl.f_cmd_state == 2u) { + if (self->private_impl.f_implicit_zero_distance) { + self->private_impl.f_distance_context = 1u; + self->private_impl.f_dist_rb_idx -= 1u; + self->private_impl.f_distance_code = self->private_impl.f_dist_rb[(self->private_impl.f_dist_rb_idx & 3u)]; + } else { + if ((self->private_impl.f_block_length[2u] == 0u) || (((uint64_t)(io2_a_src - iop_a_src)) < 8u)) { + break; + } + v_bits |= ((uint64_t)(wuffs_base__peek_u64le__no_bounds_check(iop_a_src) << v_n_bits)); + iop_a_src += ((63u - v_n_bits) >> 3u); + v_n_bits |= 56u; + v_i = ((uint32_t)((438272u + (self->private_impl.f_dist_htree_index * 920u)) + ((uint32_t)((v_bits & 255u))))); + if (v_i >= 678034u) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); + break; + } + v_e = ((uint32_t)(self->private_data.f_huffs[v_i])); + v_e_bits = (v_e & 15u); + if (v_e_bits > 8u) { + v_i = ((uint32_t)(((uint32_t)((438272u + (self->private_impl.f_dist_htree_index * 920u)) + (v_e >> 4u))) + (((uint32_t)(((v_bits >> 8u) & 32767u))) & ((((uint32_t)(1u)) << (v_e_bits - 8u)) - 1u)))); + if (v_i >= 678034u) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); + break; + } + v_e = ((uint32_t)(self->private_data.f_huffs[v_i])); + v_e_bits = ((v_e & 15u) + 8u); + } + if (v_n_bits < v_e_bits) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_bit_reader); + break; + } + v_bits >>= v_e_bits; + v_n_bits -= v_e_bits; + self->private_impl.f_symbol = (v_e >> 4u); + v_code = self->private_impl.f_symbol; + self->private_impl.f_block_length[2u] -= 1u; + self->private_impl.f_distance_context = 0u; + if (v_code < 16u) { + wuffs_brotli__decoder__take_distance_from_ring_buffer(self, v_code); + } else { + if ((v_code >= self->private_impl.f_distance_alphabet_size) || (v_code >= 1128u)) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); + break; + } + v_nbits = ((uint32_t)(wuffs_base__u8__min(self->private_data.f_dist_extra_bits[v_code], 32u))); + if (v_n_bits < v_nbits) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_bit_reader); + break; + } + v_lo = ((uint32_t)((v_bits & ((((uint64_t)(1u)) << v_nbits) - 1u)))); + self->private_impl.f_bits_value = v_lo; + v_bits >>= v_nbits; + v_n_bits -= v_nbits; + self->private_impl.f_distance_code = ((uint32_t)(((uint64_t)(((uint64_t)(self->private_data.f_dist_offset[v_code])) + ((uint64_t)(((uint64_t)(self->private_impl.f_bits_value)) << self->private_impl.f_distance_postfix_bits)))))); + } + } + self->private_impl.f_cmd_state = 3u; + } + v_max_distance = self->private_impl.f_max_distance; + if (v_max_distance != self->private_impl.f_max_backward_distance) { + v_max_distance = wuffs_base__u32__min(self->private_impl.f_pos, self->private_impl.f_max_backward_distance); + } + v_distance = self->private_impl.f_distance_code; + v_n = self->private_impl.f_copy_length; + if ((v_distance > v_max_distance) || (v_distance == 0u)) { + break; + } + v_dst_i = ((uint64_t)(self->private_impl.f_pos)); + v_src_i = ((uint64_t)((((uint32_t)(self->private_impl.f_pos - v_distance)) & ((uint32_t)(self->private_impl.f_window_size - 1u))))); + if (((v_dst_i + ((uint64_t)(v_n))) >= ((uint64_t)(self->private_impl.f_rb_size))) || ((v_src_i + ((uint64_t)(v_n))) > ((uint64_t)(self->private_impl.f_window_size))) || (((uint64_t)(self->private_impl.f_window_size)) > ((uint64_t)(a_workbuf.len)))) { + break; + } + self->private_impl.f_max_distance = v_max_distance; + self->private_impl.f_dist_rb[(self->private_impl.f_dist_rb_idx & 3u)] = v_distance; + self->private_impl.f_dist_rb_idx += 1u; + self->private_impl.f_mb_remaining -= v_n; + v_k64 = 0u; + while (v_k64 < ((uint64_t)(v_n))) { + v_n64 = ((uint64_t)(((uint64_t)(v_n)) - v_k64)); + v_n64 = wuffs_base__u64__min(v_n64, ((uint64_t)(v_distance))); + v_dlo = ((uint64_t)(v_dst_i + v_k64)); + v_dhi = ((uint64_t)(v_dlo + v_n64)); + v_slo = ((uint64_t)(v_src_i + v_k64)); + v_shi = ((uint64_t)(v_slo + v_n64)); + if ((v_n64 == 0u) || + (v_dlo > v_dhi) || + (v_dhi > ((uint64_t)(a_workbuf.len))) || + (v_slo > v_shi) || + (v_shi > ((uint64_t)(a_workbuf.len)))) { + break; + } + wuffs_private_impl__slice_u8__copy_from_slice(wuffs_base__slice_u8__subslice_ij(a_workbuf, v_dlo, v_dhi), wuffs_base__slice_u8__subslice_ij(a_workbuf, v_slo, v_shi)); + v_k64 += v_n64; + } + self->private_impl.f_pos += ((uint32_t)(v_k64)); + v_n -= ((uint32_t)(v_k64)); + while (v_n > 0u) { + wuffs_brotli__decoder__put_byte(self, a_workbuf, wuffs_brotli__decoder__get_byte(self, a_workbuf, v_distance)); + v_n -= 1u; + } + self->private_impl.f_cmd_state = 0u; + if ((self->private_impl.f_mb_remaining == 0u) || (self->private_impl.f_mb_remaining >= 2147483648u)) { + self->private_impl.f_cmd_state = 4u; + break; + } + } + label__outer__break:; + while (v_n_bits >= 8u) { + if ( ! (iop_a_src > io1_a_src)) { + status = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_bit_reader); + goto exit; + } + iop_a_src--; + v_n_bits -= 8u; + } + self->private_impl.f_bits = ((uint32_t)((v_bits & ((((uint64_t)(1u)) << v_n_bits) - 1u)))); + self->private_impl.f_n_bits = v_n_bits; + status = v_ret; + if (wuffs_base__status__is_error(&status)) { + goto exit; + } else if (wuffs_base__status__is_suspension(&status)) { + status = wuffs_base__make_status(wuffs_base__error__cannot_return_a_suspension); + goto exit; + } + goto ok; + + ok: + goto exit; + exit: + if (a_src && a_src->data.ptr) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + return status; } @@ -84794,18 +85431,24 @@ wuffs_brotli__decoder__read_distance( wuffs_base__status status = wuffs_base__make_status(NULL); uint32_t v_code = 0; + uint32_t v_fr = 0; + uint32_t v_nbits = 0; uint32_t coro_susp_point = self->private_impl.p_read_distance; if (coro_susp_point) { v_code = self->private_data.s_read_distance.v_code; + v_nbits = self->private_data.s_read_distance.v_nbits; } switch (coro_susp_point) { WUFFS_BASE__COROUTINE_SUSPENSION_POINT_0; - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(1); - status = wuffs_brotli__decoder__read_symbol(self, a_src, (438272u + (self->private_impl.f_dist_htree_index * 920u))); - if (status.repr) { - goto suspend; + v_fr = wuffs_brotli__decoder__read_symbol_fast(self, a_src, (438272u + (self->private_impl.f_dist_htree_index * 920u))); + if (v_fr != 0u) { + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(1); + status = wuffs_brotli__decoder__read_symbol(self, a_src, (438272u + (self->private_impl.f_dist_htree_index * 920u))); + if (status.repr) { + goto suspend; + } } v_code = self->private_impl.f_symbol; self->private_impl.f_block_length[2u] -= 1u; @@ -84819,10 +85462,17 @@ wuffs_brotli__decoder__read_distance( status = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); goto exit; } - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(2); - status = wuffs_brotli__decoder__read_bits32(self, a_src, ((uint32_t)(wuffs_base__u8__min(self->private_data.f_dist_extra_bits[v_code], 32u)))); - if (status.repr) { - goto suspend; + v_nbits = ((uint32_t)(wuffs_base__u8__min(self->private_data.f_dist_extra_bits[v_code], 32u))); + v_fr = 1u; + if (v_nbits <= 24u) { + v_fr = wuffs_brotli__decoder__read_bits_fast(self, a_src, v_nbits); + } + if (v_fr != 0u) { + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(2); + status = wuffs_brotli__decoder__read_bits32(self, a_src, v_nbits); + if (status.repr) { + goto suspend; + } } if (v_code >= 1128u) { status = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); @@ -84839,6 +85489,7 @@ wuffs_brotli__decoder__read_distance( suspend: self->private_impl.p_read_distance = wuffs_base__status__is_suspension(&status) ? coro_susp_point : 0; self->private_data.s_read_distance.v_code = v_code; + self->private_data.s_read_distance.v_nbits = v_nbits; goto exit; exit: diff --git a/std/brotli/README.md b/std/brotli/README.md index daa973de..d16e596b 100644 --- a/std/brotli/README.md +++ b/std/brotli/README.md @@ -55,8 +55,9 @@ Without the large-window quirk and without attached dictionaries (see the window size plus some slack. The slack mirrors the C code's `kRingBufferWriteAheadSlack`, which its fast paths use to write (and discard) a few bytes past the ring buffer's end. This -implementation does not yet have such fast paths, but it still reserves the -slack, so that adding them later will not change the `workbuf_len` API. A +implementation's fast paths (see below) do not write past the ring buffer's +end, but it still reserves the slack, so that adding such fast paths later +will not change the `workbuf_len` API. A workbuf shorter than the stream's window size plus 542 bytes is rejected with `base."#bad workbuf length"`. @@ -138,12 +139,25 @@ invariants for memory safety but Wuffs requires explicit bounds checks. - The bit reader reads input bytes only when it needs them (like the C code's "safe" `BrotliSafeReadBits` and `SafeReadSymbol` functions), so that - `transform_io` can suspend at any point. There is no "fast path" (like the C - code's `ProcessCommands`, as opposed to `SafeProcessCommands`). Both paths - produce the same output and errors. -- LZ77 copies and dictionary words are written to the ring buffer one byte at - a time, with ring buffer events (the C code's `WriteRingBuffer` and - `WrapRingBuffer` calls) at the same positions as the C code. + `transform_io` can suspend at any point. Between reads, fewer than 8 bits + are buffered. The fast path, `process_commands_fast` (like the C code's + `ProcessCommands`, as opposed to `SafeProcessCommands`), is not a coroutine. + It reads ahead (up to 8 bytes at a time) into a 64-bit bit buffer while the + input has enough bytes, and when it returns to `process_commands` (at a + block switch, a ring buffer event, a dictionary reference or when the input + runs low), it gives back the whole bytes left in that buffer, which restores + the "fewer than 8 bits" state exactly. `process_commands` tracks where it is + within a command (`cmd_state`), so that either can pick up where the other + left off. Within `process_commands`, `read_symbol_fast` and `read_bits_fast` + (which are not coroutines either) read the same bytes at the same times as + the coroutines `read_symbol` and `read_bits`, which finish the job when the + input runs out. All paths produce the same output and errors. +- Dictionary words, and LZ77 copies that wrap around the physical ring buffer + or reach the C code's (virtual) ring buffer size, are written to the ring + buffer one byte at a time, with ring buffer events (the C code's + `WriteRingBuffer` and `WrapRingBuffer` calls) at the same positions as the + C code. Other LZ77 copies are written in chunks of at most the copy distance + (so that each chunk's source bytes are final), which is equivalent. - On error, the Wuffs decoder may have already written more output to `dst` than libbrotlidec would have (libbrotlidec does not flush on error), but what both have written agrees. @@ -153,7 +167,8 @@ invariants for memory safety but Wuffs requires explicit bounds checks. - Metadata callbacks (metadata blocks are currently skipped). See the "Metadata Blocks" section below. -- Fast paths. +- More fast paths, for example for dictionary words, for copies that wrap + around the ring buffer and for block switches. - Compound dictionary matches at encoder qualities 10 and 11 (see the "Encoder" section below). diff --git a/std/brotli/decode_brotli.wuffs b/std/brotli/decode_brotli.wuffs index eccf6bd2..495e7c55 100644 --- a/std/brotli/decode_brotli.wuffs +++ b/std/brotli/decode_brotli.wuffs @@ -230,6 +230,13 @@ pub struct decoder? implements base.io_transformer( insert_length : base.u32, copy_length : base.u32, + // cmd_state is where process_commands (and process_commands_fast) + // are within a command: 0 at its start, 1 when reading its literals + // (insert_length is how many are left), 2 when reading its distance + // and 3 when applying its copy. process_commands_fast sets it to 4 + // when the metablock is done. + cmd_state : base.u32[..= 4], + // quirk_large_window is the QUIRK_LARGE_WINDOW quirk (the C code's // BROTLI_DECODER_PARAM_LARGE_WINDOW parameter). large_window is // whether the stream is a large-window stream (the C code's @@ -679,6 +686,93 @@ pri func decoder.read_symbol?(src: base.io_reader, hoff: base.u32) { this.n_bits = n_bits } +// read_bits_fast is like read_bits but is not a coroutine. It reads input +// bytes exactly when read_bits would, for as long as args.src has them. It +// returns 0 when it has set this.bits_value, or 1 when read_bits must finish +// the job (this.bits and this.n_bits hold the bits read so far). +pri func decoder.read_bits_fast!(src: base.io_reader, n: base.u32[..= 24]) base.u32 { + var n : base.u32[..= 24] + var bits : base.u32 + var n_bits : base.u32 + + n = args.n + bits = this.bits + n_bits = this.n_bits + while n_bits < n, + post n_bits >= n, + { + if args.src.length() <= 0 { + this.bits = bits + this.n_bits = n_bits + return 1 + } + assert n_bits < 24 via "a < b: a < c; c <= b"(c: n) + bits |= args.src.peek_u8_as_u32() << n_bits + args.src.skip_u32_fast!(actual: 1, worst_case: 1) + n_bits += 8 + } + this.bits_value = bits & (((1 as base.u32) << n) - 1) + this.bits = bits >> n + this.n_bits = n_bits - n + return 0 +} + +// read_symbol_fast is like read_symbol but is not a coroutine. It reads +// input bytes exactly when read_symbol would, for as long as args.src has +// them. It returns 0 when it has set this.symbol, or 1 when read_symbol must +// finish the job (this.bits and this.n_bits hold the bits read so far). In +// particular, read_symbol (not read_symbol_fast) returns any error. +pri func decoder.read_symbol_fast!(src: base.io_reader, hoff: base.u32) base.u32 { + var hoff : base.u32 + var bits : base.u32 + var n_bits : base.u32 + var i : base.u32 + var e : base.u32 + var e_bits : base.u32[..= 23] + + hoff = args.hoff + bits = this.bits + n_bits = this.n_bits + while true { + i = hoff ~mod+ (bits & 0xFF) + if i >= HUFFS_LEN { + break + } + e = this.huffs[i] as base.u32 + e_bits = e & 15 + if e_bits <= 8 { + if n_bits >= e_bits { + this.symbol = e >> 4 + this.bits = bits >> e_bits + this.n_bits = n_bits - e_bits + return 0 + } + } else if n_bits > 8 { + i = (hoff ~mod+ (e >> 4)) ~mod+ ((bits >> 8) & (((1 as base.u32) << (e_bits - 8)) - 1)) + if i >= HUFFS_LEN { + break + } + e = this.huffs[i] as base.u32 + e_bits = (e & 15) + 8 + if n_bits >= e_bits { + this.symbol = e >> 4 + this.bits = bits >> e_bits + this.n_bits = n_bits - e_bits + return 0 + } + } + if (n_bits >= 24) or (args.src.length() <= 0) { + break + } + bits |= args.src.peek_u8_as_u32() << n_bits + args.src.skip_u32_fast!(actual: 1, worst_case: 1) + n_bits += 8 + } + this.bits = bits + this.n_bits = n_bits + return 1 +} + // jump_to_byte_boundary drops the bits up to the next byte boundary and // returns whether they were all zero. Mirrors BrotliJumpToByteBoundary in // bit_reader.h. diff --git a/std/brotli/decode_metablock.wuffs b/std/brotli/decode_metablock.wuffs index f3d29d85..ae301fec 100644 --- a/std/brotli/decode_metablock.wuffs +++ b/std/brotli/decode_metablock.wuffs @@ -418,7 +418,6 @@ pri func decoder.calculate_distance_lut!() { // Mirrors ProcessCommandsInternal in decode.c. pri func decoder.process_commands?(src: base.io_reader, workbuf: slice base.u8, dictbuf: roslice base.u8) { var cmd_code : base.u32[..= 703] - var insert_length : base.u32 var p1 : base.u32[..= 0xFF] var p2 : base.u32[..= 0xFF] var mode_offset : base.u32[..= 0x600] @@ -434,37 +433,72 @@ pri func decoder.process_commands?(src: base.io_reader, workbuf: slice base.u8, var address : base.u32 var chunk : base.u32[..= 14] var j : base.u64 + var fr : base.u32 + var dst_i : base.u64 + var src_i : base.u64 + var k64 : base.u64 + var n64 : base.u64 + var dlo : base.u64 + var dhi : base.u64 + var slo : base.u64 + var shi : base.u64 + var fs : base.status while true { - // CommandBegin. - if this.block_length[1] == 0 { - // Mirrors DecodeCommandBlockSwitch. - this.decode_block_switch?(src: args.src, t: 1) - this.htree_command = this.block_type_rb[3] & 0xFF - continue + if (this.cmd_state == 0) and (args.src.length() >= 16) { + fs = this.process_commands_fast!(src: args.src, workbuf: args.workbuf) + if not fs.is_ok() { + return fs + } else if this.cmd_state == 4 { + this.cmd_state = 0 + return ok + } } - // Mirrors ReadCommandInternal. - this.read_symbol?(src: args.src, - hoff: CMD_HUFFS_BASE + (this.htree_command * CMD_HUFFS_STRIDE)) - if this.symbol >= 704 { - return "#internal error: inconsistent Huffman table" + if this.cmd_state == 0 { + // CommandBegin. + if this.block_length[1] == 0 { + // Mirrors DecodeCommandBlockSwitch. + this.decode_block_switch?(src: args.src, t: 1) + this.htree_command = this.block_type_rb[3] & 0xFF + continue + } + + // Mirrors ReadCommandInternal. + fr = this.read_symbol_fast!(src: args.src, hoff: CMD_HUFFS_BASE + (this.htree_command * CMD_HUFFS_STRIDE)) + if fr <> 0 { + this.read_symbol?(src: args.src, + hoff: CMD_HUFFS_BASE + (this.htree_command * CMD_HUFFS_STRIDE)) + } + if this.symbol >= 704 { + return "#internal error: inconsistent Huffman table" + } + cmd_code = this.symbol.min(no_more_than: 703) + this.implicit_zero_distance = CMD_IMPLICIT_ZERO_DISTANCE[cmd_code] <> 0 + this.distance_context = CMD_DISTANCE_CONTEXT[cmd_code] as base.u32 + this.update_dist_htree_index!() + fr = this.read_bits_fast!(src: args.src, n: CMD_INSERT_LEN_EXTRA_BITS[cmd_code] as base.u32) + if fr <> 0 { + this.read_bits?(src: args.src, n: CMD_INSERT_LEN_EXTRA_BITS[cmd_code] as base.u32) + } + this.insert_length = CMD_INSERT_LEN_OFFSET[cmd_code] ~mod+ this.bits_value + fr = this.read_bits_fast!(src: args.src, n: CMD_COPY_LEN_EXTRA_BITS[cmd_code] as base.u32) + if fr <> 0 { + this.read_bits?(src: args.src, n: CMD_COPY_LEN_EXTRA_BITS[cmd_code] as base.u32) + } + this.copy_length = CMD_COPY_LEN_OFFSET[cmd_code] ~mod+ this.bits_value + this.block_length[1] ~mod-= 1 + if this.insert_length > 0 { + this.mb_remaining ~mod-= this.insert_length + this.cmd_state = 1 + } else { + this.cmd_state = 2 + } } - cmd_code = this.symbol.min(no_more_than: 703) - this.implicit_zero_distance = CMD_IMPLICIT_ZERO_DISTANCE[cmd_code] <> 0 - this.distance_context = CMD_DISTANCE_CONTEXT[cmd_code] as base.u32 - this.update_dist_htree_index!() - this.read_bits?(src: args.src, n: CMD_INSERT_LEN_EXTRA_BITS[cmd_code] as base.u32) - insert_length = CMD_INSERT_LEN_OFFSET[cmd_code] ~mod+ this.bits_value - this.read_bits?(src: args.src, n: CMD_COPY_LEN_EXTRA_BITS[cmd_code] as base.u32) - this.copy_length = CMD_COPY_LEN_OFFSET[cmd_code] ~mod+ this.bits_value - this.block_length[1] ~mod-= 1 - - if insert_length > 0 { - this.mb_remaining ~mod-= insert_length + if this.cmd_state == 1 { // CommandInner. Read the literals in the command. - while insert_length > 0 { + while this.insert_length > 0 { if this.block_length[0] == 0 { // Mirrors DecodeLiteralBlockSwitch. this.decode_block_switch?(src: args.src, t: 0) @@ -478,39 +512,48 @@ pri func decoder.process_commands?(src: base.io_reader, workbuf: slice base.u8, ctx = (CONTEXT_LOOKUP[mode_offset | p1] | CONTEXT_LOOKUP[mode_offset | 0x100 | p2]) as base.u32 tree = this.context_maps[this.context_map_slice + ctx] as base.u32 - this.read_symbol?(src: args.src, hoff: LIT_HUFFS_BASE + (tree * LIT_HUFFS_STRIDE)) + fr = this.read_symbol_fast!(src: args.src, hoff: LIT_HUFFS_BASE + (tree * LIT_HUFFS_STRIDE)) + if fr <> 0 { + this.read_symbol?(src: args.src, hoff: LIT_HUFFS_BASE + (tree * LIT_HUFFS_STRIDE)) + } this.put_byte!(workbuf: args.workbuf, b: (this.symbol & 0xFF) as base.u8) this.block_length[0] ~mod-= 1 - insert_length -= 1 + this.insert_length ~mod-= 1 if this.pos == this.rb_size { this.ring_event?() } } + this.cmd_state = 2 if (this.mb_remaining == 0) or (this.mb_remaining >= 0x8000_0000) { // BROTLI_STATE_METABLOCK_DONE. + this.cmd_state = 0 return ok } } - // CommandPostDecodeLiterals. - if this.implicit_zero_distance { - // Implicit distance case. - this.distance_context = 1 - this.dist_rb_idx ~mod-= 1 - this.distance_code = this.dist_rb[this.dist_rb_idx & 3] - } else { - // Read distance code in the command, unless it was implicitly - // zero. - if this.block_length[2] == 0 { - // Mirrors DecodeDistanceBlockSwitch. - this.decode_block_switch?(src: args.src, t: 2) - this.dist_context_map_slice = (this.block_type_rb[5] & 0xFF) << 2 - this.update_dist_htree_index!() + if this.cmd_state == 2 { + // CommandPostDecodeLiterals. + if this.implicit_zero_distance { + // Implicit distance case. + this.distance_context = 1 + this.dist_rb_idx ~mod-= 1 + this.distance_code = this.dist_rb[this.dist_rb_idx & 3] + } else { + // Read distance code in the command, unless it was implicitly + // zero. + if this.block_length[2] == 0 { + // Mirrors DecodeDistanceBlockSwitch. + this.decode_block_switch?(src: args.src, t: 2) + this.dist_context_map_slice = (this.block_type_rb[5] & 0xFF) << 2 + this.update_dist_htree_index!() + } + this.read_distance?(src: args.src) } - this.read_distance?(src: args.src) + this.cmd_state = 3 } + // this.cmd_state == 3. if this.max_distance <> this.max_backward_distance { this.max_distance = this.pos.min(no_more_than: this.max_backward_distance) } @@ -604,6 +647,37 @@ pri func decoder.process_commands?(src: base.io_reader, workbuf: slice base.u8, this.dist_rb[this.dist_rb_idx & 3] = distance this.dist_rb_idx ~mod+= 1 this.mb_remaining ~mod-= i + // Fast path: when the copy neither wraps around the physical ring + // buffer (at either end) nor makes this.pos reach this.rb_size, + // copy the bytes in chunks of at most distance bytes. Each chunk's + // source bytes are final (and do not overlap the chunk's + // destination), so this matches the byte by byte loop below. + dst_i = this.pos as base.u64 + src_i = ((this.pos ~mod- distance) & (this.window_size ~mod- 1)) as base.u64 + if ((dst_i + (i as base.u64)) < (this.rb_size as base.u64)) and + ((src_i + (i as base.u64)) <= (this.window_size as base.u64)) and + ((this.window_size as base.u64) <= args.workbuf.length()) and + (distance > 0) { + k64 = 0 + while k64 < (i as base.u64) { + n64 = (i as base.u64) ~mod- k64 + n64 = n64.min(no_more_than: distance as base.u64) + dlo = dst_i ~mod+ k64 + dhi = dlo ~mod+ n64 + slo = src_i ~mod+ k64 + shi = slo ~mod+ n64 + if (n64 == 0) or (dlo > dhi) or (dhi > args.workbuf.length()) or + (slo > shi) or (shi > args.workbuf.length()) { + break + } + args.workbuf[dlo .. dhi].copy_from_slice!(s: args.workbuf[slo .. shi]) + k64 ~mod+= n64 + } + // k64 == i, unless the (unreachable) break above was taken, + // in which case the loop below copies the remaining bytes. + this.pos ~mod+= (k64 & 0xFFFF_FFFF) as base.u32 + i ~mod-= (k64 & 0xFFFF_FFFF) as base.u32 + } while i > 0 { b = this.get_byte(workbuf: args.workbuf, distance: distance) this.put_byte!(workbuf: args.workbuf, b: b) @@ -614,6 +688,7 @@ pri func decoder.process_commands?(src: base.io_reader, workbuf: slice base.u8, } } + this.cmd_state = 0 if (this.mb_remaining == 0) or (this.mb_remaining >= 0x8000_0000) { // BROTLI_STATE_METABLOCK_DONE. return ok @@ -621,6 +696,323 @@ pri func decoder.process_commands?(src: base.io_reader, workbuf: slice base.u8, } } +// process_commands_fast is a fast path for process_commands. It is not a +// coroutine. It handles commands, or parts of commands (see +// decoder.cmd_state), for as long as args.src holds enough bytes and the work +// needs no block switch, ring buffer event or dictionary reference. Then it +// returns, and process_commands picks up where it left off. +// +// It reads ahead, up to 8 bytes at a time, into a 64-bit bit buffer. When it +// returns, it gives back (undoes) the whole bytes left in that buffer. The +// coroutines read_symbol and read_bits read an input byte only when they +// need more bits, so that between reads, fewer than 8 bits are buffered. +// Giving back the whole bytes restores that state exactly, so this function +// reads the same bits and leaves the decoder in the same state as +// process_commands would, and it writes the same bytes and returns the same +// errors. It sets this.cmd_state to 4 when the metablock is done. +pri func decoder.process_commands_fast!(src: base.io_reader, workbuf: slice base.u8) base.status { + var ret : base.status + var bits : base.u64 + var n_bits : base.u32[..= 63] + var i : base.u32 + var e : base.u32 + var e_bits : base.u32[..= 23] + var cmd_code : base.u32[..= 703] + var nb : base.u32[..= 24] + var p1 : base.u32[..= 0xFF] + var p2 : base.u32[..= 0xFF] + var mode_offset : base.u32[..= 0x600] + var ctx : base.u32[..= 63] + var tree : base.u32[..= 0xFF] + var code : base.u32 + var nbits : base.u32[..= 32] + var lo : base.u32 + var max_distance : base.u32 + var distance : base.u32 + var n : base.u32 + var dst_i : base.u64 + var src_i : base.u64 + var k64 : base.u64 + var n64 : base.u64 + var dlo : base.u64 + var dhi : base.u64 + var slo : base.u64 + var shi : base.u64 + + if this.n_bits >= 8 { + return ok + } + n_bits = this.n_bits & 7 + bits = (this.bits as base.u64) & (((1 as base.u64) << n_bits) - 1) + + while.outer true { + if this.cmd_state == 0 { + // Two refills, of at most 8 bytes each. + if (this.block_length[1] == 0) or (args.src.length() < 16) { + break.outer + } + bits |= args.src.peek_u64le() ~mod<< n_bits + args.src.skip_u32_fast!(actual: (63 - n_bits) >> 3, worst_case: 8) + n_bits |= 56 + // Mirrors ReadCommandInternal. + i = (CMD_HUFFS_BASE + (this.htree_command * CMD_HUFFS_STRIDE)) ~mod+ ((bits & 0xFF) as base.u32) + if i >= HUFFS_LEN { + ret = "#internal error: inconsistent Huffman table" + break.outer + } + e = this.huffs[i] as base.u32 + e_bits = e & 15 + if e_bits > 8 { + i = ((CMD_HUFFS_BASE + (this.htree_command * CMD_HUFFS_STRIDE)) ~mod+ (e >> 4)) ~mod+ ((((bits >> 8) & 0x7FFF) as base.u32) & (((1 as base.u32) << (e_bits - 8)) - 1)) + if i >= HUFFS_LEN { + ret = "#internal error: inconsistent Huffman table" + break.outer + } + e = this.huffs[i] as base.u32 + e_bits = (e & 15) + 8 + } + if n_bits < e_bits { + ret = "#internal error: inconsistent bit reader" + break.outer + } + bits >>= e_bits + n_bits -= e_bits + if (e >> 4) >= 704 { + ret = "#internal error: inconsistent Huffman table" + break.outer + } + this.symbol = e >> 4 + cmd_code = this.symbol.min(no_more_than: 703) + this.implicit_zero_distance = CMD_IMPLICIT_ZERO_DISTANCE[cmd_code] <> 0 + this.distance_context = CMD_DISTANCE_CONTEXT[cmd_code] as base.u32 + this.update_dist_htree_index!() + nb = CMD_INSERT_LEN_EXTRA_BITS[cmd_code] as base.u32 + if n_bits < nb { + ret = "#internal error: inconsistent bit reader" + break.outer + } + this.bits_value = ((bits & (((1 as base.u64) << nb) - 1)) & 0xFFFF_FFFF) as base.u32 + bits >>= nb + n_bits -= nb + this.insert_length = CMD_INSERT_LEN_OFFSET[cmd_code] ~mod+ this.bits_value + if args.src.length() < 8 { + ret = "#internal error: inconsistent bit reader" + break.outer + } + bits |= args.src.peek_u64le() ~mod<< n_bits + args.src.skip_u32_fast!(actual: (63 - n_bits) >> 3, worst_case: 8) + n_bits |= 56 + nb = CMD_COPY_LEN_EXTRA_BITS[cmd_code] as base.u32 + if n_bits < nb { + ret = "#internal error: inconsistent bit reader" + break.outer + } + this.bits_value = ((bits & (((1 as base.u64) << nb) - 1)) & 0xFFFF_FFFF) as base.u32 + bits >>= nb + n_bits -= nb + this.copy_length = CMD_COPY_LEN_OFFSET[cmd_code] ~mod+ this.bits_value + this.block_length[1] ~mod-= 1 + if this.insert_length > 0 { + this.mb_remaining ~mod-= this.insert_length + this.cmd_state = 1 + } else { + this.cmd_state = 2 + } + } + + if this.cmd_state == 1 { + // CommandInner. Read the literals in the command. p1 and p2 are + // the last two bytes written to the ring buffer (put_byte always + // writes when the physical ring buffer fits in args.workbuf). + if (this.window_size as base.u64) > args.workbuf.length() { + break.outer + } + p1 = this.get_byte(workbuf: args.workbuf, distance: 1) as base.u32 + p2 = this.get_byte(workbuf: args.workbuf, distance: 2) as base.u32 + while this.insert_length > 0 { + if (this.block_length[0] == 0) or ((this.pos ~mod+ 1) >= this.rb_size) { + break.outer + } + if n_bits < 24 { + if args.src.length() < 8 { + break.outer + } + bits |= args.src.peek_u64le() ~mod<< n_bits + args.src.skip_u32_fast!(actual: (63 - n_bits) >> 3, worst_case: 8) + n_bits |= 56 + } + mode_offset = this.context_mode << 9 + ctx = (CONTEXT_LOOKUP[mode_offset | p1] | + CONTEXT_LOOKUP[mode_offset | 0x100 | p2]) as base.u32 + tree = this.context_maps[this.context_map_slice + ctx] as base.u32 + i = (LIT_HUFFS_BASE + (tree * LIT_HUFFS_STRIDE)) ~mod+ ((bits & 0xFF) as base.u32) + if i >= HUFFS_LEN { + ret = "#internal error: inconsistent Huffman table" + break.outer + } + e = this.huffs[i] as base.u32 + e_bits = e & 15 + if e_bits > 8 { + i = ((LIT_HUFFS_BASE + (tree * LIT_HUFFS_STRIDE)) ~mod+ (e >> 4)) ~mod+ ((((bits >> 8) & 0x7FFF) as base.u32) & (((1 as base.u32) << (e_bits - 8)) - 1)) + if i >= HUFFS_LEN { + ret = "#internal error: inconsistent Huffman table" + break.outer + } + e = this.huffs[i] as base.u32 + e_bits = (e & 15) + 8 + } + if n_bits < e_bits { + ret = "#internal error: inconsistent bit reader" + break.outer + } + bits >>= e_bits + n_bits -= e_bits + this.symbol = e >> 4 + this.put_byte!(workbuf: args.workbuf, b: (this.symbol & 0xFF) as base.u8) + p2 = p1 + p1 = this.symbol & 0xFF + this.block_length[0] ~mod-= 1 + this.insert_length ~mod-= 1 + } + this.cmd_state = 2 + if (this.mb_remaining == 0) or (this.mb_remaining >= 0x8000_0000) { + this.cmd_state = 4 + break.outer + } + } + + if this.cmd_state == 2 { + // CommandPostDecodeLiterals. + if this.implicit_zero_distance { + this.distance_context = 1 + this.dist_rb_idx ~mod-= 1 + this.distance_code = this.dist_rb[this.dist_rb_idx & 3] + } else { + // One refill: 56 bits suffice for a symbol and 32 extra bits. + if (this.block_length[2] == 0) or (args.src.length() < 8) { + break.outer + } + bits |= args.src.peek_u64le() ~mod<< n_bits + args.src.skip_u32_fast!(actual: (63 - n_bits) >> 3, worst_case: 8) + n_bits |= 56 + // Mirrors read_distance. + i = (DIST_HUFFS_BASE + (this.dist_htree_index * DIST_HUFFS_STRIDE)) ~mod+ ((bits & 0xFF) as base.u32) + if i >= HUFFS_LEN { + ret = "#internal error: inconsistent Huffman table" + break.outer + } + e = this.huffs[i] as base.u32 + e_bits = e & 15 + if e_bits > 8 { + i = ((DIST_HUFFS_BASE + (this.dist_htree_index * DIST_HUFFS_STRIDE)) ~mod+ (e >> 4)) ~mod+ ((((bits >> 8) & 0x7FFF) as base.u32) & (((1 as base.u32) << (e_bits - 8)) - 1)) + if i >= HUFFS_LEN { + ret = "#internal error: inconsistent Huffman table" + break.outer + } + e = this.huffs[i] as base.u32 + e_bits = (e & 15) + 8 + } + if n_bits < e_bits { + ret = "#internal error: inconsistent bit reader" + break.outer + } + bits >>= e_bits + n_bits -= e_bits + this.symbol = e >> 4 + code = this.symbol + this.block_length[2] ~mod-= 1 + this.distance_context = 0 + if code < 16 { + this.take_distance_from_ring_buffer!(code: code) + } else { + if (code >= this.distance_alphabet_size) or (code >= 1128) { + ret = "#internal error: inconsistent Huffman table" + break.outer + } + nbits = this.dist_extra_bits[code].min(no_more_than: 32) as base.u32 + if n_bits < nbits { + ret = "#internal error: inconsistent bit reader" + break.outer + } + // Like read_bits32, this is (lo | (hi << 16)) when nbits + // is more than 24. + lo = ((bits & (((1 as base.u64) << nbits) - 1)) & 0xFFFF_FFFF) as base.u32 + this.bits_value = lo + bits >>= nbits + n_bits -= nbits + this.distance_code = (((this.dist_offset[code] as base.u64) ~mod+ + ((this.bits_value as base.u64) ~mod<< this.distance_postfix_bits)) & 0xFFFF_FFFF) as base.u32 + } + } + this.cmd_state = 3 + } + + // this.cmd_state == 3. Only handle LZ77 copies that need no ring + // buffer event and do not wrap around the physical ring buffer. + max_distance = this.max_distance + if max_distance <> this.max_backward_distance { + max_distance = this.pos.min(no_more_than: this.max_backward_distance) + } + distance = this.distance_code + n = this.copy_length + if (distance > max_distance) or (distance == 0) { + break.outer + } + dst_i = this.pos as base.u64 + src_i = ((this.pos ~mod- distance) & (this.window_size ~mod- 1)) as base.u64 + if ((dst_i + (n as base.u64)) >= (this.rb_size as base.u64)) or + ((src_i + (n as base.u64)) > (this.window_size as base.u64)) or + ((this.window_size as base.u64) > args.workbuf.length()) { + break.outer + } + this.max_distance = max_distance + this.dist_rb[this.dist_rb_idx & 3] = distance + this.dist_rb_idx ~mod+= 1 + this.mb_remaining ~mod-= n + // Copy in chunks of at most distance bytes, like process_commands. + k64 = 0 + while k64 < (n as base.u64) { + n64 = (n as base.u64) ~mod- k64 + n64 = n64.min(no_more_than: distance as base.u64) + dlo = dst_i ~mod+ k64 + dhi = dlo ~mod+ n64 + slo = src_i ~mod+ k64 + shi = slo ~mod+ n64 + if (n64 == 0) or (dlo > dhi) or (dhi > args.workbuf.length()) or + (slo > shi) or (shi > args.workbuf.length()) { + break + } + args.workbuf[dlo .. dhi].copy_from_slice!(s: args.workbuf[slo .. shi]) + k64 ~mod+= n64 + } + this.pos ~mod+= (k64 & 0xFFFF_FFFF) as base.u32 + n ~mod-= (k64 & 0xFFFF_FFFF) as base.u32 + // n == 0, unless the (unreachable) break above was taken. This.pos + // stays below this.rb_size, so there is no ring buffer event. + while n > 0 { + this.put_byte!(workbuf: args.workbuf, b: this.get_byte(workbuf: args.workbuf, distance: distance)) + n -= 1 + } + this.cmd_state = 0 + if (this.mb_remaining == 0) or (this.mb_remaining >= 0x8000_0000) { + this.cmd_state = 4 + break.outer + } + }.outer + + // Give back the whole bytes left in the bit buffer. + while n_bits >= 8 { + if not args.src.can_undo_byte() { + return "#internal error: inconsistent bit reader" + } + args.src.undo_byte!() + n_bits -= 8 + } + this.bits = ((bits & (((1 as base.u64) << n_bits) - 1)) & 0xFFFF_FFFF) as base.u32 + this.n_bits = n_bits + return ret +} + // update_dist_htree_index sets this.dist_htree_index from the distance // context map. Mirrors part of ReadCommandInternal and // DecodeDistanceBlockSwitch in decode.c. @@ -631,10 +1023,15 @@ pri func decoder.update_dist_htree_index!() { // Mirrors ReadDistanceInternal in decode.c. pri func decoder.read_distance?(src: base.io_reader) { - var code : base.u32 + var code : base.u32 + var fr : base.u32 + var nbits : base.u32[..= 32] - this.read_symbol?(src: args.src, - hoff: DIST_HUFFS_BASE + (this.dist_htree_index * DIST_HUFFS_STRIDE)) + fr = this.read_symbol_fast!(src: args.src, hoff: DIST_HUFFS_BASE + (this.dist_htree_index * DIST_HUFFS_STRIDE)) + if fr <> 0 { + this.read_symbol?(src: args.src, + hoff: DIST_HUFFS_BASE + (this.dist_htree_index * DIST_HUFFS_STRIDE)) + } code = this.symbol this.block_length[2] ~mod-= 1 // Convert the distance code to the actual distance by possibly looking up @@ -649,7 +1046,14 @@ pri func decoder.read_distance?(src: base.io_reader) { } // The C code uses BrotliSafeReadBits32, as large-window brotli can have // more than 24 extra bits. - this.read_bits32?(src: args.src, n: this.dist_extra_bits[code].min(no_more_than: 32) as base.u32) + nbits = this.dist_extra_bits[code].min(no_more_than: 32) as base.u32 + fr = 1 + if nbits <= 24 { + fr = this.read_bits_fast!(src: args.src, n: nbits) + } + if fr <> 0 { + this.read_bits32?(src: args.src, n: nbits) + } if code >= 1128 { return "#internal error: inconsistent Huffman table" }