diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 99ff4617..a173a9cd 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -9236,6 +9236,7 @@ extern const char wuffs_brotli__error__truncated_input[]; extern const char wuffs_brotli__error__unsupported_large_window[]; extern const char wuffs_brotli__error__unsupported_multiple_custom_static_dictionaries[]; extern const char wuffs_brotli__error__unsupported_encoder_quality[]; +extern const char wuffs_brotli__error__unsupported_serialized_encoder_dictionary[]; // ---------------- Public Consts @@ -9457,6 +9458,20 @@ wuffs_brotli__encoder__transform_io( wuffs_base__io_buffer* a_src, wuffs_base__slice_u8 a_workbuf); +WUFFS_BASE__GENERATED_C_CODE +WUFFS_BASE__MAYBE_STATIC wuffs_base__status +wuffs_brotli__encoder__add_dictionary( + wuffs_brotli__encoder* self, + uint32_t a_dict_type, + wuffs_base__slice_u8 a_dict, + wuffs_base__slice_u8 a_workbuf); + +WUFFS_BASE__GENERATED_C_CODE +WUFFS_BASE__MAYBE_STATIC uint64_t +wuffs_brotli__encoder__add_dictionary_workbuf_len( + const wuffs_brotli__encoder* self, + uint64_t a_dict_len); + WUFFS_BASE__GENERATED_C_CODE WUFFS_BASE__MAYBE_STATIC uint64_t wuffs_brotli__metablock_tester__bits_entropy( @@ -10032,6 +10047,15 @@ struct wuffs_brotli__encoder__struct { uint32_t f_hr_factor_remove; uint64_t f_hr_next_ix; uint64_t f_hr_off; + uint32_t f_dict_num_chunks; + uint64_t f_dict_total_size; + uint64_t f_dict_end; + uint64_t f_dict_chunk_offsets[16]; + uint64_t f_dc_src_off[15]; + uint64_t f_dc_size[15]; + uint32_t f_dc_bucket_bits[15]; + uint32_t f_dc_slot_bits[15]; + uint64_t f_dc_num_items[15]; uint64_t f_sr_len; uint64_t f_sr_distance; uint64_t f_sr_score; @@ -10045,6 +10069,7 @@ struct wuffs_brotli__encoder__struct { uint64_t f_mb_rle_off; uint32_t p_transform_io; + uint32_t p_do_transform_io; } private_impl; struct { @@ -10120,6 +10145,9 @@ struct wuffs_brotli__encoder__struct { uint8_t f_cl_depth[32]; uint16_t f_cl_bits[32]; uint32_t f_hash_table[131072]; + uint32_t f_dp_slot_size[4096]; + uint32_t f_dp_slot_limit[4096]; + uint32_t f_dp_slot_off[4096]; uint32_t f_command_buf[131072]; uint8_t f_literal_buf[131072]; @@ -10132,7 +10160,7 @@ struct wuffs_brotli__encoder__struct { uint64_t v_mark; uint64_t v_copy_input_size; uint64_t scratch; - } s_transform_io; + } s_do_transform_io; } private_data; #ifdef __cplusplus @@ -10245,6 +10273,20 @@ struct wuffs_brotli__encoder__struct { return wuffs_brotli__encoder__transform_io(this, a_dst, a_src, a_workbuf); } + inline wuffs_base__status + add_dictionary( + uint32_t a_dict_type, + wuffs_base__slice_u8 a_dict, + wuffs_base__slice_u8 a_workbuf) { + return wuffs_brotli__encoder__add_dictionary(this, a_dict_type, a_dict, a_workbuf); + } + + inline uint64_t + add_dictionary_workbuf_len( + uint64_t a_dict_len) const { + return wuffs_brotli__encoder__add_dictionary_workbuf_len(this, a_dict_len); + } + #endif // __cplusplus }; // struct wuffs_brotli__encoder__struct @@ -43005,6 +43047,7 @@ const char wuffs_brotli__error__internal_error_inconsistent_huffman_table[] = "# const char wuffs_brotli__error__internal_error_inconsistent_ring_buffer[] = "#brotli: internal error: inconsistent ring buffer"; const char wuffs_brotli__error__unsupported_encoder_quality[] = "#brotli: unsupported encoder quality"; const char wuffs_brotli__error__internal_error_inconsistent_encoder_state[] = "#brotli: internal error: inconsistent encoder state"; +const char wuffs_brotli__error__unsupported_serialized_encoder_dictionary[] = "#brotli: unsupported serialized encoder dictionary"; // ---------------- Private Consts @@ -68543,6 +68586,10 @@ WUFFS_BROTLI__DICTIONARY[122784] WUFFS_BASE__POTENTIALLY_UNUSED = { #define WUFFS_BROTLI__F64_3 4613937818241073152u +#define WUFFS_BROTLI__ENCODER_DICTIONARY_CHUNKS_MAX 15u + +#define WUFFS_BROTLI__PREPARED_DICTIONARY_HASH_MUL_64_LONG 2297779722762296275u + #define WUFFS_BROTLI__TREE_NONE 65535u #define WUFFS_BROTLI__FRAGMENT_MAX_DISTANCE 262128u @@ -69085,6 +69132,7 @@ static wuffs_base__status wuffs_brotli__encoder__create_backward_references( wuffs_brotli__encoder* self, wuffs_base__slice_u8 a_wb, + wuffs_base__slice_u8 a_dwb, uint64_t a_num_bytes, uint64_t a_position); @@ -69656,6 +69704,15 @@ wuffs_brotli__encoder__write_metadata_header( wuffs_brotli__encoder* self, uint32_t a_block_size); +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__status +wuffs_brotli__encoder__do_transform_io( + wuffs_brotli__encoder* self, + wuffs_base__io_buffer* a_dst, + wuffs_base__io_buffer* a_src, + wuffs_base__slice_u8 a_workbuf, + wuffs_base__slice_u8 a_dwb); + WUFFS_BASE__GENERATED_C_CODE static bool wuffs_brotli__metablock_builder__histogram_pair_is_less( @@ -69889,6 +69946,7 @@ static uint64_t wuffs_brotli__encoder__extend_last_command( wuffs_brotli__encoder* self, wuffs_base__slice_u8 a_wb, + wuffs_base__slice_u8 a_dwb, uint64_t a_bytes, uint64_t a_wrapped_last_processed_pos); @@ -69958,9 +70016,68 @@ static wuffs_base__status wuffs_brotli__encoder__encode_data( wuffs_brotli__encoder* self, wuffs_base__slice_u8 a_wb, + wuffs_base__slice_u8 a_dwb, bool a_is_last, bool a_force_flush); +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__encoder__prepare_dictionary( + wuffs_brotli__encoder* self, + wuffs_base__slice_u8 a_wb, + uint64_t a_src_off, + uint64_t a_n, + uint32_t a_bucket_bits, + uint32_t a_slot_bits); + +WUFFS_BASE__GENERATED_C_CODE +static bool +wuffs_brotli__encoder__compound_dictionary_enabled( + const wuffs_brotli__encoder* self); + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__empty_struct +wuffs_brotli__encoder__lookup_compound_dictionary_match( + wuffs_brotli__encoder* self, + wuffs_base__slice_u8 a_data, + wuffs_base__slice_u8 a_dwb, + uint64_t a_cur_ix, + uint64_t a_max_length, + uint64_t a_max_ring_buffer_distance, + uint64_t a_max_distance); + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__empty_struct +wuffs_brotli__encoder__find_compound_dictionary_match( + wuffs_brotli__encoder* self, + wuffs_base__slice_u8 a_data, + wuffs_base__slice_u8 a_dwb, + uint32_t a_d, + uint64_t a_cur_ix, + uint64_t a_max_length, + uint64_t a_distance_offset, + uint64_t a_max_distance); + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__encoder__find_match_length_2( + const wuffs_brotli__encoder* self, + wuffs_base__slice_u8 a_a, + wuffs_base__slice_u8 a_b, + uint64_t a_limit); + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__encoder__extend_last_command_compound( + const wuffs_brotli__encoder* self, + wuffs_base__slice_u8 a_wb, + wuffs_base__slice_u8 a_dwb, + uint64_t a_bytes, + uint64_t a_wrapped_last_processed_pos, + uint64_t a_cmd_dist, + uint64_t a_max_distance, + uint64_t a_last_copy_len); + WUFFS_BASE__GENERATED_C_CODE static bool wuffs_brotli__encoder__set_depth( @@ -75731,6 +75848,7 @@ static wuffs_base__status wuffs_brotli__encoder__create_backward_references( wuffs_brotli__encoder* self, wuffs_base__slice_u8 a_wb, + wuffs_base__slice_u8 a_dwb, uint64_t a_num_bytes, uint64_t a_position) { uint64_t v_position = 0; @@ -75755,6 +75873,9 @@ wuffs_brotli__encoder__create_backward_references( uint64_t v_range_end = 0; uint64_t v_k_margin = 0; uint64_t v_pos_jump = 0; + uint64_t v_gap = 0; + bool v_use_compound = false; + wuffs_base__slice_u8 v_data = {0}; v_position = a_position; v_max_backward_limit = ((uint64_t)((((uint64_t)(1u)) << self->private_impl.f_lgwin) - 16u)); @@ -75771,6 +75892,11 @@ wuffs_brotli__encoder__create_backward_references( v_random_heuristics_window_size = 512u; } v_apply_random_heuristics = ((uint64_t)(v_position + v_random_heuristics_window_size)); + v_gap = self->private_impl.f_dict_total_size; + v_use_compound = wuffs_brotli__encoder__compound_dictionary_enabled(self); + if (self->private_impl.f_rb_buf_off <= ((uint64_t)(a_wb.len))) { + v_data = wuffs_base__slice_u8__subslice_i(a_wb, self->private_impl.f_rb_buf_off); + } wuffs_brotli__encoder__prepare_distance_cache(self); while (((uint64_t)(v_position + v_htl)) < v_pos_end) { v_max_length = ((uint64_t)(v_pos_end - v_position)); @@ -75785,8 +75911,17 @@ wuffs_brotli__encoder__create_backward_references( v_position, v_max_length, v_max_distance, - v_dictionary_start, + ((uint64_t)(v_dictionary_start + v_gap)), self->private_impl.f_dist_max_distance); + if (v_use_compound) { + wuffs_brotli__encoder__lookup_compound_dictionary_match(self, + v_data, + a_dwb, + v_position, + v_max_length, + v_dictionary_start, + self->private_impl.f_dist_max_distance); + } if (self->private_impl.f_sr_score > 2020u) { v_delayed_backward_references = 0u; v_max_length -= 1u; @@ -75809,8 +75944,17 @@ wuffs_brotli__encoder__create_backward_references( ((uint64_t)(v_position + 1u)), v_max_length, v_max_distance, - v_dictionary_start, + ((uint64_t)(v_dictionary_start + v_gap)), self->private_impl.f_dist_max_distance); + if (v_use_compound) { + wuffs_brotli__encoder__lookup_compound_dictionary_match(self, + v_data, + a_dwb, + ((uint64_t)(v_position + 1u)), + v_max_length, + v_dictionary_start, + self->private_impl.f_dist_max_distance); + } if (self->private_impl.f_sr_score >= ((uint64_t)(v_sr_score + 175u))) { v_position += 1u; v_insert_length += 1u; @@ -75829,8 +75973,8 @@ wuffs_brotli__encoder__create_backward_references( } v_apply_random_heuristics = ((uint64_t)(((uint64_t)(v_position + ((uint64_t)(2u * self->private_impl.f_sr_len)))) + v_random_heuristics_window_size)); v_dictionary_start = wuffs_brotli__encoder__min64(self, ((uint64_t)(v_position + v_position_offset)), v_max_backward_limit); - v_distance_code = wuffs_brotli__encoder__compute_distance_code(self, self->private_impl.f_sr_distance, v_dictionary_start); - if ((self->private_impl.f_sr_distance <= v_dictionary_start) && (v_distance_code > 0u)) { + v_distance_code = wuffs_brotli__encoder__compute_distance_code(self, self->private_impl.f_sr_distance, ((uint64_t)(v_dictionary_start + v_gap))); + if ((self->private_impl.f_sr_distance <= ((uint64_t)(v_dictionary_start + v_gap))) && (v_distance_code > 0u)) { self->private_impl.f_dist_cache[3u] = self->private_impl.f_dist_cache[2u]; self->private_impl.f_dist_cache[2u] = self->private_impl.f_dist_cache[1u]; self->private_impl.f_dist_cache[1u] = self->private_impl.f_dist_cache[0u]; @@ -78874,7 +79018,10 @@ wuffs_brotli__encoder__workbuf_len( v_lgwin, wuffs_brotli__encoder__computed_lgblock(self, v_q, v_lgwin), 22u); + } else { + return wuffs_base__utility__make_range_ii_u64(0u, 0u); } + wuffs_private_impl__u64__sat_add_indirect(&v_n, self->private_impl.f_dict_end); return wuffs_base__utility__make_range_ii_u64(v_n, v_n); } @@ -79338,6 +79485,72 @@ wuffs_brotli__encoder__transform_io( self->private_impl.active_coroutine = 0; wuffs_base__status status = wuffs_base__make_status(NULL); + wuffs_base__status v_status = wuffs_base__make_status(NULL); + + uint32_t coro_susp_point = self->private_impl.p_transform_io; + switch (coro_susp_point) { + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_0; + + while (true) { + if (self->private_impl.f_dict_end > ((uint64_t)(a_workbuf.len))) { + status = wuffs_base__make_status(wuffs_base__error__bad_workbuf_length); + goto exit; + } + { + wuffs_base__status t_0 = wuffs_brotli__encoder__do_transform_io(self, + a_dst, + a_src, + wuffs_base__slice_u8__subslice_i(a_workbuf, self->private_impl.f_dict_end), + wuffs_base__slice_u8__subslice_j(a_workbuf, self->private_impl.f_dict_end)); + v_status = t_0; + } + if (wuffs_base__status__is_ok(&v_status)) { + status = wuffs_base__make_status(NULL); + goto ok; + } else if ( ! wuffs_base__status__is_suspension(&v_status)) { + status = v_status; + if (wuffs_base__status__is_error(&status)) { + goto exit; + } else if (wuffs_base__status__is_suspension(&status)) { + status = wuffs_base__make_status(wuffs_base__error__cannot_return_a_suspension); + goto exit; + } + goto ok; + } + status = v_status; + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(1); + } + + ok: + self->private_impl.p_transform_io = 0; + goto exit; + } + + goto suspend; + suspend: + self->private_impl.p_transform_io = wuffs_base__status__is_suspension(&status) ? coro_susp_point : 0; + self->private_impl.active_coroutine = wuffs_base__status__is_suspension(&status) ? 1 : 0; + + goto exit; + exit: + if (wuffs_base__status__is_error(&status)) { + self->private_impl.magic = WUFFS_BASE__DISABLED; + } + return status; +} + +// -------- func brotli.encoder.do_transform_io + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__status +wuffs_brotli__encoder__do_transform_io( + wuffs_brotli__encoder* self, + wuffs_base__io_buffer* a_dst, + wuffs_base__io_buffer* a_src, + wuffs_base__slice_u8 a_workbuf, + wuffs_base__slice_u8 a_dwb) { + wuffs_base__status status = wuffs_base__make_status(NULL); + uint32_t v_op = 0; uint64_t v_avail = 0; uint64_t v_block_size = 0; @@ -79378,15 +79591,15 @@ wuffs_brotli__encoder__transform_io( io2_a_src = io0_a_src + a_src->meta.wi; } - uint32_t coro_susp_point = self->private_impl.p_transform_io; + uint32_t coro_susp_point = self->private_impl.p_do_transform_io; if (coro_susp_point) { - v_op = self->private_data.s_transform_io.v_op; - v_is_last = self->private_data.s_transform_io.v_is_last; - v_force_flush = self->private_data.s_transform_io.v_force_flush; - v_storage_len = self->private_data.s_transform_io.v_storage_len; - v_table_bits = self->private_data.s_transform_io.v_table_bits; - v_mark = self->private_data.s_transform_io.v_mark; - v_copy_input_size = self->private_data.s_transform_io.v_copy_input_size; + v_op = self->private_data.s_do_transform_io.v_op; + v_is_last = self->private_data.s_do_transform_io.v_is_last; + v_force_flush = self->private_data.s_do_transform_io.v_force_flush; + v_storage_len = self->private_data.s_do_transform_io.v_storage_len; + v_table_bits = self->private_data.s_do_transform_io.v_table_bits; + v_mark = self->private_data.s_do_transform_io.v_mark; + v_copy_input_size = self->private_data.s_do_transform_io.v_copy_input_size; } switch (coro_susp_point) { WUFFS_BASE__COROUTINE_SUSPENSION_POINT_0; @@ -79432,7 +79645,11 @@ wuffs_brotli__encoder__transform_io( continue; } if ((self->private_impl.f_quality >= 2u) && (self->private_impl.f_input_pos != self->private_impl.f_last_flush_pos)) { - v_status = wuffs_brotli__encoder__encode_data(self, a_workbuf, false, true); + v_status = wuffs_brotli__encoder__encode_data(self, + a_workbuf, + a_dwb, + false, + true); if ( ! wuffs_base__status__is_ok(&v_status)) { status = v_status; if (wuffs_base__status__is_error(&status)) { @@ -79506,15 +79723,15 @@ wuffs_brotli__encoder__transform_io( } } v_mark = ((uint64_t)(iop_a_src - io0_a_src)); - self->private_data.s_transform_io.scratch = v_copy_input_size; + self->private_data.s_do_transform_io.scratch = v_copy_input_size; WUFFS_BASE__COROUTINE_SUSPENSION_POINT(4); - if (self->private_data.s_transform_io.scratch > ((uint64_t)(io2_a_src - iop_a_src))) { - self->private_data.s_transform_io.scratch -= ((uint64_t)(io2_a_src - iop_a_src)); + if (self->private_data.s_do_transform_io.scratch > ((uint64_t)(io2_a_src - iop_a_src))) { + self->private_data.s_do_transform_io.scratch -= ((uint64_t)(io2_a_src - iop_a_src)); iop_a_src = io2_a_src; status = wuffs_base__make_status(wuffs_base__suspension__short_read); goto suspend; } - iop_a_src += self->private_data.s_transform_io.scratch; + iop_a_src += self->private_data.s_do_transform_io.scratch; wuffs_brotli__encoder__copy_input_to_ring_buffer(self, a_workbuf, wuffs_private_impl__io__since(v_mark, ((uint64_t)(iop_a_src - io0_a_src)), io0_a_src)); if ((self->private_impl.f_flint > 0u) && (self->private_impl.f_flint < 2147483648u)) { self->private_impl.f_flint = ((uint32_t)(self->private_impl.f_flint - ((uint32_t)((v_copy_input_size & 255u))))); @@ -79553,7 +79770,11 @@ wuffs_brotli__encoder__transform_io( v_force_flush = true; } wuffs_brotli__encoder__update_size_hint(self, ((uint64_t)(io2_a_src - iop_a_src))); - v_status = wuffs_brotli__encoder__encode_data(self, a_workbuf, v_is_last, v_force_flush); + v_status = wuffs_brotli__encoder__encode_data(self, + a_workbuf, + a_dwb, + v_is_last, + v_force_flush); if ( ! wuffs_base__status__is_ok(&v_status)) { status = v_status; if (wuffs_base__status__is_error(&status)) { @@ -79609,15 +79830,15 @@ wuffs_brotli__encoder__transform_io( self->private_impl.f_bw_ix = ((uint64_t)(self->private_impl.f_last_bytes_bits)); v_table_bits = wuffs_brotli__encoder__get_hash_table(self, v_block_size); v_mark = ((uint64_t)(iop_a_src - io0_a_src)); - self->private_data.s_transform_io.scratch = v_block_size; + self->private_data.s_do_transform_io.scratch = v_block_size; WUFFS_BASE__COROUTINE_SUSPENSION_POINT(5); - if (self->private_data.s_transform_io.scratch > ((uint64_t)(io2_a_src - iop_a_src))) { - self->private_data.s_transform_io.scratch -= ((uint64_t)(io2_a_src - iop_a_src)); + if (self->private_data.s_do_transform_io.scratch > ((uint64_t)(io2_a_src - iop_a_src))) { + self->private_data.s_do_transform_io.scratch -= ((uint64_t)(io2_a_src - iop_a_src)); iop_a_src = io2_a_src; status = wuffs_base__make_status(wuffs_base__suspension__short_read); goto suspend; } - iop_a_src += self->private_data.s_transform_io.scratch; + iop_a_src += self->private_data.s_do_transform_io.scratch; if (v_storage_len > ((uint64_t)(a_workbuf.len))) { status = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_encoder_state); goto exit; @@ -79675,21 +79896,20 @@ wuffs_brotli__encoder__transform_io( } ok: - self->private_impl.p_transform_io = 0; + self->private_impl.p_do_transform_io = 0; goto exit; } goto suspend; suspend: - self->private_impl.p_transform_io = wuffs_base__status__is_suspension(&status) ? coro_susp_point : 0; - self->private_impl.active_coroutine = wuffs_base__status__is_suspension(&status) ? 1 : 0; - self->private_data.s_transform_io.v_op = v_op; - self->private_data.s_transform_io.v_is_last = v_is_last; - self->private_data.s_transform_io.v_force_flush = v_force_flush; - self->private_data.s_transform_io.v_storage_len = v_storage_len; - self->private_data.s_transform_io.v_table_bits = v_table_bits; - self->private_data.s_transform_io.v_mark = v_mark; - self->private_data.s_transform_io.v_copy_input_size = v_copy_input_size; + self->private_impl.p_do_transform_io = wuffs_base__status__is_suspension(&status) ? coro_susp_point : 0; + self->private_data.s_do_transform_io.v_op = v_op; + self->private_data.s_do_transform_io.v_is_last = v_is_last; + self->private_data.s_do_transform_io.v_force_flush = v_force_flush; + self->private_data.s_do_transform_io.v_storage_len = v_storage_len; + self->private_data.s_do_transform_io.v_table_bits = v_table_bits; + self->private_data.s_do_transform_io.v_mark = v_mark; + self->private_data.s_do_transform_io.v_copy_input_size = v_copy_input_size; goto exit; exit: @@ -79700,9 +79920,6 @@ wuffs_brotli__encoder__transform_io( a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); } - if (wuffs_base__status__is_error(&status)) { - self->private_impl.magic = WUFFS_BASE__DISABLED; - } return status; } @@ -80872,6 +81089,7 @@ static uint64_t wuffs_brotli__encoder__extend_last_command( wuffs_brotli__encoder* self, wuffs_base__slice_u8 a_wb, + wuffs_base__slice_u8 a_dwb, uint64_t a_bytes, uint64_t a_wrapped_last_processed_pos) { uint64_t v_last = 0; @@ -80901,6 +81119,16 @@ wuffs_brotli__encoder__extend_last_command( v_n += 1u; v_p = (((uint64_t)(v_p + 1u)) & 4294967295u); } + } else { + v_n = wuffs_brotli__encoder__extend_last_command_compound(self, + a_wb, + a_dwb, + a_bytes, + a_wrapped_last_processed_pos, + v_cmd_dist, + v_max_distance, + v_last_copy_len); + v_copy_len += ((uint32_t)(v_n)); } wuffs_brotli__encoder__set_cmd(self, a_wb, @@ -81295,6 +81523,7 @@ static wuffs_base__status wuffs_brotli__encoder__encode_data( wuffs_brotli__encoder* self, wuffs_base__slice_u8 a_wb, + wuffs_base__slice_u8 a_dwb, bool a_is_last, bool a_force_flush) { wuffs_base__status v_status = wuffs_base__make_status(NULL); @@ -81349,11 +81578,19 @@ wuffs_brotli__encoder__encode_data( a_is_last); wuffs_brotli__encoder__hasher_stitch_to_previous_block(self, a_wb, v_bytes, v_wrapped_last_processed_pos); if ((self->private_impl.f_num_cmds > 0u) && (self->private_impl.f_last_insert_len == 0u)) { - v_n = wuffs_brotli__encoder__extend_last_command(self, a_wb, v_bytes, v_wrapped_last_processed_pos); + v_n = wuffs_brotli__encoder__extend_last_command(self, + a_wb, + a_dwb, + v_bytes, + v_wrapped_last_processed_pos); v_bytes -= v_n; v_wrapped_last_processed_pos = (((uint64_t)(v_wrapped_last_processed_pos + v_n)) & 4294967295u); } - v_status = wuffs_brotli__encoder__create_backward_references(self, a_wb, v_bytes, v_wrapped_last_processed_pos); + v_status = wuffs_brotli__encoder__create_backward_references(self, + a_wb, + a_dwb, + v_bytes, + v_wrapped_last_processed_pos); if ( ! wuffs_base__status__is_ok(&v_status)) { return wuffs_private_impl__status__ensure_not_a_suspension(v_status); } @@ -81431,6 +81668,562 @@ wuffs_brotli__encoder__encode_data( return wuffs_base__make_status(NULL); } +// -------- func brotli.encoder.add_dictionary + +WUFFS_BASE__GENERATED_C_CODE +WUFFS_BASE__MAYBE_STATIC wuffs_base__status +wuffs_brotli__encoder__add_dictionary( + wuffs_brotli__encoder* self, + uint32_t a_dict_type, + wuffs_base__slice_u8 a_dict, + wuffs_base__slice_u8 a_workbuf) { + if (!self) { + return wuffs_base__make_status(wuffs_base__error__bad_receiver); + } + if (self->private_impl.magic != WUFFS_BASE__MAGIC) { + return wuffs_base__make_status( + (self->private_impl.magic == WUFFS_BASE__DISABLED) + ? wuffs_base__error__disabled_by_previous_error + : wuffs_base__error__initialize_not_called); + } + + uint64_t v_n = 0; + uint64_t v_lo = 0; + uint64_t v_hi = 0; + uint32_t v_bucket_bits = 0; + uint32_t v_slot_bits = 0; + uint64_t v_num_items = 0; + uint32_t v_c = 0; + + if (self->private_impl.f_is_initialized) { + return wuffs_base__make_status(wuffs_base__error__bad_call_sequence); + } else if (a_dict_type == 1u) { + return wuffs_base__make_status(wuffs_brotli__error__unsupported_serialized_encoder_dictionary); + } else if (a_dict_type != 0u) { + return wuffs_base__make_status(wuffs_base__error__bad_argument); + } else if (self->private_impl.f_dict_num_chunks >= 15u) { + return wuffs_base__make_status(wuffs_brotli__error__too_many_compound_dictionaries); + } + v_n = ((uint64_t)(a_dict.len)); + if ((v_n > 2147483647u) || (wuffs_base__u64__sat_add(self->private_impl.f_dict_total_size, v_n) > 2147483647u)) { + return wuffs_base__make_status(wuffs_brotli__error__too_much_dictionary_data); + } else if (wuffs_brotli__encoder__add_dictionary_workbuf_len(self, v_n) > ((uint64_t)(a_workbuf.len))) { + return wuffs_base__make_status(wuffs_base__error__bad_workbuf_length); + } + v_bucket_bits = 17u; + while (((((uint64_t)(16u)) << v_bucket_bits) < v_n) && (v_bucket_bits < 22u)) { + v_bucket_bits += 1u; + } + if (v_bucket_bits >= 10u) { + v_slot_bits = (v_bucket_bits - 10u); + } + v_lo = self->private_impl.f_dict_end; + v_hi = ((uint64_t)(v_lo + v_n)); + if ((v_lo > v_hi) || (v_hi > ((uint64_t)(a_workbuf.len)))) { + return wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_encoder_state); + } + wuffs_private_impl__slice_u8__copy_from_slice(wuffs_base__slice_u8__subslice_ij(a_workbuf, v_lo, v_hi), a_dict); + v_num_items = wuffs_brotli__encoder__prepare_dictionary(self, + a_workbuf, + v_lo, + v_n, + v_bucket_bits, + v_slot_bits); + v_c = self->private_impl.f_dict_num_chunks; + if (v_c >= 15u) { + return wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_encoder_state); + } + self->private_impl.f_dc_src_off[v_c] = v_lo; + self->private_impl.f_dc_size[v_c] = v_n; + self->private_impl.f_dc_bucket_bits[v_c] = v_bucket_bits; + self->private_impl.f_dc_slot_bits[v_c] = v_slot_bits; + self->private_impl.f_dc_num_items[v_c] = v_num_items; + self->private_impl.f_dict_total_size += v_n; + self->private_impl.f_dict_chunk_offsets[(v_c + 1u)] = self->private_impl.f_dict_total_size; + self->private_impl.f_dict_num_chunks = (v_c + 1u); + self->private_impl.f_dict_end = ((uint64_t)(((uint64_t)(((uint64_t)(v_hi + (((uint64_t)(4u)) << v_slot_bits))) + (((uint64_t)(2u)) << v_bucket_bits))) + ((uint64_t)(4u * v_num_items)))); + return wuffs_base__make_status(NULL); +} + +// -------- func brotli.encoder.add_dictionary_workbuf_len + +WUFFS_BASE__GENERATED_C_CODE +WUFFS_BASE__MAYBE_STATIC uint64_t +wuffs_brotli__encoder__add_dictionary_workbuf_len( + const wuffs_brotli__encoder* self, + uint64_t a_dict_len) { + if (!self) { + return 0; + } + if ((self->private_impl.magic != WUFFS_BASE__MAGIC) && + (self->private_impl.magic != WUFFS_BASE__DISABLED)) { + return 0; + } + + uint32_t v_bucket_bits = 0; + uint32_t v_slot_bits = 0; + uint64_t v_n = 0; + uint64_t v_max_items = 0; + + if (a_dict_len > 2147483647u) { + return 18446744073709551615u; + } + v_n = a_dict_len; + v_bucket_bits = 17u; + while (((((uint64_t)(16u)) << v_bucket_bits) < v_n) && (v_bucket_bits < 22u)) { + v_bucket_bits += 1u; + } + if (v_bucket_bits >= 10u) { + v_slot_bits = (v_bucket_bits - 10u); + } + if (v_n > 7u) { + v_max_items = (v_n - 7u); + } + return wuffs_base__u64__sat_add(wuffs_base__u64__sat_add(wuffs_base__u64__sat_add(wuffs_base__u64__sat_add(wuffs_base__u64__sat_add(self->private_impl.f_dict_end, v_n), (((uint64_t)(4u)) << v_slot_bits)), (((uint64_t)(2u)) << v_bucket_bits)), ((uint64_t)(4u * v_max_items))), wuffs_base__u64__sat_add((((uint64_t)(6u)) << v_bucket_bits), ((uint64_t)(4u * v_n)))); +} + +// -------- func brotli.encoder.prepare_dictionary + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__encoder__prepare_dictionary( + wuffs_brotli__encoder* self, + wuffs_base__slice_u8 a_wb, + uint64_t a_src_off, + uint64_t a_n, + uint32_t a_bucket_bits, + uint32_t a_slot_bits) { + uint64_t v_num_slots = 0; + uint64_t v_num_buckets = 0; + uint64_t v_slot_mask = 0; + uint64_t v_slot_off = 0; + uint64_t v_heads_off = 0; + uint64_t v_items_off = 0; + uint64_t v_num_off = 0; + uint64_t v_bh_off = 0; + uint64_t v_nb_off = 0; + uint64_t v_max_items = 0; + uint64_t v_i = 0; + uint64_t v_j = 0; + uint64_t v_h = 0; + uint64_t v_key = 0; + uint32_t v_count = 0; + uint32_t v_limit = 0; + uint32_t v_size = 0; + bool v_overflow = false; + uint64_t v_total_items = 0; + uint64_t v_slot = 0; + uint64_t v_cursor = 0; + uint64_t v_pos = 0; + uint32_t v_k = 0; + + v_num_slots = (((uint64_t)(1u)) << a_slot_bits); + v_num_buckets = (((uint64_t)(1u)) << a_bucket_bits); + v_slot_mask = (v_num_slots - 1u); + if (a_n > 7u) { + v_max_items = (a_n - 7u); + } + v_slot_off = ((uint64_t)(a_src_off + a_n)); + v_heads_off = ((uint64_t)(v_slot_off + ((uint64_t)(4u * v_num_slots)))); + v_items_off = ((uint64_t)(v_heads_off + ((uint64_t)(2u * v_num_buckets)))); + v_num_off = ((uint64_t)(v_items_off + ((uint64_t)(4u * v_max_items)))); + v_bh_off = ((uint64_t)(v_num_off + ((uint64_t)(2u * v_num_buckets)))); + v_nb_off = ((uint64_t)(v_bh_off + ((uint64_t)(4u * v_num_buckets)))); + wuffs_brotli__encoder__fill_bytes(self, + a_wb, + v_num_off, + ((uint64_t)(2u * v_num_buckets)), + 0u); + v_i = 0u; + while (((uint64_t)(v_i + 7u)) < a_n) { + v_h = ((uint64_t)((wuffs_brotli__encoder__load64le(self, a_wb, ((uint64_t)(a_src_off + v_i))) & 1099511627775u) * 2297779722762296275u)); + v_key = ((v_h >> 32u) >> (32u - a_bucket_bits)); + v_count = ((uint32_t)(wuffs_brotli__encoder__peek_u16(self, a_wb, ((uint64_t)(v_num_off + ((uint64_t)(2u * v_key))))))); + if (v_count == 0u) { + wuffs_brotli__encoder__poke_u32(self, a_wb, ((uint64_t)(v_nb_off + ((uint64_t)(4u * v_i)))), 4294967295u); + } else { + wuffs_brotli__encoder__poke_u32(self, a_wb, ((uint64_t)(v_nb_off + ((uint64_t)(4u * v_i)))), wuffs_brotli__encoder__load32le(self, a_wb, ((uint64_t)(v_bh_off + ((uint64_t)(4u * v_key)))))); + } + wuffs_brotli__encoder__poke_u32(self, a_wb, ((uint64_t)(v_bh_off + ((uint64_t)(4u * v_key)))), ((uint32_t)(v_i))); + v_count += 1u; + if (v_count > 32u) { + v_count = 32u; + } + wuffs_brotli__encoder__poke_u16(self, a_wb, ((uint64_t)(v_num_off + ((uint64_t)(2u * v_key)))), ((uint16_t)(v_count))); + v_i += 1u; + } + v_i = 0u; + while (v_i < v_num_slots) { + v_limit = 32u; + while (true) { + v_count = 0u; + v_overflow = false; + v_j = v_i; + while (v_j < v_num_buckets) { + v_size = ((uint32_t)(wuffs_brotli__encoder__peek_u16(self, a_wb, ((uint64_t)(v_num_off + ((uint64_t)(2u * v_j))))))); + if (v_count >= 65535u) { + v_overflow = true; + break; + } + if (v_size > v_limit) { + v_size = v_limit; + } + v_count += v_size; + v_j += v_num_slots; + } + if ( ! v_overflow) { + self->private_data.f_dp_slot_size[(v_i & 4095u)] = v_count; + v_total_items += ((uint64_t)(v_count)); + break; + } else if (v_limit == 0u) { + self->private_data.f_dp_slot_size[(v_i & 4095u)] = 0u; + break; + } + v_limit -= 1u; + } + self->private_data.f_dp_slot_limit[(v_i & 4095u)] = v_limit; + v_i += 1u; + } + v_total_items = 0u; + v_i = 0u; + while (v_i < v_num_slots) { + wuffs_brotli__encoder__poke_u32(self, a_wb, ((uint64_t)(v_slot_off + ((uint64_t)(4u * v_i)))), ((uint32_t)(v_total_items))); + self->private_data.f_dp_slot_off[(v_i & 4095u)] = ((uint32_t)(v_total_items)); + v_total_items += ((uint64_t)(self->private_data.f_dp_slot_size[(v_i & 4095u)])); + self->private_data.f_dp_slot_size[(v_i & 4095u)] = 0u; + v_i += 1u; + } + v_i = 0u; + while (v_i < v_num_buckets) { + v_slot = (v_i & v_slot_mask & 4095u); + v_count = ((uint32_t)(wuffs_brotli__encoder__peek_u16(self, a_wb, ((uint64_t)(v_num_off + ((uint64_t)(2u * v_i))))))); + if (v_count > self->private_data.f_dp_slot_limit[v_slot]) { + v_count = self->private_data.f_dp_slot_limit[v_slot]; + } + if (v_count == 0u) { + wuffs_brotli__encoder__poke_u16(self, a_wb, ((uint64_t)(v_heads_off + ((uint64_t)(2u * v_i)))), 65535u); + v_i += 1u; + continue; + } + v_cursor = ((uint64_t)(self->private_data.f_dp_slot_size[v_slot])); + wuffs_brotli__encoder__poke_u16(self, a_wb, ((uint64_t)(v_heads_off + ((uint64_t)(2u * v_i)))), ((uint16_t)(v_cursor))); + v_cursor += ((uint64_t)(self->private_data.f_dp_slot_off[v_slot])); + self->private_data.f_dp_slot_size[v_slot] += v_count; + v_pos = ((uint64_t)(wuffs_brotli__encoder__load32le(self, a_wb, ((uint64_t)(v_bh_off + ((uint64_t)(4u * v_i))))))); + v_k = 0u; + while (v_k < v_count) { + wuffs_brotli__encoder__poke_u32(self, a_wb, ((uint64_t)(v_items_off + ((uint64_t)(4u * v_cursor)))), ((uint32_t)(v_pos))); + v_cursor += 1u; + v_pos = ((uint64_t)(wuffs_brotli__encoder__load32le(self, a_wb, ((uint64_t)(v_nb_off + ((uint64_t)(4u * v_pos))))))); + v_k += 1u; + } + v_cursor -= 1u; + wuffs_brotli__encoder__poke_u32(self, a_wb, ((uint64_t)(v_items_off + ((uint64_t)(4u * v_cursor)))), (wuffs_brotli__encoder__load32le(self, a_wb, ((uint64_t)(v_items_off + ((uint64_t)(4u * v_cursor))))) | 2147483648u)); + v_i += 1u; + } + return v_total_items; +} + +// -------- func brotli.encoder.compound_dictionary_enabled + +WUFFS_BASE__GENERATED_C_CODE +static bool +wuffs_brotli__encoder__compound_dictionary_enabled( + const wuffs_brotli__encoder* self) { + uint32_t v_htype = 0; + + if (self->private_impl.f_dict_num_chunks == 0u) { + return false; + } + v_htype = self->private_impl.f_hasher_type; + return ((v_htype == 5u) || + (v_htype == 6u) || + (v_htype == 58u) || + (v_htype == 68u) || + (v_htype == 40u) || + (v_htype == 41u) || + (v_htype == 42u) || + ((v_htype == 54u) && (self->private_impl.f_hr_jump != 0u))); +} + +// -------- func brotli.encoder.lookup_compound_dictionary_match + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__empty_struct +wuffs_brotli__encoder__lookup_compound_dictionary_match( + wuffs_brotli__encoder* self, + wuffs_base__slice_u8 a_data, + wuffs_base__slice_u8 a_dwb, + uint64_t a_cur_ix, + uint64_t a_max_length, + uint64_t a_max_ring_buffer_distance, + uint64_t a_max_distance) { + uint64_t v_base_offset = 0; + uint32_t v_d = 0; + + v_base_offset = ((uint64_t)(a_max_ring_buffer_distance + self->private_impl.f_dict_total_size)); + v_d = 0u; + while (v_d < self->private_impl.f_dict_num_chunks) { + wuffs_brotli__encoder__find_compound_dictionary_match(self, + a_data, + a_dwb, + v_d, + a_cur_ix, + a_max_length, + ((uint64_t)(v_base_offset - self->private_impl.f_dict_chunk_offsets[v_d])), + a_max_distance); + v_d += 1u; + } + return wuffs_base__make_empty_struct(); +} + +// -------- func brotli.encoder.find_compound_dictionary_match + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__empty_struct +wuffs_brotli__encoder__find_compound_dictionary_match( + wuffs_brotli__encoder* self, + wuffs_base__slice_u8 a_data, + wuffs_base__slice_u8 a_dwb, + uint32_t a_d, + uint64_t a_cur_ix, + uint64_t a_max_length, + uint64_t a_distance_offset, + uint64_t a_max_distance) { + uint64_t v_source_size = 0; + uint64_t v_boundary = 0; + uint64_t v_src_off = 0; + uint64_t v_slot_off = 0; + uint64_t v_heads_off = 0; + uint64_t v_items_off = 0; + wuffs_base__slice_u8 v_source = {0}; + wuffs_base__slice_u8 v_cur = {0}; + uint64_t v_cur_ix_masked = 0; + uint64_t v_mask = 0; + uint64_t v_best_score = 0; + uint64_t v_best_len = 0; + uint32_t v_i = 0; + uint64_t v_h = 0; + uint64_t v_key = 0; + uint64_t v_slot = 0; + uint64_t v_head = 0; + uint64_t v_chain = 0; + uint32_t v_item = 0; + uint64_t v_distance = 0; + uint64_t v_offset = 0; + uint64_t v_limit = 0; + uint64_t v_len = 0; + uint64_t v_score = 0; + uint64_t v_num_items = 0; + uint32_t v_bb = 0; + uint32_t v_sb = 0; + + v_bb = wuffs_base__u32__min(self->private_impl.f_dc_bucket_bits[a_d], 22u); + v_sb = wuffs_base__u32__min(self->private_impl.f_dc_slot_bits[a_d], 12u); + v_num_items = self->private_impl.f_dc_num_items[a_d]; + v_source_size = self->private_impl.f_dc_size[a_d]; + v_boundary = ((uint64_t)(a_distance_offset - v_source_size)); + v_src_off = self->private_impl.f_dc_src_off[a_d]; + v_slot_off = ((uint64_t)(v_src_off + v_source_size)); + v_heads_off = ((uint64_t)(v_slot_off + (((uint64_t)(4u)) << v_sb))); + v_items_off = ((uint64_t)(v_heads_off + (((uint64_t)(2u)) << v_bb))); + if ((v_src_off > v_slot_off) || (v_slot_off > ((uint64_t)(a_dwb.len)))) { + return wuffs_base__make_empty_struct(); + } + v_source = wuffs_base__slice_u8__subslice_ij(a_dwb, v_src_off, v_slot_off); + v_mask = ((uint64_t)(self->private_impl.f_rb_mask)); + v_cur_ix_masked = (a_cur_ix & v_mask); + if (v_cur_ix_masked > ((uint64_t)(a_data.len))) { + return wuffs_base__make_empty_struct(); + } + v_cur = wuffs_base__slice_u8__subslice_i(a_data, v_cur_ix_masked); + v_best_score = self->private_impl.f_sr_score; + v_best_len = self->private_impl.f_sr_len; + v_h = ((uint64_t)((wuffs_brotli__encoder__load64le(self, a_data, v_cur_ix_masked) & 1099511627775u) * 2297779722762296275u)); + v_key = ((v_h >> 32u) >> (32u - v_bb)); + v_slot = (v_key & ((((uint64_t)(1u)) << v_sb) - 1u)); + v_head = ((uint64_t)(wuffs_brotli__encoder__peek_u16(self, a_dwb, ((uint64_t)(v_heads_off + ((uint64_t)(2u * v_key))))))); + v_chain = ((uint64_t)(((uint64_t)(wuffs_brotli__encoder__load32le(self, a_dwb, ((uint64_t)(v_slot_off + ((uint64_t)(4u * v_slot))))))) + v_head)); + v_item = 0u; + if (v_head == 65535u) { + v_item = 1u; + } + v_i = 0u; + while (v_i < 4u) { + v_distance = wuffs_brotli__encoder__dist_cache_u64(self, v_i); + if ((v_distance <= v_boundary) || (v_distance > a_distance_offset)) { + v_i += 1u; + continue; + } + v_offset = ((uint64_t)(a_distance_offset - v_distance)); + v_limit = ((uint64_t)(v_source_size - v_offset)); + v_limit = wuffs_base__u64__min(v_limit, a_max_length); + v_len = 0u; + if (v_offset <= ((uint64_t)(v_source.len))) { + v_len = wuffs_brotli__encoder__find_match_length_2(self, wuffs_base__slice_u8__subslice_i(v_source, v_offset), v_cur, v_limit); + } + if (v_len >= 2u) { + v_score = wuffs_brotli__encoder__backward_reference_score_using_last_distance(self, v_len); + if (v_best_score < v_score) { + if (v_i != 0u) { + v_score -= wuffs_brotli__encoder__backward_reference_penalty_using_last_distance(self, ((uint64_t)(v_i))); + } + if (v_best_score < v_score) { + v_best_score = v_score; + if (v_len > v_best_len) { + v_best_len = v_len; + } + self->private_impl.f_sr_len = v_len; + self->private_impl.f_sr_len_code_delta = 0u; + self->private_impl.f_sr_distance = v_distance; + self->private_impl.f_sr_score = v_best_score; + } + } + } + v_i += 1u; + } + if (v_best_len < 3u) { + v_best_len = 3u; + } + while (v_item == 0u) { + if (v_chain >= v_num_items) { + break; + } + v_item = wuffs_brotli__encoder__load32le(self, a_dwb, ((uint64_t)(v_items_off + ((uint64_t)(4u * v_chain))))); + v_chain += 1u; + v_offset = ((uint64_t)((v_item & 2147483647u))); + v_item &= 2147483648u; + v_distance = ((uint64_t)(a_distance_offset - v_offset)); + v_limit = ((uint64_t)(v_source_size - v_offset)); + v_limit = wuffs_base__u64__min(v_limit, a_max_length); + if (v_distance > a_max_distance) { + continue; + } + if ((((uint64_t)(v_cur_ix_masked + v_best_len)) > v_mask) || (v_best_len >= v_limit) || (wuffs_brotli__encoder__load32le(self, a_data, ((uint64_t)(((uint64_t)(v_cur_ix_masked + v_best_len)) - 3u))) != wuffs_brotli__encoder__load32le(self, v_source, ((uint64_t)(((uint64_t)(v_offset + v_best_len)) - 3u))))) { + continue; + } + v_len = 0u; + if (v_offset <= ((uint64_t)(v_source.len))) { + v_len = wuffs_brotli__encoder__find_match_length_2(self, wuffs_base__slice_u8__subslice_i(v_source, v_offset), v_cur, v_limit); + } + if (v_len >= 4u) { + v_score = wuffs_brotli__encoder__backward_reference_score(self, v_len, v_distance); + if (v_best_score < v_score) { + v_best_score = v_score; + v_best_len = v_len; + self->private_impl.f_sr_len = v_best_len; + self->private_impl.f_sr_len_code_delta = 0u; + self->private_impl.f_sr_distance = v_distance; + self->private_impl.f_sr_score = v_best_score; + } + } + } + return wuffs_base__make_empty_struct(); +} + +// -------- func brotli.encoder.find_match_length_2 + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__encoder__find_match_length_2( + const wuffs_brotli__encoder* self, + wuffs_base__slice_u8 a_a, + wuffs_base__slice_u8 a_b, + uint64_t a_limit) { + uint64_t v_matched = 0; + uint64_t v_n = 0; + wuffs_base__slice_u8 v_a = {0}; + wuffs_base__slice_u8 v_b = {0}; + uint64_t v_x = 0; + + v_a = a_a; + v_b = a_b; + v_n = a_limit; + while ((v_n >= 8u) && (((uint64_t)(v_a.len)) >= 8u) && (((uint64_t)(v_b.len)) >= 8u)) { + v_x = (wuffs_base__peek_u64le__no_bounds_check(v_a.ptr) ^ wuffs_base__peek_u64le__no_bounds_check(v_b.ptr)); + if (v_x != 0u) { + while ((v_x & 255u) == 0u) { + v_matched += 1u; + v_x >>= 8u; + } + return v_matched; + } + v_matched += 8u; + v_n -= 8u; + v_a = wuffs_base__slice_u8__subslice_i(v_a, 8u); + v_b = wuffs_base__slice_u8__subslice_i(v_b, 8u); + } + while ((v_n > 0u) && (((uint64_t)(v_a.len)) > 0u) && (((uint64_t)(v_b.len)) > 0u)) { + if (v_a.ptr[0u] != v_b.ptr[0u]) { + break; + } + v_matched += 1u; + v_n -= 1u; + v_a = wuffs_base__slice_u8__subslice_i(v_a, 1u); + v_b = wuffs_base__slice_u8__subslice_i(v_b, 1u); + } + return v_matched; +} + +// -------- func brotli.encoder.extend_last_command_compound + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__encoder__extend_last_command_compound( + const wuffs_brotli__encoder* self, + wuffs_base__slice_u8 a_wb, + wuffs_base__slice_u8 a_dwb, + uint64_t a_bytes, + uint64_t a_wrapped_last_processed_pos, + uint64_t a_cmd_dist, + uint64_t a_max_distance, + uint64_t a_last_copy_len) { + uint64_t v_address = 0; + uint32_t v_br_index = 0; + uint64_t v_br_offset = 0; + uint64_t v_chunk_length = 0; + uint64_t v_n = 0; + uint64_t v_p = 0; + + if (((uint64_t)(((uint64_t)(a_cmd_dist - a_max_distance)) - 1u)) >= self->private_impl.f_dict_total_size) { + return 0u; + } else if (a_last_copy_len >= ((uint64_t)(a_cmd_dist - a_max_distance))) { + return 0u; + } + v_address = ((uint64_t)(((uint64_t)(self->private_impl.f_dict_total_size - ((uint64_t)(a_cmd_dist - a_max_distance)))) + a_last_copy_len)); + v_br_index = 0u; + while ((v_br_index < 15u) && (v_address >= self->private_impl.f_dict_chunk_offsets[(v_br_index + 1u)])) { + v_br_index += 1u; + } + if (v_br_index >= 15u) { + return 0u; + } + v_br_offset = ((uint64_t)(v_address - self->private_impl.f_dict_chunk_offsets[v_br_index])); + v_chunk_length = ((uint64_t)(self->private_impl.f_dict_chunk_offsets[(v_br_index + 1u)] - self->private_impl.f_dict_chunk_offsets[v_br_index])); + v_p = a_wrapped_last_processed_pos; + while (v_n < a_bytes) { + if ((v_br_offset >= v_chunk_length) || (v_br_index >= 15u)) { + break; + } + if (wuffs_brotli__encoder__rb_byte(self, a_wb, v_p) != wuffs_brotli__encoder__byte_at(self, a_dwb, ((uint64_t)(self->private_impl.f_dc_src_off[v_br_index] + v_br_offset)))) { + break; + } + v_n += 1u; + v_p = (((uint64_t)(v_p + 1u)) & 4294967295u); + v_br_offset += 1u; + if (v_br_offset == v_chunk_length) { + v_br_index += 1u; + v_br_offset = 0u; + if ((v_br_index < self->private_impl.f_dict_num_chunks) && (v_br_index < 15u)) { + v_chunk_length = ((uint64_t)(self->private_impl.f_dict_chunk_offsets[(v_br_index + 1u)] - self->private_impl.f_dict_chunk_offsets[v_br_index])); + } else { + break; + } + } + } + return v_n; +} + // -------- func brotli.encoder.set_depth WUFFS_BASE__GENERATED_C_CODE diff --git a/script/manual-test-brotli-encoder-parity.c b/script/manual-test-brotli-encoder-parity.c index b7416b50..7c07a315 100644 --- a/script/manual-test-brotli-encoder-parity.c +++ b/script/manual-test-brotli-encoder-parity.c @@ -68,6 +68,7 @@ #include #include +#include "brotli/decode.h" #include "brotli/encode.h" #define WUFFS_IMPLEMENTATION @@ -107,6 +108,26 @@ static uint64_t g_num_mismatches; static uint32_t g_qmin = 0; static uint32_t g_qmax = 9; static bool g_lw_only = false; +static bool g_dict_only = false; + +// The attached raw dictionaries (see set_dicts), for both encoders: the C +// code's prepared dictionaries and a Wuffs encoder (g_dict_template) that has +// them added, with the dictionaries in g_workbuf[.. g_dict_end]. For a small +// total dictionary size, encode_wuffs adds them to a fresh encoder instead. +#define MAX_DICTS 16 +static const uint8_t* g_dict_ptrs[MAX_DICTS]; +static size_t g_dict_lens[MAX_DICTS]; +static size_t g_num_dicts; +static size_t g_dict_total; +static BrotliEncoderPreparedDictionary* g_dict_prepared[MAX_DICTS]; +static wuffs_brotli__encoder* g_dict_template; +static uint64_t g_dict_end; +static uint64_t g_num_roundtrips; +static uint64_t g_last_wlen; // encode_wuffs' workbuf_len. +static char g_dict_desc[160]; +static uint8_t* g_foreign_dict; // test/data/enwik5. +static size_t g_foreign_dict_len; +static uint64_t g_num_smaller_checks; typedef struct { uint32_t keys[MAX_PARAMS]; @@ -342,6 +363,74 @@ brotli_enc_quirk_key(uint32_t param) { return 0; } +// grow_workbuf makes g_workbuf at least n bytes long, keeping its contents +// (and so any dictionaries at its start). +static bool // +grow_workbuf(uint64_t n) { + if (n <= g_workbuf_len) { + return true; + } else if (n > ((uint64_t)3 << 30)) { + return false; + } + uint8_t* p = realloc(g_workbuf, (size_t)n); + if (!p) { + return false; + } + g_workbuf = p; + g_workbuf_len = (size_t)n; + return true; +} + +// set_dicts sets the attached dictionaries for the following checks (none if +// n is 0), preparing them for both encoders. +static const char* // +set_dicts(size_t n, const uint8_t* const* ptrs, const size_t* lens) { + for (size_t j = 0; j < g_num_dicts; j++) { + BrotliEncoderDestroyPreparedDictionary(g_dict_prepared[j]); + g_dict_prepared[j] = NULL; + } + free(g_dict_template); + g_dict_template = NULL; + g_num_dicts = 0; + g_dict_total = 0; + g_dict_end = 0; + for (size_t j = 0; (j < n) && (j < MAX_DICTS); j++) { + g_dict_ptrs[j] = ptrs[j]; + g_dict_lens[j] = lens[j]; + g_dict_total += lens[j]; + // The quality argument does not matter for raw dictionaries. + g_dict_prepared[j] = BrotliEncoderPrepareDictionary( + BROTLI_SHARED_DICTIONARY_RAW, lens[j], ptrs[j], 11, NULL, NULL, NULL); + g_num_dicts = j + 1; + if (!g_dict_prepared[j]) { + return "BrotliEncoderPrepareDictionary failed"; + } + } + if (g_dict_total <= (256 << 10)) { + return NULL; + } + g_dict_template = wuffs_brotli__encoder__alloc(); + if (!g_dict_template) { + return "alloc failed"; + } + for (size_t j = 0; j < g_num_dicts; j++) { + uint64_t wlen = wuffs_brotli__encoder__add_dictionary_workbuf_len( + g_dict_template, g_dict_lens[j]); + if (!grow_workbuf(wlen)) { + return "malloc failed"; + } + wuffs_base__status status = wuffs_brotli__encoder__add_dictionary( + g_dict_template, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, + wuffs_base__make_slice_u8((uint8_t*)g_dict_ptrs[j], g_dict_lens[j]), + wuffs_base__make_slice_u8(g_workbuf, (size_t)wlen)); + if (status.repr) { + return status.repr; + } + } + g_dict_end = wuffs_brotli__encoder__workbuf_len(g_dict_template).max_incl; + return NULL; +} + // encode_wuffs mirrors wuffs_brotli_encode_plan in test/c/std/brotli.c. static const char* // encode_wuffs(const config* c, size_t* have_len) { @@ -350,6 +439,26 @@ encode_wuffs(const config* c, size_t* have_len) { return "alloc failed"; } const char* ret = NULL; + if (g_dict_template) { + memcpy(enc, g_dict_template, sizeof__wuffs_brotli__encoder()); + } else if (g_num_dicts > 0) { + for (size_t j = 0; j < g_num_dicts; j++) { + uint64_t n = wuffs_brotli__encoder__add_dictionary_workbuf_len( + enc, g_dict_lens[j]); + if (!grow_workbuf(n)) { + ret = "malloc failed"; + goto cleanup; + } + wuffs_base__status status = wuffs_brotli__encoder__add_dictionary( + enc, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, + wuffs_base__make_slice_u8((uint8_t*)g_dict_ptrs[j], g_dict_lens[j]), + wuffs_base__make_slice_u8(g_workbuf, (size_t)n)); + if (status.repr) { + ret = status.repr; + goto cleanup; + } + } + } wuffs_base__io_buffer dst = wuffs_base__ptr_u8__writer(g_have, MAX_OUTPUT); wuffs_base__io_buffer src = wuffs_base__ptr_u8__reader(g_src, g_src_len, false); @@ -364,18 +473,14 @@ encode_wuffs(const config* c, size_t* have_len) { } } uint64_t wlen = wuffs_brotli__encoder__workbuf_len(enc).max_incl; - if (wlen > ((uint64_t)1 << 32)) { + if (wlen > ((uint64_t)3 << 30)) { ret = "workbuf_len is too large"; goto cleanup; - } else if (wlen > g_workbuf_len) { - free(g_workbuf); - g_workbuf = malloc((size_t)wlen); - g_workbuf_len = g_workbuf ? (size_t)wlen : 0; - if (!g_workbuf) { - ret = "malloc failed"; - goto cleanup; - } + } else if (!grow_workbuf(wlen)) { + ret = "malloc failed"; + goto cleanup; } + g_last_wlen = wlen; if ((g_short_workbuf_len > 0) && (g_short_workbuf_len < wlen)) { wlen = g_short_workbuf_len; } @@ -450,6 +555,12 @@ encode_mimic(const config* c, size_t* want_len) { goto cleanup; } } + for (size_t j = 0; j < g_num_dicts; j++) { + if (!BrotliEncoderAttachPreparedDictionary(state, g_dict_prepared[j])) { + ret = "BrotliEncoderAttachPreparedDictionary failed"; + goto cleanup; + } + } const uint8_t* next_in = g_src; for (size_t i = 0; i < g_num_steps; i++) { size_t avail_in = g_step_lens[i]; @@ -498,6 +609,10 @@ describe(char* buf, size_t n, const char* name, const config* c) { c->chunking, c->flush, c->metadata_position, c->metadata_len, c->dst_chunk, c->seed); } + k = (int)strlen(buf); + if ((g_num_dicts > 0) && (k > 0) && ((size_t)k < n)) { + snprintf(buf + k, n - (size_t)k, " dicts=%s", g_dict_desc); + } } // short_workbuf_len returns the shortest workbuf (at qualities 2 to 9) that @@ -557,9 +672,12 @@ short_workbuf_len(const config* c, size_t len, uint64_t wlen) { return (n < wlen) ? (size_t)n : 0; } +static const char* // +roundtrip(const config* c, const uint8_t* data, size_t len, size_t want_len); + static void // check(const char* name, const config* c, const uint8_t* data, size_t len) { - char desc[512]; + char desc[768]; g_num_configs++; const char* err = make_plan(c, data, len); if (err) { @@ -575,16 +693,7 @@ check(const char* name, const config* c, const uint8_t* data, size_t len) { if (c->try_short && !have_err && !want_err) { // Also check the shortest workbuf that should work, and that one byte // less fails. - wuffs_brotli__encoder* enc = wuffs_brotli__encoder__alloc(); - uint64_t wlen = 0; - if (enc) { - for (size_t j = 0; j < c->num_params; j++) { - wuffs_brotli__encoder__set_quirk(enc, brotli_enc_quirk_key(c->keys[j]), - c->values[j]); - } - wlen = wuffs_brotli__encoder__workbuf_len(enc).max_incl; - free(enc); - } + uint64_t wlen = g_last_wlen; size_t short_len = short_workbuf_len(c, len, wlen); if (short_len > 0) { g_num_short_checks++; @@ -627,7 +736,121 @@ check(const char* name, const config* c, const uint8_t* data, size_t len) { "at offset %zu\n", desc, have_len, want_len, i); fflush(stdout); + return; } + if (g_num_dicts > 0) { + // A stream offset makes the output a continuation of another stream, so + // it cannot be decoded on its own. + bool stream_offset = false; + for (size_t j = 0; j < c->num_params; j++) { + if ((c->keys[j] == 9) && c->values[j]) { + stream_offset = true; + } + } + if (!stream_offset) { + g_num_roundtrips++; + const char* rt_err = roundtrip(c, data, len, want_len); + if (rt_err) { + g_num_mismatches++; + describe(desc, sizeof desc, name, c); + printf("ROUNDTRIP FAILURE %s: %s\n", desc, rt_err); + fflush(stdout); + } + } + } +} + +// roundtrip checks that both libbrotlidec and Wuffs' decoder (with the same +// dictionaries attached) decode g_want (of want_len bytes) to data. +static const char* // +roundtrip(const config* c, const uint8_t* data, size_t len, size_t want_len) { + static uint8_t* out = NULL; + static uint8_t* dec_workbuf = NULL; + static size_t dec_workbuf_len = 0; + if (!out) { + out = malloc(MAX_INPUT); + if (!out) { + return "malloc failed"; + } + } + bool lw = false; + for (size_t j = 0; j < c->num_params; j++) { + if (c->keys[j] == 6) { + lw = c->values[j] != 0; + } + } + + // libbrotlidec. + BrotliDecoderState* d = BrotliDecoderCreateInstance(NULL, NULL, NULL); + if (!d) { + return "BrotliDecoderCreateInstance failed"; + } + BrotliDecoderSetParameter(d, BROTLI_DECODER_PARAM_LARGE_WINDOW, lw ? 1 : 0); + for (size_t j = 0; j < g_num_dicts; j++) { + if (!BrotliDecoderAttachDictionary(d, BROTLI_SHARED_DICTIONARY_RAW, + g_dict_lens[j], g_dict_ptrs[j])) { + BrotliDecoderDestroyInstance(d); + return "BrotliDecoderAttachDictionary failed"; + } + } + size_t avail_in = want_len; + const uint8_t* next_in = g_want; + size_t avail_out = MAX_INPUT; + uint8_t* next_out = out; + BrotliDecoderResult r = BrotliDecoderDecompressStream( + d, &avail_in, &next_in, &avail_out, &next_out, NULL); + BrotliDecoderDestroyInstance(d); + if (r != BROTLI_DECODER_RESULT_SUCCESS) { + return "libbrotlidec failed"; + } else if (((MAX_INPUT - avail_out) != len) || (len && memcmp(out, data, len))) { + return "libbrotlidec round trip mismatch"; + } + + // Wuffs' decoder. + wuffs_brotli__decoder* dec = wuffs_brotli__decoder__alloc(); + if (!dec) { + return "alloc failed"; + } + const char* ret = NULL; + wuffs_brotli__decoder__set_quirk(dec, WUFFS_BROTLI__QUIRK_LARGE_WINDOW, + lw ? 1 : 0); + size_t need = g_dict_total + ((size_t)1 << 30) + 542; + if (need > dec_workbuf_len) { + free(dec_workbuf); + dec_workbuf = malloc(need); + dec_workbuf_len = dec_workbuf ? need : 0; + if (!dec_workbuf) { + free(dec); + return "malloc failed"; + } + } + wuffs_base__slice_u8 wb = wuffs_base__make_slice_u8(dec_workbuf, dec_workbuf_len); + for (size_t j = 0; j < g_num_dicts; j++) { + wuffs_base__status status = wuffs_brotli__decoder__add_dictionary( + dec, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, + wuffs_base__make_slice_u8((uint8_t*)g_dict_ptrs[j], g_dict_lens[j]), wb); + if (status.repr) { + ret = status.repr; + goto cleanup; + } + } + uint64_t wlen = wuffs_brotli__decoder__workbuf_len(dec).max_incl; + if (wlen > dec_workbuf_len) { + ret = "decoder workbuf_len is too large"; + goto cleanup; + } + wuffs_base__io_buffer dst = wuffs_base__ptr_u8__writer(out, MAX_INPUT); + wuffs_base__io_buffer src = wuffs_base__ptr_u8__reader(g_want, want_len, true); + wuffs_base__status status = wuffs_brotli__decoder__transform_io( + dec, &dst, &src, wuffs_base__make_slice_u8(dec_workbuf, (size_t)wlen)); + if (status.repr) { + ret = status.repr; + } else if ((dst.meta.wi != len) || (len && memcmp(out, data, len))) { + ret = "Wuffs decoder round trip mismatch"; + } +cleanup: + free(dec); + return ret; } static config // @@ -938,8 +1161,316 @@ run_input_q2(const char* name, const uint8_t* data, size_t len, uint32_t q) { } } +// run_dict_configs runs the dictionary grid's configurations for one +// quality and the current dictionaries. level 0 is the full grid, level 1 a +// smaller one and level 2 a much smaller one (for long inputs). +static void // +run_dict_configs(const char* name, const uint8_t* data, size_t len, uint32_t q, + int level) { + static const uint32_t lgwins[] = {10, 16, 22, 24}; + for (size_t w = 0; w < 4; w++) { + for (uint32_t ch = 0; ch < 5; ch += 4) { + for (uint32_t fl = 0; fl < 3; fl += 2) { + if ((level >= 1) && ((w == 1) || ((ch == 0) && (fl == 2)))) { + continue; + } else if ((level >= 2) && ((w == 0) || (ch != fl * 2))) { + continue; + } + config c = make_config(q, lgwins[w]); + c.chunking = ch; + c.flush = fl; + c.seed = 0x60000 | (w << 8) | (ch << 4) | fl | (q << 12); + check(name, &c, data, len); + } + } + } + + static const uint32_t extras[][6] = { + // Pairs of (BrotliEncoderParameter, value). A zero key ends the list. + {5, 0xFFFFFFFF}, // BROTLI_PARAM_SIZE_HINT: the exact size. + {5, 1u << 20}, // H54 (quality 4), H6 or H68. + {5, 1}, // + {9, 1}, // BROTLI_PARAM_STREAM_OFFSET. + {9, 1000}, // + {9, 1u << 30}, // + {6, 1, 2, 18}, // BROTLI_PARAM_LARGE_WINDOW. + {6, 1, 2, 30}, // + {6, 1, 2, 25, 5, 1u << 20}, // H35, H55 or H65. + {9, 1000, 6, 1, 2, 26}, // Stream offset and large window. + {3, 16}, // BROTLI_PARAM_LGBLOCK. + {3, 24, 5, 1u << 21}, // + {7, 2, 8, 12}, // BROTLI_PARAM_NPOSTFIX and NDIRECT. + {0, 2}, // FONT mode. + {4, 1}, // DISABLE_LITERAL_CONTEXT_MODELING. + }; + for (size_t i = 0; i < sizeof extras / sizeof extras[0]; i++) { + for (uint32_t ch = 0; ch < 5; ch += 4) { + if ((level >= 1) && (ch == 0)) { + continue; + } else if ((level >= 2) && (i != 1) && (i != 4) && (i != 8)) { + continue; + } + config c = make_config(q, 22); + for (size_t k = 0; (k < 6) && ((k == 0) || extras[i][k]); k += 2) { + uint32_t v = extras[i][k + 1]; + if ((extras[i][k] == 5) && (v == 0xFFFFFFFF)) { + v = (uint32_t)len; + } + add_param(&c, extras[i][k], v); + } + c.chunking = ch; + c.flush = 2; + c.seed = 0x70000 | i | (ch << 8) | (q << 12); + check(name, &c, data, len); + } + } + if (level >= 2) { + return; + } + + // Metadata blocks, limited dst lengths and a flush after every chunk. + for (uint32_t pos = 1; pos <= 2; pos++) { + config c = make_config(q, 18); + c.chunking = 4; + c.flush = 2; + c.metadata_position = pos; + c.metadata_len = (pos == 1) ? 0 : 100; + c.seed = 0x80000 | pos | (q << 12); + check(name, &c, data, len); + } + { + config c = make_config(q, 16); + c.chunking = 4; + c.flush = 0; + c.dst_chunk = (len <= (32 << 10)) ? 3 : 4096; + c.seed = 0x90000 | (q << 12); + check(name, &c, data, len); + } + if (len <= (8 << 10)) { + config c = make_config(q, 22); + c.chunking = 2; + c.flush = 1; + c.seed = 0xA0000 | (q << 12); + check(name, &c, data, len); + } +} + +// check_smaller checks that, with the current dictionaries (which should +// hold much of data), libbrotlienc's output (which Wuffs' output matches) is +// smaller than without them. +static void // +check_smaller(const char* name, const uint8_t* data, size_t len, uint32_t q) { + config c = make_config(q, 22); + check(name, &c, data, len); + size_t with_len = g_last_want_len; + size_t saved = g_num_dicts; + size_t without_len = 0; + g_num_dicts = 0; + const char* err = make_plan(&c, data, len); + if (!err) { + err = encode_mimic(&c, &without_len); + } + g_num_dicts = saved; + g_num_smaller_checks++; + if (err || (with_len >= without_len)) { + g_num_mismatches++; + printf("NOT SMALLER %s: q=%u dicts=%s: %zu bytes with, %zu without: %s\n", + name, q, g_dict_desc, with_len, without_len, err ? err : ""); + fflush(stdout); + } +} + +// run_input_dict runs the dictionary grid (several sets of raw dictionaries, +// mostly made from the input, so that there are matches) for an input. +static void // +run_input_dict(const char* name, const uint8_t* data, size_t len) { + uint64_t configs0 = g_num_configs; + uint64_t mismatches0 = g_num_mismatches; + time_t start = time(NULL); + static uint8_t noise[1000]; + fill_random(noise, sizeof noise, 0xD1C7); + static const char text[] = "the quick brown fox jumps over the lazy dog"; + bool big = len > (256 << 10); + + for (int set = 0; set < 7; set++) { + const uint8_t* ptrs[MAX_DICTS]; + size_t lens[MAX_DICTS]; + size_t n = 0; + int level = big ? 2 : 1; + switch (set) { + case 0: // A prefix of the input. + ptrs[n] = data; + lens[n++] = len / 2; + snprintf(g_dict_desc, sizeof g_dict_desc, "prefix"); + level = big ? 2 : 0; + break; + case 1: // The whole input. + ptrs[n] = data; + lens[n++] = len; + snprintf(g_dict_desc, sizeof g_dict_desc, "whole"); + level = big ? 2 : 0; + break; + case 2: // Another file. + ptrs[n] = g_foreign_dict; + lens[n++] = g_foreign_dict_len; + snprintf(g_dict_desc, sizeof g_dict_desc, "enwik5"); + break; + case 3: // Noise and the input's middle third. + ptrs[n] = noise; + lens[n++] = sizeof noise; + ptrs[n] = data + (len / 3); + lens[n++] = (len / 3); + snprintf(g_dict_desc, sizeof g_dict_desc, "noise+middle"); + break; + case 4: { // 15 (the maximum number of) pieces of the input. + if (len == 0) { + continue; + } + size_t k = (len + 14) / 15; + for (size_t j = 0; j < 15; j++) { + size_t at = (j * k) % len; + ptrs[n] = data + at; + lens[n++] = ((len - at) < k) ? (len - at) : k; + } + snprintf(g_dict_desc, sizeof g_dict_desc, "15 pieces of %zu", k); + break; + } + case 5: // Tiny dictionaries (and an empty one, first). + ptrs[n] = (const uint8_t*)text; + lens[n++] = 0; + ptrs[n] = (const uint8_t*)text; + lens[n++] = 1; + ptrs[n] = (const uint8_t*)text + 4; + lens[n++] = 7; + ptrs[n] = (const uint8_t*)text + 10; + lens[n++] = 8; + ptrs[n] = (len >= 9) ? data + (len - 9) : (const uint8_t*)text; + lens[n++] = 9; + snprintf(g_dict_desc, sizeof g_dict_desc, "tiny"); + break; + case 6: // The input's two halves, swapped. + ptrs[n] = data + (len / 2); + lens[n++] = len - (len / 2); + ptrs[n] = data; + lens[n++] = len / 2; + ptrs[n] = (const uint8_t*)text; + lens[n++] = sizeof text - 1; + snprintf(g_dict_desc, sizeof g_dict_desc, "swapped halves+text"); + break; + } + const char* err = set_dicts(n, ptrs, lens); + if (err) { + g_num_mismatches++; + printf("FAILURE %s: set_dicts(%s): %s\n", name, g_dict_desc, err); + continue; + } + for (uint32_t q = g_qmin; q <= g_qmax; q++) { + run_dict_configs(name, data, len, q, level); + if ((set == 1) && (len >= 1000) && (q >= 5)) { + check_smaller(name, data, len, q); + } + } + } + set_dicts(0, NULL, NULL); + printf("%-50s %8zu bytes: %6" PRIu64 " configs, %" PRIu64 + " mismatches (%lds)\n", + name, len, g_num_configs - configs0, g_num_mismatches - mismatches0, + (long)(time(NULL) - start)); + fflush(stdout); +} + +// run_big_dicts checks dictionaries of 2 to 40 MiB (every hash table size +// class of CreatePreparedDictionary), with pieces of a 1 MiB input copied +// into them at various offsets. +static void // +run_big_dicts(size_t which) { + static const size_t sizes[] = { + (2 << 20), (2 << 20) + 1, (3 << 20) + 77, (5 << 20) + 3, + (9 << 20) + 5, (17 << 20) + 1, (40 << 20) + 9, + }; + if (which >= sizeof sizes / sizeof sizes[0]) { + return; + } + uint64_t configs0 = g_num_configs; + uint64_t mismatches0 = g_num_mismatches; + time_t start = time(NULL); + size_t dlen = sizes[which]; + size_t len = 1 << 20; + uint8_t* input = malloc(len); + uint8_t* dict = malloc(dlen); + if (!input || !dict) { + printf("FAILURE: malloc failed\n"); + g_num_mismatches++; + free(input); + free(dict); + return; + } + fill_mixed(input, len, 0xB16 + which); + fill_mixed(dict, dlen, 0xD1C + which); + // Copy 64 KiB pieces of the input to the dictionary's start, middle and + // end (with sparse changes). + for (size_t j = 0; j < 3; j++) { + size_t at = (j == 0) ? 1234 : (j == 1) ? (dlen / 2) : (dlen - (70 << 10)); + memcpy(dict + at, input + (j * (300 << 10)), 64 << 10); + for (size_t i = 777; i < (64 << 10); i += 4099) { + dict[at + i] ^= 0x33; + } + } + char name[96]; + snprintf(name, sizeof name, "synthetic: 1 MiB with a %zu byte dictionary", + dlen); + const uint8_t* ptrs[2] = {dict, input + (900 << 10)}; + size_t lens[2] = {dlen, 1000}; + snprintf(g_dict_desc, sizeof g_dict_desc, "big %zu + 1000", dlen); + const char* err = set_dicts(2, ptrs, lens); + if (err) { + g_num_mismatches++; + printf("FAILURE %s: set_dicts: %s\n", name, err); + } else { + for (uint32_t q = (g_qmin > 2) ? g_qmin : 2; q <= g_qmax; q++) { + static const uint32_t lgwins[] = {16, 24}; + for (size_t w = 0; w < 2; w++) { + config c = make_config(q, lgwins[w]); + c.seed = 0xB0000 | (w << 8) | (q << 12); + check(name, &c, input, len); + c.chunking = 4; + c.flush = 2; + check(name, &c, input, len); + } + static const uint32_t extras[][6] = { + {6, 1, 2, 30, 5, 1u << 20}, + {9, 1000}, + {5, 1u << 20}, + }; + for (size_t i = 0; i < 3; i++) { + config c = make_config(q, 22); + for (size_t k = 0; (k < 6) && ((k == 0) || extras[i][k]); k += 2) { + add_param(&c, extras[i][k], extras[i][k + 1]); + } + c.seed = 0xC0000 | i | (q << 12); + check(name, &c, input, len); + } + if (q >= 5) { + check_smaller(name, input, len, q); + } + } + } + set_dicts(0, NULL, NULL); + free(input); + free(dict); + printf("%-50s %8zu bytes: %6" PRIu64 " configs, %" PRIu64 + " mismatches (%lds)\n", + name, len, g_num_configs - configs0, g_num_mismatches - mismatches0, + (long)(time(NULL) - start)); + fflush(stdout); +} + static void // run_input(const char* name, const uint8_t* data, size_t len) { + if (g_dict_only) { + run_input_dict(name, data, len); + return; + } uint64_t configs0 = g_num_configs; uint64_t mismatches0 = g_num_mismatches; time_t start = time(NULL); @@ -1044,6 +1575,8 @@ main(int argc, char** argv) { g_qmax = (b < 9) ? b : 9; } else if (strcmp(argv[argi], "-lw") == 0) { g_lw_only = true; + } else if (strcmp(argv[argi], "-dict") == 0) { + g_dict_only = true; } else if (strcmp(argv[argi], "-big") == 0) { big = true; } else if (sscanf(argv[argi], "-shard=%u/%u", &a, &b) == 2) { @@ -1089,7 +1622,20 @@ main(int argc, char** argv) { compare_strings); } + g_foreign_dict = read_whole_file("test/data/enwik5", &g_foreign_dict_len); + if (!g_foreign_dict) { + fprintf(stderr, "could not read test/data/enwik5\n"); + return 1; + } + int input_index = 0; + if (g_dict_only && (argi >= argc)) { + for (size_t j = 0; j < 7; j++) { + if (((input_index++) % shard_n) == shard_k) { + run_big_dicts(j); + } + } + } if (big) { // Synthetic inputs of several MiB (with the usual grid) and of 24 to 70 // MiB (with run_input_far's configurations), with repeats at distances @@ -1179,7 +1725,9 @@ main(int argc, char** argv) { } printf("TOTAL: %" PRIu64 " configs (%" PRIu64 - " also with a short workbuf), %" PRIu64 " mismatches\n", - g_num_configs, g_num_short_checks, g_num_mismatches); + " also with a short workbuf, %" PRIu64 " round trips, %" PRIu64 + " smaller-with-dictionary checks), %" PRIu64 " mismatches\n", + g_num_configs, g_num_short_checks, g_num_roundtrips, + g_num_smaller_checks, g_num_mismatches); return g_num_mismatches ? 1 : 0; } diff --git a/std/brotli/encode_backward_references.wuffs b/std/brotli/encode_backward_references.wuffs index 02a52079..6d9d84c4 100644 --- a/std/brotli/encode_backward_references.wuffs +++ b/std/brotli/encode_backward_references.wuffs @@ -224,10 +224,11 @@ pri func encoder.compute_distance_code(distance: base.u64, max_distance: base.u6 } // Mirrors CreateBackwardReferences in backward_references_inc.h (via -// BrotliCreateBackwardReferences in backward_references.c), without a -// compound dictionary and with the built-in static dictionary. New commands -// are appended at this.num_cmds. -pri func encoder.create_backward_references!(wb: slice base.u8, num_bytes: base.u64, position: base.u64) base.status { +// BrotliCreateBackwardReferences in backward_references.c), with the +// built-in static dictionary. Its ENABLE_COMPOUND_DICTIONARY variant (see +// compound_dictionary_enabled) searches the attached dictionaries (in +// args.dwb). New commands are appended at this.num_cmds. +pri func encoder.create_backward_references!(wb: slice base.u8, dwb: roslice base.u8, num_bytes: base.u64, position: base.u64) base.status { var position : base.u64 var max_backward_limit : base.u64 var position_offset : base.u64 @@ -250,6 +251,9 @@ pri func encoder.create_backward_references!(wb: slice base.u8, num_bytes: base. var range_end : base.u64 var k_margin : base.u64 var pos_jump : base.u64 + var gap : base.u64 + var use_compound : base.bool + var data : roslice base.u8 position = args.position max_backward_limit = ((1 as base.u64) << this.lgwin) ~mod- 16 @@ -266,6 +270,11 @@ pri func encoder.create_backward_references!(wb: slice base.u8, num_bytes: base. random_heuristics_window_size = 512 } apply_random_heuristics = position ~mod+ random_heuristics_window_size + gap = this.dict_total_size + use_compound = this.compound_dictionary_enabled() + if this.rb_buf_off <= args.wb.length() { + data = args.wb[this.rb_buf_off ..] + } this.prepare_distance_cache!() @@ -278,8 +287,13 @@ pri func encoder.create_backward_references!(wb: slice base.u8, num_bytes: base. this.sr_distance = 0 this.sr_score = SCORE_BASE + 100 this.find_longest_match!(wb: args.wb, cur_ix: position, max_length: max_length, - max_backward: max_distance, dictionary_distance: dictionary_start, + max_backward: max_distance, dictionary_distance: dictionary_start ~mod+ gap, max_distance: this.dist_max_distance) + if use_compound { + this.lookup_compound_dictionary_match!(data: data, dwb: args.dwb, + cur_ix: position, max_length: max_length, + max_ring_buffer_distance: dictionary_start, max_distance: this.dist_max_distance) + } if this.sr_score > (SCORE_BASE + 100) { // Found a match. Let's look for something even better ahead. delayed_backward_references = 0 @@ -299,8 +313,13 @@ pri func encoder.create_backward_references!(wb: slice base.u8, num_bytes: base. max_distance = this.min64(a: position ~mod+ 1, b: max_backward_limit) dictionary_start = this.min64(a: (position ~mod+ 1) ~mod+ position_offset, b: max_backward_limit) this.find_longest_match!(wb: args.wb, cur_ix: position ~mod+ 1, max_length: max_length, - max_backward: max_distance, dictionary_distance: dictionary_start, + max_backward: max_distance, dictionary_distance: dictionary_start ~mod+ gap, max_distance: this.dist_max_distance) + if use_compound { + this.lookup_compound_dictionary_match!(data: data, dwb: args.dwb, + cur_ix: position ~mod+ 1, max_length: max_length, + max_ring_buffer_distance: dictionary_start, max_distance: this.dist_max_distance) + } if this.sr_score >= (sr_score ~mod+ 175) { // Ok, let's just write one byte for now and start a match // from the next byte. @@ -324,8 +343,8 @@ pri func encoder.create_backward_references!(wb: slice base.u8, num_bytes: base. dictionary_start = this.min64(a: position ~mod+ position_offset, b: max_backward_limit) // The first 16 codes are special short-codes, and the minimum // offset is 1. - distance_code = this.compute_distance_code(distance: this.sr_distance, max_distance: dictionary_start) - if (this.sr_distance <= dictionary_start) and (distance_code > 0) { + distance_code = this.compute_distance_code(distance: this.sr_distance, max_distance: dictionary_start ~mod+ gap) + if (this.sr_distance <= (dictionary_start ~mod+ gap)) and (distance_code > 0) { this.dist_cache[3] = this.dist_cache[2] this.dist_cache[2] = this.dist_cache[1] this.dist_cache[1] = this.dist_cache[0] diff --git a/std/brotli/encode_brotli.wuffs b/std/brotli/encode_brotli.wuffs index 48edc6bb..11187b4a 100644 --- a/std/brotli/encode_brotli.wuffs +++ b/std/brotli/encode_brotli.wuffs @@ -194,6 +194,19 @@ pub struct encoder? implements base.io_transformer( hr_next_ix : base.u64, hr_off : base.u64, + // The attached raw dictionaries (the C code's CompoundDictionary, see + // encode_dictionary.wuffs). They use the first dict_end bytes of the + // workbuf. + dict_num_chunks : base.u32[..= 15], + dict_total_size : base.u64, + dict_end : base.u64, + dict_chunk_offsets : array[16] base.u64, + dc_src_off : array[15] base.u64, + dc_size : array[15] base.u64, + dc_bucket_bits : array[15] base.u32, + dc_slot_bits : array[15] base.u32, + dc_num_items : array[15] base.u64, + // The C code's HasherSearchResult. sr_len : base.u64, sr_distance : base.u64, @@ -328,6 +341,12 @@ pub struct encoder? implements base.io_transformer( // GetHashTable result), with up to (1 << 17) elements. hash_table : array[131072] base.u32, + // CreatePreparedDictionaryWithParams' slot_size and slot_limit + // arrays, and its slot_offsets (see encode_dictionary.wuffs). + dp_slot_size : array[4096] base.u32, + dp_slot_limit : array[4096] base.u32, + dp_slot_off : array[4096] base.u32, + // command_buf and literal_buf are the C code's command_buf_ and // literal_buf_ (quality 1 only). command_buf : array[131072] base.u32, @@ -437,7 +456,10 @@ pub func encoder.workbuf_len() base.range_ii_u64 { } else if q <= 9 { n = this.compute_layout_offset(quality: q, lgwin: lgwin, lgblock: this.computed_lgblock(quality: q, lgwin: lgwin), idx: LAYOUT_END) + } else { + return this.util.make_range_ii_u64(min_incl: 0, max_incl: 0) } + n ~sat+= this.dict_end return this.util.make_range_ii_u64(min_incl: n, max_incl: n) } @@ -799,7 +821,31 @@ pri func encoder.write_metadata_header!(block_size: base.u32) base.u64 { // qualities 0 and 1, BrotliEncoderCompressStreamFast). Each (initial or // resumed) transform_io call corresponds to one or more C calls. See // README.md for the details. +// +// The attached dictionaries (see add_dictionary) are the first dict_end +// bytes of the workbuf. do_transform_io gets the rest of the workbuf (whose +// layout is described in README.md and encode_data.wuffs) and, as args.dwb, +// the dictionaries. pub func encoder.transform_io?(dst: base.io_writer, src: base.io_reader, workbuf: slice base.u8) { + var status : base.status + + while true { + if this.dict_end > args.workbuf.length() { + return base."#bad workbuf length" + } + status =? this.do_transform_io?(dst: args.dst, src: args.src, + workbuf: args.workbuf[this.dict_end ..], + dwb: args.workbuf[.. this.dict_end]) + if status.is_ok() { + return ok + } else if not status.is_suspension() { + return status + } + yield? status + } +} + +pri func encoder.do_transform_io?(dst: base.io_writer, src: base.io_reader, workbuf: slice base.u8, dwb: roslice base.u8) { var op : base.u32 var avail : base.u64 var block_size : base.u64 @@ -863,7 +909,7 @@ pub func encoder.transform_io?(dst: base.io_writer, src: base.io_reader, workbuf continue } if (this.quality >= 2) and (this.input_pos <> this.last_flush_pos) { - status = this.encode_data!(wb: args.workbuf, is_last: false, force_flush: true) + status = this.encode_data!(wb: args.workbuf, dwb: args.dwb, is_last: false, force_flush: true) if not status.is_ok() { return status } @@ -979,7 +1025,7 @@ pub func encoder.transform_io?(dst: base.io_writer, src: base.io_reader, workbuf force_flush = true } this.update_size_hint!(available_in: args.src.length()) - status = this.encode_data!(wb: args.workbuf, is_last: is_last, force_flush: force_flush) + status = this.encode_data!(wb: args.workbuf, dwb: args.dwb, is_last: is_last, force_flush: force_flush) if not status.is_ok() { return status } diff --git a/std/brotli/encode_data.wuffs b/std/brotli/encode_data.wuffs index 7ce4ecba..de92d95d 100644 --- a/std/brotli/encode_data.wuffs +++ b/std/brotli/encode_data.wuffs @@ -474,10 +474,11 @@ pri func encoder.update_size_hint!(available_in: base.u64) { } } -// Mirrors ExtendLastCommand in encode.c (without a compound dictionary). It -// returns how many bytes it extended the last command by: the caller -// subtracts that from bytes and adds it to wrapped_last_processed_pos. -pri func encoder.extend_last_command!(wb: slice base.u8, bytes: base.u64, wrapped_last_processed_pos: base.u64) base.u64 { +// Mirrors ExtendLastCommand in encode.c (see extend_last_command_compound +// for its compound dictionary branch). It returns how many bytes it extended +// the last command by: the caller subtracts that from bytes and adds it to +// wrapped_last_processed_pos. +pri func encoder.extend_last_command!(wb: slice base.u8, dwb: roslice base.u8, bytes: base.u64, wrapped_last_processed_pos: base.u64) base.u64 { var last : base.u64 var max_backward_distance : base.u64 var copy_len : base.u32 @@ -506,6 +507,11 @@ pri func encoder.extend_last_command!(wb: slice base.u8, bytes: base.u64, wrappe n ~mod+= 1 p = (p ~mod+ 1) & 0xFFFF_FFFF } + } else { + n = this.extend_last_command_compound(wb: args.wb, dwb: args.dwb, bytes: args.bytes, + wrapped_last_processed_pos: args.wrapped_last_processed_pos, + cmd_dist: cmd_dist, max_distance: max_distance, last_copy_len: last_copy_len) + copy_len ~mod+= (n & 0xFFFF_FFFF) as base.u32 } // The copy length is at most the metablock size, and thus // expressible. @@ -857,7 +863,7 @@ pri func encoder.write_meta_block_internal!(wb: slice base.u8, last_flush_pos: b // Mirrors EncodeData in encode.c, for qualities 2 to 9. On a new metablock, // it sets the pending output (this.out_etc) to the storage (or tiny_buf). -pri func encoder.encode_data!(wb: slice base.u8, is_last: base.bool, force_flush: base.bool) base.status { +pri func encoder.encode_data!(wb: slice base.u8, dwb: roslice base.u8, is_last: base.bool, force_flush: base.bool) base.status { var status : base.status var delta : base.u64 var bytes : base.u64 @@ -916,13 +922,13 @@ pri func encoder.encode_data!(wb: slice base.u8, is_last: base.bool, force_flush this.hasher_stitch_to_previous_block!(wb: args.wb, num_bytes: bytes, position: wrapped_last_processed_pos) if (this.num_cmds > 0) and (this.last_insert_len == 0) { - n = this.extend_last_command!(wb: args.wb, bytes: bytes, + n = this.extend_last_command!(wb: args.wb, dwb: args.dwb, bytes: bytes, wrapped_last_processed_pos: wrapped_last_processed_pos) bytes ~mod-= n wrapped_last_processed_pos = (wrapped_last_processed_pos ~mod+ n) & 0xFFFF_FFFF } - status = this.create_backward_references!(wb: args.wb, num_bytes: bytes, position: wrapped_last_processed_pos) + status = this.create_backward_references!(wb: args.wb, dwb: args.dwb, num_bytes: bytes, position: wrapped_last_processed_pos) if not status.is_ok() { return status } diff --git a/std/brotli/encode_dictionary.wuffs b/std/brotli/encode_dictionary.wuffs new file mode 100644 index 00000000..4ada69a8 --- /dev/null +++ b/std/brotli/encode_dictionary.wuffs @@ -0,0 +1,568 @@ +// Copyright 2026 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. +// +// SPDX-License-Identifier: Apache-2.0 OR MIT + +// -------- + +// This file mirrors the encoder's raw (prefix) dictionaries: compound +// dictionary.c (CreatePreparedDictionary and AttachPreparedDictionary), the +// BROTLI_SHARED_DICTIONARY_RAW cases of BrotliEncoderPrepareDictionary and +// BrotliEncoderAttachPreparedDictionary in encode.c and the compound +// dictionary search of hash.h (FindCompoundDictionaryMatch and +// LookupCompoundDictionaryMatch). +// +// The dictionaries live at the start of the workbuf (see README.md). Each +// one (a "chunk", the C code's PreparedDictionary) is, starting at its +// dc_src_off workbuf offset: +// - the dictionary bytes (the C code's source, dc_size bytes), +// - slot_offsets: (1 << dc_slot_bits) u32le values, +// - heads: (1 << dc_bucket_bits) u16le values and +// - items: dc_num_items u32le values. + +pub status "#unsupported serialized encoder dictionary" + +// ENCODER_DICTIONARY_CHUNKS_MAX mirrors SHARED_BROTLI_MAX_COMPOUND_DICTS in +// shared_dictionary.h. +pri const ENCODER_DICTIONARY_CHUNKS_MAX : base.u32 = 15 + +// Mirrors kPreparedDictionaryHashMul64Long in compound_dictionary.h. +pri const PREPARED_DICTIONARY_HASH_MUL_64_LONG : base.u64 = 0x1FE3_5A7B_D357_9BD3 + +// add_dictionary attaches a raw (LZ77 prefix) dictionary, like +// BrotliEncoderPrepareDictionary (with BROTLI_SHARED_DICTIONARY_RAW) +// followed by BrotliEncoderAttachPreparedDictionary in encode.c. It must be +// called before the first transform_io call. It can be called up to 15 times +// (the C code's SHARED_BROTLI_MAX_COMPOUND_DICTS). +// +// args.dict_type is DICTIONARY_TYPE_RAW or DICTIONARY_TYPE_SERIALIZED. Like +// a default (not BROTLI_EXPERIMENTAL) libbrotlienc build, whose +// BrotliEncoderPrepareDictionary returns NULL for a serialized dictionary, +// serialized dictionaries are rejected. +// +// Wuffs structs cannot hold pointers, so the dictionary's bytes are copied +// into the workbuf, followed by its prepared hash tables (the C code's +// PreparedDictionary), after the dictionaries added so far. args.workbuf must +// be at least add_dictionary_workbuf_len(args.dict.length()) bytes long +// (preparing the hash tables needs some scratch space after them) and every +// later add_dictionary and transform_io call must be passed the same workbuf +// (or at least one whose prefix holds the same bytes). After the last +// add_dictionary call, workbuf_len gives the workbuf length that +// transform_io needs. +// +// On failure, the encoder's state is unchanged. +pub func encoder.add_dictionary!(dict_type: base.u32, dict: roslice base.u8, workbuf: slice base.u8) base.status { + var n : base.u64 + var lo : base.u64 + var hi : base.u64 + var bucket_bits : base.u32[..= 22] + var slot_bits : base.u32[..= 12] + var num_items : base.u64 + var c : base.u32 + + if this.is_initialized { + return base."#bad call sequence" + } else if args.dict_type == DICTIONARY_TYPE_SERIALIZED { + return "#unsupported serialized encoder dictionary" + } else if args.dict_type <> DICTIONARY_TYPE_RAW { + return base."#bad argument" + } else if this.dict_num_chunks >= ENCODER_DICTIONARY_CHUNKS_MAX { + // Mirrors AttachPreparedDictionary in compound_dictionary.c. + return "#too many compound dictionaries" + } + n = args.dict.length() + if (n > DICT_TOTAL_MAX) or ((this.dict_total_size ~sat+ n) > DICT_TOTAL_MAX) { + return "#too much dictionary data" + } else if this.add_dictionary_workbuf_len(dict_len: n) > args.workbuf.length() { + return base."#bad workbuf length" + } + + // Mirrors CreatePreparedDictionary in compound_dictionary.c. + bucket_bits = 17 + while (((0x10 as base.u64) << bucket_bits) < n) and (bucket_bits < 22) { + bucket_bits += 1 + } + if bucket_bits >= 10 { + slot_bits = bucket_bits - 10 + } + + // Copy the dictionary bytes into the workbuf. + lo = this.dict_end + hi = lo ~mod+ n + if (lo > hi) or (hi > args.workbuf.length()) { + return "#internal error: inconsistent encoder state" + } + args.workbuf[lo .. hi].copy_from_slice!(s: args.dict) + + num_items = this.prepare_dictionary!(wb: args.workbuf, src_off: lo, n: n, + bucket_bits: bucket_bits, slot_bits: slot_bits) + + // Mirrors AttachPreparedDictionary in compound_dictionary.c. + c = this.dict_num_chunks + if c >= 15 { + return "#internal error: inconsistent encoder state" + } + this.dc_src_off[c] = lo + this.dc_size[c] = n + this.dc_bucket_bits[c] = bucket_bits + this.dc_slot_bits[c] = slot_bits + this.dc_num_items[c] = num_items + this.dict_total_size ~mod+= n + this.dict_chunk_offsets[c + 1] = this.dict_total_size + this.dict_num_chunks = c + 1 + this.dict_end = ((hi ~mod+ ((4 as base.u64) << slot_bits)) ~mod+ + ((2 as base.u64) << bucket_bits)) ~mod+ (4 ~mod* num_items) + return ok +} + +// add_dictionary_workbuf_len returns the workbuf length that an +// add_dictionary call, for a dictionary of dict_len bytes, needs: the +// dictionaries added so far, plus that dictionary's bytes, its hash tables +// (assuming the most items) and the scratch space for preparing them. It +// returns 0xFFFF_FFFF_FFFF_FFFF if dict_len is too large. +pub func encoder.add_dictionary_workbuf_len(dict_len: base.u64) base.u64 { + var bucket_bits : base.u32[..= 22] + var slot_bits : base.u32[..= 12] + var n : base.u64 + var max_items : base.u64 + + if args.dict_len > DICT_TOTAL_MAX { + return 0xFFFF_FFFF_FFFF_FFFF + } + n = args.dict_len + bucket_bits = 17 + while (((0x10 as base.u64) << bucket_bits) < n) and (bucket_bits < 22) { + bucket_bits += 1 + } + if bucket_bits >= 10 { + slot_bits = bucket_bits - 10 + } + if n > 7 { + max_items = n - 7 + } + // The dictionary bytes, slot_offsets, heads and (at most) items, then the + // scratch space: num (u16), bucket_heads (u32) and next_bucket (u32). + return ((((this.dict_end ~sat+ n) ~sat+ ((4 as base.u64) << slot_bits)) ~sat+ + ((2 as base.u64) << bucket_bits)) ~sat+ (4 ~mod* max_items)) ~sat+ + (((6 as base.u64) << bucket_bits) ~sat+ (4 ~mod* n)) +} + +// prepare_dictionary mirrors CreatePreparedDictionaryWithParams in +// compound_dictionary.c, with hash_bits 40 and bucket_limit 32, for the n +// dictionary bytes at args.wb[args.src_off ..]. It writes the slot_offsets, +// heads and items arrays after those bytes (with the scratch space after the +// largest possible items array) and returns the number of items. +pri func encoder.prepare_dictionary!(wb: slice base.u8, src_off: base.u64, n: base.u64, bucket_bits: base.u32[..= 22], slot_bits: base.u32[..= 12]) base.u64 { + var num_slots : base.u64 + var num_buckets : base.u64 + var slot_mask : base.u64 + var slot_off : base.u64 + var heads_off : base.u64 + var items_off : base.u64 + var num_off : base.u64 + var bh_off : base.u64 + var nb_off : base.u64 + var max_items : base.u64 + var i : base.u64 + var j : base.u64 + var h : base.u64 + var key : base.u64 + var count : base.u32 + var limit : base.u32 + var size : base.u32 + var overflow : base.bool + var total_items : base.u64 + var slot : base.u64 + var cursor : base.u64 + var pos : base.u64 + var k : base.u32 + + num_slots = (1 as base.u64) << args.slot_bits + num_buckets = (1 as base.u64) << args.bucket_bits + slot_mask = num_slots - 1 + if args.n > 7 { + max_items = args.n - 7 + } + slot_off = args.src_off ~mod+ args.n + heads_off = slot_off ~mod+ (4 ~mod* num_slots) + items_off = heads_off ~mod+ (2 ~mod* num_buckets) + num_off = items_off ~mod+ (4 ~mod* max_items) + bh_off = num_off ~mod+ (2 ~mod* num_buckets) + nb_off = bh_off ~mod+ (4 ~mod* num_buckets) + + // Step 1: create "bloated" hasher. + this.fill_bytes!(s: args.wb, i: num_off, n: 2 ~mod* num_buckets, v: 0) + i = 0 + while (i ~mod+ 7) < args.n { + h = (this.load64le(s: args.wb, i: args.src_off ~mod+ i) & 0xFF_FFFF_FFFF) ~mod* + PREPARED_DICTIONARY_HASH_MUL_64_LONG + key = (h >> 32) >> (32 - args.bucket_bits) + count = this.peek_u16(s: args.wb, i: num_off ~mod+ (2 ~mod* key)) as base.u32 + if count == 0 { + this.poke_u32!(s: args.wb, i: nb_off ~mod+ (4 ~mod* i), v: 0xFFFF_FFFF) + } else { + this.poke_u32!(s: args.wb, i: nb_off ~mod+ (4 ~mod* i), + v: this.load32le(s: args.wb, i: bh_off ~mod+ (4 ~mod* key))) + } + this.poke_u32!(s: args.wb, i: bh_off ~mod+ (4 ~mod* key), v: (i & 0xFFFF_FFFF) as base.u32) + count ~mod+= 1 + if count > 32 { + count = 32 + } + this.poke_u16!(s: args.wb, i: num_off ~mod+ (2 ~mod* key), v: (count & 0xFFFF) as base.u16) + i ~mod+= 1 + } + + // Step 2: find slot limits. + i = 0 + while i < num_slots { + limit = 32 + while true { + count = 0 + overflow = false + j = i + while j < num_buckets { + size = this.peek_u16(s: args.wb, i: num_off ~mod+ (2 ~mod* j)) as base.u32 + // Last chain may span behind 64K limit; overflow happens + // only if we are about to use 0xFFFF+ as item offset. + if count >= 0xFFFF { + overflow = true + break + } + if size > limit { + size = limit + } + count ~mod+= size + j ~mod+= num_slots + } + if not overflow { + this.dp_slot_size[i & 4095] = count + total_items ~mod+= count as base.u64 + break + } else if limit == 0 { + // Unreachable: with a zero limit, count stays zero. + this.dp_slot_size[i & 4095] = 0 + break + } + limit -= 1 + } + this.dp_slot_limit[i & 4095] = limit + i ~mod+= 1 + } + + // Step 3: transfer data to "slim" hasher. + total_items = 0 + i = 0 + while i < num_slots { + this.poke_u32!(s: args.wb, i: slot_off ~mod+ (4 ~mod* i), v: (total_items & 0xFFFF_FFFF) as base.u32) + this.dp_slot_off[i & 4095] = (total_items & 0xFFFF_FFFF) as base.u32 + total_items ~mod+= this.dp_slot_size[i & 4095] as base.u64 + this.dp_slot_size[i & 4095] = 0 + i ~mod+= 1 + } + i = 0 + while i < num_buckets { + slot = i & slot_mask & 4095 + count = this.peek_u16(s: args.wb, i: num_off ~mod+ (2 ~mod* i)) as base.u32 + if count > this.dp_slot_limit[slot] { + count = this.dp_slot_limit[slot] + } + if count == 0 { + this.poke_u16!(s: args.wb, i: heads_off ~mod+ (2 ~mod* i), v: 0xFFFF) + i ~mod+= 1 + continue + } + cursor = this.dp_slot_size[slot] as base.u64 + this.poke_u16!(s: args.wb, i: heads_off ~mod+ (2 ~mod* i), v: (cursor & 0xFFFF) as base.u16) + cursor ~mod+= this.dp_slot_off[slot] as base.u64 + this.dp_slot_size[slot] ~mod+= count + pos = this.load32le(s: args.wb, i: bh_off ~mod+ (4 ~mod* i)) as base.u64 + k = 0 + while k < count { + this.poke_u32!(s: args.wb, i: items_off ~mod+ (4 ~mod* cursor), v: (pos & 0xFFFF_FFFF) as base.u32) + cursor ~mod+= 1 + pos = this.load32le(s: args.wb, i: nb_off ~mod+ (4 ~mod* pos)) as base.u64 + k ~mod+= 1 + } + cursor ~mod-= 1 + this.poke_u32!(s: args.wb, i: items_off ~mod+ (4 ~mod* cursor), + v: this.load32le(s: args.wb, i: items_off ~mod+ (4 ~mod* cursor)) | 0x8000_0000) + i ~mod+= 1 + } + return total_items +} + +// compound_dictionary_enabled returns whether BrotliCreateBackwardReferences +// in backward_references.c uses its ENABLE_COMPOUND_DICTIONARY variant: when +// any dictionary is attached and the hasher is H5, H6, H58, H68, H40, H41, +// H42, H55 or H65 (H55 and H65 are the composite hashers H54 and H6 plus a +// rolling hasher). The other hashers (H2, H3, H4, H54 and H35) ignore the +// compound dictionary (but not its total size). +pri func encoder.compound_dictionary_enabled() base.bool { + var htype : base.u32 + + if this.dict_num_chunks == 0 { + return false + } + htype = this.hasher_type + return (htype == 5) or (htype == 6) or (htype == 58) or (htype == 68) or + (htype == 40) or (htype == 41) or (htype == 42) or + ((htype == 54) and (this.hr_jump <> 0)) +} + +// Mirrors LookupCompoundDictionaryMatch in hash.h. args.data is the ring +// buffer (the C code's data) and args.dwb holds the dictionaries. +pri func encoder.lookup_compound_dictionary_match!(data: roslice base.u8, dwb: roslice base.u8, cur_ix: base.u64, max_length: base.u64, max_ring_buffer_distance: base.u64, max_distance: base.u64) { + var base_offset : base.u64 + var d : base.u32 + + base_offset = args.max_ring_buffer_distance ~mod+ this.dict_total_size + d = 0 + while d < this.dict_num_chunks { + assert d < 15 via "a < b: a < c; c <= b"(c: this.dict_num_chunks) + this.find_compound_dictionary_match!(data: args.data, dwb: args.dwb, d: d, + cur_ix: args.cur_ix, max_length: args.max_length, + distance_offset: base_offset ~mod- this.dict_chunk_offsets[d], + max_distance: args.max_distance) + d += 1 + } +} + +// Mirrors FindCompoundDictionaryMatch in hash.h, for the d'th chunk. The +// search result is in (and starts from) the this.sr_etc fields. +pri func encoder.find_compound_dictionary_match!(data: roslice base.u8, dwb: roslice base.u8, d: base.u32[..= 14], cur_ix: base.u64, max_length: base.u64, distance_offset: base.u64, max_distance: base.u64) { + var source_size : base.u64 + var boundary : base.u64 + var src_off : base.u64 + var slot_off : base.u64 + var heads_off : base.u64 + var items_off : base.u64 + var source : roslice base.u8 + var cur : roslice base.u8 + var cur_ix_masked : base.u64 + var mask : base.u64 + var best_score : base.u64 + var best_len : base.u64 + var i : base.u32 + var h : base.u64 + var key : base.u64 + var slot : base.u64 + var head : base.u64 + var chain : base.u64 + var item : base.u32 + var distance : base.u64 + var offset : base.u64 + var limit : base.u64 + var len : base.u64 + var score : base.u64 + var num_items : base.u64 + var bb : base.u32[..= 22] + var sb : base.u32[..= 12] + + bb = this.dc_bucket_bits[args.d].min(no_more_than: 22) + sb = this.dc_slot_bits[args.d].min(no_more_than: 12) + num_items = this.dc_num_items[args.d] + source_size = this.dc_size[args.d] + boundary = args.distance_offset ~mod- source_size + src_off = this.dc_src_off[args.d] + slot_off = src_off ~mod+ source_size + heads_off = slot_off ~mod+ ((4 as base.u64) << sb) + items_off = heads_off ~mod+ ((2 as base.u64) << bb) + if (src_off > slot_off) or (slot_off > args.dwb.length()) { + return nothing + } + source = args.dwb[src_off .. slot_off] + + mask = this.rb_mask as base.u64 + cur_ix_masked = args.cur_ix & mask + if cur_ix_masked > args.data.length() { + return nothing + } + cur = args.data[cur_ix_masked ..] + best_score = this.sr_score + best_len = this.sr_len + h = (this.load64le(s: args.data, i: cur_ix_masked) & 0xFF_FFFF_FFFF) ~mod* + PREPARED_DICTIONARY_HASH_MUL_64_LONG + key = (h >> 32) >> (32 - bb) + slot = key & (((1 as base.u64) << sb) - 1) + head = this.peek_u16(s: args.dwb, i: heads_off ~mod+ (2 ~mod* key)) as base.u64 + chain = (this.load32le(s: args.dwb, i: slot_off ~mod+ (4 ~mod* slot)) as base.u64) ~mod+ head + item = 0 + if head == 0xFFFF { + item = 1 + } + + i = 0 + while i < 4 { + distance = this.dist_cache_u64(i: i) + if (distance <= boundary) or (distance > args.distance_offset) { + i ~mod+= 1 + continue + } + offset = args.distance_offset ~mod- distance + limit = source_size ~mod- offset + limit = limit.min(no_more_than: args.max_length) + len = 0 + if offset <= source.length() { + len = this.find_match_length_2(a: source[offset ..], b: cur, limit: limit) + } + if len >= 2 { + score = this.backward_reference_score_using_last_distance(copy_length: len) + if best_score < score { + if i <> 0 { + score ~mod-= this.backward_reference_penalty_using_last_distance(distance_short_code: i as base.u64) + } + if best_score < score { + best_score = score + if len > best_len { + best_len = len + } + this.sr_len = len + this.sr_len_code_delta = 0 + this.sr_distance = distance + this.sr_score = best_score + } + } + } + i ~mod+= 1 + } + // we require matches of len >4, so increase best_len to 3, so we can + // compare 4 bytes all the time. + if best_len < 3 { + best_len = 3 + } + while item == 0 { + if chain >= num_items { + // Unreachable: every chain ends with a 0x8000_0000 bit. + break + } + item = this.load32le(s: args.dwb, i: items_off ~mod+ (4 ~mod* chain)) + chain ~mod+= 1 + offset = (item & 0x7FFF_FFFF) as base.u64 + item &= 0x8000_0000 + distance = args.distance_offset ~mod- offset + limit = source_size ~mod- offset + limit = limit.min(no_more_than: args.max_length) + if distance > args.max_distance { + continue + } + if ((cur_ix_masked ~mod+ best_len) > mask) or (best_len >= limit) or + (this.load32le(s: args.data, i: (cur_ix_masked ~mod+ best_len) ~mod- 3) <> + this.load32le(s: source, i: (offset ~mod+ best_len) ~mod- 3)) { + continue + } + len = 0 + if offset <= source.length() { + len = this.find_match_length_2(a: source[offset ..], b: cur, limit: limit) + } + if len >= 4 { + score = this.backward_reference_score(copy_length: len, backward_reference_offset: distance) + if best_score < score { + best_score = score + best_len = len + this.sr_len = best_len + this.sr_len_code_delta = 0 + this.sr_distance = distance + this.sr_score = best_score + } + } + } +} + +// find_match_length_2 mirrors FindMatchLengthWithLimit in +// find_match_length.h: the length of the common prefix of a and b, up to +// limit bytes. +pri func encoder.find_match_length_2(a: roslice base.u8, b: roslice base.u8, limit: base.u64) base.u64 { + var matched : base.u64 + var n : base.u64 + var a : roslice base.u8 + var b : roslice base.u8 + var x : base.u64 + + a = args.a + b = args.b + n = args.limit + while (n >= 8) and (a.length() >= 8) and (b.length() >= 8) { + x = a.peek_u64le() ^ b.peek_u64le() + if x <> 0 { + while (x & 0xFF) == 0 { + matched ~mod+= 1 + x >>= 8 + } + return matched + } + matched ~mod+= 8 + n -= 8 + a = a[8 ..] + b = b[8 ..] + } + while (n > 0) and (a.length() > 0) and (b.length() > 0) { + if a[0] <> b[0] { + break + } + matched ~mod+= 1 + n -= 1 + a = a[1 ..] + b = b[1 ..] + } + return matched +} + +// extend_last_command_compound mirrors the compound dictionary branch of +// ExtendLastCommand in encode.c: extending the last command's copy (of +// distance cmd_dist, beyond max_distance) from the compound dictionary. It +// returns how many bytes it extended the copy by. +pri func encoder.extend_last_command_compound(wb: roslice base.u8, dwb: roslice base.u8, bytes: base.u64, wrapped_last_processed_pos: base.u64, cmd_dist: base.u64, max_distance: base.u64, last_copy_len: base.u64) base.u64 { + var address : base.u64 + var br_index : base.u32 + var br_offset : base.u64 + var chunk_length : base.u64 + var n : base.u64 + var p : base.u64 + + if ((args.cmd_dist ~mod- args.max_distance) ~mod- 1) >= this.dict_total_size { + return 0 + } else if args.last_copy_len >= (args.cmd_dist ~mod- args.max_distance) { + return 0 + } + address = (this.dict_total_size ~mod- (args.cmd_dist ~mod- args.max_distance)) ~mod+ args.last_copy_len + br_index = 0 + while (br_index < 15) and (address >= this.dict_chunk_offsets[br_index + 1]) { + br_index += 1 + } + if br_index >= 15 { + return 0 + } + br_offset = address ~mod- this.dict_chunk_offsets[br_index] + chunk_length = this.dict_chunk_offsets[br_index + 1] ~mod- this.dict_chunk_offsets[br_index] + p = args.wrapped_last_processed_pos + while n < args.bytes { + // The C code reads past an empty chunk's end. Here, such a read + // stops the extension (see README.md). + if (br_offset >= chunk_length) or (br_index >= 15) { + break + } + if this.rb_byte(wb: args.wb, pos: p) <> + this.byte_at(s: args.dwb, i: this.dc_src_off[br_index] ~mod+ br_offset) { + break + } + n ~mod+= 1 + p = (p ~mod+ 1) & 0xFFFF_FFFF + br_offset ~mod+= 1 + if br_offset == chunk_length { + br_index += 1 + br_offset = 0 + if (br_index < this.dict_num_chunks) and (br_index < 15) { + chunk_length = this.dict_chunk_offsets[br_index + 1] ~mod- this.dict_chunk_offsets[br_index] + } else { + break + } + } + } + return n +}