mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-17 16:32:11 +01:00
Merge claude/project-thread-0izcgq (q10/q11 dictionaries)
This commit is contained in:
11 files changed
+898
-135
No files matched your search
@@ -46,7 +46,7 @@ first `transform_io` call.
|
||||
`min_incl` is a smaller practical bound), but a shorter workbuf works for
|
||||
short inputs: see the README's "Encoder Workbuf Layout" section. Up to 15
|
||||
raw dictionaries can be attached (like libbrotlienc's
|
||||
`BrotliEncoderAttachPreparedDictionary`) at qualities 0 to 9, see the
|
||||
README's "Encoder Dictionaries" section. They live at the start of the
|
||||
`BrotliEncoderAttachPreparedDictionary`; like the C code, qualities 0
|
||||
and 1 ignore them), see the README's "Encoder Dictionaries" section. They live at the start of the
|
||||
workbuf, with their prepared hash tables, and `workbuf_len` grows
|
||||
accordingly.
|
||||
@@ -9235,7 +9235,6 @@ extern const char wuffs_brotli__error__too_much_dictionary_data[];
|
||||
extern const char wuffs_brotli__error__truncated_input[];
|
||||
extern const char wuffs_brotli__error__unsupported_large_window[];
|
||||
extern const char wuffs_brotli__error__unsupported_multiple_custom_static_dictionaries[];
|
||||
extern const char wuffs_brotli__error__unsupported_encoder_quality[];
|
||||
extern const char wuffs_brotli__error__unsupported_serialized_encoder_dictionary[];
|
||||
|
||||
// ---------------- Public Consts
|
||||
@@ -43080,7 +43079,6 @@ const char wuffs_brotli__error__unsupported_multiple_custom_static_dictionaries[
|
||||
const char wuffs_brotli__error__internal_error_inconsistent_bit_reader[] = "#brotli: internal error: inconsistent bit reader";
|
||||
const char wuffs_brotli__error__internal_error_inconsistent_huffman_table[] = "#brotli: internal error: inconsistent Huffman table";
|
||||
const char wuffs_brotli__error__internal_error_inconsistent_ring_buffer[] = "#brotli: internal error: inconsistent ring buffer";
|
||||
const char wuffs_brotli__error__unsupported_encoder_quality[] = "#brotli: unsupported encoder quality";
|
||||
const char wuffs_brotli__error__internal_error_inconsistent_encoder_state[] = "#brotli: internal error: inconsistent encoder state";
|
||||
const char wuffs_brotli__error__unsupported_serialized_encoder_dictionary[] = "#brotli: unsupported serialized encoder dictionary";
|
||||
|
||||
@@ -76777,6 +76775,10 @@ WUFFS_BROTLI__DICTIONARY[122784] WUFFS_BASE__POTENTIALLY_UNUSED = {
|
||||
|
||||
#define WUFFS_BROTLI__HQ_MATCHES_PER_BYTE_MAX 100u
|
||||
|
||||
#define WUFFS_BROTLI__HQ_MATCHES_PER_BYTE_MAX_DICT 164u
|
||||
|
||||
#define WUFFS_BROTLI__HQ_SHADOW_MATCHES 256u
|
||||
|
||||
#define WUFFS_BROTLI__Z_INFINITY 2130692510u
|
||||
|
||||
#define WUFFS_BROTLI__Z_F32_1 1065353216u
|
||||
@@ -78227,6 +78229,35 @@ wuffs_brotli__encoder__find_compound_dictionary_match(
|
||||
uint64_t a_distance_offset,
|
||||
uint64_t a_max_distance);
|
||||
|
||||
WUFFS_BASE__GENERATED_C_CODE
|
||||
static uint64_t
|
||||
wuffs_brotli__encoder__lookup_all_compound_dictionary_matches(
|
||||
wuffs_brotli__encoder* self,
|
||||
wuffs_base__slice_u8 a_wb,
|
||||
wuffs_base__slice_u8 a_dwb,
|
||||
uint64_t a_cur_ix,
|
||||
uint64_t a_min_length,
|
||||
uint64_t a_max_length,
|
||||
uint64_t a_max_ring_buffer_distance,
|
||||
uint64_t a_max_distance,
|
||||
uint64_t a_matches_off,
|
||||
uint64_t a_match_limit);
|
||||
|
||||
WUFFS_BASE__GENERATED_C_CODE
|
||||
static uint64_t
|
||||
wuffs_brotli__encoder__find_all_compound_dictionary_matches(
|
||||
wuffs_brotli__encoder* self,
|
||||
wuffs_base__slice_u8 a_wb,
|
||||
wuffs_base__slice_u8 a_dwb,
|
||||
uint32_t a_d,
|
||||
uint64_t a_cur_ix,
|
||||
uint64_t a_min_length,
|
||||
uint64_t a_max_length,
|
||||
uint64_t a_distance_offset,
|
||||
uint64_t a_max_distance,
|
||||
uint64_t a_matches_off,
|
||||
uint64_t a_match_limit);
|
||||
|
||||
WUFFS_BASE__GENERATED_C_CODE
|
||||
static uint64_t
|
||||
wuffs_brotli__encoder__find_match_length_2(
|
||||
@@ -80048,6 +80079,7 @@ static uint64_t
|
||||
wuffs_brotli__encoder__update_nodes(
|
||||
wuffs_brotli__encoder* self,
|
||||
wuffs_base__slice_u8 a_wb,
|
||||
wuffs_base__slice_u8 a_dwb,
|
||||
uint64_t a_num_bytes,
|
||||
uint64_t a_block_start,
|
||||
uint64_t a_pos,
|
||||
@@ -80055,6 +80087,19 @@ wuffs_brotli__encoder__update_nodes(
|
||||
uint64_t a_num_matches,
|
||||
uint64_t a_matches_off);
|
||||
|
||||
WUFFS_BASE__GENERATED_C_CODE
|
||||
static uint64_t
|
||||
wuffs_brotli__encoder__compound_last_distance_match_len(
|
||||
const wuffs_brotli__encoder* self,
|
||||
wuffs_base__slice_u8 a_wb,
|
||||
wuffs_base__slice_u8 a_dwb,
|
||||
uint64_t a_cur_ix_masked,
|
||||
uint64_t a_dictionary_start,
|
||||
uint64_t a_backward,
|
||||
uint64_t a_best_len,
|
||||
uint64_t a_max_len,
|
||||
uint8_t a_continuation);
|
||||
|
||||
WUFFS_BASE__GENERATED_C_CODE
|
||||
static uint64_t
|
||||
wuffs_brotli__encoder__compute_shortest_path_from_nodes(
|
||||
@@ -80076,6 +80121,7 @@ static uint64_t
|
||||
wuffs_brotli__encoder__zopfli_iterate(
|
||||
wuffs_brotli__encoder* self,
|
||||
wuffs_base__slice_u8 a_wb,
|
||||
wuffs_base__slice_u8 a_dwb,
|
||||
uint64_t a_num_bytes,
|
||||
uint64_t a_position);
|
||||
|
||||
@@ -80091,6 +80137,7 @@ static uint64_t
|
||||
wuffs_brotli__encoder__zopfli_compute_shortest_path(
|
||||
wuffs_brotli__encoder* self,
|
||||
wuffs_base__slice_u8 a_wb,
|
||||
wuffs_base__slice_u8 a_dwb,
|
||||
uint64_t a_num_bytes,
|
||||
uint64_t a_position);
|
||||
|
||||
@@ -80099,6 +80146,7 @@ static wuffs_base__status
|
||||
wuffs_brotli__encoder__create_zopfli_backward_references(
|
||||
wuffs_brotli__encoder* self,
|
||||
wuffs_base__slice_u8 a_wb,
|
||||
wuffs_base__slice_u8 a_dwb,
|
||||
uint64_t a_num_bytes,
|
||||
uint64_t a_position);
|
||||
|
||||
@@ -80107,9 +80155,21 @@ static wuffs_base__status
|
||||
wuffs_brotli__encoder__create_hq_zopfli_backward_references(
|
||||
wuffs_brotli__encoder* self,
|
||||
wuffs_base__slice_u8 a_wb,
|
||||
wuffs_base__slice_u8 a_dwb,
|
||||
uint64_t a_num_bytes,
|
||||
uint64_t a_position);
|
||||
|
||||
WUFFS_BASE__GENERATED_C_CODE
|
||||
static wuffs_base__empty_struct
|
||||
wuffs_brotli__encoder__merge_matches(
|
||||
wuffs_brotli__encoder* self,
|
||||
wuffs_base__slice_u8 a_wb,
|
||||
uint64_t a_dst,
|
||||
uint64_t a_src1,
|
||||
uint64_t a_len1,
|
||||
uint64_t a_src2,
|
||||
uint64_t a_len2);
|
||||
|
||||
WUFFS_BASE__GENERATED_C_CODE
|
||||
static uint32_t
|
||||
wuffs_brotli__encoder__min32_543(
|
||||
@@ -88414,10 +88474,6 @@ wuffs_brotli__encoder__do_transform_io(
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_0;
|
||||
|
||||
wuffs_brotli__encoder__ensure_initialized(self);
|
||||
if ((self->private_impl.f_quality >= 10u) && (self->private_impl.f_dict_num_chunks > 0u)) {
|
||||
status = wuffs_base__make_status(wuffs_brotli__error__unsupported_encoder_quality);
|
||||
goto exit;
|
||||
}
|
||||
wuffs_brotli__encoder__hq_extend_var_region(self, ((uint64_t)(a_workbuf.len)));
|
||||
if ((self->private_impl.f_quality >= 2u) && (((uint64_t)(a_workbuf.len)) < self->private_impl.f_workbuf_end)) {
|
||||
self->private_impl.f_rb_short_workbuf = true;
|
||||
@@ -90431,9 +90487,17 @@ wuffs_brotli__encoder__encode_data(
|
||||
v_wrapped_last_processed_pos = (((uint64_t)(v_wrapped_last_processed_pos + v_n)) & 4294967295u);
|
||||
}
|
||||
if (self->private_impl.f_quality == 10u) {
|
||||
v_status = wuffs_brotli__encoder__create_zopfli_backward_references(self, a_wb, v_bytes, v_wrapped_last_processed_pos);
|
||||
v_status = wuffs_brotli__encoder__create_zopfli_backward_references(self,
|
||||
a_wb,
|
||||
a_dwb,
|
||||
v_bytes,
|
||||
v_wrapped_last_processed_pos);
|
||||
} else if (self->private_impl.f_quality == 11u) {
|
||||
v_status = wuffs_brotli__encoder__create_hq_zopfli_backward_references(self, a_wb, v_bytes, v_wrapped_last_processed_pos);
|
||||
v_status = wuffs_brotli__encoder__create_hq_zopfli_backward_references(self,
|
||||
a_wb,
|
||||
a_dwb,
|
||||
v_bytes,
|
||||
v_wrapped_last_processed_pos);
|
||||
} else {
|
||||
v_status = wuffs_brotli__encoder__create_backward_references(self,
|
||||
a_wb,
|
||||
@@ -90974,6 +91038,163 @@ wuffs_brotli__encoder__find_compound_dictionary_match(
|
||||
return wuffs_base__make_empty_struct();
|
||||
}
|
||||
|
||||
// -------- func brotli.encoder.lookup_all_compound_dictionary_matches
|
||||
|
||||
WUFFS_BASE__GENERATED_C_CODE
|
||||
static uint64_t
|
||||
wuffs_brotli__encoder__lookup_all_compound_dictionary_matches(
|
||||
wuffs_brotli__encoder* self,
|
||||
wuffs_base__slice_u8 a_wb,
|
||||
wuffs_base__slice_u8 a_dwb,
|
||||
uint64_t a_cur_ix,
|
||||
uint64_t a_min_length,
|
||||
uint64_t a_max_length,
|
||||
uint64_t a_max_ring_buffer_distance,
|
||||
uint64_t a_max_distance,
|
||||
uint64_t a_matches_off,
|
||||
uint64_t a_match_limit) {
|
||||
uint64_t v_base_offset = 0;
|
||||
uint64_t v_min_length = 0;
|
||||
uint64_t v_total_found = 0;
|
||||
uint32_t v_d = 0;
|
||||
|
||||
v_base_offset = ((uint64_t)(a_max_ring_buffer_distance + self->private_impl.f_dict_total_size));
|
||||
v_min_length = a_min_length;
|
||||
v_d = 0u;
|
||||
while (v_d < self->private_impl.f_dict_num_chunks) {
|
||||
v_total_found += wuffs_brotli__encoder__find_all_compound_dictionary_matches(self,
|
||||
a_wb,
|
||||
a_dwb,
|
||||
v_d,
|
||||
a_cur_ix,
|
||||
v_min_length,
|
||||
a_max_length,
|
||||
((uint64_t)(v_base_offset - self->private_impl.f_dict_chunk_offsets[v_d])),
|
||||
a_max_distance,
|
||||
((uint64_t)(a_matches_off + ((uint64_t)(v_total_found * 8u)))),
|
||||
((uint64_t)(a_match_limit - v_total_found)));
|
||||
if (v_total_found == a_match_limit) {
|
||||
break;
|
||||
}
|
||||
if (v_total_found > 0u) {
|
||||
v_min_length = wuffs_brotli__encoder__bm_length(self, a_wb, ((uint64_t)(a_matches_off + ((uint64_t)(((uint64_t)(v_total_found - 1u)) * 8u)))));
|
||||
}
|
||||
v_d += 1u;
|
||||
}
|
||||
return v_total_found;
|
||||
}
|
||||
|
||||
// -------- func brotli.encoder.find_all_compound_dictionary_matches
|
||||
|
||||
WUFFS_BASE__GENERATED_C_CODE
|
||||
static uint64_t
|
||||
wuffs_brotli__encoder__find_all_compound_dictionary_matches(
|
||||
wuffs_brotli__encoder* self,
|
||||
wuffs_base__slice_u8 a_wb,
|
||||
wuffs_base__slice_u8 a_dwb,
|
||||
uint32_t a_d,
|
||||
uint64_t a_cur_ix,
|
||||
uint64_t a_min_length,
|
||||
uint64_t a_max_length,
|
||||
uint64_t a_distance_offset,
|
||||
uint64_t a_max_distance,
|
||||
uint64_t a_matches_off,
|
||||
uint64_t a_match_limit) {
|
||||
uint64_t v_source_size = 0;
|
||||
uint64_t v_src_off = 0;
|
||||
uint64_t v_slot_off = 0;
|
||||
uint64_t v_heads_off = 0;
|
||||
uint64_t v_items_off = 0;
|
||||
wuffs_base__slice_u8 v_source = {0};
|
||||
wuffs_base__slice_u8 v_data = {0};
|
||||
wuffs_base__slice_u8 v_cur = {0};
|
||||
uint64_t v_cur_ix_masked = 0;
|
||||
uint64_t v_mask = 0;
|
||||
uint64_t v_best_len = 0;
|
||||
uint64_t v_h = 0;
|
||||
uint64_t v_key = 0;
|
||||
uint64_t v_slot = 0;
|
||||
uint64_t v_head = 0;
|
||||
uint64_t v_chain = 0;
|
||||
uint32_t v_item = 0;
|
||||
uint64_t v_distance = 0;
|
||||
uint64_t v_offset = 0;
|
||||
uint64_t v_limit = 0;
|
||||
uint64_t v_len = 0;
|
||||
uint64_t v_found = 0;
|
||||
uint64_t v_num_items = 0;
|
||||
uint32_t v_bb = 0;
|
||||
uint32_t v_sb = 0;
|
||||
|
||||
v_bb = wuffs_base__u32__min(self->private_impl.f_dc_bucket_bits[a_d], 22u);
|
||||
v_sb = wuffs_base__u32__min(self->private_impl.f_dc_slot_bits[a_d], 12u);
|
||||
v_num_items = self->private_impl.f_dc_num_items[a_d];
|
||||
v_source_size = self->private_impl.f_dc_size[a_d];
|
||||
v_src_off = self->private_impl.f_dc_src_off[a_d];
|
||||
v_slot_off = ((uint64_t)(v_src_off + v_source_size));
|
||||
v_heads_off = ((uint64_t)(v_slot_off + (((uint64_t)(4u)) << v_sb)));
|
||||
v_items_off = ((uint64_t)(v_heads_off + (((uint64_t)(2u)) << v_bb)));
|
||||
if ((v_src_off > v_slot_off) || (v_slot_off > ((uint64_t)(a_dwb.len)))) {
|
||||
return 0u;
|
||||
}
|
||||
v_source = wuffs_base__slice_u8__subslice_ij(a_dwb, v_src_off, v_slot_off);
|
||||
if (self->private_impl.f_rb_buf_off > ((uint64_t)(a_wb.len))) {
|
||||
return 0u;
|
||||
}
|
||||
v_data = wuffs_base__slice_u8__subslice_i(a_wb, self->private_impl.f_rb_buf_off);
|
||||
v_mask = ((uint64_t)(self->private_impl.f_rb_mask));
|
||||
v_cur_ix_masked = (a_cur_ix & v_mask);
|
||||
if (v_cur_ix_masked > ((uint64_t)(v_data.len))) {
|
||||
return 0u;
|
||||
}
|
||||
v_cur = wuffs_base__slice_u8__subslice_i(v_data, v_cur_ix_masked);
|
||||
v_best_len = a_min_length;
|
||||
v_h = ((uint64_t)((wuffs_brotli__encoder__load64le(self, v_data, v_cur_ix_masked) & 1099511627775u) * 2297779722762296275u));
|
||||
v_key = ((v_h >> 32u) >> (32u - v_bb));
|
||||
v_slot = (v_key & ((((uint64_t)(1u)) << v_sb) - 1u));
|
||||
v_head = ((uint64_t)(wuffs_brotli__encoder__peek_u16(self, a_dwb, ((uint64_t)(v_heads_off + ((uint64_t)(2u * v_key)))))));
|
||||
v_chain = ((uint64_t)(((uint64_t)(wuffs_brotli__encoder__load32le(self, a_dwb, ((uint64_t)(v_slot_off + ((uint64_t)(4u * v_slot))))))) + v_head));
|
||||
v_item = 0u;
|
||||
if (v_head == 65535u) {
|
||||
v_item = 1u;
|
||||
}
|
||||
while (v_item == 0u) {
|
||||
if (v_chain >= v_num_items) {
|
||||
break;
|
||||
}
|
||||
v_item = wuffs_brotli__encoder__load32le(self, a_dwb, ((uint64_t)(v_items_off + ((uint64_t)(4u * v_chain)))));
|
||||
v_chain += 1u;
|
||||
v_offset = ((uint64_t)((v_item & 2147483647u)));
|
||||
v_item &= 2147483648u;
|
||||
v_distance = ((uint64_t)(a_distance_offset - v_offset));
|
||||
v_limit = ((uint64_t)(v_source_size - v_offset));
|
||||
v_limit = wuffs_base__u64__min(v_limit, a_max_length);
|
||||
if (v_distance > a_max_distance) {
|
||||
continue;
|
||||
}
|
||||
if ((((uint64_t)(v_cur_ix_masked + v_best_len)) > v_mask) || (v_best_len >= v_limit) || (wuffs_brotli__encoder__byte_at(self, v_data, ((uint64_t)(v_cur_ix_masked + v_best_len))) != wuffs_brotli__encoder__byte_at(self, v_source, ((uint64_t)(v_offset + v_best_len))))) {
|
||||
continue;
|
||||
}
|
||||
v_len = 0u;
|
||||
if (v_offset <= ((uint64_t)(v_source.len))) {
|
||||
v_len = wuffs_brotli__encoder__find_match_length_2(self, wuffs_base__slice_u8__subslice_i(v_source, v_offset), v_cur, v_limit);
|
||||
}
|
||||
if (v_len > v_best_len) {
|
||||
v_best_len = v_len;
|
||||
wuffs_brotli__encoder__set_backward_match(self,
|
||||
a_wb,
|
||||
((uint64_t)(a_matches_off + ((uint64_t)(v_found * 8u)))),
|
||||
v_distance,
|
||||
((uint64_t)(v_len * 32u)));
|
||||
v_found += 1u;
|
||||
if (v_found == a_match_limit) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return v_found;
|
||||
}
|
||||
|
||||
// -------- func brotli.encoder.find_match_length_2
|
||||
|
||||
WUFFS_BASE__GENERATED_C_CODE
|
||||
@@ -98527,12 +98748,16 @@ wuffs_brotli__encoder__hq_zopfli_len(
|
||||
uint32_t a_lgblock,
|
||||
uint64_t a_matches_per_byte) {
|
||||
uint64_t v_n = 0;
|
||||
uint64_t v_shadow = 0;
|
||||
|
||||
v_n = (((uint64_t)(1u)) << a_lgblock);
|
||||
if (a_quality <= 10u) {
|
||||
return (((16u * (v_n + 1u)) + (4u * (v_n + 2u))) + 3072u);
|
||||
}
|
||||
return wuffs_base__u64__sat_add((((16u * (v_n + 1u)) + (4u * (v_n + 2u))) + (4u * v_n)), ((uint64_t)(8u * wuffs_base__u64__sat_add(((uint64_t)(a_matches_per_byte * v_n)), 128u))));
|
||||
if (self->private_impl.f_dict_num_chunks > 0u) {
|
||||
v_shadow = 256u;
|
||||
}
|
||||
return wuffs_base__u64__sat_add((((16u * (v_n + 1u)) + (4u * (v_n + 2u))) + (4u * v_n)), ((uint64_t)(8u * wuffs_base__u64__sat_add(wuffs_base__u64__sat_add(((uint64_t)(a_matches_per_byte * v_n)), 128u), v_shadow))));
|
||||
}
|
||||
|
||||
// -------- func brotli.encoder.hq_var_len
|
||||
@@ -98549,7 +98774,11 @@ wuffs_brotli__encoder__hq_var_len(
|
||||
uint64_t v_m = 0;
|
||||
|
||||
if (a_worst_case) {
|
||||
v_z = wuffs_brotli__encoder__hq_zopfli_len(self, a_quality, a_lgblock, 100u);
|
||||
if (self->private_impl.f_dict_num_chunks > 0u) {
|
||||
v_z = wuffs_brotli__encoder__hq_zopfli_len(self, a_quality, a_lgblock, 164u);
|
||||
} else {
|
||||
v_z = wuffs_brotli__encoder__hq_zopfli_len(self, a_quality, a_lgblock, 100u);
|
||||
}
|
||||
v_m = wuffs_brotli__metablock_builder__workbuf_len_worst_case(&self->private_data.f_mb, wuffs_brotli__encoder__hq_max_literals(self, a_lgwin, a_lgblock), wuffs_brotli__encoder__commands_capacity(self, a_quality, a_lgwin, a_lgblock));
|
||||
} else {
|
||||
v_z = wuffs_brotli__encoder__hq_zopfli_len(self, a_quality, a_lgblock, 16u);
|
||||
@@ -100016,7 +100245,7 @@ wuffs_brotli__encoder__compute_distance_shortcut(
|
||||
v_dist = ((uint64_t)(wuffs_brotli__encoder__zn_distance(self, a_wb, a_pos)));
|
||||
if (a_pos == 0u) {
|
||||
return 0u;
|
||||
} else if ((((uint64_t)(v_dist + v_c_len)) <= ((uint64_t)(a_block_start + a_pos))) && (v_dist <= a_max_backward_limit) && (wuffs_brotli__encoder__zn_distance_code(self, a_wb, a_pos) > 0u)) {
|
||||
} else if ((((uint64_t)(v_dist + v_c_len)) <= ((uint64_t)(((uint64_t)(a_block_start + a_pos)) + self->private_impl.f_dict_total_size))) && (v_dist <= ((uint64_t)(a_max_backward_limit + self->private_impl.f_dict_total_size))) && (wuffs_brotli__encoder__zn_distance_code(self, a_wb, a_pos) > 0u)) {
|
||||
return ((uint32_t)(a_pos));
|
||||
}
|
||||
return wuffs_brotli__encoder__zn_u(self, a_wb, ((uint64_t)(((uint64_t)(a_pos - v_c_len)) - v_i_len)));
|
||||
@@ -100091,6 +100320,7 @@ static uint64_t
|
||||
wuffs_brotli__encoder__update_nodes(
|
||||
wuffs_brotli__encoder* self,
|
||||
wuffs_base__slice_u8 a_wb,
|
||||
wuffs_base__slice_u8 a_dwb,
|
||||
uint64_t a_num_bytes,
|
||||
uint64_t a_block_start,
|
||||
uint64_t a_pos,
|
||||
@@ -100136,8 +100366,10 @@ wuffs_brotli__encoder__update_nodes(
|
||||
uint32_t v_distnumextra = 0;
|
||||
uint64_t v_max_match_len = 0;
|
||||
uint64_t v_len_code = 0;
|
||||
uint64_t v_gap = 0;
|
||||
|
||||
v_mask = ((uint64_t)(self->private_impl.f_rb_mask));
|
||||
v_gap = self->private_impl.f_dict_total_size;
|
||||
v_cur_ix = ((uint64_t)(a_block_start + a_pos));
|
||||
v_cur_ix_masked = (v_cur_ix & v_mask);
|
||||
v_max_distance = wuffs_base__u64__min(v_cur_ix, a_max_backward_limit);
|
||||
@@ -100183,7 +100415,7 @@ wuffs_brotli__encoder__update_nodes(
|
||||
if (((uint64_t)(v_cur_ix_masked + v_best_len)) > v_mask) {
|
||||
break;
|
||||
}
|
||||
if (v_backward > v_dictionary_start) {
|
||||
if (v_backward > ((uint64_t)(v_dictionary_start + v_gap))) {
|
||||
v_j += 1u;
|
||||
continue;
|
||||
}
|
||||
@@ -100204,6 +100436,16 @@ wuffs_brotli__encoder__update_nodes(
|
||||
v_cur_ix_masked,
|
||||
v_max_len);
|
||||
}
|
||||
} else if (v_backward > v_dictionary_start) {
|
||||
v_len = wuffs_brotli__encoder__compound_last_distance_match_len(self,
|
||||
a_wb,
|
||||
a_dwb,
|
||||
v_cur_ix_masked,
|
||||
v_dictionary_start,
|
||||
v_backward,
|
||||
v_best_len,
|
||||
v_max_len,
|
||||
v_continuation);
|
||||
} else {
|
||||
v_j += 1u;
|
||||
continue;
|
||||
@@ -100245,7 +100487,7 @@ wuffs_brotli__encoder__update_nodes(
|
||||
while (v_j < a_num_matches) {
|
||||
v_moff = ((uint64_t)(a_matches_off + ((uint64_t)(v_j * 8u))));
|
||||
v_dist = wuffs_brotli__encoder__bm_distance(self, a_wb, v_moff);
|
||||
v_is_dictionary_match = (v_dist > v_dictionary_start);
|
||||
v_is_dictionary_match = (v_dist > ((uint64_t)(v_dictionary_start + v_gap)));
|
||||
v_dist_code = ((uint64_t)(v_dist + 15u));
|
||||
v_pe = wuffs_brotli__metablock_builder__prefix_encode_copy_distance(&self->private_data.f_mb, v_dist_code, ((uint64_t)(self->private_impl.f_num_direct_distance_codes)), self->private_impl.f_distance_postfix_bits);
|
||||
v_dist_symbol = ((uint32_t)((v_pe & 65535u)));
|
||||
@@ -100284,6 +100526,52 @@ wuffs_brotli__encoder__update_nodes(
|
||||
return v_result;
|
||||
}
|
||||
|
||||
// -------- func brotli.encoder.compound_last_distance_match_len
|
||||
|
||||
WUFFS_BASE__GENERATED_C_CODE
|
||||
static uint64_t
|
||||
wuffs_brotli__encoder__compound_last_distance_match_len(
|
||||
const wuffs_brotli__encoder* self,
|
||||
wuffs_base__slice_u8 a_wb,
|
||||
wuffs_base__slice_u8 a_dwb,
|
||||
uint64_t a_cur_ix_masked,
|
||||
uint64_t a_dictionary_start,
|
||||
uint64_t a_backward,
|
||||
uint64_t a_best_len,
|
||||
uint64_t a_max_len,
|
||||
uint8_t a_continuation) {
|
||||
uint32_t v_d = 0;
|
||||
uint64_t v_offset = 0;
|
||||
uint64_t v_limit = 0;
|
||||
uint64_t v_src_lo = 0;
|
||||
uint64_t v_src_hi = 0;
|
||||
wuffs_base__slice_u8 v_source = {0};
|
||||
|
||||
v_offset = ((uint64_t)(a_dictionary_start + self->private_impl.f_dict_total_size));
|
||||
while (v_d < 14u) {
|
||||
if (((v_d + 1u) >= self->private_impl.f_dict_num_chunks) || (v_offset < ((uint64_t)(a_backward + self->private_impl.f_dict_chunk_offsets[(v_d + 1u)])))) {
|
||||
break;
|
||||
}
|
||||
v_d += 1u;
|
||||
}
|
||||
v_offset = ((uint64_t)(((uint64_t)(v_offset - self->private_impl.f_dict_chunk_offsets[v_d])) - a_backward));
|
||||
v_limit = ((uint64_t)(((uint64_t)(self->private_impl.f_dict_chunk_offsets[(v_d + 1u)] - self->private_impl.f_dict_chunk_offsets[v_d])) - v_offset));
|
||||
v_limit = wuffs_base__u64__min(v_limit, a_max_len);
|
||||
v_src_lo = self->private_impl.f_dc_src_off[v_d];
|
||||
v_src_hi = ((uint64_t)(v_src_lo + self->private_impl.f_dc_size[v_d]));
|
||||
if ((v_src_lo > v_src_hi) || (v_src_hi > ((uint64_t)(a_dwb.len)))) {
|
||||
return 0u;
|
||||
}
|
||||
v_source = wuffs_base__slice_u8__subslice_ij(a_dwb, v_src_lo, v_src_hi);
|
||||
if ((a_best_len >= v_limit) || (a_continuation != wuffs_brotli__encoder__byte_at(self, v_source, ((uint64_t)(v_offset + a_best_len))))) {
|
||||
return 0u;
|
||||
}
|
||||
if ((v_offset > ((uint64_t)(v_source.len))) || (((uint64_t)(self->private_impl.f_rb_buf_off + a_cur_ix_masked)) > ((uint64_t)(a_wb.len)))) {
|
||||
return 0u;
|
||||
}
|
||||
return wuffs_brotli__encoder__find_match_length_2(self, wuffs_base__slice_u8__subslice_i(v_source, v_offset), wuffs_base__slice_u8__subslice_i(a_wb, ((uint64_t)(self->private_impl.f_rb_buf_off + a_cur_ix_masked))), v_limit);
|
||||
}
|
||||
|
||||
// -------- func brotli.encoder.compute_shortest_path_from_nodes
|
||||
|
||||
WUFFS_BASE__GENERATED_C_CODE
|
||||
@@ -100354,7 +100642,7 @@ wuffs_brotli__encoder__zopfli_create_commands(
|
||||
v_distance = ((uint64_t)(wuffs_brotli__encoder__zn_distance(self, a_wb, v_next)));
|
||||
v_len_code = wuffs_brotli__encoder__zn_length_code(self, a_wb, v_next);
|
||||
v_dictionary_start = wuffs_brotli__encoder__min64(self, ((uint64_t)(((uint64_t)(a_block_start + v_pos)) + ((uint64_t)(self->private_impl.f_stream_offset)))), v_max_backward_limit);
|
||||
v_is_dictionary = (v_distance > v_dictionary_start);
|
||||
v_is_dictionary = (v_distance > ((uint64_t)(v_dictionary_start + self->private_impl.f_dict_total_size)));
|
||||
v_dist_code = wuffs_brotli__encoder__zn_distance_code(self, a_wb, v_next);
|
||||
wuffs_brotli__encoder__init_command(self,
|
||||
a_wb,
|
||||
@@ -100384,6 +100672,7 @@ static uint64_t
|
||||
wuffs_brotli__encoder__zopfli_iterate(
|
||||
wuffs_brotli__encoder* self,
|
||||
wuffs_base__slice_u8 a_wb,
|
||||
wuffs_base__slice_u8 a_dwb,
|
||||
uint64_t a_num_bytes,
|
||||
uint64_t a_position) {
|
||||
uint64_t v_max_backward_limit = 0;
|
||||
@@ -100407,6 +100696,7 @@ wuffs_brotli__encoder__zopfli_iterate(
|
||||
v_nm = ((uint64_t)(wuffs_brotli__encoder__load32le(self, a_wb, ((uint64_t)(self->private_impl.f_z_nm_off + ((uint64_t)(v_i * 4u)))))));
|
||||
v_skip = wuffs_brotli__encoder__update_nodes(self,
|
||||
a_wb,
|
||||
a_dwb,
|
||||
a_num_bytes,
|
||||
a_position,
|
||||
v_i,
|
||||
@@ -100486,6 +100776,7 @@ static uint64_t
|
||||
wuffs_brotli__encoder__zopfli_compute_shortest_path(
|
||||
wuffs_brotli__encoder* self,
|
||||
wuffs_base__slice_u8 a_wb,
|
||||
wuffs_base__slice_u8 a_dwb,
|
||||
uint64_t a_num_bytes,
|
||||
uint64_t a_position) {
|
||||
uint64_t v_max_backward_limit = 0;
|
||||
@@ -100498,8 +100789,14 @@ wuffs_brotli__encoder__zopfli_compute_shortest_path(
|
||||
uint64_t v_skip = 0;
|
||||
uint64_t v_num_matches = 0;
|
||||
uint64_t v_ml = 0;
|
||||
uint64_t v_lz_matches_off = 0;
|
||||
uint64_t v_cd_matches = 0;
|
||||
|
||||
v_max_backward_limit = ((uint64_t)((((uint64_t)(1u)) << self->private_impl.f_lgwin) - 16u));
|
||||
v_lz_matches_off = self->private_impl.f_z_m_off;
|
||||
if (self->private_impl.f_dict_num_chunks != 0u) {
|
||||
v_lz_matches_off += 2048u;
|
||||
}
|
||||
v_max_zopfli_len = 150u;
|
||||
if (self->private_impl.f_quality > 10u) {
|
||||
v_max_zopfli_len = 325u;
|
||||
@@ -100523,8 +100820,28 @@ wuffs_brotli__encoder__zopfli_compute_shortest_path(
|
||||
v_pos,
|
||||
((uint64_t)(a_num_bytes - v_i)),
|
||||
v_max_distance,
|
||||
v_dictionary_start,
|
||||
self->private_impl.f_z_m_off);
|
||||
((uint64_t)(v_dictionary_start + self->private_impl.f_dict_total_size)),
|
||||
v_lz_matches_off);
|
||||
if (self->private_impl.f_dict_num_chunks != 0u) {
|
||||
v_cd_matches = wuffs_brotli__encoder__lookup_all_compound_dictionary_matches(self,
|
||||
a_wb,
|
||||
a_dwb,
|
||||
v_pos,
|
||||
3u,
|
||||
((uint64_t)(a_num_bytes - v_i)),
|
||||
v_dictionary_start,
|
||||
self->private_impl.f_dist_max_distance,
|
||||
((uint64_t)(v_lz_matches_off - 512u)),
|
||||
64u);
|
||||
wuffs_brotli__encoder__merge_matches(self,
|
||||
a_wb,
|
||||
self->private_impl.f_z_m_off,
|
||||
((uint64_t)(v_lz_matches_off - 512u)),
|
||||
v_cd_matches,
|
||||
v_lz_matches_off,
|
||||
v_num_matches);
|
||||
v_num_matches += v_cd_matches;
|
||||
}
|
||||
if (v_num_matches > 0u) {
|
||||
v_ml = wuffs_brotli__encoder__bm_length(self, a_wb, ((uint64_t)(self->private_impl.f_z_m_off + ((uint64_t)(((uint64_t)(v_num_matches - 1u)) * 8u)))));
|
||||
if (v_ml > v_max_zopfli_len) {
|
||||
@@ -100534,6 +100851,7 @@ wuffs_brotli__encoder__zopfli_compute_shortest_path(
|
||||
}
|
||||
v_skip = wuffs_brotli__encoder__update_nodes(self,
|
||||
a_wb,
|
||||
a_dwb,
|
||||
a_num_bytes,
|
||||
a_position,
|
||||
v_i,
|
||||
@@ -100578,6 +100896,7 @@ static wuffs_base__status
|
||||
wuffs_brotli__encoder__create_zopfli_backward_references(
|
||||
wuffs_brotli__encoder* self,
|
||||
wuffs_base__slice_u8 a_wb,
|
||||
wuffs_base__slice_u8 a_dwb,
|
||||
uint64_t a_num_bytes,
|
||||
uint64_t a_position) {
|
||||
uint64_t v_cmds_base = 0;
|
||||
@@ -100590,7 +100909,11 @@ wuffs_brotli__encoder__create_zopfli_backward_references(
|
||||
}
|
||||
v_cmds_base = self->private_impl.f_num_cmds;
|
||||
wuffs_brotli__encoder__init_zopfli_nodes(self, a_wb, ((uint64_t)(a_num_bytes + 1u)));
|
||||
v_n = wuffs_brotli__encoder__zopfli_compute_shortest_path(self, a_wb, a_num_bytes, a_position);
|
||||
v_n = wuffs_brotli__encoder__zopfli_compute_shortest_path(self,
|
||||
a_wb,
|
||||
a_dwb,
|
||||
a_num_bytes,
|
||||
a_position);
|
||||
if (((uint64_t)(v_cmds_base + v_n)) > self->private_impl.f_cmds_cap) {
|
||||
return wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_encoder_state);
|
||||
}
|
||||
@@ -100610,6 +100933,7 @@ static wuffs_base__status
|
||||
wuffs_brotli__encoder__create_hq_zopfli_backward_references(
|
||||
wuffs_brotli__encoder* self,
|
||||
wuffs_base__slice_u8 a_wb,
|
||||
wuffs_base__slice_u8 a_dwb,
|
||||
uint64_t a_num_bytes,
|
||||
uint64_t a_position) {
|
||||
uint64_t v_max_backward_limit = 0;
|
||||
@@ -100633,6 +100957,8 @@ wuffs_brotli__encoder__create_hq_zopfli_backward_references(
|
||||
uint32_t v_k = 0;
|
||||
uint64_t v_n = 0;
|
||||
bool v_ok_offsets = false;
|
||||
uint64_t v_shadow_matches = 0;
|
||||
uint64_t v_cd_matches = 0;
|
||||
|
||||
v_ok_offsets = wuffs_brotli__encoder__z_set_offsets(self, wuffs_base__u64__min(self->private_impl.f_var_end, ((uint64_t)(a_wb.len))), a_num_bytes);
|
||||
if ( ! v_ok_offsets) {
|
||||
@@ -100643,13 +100969,16 @@ wuffs_brotli__encoder__create_hq_zopfli_backward_references(
|
||||
if (a_num_bytes >= 128u) {
|
||||
v_store_end = ((uint64_t)(((uint64_t)(((uint64_t)(a_position + a_num_bytes)) - 128u)) + 1u));
|
||||
}
|
||||
if (self->private_impl.f_dict_num_chunks != 0u) {
|
||||
v_shadow_matches = 256u;
|
||||
}
|
||||
v_i = 0u;
|
||||
while (((uint64_t)(v_i + 3u)) < a_num_bytes) {
|
||||
v_pos = ((uint64_t)(a_position + v_i));
|
||||
v_max_distance = wuffs_base__u64__min(v_pos, v_max_backward_limit);
|
||||
v_dictionary_start = wuffs_brotli__encoder__min64(self, ((uint64_t)(v_pos + ((uint64_t)(self->private_impl.f_stream_offset)))), v_max_backward_limit);
|
||||
v_max_length = ((uint64_t)(a_num_bytes - v_i));
|
||||
if (((uint64_t)(v_cur_match_pos + 128u)) > self->private_impl.f_z_m_cap) {
|
||||
if (((uint64_t)(((uint64_t)(v_cur_match_pos + 128u)) + v_shadow_matches)) > self->private_impl.f_z_m_cap) {
|
||||
return wuffs_base__make_status(wuffs_base__error__bad_workbuf_length);
|
||||
}
|
||||
v_num_found_matches = wuffs_brotli__encoder__h10_find_all_matches(self,
|
||||
@@ -100657,8 +100986,28 @@ wuffs_brotli__encoder__create_hq_zopfli_backward_references(
|
||||
v_pos,
|
||||
v_max_length,
|
||||
v_max_distance,
|
||||
v_dictionary_start,
|
||||
((uint64_t)(self->private_impl.f_z_m_off + ((uint64_t)(v_cur_match_pos * 8u)))));
|
||||
((uint64_t)(v_dictionary_start + self->private_impl.f_dict_total_size)),
|
||||
((uint64_t)(self->private_impl.f_z_m_off + ((uint64_t)(((uint64_t)(v_cur_match_pos + v_shadow_matches)) * 8u)))));
|
||||
if (self->private_impl.f_dict_num_chunks != 0u) {
|
||||
v_cd_matches = wuffs_brotli__encoder__lookup_all_compound_dictionary_matches(self,
|
||||
a_wb,
|
||||
a_dwb,
|
||||
v_pos,
|
||||
3u,
|
||||
v_max_length,
|
||||
v_dictionary_start,
|
||||
self->private_impl.f_dist_max_distance,
|
||||
((uint64_t)(self->private_impl.f_z_m_off + ((uint64_t)(((uint64_t)(((uint64_t)(v_cur_match_pos + v_shadow_matches)) - 64u)) * 8u)))),
|
||||
64u);
|
||||
wuffs_brotli__encoder__merge_matches(self,
|
||||
a_wb,
|
||||
((uint64_t)(self->private_impl.f_z_m_off + ((uint64_t)(v_cur_match_pos * 8u)))),
|
||||
((uint64_t)(self->private_impl.f_z_m_off + ((uint64_t)(((uint64_t)(((uint64_t)(v_cur_match_pos + v_shadow_matches)) - 64u)) * 8u)))),
|
||||
v_cd_matches,
|
||||
((uint64_t)(self->private_impl.f_z_m_off + ((uint64_t)(((uint64_t)(v_cur_match_pos + v_shadow_matches)) * 8u)))),
|
||||
v_num_found_matches);
|
||||
v_num_found_matches += v_cd_matches;
|
||||
}
|
||||
v_cur_match_end = ((uint64_t)(v_cur_match_pos + v_num_found_matches));
|
||||
wuffs_brotli__encoder__poke_u32(self, a_wb, ((uint64_t)(self->private_impl.f_z_nm_off + ((uint64_t)(v_i * 4u)))), ((uint32_t)(v_num_found_matches)));
|
||||
if (v_num_found_matches > 0u) {
|
||||
@@ -100711,7 +101060,11 @@ wuffs_brotli__encoder__create_hq_zopfli_backward_references(
|
||||
self->private_impl.f_dist_cache[v_k] = v_orig_dist_cache[v_k];
|
||||
v_k += 1u;
|
||||
}
|
||||
v_n = wuffs_brotli__encoder__zopfli_iterate(self, a_wb, a_num_bytes, a_position);
|
||||
v_n = wuffs_brotli__encoder__zopfli_iterate(self,
|
||||
a_wb,
|
||||
a_dwb,
|
||||
a_num_bytes,
|
||||
a_position);
|
||||
if (((uint64_t)(v_orig_num_commands + v_n)) > self->private_impl.f_cmds_cap) {
|
||||
return wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_encoder_state);
|
||||
}
|
||||
@@ -100726,6 +101079,60 @@ wuffs_brotli__encoder__create_hq_zopfli_backward_references(
|
||||
return wuffs_base__make_status(NULL);
|
||||
}
|
||||
|
||||
// -------- func brotli.encoder.merge_matches
|
||||
|
||||
WUFFS_BASE__GENERATED_C_CODE
|
||||
static wuffs_base__empty_struct
|
||||
wuffs_brotli__encoder__merge_matches(
|
||||
wuffs_brotli__encoder* self,
|
||||
wuffs_base__slice_u8 a_wb,
|
||||
uint64_t a_dst,
|
||||
uint64_t a_src1,
|
||||
uint64_t a_len1,
|
||||
uint64_t a_src2,
|
||||
uint64_t a_len2) {
|
||||
uint64_t v_dst = 0;
|
||||
uint64_t v_src1 = 0;
|
||||
uint64_t v_len1 = 0;
|
||||
uint64_t v_src2 = 0;
|
||||
uint64_t v_len2 = 0;
|
||||
uint64_t v_l1 = 0;
|
||||
uint64_t v_l2 = 0;
|
||||
|
||||
v_dst = a_dst;
|
||||
v_src1 = a_src1;
|
||||
v_len1 = a_len1;
|
||||
v_src2 = a_src2;
|
||||
v_len2 = a_len2;
|
||||
while ((v_len1 > 0u) && (v_len2 > 0u)) {
|
||||
v_l1 = wuffs_brotli__encoder__bm_length(self, a_wb, v_src1);
|
||||
v_l2 = wuffs_brotli__encoder__bm_length(self, a_wb, v_src2);
|
||||
if ((v_l1 < v_l2) || ((v_l1 == v_l2) && (wuffs_brotli__encoder__bm_distance(self, a_wb, v_src1) < wuffs_brotli__encoder__bm_distance(self, a_wb, v_src2)))) {
|
||||
wuffs_brotli__encoder__bm_copy(self, a_wb, v_dst, v_src1);
|
||||
v_src1 += 8u;
|
||||
v_len1 -= 1u;
|
||||
} else {
|
||||
wuffs_brotli__encoder__bm_copy(self, a_wb, v_dst, v_src2);
|
||||
v_src2 += 8u;
|
||||
v_len2 -= 1u;
|
||||
}
|
||||
v_dst += 8u;
|
||||
}
|
||||
while (v_len1 > 0u) {
|
||||
wuffs_brotli__encoder__bm_copy(self, a_wb, v_dst, v_src1);
|
||||
v_src1 += 8u;
|
||||
v_dst += 8u;
|
||||
v_len1 -= 1u;
|
||||
}
|
||||
while (v_len2 > 0u) {
|
||||
wuffs_brotli__encoder__bm_copy(self, a_wb, v_dst, v_src2);
|
||||
v_src2 += 8u;
|
||||
v_dst += 8u;
|
||||
v_len2 -= 1u;
|
||||
}
|
||||
return wuffs_base__make_empty_struct();
|
||||
}
|
||||
|
||||
// -------- func brotli.encoder.min32_543
|
||||
|
||||
WUFFS_BASE__GENERATED_C_CODE
|
||||
|
||||
@@ -1388,7 +1388,10 @@ run_input_dict(const char* name, const uint8_t* data, size_t len) {
|
||||
continue;
|
||||
}
|
||||
for (uint32_t q = g_qmin; q <= g_qmax; q++) {
|
||||
run_dict_configs(name, data, len, q, level);
|
||||
// Qualities 10 and 11 (which are much slower) use the smallest grid
|
||||
// for inputs longer than 64 KiB.
|
||||
run_dict_configs(name, data, len, q,
|
||||
((q >= 10) && (len > (64 << 10))) ? 2 : level);
|
||||
if ((set == 1) && (len >= 1000) && (q >= 5)) {
|
||||
check_smaller(name, data, len, q);
|
||||
}
|
||||
|
||||
+38
-12
@@ -174,8 +174,6 @@ invariants for memory safety but Wuffs requires explicit bounds checks.
|
||||
- Metadata callbacks (metadata blocks are currently skipped). See the
|
||||
"Metadata Blocks" section below.
|
||||
- Fast paths.
|
||||
- Compound dictionary matches at encoder qualities 10 and 11 (see the
|
||||
"Encoder" section below).
|
||||
|
||||
|
||||
## Encoder
|
||||
@@ -206,10 +204,7 @@ hasher, `BrotliCreateZopfliBackwardReferences` at quality 10 and
|
||||
below) works at qualities 3 to 11 with lgwin 10 to 30. Like the C code,
|
||||
qualities 0 to 2 ignore the large-window quirk.
|
||||
Raw (LZ77 prefix) dictionaries (see "Encoder Dictionaries" below) work at
|
||||
qualities 0 to 9, and, like the C code, qualities 0 and 1 ignore them. At
|
||||
qualities 10 and 11, `transform_io` returns `"#unsupported encoder quality"`
|
||||
if a dictionary is attached (the Zopfli path's dictionary lookups are not
|
||||
implemented yet).
|
||||
all qualities, and, like the C code, qualities 0 and 1 ignore them.
|
||||
|
||||
The source files are:
|
||||
|
||||
@@ -537,6 +532,19 @@ literals) and:
|
||||
- `M` is the `metablock_builder`'s `workbuf_len_worst_case(L, cmds_cap)`
|
||||
(`max_incl`) or `workbuf_len_practical(L, cmds_cap)` (`min_incl`).
|
||||
|
||||
With raw dictionaries attached (see "Encoder Dictionaries" below), both
|
||||
`min_incl` and `max_incl` also include the dictionaries' `D` bytes, and, at
|
||||
quality 11, `Z` has room for 256 more `BackwardMatch`es (the C code's
|
||||
`shadow_matches` in `BrotliCreateHqZopfliBackwardReferences`: scratch space
|
||||
past the current position's matches, where the binary tree's and the
|
||||
compound dictionary's matches are found before `MergeMatches` merges them)
|
||||
and, for `max_incl`, up to 164 `BackwardMatch`es per byte (64 more, the
|
||||
C code's `match_limit`, from the compound dictionary). At quality 10, the
|
||||
C code's matches buffer (`2 * (MAX_NUM_MATCHES_H10 + 64)` entries, for one
|
||||
position) already has room for its `lz_matches_offset` of
|
||||
`MAX_NUM_MATCHES_H10 + 128`, so `Z` is unchanged. In practice `M` is the
|
||||
larger of the two, so only `D` changes the table below.
|
||||
|
||||
`workbuf_len_practical` is `workbuf_len_worst_case` with these caps:
|
||||
`FindBlocks` makes at most `(k / 16) + 1` blocks for `k` literals, commands
|
||||
or distances (instead of one block per symbol), `ClusterBlocks`' first phase
|
||||
@@ -574,7 +582,11 @@ words, at most the C code's `max_distance` check in `FindAllMatchesH10` and
|
||||
less than `(1 << 24) + (1 << 17)`, plus 15 for the distance code), which
|
||||
satisfies the `metablock_builder`'s precondition (distance codes at most the
|
||||
distance params' `max_distance`, which is at least `(1 << 26) - 4` without
|
||||
large-window brotli).
|
||||
large-window brotli). With raw dictionaries attached, the static dictionary
|
||||
words start `gap` further away and compound dictionary matches can be
|
||||
further away too, but explicit distances are still at most `max_distance`
|
||||
(`FindAllMatchesH10` and `FindAllCompoundDictionaryMatches` skip longer
|
||||
ones, and distance cache hits use short distance codes), as in the C code.
|
||||
|
||||
|
||||
### Encoder Operation Mapping
|
||||
@@ -754,6 +766,19 @@ The compound dictionary then affects compression like in the C code:
|
||||
that are within `params.dist.max_distance` (`BROTLI_MAX_DISTANCE`, or
|
||||
`BROTLI_MAX_ALLOWED_DISTANCE` for large-window brotli) and that do not
|
||||
cross a chunk's end.
|
||||
- At qualities 10 and 11 (`BrotliCreateZopfliBackwardReferences` and
|
||||
`BrotliCreateHqZopfliBackwardReferences` in `backward_references_hq.c`),
|
||||
for each position, `LookupAllCompoundDictionaryMatches` finds up to 64
|
||||
matches (of length 4 or more, each longer than the previous one, over the
|
||||
chunks in order) in the dictionaries, which `MergeMatches` merges (by
|
||||
length, then distance) with the H10 binary tree's and static dictionary's
|
||||
matches. The binary tree's static dictionary matches start at
|
||||
`dictionary_start + gap`, and `UpdateNodes`, `EvaluateNode` (via
|
||||
`ComputeDistanceShortcut`) and `BrotliZopfliCreateCommands` treat
|
||||
distances up to `dictionary_start + gap` as non-static-dictionary ones
|
||||
(which update the distance cache). `UpdateNodes` also tries the distance
|
||||
cache's distances that land in the compound dictionary (finding the chunk
|
||||
and matching up to its end).
|
||||
- With `QUIRK_STREAM_OFFSET`, `dictionary_start` is `min(position +
|
||||
stream_offset, max_backward_limit)`, as in the C code (whose output then
|
||||
continues another stream and can only be decoded after it).
|
||||
@@ -829,7 +854,7 @@ dictionary grid (see "Encoder Dictionaries" above): for each input, seven
|
||||
sets of raw dictionaries (a prefix of the input, the whole input,
|
||||
`test/data/enwik5`, noise plus the input's middle third, 15 pieces of the
|
||||
input, tiny and empty dictionaries, and the input's swapped halves plus some
|
||||
text), with qualities 0 to 9, lgwins, chunkings, flushes, metadata blocks,
|
||||
text), with qualities 0 to 11, lgwins, chunkings, flushes, metadata blocks,
|
||||
size hints, stream offsets, large window and other parameters, plus
|
||||
dictionaries of 2 MiB to 40 MiB (every hash table size) with a 1 MiB input.
|
||||
Each dictionary configuration's output is also decoded by libbrotlidec and
|
||||
@@ -838,7 +863,11 @@ stream offset, and the whole-input and big dictionaries check that the
|
||||
output is smaller than without them (at qualities 5 and above, unless it is
|
||||
already tiny). With `-dict` (and `-q=0-9`) it runs 419,802 configurations,
|
||||
all byte-exact against libbrotlienc v1.2.1 on x86-64, with 356,546 round
|
||||
trips through both decoders. With
|
||||
trips through both decoders. With `-dict -q=10-11` (which uses the
|
||||
smallest dictionary grid for inputs longer than 64 KiB), shards 0 to 47 of
|
||||
90 (except 43 and 46: 98 inputs, including all of the 2 to 40 MiB
|
||||
dictionaries) ran 39,800 configurations, also all byte-exact, with 33,814
|
||||
round trips through both decoders and 80 smaller-with-dictionary checks. With
|
||||
`-q=0-9` (the default) and without `-dict` it runs 814,706
|
||||
configurations (96,688 of them in the large-window grid), all byte-exact
|
||||
against libbrotlienc v1.2.1 on x86-64. With `-lw -big` it runs 720 more
|
||||
@@ -1047,9 +1076,6 @@ metablock) for as long as it needs the results.
|
||||
- Speed: at qualities 10 and 11 the Wuffs encoder is several times slower
|
||||
than libbrotlienc, mostly because of the soft float arithmetic in the
|
||||
Zopfli cost model, the block splitter and histogram clustering.
|
||||
- Compound dictionary matches at qualities 10 and 11 (the Zopfli path's
|
||||
`FindCompoundDictionaryMatch` lookups); attaching a dictionary at those
|
||||
qualities currently returns `"#unsupported encoder quality"`.
|
||||
- Serialized (shared) dictionaries, which only an experimental
|
||||
(`BROTLI_EXPERIMENTAL`) libbrotlienc build supports.
|
||||
- Changing parameters mid-stream.
|
||||
|
||||
@@ -31,8 +31,6 @@
|
||||
|
||||
use "std/softfloat"
|
||||
|
||||
pub status "#unsupported encoder quality"
|
||||
|
||||
pri status "#internal error: inconsistent encoder state"
|
||||
|
||||
pub const ENCODER_DST_HISTORY_RETAIN_LENGTH_MAX_INCL_WORST_CASE : base.u64 = 0
|
||||
@@ -916,11 +914,6 @@ pri func encoder.do_transform_io?(dst: base.io_writer, src: base.io_reader, work
|
||||
var total : base.u64
|
||||
|
||||
this.ensure_initialized!()
|
||||
if (this.quality >= 10) and (this.dict_num_chunks > 0) {
|
||||
// TODO: compound dictionaries at qualities 10 and 11 (the Zopfli
|
||||
// path's dictionary lookups).
|
||||
return "#unsupported encoder quality"
|
||||
}
|
||||
this.hq_extend_var_region!(wb_len: args.workbuf.length())
|
||||
if (this.quality >= 2) and (args.workbuf.length() < this.workbuf_end) {
|
||||
// A workbuf shorter than workbuf_len works if it is long enough for
|
||||
|
||||
@@ -954,9 +954,9 @@ pri func encoder.encode_data!(wb: slice base.u8, dwb: roslice base.u8, is_last:
|
||||
}
|
||||
|
||||
if this.quality == 10 {
|
||||
status = this.create_zopfli_backward_references!(wb: args.wb, num_bytes: bytes, position: wrapped_last_processed_pos)
|
||||
status = this.create_zopfli_backward_references!(wb: args.wb, dwb: args.dwb, num_bytes: bytes, position: wrapped_last_processed_pos)
|
||||
} else if this.quality == 11 {
|
||||
status = this.create_hq_zopfli_backward_references!(wb: args.wb, num_bytes: bytes, position: wrapped_last_processed_pos)
|
||||
status = this.create_hq_zopfli_backward_references!(wb: args.wb, dwb: args.dwb, num_bytes: bytes, position: wrapped_last_processed_pos)
|
||||
} else {
|
||||
status = this.create_backward_references!(wb: args.wb, dwb: args.dwb, num_bytes: bytes, position: wrapped_last_processed_pos)
|
||||
}
|
||||
|
||||
@@ -474,6 +474,144 @@ pri func encoder.find_compound_dictionary_match!(data: roslice base.u8, dwb: ros
|
||||
}
|
||||
}
|
||||
|
||||
// Mirrors LookupAllCompoundDictionaryMatches in hash.h (for qualities 10 and
|
||||
// 11). args.wb[this.rb_buf_off ..] is the ring buffer (the C code's data),
|
||||
// args.dwb holds the dictionaries and the matches go to
|
||||
// args.wb[matches_off ..]. It returns how many matches it found.
|
||||
pri func encoder.lookup_all_compound_dictionary_matches!(wb: slice base.u8, dwb: roslice base.u8, cur_ix: base.u64, min_length: base.u64, max_length: base.u64,
|
||||
max_ring_buffer_distance: base.u64, max_distance: base.u64, matches_off: base.u64, match_limit: base.u64) base.u64 {
|
||||
var base_offset : base.u64
|
||||
var min_length : base.u64
|
||||
var total_found : base.u64
|
||||
var d : base.u32
|
||||
|
||||
base_offset = args.max_ring_buffer_distance ~mod+ this.dict_total_size
|
||||
min_length = args.min_length
|
||||
d = 0
|
||||
while d < this.dict_num_chunks {
|
||||
assert d < 15 via "a < b: a < c; c <= b"(c: this.dict_num_chunks)
|
||||
total_found ~mod+= this.find_all_compound_dictionary_matches!(wb: args.wb, dwb: args.dwb, d: d,
|
||||
cur_ix: args.cur_ix, min_length: min_length, max_length: args.max_length,
|
||||
distance_offset: base_offset ~mod- this.dict_chunk_offsets[d],
|
||||
max_distance: args.max_distance,
|
||||
matches_off: args.matches_off ~mod+ (total_found ~mod* 8),
|
||||
match_limit: args.match_limit ~mod- total_found)
|
||||
if total_found == args.match_limit {
|
||||
break
|
||||
}
|
||||
if total_found > 0 {
|
||||
min_length = this.bm_length(wb: args.wb, off: args.matches_off ~mod+ ((total_found ~mod- 1) ~mod* 8))
|
||||
}
|
||||
d += 1
|
||||
}
|
||||
return total_found
|
||||
}
|
||||
|
||||
// Mirrors FindAllCompoundDictionaryMatches in hash.h, for the d'th chunk.
|
||||
// args.wb[this.rb_buf_off ..] is the ring buffer (the C code's data) and
|
||||
// the matches go to args.wb[matches_off ..]. It returns how many matches it
|
||||
// found.
|
||||
pri func encoder.find_all_compound_dictionary_matches!(wb: slice base.u8, dwb: roslice base.u8, d: base.u32[..= 14], cur_ix: base.u64,
|
||||
min_length: base.u64, max_length: base.u64, distance_offset: base.u64, max_distance: base.u64,
|
||||
matches_off: base.u64, match_limit: base.u64) base.u64 {
|
||||
var source_size : base.u64
|
||||
var src_off : base.u64
|
||||
var slot_off : base.u64
|
||||
var heads_off : base.u64
|
||||
var items_off : base.u64
|
||||
var source : roslice base.u8
|
||||
var data : roslice base.u8
|
||||
var cur : roslice base.u8
|
||||
var cur_ix_masked : base.u64
|
||||
var mask : base.u64
|
||||
var best_len : base.u64
|
||||
var h : base.u64
|
||||
var key : base.u64
|
||||
var slot : base.u64
|
||||
var head : base.u64
|
||||
var chain : base.u64
|
||||
var item : base.u32
|
||||
var distance : base.u64
|
||||
var offset : base.u64
|
||||
var limit : base.u64
|
||||
var len : base.u64
|
||||
var found : base.u64
|
||||
var num_items : base.u64
|
||||
var bb : base.u32[..= 22]
|
||||
var sb : base.u32[..= 12]
|
||||
|
||||
bb = this.dc_bucket_bits[args.d].min(no_more_than: 22)
|
||||
sb = this.dc_slot_bits[args.d].min(no_more_than: 12)
|
||||
num_items = this.dc_num_items[args.d]
|
||||
source_size = this.dc_size[args.d]
|
||||
src_off = this.dc_src_off[args.d]
|
||||
slot_off = src_off ~mod+ source_size
|
||||
heads_off = slot_off ~mod+ ((4 as base.u64) << sb)
|
||||
items_off = heads_off ~mod+ ((2 as base.u64) << bb)
|
||||
if (src_off > slot_off) or (slot_off > args.dwb.length()) {
|
||||
return 0
|
||||
}
|
||||
source = args.dwb[src_off .. slot_off]
|
||||
if this.rb_buf_off > args.wb.length() {
|
||||
return 0
|
||||
}
|
||||
data = args.wb[this.rb_buf_off ..]
|
||||
|
||||
mask = this.rb_mask as base.u64
|
||||
cur_ix_masked = args.cur_ix & mask
|
||||
if cur_ix_masked > data.length() {
|
||||
return 0
|
||||
}
|
||||
cur = data[cur_ix_masked ..]
|
||||
best_len = args.min_length
|
||||
h = (this.load64le(s: data, i: cur_ix_masked) & 0xFF_FFFF_FFFF) ~mod*
|
||||
PREPARED_DICTIONARY_HASH_MUL_64_LONG
|
||||
key = (h >> 32) >> (32 - bb)
|
||||
slot = key & (((1 as base.u64) << sb) - 1)
|
||||
head = this.peek_u16(s: args.dwb, i: heads_off ~mod+ (2 ~mod* key)) as base.u64
|
||||
chain = (this.load32le(s: args.dwb, i: slot_off ~mod+ (4 ~mod* slot)) as base.u64) ~mod+ head
|
||||
item = 0
|
||||
if head == 0xFFFF {
|
||||
item = 1
|
||||
}
|
||||
|
||||
while item == 0 {
|
||||
if chain >= num_items {
|
||||
// Unreachable: every chain ends with a 0x8000_0000 bit.
|
||||
break
|
||||
}
|
||||
item = this.load32le(s: args.dwb, i: items_off ~mod+ (4 ~mod* chain))
|
||||
chain ~mod+= 1
|
||||
offset = (item & 0x7FFF_FFFF) as base.u64
|
||||
item &= 0x8000_0000
|
||||
distance = args.distance_offset ~mod- offset
|
||||
limit = source_size ~mod- offset
|
||||
limit = limit.min(no_more_than: args.max_length)
|
||||
if distance > args.max_distance {
|
||||
continue
|
||||
}
|
||||
if ((cur_ix_masked ~mod+ best_len) > mask) or (best_len >= limit) or
|
||||
(this.byte_at(s: data, i: cur_ix_masked ~mod+ best_len) <>
|
||||
this.byte_at(s: source, i: offset ~mod+ best_len)) {
|
||||
continue
|
||||
}
|
||||
len = 0
|
||||
if offset <= source.length() {
|
||||
len = this.find_match_length_2(a: source[offset ..], b: cur, limit: limit)
|
||||
}
|
||||
if len > best_len {
|
||||
best_len = len
|
||||
this.set_backward_match!(wb: args.wb, off: args.matches_off ~mod+ (found ~mod* 8),
|
||||
distance: distance, length_and_code: len ~mod* 32)
|
||||
found ~mod+= 1
|
||||
if found == args.match_limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return found
|
||||
}
|
||||
|
||||
// find_match_length_2 mirrors FindMatchLengthWithLimit in
|
||||
// find_match_length.h: the length of the common prefix of a and b, up to
|
||||
// limit bytes.
|
||||
|
||||
@@ -30,8 +30,7 @@
|
||||
// When this quirk is set, it sets the encoder's BROTLI_PARAM_QUALITY
|
||||
// parameter: 0 to 11. The default is 11. Like the C code, out of range values
|
||||
// are clamped (and a value with its high bit set is negative, so it is
|
||||
// clamped to 0). Only qualities 0 and 1 are currently supported: transform_io
|
||||
// returns "#unsupported encoder quality" for the others.
|
||||
// clamped to 0).
|
||||
//
|
||||
// It is not named QUIRK_QUALITY because the base package already has a
|
||||
// base.QUIRK_QUALITY key (with different value semantics).
|
||||
|
||||
@@ -38,6 +38,18 @@ pri const CONTEXT_SIGNED : base.u32 = 3
|
||||
pri const HQ_MATCHES_PER_BYTE_BUDGET : base.u64 = 16
|
||||
pri const HQ_MATCHES_PER_BYTE_MAX : base.u64 = 100
|
||||
|
||||
// HQ_MATCHES_PER_BYTE_MAX_DICT is HQ_MATCHES_PER_BYTE_MAX plus up to 64 (the
|
||||
// C code's match_limit) compound dictionary matches, when a dictionary is
|
||||
// attached. HQ_SHADOW_MATCHES is the C code's shadow_matches (in
|
||||
// BrotliCreateHqZopfliBackwardReferences): with a dictionary attached, that
|
||||
// many more BackwardMatches past the current position's are scratch space
|
||||
// for the binary tree's and the compound dictionaries' matches before
|
||||
// MergeMatches. Quality 10's lz_matches_offset (in
|
||||
// BrotliZopfliComputeShortestPath) has the same value but fits in its
|
||||
// matches buffer of 2 * (MAX_NUM_MATCHES_H10 + 64) BackwardMatches.
|
||||
pri const HQ_MATCHES_PER_BYTE_MAX_DICT : base.u64 = 164
|
||||
pri const HQ_SHADOW_MATCHES : base.u64 = 256
|
||||
|
||||
// hq_max_literals returns an upper bound on the number of literals in a
|
||||
// metablock at qualities 10 and 11: EncodeData merges another input block
|
||||
// only while there are fewer than (max_len / 8) literals.
|
||||
@@ -50,16 +62,21 @@ pri func encoder.hq_max_literals(lgwin: base.u32[..= 30], lgblock: base.u32[..=
|
||||
|
||||
// hq_zopfli_len returns how much of the variable region the Zopfli backward
|
||||
// references need, for an input block of (1 << lgblock) bytes, with (at
|
||||
// quality 11) matches_per_byte BackwardMatches per byte.
|
||||
// quality 11) matches_per_byte BackwardMatches per byte (plus, if a
|
||||
// dictionary is attached, HQ_SHADOW_MATCHES).
|
||||
pri func encoder.hq_zopfli_len(quality: base.u32[..= 11], lgblock: base.u32[..= 30], matches_per_byte: base.u64) base.u64 {
|
||||
var n : base.u64
|
||||
var n : base.u64
|
||||
var shadow : base.u64
|
||||
|
||||
n = (1 as base.u64) << args.lgblock
|
||||
if args.quality <= 10 {
|
||||
return ((16 * (n + 1)) + (4 * (n + 2))) + (8 * 2 * (MAX_NUM_MATCHES_H10 + 64))
|
||||
}
|
||||
if this.dict_num_chunks > 0 {
|
||||
shadow = HQ_SHADOW_MATCHES
|
||||
}
|
||||
return (((16 * (n + 1)) + (4 * (n + 2))) + (4 * n)) ~sat+
|
||||
(8 ~mod* ((args.matches_per_byte ~mod* n) ~sat+ MAX_NUM_MATCHES_H10))
|
||||
(8 ~mod* (((args.matches_per_byte ~mod* n) ~sat+ MAX_NUM_MATCHES_H10) ~sat+ shadow))
|
||||
}
|
||||
|
||||
// hq_var_len returns the minimum (if not worst_case) or the worst case (if
|
||||
@@ -69,7 +86,11 @@ pri func encoder.hq_var_len(quality: base.u32[..= 11], lgwin: base.u32[..= 30],
|
||||
var m : base.u64
|
||||
|
||||
if args.worst_case {
|
||||
z = this.hq_zopfli_len(quality: args.quality, lgblock: args.lgblock, matches_per_byte: HQ_MATCHES_PER_BYTE_MAX)
|
||||
if this.dict_num_chunks > 0 {
|
||||
z = this.hq_zopfli_len(quality: args.quality, lgblock: args.lgblock, matches_per_byte: HQ_MATCHES_PER_BYTE_MAX_DICT)
|
||||
} else {
|
||||
z = this.hq_zopfli_len(quality: args.quality, lgblock: args.lgblock, matches_per_byte: HQ_MATCHES_PER_BYTE_MAX)
|
||||
}
|
||||
m = this.mb.workbuf_len_worst_case(
|
||||
num_literals: this.hq_max_literals(lgwin: args.lgwin, lgblock: args.lgblock),
|
||||
num_commands: this.commands_capacity(quality: args.quality, lgwin: args.lgwin, lgblock: args.lgblock))
|
||||
|
||||
+158
-26
@@ -489,8 +489,8 @@ pri func encoder.compute_minimum_copy_length(wb: roslice base.u8, start_cost: ba
|
||||
return len
|
||||
}
|
||||
|
||||
// Mirrors ComputeDistanceShortcut in backward_references_hq.c (with a gap
|
||||
// of 0).
|
||||
// Mirrors ComputeDistanceShortcut in backward_references_hq.c. The gap is
|
||||
// this.dict_total_size.
|
||||
pri func encoder.compute_distance_shortcut(wb: roslice base.u8, block_start: base.u64, pos: base.u64, max_backward_limit: base.u64) base.u32 {
|
||||
var c_len : base.u64
|
||||
var i_len : base.u64
|
||||
@@ -507,8 +507,8 @@ pri func encoder.compute_distance_shortcut(wb: roslice base.u8, block_start: bas
|
||||
// distances.
|
||||
if args.pos == 0 {
|
||||
return 0
|
||||
} else if ((dist ~mod+ c_len) <= (args.block_start ~mod+ args.pos)) and
|
||||
(dist <= args.max_backward_limit) and
|
||||
} else if ((dist ~mod+ c_len) <= ((args.block_start ~mod+ args.pos) ~mod+ this.dict_total_size)) and
|
||||
(dist <= (args.max_backward_limit ~mod+ this.dict_total_size)) and
|
||||
(this.zn_distance_code(wb: args.wb, i: args.pos) > 0) {
|
||||
return (args.pos & 0xFFFF_FFFF) as base.u32
|
||||
}
|
||||
@@ -542,7 +542,8 @@ pri func encoder.compute_distance_cache!(wb: roslice base.u8, pos: base.u64) {
|
||||
}
|
||||
}
|
||||
|
||||
// Mirrors EvaluateNode in backward_references_hq.c.
|
||||
// Mirrors EvaluateNode in backward_references_hq.c. The gap is
|
||||
// this.dict_total_size.
|
||||
pri func encoder.evaluate_node!(wb: slice base.u8, block_start: base.u64, pos: base.u64, max_backward_limit: base.u64) {
|
||||
var node_cost : base.u32
|
||||
var lc : base.u32
|
||||
@@ -562,8 +563,9 @@ pri func encoder.evaluate_node!(wb: slice base.u8, block_start: base.u64, pos: b
|
||||
}
|
||||
|
||||
// Mirrors UpdateNodes in backward_references_hq.c. The num_matches matches
|
||||
// are at args.wb[matches_off ..]. It returns the longest copy length.
|
||||
pri func encoder.update_nodes!(wb: slice base.u8, num_bytes: base.u64, block_start: base.u64, pos: base.u64,
|
||||
// are at args.wb[matches_off ..] and args.dwb holds the dictionaries. The
|
||||
// gap is this.dict_total_size. It returns the longest copy length.
|
||||
pri func encoder.update_nodes!(wb: slice base.u8, dwb: roslice base.u8, num_bytes: base.u64, block_start: base.u64, pos: base.u64,
|
||||
max_backward_limit: base.u64, num_matches: base.u64, matches_off: base.u64) base.u64 {
|
||||
var mask : base.u64
|
||||
var cur_ix : base.u64
|
||||
@@ -604,8 +606,10 @@ pri func encoder.update_nodes!(wb: slice base.u8, num_bytes: base.u64, block_sta
|
||||
var distnumextra : base.u32
|
||||
var max_match_len : base.u64
|
||||
var len_code : base.u64
|
||||
var gap : base.u64
|
||||
|
||||
mask = this.rb_mask as base.u64
|
||||
gap = this.dict_total_size
|
||||
cur_ix = args.block_start ~mod+ args.pos
|
||||
cur_ix_masked = cur_ix & mask
|
||||
max_distance = cur_ix.min(no_more_than: args.max_backward_limit)
|
||||
@@ -656,7 +660,7 @@ pri func encoder.update_nodes!(wb: slice base.u8, num_bytes: base.u64, block_sta
|
||||
if (cur_ix_masked ~mod+ best_len) > mask {
|
||||
break
|
||||
}
|
||||
if backward > dictionary_start {
|
||||
if backward > (dictionary_start ~mod+ gap) {
|
||||
// Word dictionary -> ignore.
|
||||
j ~mod+= 1
|
||||
continue
|
||||
@@ -677,11 +681,13 @@ pri func encoder.update_nodes!(wb: slice base.u8, num_bytes: base.u64, block_sta
|
||||
len = this.find_match_length_with_limit(s: args.wb[this.rb_buf_off ..],
|
||||
i1: prev_ix, i2: cur_ix_masked, limit: max_len)
|
||||
}
|
||||
} else if backward > dictionary_start {
|
||||
len = this.compound_last_distance_match_len(wb: args.wb, dwb: args.dwb,
|
||||
cur_ix_masked: cur_ix_masked, dictionary_start: dictionary_start, backward: backward,
|
||||
best_len: best_len, max_len: max_len, continuation: continuation)
|
||||
} else {
|
||||
// "Gray" area. It is addressable by decoder, but this encoder
|
||||
// instance does not have that data -> should not touch it.
|
||||
// (Without a compound dictionary, backward > dictionary_start
|
||||
// was handled above.)
|
||||
j ~mod+= 1
|
||||
continue
|
||||
}
|
||||
@@ -723,7 +729,7 @@ pri func encoder.update_nodes!(wb: slice base.u8, num_bytes: base.u64, block_sta
|
||||
while j < args.num_matches {
|
||||
moff = args.matches_off ~mod+ (j ~mod* 8)
|
||||
dist = this.bm_distance(wb: args.wb, off: moff)
|
||||
is_dictionary_match = dist > dictionary_start
|
||||
is_dictionary_match = dist > (dictionary_start ~mod+ gap)
|
||||
// We already tried all possible last distance matches, so we can
|
||||
// use normal distance code here.
|
||||
dist_code = dist ~mod+ 15
|
||||
@@ -769,6 +775,51 @@ pri func encoder.update_nodes!(wb: slice base.u8, num_bytes: base.u64, block_sta
|
||||
return result
|
||||
}
|
||||
|
||||
// compound_last_distance_match_len mirrors the compound dictionary branch
|
||||
// of UpdateNodes' last distance loop in backward_references_hq.c, for a
|
||||
// backward distance greater than dictionary_start (and at most
|
||||
// dictionary_start plus the gap, this.dict_total_size). It returns the C
|
||||
// code's len, or 0 where the C code continues with the next distance (which
|
||||
// is equivalent, as UpdateNodes then tries no copy lengths).
|
||||
pri func encoder.compound_last_distance_match_len(wb: roslice base.u8, dwb: roslice base.u8, cur_ix_masked: base.u64,
|
||||
dictionary_start: base.u64, backward: base.u64, best_len: base.u64, max_len: base.u64, continuation: base.u8) base.u64 {
|
||||
var d : base.u32[..= 14]
|
||||
var offset : base.u64
|
||||
var limit : base.u64
|
||||
var src_lo : base.u64
|
||||
var src_hi : base.u64
|
||||
var source : roslice base.u8
|
||||
|
||||
offset = args.dictionary_start ~mod+ this.dict_total_size
|
||||
// The C code's loop condition is only the second check below: it fails
|
||||
// for the last chunk, as backward > dictionary_start.
|
||||
while d < 14 {
|
||||
if ((d + 1) >= this.dict_num_chunks) or
|
||||
(offset < (args.backward ~mod+ this.dict_chunk_offsets[d + 1])) {
|
||||
break
|
||||
}
|
||||
d += 1
|
||||
}
|
||||
offset = (offset ~mod- this.dict_chunk_offsets[d]) ~mod- args.backward
|
||||
limit = (this.dict_chunk_offsets[d + 1] ~mod- this.dict_chunk_offsets[d]) ~mod- offset
|
||||
limit = limit.min(no_more_than: args.max_len)
|
||||
src_lo = this.dc_src_off[d]
|
||||
src_hi = src_lo ~mod+ this.dc_size[d]
|
||||
if (src_lo > src_hi) or (src_hi > args.dwb.length()) {
|
||||
return 0
|
||||
}
|
||||
source = args.dwb[src_lo .. src_hi]
|
||||
if (args.best_len >= limit) or
|
||||
(args.continuation <> this.byte_at(s: source, i: offset ~mod+ args.best_len)) {
|
||||
return 0
|
||||
}
|
||||
if (offset > source.length()) or ((this.rb_buf_off ~mod+ args.cur_ix_masked) > args.wb.length()) {
|
||||
return 0
|
||||
}
|
||||
return this.find_match_length_2(a: source[offset ..],
|
||||
b: args.wb[this.rb_buf_off ~mod+ args.cur_ix_masked ..], limit: limit)
|
||||
}
|
||||
|
||||
// Mirrors ComputeShortestPathFromNodes in backward_references_hq.c.
|
||||
pri func encoder.compute_shortest_path_from_nodes!(wb: slice base.u8, num_bytes: base.u64) base.u64 {
|
||||
var index : base.u64
|
||||
@@ -797,7 +848,8 @@ pri func encoder.compute_shortest_path_from_nodes!(wb: slice base.u8, num_bytes:
|
||||
}
|
||||
|
||||
// Mirrors BrotliZopfliCreateCommands in backward_references_hq.c. The
|
||||
// commands go to command indexes cmds_base and up.
|
||||
// commands go to command indexes cmds_base and up. The gap is
|
||||
// this.dict_total_size.
|
||||
pri func encoder.zopfli_create_commands!(wb: slice base.u8, num_bytes: base.u64, block_start: base.u64, cmds_base: base.u64) {
|
||||
var max_backward_limit : base.u64
|
||||
var pos : base.u64
|
||||
@@ -827,7 +879,7 @@ pri func encoder.zopfli_create_commands!(wb: slice base.u8, num_bytes: base.u64,
|
||||
distance = this.zn_distance(wb: args.wb, i: next) as base.u64
|
||||
len_code = this.zn_length_code(wb: args.wb, i: next)
|
||||
dictionary_start = this.min64(a: (args.block_start ~mod+ pos) ~mod+ (this.stream_offset as base.u64), b: max_backward_limit)
|
||||
is_dictionary = distance > dictionary_start
|
||||
is_dictionary = distance > (dictionary_start ~mod+ this.dict_total_size)
|
||||
dist_code = this.zn_distance_code(wb: args.wb, i: next)
|
||||
this.init_command!(wb: args.wb, n: args.cmds_base ~mod+ i, insertlen: insert_length,
|
||||
copylen: copy_length, copylen_code_delta: len_code ~mod- copy_length, distance_code: dist_code)
|
||||
@@ -846,7 +898,7 @@ pri func encoder.zopfli_create_commands!(wb: slice base.u8, num_bytes: base.u64,
|
||||
|
||||
// Mirrors ZopfliIterate in backward_references_hq.c. The num_matches array
|
||||
// is at this.z_nm_off and the matches at this.z_m_off.
|
||||
pri func encoder.zopfli_iterate!(wb: slice base.u8, num_bytes: base.u64, position: base.u64) base.u64 {
|
||||
pri func encoder.zopfli_iterate!(wb: slice base.u8, dwb: roslice base.u8, num_bytes: base.u64, position: base.u64) base.u64 {
|
||||
var max_backward_limit : base.u64
|
||||
var max_zopfli_len : base.u64
|
||||
var cur_match_pos : base.u64
|
||||
@@ -866,7 +918,7 @@ pri func encoder.zopfli_iterate!(wb: slice base.u8, num_bytes: base.u64, positio
|
||||
i = 0
|
||||
while (i ~mod+ 3) < args.num_bytes {
|
||||
nm = this.load32le(s: args.wb, i: this.z_nm_off ~mod+ (i ~mod* 4)) as base.u64
|
||||
skip = this.update_nodes!(wb: args.wb, num_bytes: args.num_bytes, block_start: args.position, pos: i,
|
||||
skip = this.update_nodes!(wb: args.wb, dwb: args.dwb, num_bytes: args.num_bytes, block_start: args.position, pos: i,
|
||||
max_backward_limit: max_backward_limit, num_matches: nm,
|
||||
matches_off: this.z_m_off ~mod+ (cur_match_pos ~mod* 8))
|
||||
if skip < BROTLI_LONG_COPY_QUICK_STEP {
|
||||
@@ -929,8 +981,9 @@ pri func encoder.z_set_offsets!(wb_len: base.u64, num_bytes: base.u64) base.bool
|
||||
}
|
||||
|
||||
// Mirrors BrotliZopfliComputeShortestPath in backward_references_hq.c. The
|
||||
// nodes were initialized by the caller.
|
||||
pri func encoder.zopfli_compute_shortest_path!(wb: slice base.u8, num_bytes: base.u64, position: base.u64) base.u64 {
|
||||
// nodes were initialized by the caller. The matches buffer (of 2 *
|
||||
// (MAX_NUM_MATCHES_H10 + 64) BackwardMatches) is at this.z_m_off.
|
||||
pri func encoder.zopfli_compute_shortest_path!(wb: slice base.u8, dwb: roslice base.u8, num_bytes: base.u64, position: base.u64) base.u64 {
|
||||
var max_backward_limit : base.u64
|
||||
var max_zopfli_len : base.u64
|
||||
var store_end : base.u64
|
||||
@@ -941,8 +994,15 @@ pri func encoder.zopfli_compute_shortest_path!(wb: slice base.u8, num_bytes: bas
|
||||
var skip : base.u64
|
||||
var num_matches : base.u64
|
||||
var ml : base.u64
|
||||
var lz_matches_off : base.u64
|
||||
var cd_matches : base.u64
|
||||
|
||||
max_backward_limit = ((1 as base.u64) << this.lgwin) ~mod- 16
|
||||
// lz_matches_off is the C code's lz_matches_offset, in bytes.
|
||||
lz_matches_off = this.z_m_off
|
||||
if this.dict_num_chunks <> 0 {
|
||||
lz_matches_off ~mod+= 8 * (MAX_NUM_MATCHES_H10 + 128)
|
||||
}
|
||||
max_zopfli_len = MAX_ZOPFLI_LEN_QUALITY_10
|
||||
if this.quality > 10 {
|
||||
max_zopfli_len = MAX_ZOPFLI_LEN_QUALITY_11
|
||||
@@ -962,7 +1022,18 @@ pri func encoder.zopfli_compute_shortest_path!(wb: slice base.u8, num_bytes: bas
|
||||
max_distance = pos.min(no_more_than: max_backward_limit)
|
||||
dictionary_start = this.min64(a: pos ~mod+ (this.stream_offset as base.u64), b: max_backward_limit)
|
||||
num_matches = this.h10_find_all_matches!(wb: args.wb, cur_ix: pos, max_length: args.num_bytes ~mod- i,
|
||||
max_backward: max_distance, dictionary_distance: dictionary_start, matches_off: this.z_m_off)
|
||||
max_backward: max_distance, dictionary_distance: dictionary_start ~mod+ this.dict_total_size,
|
||||
matches_off: lz_matches_off)
|
||||
if this.dict_num_chunks <> 0 {
|
||||
cd_matches = this.lookup_all_compound_dictionary_matches!(wb: args.wb, dwb: args.dwb,
|
||||
cur_ix: pos, min_length: 3, max_length: args.num_bytes ~mod- i,
|
||||
max_ring_buffer_distance: dictionary_start, max_distance: this.dist_max_distance,
|
||||
matches_off: lz_matches_off ~mod- (8 * 64), match_limit: 64)
|
||||
this.merge_matches!(wb: args.wb, dst: this.z_m_off,
|
||||
src1: lz_matches_off ~mod- (8 * 64), len1: cd_matches,
|
||||
src2: lz_matches_off, len2: num_matches)
|
||||
num_matches ~mod+= cd_matches
|
||||
}
|
||||
if num_matches > 0 {
|
||||
ml = this.bm_length(wb: args.wb, off: this.z_m_off ~mod+ ((num_matches ~mod- 1) ~mod* 8))
|
||||
if ml > max_zopfli_len {
|
||||
@@ -970,7 +1041,7 @@ pri func encoder.zopfli_compute_shortest_path!(wb: slice base.u8, num_bytes: bas
|
||||
num_matches = 1
|
||||
}
|
||||
}
|
||||
skip = this.update_nodes!(wb: args.wb, num_bytes: args.num_bytes, block_start: args.position, pos: i,
|
||||
skip = this.update_nodes!(wb: args.wb, dwb: args.dwb, num_bytes: args.num_bytes, block_start: args.position, pos: i,
|
||||
max_backward_limit: max_backward_limit, num_matches: num_matches, matches_off: this.z_m_off)
|
||||
if skip < BROTLI_LONG_COPY_QUICK_STEP {
|
||||
skip = 0
|
||||
@@ -1002,7 +1073,7 @@ pri func encoder.zopfli_compute_shortest_path!(wb: slice base.u8, num_bytes: bas
|
||||
}
|
||||
|
||||
// Mirrors BrotliCreateZopfliBackwardReferences in backward_references_hq.c.
|
||||
pri func encoder.create_zopfli_backward_references!(wb: slice base.u8, num_bytes: base.u64, position: base.u64) base.status {
|
||||
pri func encoder.create_zopfli_backward_references!(wb: slice base.u8, dwb: roslice base.u8, num_bytes: base.u64, position: base.u64) base.status {
|
||||
var cmds_base : base.u64
|
||||
var n : base.u64
|
||||
var ok_offsets : base.bool
|
||||
@@ -1013,7 +1084,7 @@ pri func encoder.create_zopfli_backward_references!(wb: slice base.u8, num_bytes
|
||||
}
|
||||
cmds_base = this.num_cmds
|
||||
this.init_zopfli_nodes!(wb: args.wb, length: args.num_bytes ~mod+ 1)
|
||||
n = this.zopfli_compute_shortest_path!(wb: args.wb, num_bytes: args.num_bytes, position: args.position)
|
||||
n = this.zopfli_compute_shortest_path!(wb: args.wb, dwb: args.dwb, num_bytes: args.num_bytes, position: args.position)
|
||||
if (cmds_base ~mod+ n) > this.cmds_cap {
|
||||
return "#internal error: inconsistent encoder state"
|
||||
}
|
||||
@@ -1024,7 +1095,7 @@ pri func encoder.create_zopfli_backward_references!(wb: slice base.u8, num_bytes
|
||||
|
||||
// Mirrors BrotliCreateHqZopfliBackwardReferences in
|
||||
// backward_references_hq.c.
|
||||
pri func encoder.create_hq_zopfli_backward_references!(wb: slice base.u8, num_bytes: base.u64, position: base.u64) base.status {
|
||||
pri func encoder.create_hq_zopfli_backward_references!(wb: slice base.u8, dwb: roslice base.u8, num_bytes: base.u64, position: base.u64) base.status {
|
||||
var max_backward_limit : base.u64
|
||||
var store_end : base.u64
|
||||
var cur_match_pos : base.u64
|
||||
@@ -1046,6 +1117,8 @@ pri func encoder.create_hq_zopfli_backward_references!(wb: slice base.u8, num_by
|
||||
var k : base.u32
|
||||
var n : base.u64
|
||||
var ok_offsets : base.bool
|
||||
var shadow_matches : base.u64
|
||||
var cd_matches : base.u64
|
||||
|
||||
ok_offsets = this.z_set_offsets!(wb_len: this.var_end.min(no_more_than: args.wb.length()), num_bytes: args.num_bytes)
|
||||
if not ok_offsets {
|
||||
@@ -1056,6 +1129,9 @@ pri func encoder.create_hq_zopfli_backward_references!(wb: slice base.u8, num_by
|
||||
if args.num_bytes >= H10_MAX_TREE_COMP_LENGTH {
|
||||
store_end = ((args.position ~mod+ args.num_bytes) ~mod- H10_MAX_TREE_COMP_LENGTH) ~mod+ 1
|
||||
}
|
||||
if this.dict_num_chunks <> 0 {
|
||||
shadow_matches = MAX_NUM_MATCHES_H10 + 128
|
||||
}
|
||||
i = 0
|
||||
while (i ~mod+ 3) < args.num_bytes {
|
||||
pos = args.position ~mod+ i
|
||||
@@ -1063,12 +1139,23 @@ pri func encoder.create_hq_zopfli_backward_references!(wb: slice base.u8, num_by
|
||||
dictionary_start = this.min64(a: pos ~mod+ (this.stream_offset as base.u64), b: max_backward_limit)
|
||||
max_length = args.num_bytes ~mod- i
|
||||
// Ensure that we have enough free slots.
|
||||
if (cur_match_pos ~mod+ MAX_NUM_MATCHES_H10) > this.z_m_cap {
|
||||
if ((cur_match_pos ~mod+ MAX_NUM_MATCHES_H10) ~mod+ shadow_matches) > this.z_m_cap {
|
||||
return base."#bad workbuf length"
|
||||
}
|
||||
num_found_matches = this.h10_find_all_matches!(wb: args.wb, cur_ix: pos, max_length: max_length,
|
||||
max_backward: max_distance, dictionary_distance: dictionary_start,
|
||||
matches_off: this.z_m_off ~mod+ (cur_match_pos ~mod* 8))
|
||||
max_backward: max_distance, dictionary_distance: dictionary_start ~mod+ this.dict_total_size,
|
||||
matches_off: this.z_m_off ~mod+ ((cur_match_pos ~mod+ shadow_matches) ~mod* 8))
|
||||
if this.dict_num_chunks <> 0 {
|
||||
cd_matches = this.lookup_all_compound_dictionary_matches!(wb: args.wb, dwb: args.dwb,
|
||||
cur_ix: pos, min_length: 3, max_length: max_length,
|
||||
max_ring_buffer_distance: dictionary_start, max_distance: this.dist_max_distance,
|
||||
matches_off: this.z_m_off ~mod+ (((cur_match_pos ~mod+ shadow_matches) ~mod- 64) ~mod* 8),
|
||||
match_limit: 64)
|
||||
this.merge_matches!(wb: args.wb, dst: this.z_m_off ~mod+ (cur_match_pos ~mod* 8),
|
||||
src1: this.z_m_off ~mod+ (((cur_match_pos ~mod+ shadow_matches) ~mod- 64) ~mod* 8), len1: cd_matches,
|
||||
src2: this.z_m_off ~mod+ ((cur_match_pos ~mod+ shadow_matches) ~mod* 8), len2: num_found_matches)
|
||||
num_found_matches ~mod+= cd_matches
|
||||
}
|
||||
cur_match_end = cur_match_pos ~mod+ num_found_matches
|
||||
this.poke_u32!(s: args.wb, i: this.z_nm_off ~mod+ (i ~mod* 4), v: (num_found_matches & 0xFFFF_FFFF) as base.u32)
|
||||
if num_found_matches > 0 {
|
||||
@@ -1121,7 +1208,7 @@ pri func encoder.create_hq_zopfli_backward_references!(wb: slice base.u8, num_by
|
||||
this.dist_cache[k] = orig_dist_cache[k]
|
||||
k ~mod+= 1
|
||||
}
|
||||
n = this.zopfli_iterate!(wb: args.wb, num_bytes: args.num_bytes, position: args.position)
|
||||
n = this.zopfli_iterate!(wb: args.wb, dwb: args.dwb, num_bytes: args.num_bytes, position: args.position)
|
||||
if (orig_num_commands ~mod+ n) > this.cmds_cap {
|
||||
return "#internal error: inconsistent encoder state"
|
||||
}
|
||||
@@ -1133,6 +1220,51 @@ pri func encoder.create_hq_zopfli_backward_references!(wb: slice base.u8, num_by
|
||||
return ok
|
||||
}
|
||||
|
||||
// Mirrors MergeMatches in backward_references_hq.c: it merges the len1
|
||||
// BackwardMatches at args.wb[src1 ..] and the len2 at args.wb[src2 ..] (each
|
||||
// sorted by length) into args.wb[dst ..].
|
||||
pri func encoder.merge_matches!(wb: slice base.u8, dst: base.u64, src1: base.u64, len1: base.u64, src2: base.u64, len2: base.u64) {
|
||||
var dst : base.u64
|
||||
var src1 : base.u64
|
||||
var len1 : base.u64
|
||||
var src2 : base.u64
|
||||
var len2 : base.u64
|
||||
var l1 : base.u64
|
||||
var l2 : base.u64
|
||||
|
||||
dst = args.dst
|
||||
src1 = args.src1
|
||||
len1 = args.len1
|
||||
src2 = args.src2
|
||||
len2 = args.len2
|
||||
while (len1 > 0) and (len2 > 0) {
|
||||
l1 = this.bm_length(wb: args.wb, off: src1)
|
||||
l2 = this.bm_length(wb: args.wb, off: src2)
|
||||
if (l1 < l2) or ((l1 == l2) and (this.bm_distance(wb: args.wb, off: src1) < this.bm_distance(wb: args.wb, off: src2))) {
|
||||
this.bm_copy!(wb: args.wb, dst: dst, src: src1)
|
||||
src1 ~mod+= 8
|
||||
len1 -= 1
|
||||
} else {
|
||||
this.bm_copy!(wb: args.wb, dst: dst, src: src2)
|
||||
src2 ~mod+= 8
|
||||
len2 -= 1
|
||||
}
|
||||
dst ~mod+= 8
|
||||
}
|
||||
while len1 > 0 {
|
||||
this.bm_copy!(wb: args.wb, dst: dst, src: src1)
|
||||
src1 ~mod+= 8
|
||||
dst ~mod+= 8
|
||||
len1 -= 1
|
||||
}
|
||||
while len2 > 0 {
|
||||
this.bm_copy!(wb: args.wb, dst: dst, src: src2)
|
||||
src2 ~mod+= 8
|
||||
dst ~mod+= 8
|
||||
len2 -= 1
|
||||
}
|
||||
}
|
||||
|
||||
// min32_543 returns min(a, 543).
|
||||
pri func encoder.min32_543(a: base.u32) base.u32[..= 543] {
|
||||
if args.a < 543 {
|
||||
|
||||
+102
-58
@@ -1604,30 +1604,54 @@ test_wuffs_brotli_encode_parameters() {
|
||||
BROTLI_ENC_CHECK(status.repr == wuffs_base__error__bad_workbuf_length,
|
||||
"short workbuf at quality 10..11");
|
||||
}
|
||||
// Compound dictionaries are not supported (yet) at qualities 10 and 11.
|
||||
{
|
||||
// Compound dictionaries at qualities 10 and 11. The want bytes are
|
||||
// libbrotlienc's output (with the dictionary prepared and attached, lgwin
|
||||
// 16 and BROTLI_OPERATION_FINISH), which is the same at both qualities.
|
||||
for (uint32_t q = 10; q <= 11; q++) {
|
||||
free(enc);
|
||||
enc = wuffs_brotli__encoder__alloc();
|
||||
if (!enc) {
|
||||
RETURN_FAIL("alloc failed");
|
||||
}
|
||||
wuffs_brotli__encoder__set_quirk(enc, WUFFS_BROTLI__QUIRK_ENCODER_QUALITY,
|
||||
q);
|
||||
wuffs_brotli__encoder__set_quirk(enc, WUFFS_BROTLI__QUIRK_LGWIN, 16);
|
||||
static const char dict_bytes[] = "a raw dictionary, for quality 11";
|
||||
static const char input[] = "for quality 10 or 11, a raw dictionary";
|
||||
static const uint8_t want[] = {
|
||||
0xA2, 0x04, 0x00, 0xBF, 0x34, 0x08, 0x0B, 0x4C, 0x8C, 0x8C,
|
||||
0x20, 0x18, 0x71, 0xEA, 0x81, 0x22, 0x4A, 0xEC, 0x4B,
|
||||
};
|
||||
wuffs_base__slice_u8 d = wuffs_base__make_slice_u8(
|
||||
(uint8_t*)dict_bytes, sizeof dict_bytes - 1);
|
||||
// The workbuf (whose prefix holds the prepared dictionary) is longer
|
||||
// than g_work_slice_u8, so it is malloc'ed and then realloc'ed.
|
||||
uint64_t n = wuffs_brotli__encoder__add_dictionary_workbuf_len(enc, d.len);
|
||||
BROTLI_ENC_CHECK(n <= g_work_slice_u8.len, "add_dictionary_workbuf_len");
|
||||
uint8_t* p = malloc((size_t)n);
|
||||
BROTLI_ENC_CHECK(p, "malloc failed");
|
||||
status = wuffs_brotli__encoder__add_dictionary(
|
||||
enc, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, d,
|
||||
wuffs_base__make_slice_u8(g_work_slice_u8.ptr, (size_t)n));
|
||||
BROTLI_ENC_CHECK(!status.repr, "add_dictionary at quality 11");
|
||||
src = make_io_buffer_from_string("abc", 3);
|
||||
wuffs_base__make_slice_u8(p, (size_t)n));
|
||||
if (status.repr) {
|
||||
free(p);
|
||||
}
|
||||
BROTLI_ENC_CHECK(!status.repr, "add_dictionary at quality 10..11");
|
||||
n = wuffs_brotli__encoder__workbuf_len(enc).min_incl;
|
||||
uint8_t* p2 = realloc(p, (size_t)n);
|
||||
if (!p2) {
|
||||
free(p);
|
||||
}
|
||||
BROTLI_ENC_CHECK(p2, "realloc failed");
|
||||
src = make_io_buffer_from_string(input, sizeof input - 1);
|
||||
src.meta.closed = true;
|
||||
dst.meta.wi = 0;
|
||||
status = wuffs_brotli__encoder__transform_io(enc, &dst, &src,
|
||||
g_work_slice_u8);
|
||||
BROTLI_ENC_CHECK(
|
||||
status.repr == wuffs_brotli__error__unsupported_encoder_quality,
|
||||
"dictionary at quality 11");
|
||||
status = wuffs_brotli__encoder__transform_io(
|
||||
enc, &dst, &src, wuffs_base__make_slice_u8(p2, (size_t)n));
|
||||
free(p2);
|
||||
BROTLI_ENC_CHECK(!status.repr, "dictionary at quality 10..11");
|
||||
BROTLI_ENC_CHECK((dst.meta.wi == sizeof want) &&
|
||||
!memcmp(dst.data.ptr, want, sizeof want),
|
||||
"dictionary at quality 10..11: output bytes");
|
||||
}
|
||||
src = make_io_buffer_from_string("", 0);
|
||||
dst.meta.wi = 0;
|
||||
@@ -4186,11 +4210,16 @@ do_test_brotli_encode_dictionary_grid(bool mimic) {
|
||||
r_len);
|
||||
const uint8_t* data = (set == 4) ? r : (a + (a_len / 2));
|
||||
size_t data_len = (set == 4) ? r_len : (a_len - (a_len / 2));
|
||||
for (uint32_t q = 0; (q <= 9) && !ret; q++) {
|
||||
for (uint32_t q = 0; (q <= 11) && !ret; q++) {
|
||||
for (size_t i = 0; (i < WUFFS_TESTLIB_ARRAY_SIZE(configs)) && !ret;
|
||||
i++) {
|
||||
if (!mimic && (i >= 2) && (q & 1)) {
|
||||
continue;
|
||||
} else if ((q >= 10) &&
|
||||
((i == 2) || (i == 7) || (!mimic && (i != 0)) ||
|
||||
((q == 11) && (i & 1)))) {
|
||||
// Qualities 10 and 11 are slow, so they skip some configs.
|
||||
continue;
|
||||
}
|
||||
brotli_enc_config c = make_brotli_enc_config(q, configs[i][0]);
|
||||
if (configs[i][4]) {
|
||||
@@ -4263,10 +4292,17 @@ do_test_brotli_encode_dictionary_big(bool mimic) {
|
||||
wuffs_base__make_slice_u8(dict, dlen),
|
||||
wuffs_base__make_slice_u8(data + (len - 1000), 1000),
|
||||
};
|
||||
static const uint32_t qs[] = {2, 4, 5, 9};
|
||||
static const uint32_t qs[] = {2, 4, 5, 9, 10, 11};
|
||||
for (size_t i = 0; (i < WUFFS_TESTLIB_ARRAY_SIZE(qs)) && !ret; i++) {
|
||||
// Without mimic, and for the largest dictionary, a smaller grid.
|
||||
if ((!mimic && ((k & 1) == 0)) || ((!mimic || (k == 3)) && (i != 2))) {
|
||||
// Without mimic, and for the largest dictionary, a smaller grid. The
|
||||
// slow qualities 10 and 11 only run with mimic, for one dictionary
|
||||
// each.
|
||||
if (qs[i] >= 10) {
|
||||
if (!mimic || (k != ((qs[i] == 10) ? 3 : 1))) {
|
||||
continue;
|
||||
}
|
||||
} else if ((!mimic && ((k & 1) == 0)) ||
|
||||
((!mimic || (k == 3)) && (i != 2))) {
|
||||
continue;
|
||||
}
|
||||
brotli_enc_config c = make_brotli_enc_config(qs[i], 24);
|
||||
@@ -4279,13 +4315,17 @@ do_test_brotli_encode_dictionary_big(bool mimic) {
|
||||
char s[256];
|
||||
snprintf(s, sizeof s, "dictionary of %zu bytes: q=%u", dlen, qs[i]);
|
||||
size_t have_len = 0;
|
||||
ret = do_test_brotli_encode_dictionary(s, &c, data, len, 2, dicts,
|
||||
mimic, &have_len);
|
||||
// At qualities 10 and 11, only the first 128 KiB of the input (which
|
||||
// has the first piece) is encoded.
|
||||
ret = do_test_brotli_encode_dictionary(
|
||||
s, &c, data, (qs[i] >= 10) ? (128 << 10) : len, 2, dicts, mimic,
|
||||
&have_len);
|
||||
// The three 64 KiB pieces (of 3.7 bits of entropy per byte) should be
|
||||
// found at quality 5 and above, even when they are more than 16 MiB
|
||||
// (lgwin 24's window) into the dictionary.
|
||||
// (lgwin 24's window) into the dictionary. (At the slow qualities 10
|
||||
// and 11, this is left to the parity check.)
|
||||
size_t want_max = 0;
|
||||
if (!ret && (qs[i] >= 5)) {
|
||||
if (!ret && (qs[i] >= 5) && (qs[i] <= 9)) {
|
||||
c = make_brotli_enc_config(qs[i], 24);
|
||||
ret = do_test_brotli_encode_dictionary(s, &c, data, len, 0, NULL,
|
||||
false, &want_max);
|
||||
@@ -4359,47 +4399,51 @@ test_mimic_brotli_encode_dictionary_rejections() {
|
||||
{1, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, 0, NULL, NULL},
|
||||
{3, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, 0, NULL, NULL},
|
||||
};
|
||||
for (size_t i = 0; i < WUFFS_TESTLIB_ARRAY_SIZE(cases); i++) {
|
||||
uint32_t dict_types[16];
|
||||
wuffs_base__slice_u8 dicts[16];
|
||||
for (size_t j = 0; j < cases[i].num_dicts; j++) {
|
||||
dict_types[j] = WUFFS_BROTLI__DICTIONARY_TYPE_RAW;
|
||||
dicts[j] = wuffs_base__make_slice_u8((uint8_t*)text + j, 20);
|
||||
}
|
||||
size_t last = cases[i].num_dicts - 1;
|
||||
dict_types[last] = cases[i].type;
|
||||
dicts[last] = wuffs_base__make_slice_u8(
|
||||
(uint8_t*)((cases[i].type == WUFFS_BROTLI__DICTIONARY_TYPE_RAW)
|
||||
? text
|
||||
: serialized),
|
||||
cases[i].len);
|
||||
// Quality 5 and the Zopfli qualities 10 and 11.
|
||||
static const uint32_t qs[] = {5, 10, 11};
|
||||
for (size_t iq = 0; iq < WUFFS_TESTLIB_ARRAY_SIZE(qs); iq++) {
|
||||
for (size_t i = 0; i < WUFFS_TESTLIB_ARRAY_SIZE(cases); i++) {
|
||||
uint32_t dict_types[16];
|
||||
wuffs_base__slice_u8 dicts[16];
|
||||
for (size_t j = 0; j < cases[i].num_dicts; j++) {
|
||||
dict_types[j] = WUFFS_BROTLI__DICTIONARY_TYPE_RAW;
|
||||
dicts[j] = wuffs_base__make_slice_u8((uint8_t*)text + j, 20);
|
||||
}
|
||||
size_t last = cases[i].num_dicts - 1;
|
||||
dict_types[last] = cases[i].type;
|
||||
dicts[last] = wuffs_base__make_slice_u8(
|
||||
(uint8_t*)((cases[i].type == WUFFS_BROTLI__DICTIONARY_TYPE_RAW)
|
||||
? text
|
||||
: serialized),
|
||||
cases[i].len);
|
||||
|
||||
brotli_enc_config c = make_brotli_enc_config(5, 22);
|
||||
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
|
||||
.data = g_src_slice_u8,
|
||||
});
|
||||
CHECK_STRING(make_brotli_enc_plan(&src, &c, text, sizeof text - 1));
|
||||
wuffs_base__io_buffer dsts[2] = {
|
||||
((wuffs_base__io_buffer){.data = g_have_slice_u8}),
|
||||
((wuffs_base__io_buffer){.data = g_want_slice_u8}),
|
||||
};
|
||||
for (int k = 0; k < 2; k++) {
|
||||
wuffs_base__io_buffer src1 = src;
|
||||
const char* have =
|
||||
(k ? mimic_brotli_encode_with_dictionaries
|
||||
: wuffs_brotli_encode_plan_with_dictionaries)(
|
||||
&dsts[k], &src1, c.param_keys, c.param_values, c.num_params,
|
||||
cases[i].num_dicts, dict_types, dicts, g_brotli_enc_step_ops,
|
||||
g_brotli_enc_step_lens, g_brotli_enc_num_steps, c.dst_chunk);
|
||||
const char* want = k ? cases[i].want_mimic : cases[i].want_wuffs;
|
||||
if (have != want) {
|
||||
RETURN_FAIL("cases[%zu] (%s): have \"%s\", want \"%s\"", i,
|
||||
k ? "mimic" : "wuffs", have ? have : "",
|
||||
want ? want : "");
|
||||
brotli_enc_config c = make_brotli_enc_config(qs[iq], 22);
|
||||
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
|
||||
.data = g_src_slice_u8,
|
||||
});
|
||||
CHECK_STRING(make_brotli_enc_plan(&src, &c, text, sizeof text - 1));
|
||||
wuffs_base__io_buffer dsts[2] = {
|
||||
((wuffs_base__io_buffer){.data = g_have_slice_u8}),
|
||||
((wuffs_base__io_buffer){.data = g_want_slice_u8}),
|
||||
};
|
||||
for (int k = 0; k < 2; k++) {
|
||||
wuffs_base__io_buffer src1 = src;
|
||||
const char* have =
|
||||
(k ? mimic_brotli_encode_with_dictionaries
|
||||
: wuffs_brotli_encode_plan_with_dictionaries)(
|
||||
&dsts[k], &src1, c.param_keys, c.param_values, c.num_params,
|
||||
cases[i].num_dicts, dict_types, dicts, g_brotli_enc_step_ops,
|
||||
g_brotli_enc_step_lens, g_brotli_enc_num_steps, c.dst_chunk);
|
||||
const char* want = k ? cases[i].want_mimic : cases[i].want_wuffs;
|
||||
if (have != want) {
|
||||
RETURN_FAIL("q=%u cases[%zu] (%s): have \"%s\", want \"%s\"",
|
||||
qs[iq], i, k ? "mimic" : "wuffs", have ? have : "",
|
||||
want ? want : "");
|
||||
}
|
||||
}
|
||||
if (!cases[i].want_wuffs) {
|
||||
CHECK_STRING(check_io_buffers_equal("", &dsts[0], &dsts[1]));
|
||||
}
|
||||
}
|
||||
if (!cases[i].want_wuffs) {
|
||||
CHECK_STRING(check_io_buffers_equal("", &dsts[0], &dsts[1]));
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
|
||||
Reference in new issue
Block a user