diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index e5b35dc4..c78ddcfb 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -10025,6 +10025,12 @@ struct wuffs_brotli__encoder__struct { uint64_t f_dict_num_lookups; uint64_t f_dict_num_matches; uint64_t f_fc_max_hops; + uint32_t f_hr_jump; + bool f_hr_fresh; + uint32_t f_hr_state; + uint32_t f_hr_factor_remove; + uint64_t f_hr_next_ix; + uint64_t f_hr_off; uint64_t f_sr_len; uint64_t f_sr_distance; uint64_t f_sr_score; @@ -68546,6 +68552,16 @@ WUFFS_BROTLI__DICTIONARY[122784] WUFFS_BASE__POTENTIALLY_UNUSED = { #define WUFFS_BROTLI__SCORE_BASE 1920u +#define WUFFS_BROTLI__ROLLING_CHUNKLEN 32u + +#define WUFFS_BROTLI__ROLLING_NUMBUCKETS 16777216u + +#define WUFFS_BROTLI__ROLLING_MASK 1073741823u + +#define WUFFS_BROTLI__ROLLING_HASH_MUL_32 69069u + +#define WUFFS_BROTLI__ROLLING_INVALID_POS 4294967295u + #define WUFFS_BROTLI__MAX_NUMBER_OF_HISTOGRAMS 256u #define WUFFS_BROTLI__INVALID_INDEX 4294967295u @@ -70596,6 +70612,42 @@ wuffs_brotli__encoder__find_longest_match_forgetful( uint64_t a_dictionary_distance, uint64_t a_max_distance); +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__encoder__hasher_rolling_alloc_size( + const wuffs_brotli__encoder* self); + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__empty_struct +wuffs_brotli__encoder__hasher_rolling_initialize( + wuffs_brotli__encoder* self, + wuffs_base__slice_u8 a_wb); + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__empty_struct +wuffs_brotli__encoder__hasher_rolling_prepare( + wuffs_brotli__encoder* self, + wuffs_base__slice_u8 a_wb, + uint64_t a_input_size, + uint64_t a_data_off); + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__empty_struct +wuffs_brotli__encoder__hasher_rolling_stitch_to_previous_block( + wuffs_brotli__encoder* self, + wuffs_base__slice_u8 a_wb, + uint64_t a_num_bytes, + uint64_t a_position); + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__empty_struct +wuffs_brotli__encoder__hasher_rolling_find_longest_match( + wuffs_brotli__encoder* self, + wuffs_base__slice_u8 a_wb, + uint64_t a_cur_ix, + uint64_t a_max_length, + uint64_t a_max_backward); + WUFFS_BASE__GENERATED_C_CODE static wuffs_base__empty_struct wuffs_brotli__bit_cost__histogram_clear( @@ -78813,7 +78865,7 @@ wuffs_brotli__encoder__workbuf_len( v_lgwin = wuffs_brotli__encoder__sanitized_lgwin(self, v_q); if (v_q <= 1u) { v_n = ((((uint64_t)(2u)) << v_lgwin) + 503u + 16u); - } else if ((v_q <= 9u) && ((v_q == 2u) || (self->private_impl.f_p_large_window == 0u))) { + } else if (v_q <= 9u) { v_n = wuffs_brotli__encoder__compute_layout_offset(self, v_q, v_lgwin, @@ -79336,7 +79388,7 @@ wuffs_brotli__encoder__transform_io( WUFFS_BASE__COROUTINE_SUSPENSION_POINT_0; wuffs_brotli__encoder__ensure_initialized(self); - if ((self->private_impl.f_quality > 9u) || (self->private_impl.f_large_window && (self->private_impl.f_quality > 2u))) { + if (self->private_impl.f_quality > 9u) { status = wuffs_base__make_status(wuffs_brotli__error__unsupported_encoder_quality); goto exit; } else if ((self->private_impl.f_quality >= 2u) && (((uint64_t)(a_workbuf.len)) < self->private_impl.f_workbuf_end)) { @@ -80332,6 +80384,7 @@ wuffs_brotli__encoder__hasher_total_size( uint32_t v_htype = 0; uint32_t v_bucket_bits = 0; uint32_t v_block_bits = 0; + uint64_t v_rolling = 0; if ((a_quality == 4u) && (a_size_hint >= 1048576u)) { v_htype = 54u; @@ -80360,7 +80413,16 @@ wuffs_brotli__encoder__hasher_total_size( } v_block_bits = (a_quality - 1u); } - return ((uint64_t)(((uint64_t)(wuffs_brotli__encoder__hasher_alloc_size(self, + v_rolling = 0u; + if (a_lgwin > 24u) { + if ((v_htype == 3u) || (v_htype == 54u)) { + v_rolling = wuffs_brotli__encoder__hasher_rolling_alloc_size(self); + } else if ((v_htype == 6u) || (v_htype == 68u)) { + v_htype = 6u; + v_rolling = wuffs_brotli__encoder__hasher_rolling_alloc_size(self); + } + } + return ((uint64_t)(((uint64_t)(((uint64_t)(wuffs_brotli__encoder__hasher_alloc_size(self, v_htype, v_bucket_bits, v_block_bits, @@ -80372,7 +80434,7 @@ wuffs_brotli__encoder__hasher_total_size( v_htype, v_bucket_bits, v_block_bits, - 2u))); + 2u))) + v_rolling)); } // -------- func brotli.encoder.hash_region_size @@ -83855,6 +83917,15 @@ wuffs_brotli__encoder__choose_hasher( self->private_impl.f_h_num_last_distances = 10u; } } + self->private_impl.f_hr_jump = 0u; + if (self->private_impl.f_lgwin > 24u) { + if ((self->private_impl.f_hasher_type == 3u) || (self->private_impl.f_hasher_type == 54u)) { + self->private_impl.f_hr_jump = 4u; + } else if ((self->private_impl.f_hasher_type == 6u) || (self->private_impl.f_hasher_type == 68u)) { + self->private_impl.f_hasher_type = 6u; + self->private_impl.f_hr_jump = 1u; + } + } return wuffs_base__make_empty_struct(); } @@ -83948,6 +84019,8 @@ wuffs_brotli__encoder__hasher_setup( self->private_impl.f_h_bucket_bits, self->private_impl.f_h_block_bits, 1u))); + self->private_impl.f_hr_off = self->private_impl.f_h_off2; + self->private_impl.f_hr_fresh = true; self->private_impl.f_h_is_prepared = false; self->private_impl.f_h_is_setup = true; } @@ -84067,6 +84140,13 @@ wuffs_brotli__encoder__hasher_prepare( v_j += 1u; } } + if (self->private_impl.f_hr_jump != 0u) { + if (self->private_impl.f_hr_fresh) { + self->private_impl.f_hr_fresh = false; + wuffs_brotli__encoder__hasher_rolling_initialize(self, a_wb); + } + wuffs_brotli__encoder__hasher_rolling_prepare(self, a_wb, a_input_size, self->private_impl.f_rb_buf_off); + } return wuffs_base__make_empty_struct(); } @@ -84236,6 +84316,9 @@ wuffs_brotli__encoder__hasher_stitch_to_previous_block( wuffs_brotli__encoder__hasher_store(self, a_wb, ((uint64_t)(a_position - 2u))); wuffs_brotli__encoder__hasher_store(self, a_wb, ((uint64_t)(a_position - 1u))); } + if (self->private_impl.f_hr_jump != 0u) { + wuffs_brotli__encoder__hasher_rolling_stitch_to_previous_block(self, a_wb, a_num_bytes, a_position); + } return wuffs_base__make_empty_struct(); } @@ -84488,6 +84571,13 @@ wuffs_brotli__encoder__find_longest_match( a_dictionary_distance, a_max_distance); } + if (self->private_impl.f_hr_jump != 0u) { + wuffs_brotli__encoder__hasher_rolling_find_longest_match(self, + a_wb, + a_cur_ix, + a_max_length, + a_max_backward); + } return wuffs_base__make_empty_struct(); } @@ -85084,6 +85174,177 @@ wuffs_brotli__encoder__find_longest_match_forgetful( return wuffs_base__make_empty_struct(); } +// -------- func brotli.encoder.hasher_rolling_alloc_size + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__encoder__hasher_rolling_alloc_size( + const wuffs_brotli__encoder* self) { + return (4u * ((uint64_t)(16777216u))); +} + +// -------- func brotli.encoder.hasher_rolling_initialize + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__empty_struct +wuffs_brotli__encoder__hasher_rolling_initialize( + wuffs_brotli__encoder* self, + wuffs_base__slice_u8 a_wb) { + uint64_t v_i = 0; + + self->private_impl.f_hr_state = 0u; + self->private_impl.f_hr_next_ix = 0u; + self->private_impl.f_hr_factor_remove = 1u; + v_i = 0u; + while (v_i < 32u) { + self->private_impl.f_hr_factor_remove *= 69069u; + v_i += ((uint64_t)(self->private_impl.f_hr_jump)); + } + wuffs_brotli__encoder__fill_bytes(self, + a_wb, + self->private_impl.f_hr_off, + wuffs_brotli__encoder__hasher_rolling_alloc_size(self), + 255u); + return wuffs_base__make_empty_struct(); +} + +// -------- func brotli.encoder.hasher_rolling_prepare + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__empty_struct +wuffs_brotli__encoder__hasher_rolling_prepare( + wuffs_brotli__encoder* self, + wuffs_base__slice_u8 a_wb, + uint64_t a_input_size, + uint64_t a_data_off) { + uint64_t v_i = 0; + + if (a_input_size < 32u) { + return wuffs_base__make_empty_struct(); + } + self->private_impl.f_hr_state = 0u; + v_i = 0u; + while (v_i < 32u) { + self->private_impl.f_hr_state = ((uint32_t)(((uint32_t)(69069u * self->private_impl.f_hr_state)) + (((uint32_t)(wuffs_brotli__encoder__byte_at(self, a_wb, ((uint64_t)(a_data_off + v_i))))) + 1u))); + v_i += ((uint64_t)(self->private_impl.f_hr_jump)); + } + return wuffs_base__make_empty_struct(); +} + +// -------- func brotli.encoder.hasher_rolling_stitch_to_previous_block + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__empty_struct +wuffs_brotli__encoder__hasher_rolling_stitch_to_previous_block( + wuffs_brotli__encoder* self, + wuffs_base__slice_u8 a_wb, + uint64_t a_num_bytes, + uint64_t a_position) { + uint64_t v_position = 0; + uint64_t v_position_masked = 0; + uint64_t v_available = 0; + uint64_t v_jm1 = 0; + uint64_t v_diff = 0; + uint64_t v_mask = 0; + + v_position = a_position; + v_available = a_num_bytes; + v_jm1 = ((uint64_t)(((uint64_t)(self->private_impl.f_hr_jump)) - 1u)); + if ((v_position & v_jm1) != 0u) { + v_diff = ((uint64_t)(((uint64_t)(self->private_impl.f_hr_jump)) - (v_position & v_jm1))); + if (v_diff > v_available) { + v_available = 0u; + } else { + v_available -= v_diff; + } + v_position += v_diff; + } + v_mask = ((uint64_t)(self->private_impl.f_rb_mask)); + v_position_masked = (v_position & v_mask); + if (v_available > ((uint64_t)(v_mask - v_position_masked))) { + v_available = ((uint64_t)(v_mask - v_position_masked)); + } + wuffs_brotli__encoder__hasher_rolling_prepare(self, a_wb, v_available, ((uint64_t)(self->private_impl.f_rb_buf_off + v_position_masked))); + self->private_impl.f_hr_next_ix = v_position; + return wuffs_base__make_empty_struct(); +} + +// -------- func brotli.encoder.hasher_rolling_find_longest_match + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__empty_struct +wuffs_brotli__encoder__hasher_rolling_find_longest_match( + wuffs_brotli__encoder* self, + wuffs_base__slice_u8 a_wb, + uint64_t a_cur_ix, + uint64_t a_max_length, + uint64_t a_max_backward) { + wuffs_base__slice_u8 v_data = {0}; + uint64_t v_mask = 0; + uint64_t v_cur_ix_masked = 0; + uint64_t v_jump = 0; + uint64_t v_pos = 0; + uint32_t v_code = 0; + uint8_t v_rem = 0; + uint8_t v_add = 0; + uint64_t v_found_ix = 0; + uint64_t v_backward = 0; + uint64_t v_found_ix_masked = 0; + uint64_t v_len = 0; + uint64_t v_score = 0; + uint64_t v_t = 0; + + if (self->private_impl.f_rb_buf_off > ((uint64_t)(a_wb.len))) { + return wuffs_base__make_empty_struct(); + } + v_data = wuffs_base__slice_u8__subslice_i(a_wb, self->private_impl.f_rb_buf_off); + v_mask = ((uint64_t)(self->private_impl.f_rb_mask)); + v_cur_ix_masked = (a_cur_ix & v_mask); + v_jump = ((uint64_t)(self->private_impl.f_hr_jump)); + if ((a_cur_ix & ((uint64_t)(v_jump - 1u))) != 0u) { + return wuffs_base__make_empty_struct(); + } + if (a_max_length < 32u) { + return wuffs_base__make_empty_struct(); + } + v_pos = self->private_impl.f_hr_next_ix; + while (v_pos <= a_cur_ix) { + v_code = (self->private_impl.f_hr_state & 1073741823u); + v_rem = wuffs_brotli__encoder__byte_at(self, v_data, (v_pos & v_mask)); + v_add = wuffs_brotli__encoder__byte_at(self, v_data, (((uint64_t)(v_pos + 32u)) & v_mask)); + v_found_ix = ((uint64_t)(4294967295u)); + self->private_impl.f_hr_state = ((uint32_t)(((uint32_t)(((uint32_t)(69069u * self->private_impl.f_hr_state)) + (((uint32_t)(v_add)) + 1u))) - ((uint32_t)(self->private_impl.f_hr_factor_remove * (((uint32_t)(v_rem)) + 1u))))); + if (v_code < 16777216u) { + v_t = ((uint64_t)(self->private_impl.f_hr_off + (4u * ((uint64_t)(v_code))))); + v_found_ix = ((uint64_t)(wuffs_brotli__encoder__load32le(self, a_wb, v_t))); + wuffs_brotli__encoder__poke_u32(self, a_wb, v_t, ((uint32_t)(v_pos))); + if ((v_pos == a_cur_ix) && (v_found_ix != ((uint64_t)(4294967295u)))) { + v_backward = (((uint64_t)(a_cur_ix - v_found_ix)) & 4294967295u); + if (v_backward <= a_max_backward) { + v_found_ix_masked = (v_found_ix & v_mask); + v_len = wuffs_brotli__encoder__find_match_length_with_limit(self, + v_data, + v_found_ix_masked, + v_cur_ix_masked, + a_max_length); + if ((v_len >= 4u) && (v_len > self->private_impl.f_sr_len)) { + v_score = wuffs_brotli__encoder__backward_reference_score(self, v_len, v_backward); + if (v_score > self->private_impl.f_sr_score) { + self->private_impl.f_sr_len = v_len; + self->private_impl.f_sr_distance = v_backward; + self->private_impl.f_sr_score = v_score; + self->private_impl.f_sr_len_code_delta = 0u; + } + } + } + } + } + v_pos += v_jump; + } + self->private_impl.f_hr_next_ix = ((uint64_t)(a_cur_ix + v_jump)); + return wuffs_base__make_empty_struct(); +} + // -------- func brotli.bit_cost.histogram_clear WUFFS_BASE__GENERATED_C_CODE diff --git a/script/manual-test-brotli-encoder-parity.c b/script/manual-test-brotli-encoder-parity.c index 9d1d5ba7..c7347037 100644 --- a/script/manual-test-brotli-encoder-parity.c +++ b/script/manual-test-brotli-encoder-parity.c @@ -96,6 +96,7 @@ static uint64_t g_num_mismatches; static uint32_t g_qmin = 0; static uint32_t g_qmax = 9; +static bool g_lw_only = false; typedef struct { uint32_t keys[MAX_PARAMS]; @@ -543,6 +544,93 @@ run_input_q01(const char* name, const uint8_t* data, size_t len, uint32_t q) { } } +// run_input_lw runs the large-window (BROTLI_PARAM_LARGE_WINDOW) grid for a +// quality from 2 to 9. At quality 2 (and below), the C code ignores the +// large-window parameter. At qualities 3 to 9, lgwin can be up to 30 and lgwin +// above 24 selects the composite hashers (H35, H55 and H65) instead of H3, +// H54, H6 and H68 (and only those: H5, H58 and H40 to H42 stay as they are). +static void // +run_input_lw(const char* name, const uint8_t* data, size_t len, uint32_t q) { + static const size_t metadata_lens[] = {0, 1, 100, 70000}; + bool big = len > (512 << 10); + + // lgwin and chunking. + static const uint32_t lgwins[] = {10, 18, 24, 25, 26, 27, 28, 29, 30}; + for (size_t w = 0; w < 9; w++) { + if (big && (lgwins[w] != 24) && (lgwins[w] != 25) && (lgwins[w] != 28) && + (lgwins[w] != 30)) { + continue; + } + for (uint32_t ch = 0; ch < 5; ch += 4) { + for (uint32_t fl = 0; fl < 3; fl += 2) { + if (big && (fl == 0)) { + continue; + } + config c = make_config(q, lgwins[w]); + add_param(&c, 6, 1); // BROTLI_PARAM_LARGE_WINDOW. + c.chunking = ch; + c.flush = fl; + c.seed = 0x10000 | (w << 8) | (ch << 4) | fl | (q << 12); + check(name, &c, data, len); + } + } + } + + // Other parameters, alone and in combination. + static const uint32_t extras[][6] = { + // Pairs of (BrotliEncoderParameter, value). A zero key ends the list. + {5, 1u << 20}, // BROTLI_PARAM_SIZE_HINT: H55 and H65. + {5, 0xFFFFFFFF}, // Replaced by the exact size. + {5, 1u << 30}, // + {3, 16}, // BROTLI_PARAM_LGBLOCK. + {3, 18}, // + {3, 24, 5, 1u << 21}, // ... and a big size hint. + {4, 1, 5, 1u << 20}, // DISABLE_LITERAL_CONTEXT_MODELING. + {7, 3, 8, 120}, // BROTLI_PARAM_NPOSTFIX and NDIRECT. + {7, 1, 8, 4, 5, 1u << 20}, // + {0, 2}, // FONT mode. + {0, 1, 5, 1u << 22}, // TEXT mode and a big size hint. + {9, 1000}, // BROTLI_PARAM_STREAM_OFFSET. + {9, 1u << 30, 5, 1u << 20}, + {2, 0xFFFFFFFF}, // LGWIN (int)-1, so 10. + {2, 31}, // LGWIN 31, so 30. + }; + for (size_t i = 0; i < sizeof extras / sizeof extras[0]; i++) { + for (size_t w = 0; w < 2; w++) { + static const uint32_t ws[] = {25, 30}; + if (big && (w != 0)) { + continue; + } + config c = make_config(q, ws[w]); + add_param(&c, 6, 1); // BROTLI_PARAM_LARGE_WINDOW. + for (size_t k = 0; (k < 6) && ((k == 0) || extras[i][k]); k += 2) { + uint32_t v = extras[i][k + 1]; + if ((extras[i][k] == 5) && (v == 0xFFFFFFFF)) { + v = (uint32_t)len; + } + add_param(&c, extras[i][k], v); + } + c.chunking = 4; + c.flush = 2; + c.seed = 0x20000 | i | (w << 8) | (q << 12); + check(name, &c, data, len); + } + } + + // Metadata blocks and limited dst lengths. + for (size_t i = 0; i < 4; i += 3) { + config c = make_config(q, 26); + add_param(&c, 6, 1); // BROTLI_PARAM_LARGE_WINDOW. + c.chunking = 4; + c.flush = 2; + c.metadata_position = 2; + c.metadata_len = metadata_lens[i]; + c.dst_chunk = (len <= (32 << 10)) ? 3 : 4096; + c.seed = 0x30000 | i | (q << 12); + check(name, &c, data, len); + } +} + // run_input_q2 runs the grid for a quality from 2 to 9. static void // run_input_q2(const char* name, const uint8_t* data, size_t len, uint32_t q) { @@ -649,17 +737,7 @@ run_input_q2(const char* name, const uint8_t* data, size_t len, uint32_t q) { } } } - if (q == 2) { - // Large window is supported (by the Wuffs encoder) only at quality 2. - for (uint32_t w = 22; w <= 30; w += 4) { - config c = make_config(q, w); - add_param(&c, 6, 1); // BROTLI_PARAM_LARGE_WINDOW. - c.chunking = 4; - c.flush = 2; - c.seed = w; - check(name, &c, data, len); - } - } + run_input_lw(name, data, len, q); if (len <= (256 << 10)) { static const size_t dst_chunks[] = {1, 2, 3, 7, 64, 4096}; for (size_t i = 0; i < 6; i++) { @@ -684,7 +762,11 @@ run_input(const char* name, const uint8_t* data, size_t len) { uint64_t mismatches0 = g_num_mismatches; time_t start = time(NULL); for (uint32_t q = g_qmin; q <= g_qmax; q++) { - if (q <= 1) { + if (g_lw_only) { + if (q >= 2) { + run_input_lw(name, data, len, q); + } + } else if (q <= 1) { run_input_q01(name, data, len, q); } else { run_input_q2(name, data, len, q); @@ -742,6 +824,8 @@ main(int argc, char** argv) { if (sscanf(argv[argi], "-q=%u-%u", &a, &b) == 2) { g_qmin = a; g_qmax = (b < 9) ? b : 9; + } else if (strcmp(argv[argi], "-lw") == 0) { + g_lw_only = true; } else if (sscanf(argv[argi], "-shard=%u/%u", &a, &b) == 2) { shard_k = (int)a; shard_n = (b > 0) ? (int)b : 1; diff --git a/std/brotli/encode_brotli.wuffs b/std/brotli/encode_brotli.wuffs index 9c299228..b1e629cc 100644 --- a/std/brotli/encode_brotli.wuffs +++ b/std/brotli/encode_brotli.wuffs @@ -178,6 +178,16 @@ pub struct encoder? implements base.io_transformer( dict_num_matches : base.u64, fc_max_hops : base.u64, + // The large-window composite hashers' rolling hasher (the C code's + // HashRolling). See encode_hasher_composite.wuffs. hr_jump is 0 when + // the hasher is not a composite one. + hr_jump : base.u32[..= 4], + hr_fresh : base.bool, + hr_state : base.u32, + hr_factor_remove : base.u32, + hr_next_ix : base.u64, + hr_off : base.u64, + // The C code's HasherSearchResult. sr_len : base.u64, sr_distance : base.u64, @@ -418,7 +428,7 @@ pub func encoder.workbuf_len() base.range_ii_u64 { lgwin = this.sanitized_lgwin(quality: q) if q <= 1 { n = ((2 as base.u64) << lgwin) + 503 + STORAGE_SLACK - } else if (q <= 9) and ((q == 2) or (this.p_large_window == 0)) { + } else if q <= 9 { n = this.compute_layout_offset(quality: q, lgwin: lgwin, lgblock: this.computed_lgblock(quality: q, lgwin: lgwin), idx: LAYOUT_END) } @@ -801,9 +811,8 @@ pub func encoder.transform_io?(dst: base.io_writer, src: base.io_reader, workbuf var copy_input_size : base.u64 this.ensure_initialized!() - if (this.quality > 9) or (this.large_window and (this.quality > 2)) { - // TODO: implement Zopfli (qualities 10 and 11) and large-window - // brotli at qualities 3 and above. + if this.quality > 9 { + // TODO: implement Zopfli (qualities 10 and 11). return "#unsupported encoder quality" } else if (this.quality >= 2) and (args.workbuf.length() < this.workbuf_end) { return base."#bad workbuf length" diff --git a/std/brotli/encode_data.wuffs b/std/brotli/encode_data.wuffs index 0e1e98f9..71fdb728 100644 --- a/std/brotli/encode_data.wuffs +++ b/std/brotli/encode_data.wuffs @@ -90,11 +90,13 @@ pri func encoder.max_metablock_size(lgwin: base.u32[..= 30], lgblock: base.u32[. // hasher_total_size returns the sum of FN(HashMemAllocInBytes)'s // alloc_size[i] for the hasher that ChooseHasher picks for the given -// quality, lgwin and size_hint. +// quality, lgwin and size_hint (including, for lgwin above 24, the +// large-window composite hashers' rolling hasher table). pri func encoder.hasher_total_size(quality: base.u32[..= 11], lgwin: base.u32[..= 30], size_hint: base.u32) base.u64 { var htype : base.u32 var bucket_bits : base.u32[..= 24] var block_bits : base.u32[..= 16] + var rolling : base.u64 if (args.quality == 4) and (args.size_hint >= 0x10_0000) { htype = 54 @@ -123,9 +125,21 @@ pri func encoder.hasher_total_size(quality: base.u32[..= 11], lgwin: base.u32[.. } block_bits = args.quality - 1 } - return (this.hasher_alloc_size(htype: htype, bucket_bits: bucket_bits, block_bits: block_bits, i: 0) ~mod+ + rolling = 0 + if args.lgwin > 24 { + // The large-window composite hashers H35, H55 and H65 (see + // choose_hasher): HASHER_A plus the rolling hasher's table. + if (htype == 3) or (htype == 54) { + rolling = this.hasher_rolling_alloc_size() + } else if (htype == 6) or (htype == 68) { + htype = 6 + rolling = this.hasher_rolling_alloc_size() + } + } + return ((this.hasher_alloc_size(htype: htype, bucket_bits: bucket_bits, block_bits: block_bits, i: 0) ~mod+ this.hasher_alloc_size(htype: htype, bucket_bits: bucket_bits, block_bits: block_bits, i: 1)) ~mod+ - this.hasher_alloc_size(htype: htype, bucket_bits: bucket_bits, block_bits: block_bits, i: 2) + this.hasher_alloc_size(htype: htype, bucket_bits: bucket_bits, block_bits: block_bits, i: 2)) ~mod+ + rolling } // hash_region_size returns the workbuf length reserved for the hasher's diff --git a/std/brotli/encode_hasher.wuffs b/std/brotli/encode_hasher.wuffs index 570c8597..0ae661b4 100644 --- a/std/brotli/encode_hasher.wuffs +++ b/std/brotli/encode_hasher.wuffs @@ -102,7 +102,7 @@ pri func encoder.fc_bank_bits() base.u32[..= 16] { } // Mirrors ChooseHasher in quality.h (with BROTLI_MAX_SIMD_QUALITY defined as -// 6), for qualities 2 to 9 and lgwin up to 24. +// 6), for qualities 2 to 9. pri func encoder.choose_hasher!() { var q : base.u32[..= 11] @@ -147,6 +147,25 @@ pri func encoder.choose_hasher!() { this.h_num_last_distances = 10 } } + + this.hr_jump = 0 + if this.lgwin > 24 { + // Different hashers for large window brotli: not for qualities <= 2, + // these are too fast for large window. Not for qualities >= 10: + // their hasher already works well with large window. So the changes + // are: + // H3 --> H35: for quality 3. + // H54 --> H55: for quality 4 with size hint > 1MB + // H6/H68 --> H65: for qualities 5, 6, 7, 8, 9. + // See encode_hasher_composite.wuffs: the composite hasher's + // HASHER_A stays in this.hasher_type. + if (this.hasher_type == 3) or (this.hasher_type == 54) { + this.hr_jump = 4 + } else if (this.hasher_type == 6) or (this.hasher_type == 68) { + this.hasher_type = 6 + this.hr_jump = 1 + } + } } // hasher_alloc_size mirrors FN(HashMemAllocInBytes) (alloc_size[i], for i in @@ -220,6 +239,9 @@ pri func encoder.hasher_setup!(wb: slice base.u8, position: base.u64, input_size htype: this.hasher_type, bucket_bits: this.h_bucket_bits, block_bits: this.h_block_bits, i: 0) this.h_off2 = this.h_off1 ~mod+ this.hasher_alloc_size( htype: this.hasher_type, bucket_bits: this.h_bucket_bits, block_bits: this.h_block_bits, i: 1) + // The composite hashers' HASHER_B memory follows HASHER_A's. + this.hr_off = this.h_off2 + this.hr_fresh = true this.h_is_prepared = false this.h_is_setup = true } @@ -306,6 +328,18 @@ pri func encoder.hasher_prepare!(wb: slice base.u8, one_shot: base.bool, input_s j ~mod+= 1 } } + + if this.hr_jump <> 0 { + // The composite hasher's FN(Prepare) (in hash_composite_inc.h), for + // HASHER_B. HASHER_B's FN(Initialize) is deferred to the first + // FN(Prepare). HASHER_A's FN(Initialize) does not touch the hasher + // memory, so doing it earlier (in hasher_setup) is equivalent. + if this.hr_fresh { + this.hr_fresh = false + this.hasher_rolling_initialize!(wb: args.wb) + } + this.hasher_rolling_prepare!(wb: args.wb, input_size: args.input_size, data_off: this.rb_buf_off) + } } // Mirrors FN(HashBytes) in hash_longest_match_quickly_inc.h. @@ -433,6 +467,10 @@ pri func encoder.hasher_stitch_to_previous_block!(wb: slice base.u8, num_bytes: this.hasher_store!(wb: args.wb, ix: args.position ~mod- 2) this.hasher_store!(wb: args.wb, ix: args.position ~mod- 1) } + if this.hr_jump <> 0 { + // The composite hasher's HASHER_B. + this.hasher_rolling_stitch_to_previous_block!(wb: args.wb, num_bytes: args.num_bytes, position: args.position) + } } // Mirrors PrepareDistanceCache in hash.h, via each hasher's @@ -598,6 +636,11 @@ pri func encoder.find_longest_match!(wb: slice base.u8, cur_ix: base.u64, max_le this.find_longest_match_forgetful!(wb: args.wb, cur_ix: args.cur_ix, max_length: args.max_length, max_backward: args.max_backward, dictionary_distance: args.dictionary_distance, max_distance: args.max_distance) } + if this.hr_jump <> 0 { + // The composite hasher's HASHER_B. + this.hasher_rolling_find_longest_match!(wb: args.wb, cur_ix: args.cur_ix, max_length: args.max_length, + max_backward: args.max_backward) + } } // Mirrors FN(FindLongestMatch) in hash_longest_match_quickly_inc.h. diff --git a/std/brotli/encode_hasher_composite.wuffs b/std/brotli/encode_hasher_composite.wuffs new file mode 100644 index 00000000..21fa6674 --- /dev/null +++ b/std/brotli/encode_hasher_composite.wuffs @@ -0,0 +1,208 @@ +// Copyright 2026 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. +// +// SPDX-License-Identifier: Apache-2.0 OR MIT + +// -------- + +// This file mirrors the C code's large-window hashers: the composite hashers +// (hash_composite_inc.h) H35 (H3 + HROLLING_FAST), H55 (H54 + HROLLING_FAST) +// and H65 (H6 + HROLLING), and the rolling hashers (hash_rolling_inc.h) +// HROLLING_FAST (JUMP 4) and HROLLING (JUMP 1), both with CHUNKLEN 32, +// NUMBUCKETS (1 << 24) and MASK ((NUMBUCKETS * 64) - 1). +// +// A composite hasher is represented by its first hasher (HASHER_A, H3, H54 or +// H6) in this.hasher_type and its rolling hasher (HASHER_B) by a non-zero +// this.hr_jump (4 for HROLLING_FAST, 1 for HROLLING). The composite's FN(etc) +// functions call HASHER_A's and then HASHER_B's, so the encode_hasher.wuffs +// functions (which implement HASHER_A) call the hasher_rolling_etc functions +// below after their own work when this.hr_jump is non-zero. +// +// The composite hasher's memory is HASHER_A's alloc_size[0] and [1] followed +// by HASHER_B's alloc_size[0] (the rolling table of NUMBUCKETS u32le values), +// at the this.hr_off workbuf offset. HASHER_A's HasherCommon is a copy of the +// composite's (the C code's ha_common), whose dict_num_lookups and +// dict_num_matches start at zero, like the composite's own, so HASHER_A can +// keep using this.dict_num_lookups and this.dict_num_matches. + +// Mirrors CHUNKLEN in hash.h, for the rolling hashers. +pri const ROLLING_CHUNKLEN : base.u64 = 32 + +// Mirrors NUMBUCKETS in hash.h, for the rolling hashers. +pri const ROLLING_NUMBUCKETS : base.u32 = 0x100_0000 + +// Mirrors MASK in hash.h, for the rolling hashers. +pri const ROLLING_MASK : base.u32 = 0x3FFF_FFFF + +// Mirrors FN(kRollingHashMul32) in hash_rolling_inc.h. +pri const ROLLING_HASH_MUL_32 : base.u32 = 69069 + +// Mirrors FN(kInvalidPos) in hash_rolling_inc.h. +pri const ROLLING_INVALID_POS : base.u32 = 0xFFFF_FFFF + +// hasher_rolling_alloc_size mirrors FN(HashMemAllocInBytes) in +// hash_rolling_inc.h: alloc_size[0], the table. +pri func encoder.hasher_rolling_alloc_size() base.u64 { + return 4 * (ROLLING_NUMBUCKETS as base.u64) +} + +// Mirrors FN(Initialize) in hash_rolling_inc.h. The C code's composite +// hashers call it (via their deferred initialization) in their first +// FN(Prepare), which HasherSetup calls right after FN(Initialize). +pri func encoder.hasher_rolling_initialize!(wb: slice base.u8) { + var i : base.u64 + + this.hr_state = 0 + this.hr_next_ix = 0 + + // Compute the factor of the oldest byte to remove: factor**steps modulo + // 0xffffffff (the multiplications rely on 32-bit overflow). + this.hr_factor_remove = 1 + i = 0 + while i < ROLLING_CHUNKLEN { + this.hr_factor_remove ~mod*= ROLLING_HASH_MUL_32 + i ~mod+= this.hr_jump as base.u64 + } + + // Every table entry is ROLLING_INVALID_POS. + this.fill_bytes!(s: args.wb, i: this.hr_off, n: this.hasher_rolling_alloc_size(), v: 0xFF) +} + +// Mirrors FN(Prepare) in hash_rolling_inc.h, where data is the workbuf's +// bytes starting at offset data_off. +pri func encoder.hasher_rolling_prepare!(wb: slice base.u8, input_size: base.u64, data_off: base.u64) { + var i : base.u64 + + // Too small size, cannot use this hasher. + if args.input_size < ROLLING_CHUNKLEN { + return nothing + } + this.hr_state = 0 + i = 0 + while i < ROLLING_CHUNKLEN { + // FN(HashRollingFunctionInitial). + this.hr_state = (ROLLING_HASH_MUL_32 ~mod* this.hr_state) ~mod+ + ((this.byte_at(s: args.wb, i: args.data_off ~mod+ i) as base.u32) + 1) + i ~mod+= this.hr_jump as base.u64 + } +} + +// Mirrors FN(StitchToPreviousBlock) in hash_rolling_inc.h. +pri func encoder.hasher_rolling_stitch_to_previous_block!(wb: slice base.u8, num_bytes: base.u64, position: base.u64) { + var position : base.u64 + var position_masked : base.u64 + var available : base.u64 + var jm1 : base.u64 + var diff : base.u64 + var mask : base.u64 + + // In this case we must re-initialize the hasher from scratch from the + // current position. + position = args.position + available = args.num_bytes + jm1 = (this.hr_jump as base.u64) ~mod- 1 + if (position & jm1) <> 0 { + diff = (this.hr_jump as base.u64) ~mod- (position & jm1) + if diff > available { + available = 0 + } else { + available ~mod-= diff + } + position ~mod+= diff + } + mask = this.rb_mask as base.u64 + position_masked = position & mask + // Wrapping around ringbuffer not handled. + if available > (mask ~mod- position_masked) { + available = mask ~mod- position_masked + } + this.hasher_rolling_prepare!(wb: args.wb, input_size: available, + data_off: this.rb_buf_off ~mod+ position_masked) + this.hr_next_ix = position +} + +// Mirrors FN(FindLongestMatch) in hash_rolling_inc.h. The search result is in +// the this.sr_etc fields. +pri func encoder.hasher_rolling_find_longest_match!(wb: slice base.u8, cur_ix: base.u64, max_length: base.u64, max_backward: base.u64) { + var data : roslice base.u8 + var mask : base.u64 + var cur_ix_masked : base.u64 + var jump : base.u64 + var pos : base.u64 + var code : base.u32 + var rem : base.u8 + var add : base.u8 + var found_ix : base.u64 + var backward : base.u64 + var found_ix_masked : base.u64 + var len : base.u64 + var score : base.u64 + var t : base.u64 + + if this.rb_buf_off > args.wb.length() { + return nothing + } + data = args.wb[this.rb_buf_off ..] + mask = this.rb_mask as base.u64 + cur_ix_masked = args.cur_ix & mask + jump = this.hr_jump as base.u64 + + if (args.cur_ix & (jump ~mod- 1)) <> 0 { + return nothing + } + + // Not enough lookahead. + if args.max_length < ROLLING_CHUNKLEN { + return nothing + } + + pos = this.hr_next_ix + while pos <= args.cur_ix { + code = this.hr_state & ROLLING_MASK + + rem = this.byte_at(s: data, i: pos & mask) + add = this.byte_at(s: data, i: (pos ~mod+ ROLLING_CHUNKLEN) & mask) + found_ix = ROLLING_INVALID_POS as base.u64 + + // FN(HashRollingFunction). + this.hr_state = ((ROLLING_HASH_MUL_32 ~mod* this.hr_state) ~mod+ ((add as base.u32) + 1)) ~mod- + (this.hr_factor_remove ~mod* ((rem as base.u32) + 1)) + + if code < ROLLING_NUMBUCKETS { + t = this.hr_off ~mod+ (4 * (code as base.u64)) + found_ix = this.load32le(s: args.wb, i: t) as base.u64 + this.poke_u32!(s: args.wb, i: t, v: (pos & 0xFFFF_FFFF) as base.u32) + if (pos == args.cur_ix) and (found_ix <> (ROLLING_INVALID_POS as base.u64)) { + // The cast to 32-bit makes backward distances up to 4GB work + // even if cur_ix is above 4GB, despite using 32-bit values in + // the table. + backward = (args.cur_ix ~mod- found_ix) & 0xFFFF_FFFF + if backward <= args.max_backward { + found_ix_masked = found_ix & mask + len = this.find_match_length_with_limit(s: data, + i1: found_ix_masked, i2: cur_ix_masked, limit: args.max_length) + if (len >= 4) and (len > this.sr_len) { + score = this.backward_reference_score(copy_length: len, backward_reference_offset: backward) + if score > this.sr_score { + this.sr_len = len + this.sr_distance = backward + this.sr_score = score + this.sr_len_code_delta = 0 + } + } + } + } + } + pos ~mod+= jump + } + + this.hr_next_ix = args.cur_ix ~mod+ jump + + // NOTE: this hasher does not search in the dictionary. It is used as + // backup-hasher, the main hasher already searches in it. +}