From fbae955a74584e2b4051d62a197ca0cb08d30b78 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 08:50:15 +0000 Subject: [PATCH] std/brotli: speed up qualities 0 to 9 and the decoder Encoder (output stays byte-exact with libbrotlienc v1.2.1): - H58/H68 (q5-6): match the tags 8 at a time (SWAR, like GetMatchingTagMask's non-SSE2 code) and visit the matches with a de Bruijn ctz; inline the bucket entry check into the H5/H6 and H58/H68 loops, caching the 4 bytes compared at best_len; a dedicated H58/H68 store and hash. - find_last_distances caches the current byte at best_len. - ctz64 and a branch-free log2_floor_non_zero (de Bruijn tables), and find_match_length_with_limit uses ctz. - load32le, load64le, peek_u16, poke_u16 and poke_u32 use one bounds check each. Decoder (same output and statuses): - The literal fast path keeps pos and the counts in local variables. - Copies of at most 16 bytes, with a distance of at least 16, copy 16 bytes (like the C code; the window gap keeps the extra bytes unread). script/manual-test-brotli-float.c also checks log2_floor_non_zero (every u32) and ctz64. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UZ978BikvkegL2JUkqoB1e --- release/c/wuffs-unsupported-snapshot.c | 548 ++++++++++++++++--------- script/manual-test-brotli-float.c | 41 ++ std/brotli/README.md | 24 +- std/brotli/decode_metablock.wuffs | 103 ++++- std/brotli/encode_bit_writer.wuffs | 62 ++- std/brotli/encode_hasher.wuffs | 254 +++++++----- std/brotli/encode_metablock.wuffs | 16 +- std/brotli/encode_util.wuffs | 45 +- 8 files changed, 716 insertions(+), 377 deletions(-) diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 87d7f819..6001456e 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -77153,6 +77153,14 @@ WUFFS_BROTLI__DICTIONARY[122784] WUFFS_BASE__POTENTIALLY_UNUSED = { #define WUFFS_BROTLI__F64_1E99 6088095589093318446u +static const uint32_t +WUFFS_BROTLI__LOG2_DEBRUIJN32[32] WUFFS_BASE__POTENTIALLY_UNUSED = { + 0u, 9u, 1u, 10u, 13u, 21u, 2u, 29u, + 11u, 14u, 16u, 18u, 22u, 25u, 3u, 30u, + 8u, 12u, 20u, 28u, 15u, 17u, 24u, 7u, + 19u, 27u, 23u, 6u, 26u, 5u, 4u, 31u, +}; + #define WUFFS_BROTLI__LITERAL_BLOCK_SWITCH_COST 4628602664527698330u #define WUFFS_BROTLI__COMMAND_BLOCK_SWITCH_COST 4623789442425946112u @@ -77339,6 +77347,18 @@ WUFFS_BROTLI__F32_OF_SMALL_INT[32] WUFFS_BASE__POTENTIALLY_UNUSED = { #define WUFFS_BROTLI__HQ_SHADOW_MATCHES 256u +static const uint8_t +WUFFS_BROTLI__CTZ64_DEBRUIJN[64] WUFFS_BASE__POTENTIALLY_UNUSED = { + 0u, 1u, 48u, 2u, 57u, 49u, 28u, 3u, + 61u, 58u, 50u, 42u, 38u, 29u, 17u, 4u, + 62u, 55u, 59u, 36u, 53u, 51u, 43u, 22u, + 45u, 39u, 33u, 30u, 24u, 18u, 12u, 5u, + 63u, 47u, 56u, 27u, 60u, 41u, 37u, 16u, + 54u, 35u, 52u, 21u, 44u, 32u, 23u, 11u, + 46u, 26u, 40u, 15u, 34u, 20u, 31u, 10u, + 25u, 14u, 19u, 9u, 13u, 8u, 7u, 6u, +}; + #define WUFFS_BROTLI__Z_INFINITY 2130692510u #define WUFFS_BROTLI__Z_F32_1 1065353216u @@ -77759,6 +77779,14 @@ wuffs_brotli__decoder__take_distance_from_ring_buffer( wuffs_brotli__decoder* self, uint32_t a_code); +WUFFS_BASE__GENERATED_C_CODE +static bool +wuffs_brotli__decoder__copy16( + wuffs_brotli__decoder* self, + wuffs_base__slice_u8 a_workbuf, + uint64_t a_d, + uint64_t a_s); + WUFFS_BASE__GENERATED_C_CODE static uint32_t wuffs_brotli__encoder__cmd_insert_len( @@ -79652,6 +79680,20 @@ wuffs_brotli__encoder__hasher_store( wuffs_base__slice_u8 a_wb, uint64_t a_ix); +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__empty_struct +wuffs_brotli__encoder__hasher_store_h58( + wuffs_brotli__encoder* self, + wuffs_base__slice_u8 a_wb, + uint64_t a_ix); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__encoder__hash_h58( + const wuffs_brotli__encoder* self, + wuffs_base__slice_u8 a_wb, + uint64_t a_ix); + WUFFS_BASE__GENERATED_C_CODE static wuffs_base__empty_struct wuffs_brotli__encoder__hasher_store_range( @@ -79755,17 +79797,6 @@ wuffs_brotli__encoder__find_last_distances( uint64_t a_max_length, uint64_t a_max_backward); -WUFFS_BASE__GENERATED_C_CODE -static uint64_t -wuffs_brotli__encoder__try_bucket_entry( - wuffs_brotli__encoder* self, - wuffs_base__slice_u8 a_data, - uint64_t a_cur_ix, - uint64_t a_prev_ix_in, - uint64_t a_best_len, - uint64_t a_max_length, - uint64_t a_max_backward); - WUFFS_BASE__GENERATED_C_CODE static wuffs_base__empty_struct wuffs_brotli__encoder__find_longest_match_h5( @@ -80547,6 +80578,12 @@ wuffs_brotli__encoder__hq_extend_var_region( wuffs_brotli__encoder* self, uint64_t a_wb_len); +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__encoder__ctz64( + const wuffs_brotli__encoder* self, + uint64_t a_x); + WUFFS_BASE__GENERATED_C_CODE static uint8_t wuffs_brotli__encoder__byte_at( @@ -85534,6 +85571,13 @@ wuffs_brotli__decoder__process_commands_fast( uint64_t v_dhi = 0; uint64_t v_slo = 0; uint64_t v_shi = 0; + uint32_t v_cm_slice = 0; + uint32_t v_wmask = 0; + uint32_t v_pos = 0; + uint32_t v_lit_n = 0; + uint32_t v_lit_i = 0; + uint64_t v_wi = 0; + bool v_copied16 = false; const uint8_t* iop_a_src = NULL; const uint8_t* io0_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; @@ -85630,25 +85674,32 @@ wuffs_brotli__decoder__process_commands_fast( } v_p1 = ((uint32_t)(wuffs_brotli__decoder__get_byte(self, a_workbuf, 1u))); v_p2 = ((uint32_t)(wuffs_brotli__decoder__get_byte(self, a_workbuf, 2u))); - while (self->private_impl.f_insert_length > 0u) { - if ((self->private_impl.f_block_length[0u] == 0u) || (((uint32_t)(self->private_impl.f_pos + 1u)) >= self->private_impl.f_rb_size)) { - goto label__outer__break; - } + v_mode_offset = (self->private_impl.f_context_mode << 9u); + v_cm_slice = self->private_impl.f_context_map_slice; + v_wmask = ((uint32_t)(self->private_impl.f_window_size - 1u)); + v_pos = self->private_impl.f_pos; + v_lit_n = wuffs_base__u32__min(self->private_impl.f_insert_length, self->private_impl.f_block_length[0u]); + if (((uint32_t)(v_pos + 1u)) >= self->private_impl.f_rb_size) { + v_lit_n = 0u; + } else { + v_lit_n = wuffs_base__u32__min(v_lit_n, ((uint32_t)(((uint32_t)(self->private_impl.f_rb_size - v_pos)) - 1u))); + } + v_lit_i = 0u; + while (v_lit_i < v_lit_n) { if (v_n_bits < 24u) { if (((uint64_t)(io2_a_src - iop_a_src)) < 8u) { - goto label__outer__break; + break; } v_bits |= ((uint64_t)(wuffs_base__peek_u64le__no_bounds_check(iop_a_src) << v_n_bits)); iop_a_src += ((63u - v_n_bits) >> 3u); v_n_bits |= 56u; } - v_mode_offset = (self->private_impl.f_context_mode << 9u); v_ctx = ((uint32_t)(((uint8_t)(WUFFS_BROTLI__CONTEXT_LOOKUP[(v_mode_offset | v_p1)] | WUFFS_BROTLI__CONTEXT_LOOKUP[(v_mode_offset | 256u | v_p2)])))); - v_tree = ((uint32_t)(self->private_data.f_context_maps[(self->private_impl.f_context_map_slice + v_ctx)])); + v_tree = ((uint32_t)(self->private_data.f_context_maps[(v_cm_slice + v_ctx)])); v_i = ((uint32_t)((v_tree * 632u) + ((uint32_t)((v_bits & 255u))))); if (v_i >= 678034u) { v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); - goto label__outer__break; + break; } v_e = ((uint32_t)(self->private_data.f_huffs[v_i])); v_e_bits = (v_e & 15u); @@ -85656,23 +85707,34 @@ wuffs_brotli__decoder__process_commands_fast( v_i = ((uint32_t)(((uint32_t)((v_tree * 632u) + (v_e >> 4u))) + (((uint32_t)(((v_bits >> 8u) & 32767u))) & ((((uint32_t)(1u)) << (v_e_bits - 8u)) - 1u)))); if (v_i >= 678034u) { v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); - goto label__outer__break; + break; } v_e = ((uint32_t)(self->private_data.f_huffs[v_i])); v_e_bits = ((v_e & 15u) + 8u); } if (v_n_bits < v_e_bits) { v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_bit_reader); - goto label__outer__break; + break; } v_bits >>= v_e_bits; v_n_bits -= v_e_bits; - self->private_impl.f_symbol = (v_e >> 4u); - wuffs_brotli__decoder__put_byte(self, a_workbuf, ((uint8_t)(self->private_impl.f_symbol))); v_p2 = v_p1; - v_p1 = (self->private_impl.f_symbol & 255u); - self->private_impl.f_block_length[0u] -= 1u; - self->private_impl.f_insert_length -= 1u; + v_p1 = ((v_e >> 4u) & 255u); + v_wi = ((uint64_t)((v_pos & v_wmask))); + if (v_wi < ((uint64_t)(a_workbuf.len))) { + a_workbuf.ptr[v_wi] = ((uint8_t)(v_p1)); + } + v_pos += 1u; + v_lit_i += 1u; + } + if (v_lit_i > 0u) { + self->private_impl.f_symbol = v_p1; + self->private_impl.f_pos = v_pos; + self->private_impl.f_block_length[0u] -= v_lit_i; + self->private_impl.f_insert_length -= v_lit_i; + } + if (wuffs_base__status__is_error(&v_ret) || (self->private_impl.f_insert_length > 0u)) { + break; } self->private_impl.f_cmd_state = 2u; if ((self->private_impl.f_mb_remaining == 0u) || (self->private_impl.f_mb_remaining >= 2147483648u)) { @@ -85758,6 +85820,15 @@ wuffs_brotli__decoder__process_commands_fast( self->private_impl.f_dist_rb_idx += 1u; self->private_impl.f_mb_remaining -= v_n; v_k64 = 0u; + if ((v_n <= 16u) && + (v_distance >= 16u) && + (((uint64_t)(v_dst_i + 16u)) <= ((uint64_t)(self->private_impl.f_rb_size))) && + (((uint64_t)(v_src_i + 16u)) <= ((uint64_t)(self->private_impl.f_window_size)))) { + v_copied16 = wuffs_brotli__decoder__copy16(self, a_workbuf, v_dst_i, v_src_i); + if (v_copied16) { + v_k64 = ((uint64_t)(v_n)); + } + } while (v_k64 < ((uint64_t)(v_n))) { v_n64 = ((uint64_t)(((uint64_t)(v_n)) - v_k64)); v_n64 = wuffs_base__u64__min(v_n64, ((uint64_t)(v_distance))); @@ -85787,7 +85858,6 @@ wuffs_brotli__decoder__process_commands_fast( break; } } - label__outer__break:; while (v_n_bits >= 8u) { if ( ! (iop_a_src > io1_a_src)) { status = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_bit_reader); @@ -85938,6 +86008,43 @@ wuffs_brotli__decoder__take_distance_from_ring_buffer( return wuffs_base__make_empty_struct(); } +// -------- func brotli.decoder.copy16 + +WUFFS_BASE__GENERATED_C_CODE +static bool +wuffs_brotli__decoder__copy16( + wuffs_brotli__decoder* self, + wuffs_base__slice_u8 a_workbuf, + uint64_t a_d, + uint64_t a_s) { + uint64_t v_lo = 0; + uint64_t v_hi = 0; + wuffs_base__slice_u8 v_q = {0}; + wuffs_base__slice_u8 v_r = {0}; + + if ((a_s > ((uint64_t)(a_workbuf.len))) || (a_d > ((uint64_t)(a_workbuf.len)))) { + return false; + } + v_q = wuffs_base__slice_u8__subslice_i(a_workbuf, a_s); + if (((uint64_t)(v_q.len)) >= 16u) { + v_lo = wuffs_base__peek_u64le__no_bounds_check(v_q.ptr); + v_q = wuffs_base__slice_u8__subslice_i(v_q, 8u); + if (((uint64_t)(v_q.len)) >= 8u) { + v_hi = wuffs_base__peek_u64le__no_bounds_check(v_q.ptr); + v_q = wuffs_base__slice_u8__subslice_i(a_workbuf, a_d); + if (((uint64_t)(v_q.len)) >= 16u) { + v_r = wuffs_base__slice_u8__subslice_i(v_q, 8u); + wuffs_base__poke_u64le__no_bounds_check(v_q.ptr, v_lo); + if (((uint64_t)(v_r.len)) >= 8u) { + wuffs_base__poke_u64le__no_bounds_check(v_r.ptr, v_hi); + return true; + } + } + } + } + return false; +} + // -------- func brotli.encoder.cmd_insert_len WUFFS_BASE__GENERATED_C_CODE @@ -88206,29 +88313,14 @@ wuffs_brotli__encoder__log2_floor_non_zero( const wuffs_brotli__encoder* self, uint64_t a_n) { uint32_t v_x = 0; - uint32_t v_r = 0; v_x = ((uint32_t)(a_n)); - if (v_x >= 65536u) { - v_x >>= 16u; - v_r = 16u; - } - if (v_x >= 256u) { - v_x >>= 8u; - v_r += 8u; - } - if (v_x >= 16u) { - v_x >>= 4u; - v_r += 4u; - } - if (v_x >= 4u) { - v_x >>= 2u; - v_r += 2u; - } - if (v_x >= 2u) { - v_r += 1u; - } - return (v_r & 31u); + v_x |= (v_x >> 1u); + v_x |= (v_x >> 2u); + v_x |= (v_x >> 4u); + v_x |= (v_x >> 8u); + v_x |= (v_x >> 16u); + return WUFFS_BROTLI__LOG2_DEBRUIJN32[(((uint32_t)(v_x * 130329821u)) >> 27u)]; } // -------- func brotli.encoder.fast_log2 @@ -88279,12 +88371,11 @@ wuffs_brotli__encoder__load64le( const wuffs_brotli__encoder* self, wuffs_base__slice_u8 a_s, uint64_t a_i) { - wuffs_base__slice_u8 v_q = {0}; - - if (a_i < ((uint64_t)(a_s.len))) { - v_q = wuffs_base__slice_u8__subslice_i(a_s, a_i); - if (((uint64_t)(v_q.len)) >= 8u) { - return wuffs_base__peek_u64le__no_bounds_check(v_q.ptr); + if (a_i < 18446744073709486080u) { + if ((a_i + 8u) <= ((uint64_t)(a_s.len))) { + return wuffs_base__peek_u64le__no_bounds_check(wuffs_base__slice_u8__subslice_ij(a_s, + a_i, + (a_i + 8u)).ptr); } } return 0u; @@ -88298,12 +88389,11 @@ wuffs_brotli__encoder__load32le( const wuffs_brotli__encoder* self, wuffs_base__slice_u8 a_s, uint64_t a_i) { - wuffs_base__slice_u8 v_q = {0}; - - if (a_i < ((uint64_t)(a_s.len))) { - v_q = wuffs_base__slice_u8__subslice_i(a_s, a_i); - if (((uint64_t)(v_q.len)) >= 4u) { - return wuffs_base__peek_u32le__no_bounds_check(v_q.ptr); + if (a_i < 18446744073709486080u) { + if ((a_i + 4u) <= ((uint64_t)(a_s.len))) { + return wuffs_base__peek_u32le__no_bounds_check(wuffs_base__slice_u8__subslice_ij(a_s, + a_i, + (a_i + 4u)).ptr); } } return 0u; @@ -88334,11 +88424,7 @@ wuffs_brotli__encoder__find_match_length_with_limit( while ((v_n >= 8u) && (((uint64_t)(v_a.len)) >= 8u) && (((uint64_t)(v_b.len)) >= 8u)) { v_x = (wuffs_base__peek_u64le__no_bounds_check(v_a.ptr) ^ wuffs_base__peek_u64le__no_bounds_check(v_b.ptr)); if (v_x != 0u) { - while ((v_x & 255u) == 0u) { - v_matched += 1u; - v_x >>= 8u; - } - return v_matched; + return ((uint64_t)(v_matched + (wuffs_brotli__encoder__ctz64(self, v_x) >> 3u))); } v_matched += 8u; v_n -= 8u; @@ -96842,7 +96928,6 @@ wuffs_brotli__encoder__hasher_store( uint64_t v_bsize = 0; uint64_t v_off = 0; uint16_t v_num = 0; - uint32_t v_hash = 0; uint64_t v_bank = 0; uint64_t v_idx = 0; uint64_t v_delta = 0; @@ -96850,6 +96935,10 @@ wuffs_brotli__encoder__hasher_store( uint32_t v_fc_bits = 0; v_htype = self->private_impl.f_hasher_type; + if ((v_htype == 58u) || (v_htype == 68u)) { + wuffs_brotli__encoder__hasher_store_h58(self, a_wb, a_ix); + return wuffs_base__make_empty_struct(); + } if (self->private_impl.f_rb_buf_off > ((uint64_t)(a_wb.len))) { return wuffs_base__make_empty_struct(); } @@ -96868,14 +96957,6 @@ wuffs_brotli__encoder__hasher_store( v_off = ((uint64_t)((((uint64_t)(v_num)) & ((((uint64_t)(1u)) << self->private_impl.f_h_block_bits) - 1u)) + (((uint64_t)(v_key)) << self->private_impl.f_h_block_bits))); wuffs_brotli__encoder__poke_u16(self, a_wb, ((uint64_t)(self->private_impl.f_h_off0 + ((uint64_t)(2u * ((uint64_t)(v_key)))))), ((uint16_t)(v_num + 1u))); wuffs_brotli__encoder__poke_u32(self, a_wb, ((uint64_t)(self->private_impl.f_h_off1 + ((uint64_t)(4u * v_off)))), ((uint32_t)(a_ix))); - } else if ((v_htype == 58u) || (v_htype == 68u)) { - v_hash = wuffs_brotli__encoder__hash_longest(self, v_data, (a_ix & ((uint64_t)(self->private_impl.f_rb_mask)))); - v_key = (v_hash >> 8u); - v_num = wuffs_brotli__encoder__peek_u16(self, a_wb, ((uint64_t)(self->private_impl.f_h_off0 + ((uint64_t)(2u * ((uint64_t)(v_key))))))); - v_off = ((uint64_t)((((uint64_t)(v_num)) & ((((uint64_t)(1u)) << self->private_impl.f_h_block_bits) - 1u)) + (((uint64_t)(v_key)) << self->private_impl.f_h_block_bits))); - wuffs_brotli__encoder__poke_u16(self, a_wb, ((uint64_t)(self->private_impl.f_h_off0 + ((uint64_t)(2u * ((uint64_t)(v_key)))))), ((uint16_t)(v_num - 1u))); - wuffs_brotli__encoder__poke_u32(self, a_wb, ((uint64_t)(self->private_impl.f_h_off2 + ((uint64_t)(4u * v_off)))), ((uint32_t)(a_ix))); - wuffs_brotli__encoder__set_byte(self, a_wb, ((uint64_t)(self->private_impl.f_h_off1 + v_off)), ((uint8_t)(v_hash))); } else if ((v_htype == 40u) || (v_htype == 41u) || (v_htype == 42u)) { v_key = wuffs_brotli__encoder__hash_forgetful(self, v_data, (a_ix & ((uint64_t)(self->private_impl.f_rb_mask)))); v_fc_bits = wuffs_brotli__encoder__fc_bank_bits(self); @@ -96904,6 +96985,48 @@ wuffs_brotli__encoder__hasher_store( return wuffs_base__make_empty_struct(); } +// -------- func brotli.encoder.hasher_store_h58 + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__empty_struct +wuffs_brotli__encoder__hasher_store_h58( + wuffs_brotli__encoder* self, + wuffs_base__slice_u8 a_wb, + uint64_t a_ix) { + uint32_t v_hash = 0; + uint64_t v_key = 0; + uint64_t v_koff = 0; + uint16_t v_num = 0; + uint64_t v_off = 0; + + v_hash = wuffs_brotli__encoder__hash_h58(self, a_wb, a_ix); + v_key = ((uint64_t)((v_hash >> 8u))); + v_koff = ((uint64_t)(self->private_impl.f_h_off0 + ((uint64_t)(2u * v_key)))); + v_num = wuffs_brotli__encoder__peek_u16(self, a_wb, v_koff); + v_off = ((uint64_t)((((uint64_t)(v_num)) & ((((uint64_t)(1u)) << self->private_impl.f_h_block_bits) - 1u)) + (v_key << self->private_impl.f_h_block_bits))); + wuffs_brotli__encoder__poke_u16(self, a_wb, v_koff, ((uint16_t)(v_num - 1u))); + wuffs_brotli__encoder__poke_u32(self, a_wb, ((uint64_t)(self->private_impl.f_h_off2 + ((uint64_t)(4u * v_off)))), ((uint32_t)(a_ix))); + wuffs_brotli__encoder__set_byte(self, a_wb, ((uint64_t)(self->private_impl.f_h_off1 + v_off)), ((uint8_t)(v_hash))); + return wuffs_base__make_empty_struct(); +} + +// -------- func brotli.encoder.hash_h58 + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__encoder__hash_h58( + const wuffs_brotli__encoder* self, + wuffs_base__slice_u8 a_wb, + uint64_t a_ix) { + uint64_t v_i = 0; + + v_i = ((uint64_t)(self->private_impl.f_rb_buf_off + (a_ix & ((uint64_t)(self->private_impl.f_rb_mask))))); + if (self->private_impl.f_hasher_type == 58u) { + return (((uint32_t)(wuffs_brotli__encoder__load32le(self, a_wb, v_i) * 506832829u)) >> ((24u - self->private_impl.f_h_bucket_bits) & 31u)); + } + return ((uint32_t)((((uint64_t)(wuffs_brotli__encoder__load64le(self, a_wb, v_i) * 8922571613522624512u)) >> 41u))); +} + // -------- func brotli.encoder.hasher_store_range WUFFS_BASE__GENERATED_C_CODE @@ -97384,6 +97507,8 @@ wuffs_brotli__encoder__find_last_distances( uint64_t v_prev_ix = 0; uint64_t v_len = 0; uint64_t v_score = 0; + uint8_t v_cur_byte = 0; + uint32_t v_n = 0; v_mask = ((uint64_t)(self->private_impl.f_rb_mask)); v_cur_ix_masked = (a_cur_ix & v_mask); @@ -97391,15 +97516,13 @@ wuffs_brotli__encoder__find_last_distances( v_best_len = self->private_impl.f_sr_len; self->private_impl.f_sr_len = 0u; self->private_impl.f_sr_len_code_delta = 0u; + v_cur_byte = wuffs_brotli__encoder__byte_at(self, a_data, ((uint64_t)(v_cur_ix_masked + v_best_len))); + v_n = self->private_impl.f_h_num_last_distances; v_i = 0u; - while (v_i < self->private_impl.f_h_num_last_distances) { + while (v_i < v_n) { v_backward = wuffs_brotli__encoder__dist_cache_u64(self, v_i); v_prev_ix = ((uint64_t)(a_cur_ix - v_backward)); - if (v_prev_ix >= a_cur_ix) { - v_i += 1u; - continue; - } - if (v_backward > a_max_backward) { + if ((v_prev_ix >= a_cur_ix) || (v_backward > a_max_backward)) { v_i += 1u; continue; } @@ -97407,7 +97530,7 @@ wuffs_brotli__encoder__find_last_distances( if (((uint64_t)(v_cur_ix_masked + v_best_len)) > v_mask) { break; } - if ((((uint64_t)(v_prev_ix + v_best_len)) > v_mask) || (wuffs_brotli__encoder__byte_at(self, a_data, ((uint64_t)(v_cur_ix_masked + v_best_len))) != wuffs_brotli__encoder__byte_at(self, a_data, ((uint64_t)(v_prev_ix + v_best_len))))) { + if ((((uint64_t)(v_prev_ix + v_best_len)) > v_mask) || (v_cur_byte != wuffs_brotli__encoder__byte_at(self, a_data, ((uint64_t)(v_prev_ix + v_best_len))))) { v_i += 1u; continue; } @@ -97428,6 +97551,7 @@ wuffs_brotli__encoder__find_last_distances( self->private_impl.f_sr_len = v_best_len; self->private_impl.f_sr_distance = v_backward; self->private_impl.f_sr_score = v_best_score; + v_cur_byte = wuffs_brotli__encoder__byte_at(self, a_data, ((uint64_t)(v_cur_ix_masked + v_best_len))); } } } @@ -97436,67 +97560,6 @@ wuffs_brotli__encoder__find_last_distances( return v_best_len; } -// -------- func brotli.encoder.try_bucket_entry - -WUFFS_BASE__GENERATED_C_CODE -static uint64_t -wuffs_brotli__encoder__try_bucket_entry( - wuffs_brotli__encoder* self, - wuffs_base__slice_u8 a_data, - uint64_t a_cur_ix, - uint64_t a_prev_ix_in, - uint64_t a_best_len, - uint64_t a_max_length, - uint64_t a_max_backward) { - uint64_t v_mask = 0; - uint64_t v_cur_ix_masked = 0; - uint64_t v_prev_ix = 0; - uint64_t v_backward = 0; - uint64_t v_len = 0; - uint64_t v_score = 0; - - v_mask = ((uint64_t)(self->private_impl.f_rb_mask)); - v_cur_ix_masked = (a_cur_ix & v_mask); - v_backward = ((uint64_t)(a_cur_ix - a_prev_ix_in)); - if (v_backward > a_max_backward) { - return 1u; - } - v_prev_ix = (a_prev_ix_in & v_mask); - if (((uint64_t)(v_cur_ix_masked + a_best_len)) > v_mask) { - return 1u; - } - if ((((uint64_t)(v_prev_ix + a_best_len)) > v_mask) || (wuffs_brotli__encoder__load32le(self, a_data, ((uint64_t)(((uint64_t)(v_cur_ix_masked + a_best_len)) - 3u))) != wuffs_brotli__encoder__load32le(self, a_data, ((uint64_t)(((uint64_t)(v_prev_ix + a_best_len)) - 3u))))) { - return 0u; - } - if ((self->private_impl.f_hasher_type == 6u) || (self->private_impl.f_hasher_type == 68u)) { - if (wuffs_brotli__encoder__load32le(self, a_data, v_cur_ix_masked) != wuffs_brotli__encoder__load32le(self, a_data, v_prev_ix)) { - return 0u; - } - v_len = ((uint64_t)(wuffs_brotli__encoder__find_match_length_with_limit(self, - a_data, - ((uint64_t)(v_prev_ix + 4u)), - ((uint64_t)(v_cur_ix_masked + 4u)), - ((uint64_t)(a_max_length - 4u))) + 4u)); - } else { - v_len = wuffs_brotli__encoder__find_match_length_with_limit(self, - a_data, - v_prev_ix, - v_cur_ix_masked, - a_max_length); - if (v_len < 4u) { - return 0u; - } - } - v_score = wuffs_brotli__encoder__backward_reference_score(self, v_len, v_backward); - if (self->private_impl.f_sr_score < v_score) { - self->private_impl.f_sr_len = v_len; - self->private_impl.f_sr_distance = v_backward; - self->private_impl.f_sr_score = v_score; - return ((uint64_t)(2u + v_len)); - } - return 0u; -} - // -------- func brotli.encoder.find_longest_match_h5 WUFFS_BASE__GENERATED_C_CODE @@ -97520,7 +97583,13 @@ wuffs_brotli__encoder__find_longest_match_h5( uint64_t v_bucket = 0; uint64_t v_down = 0; uint64_t v_i = 0; - uint64_t v_r = 0; + uint64_t v_mask = 0; + bool v_is64 = false; + uint32_t v_cur4 = 0; + uint64_t v_prev_ix = 0; + uint64_t v_backward = 0; + uint64_t v_len = 0; + uint64_t v_score = 0; if (self->private_impl.f_rb_buf_off > ((uint64_t)(a_wb.len))) { return wuffs_base__make_empty_struct(); @@ -97545,20 +97614,50 @@ wuffs_brotli__encoder__find_longest_match_h5( if (((uint64_t)(v_num)) > v_block_size) { v_down = ((uint64_t)(((uint64_t)(v_num)) - v_block_size)); } + v_mask = ((uint64_t)(self->private_impl.f_rb_mask)); + v_is64 = (self->private_impl.f_hasher_type == 6u); + v_cur4 = wuffs_brotli__encoder__load32le(self, v_data, ((uint64_t)(((uint64_t)(v_cur_ix_masked + v_best_len)) - 3u))); v_i = ((uint64_t)(v_num)); while (v_i > v_down) { v_i -= 1u; - v_r = wuffs_brotli__encoder__try_bucket_entry(self, - v_data, - a_cur_ix, - ((uint64_t)(wuffs_brotli__encoder__load32le(self, a_wb, ((uint64_t)(v_bucket + ((uint64_t)(4u * (v_i & v_block_mask)))))))), - v_best_len, - a_max_length, - a_max_backward); - if (v_r == 1u) { + v_prev_ix = ((uint64_t)(wuffs_brotli__encoder__load32le(self, a_wb, ((uint64_t)(v_bucket + ((uint64_t)(4u * (v_i & v_block_mask)))))))); + v_backward = ((uint64_t)(a_cur_ix - v_prev_ix)); + if (v_backward > a_max_backward) { break; - } else if (v_r >= 2u) { - v_best_len = ((uint64_t)(v_r - 2u)); + } + v_prev_ix &= v_mask; + if (((uint64_t)(v_cur_ix_masked + v_best_len)) > v_mask) { + break; + } + if ((((uint64_t)(v_prev_ix + v_best_len)) > v_mask) || (v_cur4 != wuffs_brotli__encoder__load32le(self, v_data, ((uint64_t)(((uint64_t)(v_prev_ix + v_best_len)) - 3u))))) { + continue; + } + if (v_is64) { + if (wuffs_brotli__encoder__load32le(self, v_data, v_cur_ix_masked) != wuffs_brotli__encoder__load32le(self, v_data, v_prev_ix)) { + continue; + } + v_len = ((uint64_t)(wuffs_brotli__encoder__find_match_length_with_limit(self, + v_data, + ((uint64_t)(v_prev_ix + 4u)), + ((uint64_t)(v_cur_ix_masked + 4u)), + ((uint64_t)(a_max_length - 4u))) + 4u)); + } else { + v_len = wuffs_brotli__encoder__find_match_length_with_limit(self, + v_data, + v_prev_ix, + v_cur_ix_masked, + a_max_length); + if (v_len < 4u) { + continue; + } + } + v_score = wuffs_brotli__encoder__backward_reference_score(self, v_len, v_backward); + if (self->private_impl.f_sr_score < v_score) { + self->private_impl.f_sr_len = v_len; + self->private_impl.f_sr_distance = v_backward; + self->private_impl.f_sr_score = v_score; + v_best_len = v_len; + v_cur4 = wuffs_brotli__encoder__load32le(self, v_data, ((uint64_t)(((uint64_t)(v_cur_ix_masked + v_best_len)) - 3u))); } } wuffs_brotli__encoder__poke_u32(self, a_wb, ((uint64_t)(v_bucket + ((uint64_t)(4u * (((uint64_t)(v_num)) & v_block_mask))))), ((uint32_t)(a_cur_ix))); @@ -97604,7 +97703,17 @@ wuffs_brotli__encoder__find_longest_match_h58( uint64_t v_limit = 0; uint64_t v_k = 0; uint64_t v_rb_index = 0; - uint64_t v_r = 0; + uint64_t v_splat = 0; + uint64_t v_matches = 0; + uint64_t v_chunk = 0; + uint64_t v_j = 0; + uint64_t v_mask = 0; + bool v_is64 = false; + uint32_t v_cur4 = 0; + uint64_t v_prev_ix = 0; + uint64_t v_backward = 0; + uint64_t v_len = 0; + uint64_t v_score = 0; if (self->private_impl.f_rb_buf_off > ((uint64_t)(a_wb.len))) { return wuffs_base__make_empty_struct(); @@ -97612,7 +97721,7 @@ wuffs_brotli__encoder__find_longest_match_h58( v_data = wuffs_base__slice_u8__subslice_i(a_wb, self->private_impl.f_rb_buf_off); v_cur_ix_masked = (a_cur_ix & ((uint64_t)(self->private_impl.f_rb_mask))); v_min_score = self->private_impl.f_sr_score; - v_hash = wuffs_brotli__encoder__hash_longest(self, v_data, v_cur_ix_masked); + v_hash = wuffs_brotli__encoder__hash_h58(self, a_wb, a_cur_ix); v_key = ((uint64_t)((v_hash >> 8u))); v_block_size = (((uint64_t)(1u)) << self->private_impl.f_h_block_bits); v_block_mask = ((uint64_t)(v_block_size - 1u)); @@ -97634,24 +97743,68 @@ wuffs_brotli__encoder__find_longest_match_h58( if (v_block_size > ((uint64_t)(v_n))) { v_limit = ((uint64_t)(v_n)); } - v_k = 0u; - while (v_k < v_limit) { + v_splat = ((uint64_t)(((uint64_t)(v_tag)) * 72340172838076673u)); + v_matches = 0u; + v_j = 0u; + while (v_j < v_block_size) { + v_chunk = (wuffs_brotli__encoder__load64le(self, a_wb, ((uint64_t)(v_tag_bucket + v_j))) ^ v_splat); + v_chunk = ((((uint64_t)((v_chunk | 9259542123273814144u) - 72340172838076673u)) | v_chunk) & 9259542123273814144u); + v_chunk = ((v_chunk ^ 9259542123273814144u) >> 7u); + v_matches |= ((uint64_t)((((uint64_t)(v_chunk * 72624976668147840u)) >> 56u) << (v_j & 63u))); + v_j += 8u; + } + if (v_head > 0u) { + v_matches = (((v_matches >> (v_head & 63u)) | ((uint64_t)(v_matches << (((uint64_t)(v_block_size - v_head)) & 63u)))) & (((uint64_t)(18446744073709551615u)) >> (((uint64_t)(64u - v_block_size)) & 63u))); + } + if (v_limit < 64u) { + v_matches &= ((uint64_t)((((uint64_t)(1u)) << (v_limit & 63u)) - 1u)); + } + v_mask = ((uint64_t)(self->private_impl.f_rb_mask)); + v_is64 = (self->private_impl.f_hasher_type == 68u); + v_cur4 = wuffs_brotli__encoder__load32le(self, v_data, ((uint64_t)(((uint64_t)(v_cur_ix_masked + v_best_len)) - 3u))); + while (v_matches != 0u) { + v_k = wuffs_brotli__encoder__ctz64(self, v_matches); + v_matches &= ((uint64_t)(v_matches - 1u)); v_rb_index = (((uint64_t)(v_head + v_k)) & v_block_mask); - if (wuffs_brotli__encoder__byte_at(self, a_wb, ((uint64_t)(v_tag_bucket + v_rb_index))) == v_tag) { - v_r = wuffs_brotli__encoder__try_bucket_entry(self, + v_prev_ix = ((uint64_t)(wuffs_brotli__encoder__load32le(self, a_wb, ((uint64_t)(v_bucket + ((uint64_t)(4u * v_rb_index))))))); + v_backward = ((uint64_t)(a_cur_ix - v_prev_ix)); + if (v_backward > a_max_backward) { + break; + } + v_prev_ix &= v_mask; + if (((uint64_t)(v_cur_ix_masked + v_best_len)) > v_mask) { + break; + } + if ((((uint64_t)(v_prev_ix + v_best_len)) > v_mask) || (v_cur4 != wuffs_brotli__encoder__load32le(self, v_data, ((uint64_t)(((uint64_t)(v_prev_ix + v_best_len)) - 3u))))) { + continue; + } + if (v_is64) { + if (wuffs_brotli__encoder__load32le(self, v_data, v_cur_ix_masked) != wuffs_brotli__encoder__load32le(self, v_data, v_prev_ix)) { + continue; + } + v_len = ((uint64_t)(wuffs_brotli__encoder__find_match_length_with_limit(self, v_data, - a_cur_ix, - ((uint64_t)(wuffs_brotli__encoder__load32le(self, a_wb, ((uint64_t)(v_bucket + ((uint64_t)(4u * v_rb_index))))))), - v_best_len, - a_max_length, - a_max_backward); - if (v_r == 1u) { - break; - } else if (v_r >= 2u) { - v_best_len = ((uint64_t)(v_r - 2u)); + ((uint64_t)(v_prev_ix + 4u)), + ((uint64_t)(v_cur_ix_masked + 4u)), + ((uint64_t)(a_max_length - 4u))) + 4u)); + } else { + v_len = wuffs_brotli__encoder__find_match_length_with_limit(self, + v_data, + v_prev_ix, + v_cur_ix_masked, + a_max_length); + if (v_len < 4u) { + continue; } } - v_k += 1u; + v_score = wuffs_brotli__encoder__backward_reference_score(self, v_len, v_backward); + if (self->private_impl.f_sr_score < v_score) { + self->private_impl.f_sr_len = v_len; + self->private_impl.f_sr_distance = v_backward; + self->private_impl.f_sr_score = v_score; + v_best_len = v_len; + v_cur4 = wuffs_brotli__encoder__load32le(self, v_data, ((uint64_t)(((uint64_t)(v_cur_ix_masked + v_best_len)) - 3u))); + } } wuffs_brotli__encoder__poke_u32(self, a_wb, ((uint64_t)(v_bucket + ((uint64_t)(4u * (((uint64_t)(v_num)) & v_block_mask))))), ((uint32_t)(a_cur_ix))); wuffs_brotli__encoder__set_byte(self, a_wb, ((uint64_t)(v_tag_bucket + (((uint64_t)(v_num)) & v_block_mask))), v_tag); @@ -98976,16 +99129,14 @@ wuffs_brotli__metablock_builder__log2_floor_non_zero( const wuffs_brotli__metablock_builder* self, uint64_t a_n) { uint32_t v_x = 0; - uint32_t v_r = 0; v_x = ((uint32_t)(a_n)); - while (v_x > 1u) { - v_x >>= 1u; - if (v_r < 31u) { - v_r += 1u; - } - } - return v_r; + v_x |= (v_x >> 1u); + v_x |= (v_x >> 2u); + v_x |= (v_x >> 4u); + v_x |= (v_x >> 8u); + v_x |= (v_x >> 16u); + return WUFFS_BROTLI__LOG2_DEBRUIJN32[(((uint32_t)(v_x * 130329821u)) >> 27u)]; } // -------- func brotli.metablock_builder.prefix_encode_copy_distance @@ -101839,6 +101990,16 @@ wuffs_brotli__metablock_tester__get( return 0u; } +// -------- func brotli.encoder.ctz64 + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__encoder__ctz64( + const wuffs_brotli__encoder* self, + uint64_t a_x) { + return ((uint64_t)(WUFFS_BROTLI__CTZ64_DEBRUIJN[(((uint64_t)((a_x & ((uint64_t)(0u - a_x))) * 285870213051386505u)) >> 58u)])); +} + // -------- func brotli.encoder.byte_at WUFFS_BASE__GENERATED_C_CODE @@ -101861,12 +102022,11 @@ wuffs_brotli__encoder__peek_u16( const wuffs_brotli__encoder* self, wuffs_base__slice_u8 a_s, uint64_t a_i) { - wuffs_base__slice_u8 v_q = {0}; - - if (a_i < ((uint64_t)(a_s.len))) { - v_q = wuffs_base__slice_u8__subslice_i(a_s, a_i); - if (((uint64_t)(v_q.len)) >= 2u) { - return wuffs_base__peek_u16le__no_bounds_check(v_q.ptr); + if (a_i < 18446744073709486080u) { + if ((a_i + 2u) <= ((uint64_t)(a_s.len))) { + return wuffs_base__peek_u16le__no_bounds_check(wuffs_base__slice_u8__subslice_ij(a_s, + a_i, + (a_i + 2u)).ptr); } } return 0u; @@ -101881,12 +102041,11 @@ wuffs_brotli__encoder__poke_u16( wuffs_base__slice_u8 a_s, uint64_t a_i, uint16_t a_v) { - wuffs_base__slice_u8 v_q = {0}; - - if (a_i < ((uint64_t)(a_s.len))) { - v_q = wuffs_base__slice_u8__subslice_i(a_s, a_i); - if (((uint64_t)(v_q.len)) >= 2u) { - wuffs_base__poke_u16le__no_bounds_check(v_q.ptr, a_v); + if (a_i < 18446744073709486080u) { + if ((a_i + 2u) <= ((uint64_t)(a_s.len))) { + wuffs_base__poke_u16le__no_bounds_check(wuffs_base__slice_u8__subslice_ij(a_s, + a_i, + (a_i + 2u)).ptr, a_v); } } return wuffs_base__make_empty_struct(); @@ -101901,12 +102060,11 @@ wuffs_brotli__encoder__poke_u32( wuffs_base__slice_u8 a_s, uint64_t a_i, uint32_t a_v) { - wuffs_base__slice_u8 v_q = {0}; - - if (a_i < ((uint64_t)(a_s.len))) { - v_q = wuffs_base__slice_u8__subslice_i(a_s, a_i); - if (((uint64_t)(v_q.len)) >= 4u) { - wuffs_base__poke_u32le__no_bounds_check(v_q.ptr, a_v); + if (a_i < 18446744073709486080u) { + if ((a_i + 4u) <= ((uint64_t)(a_s.len))) { + wuffs_base__poke_u32le__no_bounds_check(wuffs_base__slice_u8__subslice_ij(a_s, + a_i, + (a_i + 4u)).ptr, a_v); } } return wuffs_base__make_empty_struct(); diff --git a/script/manual-test-brotli-float.c b/script/manual-test-brotli-float.c index 22953fc2..561bb28d 100644 --- a/script/manual-test-brotli-float.c +++ b/script/manual-test-brotli-float.c @@ -23,6 +23,9 @@ // gcc -O2 -std=c99 manual-test-brotli-float.c -o /tmp/brfloat // /tmp/brfloat # Default: 100 million operations per op. // /tmp/brfloat 1000000 # Or pass the number of operations per op. +// It also checks the integer bit helpers (log2_floor_non_zero, for every u32, +// and ctz64) against the compiler's __builtin_clz and __builtin_ctzll. +// // It prints one line per op and the first few mismatches (if any), and exits // with a non-zero status if there were any mismatches. NaN results only need // to be NaN (on both sides). @@ -46,6 +49,7 @@ #error "Build without -ffast-math and without -mfma / -march=native" #endif +static wuffs_brotli__encoder* g_enc; static wuffs_brotli__bit_cost* g_bc; static uint64_t g_rng = 0x123456789ABCDEF1ull; @@ -334,6 +338,41 @@ check_conversions(uint64_t n) { g_total_mismatches += mismatches; } +static void // +check_bit_helpers(uint64_t n) { + uint64_t mismatches = 0; + wuffs_brotli__encoder* enc = g_enc; + // log2_floor_non_zero truncates its argument to 32 bits (and returns 0 for + // 0), so check every u32 value, with random high bits. + uint32_t v = 0; + do { + uint64_t x = ((uint64_t)(rnd() & 0xFFFFFFFF00000000ull)) | v; + uint32_t want = v ? (31 - (uint32_t)__builtin_clz(v)) : 0; + uint32_t have0 = wuffs_brotli__encoder__log2_floor_non_zero(enc, x); + uint32_t have1 = wuffs_brotli__metablock_builder__log2_floor_non_zero( + &enc->private_data.f_mb, x); + if (((have0 != want) || (have1 != want)) && (mismatches++ < 5)) { + printf("log2_floor_non_zero(0x%016" PRIX64 "): have %" PRIu32 + " and %" PRIu32 ", want %" PRIu32 "\n", x, have0, have1, want); + } + } while (++v != 0); + for (uint64_t i = 0; i < n; i++) { + uint64_t x = (i < 64) ? (1ull << i) : rnd_int(); + if (x == 0) { + continue; + } + uint64_t want = (uint64_t)__builtin_ctzll(x); + uint64_t have = wuffs_brotli__encoder__ctz64(enc, x); + if ((have != want) && (mismatches++ < 5)) { + printf("ctz64(0x%016" PRIX64 "): have %" PRIu64 ", want %" PRIu64 "\n", + x, have, want); + } + } + printf("%-8s %12" PRIu64 " iterations, %8" PRIu64 " mismatches\n", "bits", + n + (uint64_t)0x100000000ull, mismatches); + g_total_mismatches += mismatches; +} + int // main(int argc, char** argv) { uint64_t n = 100000000; @@ -345,6 +384,7 @@ main(int argc, char** argv) { printf("alloc failed\n"); return 1; } + g_enc = enc; g_bc = &enc->private_data.f_bc; check_f32_add(n); @@ -359,6 +399,7 @@ main(int argc, char** argv) { check_f64_gt(n); check_f64_ge(n); check_conversions(n); + check_bit_helpers(n); free(enc); if (g_total_mismatches) { diff --git a/std/brotli/README.md b/std/brotli/README.md index ba407481..4e3c9f9f 100644 --- a/std/brotli/README.md +++ b/std/brotli/README.md @@ -883,7 +883,9 @@ by both this package's decoder and libbrotlidec. `script/manual-test-brotli-float.c` checks the encoder's package-private float fast paths (`encode_float.wuffs`, which spare the hot loops the cross-package `std/softfloat` calls) against the C compiler's hardware float -and double arithmetic, for operands biased towards their edge cases. +and double arithmetic, for operands biased towards their edge cases, and +checks the integer bit helpers (`log2_floor_non_zero` for every 32-bit value, +and `ctz64`). `script/manual-test-brotli-encoder-parity.c` runs a larger grid over every file in `test/data/brotli`, some larger files in `test/data` (up to about 400 KiB) and some synthetic inputs (up to 1 MiB). Its `-q=LO-HI` and @@ -1131,13 +1133,19 @@ metablock) for as long as it needs the results. - `workbuf_len` is a worst case (or, at qualities 10 and 11, a practical bound). A future option could take the input length (like the C code's size hint) to give a tighter bound. -- Speed. Compressing the 3 MiB `test/data/harvesters.bmp` in one call (on - x86-64, lgwin 22), the Wuffs encoder takes about 1.2 times libbrotlienc's - time at quality 5, 1.7 times at quality 9, 4 times at quality 10 and 3.5 - times at quality 11 (on text, 2 to 3 times at qualities 10 and 11), mostly - because of the soft float arithmetic in the Zopfli cost model, the block - splitter and histogram clustering. Decoding takes about 1.2 times - libbrotlidec's time. +- Speed. On a mix of text, JSON, map data, an image and an ISO image (on + x86-64, lgwin 22, one call each, reusing the workbuf), the Wuffs encoder + takes about 1.2 to 1.8 times libbrotlienc's time at qualities 0 to 8 (about + 1.8 times at the qualities 4 to 6 that web servers use for on-the-fly + compression) and about the same time at quality 9. At qualities 10 and 11 + it takes about 2 times libbrotlienc's time on text and 3.5 to 4 times on + images (such as the 3 MiB `test/data/harvesters.bmp`), mostly because of + the soft float arithmetic in the Zopfli cost model, the block splitter and + histogram clustering. Decoding takes about 1.2 to 1.6 times libbrotlidec's + time. The workbuf is sized for the window (about 48 to 62 MiB at lgwin 22 + and qualities 5 to 9), so callers that compress many small inputs should + reuse it: allocating and first touching a fresh one for each call costs + more than the compression of a small input. - Serialized (shared) dictionaries, which only an experimental (`BROTLI_EXPERIMENTAL`) libbrotlienc build supports. - Changing parameters mid-stream. diff --git a/std/brotli/decode_metablock.wuffs b/std/brotli/decode_metablock.wuffs index ae301fec..26787df2 100644 --- a/std/brotli/decode_metablock.wuffs +++ b/std/brotli/decode_metablock.wuffs @@ -738,6 +738,13 @@ pri func decoder.process_commands_fast!(src: base.io_reader, workbuf: slice base var dhi : base.u64 var slo : base.u64 var shi : base.u64 + var cm_slice : base.u32[..= 16320] + var wmask : base.u32 + var pos : base.u32 + var lit_n : base.u32 + var lit_i : base.u32 + var wi : base.u64 + var copied16 : base.bool if this.n_bits >= 8 { return ok @@ -829,26 +836,38 @@ pri func decoder.process_commands_fast!(src: base.io_reader, workbuf: slice base } p1 = this.get_byte(workbuf: args.workbuf, distance: 1) as base.u32 p2 = this.get_byte(workbuf: args.workbuf, distance: 2) as base.u32 - while this.insert_length > 0 { - if (this.block_length[0] == 0) or ((this.pos ~mod+ 1) >= this.rb_size) { - break.outer - } + mode_offset = this.context_mode << 9 + cm_slice = this.context_map_slice + wmask = this.window_size ~mod- 1 + pos = this.pos + // lit_n is how many literals the loop below decodes before this + // command's literals, this block's literals or the ring buffer + // (before the next ring_event) run out. The loop keeps pos, the + // counts and the symbol in local variables, and writes them back + // to this.etc fields before every break. + lit_n = this.insert_length.min(no_more_than: this.block_length[0]) + if (pos ~mod+ 1) >= this.rb_size { + lit_n = 0 + } else { + lit_n = lit_n.min(no_more_than: (this.rb_size ~mod- pos) ~mod- 1) + } + lit_i = 0 + while lit_i < lit_n { if n_bits < 24 { if args.src.length() < 8 { - break.outer + break } bits |= args.src.peek_u64le() ~mod<< n_bits args.src.skip_u32_fast!(actual: (63 - n_bits) >> 3, worst_case: 8) n_bits |= 56 } - mode_offset = this.context_mode << 9 ctx = (CONTEXT_LOOKUP[mode_offset | p1] | CONTEXT_LOOKUP[mode_offset | 0x100 | p2]) as base.u32 - tree = this.context_maps[this.context_map_slice + ctx] as base.u32 + tree = this.context_maps[cm_slice + ctx] as base.u32 i = (LIT_HUFFS_BASE + (tree * LIT_HUFFS_STRIDE)) ~mod+ ((bits & 0xFF) as base.u32) if i >= HUFFS_LEN { ret = "#internal error: inconsistent Huffman table" - break.outer + break } e = this.huffs[i] as base.u32 e_bits = e & 15 @@ -856,23 +875,34 @@ pri func decoder.process_commands_fast!(src: base.io_reader, workbuf: slice base i = ((LIT_HUFFS_BASE + (tree * LIT_HUFFS_STRIDE)) ~mod+ (e >> 4)) ~mod+ ((((bits >> 8) & 0x7FFF) as base.u32) & (((1 as base.u32) << (e_bits - 8)) - 1)) if i >= HUFFS_LEN { ret = "#internal error: inconsistent Huffman table" - break.outer + break } e = this.huffs[i] as base.u32 e_bits = (e & 15) + 8 } if n_bits < e_bits { ret = "#internal error: inconsistent bit reader" - break.outer + break } bits >>= e_bits n_bits -= e_bits - this.symbol = e >> 4 - this.put_byte!(workbuf: args.workbuf, b: (this.symbol & 0xFF) as base.u8) p2 = p1 - p1 = this.symbol & 0xFF - this.block_length[0] ~mod-= 1 - this.insert_length ~mod-= 1 + p1 = (e >> 4) & 0xFF + wi = (pos & wmask) as base.u64 + if wi < args.workbuf.length() { + args.workbuf[wi] = p1 as base.u8 + } + pos ~mod+= 1 + lit_i ~mod+= 1 + } + if lit_i > 0 { + this.symbol = p1 + this.pos = pos + this.block_length[0] ~mod-= lit_i + this.insert_length ~mod-= lit_i + } + if (ret.is_error()) or (this.insert_length > 0) { + break.outer } this.cmd_state = 2 if (this.mb_remaining == 0) or (this.mb_remaining >= 0x8000_0000) { @@ -971,6 +1001,18 @@ pri func decoder.process_commands_fast!(src: base.io_reader, workbuf: slice base this.mb_remaining ~mod-= n // Copy in chunks of at most distance bytes, like process_commands. k64 = 0 + if (n <= 16) and (distance >= 16) and + ((dst_i ~mod+ 16) <= (this.rb_size as base.u64)) and + ((src_i ~mod+ 16) <= (this.window_size as base.u64)) { + // Like the C code, copy 16 bytes, overwriting up to 16 - n bytes + // after the copy's end. Those bytes are more than the maximum + // backward distance (BROTLI_WINDOW_GAP is 16) away from any later + // position, so they are never read before being rewritten. + copied16 = this.copy16!(workbuf: args.workbuf, d: dst_i, s: src_i) + if copied16 { + k64 = n as base.u64 + } + } while k64 < (n as base.u64) { n64 = (n as base.u64) ~mod- k64 n64 = n64.min(no_more_than: distance as base.u64) @@ -1098,3 +1140,34 @@ pri func decoder.take_distance_from_ring_buffer!(code: base.u32[..= 15]) { this.distance_code = v - 3 } } + +// copy16 copies the 16 bytes at workbuf[s ..] to workbuf[d ..], if both are +// in bounds (and returns whether it did). The two ranges must not overlap. +pri func decoder.copy16!(workbuf: slice base.u8, d: base.u64, s: base.u64) base.bool { + var lo : base.u64 + var hi : base.u64 + var q : slice base.u8 + var r : slice base.u8 + + if (args.s > args.workbuf.length()) or (args.d > args.workbuf.length()) { + return false + } + q = args.workbuf[args.s ..] + if q.length() >= 16 { + lo = q.peek_u64le() + q = q[8 ..] + if q.length() >= 8 { + hi = q.peek_u64le() + q = args.workbuf[args.d ..] + if q.length() >= 16 { + r = q[8 ..] + q.poke_u64le!(a: lo) + if r.length() >= 8 { + r.poke_u64le!(a: hi) + return true + } + } + } + } + return false +} diff --git a/std/brotli/encode_bit_writer.wuffs b/std/brotli/encode_bit_writer.wuffs index d6acd49f..9abf7020 100644 --- a/std/brotli/encode_bit_writer.wuffs +++ b/std/brotli/encode_bit_writer.wuffs @@ -125,33 +125,27 @@ pri func encoder.store_meta_block_header!(len: base.u64, is_uncompressed: base.b } } +// LOG2_DEBRUIJN32 maps (v * 0x07C4_ACDD) >> 27, for v = (1 << (k + 1)) - 1, +// to k. +pri const LOG2_DEBRUIJN32 : roarray[32] base.u32[..= 31] = [ + 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, + 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31, +] + // Mirrors Log2FloorNonZero in fast_log.h, which (with BROTLI_BSR32) truncates // its argument to 32 bits. It returns 0 for an argument of 0. pri func encoder.log2_floor_non_zero(n: base.u64) base.u32[..= 31] { var x : base.u32 - var r : base.u32 + // Smear the highest set bit of the low 32 bits of n rightwards, then + // look up its position. x = (args.n & 0xFFFF_FFFF) as base.u32 - if x >= 0x1_0000 { - x >>= 16 - r = 16 - } - if x >= 0x100 { - x >>= 8 - r ~mod+= 8 - } - if x >= 0x10 { - x >>= 4 - r ~mod+= 4 - } - if x >= 4 { - x >>= 2 - r ~mod+= 2 - } - if x >= 2 { - r ~mod+= 1 - } - return r & 31 + x |= x >> 1 + x |= x >> 2 + x |= x >> 4 + x |= x >> 8 + x |= x >> 16 + return LOG2_DEBRUIJN32[(x ~mod* 0x07C4_ACDD) >> 27] } // Mirrors FastLog2 in fast_log.h. The result is the bit pattern of an IEEE @@ -193,12 +187,10 @@ pri func encoder.bits_entropy_of_histogram() base.u64 { // code's BROTLI_UNALIGNED_LOAD64LE. The C code never reads out of bounds, so // the zero result for an out of bounds read should never be used. pri func encoder.load64le(s: roslice base.u8, i: base.u64) base.u64 { - var q : roslice base.u8 - - if args.i < args.s.length() { - q = args.s[args.i ..] - if q.length() >= 8 { - return q.peek_u64le() + if args.i < 0xFFFF_FFFF_FFFF_0000 { + if (args.i + 8) <= args.s.length() { + assert args.i <= (args.i + 8) via "a <= (a + b): 0 <= b"(b: 8) + return args.s[args.i .. args.i + 8].peek_u64le() } } return 0 @@ -206,12 +198,10 @@ pri func encoder.load64le(s: roslice base.u8, i: base.u64) base.u64 { // load32le is like load64le but for 4 bytes. pri func encoder.load32le(s: roslice base.u8, i: base.u64) base.u32 { - var q : roslice base.u8 - - if args.i < args.s.length() { - q = args.s[args.i ..] - if q.length() >= 4 { - return q.peek_u32le() + if args.i < 0xFFFF_FFFF_FFFF_0000 { + if (args.i + 4) <= args.s.length() { + assert args.i <= (args.i + 4) via "a <= (a + b): 0 <= b"(b: 4) + return args.s[args.i .. args.i + 4].peek_u32le() } } return 0 @@ -235,11 +225,7 @@ pri func encoder.find_match_length_with_limit(s: roslice base.u8, i1: base.u64, while (n >= 8) and (a.length() >= 8) and (b.length() >= 8) { x = a.peek_u64le() ^ b.peek_u64le() if x <> 0 { - while (x & 0xFF) == 0 { - matched ~mod+= 1 - x >>= 8 - } - return matched + return matched ~mod+ (this.ctz64(x: x) >> 3) } matched ~mod+= 8 n -= 8 diff --git a/std/brotli/encode_hasher.wuffs b/std/brotli/encode_hasher.wuffs index b51e3b5e..1e64c3dd 100644 --- a/std/brotli/encode_hasher.wuffs +++ b/std/brotli/encode_hasher.wuffs @@ -400,7 +400,6 @@ pri func encoder.hasher_store!(wb: slice base.u8, ix: base.u64) { var bsize : base.u64 var off : base.u64 var num : base.u16 - var hash : base.u32 var bank : base.u64 var idx : base.u64 var delta : base.u64 @@ -408,6 +407,10 @@ pri func encoder.hasher_store!(wb: slice base.u8, ix: base.u64) { var fc_bits : base.u32[..= 16] htype = this.hasher_type + if (htype == 58) or (htype == 68) { + this.hasher_store_h58!(wb: args.wb, ix: args.ix) + return nothing + } if this.rb_buf_off > args.wb.length() { return nothing } @@ -427,16 +430,6 @@ pri func encoder.hasher_store!(wb: slice base.u8, ix: base.u64) { this.poke_u16!(s: args.wb, i: this.h_off0 ~mod+ (2 ~mod* (key as base.u64)), v: num ~mod+ 1) this.poke_u32!(s: args.wb, i: this.h_off1 ~mod+ (4 ~mod* off), v: (args.ix & 0xFFFF_FFFF) as base.u32) - } else if (htype == 58) or (htype == 68) { - hash = this.hash_longest(data: data, i: args.ix & (this.rb_mask as base.u64)) - key = hash >> 8 - num = this.peek_u16(s: args.wb, i: this.h_off0 ~mod+ (2 ~mod* (key as base.u64))) - off = ((num as base.u64) & (((1 as base.u64) << this.h_block_bits) - 1)) ~mod+ - ((key as base.u64) << this.h_block_bits) - this.poke_u16!(s: args.wb, i: this.h_off0 ~mod+ (2 ~mod* (key as base.u64)), v: num ~mod- 1) - this.poke_u32!(s: args.wb, i: this.h_off2 ~mod+ (4 ~mod* off), v: (args.ix & 0xFFFF_FFFF) as base.u32) - this.set_byte!(s: args.wb, i: this.h_off1 ~mod+ off, v: (hash & 0xFF) as base.u8) - } else if (htype == 40) or (htype == 41) or (htype == 42) { key = this.hash_forgetful(data: data, i: args.ix & (this.rb_mask as base.u64)) fc_bits = this.fc_bank_bits() @@ -457,6 +450,36 @@ pri func encoder.hasher_store!(wb: slice base.u8, ix: base.u64) { } } +// hasher_store_h58 is FN(Store) for H58 and H68. +pri func encoder.hasher_store_h58!(wb: slice base.u8, ix: base.u64) { + var hash : base.u32 + var key : base.u64 + var koff : base.u64 + var num : base.u16 + var off : base.u64 + + hash = this.hash_h58(wb: args.wb, ix: args.ix) + key = (hash >> 8) as base.u64 + koff = this.h_off0 ~mod+ (2 ~mod* key) + num = this.peek_u16(s: args.wb, i: koff) + off = ((num as base.u64) & (((1 as base.u64) << this.h_block_bits) - 1)) ~mod+ (key << this.h_block_bits) + this.poke_u16!(s: args.wb, i: koff, v: num ~mod- 1) + this.poke_u32!(s: args.wb, i: this.h_off2 ~mod+ (4 ~mod* off), v: (args.ix & 0xFFFF_FFFF) as base.u32) + this.set_byte!(s: args.wb, i: this.h_off1 ~mod+ off, v: (hash & 0xFF) as base.u8) +} + +// hash_h58 is hash_longest for H58 and H68, of the ring buffer bytes at +// position ix (masked). +pri func encoder.hash_h58(wb: roslice base.u8, ix: base.u64) base.u32 { + var i : base.u64 + + i = this.rb_buf_off ~mod+ (args.ix & (this.rb_mask as base.u64)) + if this.hasher_type == 58 { + return (this.load32le(s: args.wb, i: i) ~mod* HASH_MUL_32) >> ((24 - this.h_bucket_bits) & 31) + } + return ((this.load64le(s: args.wb, i: i) ~mod* ((HASH_MUL_64 << 24) & 0xFFFF_FFFF_FFFF_FFFF)) >> 41) as base.u32 +} + // Mirrors FN(StoreRange) of every hasher. pri func encoder.hasher_store_range!(wb: slice base.u8, ix_start: base.u64, ix_end: base.u64) { var i : base.u64 @@ -803,6 +826,8 @@ pri func encoder.find_last_distances!(data: roslice base.u8, cur_ix: base.u64, m var prev_ix : base.u64 var len : base.u64 var score : base.u64 + var cur_byte : base.u8 + var n : base.u32 mask = this.rb_mask as base.u64 cur_ix_masked = args.cur_ix & mask @@ -811,15 +836,14 @@ pri func encoder.find_last_distances!(data: roslice base.u8, cur_ix: base.u64, m this.sr_len = 0 this.sr_len_code_delta = 0 + // cur_byte is data[cur_ix_masked + best_len], updated with best_len. + cur_byte = this.byte_at(s: args.data, i: cur_ix_masked ~mod+ best_len) + n = this.h_num_last_distances i = 0 - while i < this.h_num_last_distances { + while i < n { backward = this.dist_cache_u64(i: i) prev_ix = args.cur_ix ~mod- backward - if prev_ix >= args.cur_ix { - i ~mod+= 1 - continue - } - if backward > args.max_backward { + if (prev_ix >= args.cur_ix) or (backward > args.max_backward) { i ~mod+= 1 continue } @@ -828,7 +852,7 @@ pri func encoder.find_last_distances!(data: roslice base.u8, cur_ix: base.u64, m break } if ((prev_ix ~mod+ best_len) > mask) or - (this.byte_at(s: args.data, i: cur_ix_masked ~mod+ best_len) <> this.byte_at(s: args.data, i: prev_ix ~mod+ best_len)) { + (cur_byte <> this.byte_at(s: args.data, i: prev_ix ~mod+ best_len)) { i ~mod+= 1 continue } @@ -845,6 +869,7 @@ pri func encoder.find_last_distances!(data: roslice base.u8, cur_ix: base.u64, m this.sr_len = best_len this.sr_distance = backward this.sr_score = best_score + cur_byte = this.byte_at(s: args.data, i: cur_ix_masked ~mod+ best_len) } } } @@ -853,55 +878,6 @@ pri func encoder.find_last_distances!(data: roslice base.u8, cur_ix: base.u64, m return best_len } -// try_bucket_entry is the body of the bucket loop shared by the H5, H6, H58 -// and H68 FN(FindLongestMatch) functions, for one prev_ix candidate. It -// returns 0 to continue, 1 to break, or 2 + the new best_len. -pri func encoder.try_bucket_entry!(data: roslice base.u8, cur_ix: base.u64, prev_ix_in: base.u64, best_len: base.u64, max_length: base.u64, max_backward: base.u64) base.u64 { - var mask : base.u64 - var cur_ix_masked : base.u64 - var prev_ix : base.u64 - var backward : base.u64 - var len : base.u64 - var score : base.u64 - - mask = this.rb_mask as base.u64 - cur_ix_masked = args.cur_ix & mask - backward = args.cur_ix ~mod- args.prev_ix_in - if backward > args.max_backward { - return 1 - } - prev_ix = args.prev_ix_in & mask - if (cur_ix_masked ~mod+ args.best_len) > mask { - return 1 - } - if ((prev_ix ~mod+ args.best_len) > mask) or - (this.load32le(s: args.data, i: (cur_ix_masked ~mod+ args.best_len) ~mod- 3) <> - this.load32le(s: args.data, i: (prev_ix ~mod+ args.best_len) ~mod- 3)) { - return 0 - } - if (this.hasher_type == 6) or (this.hasher_type == 68) { - if this.load32le(s: args.data, i: cur_ix_masked) <> this.load32le(s: args.data, i: prev_ix) { - return 0 - } - len = this.find_match_length_with_limit(s: args.data, i1: prev_ix ~mod+ 4, i2: cur_ix_masked ~mod+ 4, - limit: args.max_length ~mod- 4) ~mod+ 4 - } else { - len = this.find_match_length_with_limit(s: args.data, i1: prev_ix, i2: cur_ix_masked, limit: args.max_length) - if len < 4 { - return 0 - } - } - score = this.backward_reference_score(copy_length: len, backward_reference_offset: backward) - if this.sr_score < score { - // The C code's best_score is always equal to out->score here. - this.sr_len = len - this.sr_distance = backward - this.sr_score = score - return 2 ~mod+ len - } - return 0 -} - // Mirrors FN(FindLongestMatch) in hash_longest_match_inc.h (H5) and // hash_longest_match64_inc.h (H6). pri func encoder.find_longest_match_h5!(wb: slice base.u8, cur_ix: base.u64, max_length: base.u64, max_backward: base.u64, dictionary_distance: base.u64, max_distance: base.u64) { @@ -916,7 +892,13 @@ pri func encoder.find_longest_match_h5!(wb: slice base.u8, cur_ix: base.u64, max var bucket : base.u64 var down : base.u64 var i : base.u64 - var r : base.u64 + var mask : base.u64 + var is64 : base.bool + var cur4 : base.u32 + var prev_ix : base.u64 + var backward : base.u64 + var len : base.u64 + var score : base.u64 if this.rb_buf_off > args.wb.length() { return nothing @@ -938,20 +920,46 @@ pri func encoder.find_longest_match_h5!(wb: slice base.u8, cur_ix: base.u64, max if (num as base.u64) > block_size { down = (num as base.u64) ~mod- block_size } + // Try the bucket's entries, newest first. Like the C code, this + // compares 4 bytes ending at best_len + 1 before the full match. + mask = this.rb_mask as base.u64 + is64 = this.hasher_type == 6 + cur4 = this.load32le(s: data, i: (cur_ix_masked ~mod+ best_len) ~mod- 3) i = num as base.u64 while i > down { i ~mod-= 1 - r = this.try_bucket_entry!( - data: data, - cur_ix: args.cur_ix, - prev_ix_in: this.load32le(s: args.wb, i: bucket ~mod+ (4 ~mod* (i & block_mask))) as base.u64, - best_len: best_len, - max_length: args.max_length, - max_backward: args.max_backward) - if r == 1 { + prev_ix = this.load32le(s: args.wb, i: bucket ~mod+ (4 ~mod* (i & block_mask))) as base.u64 + backward = args.cur_ix ~mod- prev_ix + if backward > args.max_backward { break - } else if r >= 2 { - best_len = r ~mod- 2 + } + prev_ix &= mask + if (cur_ix_masked ~mod+ best_len) > mask { + break + } + if ((prev_ix ~mod+ best_len) > mask) or + (cur4 <> this.load32le(s: data, i: (prev_ix ~mod+ best_len) ~mod- 3)) { + continue + } + if is64 { + if this.load32le(s: data, i: cur_ix_masked) <> this.load32le(s: data, i: prev_ix) { + continue + } + len = this.find_match_length_with_limit(s: data, i1: prev_ix ~mod+ 4, i2: cur_ix_masked ~mod+ 4, + limit: args.max_length ~mod- 4) ~mod+ 4 + } else { + len = this.find_match_length_with_limit(s: data, i1: prev_ix, i2: cur_ix_masked, limit: args.max_length) + if len < 4 { + continue + } + } + score = this.backward_reference_score(copy_length: len, backward_reference_offset: backward) + if this.sr_score < score { + this.sr_len = len + this.sr_distance = backward + this.sr_score = score + best_len = len + cur4 = this.load32le(s: data, i: (cur_ix_masked ~mod+ best_len) ~mod- 3) } } this.poke_u32!(s: args.wb, i: bucket ~mod+ (4 ~mod* ((num as base.u64) & block_mask)), v: (args.cur_ix & 0xFFFF_FFFF) as base.u32) @@ -984,7 +992,17 @@ pri func encoder.find_longest_match_h58!(wb: slice base.u8, cur_ix: base.u64, ma var limit : base.u64 var k : base.u64 var rb_index : base.u64 - var r : base.u64 + var splat : base.u64 + var matches : base.u64 + var chunk : base.u64 + var j : base.u64 + var mask : base.u64 + var is64 : base.bool + var cur4 : base.u32 + var prev_ix : base.u64 + var backward : base.u64 + var len : base.u64 + var score : base.u64 if this.rb_buf_off > args.wb.length() { return nothing @@ -992,7 +1010,7 @@ pri func encoder.find_longest_match_h58!(wb: slice base.u8, cur_ix: base.u64, ma data = args.wb[this.rb_buf_off ..] cur_ix_masked = args.cur_ix & (this.rb_mask as base.u64) min_score = this.sr_score - hash = this.hash_longest(data: data, i: cur_ix_masked) + hash = this.hash_h58(wb: args.wb, ix: args.cur_ix) key = (hash >> 8) as base.u64 block_size = (1 as base.u64) << this.h_block_bits block_mask = block_size ~mod- 1 @@ -1014,24 +1032,70 @@ pri func encoder.find_longest_match_h58!(wb: slice base.u8, cur_ix: base.u64, ma if block_size > (n as base.u64) { limit = n as base.u64 } - k = 0 - while k < limit { + // Like GetMatchingTagMask's non-SSE2 code: bit j of matches is whether + // the tag of entry j matches, 8 entries at a time. + splat = (tag as base.u64) ~mod* 0x0101_0101_0101_0101 + matches = 0 + j = 0 + while j < block_size { + chunk = this.load64le(s: args.wb, i: tag_bucket ~mod+ j) ^ splat + // The 0x80 bit of each byte is set when that byte is non-zero. + chunk = (((chunk | 0x8080_8080_8080_8080) ~mod- 0x0101_0101_0101_0101) | chunk) & 0x8080_8080_8080_8080 + chunk = (chunk ^ 0x8080_8080_8080_8080) >> 7 + matches |= ((chunk ~mod* 0x0102_0408_1020_4080) >> 56) ~mod<< (j & 63) + j ~mod+= 8 + } + // Rotate so that bit k is entry (head + k) & block_mask. + if head > 0 { + matches = ((matches >> (head & 63)) | (matches ~mod<< ((block_size ~mod- head) & 63))) & + ((0xFFFF_FFFF_FFFF_FFFF as base.u64) >> ((64 ~mod- block_size) & 63)) + } + if limit < 64 { + matches &= ((1 as base.u64) << (limit & 63)) ~mod- 1 + } + // Try the entries whose tags match, in the C code's order (from head). + // Like the C code, this compares 4 bytes ending at best_len + 1 before + // the full match. + mask = this.rb_mask as base.u64 + is64 = this.hasher_type == 68 + cur4 = this.load32le(s: data, i: (cur_ix_masked ~mod+ best_len) ~mod- 3) + while matches <> 0 { + k = this.ctz64(x: matches) + matches &= matches ~mod- 1 rb_index = (head ~mod+ k) & block_mask - if this.byte_at(s: args.wb, i: tag_bucket ~mod+ rb_index) == tag { - r = this.try_bucket_entry!( - data: data, - cur_ix: args.cur_ix, - prev_ix_in: this.load32le(s: args.wb, i: bucket ~mod+ (4 ~mod* rb_index)) as base.u64, - best_len: best_len, - max_length: args.max_length, - max_backward: args.max_backward) - if r == 1 { - break - } else if r >= 2 { - best_len = r ~mod- 2 + prev_ix = this.load32le(s: args.wb, i: bucket ~mod+ (4 ~mod* rb_index)) as base.u64 + backward = args.cur_ix ~mod- prev_ix + if backward > args.max_backward { + break + } + prev_ix &= mask + if (cur_ix_masked ~mod+ best_len) > mask { + break + } + if ((prev_ix ~mod+ best_len) > mask) or + (cur4 <> this.load32le(s: data, i: (prev_ix ~mod+ best_len) ~mod- 3)) { + continue + } + if is64 { + if this.load32le(s: data, i: cur_ix_masked) <> this.load32le(s: data, i: prev_ix) { + continue + } + len = this.find_match_length_with_limit(s: data, i1: prev_ix ~mod+ 4, i2: cur_ix_masked ~mod+ 4, + limit: args.max_length ~mod- 4) ~mod+ 4 + } else { + len = this.find_match_length_with_limit(s: data, i1: prev_ix, i2: cur_ix_masked, limit: args.max_length) + if len < 4 { + continue } } - k ~mod+= 1 + score = this.backward_reference_score(copy_length: len, backward_reference_offset: backward) + if this.sr_score < score { + this.sr_len = len + this.sr_distance = backward + this.sr_score = score + best_len = len + cur4 = this.load32le(s: data, i: (cur_ix_masked ~mod+ best_len) ~mod- 3) + } } this.poke_u32!(s: args.wb, i: bucket ~mod+ (4 ~mod* ((num as base.u64) & block_mask)), v: (args.cur_ix & 0xFFFF_FFFF) as base.u32) this.set_byte!(s: args.wb, i: tag_bucket ~mod+ ((num as base.u64) & block_mask), v: tag) diff --git a/std/brotli/encode_metablock.wuffs b/std/brotli/encode_metablock.wuffs index fbc42351..4f352e31 100644 --- a/std/brotli/encode_metablock.wuffs +++ b/std/brotli/encode_metablock.wuffs @@ -283,16 +283,16 @@ pri func metablock_builder.h_population_cost!(wb: roslice base.u8, h: base.u64, // Mirrors Log2FloorNonZero in fast_log.h (which truncates n to 32 bits). pri func metablock_builder.log2_floor_non_zero(n: base.u64) base.u32[..= 31] { var x : base.u32 - var r : base.u32[..= 31] + // Smear the highest set bit of the low 32 bits of n rightwards, then + // look up its position. x = (args.n & 0xFFFF_FFFF) as base.u32 - while x > 1 { - x >>= 1 - if r < 31 { - r += 1 - } - } - return r + x |= x >> 1 + x |= x >> 2 + x |= x >> 4 + x |= x >> 8 + x |= x >> 16 + return LOG2_DEBRUIJN32[(x ~mod* 0x07C4_ACDD) >> 27] } // Mirrors PrefixEncodeCopyDistance in prefix.h. It returns the code (the diff --git a/std/brotli/encode_util.wuffs b/std/brotli/encode_util.wuffs index 2d10a076..73271f1f 100644 --- a/std/brotli/encode_util.wuffs +++ b/std/brotli/encode_util.wuffs @@ -18,6 +18,21 @@ // out of bounds branches here (which do nothing, or return 0) should never be // taken. +// CTZ64_DEBRUIJN maps ((x & -x) * 0x03F7_9D71_B4CB_0A89) >> 58 to the +// number of trailing zero bits of the non-zero x. +pri const CTZ64_DEBRUIJN : roarray[64] base.u8[..= 63] = [ + 0, 1, 48, 2, 57, 49, 28, 3, 61, 58, 50, 42, 38, 29, 17, 4, + 62, 55, 59, 36, 53, 51, 43, 22, 45, 39, 33, 30, 24, 18, 12, 5, + 63, 47, 56, 27, 60, 41, 37, 16, 54, 35, 52, 21, 44, 32, 23, 11, + 46, 26, 40, 15, 34, 20, 31, 10, 25, 14, 19, 9, 13, 8, 7, 6, +] + +// ctz64 returns the number of trailing zero bits of x, which must be +// non-zero (for x == 0 it returns 0). +pri func encoder.ctz64(x: base.u64) base.u64[..= 63] { + return CTZ64_DEBRUIJN[((args.x & (0 ~mod- args.x)) ~mod* 0x03F7_9D71_B4CB_0A89) >> 58] as base.u64 +} + // byte_at returns s[i], or 0 if i is out of bounds. pri func encoder.byte_at(s: roslice base.u8, i: base.u64) base.u8 { if args.i < args.s.length() { @@ -28,12 +43,10 @@ pri func encoder.byte_at(s: roslice base.u8, i: base.u64) base.u8 { // peek_u16 returns the 2 bytes (little-endian) of s starting at i. pri func encoder.peek_u16(s: roslice base.u8, i: base.u64) base.u16 { - var q : roslice base.u8 - - if args.i < args.s.length() { - q = args.s[args.i ..] - if q.length() >= 2 { - return q.peek_u16le() + if args.i < 0xFFFF_FFFF_FFFF_0000 { + if (args.i + 2) <= args.s.length() { + assert args.i <= (args.i + 2) via "a <= (a + b): 0 <= b"(b: 2) + return args.s[args.i .. args.i + 2].peek_u16le() } } return 0 @@ -41,24 +54,20 @@ pri func encoder.peek_u16(s: roslice base.u8, i: base.u64) base.u16 { // poke_u16 sets the 2 bytes (little-endian) of s starting at i. pri func encoder.poke_u16!(s: slice base.u8, i: base.u64, v: base.u16) { - var q : slice base.u8 - - if args.i < args.s.length() { - q = args.s[args.i ..] - if q.length() >= 2 { - q.poke_u16le!(a: args.v) + if args.i < 0xFFFF_FFFF_FFFF_0000 { + if (args.i + 2) <= args.s.length() { + assert args.i <= (args.i + 2) via "a <= (a + b): 0 <= b"(b: 2) + args.s[args.i .. args.i + 2].poke_u16le!(a: args.v) } } } // poke_u32 sets the 4 bytes (little-endian) of s starting at i. pri func encoder.poke_u32!(s: slice base.u8, i: base.u64, v: base.u32) { - var q : slice base.u8 - - if args.i < args.s.length() { - q = args.s[args.i ..] - if q.length() >= 4 { - q.poke_u32le!(a: args.v) + if args.i < 0xFFFF_FFFF_FFFF_0000 { + if (args.i + 4) <= args.s.length() { + assert args.i <= (args.i + 4) via "a <= (a + b): 0 <= b"(b: 4) + args.s[args.i .. args.i + 4].poke_u32le!(a: args.v) } } }