std/brotli: speed up qualities 0 to 9 and the decoder

Encoder (output stays byte-exact with libbrotlienc v1.2.1):
- H58/H68 (q5-6): match the tags 8 at a time (SWAR, like
  GetMatchingTagMask's non-SSE2 code) and visit the matches with a de
  Bruijn ctz; inline the bucket entry check into the H5/H6 and H58/H68
  loops, caching the 4 bytes compared at best_len; a dedicated H58/H68
  store and hash.
- find_last_distances caches the current byte at best_len.
- ctz64 and a branch-free log2_floor_non_zero (de Bruijn tables), and
  find_match_length_with_limit uses ctz.
- load32le, load64le, peek_u16, poke_u16 and poke_u32 use one bounds
  check each.

Decoder (same output and statuses):
- The literal fast path keeps pos and the counts in local variables.
- Copies of at most 16 bytes, with a distance of at least 16, copy 16
  bytes (like the C code; the window gap keeps the extra bytes unread).

script/manual-test-brotli-float.c also checks log2_floor_non_zero (every
u32) and ctz64.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UZ978BikvkegL2JUkqoB1e
This commit is contained in:
Claude committed 2026-10-28 08:50:15 +00:00
1 parent 044437a8ec
commit fbae955a74
8 files changed
+713 -374

No files matched your search

+352 -194
View File
@@ -77153,6 +77153,14 @@ WUFFS_BROTLI__DICTIONARY[122784] WUFFS_BASE__POTENTIALLY_UNUSED = {
#define WUFFS_BROTLI__F64_1E99 6088095589093318446u
static const uint32_t
WUFFS_BROTLI__LOG2_DEBRUIJN32[32] WUFFS_BASE__POTENTIALLY_UNUSED = {
0u, 9u, 1u, 10u, 13u, 21u, 2u, 29u,
11u, 14u, 16u, 18u, 22u, 25u, 3u, 30u,
8u, 12u, 20u, 28u, 15u, 17u, 24u, 7u,
19u, 27u, 23u, 6u, 26u, 5u, 4u, 31u,
};
#define WUFFS_BROTLI__LITERAL_BLOCK_SWITCH_COST 4628602664527698330u
#define WUFFS_BROTLI__COMMAND_BLOCK_SWITCH_COST 4623789442425946112u
@@ -77339,6 +77347,18 @@ WUFFS_BROTLI__F32_OF_SMALL_INT[32] WUFFS_BASE__POTENTIALLY_UNUSED = {
#define WUFFS_BROTLI__HQ_SHADOW_MATCHES 256u
static const uint8_t
WUFFS_BROTLI__CTZ64_DEBRUIJN[64] WUFFS_BASE__POTENTIALLY_UNUSED = {
0u, 1u, 48u, 2u, 57u, 49u, 28u, 3u,
61u, 58u, 50u, 42u, 38u, 29u, 17u, 4u,
62u, 55u, 59u, 36u, 53u, 51u, 43u, 22u,
45u, 39u, 33u, 30u, 24u, 18u, 12u, 5u,
63u, 47u, 56u, 27u, 60u, 41u, 37u, 16u,
54u, 35u, 52u, 21u, 44u, 32u, 23u, 11u,
46u, 26u, 40u, 15u, 34u, 20u, 31u, 10u,
25u, 14u, 19u, 9u, 13u, 8u, 7u, 6u,
};
#define WUFFS_BROTLI__Z_INFINITY 2130692510u
#define WUFFS_BROTLI__Z_F32_1 1065353216u
@@ -77759,6 +77779,14 @@ wuffs_brotli__decoder__take_distance_from_ring_buffer(
wuffs_brotli__decoder* self,
uint32_t a_code);
WUFFS_BASE__GENERATED_C_CODE
static bool
wuffs_brotli__decoder__copy16(
wuffs_brotli__decoder* self,
wuffs_base__slice_u8 a_workbuf,
uint64_t a_d,
uint64_t a_s);
WUFFS_BASE__GENERATED_C_CODE
static uint32_t
wuffs_brotli__encoder__cmd_insert_len(
@@ -79652,6 +79680,20 @@ wuffs_brotli__encoder__hasher_store(
wuffs_base__slice_u8 a_wb,
uint64_t a_ix);
WUFFS_BASE__GENERATED_C_CODE
static wuffs_base__empty_struct
wuffs_brotli__encoder__hasher_store_h58(
wuffs_brotli__encoder* self,
wuffs_base__slice_u8 a_wb,
uint64_t a_ix);
WUFFS_BASE__GENERATED_C_CODE
static uint32_t
wuffs_brotli__encoder__hash_h58(
const wuffs_brotli__encoder* self,
wuffs_base__slice_u8 a_wb,
uint64_t a_ix);
WUFFS_BASE__GENERATED_C_CODE
static wuffs_base__empty_struct
wuffs_brotli__encoder__hasher_store_range(
@@ -79755,17 +79797,6 @@ wuffs_brotli__encoder__find_last_distances(
uint64_t a_max_length,
uint64_t a_max_backward);
WUFFS_BASE__GENERATED_C_CODE
static uint64_t
wuffs_brotli__encoder__try_bucket_entry(
wuffs_brotli__encoder* self,
wuffs_base__slice_u8 a_data,
uint64_t a_cur_ix,
uint64_t a_prev_ix_in,
uint64_t a_best_len,
uint64_t a_max_length,
uint64_t a_max_backward);
WUFFS_BASE__GENERATED_C_CODE
static wuffs_base__empty_struct
wuffs_brotli__encoder__find_longest_match_h5(
@@ -80547,6 +80578,12 @@ wuffs_brotli__encoder__hq_extend_var_region(
wuffs_brotli__encoder* self,
uint64_t a_wb_len);
WUFFS_BASE__GENERATED_C_CODE
static uint64_t
wuffs_brotli__encoder__ctz64(
const wuffs_brotli__encoder* self,
uint64_t a_x);
WUFFS_BASE__GENERATED_C_CODE
static uint8_t
wuffs_brotli__encoder__byte_at(
@@ -85534,6 +85571,13 @@ wuffs_brotli__decoder__process_commands_fast(
uint64_t v_dhi = 0;
uint64_t v_slo = 0;
uint64_t v_shi = 0;
uint32_t v_cm_slice = 0;
uint32_t v_wmask = 0;
uint32_t v_pos = 0;
uint32_t v_lit_n = 0;
uint32_t v_lit_i = 0;
uint64_t v_wi = 0;
bool v_copied16 = false;
const uint8_t* iop_a_src = NULL;
const uint8_t* io0_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL;
@@ -85630,25 +85674,32 @@ wuffs_brotli__decoder__process_commands_fast(
}
v_p1 = ((uint32_t)(wuffs_brotli__decoder__get_byte(self, a_workbuf, 1u)));
v_p2 = ((uint32_t)(wuffs_brotli__decoder__get_byte(self, a_workbuf, 2u)));
while (self->private_impl.f_insert_length > 0u) {
if ((self->private_impl.f_block_length[0u] == 0u) || (((uint32_t)(self->private_impl.f_pos + 1u)) >= self->private_impl.f_rb_size)) {
goto label__outer__break;
v_mode_offset = (self->private_impl.f_context_mode << 9u);
v_cm_slice = self->private_impl.f_context_map_slice;
v_wmask = ((uint32_t)(self->private_impl.f_window_size - 1u));
v_pos = self->private_impl.f_pos;
v_lit_n = wuffs_base__u32__min(self->private_impl.f_insert_length, self->private_impl.f_block_length[0u]);
if (((uint32_t)(v_pos + 1u)) >= self->private_impl.f_rb_size) {
v_lit_n = 0u;
} else {
v_lit_n = wuffs_base__u32__min(v_lit_n, ((uint32_t)(((uint32_t)(self->private_impl.f_rb_size - v_pos)) - 1u)));
}
v_lit_i = 0u;
while (v_lit_i < v_lit_n) {
if (v_n_bits < 24u) {
if (((uint64_t)(io2_a_src - iop_a_src)) < 8u) {
goto label__outer__break;
break;
}
v_bits |= ((uint64_t)(wuffs_base__peek_u64le__no_bounds_check(iop_a_src) << v_n_bits));
iop_a_src += ((63u - v_n_bits) >> 3u);
v_n_bits |= 56u;
}
v_mode_offset = (self->private_impl.f_context_mode << 9u);
v_ctx = ((uint32_t)(((uint8_t)(WUFFS_BROTLI__CONTEXT_LOOKUP[(v_mode_offset | v_p1)] | WUFFS_BROTLI__CONTEXT_LOOKUP[(v_mode_offset | 256u | v_p2)]))));
v_tree = ((uint32_t)(self->private_data.f_context_maps[(self->private_impl.f_context_map_slice + v_ctx)]));
v_tree = ((uint32_t)(self->private_data.f_context_maps[(v_cm_slice + v_ctx)]));
v_i = ((uint32_t)((v_tree * 632u) + ((uint32_t)((v_bits & 255u)))));
if (v_i >= 678034u) {
v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table);
goto label__outer__break;
break;
}
v_e = ((uint32_t)(self->private_data.f_huffs[v_i]));
v_e_bits = (v_e & 15u);
@@ -85656,23 +85707,34 @@ wuffs_brotli__decoder__process_commands_fast(
v_i = ((uint32_t)(((uint32_t)((v_tree * 632u) + (v_e >> 4u))) + (((uint32_t)(((v_bits >> 8u) & 32767u))) & ((((uint32_t)(1u)) << (v_e_bits - 8u)) - 1u))));
if (v_i >= 678034u) {
v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table);
goto label__outer__break;
break;
}
v_e = ((uint32_t)(self->private_data.f_huffs[v_i]));
v_e_bits = ((v_e & 15u) + 8u);
}
if (v_n_bits < v_e_bits) {
v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_bit_reader);
goto label__outer__break;
break;
}
v_bits >>= v_e_bits;
v_n_bits -= v_e_bits;
self->private_impl.f_symbol = (v_e >> 4u);
wuffs_brotli__decoder__put_byte(self, a_workbuf, ((uint8_t)(self->private_impl.f_symbol)));
v_p2 = v_p1;
v_p1 = (self->private_impl.f_symbol & 255u);
self->private_impl.f_block_length[0u] -= 1u;
self->private_impl.f_insert_length -= 1u;
v_p1 = ((v_e >> 4u) & 255u);
v_wi = ((uint64_t)((v_pos & v_wmask)));
if (v_wi < ((uint64_t)(a_workbuf.len))) {
a_workbuf.ptr[v_wi] = ((uint8_t)(v_p1));
}
v_pos += 1u;
v_lit_i += 1u;
}
if (v_lit_i > 0u) {
self->private_impl.f_symbol = v_p1;
self->private_impl.f_pos = v_pos;
self->private_impl.f_block_length[0u] -= v_lit_i;
self->private_impl.f_insert_length -= v_lit_i;
}
if (wuffs_base__status__is_error(&v_ret) || (self->private_impl.f_insert_length > 0u)) {
break;
}
self->private_impl.f_cmd_state = 2u;
if ((self->private_impl.f_mb_remaining == 0u) || (self->private_impl.f_mb_remaining >= 2147483648u)) {
@@ -85758,6 +85820,15 @@ wuffs_brotli__decoder__process_commands_fast(
self->private_impl.f_dist_rb_idx += 1u;
self->private_impl.f_mb_remaining -= v_n;
v_k64 = 0u;
if ((v_n <= 16u) &&
(v_distance >= 16u) &&
(((uint64_t)(v_dst_i + 16u)) <= ((uint64_t)(self->private_impl.f_rb_size))) &&
(((uint64_t)(v_src_i + 16u)) <= ((uint64_t)(self->private_impl.f_window_size)))) {
v_copied16 = wuffs_brotli__decoder__copy16(self, a_workbuf, v_dst_i, v_src_i);
if (v_copied16) {
v_k64 = ((uint64_t)(v_n));
}
}
while (v_k64 < ((uint64_t)(v_n))) {
v_n64 = ((uint64_t)(((uint64_t)(v_n)) - v_k64));
v_n64 = wuffs_base__u64__min(v_n64, ((uint64_t)(v_distance)));
@@ -85787,7 +85858,6 @@ wuffs_brotli__decoder__process_commands_fast(
break;
}
}
label__outer__break:;
while (v_n_bits >= 8u) {
if ( ! (iop_a_src > io1_a_src)) {
status = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_bit_reader);
@@ -85938,6 +86008,43 @@ wuffs_brotli__decoder__take_distance_from_ring_buffer(
return wuffs_base__make_empty_struct();
}
// -------- func brotli.decoder.copy16
WUFFS_BASE__GENERATED_C_CODE
static bool
wuffs_brotli__decoder__copy16(
wuffs_brotli__decoder* self,
wuffs_base__slice_u8 a_workbuf,
uint64_t a_d,
uint64_t a_s) {
uint64_t v_lo = 0;
uint64_t v_hi = 0;
wuffs_base__slice_u8 v_q = {0};
wuffs_base__slice_u8 v_r = {0};
if ((a_s > ((uint64_t)(a_workbuf.len))) || (a_d > ((uint64_t)(a_workbuf.len)))) {
return false;
}
v_q = wuffs_base__slice_u8__subslice_i(a_workbuf, a_s);
if (((uint64_t)(v_q.len)) >= 16u) {
v_lo = wuffs_base__peek_u64le__no_bounds_check(v_q.ptr);
v_q = wuffs_base__slice_u8__subslice_i(v_q, 8u);
if (((uint64_t)(v_q.len)) >= 8u) {
v_hi = wuffs_base__peek_u64le__no_bounds_check(v_q.ptr);
v_q = wuffs_base__slice_u8__subslice_i(a_workbuf, a_d);
if (((uint64_t)(v_q.len)) >= 16u) {
v_r = wuffs_base__slice_u8__subslice_i(v_q, 8u);
wuffs_base__poke_u64le__no_bounds_check(v_q.ptr, v_lo);
if (((uint64_t)(v_r.len)) >= 8u) {
wuffs_base__poke_u64le__no_bounds_check(v_r.ptr, v_hi);
return true;
}
}
}
}
return false;
}
// -------- func brotli.encoder.cmd_insert_len
WUFFS_BASE__GENERATED_C_CODE
@@ -88206,29 +88313,14 @@ wuffs_brotli__encoder__log2_floor_non_zero(
const wuffs_brotli__encoder* self,
uint64_t a_n) {
uint32_t v_x = 0;
uint32_t v_r = 0;
v_x = ((uint32_t)(a_n));
if (v_x >= 65536u) {
v_x >>= 16u;
v_r = 16u;
}
if (v_x >= 256u) {
v_x >>= 8u;
v_r += 8u;
}
if (v_x >= 16u) {
v_x >>= 4u;
v_r += 4u;
}
if (v_x >= 4u) {
v_x >>= 2u;
v_r += 2u;
}
if (v_x >= 2u) {
v_r += 1u;
}
return (v_r & 31u);
v_x |= (v_x >> 1u);
v_x |= (v_x >> 2u);
v_x |= (v_x >> 4u);
v_x |= (v_x >> 8u);
v_x |= (v_x >> 16u);
return WUFFS_BROTLI__LOG2_DEBRUIJN32[(((uint32_t)(v_x * 130329821u)) >> 27u)];
}
// -------- func brotli.encoder.fast_log2
@@ -88279,12 +88371,11 @@ wuffs_brotli__encoder__load64le(
const wuffs_brotli__encoder* self,
wuffs_base__slice_u8 a_s,
uint64_t a_i) {
wuffs_base__slice_u8 v_q = {0};
if (a_i < ((uint64_t)(a_s.len))) {
v_q = wuffs_base__slice_u8__subslice_i(a_s, a_i);
if (((uint64_t)(v_q.len)) >= 8u) {
return wuffs_base__peek_u64le__no_bounds_check(v_q.ptr);
if (a_i < 18446744073709486080u) {
if ((a_i + 8u) <= ((uint64_t)(a_s.len))) {
return wuffs_base__peek_u64le__no_bounds_check(wuffs_base__slice_u8__subslice_ij(a_s,
a_i,
(a_i + 8u)).ptr);
}
}
return 0u;
@@ -88298,12 +88389,11 @@ wuffs_brotli__encoder__load32le(
const wuffs_brotli__encoder* self,
wuffs_base__slice_u8 a_s,
uint64_t a_i) {
wuffs_base__slice_u8 v_q = {0};
if (a_i < ((uint64_t)(a_s.len))) {
v_q = wuffs_base__slice_u8__subslice_i(a_s, a_i);
if (((uint64_t)(v_q.len)) >= 4u) {
return wuffs_base__peek_u32le__no_bounds_check(v_q.ptr);
if (a_i < 18446744073709486080u) {
if ((a_i + 4u) <= ((uint64_t)(a_s.len))) {
return wuffs_base__peek_u32le__no_bounds_check(wuffs_base__slice_u8__subslice_ij(a_s,
a_i,
(a_i + 4u)).ptr);
}
}
return 0u;
@@ -88334,11 +88424,7 @@ wuffs_brotli__encoder__find_match_length_with_limit(
while ((v_n >= 8u) && (((uint64_t)(v_a.len)) >= 8u) && (((uint64_t)(v_b.len)) >= 8u)) {
v_x = (wuffs_base__peek_u64le__no_bounds_check(v_a.ptr) ^ wuffs_base__peek_u64le__no_bounds_check(v_b.ptr));
if (v_x != 0u) {
while ((v_x & 255u) == 0u) {
v_matched += 1u;
v_x >>= 8u;
}
return v_matched;
return ((uint64_t)(v_matched + (wuffs_brotli__encoder__ctz64(self, v_x) >> 3u)));
}
v_matched += 8u;
v_n -= 8u;
@@ -96842,7 +96928,6 @@ wuffs_brotli__encoder__hasher_store(
uint64_t v_bsize = 0;
uint64_t v_off = 0;
uint16_t v_num = 0;
uint32_t v_hash = 0;
uint64_t v_bank = 0;
uint64_t v_idx = 0;
uint64_t v_delta = 0;
@@ -96850,6 +96935,10 @@ wuffs_brotli__encoder__hasher_store(
uint32_t v_fc_bits = 0;
v_htype = self->private_impl.f_hasher_type;
if ((v_htype == 58u) || (v_htype == 68u)) {
wuffs_brotli__encoder__hasher_store_h58(self, a_wb, a_ix);
return wuffs_base__make_empty_struct();
}
if (self->private_impl.f_rb_buf_off > ((uint64_t)(a_wb.len))) {
return wuffs_base__make_empty_struct();
}
@@ -96868,14 +96957,6 @@ wuffs_brotli__encoder__hasher_store(
v_off = ((uint64_t)((((uint64_t)(v_num)) & ((((uint64_t)(1u)) << self->private_impl.f_h_block_bits) - 1u)) + (((uint64_t)(v_key)) << self->private_impl.f_h_block_bits)));
wuffs_brotli__encoder__poke_u16(self, a_wb, ((uint64_t)(self->private_impl.f_h_off0 + ((uint64_t)(2u * ((uint64_t)(v_key)))))), ((uint16_t)(v_num + 1u)));
wuffs_brotli__encoder__poke_u32(self, a_wb, ((uint64_t)(self->private_impl.f_h_off1 + ((uint64_t)(4u * v_off)))), ((uint32_t)(a_ix)));
} else if ((v_htype == 58u) || (v_htype == 68u)) {
v_hash = wuffs_brotli__encoder__hash_longest(self, v_data, (a_ix & ((uint64_t)(self->private_impl.f_rb_mask))));
v_key = (v_hash >> 8u);
v_num = wuffs_brotli__encoder__peek_u16(self, a_wb, ((uint64_t)(self->private_impl.f_h_off0 + ((uint64_t)(2u * ((uint64_t)(v_key)))))));
v_off = ((uint64_t)((((uint64_t)(v_num)) & ((((uint64_t)(1u)) << self->private_impl.f_h_block_bits) - 1u)) + (((uint64_t)(v_key)) << self->private_impl.f_h_block_bits)));
wuffs_brotli__encoder__poke_u16(self, a_wb, ((uint64_t)(self->private_impl.f_h_off0 + ((uint64_t)(2u * ((uint64_t)(v_key)))))), ((uint16_t)(v_num - 1u)));
wuffs_brotli__encoder__poke_u32(self, a_wb, ((uint64_t)(self->private_impl.f_h_off2 + ((uint64_t)(4u * v_off)))), ((uint32_t)(a_ix)));
wuffs_brotli__encoder__set_byte(self, a_wb, ((uint64_t)(self->private_impl.f_h_off1 + v_off)), ((uint8_t)(v_hash)));
} else if ((v_htype == 40u) || (v_htype == 41u) || (v_htype == 42u)) {
v_key = wuffs_brotli__encoder__hash_forgetful(self, v_data, (a_ix & ((uint64_t)(self->private_impl.f_rb_mask))));
v_fc_bits = wuffs_brotli__encoder__fc_bank_bits(self);
@@ -96904,6 +96985,48 @@ wuffs_brotli__encoder__hasher_store(
return wuffs_base__make_empty_struct();
}
// -------- func brotli.encoder.hasher_store_h58
WUFFS_BASE__GENERATED_C_CODE
static wuffs_base__empty_struct
wuffs_brotli__encoder__hasher_store_h58(
wuffs_brotli__encoder* self,
wuffs_base__slice_u8 a_wb,
uint64_t a_ix) {
uint32_t v_hash = 0;
uint64_t v_key = 0;
uint64_t v_koff = 0;
uint16_t v_num = 0;
uint64_t v_off = 0;
v_hash = wuffs_brotli__encoder__hash_h58(self, a_wb, a_ix);
v_key = ((uint64_t)((v_hash >> 8u)));
v_koff = ((uint64_t)(self->private_impl.f_h_off0 + ((uint64_t)(2u * v_key))));
v_num = wuffs_brotli__encoder__peek_u16(self, a_wb, v_koff);
v_off = ((uint64_t)((((uint64_t)(v_num)) & ((((uint64_t)(1u)) << self->private_impl.f_h_block_bits) - 1u)) + (v_key << self->private_impl.f_h_block_bits)));
wuffs_brotli__encoder__poke_u16(self, a_wb, v_koff, ((uint16_t)(v_num - 1u)));
wuffs_brotli__encoder__poke_u32(self, a_wb, ((uint64_t)(self->private_impl.f_h_off2 + ((uint64_t)(4u * v_off)))), ((uint32_t)(a_ix)));
wuffs_brotli__encoder__set_byte(self, a_wb, ((uint64_t)(self->private_impl.f_h_off1 + v_off)), ((uint8_t)(v_hash)));
return wuffs_base__make_empty_struct();
}
// -------- func brotli.encoder.hash_h58
WUFFS_BASE__GENERATED_C_CODE
static uint32_t
wuffs_brotli__encoder__hash_h58(
const wuffs_brotli__encoder* self,
wuffs_base__slice_u8 a_wb,
uint64_t a_ix) {
uint64_t v_i = 0;
v_i = ((uint64_t)(self->private_impl.f_rb_buf_off + (a_ix & ((uint64_t)(self->private_impl.f_rb_mask)))));
if (self->private_impl.f_hasher_type == 58u) {
return (((uint32_t)(wuffs_brotli__encoder__load32le(self, a_wb, v_i) * 506832829u)) >> ((24u - self->private_impl.f_h_bucket_bits) & 31u));
}
return ((uint32_t)((((uint64_t)(wuffs_brotli__encoder__load64le(self, a_wb, v_i) * 8922571613522624512u)) >> 41u)));
}
// -------- func brotli.encoder.hasher_store_range
WUFFS_BASE__GENERATED_C_CODE
@@ -97384,6 +97507,8 @@ wuffs_brotli__encoder__find_last_distances(
uint64_t v_prev_ix = 0;
uint64_t v_len = 0;
uint64_t v_score = 0;
uint8_t v_cur_byte = 0;
uint32_t v_n = 0;
v_mask = ((uint64_t)(self->private_impl.f_rb_mask));
v_cur_ix_masked = (a_cur_ix & v_mask);
@@ -97391,15 +97516,13 @@ wuffs_brotli__encoder__find_last_distances(
v_best_len = self->private_impl.f_sr_len;
self->private_impl.f_sr_len = 0u;
self->private_impl.f_sr_len_code_delta = 0u;
v_cur_byte = wuffs_brotli__encoder__byte_at(self, a_data, ((uint64_t)(v_cur_ix_masked + v_best_len)));
v_n = self->private_impl.f_h_num_last_distances;
v_i = 0u;
while (v_i < self->private_impl.f_h_num_last_distances) {
while (v_i < v_n) {
v_backward = wuffs_brotli__encoder__dist_cache_u64(self, v_i);
v_prev_ix = ((uint64_t)(a_cur_ix - v_backward));
if (v_prev_ix >= a_cur_ix) {
v_i += 1u;
continue;
}
if (v_backward > a_max_backward) {
if ((v_prev_ix >= a_cur_ix) || (v_backward > a_max_backward)) {
v_i += 1u;
continue;
}
@@ -97407,7 +97530,7 @@ wuffs_brotli__encoder__find_last_distances(
if (((uint64_t)(v_cur_ix_masked + v_best_len)) > v_mask) {
break;
}
if ((((uint64_t)(v_prev_ix + v_best_len)) > v_mask) || (wuffs_brotli__encoder__byte_at(self, a_data, ((uint64_t)(v_cur_ix_masked + v_best_len))) != wuffs_brotli__encoder__byte_at(self, a_data, ((uint64_t)(v_prev_ix + v_best_len))))) {
if ((((uint64_t)(v_prev_ix + v_best_len)) > v_mask) || (v_cur_byte != wuffs_brotli__encoder__byte_at(self, a_data, ((uint64_t)(v_prev_ix + v_best_len))))) {
v_i += 1u;
continue;
}
@@ -97428,6 +97551,7 @@ wuffs_brotli__encoder__find_last_distances(
self->private_impl.f_sr_len = v_best_len;
self->private_impl.f_sr_distance = v_backward;
self->private_impl.f_sr_score = v_best_score;
v_cur_byte = wuffs_brotli__encoder__byte_at(self, a_data, ((uint64_t)(v_cur_ix_masked + v_best_len)));
}
}
}
@@ -97436,67 +97560,6 @@ wuffs_brotli__encoder__find_last_distances(
return v_best_len;
}
// -------- func brotli.encoder.try_bucket_entry
WUFFS_BASE__GENERATED_C_CODE
static uint64_t
wuffs_brotli__encoder__try_bucket_entry(
wuffs_brotli__encoder* self,
wuffs_base__slice_u8 a_data,
uint64_t a_cur_ix,
uint64_t a_prev_ix_in,
uint64_t a_best_len,
uint64_t a_max_length,
uint64_t a_max_backward) {
uint64_t v_mask = 0;
uint64_t v_cur_ix_masked = 0;
uint64_t v_prev_ix = 0;
uint64_t v_backward = 0;
uint64_t v_len = 0;
uint64_t v_score = 0;
v_mask = ((uint64_t)(self->private_impl.f_rb_mask));
v_cur_ix_masked = (a_cur_ix & v_mask);
v_backward = ((uint64_t)(a_cur_ix - a_prev_ix_in));
if (v_backward > a_max_backward) {
return 1u;
}
v_prev_ix = (a_prev_ix_in & v_mask);
if (((uint64_t)(v_cur_ix_masked + a_best_len)) > v_mask) {
return 1u;
}
if ((((uint64_t)(v_prev_ix + a_best_len)) > v_mask) || (wuffs_brotli__encoder__load32le(self, a_data, ((uint64_t)(((uint64_t)(v_cur_ix_masked + a_best_len)) - 3u))) != wuffs_brotli__encoder__load32le(self, a_data, ((uint64_t)(((uint64_t)(v_prev_ix + a_best_len)) - 3u))))) {
return 0u;
}
if ((self->private_impl.f_hasher_type == 6u) || (self->private_impl.f_hasher_type == 68u)) {
if (wuffs_brotli__encoder__load32le(self, a_data, v_cur_ix_masked) != wuffs_brotli__encoder__load32le(self, a_data, v_prev_ix)) {
return 0u;
}
v_len = ((uint64_t)(wuffs_brotli__encoder__find_match_length_with_limit(self,
a_data,
((uint64_t)(v_prev_ix + 4u)),
((uint64_t)(v_cur_ix_masked + 4u)),
((uint64_t)(a_max_length - 4u))) + 4u));
} else {
v_len = wuffs_brotli__encoder__find_match_length_with_limit(self,
a_data,
v_prev_ix,
v_cur_ix_masked,
a_max_length);
if (v_len < 4u) {
return 0u;
}
}
v_score = wuffs_brotli__encoder__backward_reference_score(self, v_len, v_backward);
if (self->private_impl.f_sr_score < v_score) {
self->private_impl.f_sr_len = v_len;
self->private_impl.f_sr_distance = v_backward;
self->private_impl.f_sr_score = v_score;
return ((uint64_t)(2u + v_len));
}
return 0u;
}
// -------- func brotli.encoder.find_longest_match_h5
WUFFS_BASE__GENERATED_C_CODE
@@ -97520,7 +97583,13 @@ wuffs_brotli__encoder__find_longest_match_h5(
uint64_t v_bucket = 0;
uint64_t v_down = 0;
uint64_t v_i = 0;
uint64_t v_r = 0;
uint64_t v_mask = 0;
bool v_is64 = false;
uint32_t v_cur4 = 0;
uint64_t v_prev_ix = 0;
uint64_t v_backward = 0;
uint64_t v_len = 0;
uint64_t v_score = 0;
if (self->private_impl.f_rb_buf_off > ((uint64_t)(a_wb.len))) {
return wuffs_base__make_empty_struct();
@@ -97545,20 +97614,50 @@ wuffs_brotli__encoder__find_longest_match_h5(
if (((uint64_t)(v_num)) > v_block_size) {
v_down = ((uint64_t)(((uint64_t)(v_num)) - v_block_size));
}
v_mask = ((uint64_t)(self->private_impl.f_rb_mask));
v_is64 = (self->private_impl.f_hasher_type == 6u);
v_cur4 = wuffs_brotli__encoder__load32le(self, v_data, ((uint64_t)(((uint64_t)(v_cur_ix_masked + v_best_len)) - 3u)));
v_i = ((uint64_t)(v_num));
while (v_i > v_down) {
v_i -= 1u;
v_r = wuffs_brotli__encoder__try_bucket_entry(self,
v_data,
a_cur_ix,
((uint64_t)(wuffs_brotli__encoder__load32le(self, a_wb, ((uint64_t)(v_bucket + ((uint64_t)(4u * (v_i & v_block_mask)))))))),
v_best_len,
a_max_length,
a_max_backward);
if (v_r == 1u) {
v_prev_ix = ((uint64_t)(wuffs_brotli__encoder__load32le(self, a_wb, ((uint64_t)(v_bucket + ((uint64_t)(4u * (v_i & v_block_mask))))))));
v_backward = ((uint64_t)(a_cur_ix - v_prev_ix));
if (v_backward > a_max_backward) {
break;
} else if (v_r >= 2u) {
v_best_len = ((uint64_t)(v_r - 2u));
}
v_prev_ix &= v_mask;
if (((uint64_t)(v_cur_ix_masked + v_best_len)) > v_mask) {
break;
}
if ((((uint64_t)(v_prev_ix + v_best_len)) > v_mask) || (v_cur4 != wuffs_brotli__encoder__load32le(self, v_data, ((uint64_t)(((uint64_t)(v_prev_ix + v_best_len)) - 3u))))) {
continue;
}
if (v_is64) {
if (wuffs_brotli__encoder__load32le(self, v_data, v_cur_ix_masked) != wuffs_brotli__encoder__load32le(self, v_data, v_prev_ix)) {
continue;
}
v_len = ((uint64_t)(wuffs_brotli__encoder__find_match_length_with_limit(self,
v_data,
((uint64_t)(v_prev_ix + 4u)),
((uint64_t)(v_cur_ix_masked + 4u)),
((uint64_t)(a_max_length - 4u))) + 4u));
} else {
v_len = wuffs_brotli__encoder__find_match_length_with_limit(self,
v_data,
v_prev_ix,
v_cur_ix_masked,
a_max_length);
if (v_len < 4u) {
continue;
}
}
v_score = wuffs_brotli__encoder__backward_reference_score(self, v_len, v_backward);
if (self->private_impl.f_sr_score < v_score) {
self->private_impl.f_sr_len = v_len;
self->private_impl.f_sr_distance = v_backward;
self->private_impl.f_sr_score = v_score;
v_best_len = v_len;
v_cur4 = wuffs_brotli__encoder__load32le(self, v_data, ((uint64_t)(((uint64_t)(v_cur_ix_masked + v_best_len)) - 3u)));
}
}
wuffs_brotli__encoder__poke_u32(self, a_wb, ((uint64_t)(v_bucket + ((uint64_t)(4u * (((uint64_t)(v_num)) & v_block_mask))))), ((uint32_t)(a_cur_ix)));
@@ -97604,7 +97703,17 @@ wuffs_brotli__encoder__find_longest_match_h58(
uint64_t v_limit = 0;
uint64_t v_k = 0;
uint64_t v_rb_index = 0;
uint64_t v_r = 0;
uint64_t v_splat = 0;
uint64_t v_matches = 0;
uint64_t v_chunk = 0;
uint64_t v_j = 0;
uint64_t v_mask = 0;
bool v_is64 = false;
uint32_t v_cur4 = 0;
uint64_t v_prev_ix = 0;
uint64_t v_backward = 0;
uint64_t v_len = 0;
uint64_t v_score = 0;
if (self->private_impl.f_rb_buf_off > ((uint64_t)(a_wb.len))) {
return wuffs_base__make_empty_struct();
@@ -97612,7 +97721,7 @@ wuffs_brotli__encoder__find_longest_match_h58(
v_data = wuffs_base__slice_u8__subslice_i(a_wb, self->private_impl.f_rb_buf_off);
v_cur_ix_masked = (a_cur_ix & ((uint64_t)(self->private_impl.f_rb_mask)));
v_min_score = self->private_impl.f_sr_score;
v_hash = wuffs_brotli__encoder__hash_longest(self, v_data, v_cur_ix_masked);
v_hash = wuffs_brotli__encoder__hash_h58(self, a_wb, a_cur_ix);
v_key = ((uint64_t)((v_hash >> 8u)));
v_block_size = (((uint64_t)(1u)) << self->private_impl.f_h_block_bits);
v_block_mask = ((uint64_t)(v_block_size - 1u));
@@ -97634,24 +97743,68 @@ wuffs_brotli__encoder__find_longest_match_h58(
if (v_block_size > ((uint64_t)(v_n))) {
v_limit = ((uint64_t)(v_n));
}
v_k = 0u;
while (v_k < v_limit) {
v_splat = ((uint64_t)(((uint64_t)(v_tag)) * 72340172838076673u));
v_matches = 0u;
v_j = 0u;
while (v_j < v_block_size) {
v_chunk = (wuffs_brotli__encoder__load64le(self, a_wb, ((uint64_t)(v_tag_bucket + v_j))) ^ v_splat);
v_chunk = ((((uint64_t)((v_chunk | 9259542123273814144u) - 72340172838076673u)) | v_chunk) & 9259542123273814144u);
v_chunk = ((v_chunk ^ 9259542123273814144u) >> 7u);
v_matches |= ((uint64_t)((((uint64_t)(v_chunk * 72624976668147840u)) >> 56u) << (v_j & 63u)));
v_j += 8u;
}
if (v_head > 0u) {
v_matches = (((v_matches >> (v_head & 63u)) | ((uint64_t)(v_matches << (((uint64_t)(v_block_size - v_head)) & 63u)))) & (((uint64_t)(18446744073709551615u)) >> (((uint64_t)(64u - v_block_size)) & 63u)));
}
if (v_limit < 64u) {
v_matches &= ((uint64_t)((((uint64_t)(1u)) << (v_limit & 63u)) - 1u));
}
v_mask = ((uint64_t)(self->private_impl.f_rb_mask));
v_is64 = (self->private_impl.f_hasher_type == 68u);
v_cur4 = wuffs_brotli__encoder__load32le(self, v_data, ((uint64_t)(((uint64_t)(v_cur_ix_masked + v_best_len)) - 3u)));
while (v_matches != 0u) {
v_k = wuffs_brotli__encoder__ctz64(self, v_matches);
v_matches &= ((uint64_t)(v_matches - 1u));
v_rb_index = (((uint64_t)(v_head + v_k)) & v_block_mask);
if (wuffs_brotli__encoder__byte_at(self, a_wb, ((uint64_t)(v_tag_bucket + v_rb_index))) == v_tag) {
v_r = wuffs_brotli__encoder__try_bucket_entry(self,
v_data,
a_cur_ix,
((uint64_t)(wuffs_brotli__encoder__load32le(self, a_wb, ((uint64_t)(v_bucket + ((uint64_t)(4u * v_rb_index))))))),
v_best_len,
a_max_length,
a_max_backward);
if (v_r == 1u) {
v_prev_ix = ((uint64_t)(wuffs_brotli__encoder__load32le(self, a_wb, ((uint64_t)(v_bucket + ((uint64_t)(4u * v_rb_index)))))));
v_backward = ((uint64_t)(a_cur_ix - v_prev_ix));
if (v_backward > a_max_backward) {
break;
} else if (v_r >= 2u) {
v_best_len = ((uint64_t)(v_r - 2u));
}
v_prev_ix &= v_mask;
if (((uint64_t)(v_cur_ix_masked + v_best_len)) > v_mask) {
break;
}
if ((((uint64_t)(v_prev_ix + v_best_len)) > v_mask) || (v_cur4 != wuffs_brotli__encoder__load32le(self, v_data, ((uint64_t)(((uint64_t)(v_prev_ix + v_best_len)) - 3u))))) {
continue;
}
if (v_is64) {
if (wuffs_brotli__encoder__load32le(self, v_data, v_cur_ix_masked) != wuffs_brotli__encoder__load32le(self, v_data, v_prev_ix)) {
continue;
}
v_len = ((uint64_t)(wuffs_brotli__encoder__find_match_length_with_limit(self,
v_data,
((uint64_t)(v_prev_ix + 4u)),
((uint64_t)(v_cur_ix_masked + 4u)),
((uint64_t)(a_max_length - 4u))) + 4u));
} else {
v_len = wuffs_brotli__encoder__find_match_length_with_limit(self,
v_data,
v_prev_ix,
v_cur_ix_masked,
a_max_length);
if (v_len < 4u) {
continue;
}
}
v_k += 1u;
v_score = wuffs_brotli__encoder__backward_reference_score(self, v_len, v_backward);
if (self->private_impl.f_sr_score < v_score) {
self->private_impl.f_sr_len = v_len;
self->private_impl.f_sr_distance = v_backward;
self->private_impl.f_sr_score = v_score;
v_best_len = v_len;
v_cur4 = wuffs_brotli__encoder__load32le(self, v_data, ((uint64_t)(((uint64_t)(v_cur_ix_masked + v_best_len)) - 3u)));
}
}
wuffs_brotli__encoder__poke_u32(self, a_wb, ((uint64_t)(v_bucket + ((uint64_t)(4u * (((uint64_t)(v_num)) & v_block_mask))))), ((uint32_t)(a_cur_ix)));
wuffs_brotli__encoder__set_byte(self, a_wb, ((uint64_t)(v_tag_bucket + (((uint64_t)(v_num)) & v_block_mask))), v_tag);
@@ -98976,16 +99129,14 @@ wuffs_brotli__metablock_builder__log2_floor_non_zero(
const wuffs_brotli__metablock_builder* self,
uint64_t a_n) {
uint32_t v_x = 0;
uint32_t v_r = 0;
v_x = ((uint32_t)(a_n));
while (v_x > 1u) {
v_x >>= 1u;
if (v_r < 31u) {
v_r += 1u;
}
}
return v_r;
v_x |= (v_x >> 1u);
v_x |= (v_x >> 2u);
v_x |= (v_x >> 4u);
v_x |= (v_x >> 8u);
v_x |= (v_x >> 16u);
return WUFFS_BROTLI__LOG2_DEBRUIJN32[(((uint32_t)(v_x * 130329821u)) >> 27u)];
}
// -------- func brotli.metablock_builder.prefix_encode_copy_distance
@@ -101839,6 +101990,16 @@ wuffs_brotli__metablock_tester__get(
return 0u;
}
// -------- func brotli.encoder.ctz64
WUFFS_BASE__GENERATED_C_CODE
static uint64_t
wuffs_brotli__encoder__ctz64(
const wuffs_brotli__encoder* self,
uint64_t a_x) {
return ((uint64_t)(WUFFS_BROTLI__CTZ64_DEBRUIJN[(((uint64_t)((a_x & ((uint64_t)(0u - a_x))) * 285870213051386505u)) >> 58u)]));
}
// -------- func brotli.encoder.byte_at
WUFFS_BASE__GENERATED_C_CODE
@@ -101861,12 +102022,11 @@ wuffs_brotli__encoder__peek_u16(
const wuffs_brotli__encoder* self,
wuffs_base__slice_u8 a_s,
uint64_t a_i) {
wuffs_base__slice_u8 v_q = {0};
if (a_i < ((uint64_t)(a_s.len))) {
v_q = wuffs_base__slice_u8__subslice_i(a_s, a_i);
if (((uint64_t)(v_q.len)) >= 2u) {
return wuffs_base__peek_u16le__no_bounds_check(v_q.ptr);
if (a_i < 18446744073709486080u) {
if ((a_i + 2u) <= ((uint64_t)(a_s.len))) {
return wuffs_base__peek_u16le__no_bounds_check(wuffs_base__slice_u8__subslice_ij(a_s,
a_i,
(a_i + 2u)).ptr);
}
}
return 0u;
@@ -101881,12 +102041,11 @@ wuffs_brotli__encoder__poke_u16(
wuffs_base__slice_u8 a_s,
uint64_t a_i,
uint16_t a_v) {
wuffs_base__slice_u8 v_q = {0};
if (a_i < ((uint64_t)(a_s.len))) {
v_q = wuffs_base__slice_u8__subslice_i(a_s, a_i);
if (((uint64_t)(v_q.len)) >= 2u) {
wuffs_base__poke_u16le__no_bounds_check(v_q.ptr, a_v);
if (a_i < 18446744073709486080u) {
if ((a_i + 2u) <= ((uint64_t)(a_s.len))) {
wuffs_base__poke_u16le__no_bounds_check(wuffs_base__slice_u8__subslice_ij(a_s,
a_i,
(a_i + 2u)).ptr, a_v);
}
}
return wuffs_base__make_empty_struct();
@@ -101901,12 +102060,11 @@ wuffs_brotli__encoder__poke_u32(
wuffs_base__slice_u8 a_s,
uint64_t a_i,
uint32_t a_v) {
wuffs_base__slice_u8 v_q = {0};
if (a_i < ((uint64_t)(a_s.len))) {
v_q = wuffs_base__slice_u8__subslice_i(a_s, a_i);
if (((uint64_t)(v_q.len)) >= 4u) {
wuffs_base__poke_u32le__no_bounds_check(v_q.ptr, a_v);
if (a_i < 18446744073709486080u) {
if ((a_i + 4u) <= ((uint64_t)(a_s.len))) {
wuffs_base__poke_u32le__no_bounds_check(wuffs_base__slice_u8__subslice_ij(a_s,
a_i,
(a_i + 4u)).ptr, a_v);
}
}
return wuffs_base__make_empty_struct();
+41
View File
@@ -23,6 +23,9 @@
// gcc -O2 -std=c99 manual-test-brotli-float.c -o /tmp/brfloat
// /tmp/brfloat # Default: 100 million operations per op.
// /tmp/brfloat 1000000 # Or pass the number of operations per op.
// It also checks the integer bit helpers (log2_floor_non_zero, for every u32,
// and ctz64) against the compiler's __builtin_clz and __builtin_ctzll.
//
// It prints one line per op and the first few mismatches (if any), and exits
// with a non-zero status if there were any mismatches. NaN results only need
// to be NaN (on both sides).
@@ -46,6 +49,7 @@
#error "Build without -ffast-math and without -mfma / -march=native"
#endif
static wuffs_brotli__encoder* g_enc;
static wuffs_brotli__bit_cost* g_bc;
static uint64_t g_rng = 0x123456789ABCDEF1ull;
@@ -334,6 +338,41 @@ check_conversions(uint64_t n) {
g_total_mismatches += mismatches;
}
static void //
check_bit_helpers(uint64_t n) {
uint64_t mismatches = 0;
wuffs_brotli__encoder* enc = g_enc;
// log2_floor_non_zero truncates its argument to 32 bits (and returns 0 for
// 0), so check every u32 value, with random high bits.
uint32_t v = 0;
do {
uint64_t x = ((uint64_t)(rnd() & 0xFFFFFFFF00000000ull)) | v;
uint32_t want = v ? (31 - (uint32_t)__builtin_clz(v)) : 0;
uint32_t have0 = wuffs_brotli__encoder__log2_floor_non_zero(enc, x);
uint32_t have1 = wuffs_brotli__metablock_builder__log2_floor_non_zero(
&enc->private_data.f_mb, x);
if (((have0 != want) || (have1 != want)) && (mismatches++ < 5)) {
printf("log2_floor_non_zero(0x%016" PRIX64 "): have %" PRIu32
" and %" PRIu32 ", want %" PRIu32 "\n", x, have0, have1, want);
}
} while (++v != 0);
for (uint64_t i = 0; i < n; i++) {
uint64_t x = (i < 64) ? (1ull << i) : rnd_int();
if (x == 0) {
continue;
}
uint64_t want = (uint64_t)__builtin_ctzll(x);
uint64_t have = wuffs_brotli__encoder__ctz64(enc, x);
if ((have != want) && (mismatches++ < 5)) {
printf("ctz64(0x%016" PRIX64 "): have %" PRIu64 ", want %" PRIu64 "\n",
x, have, want);
}
}
printf("%-8s %12" PRIu64 " iterations, %8" PRIu64 " mismatches\n", "bits",
n + (uint64_t)0x100000000ull, mismatches);
g_total_mismatches += mismatches;
}
int //
main(int argc, char** argv) {
uint64_t n = 100000000;
@@ -345,6 +384,7 @@ main(int argc, char** argv) {
printf("alloc failed\n");
return 1;
}
g_enc = enc;
g_bc = &enc->private_data.f_bc;
check_f32_add(n);
@@ -359,6 +399,7 @@ main(int argc, char** argv) {
check_f64_gt(n);
check_f64_ge(n);
check_conversions(n);
check_bit_helpers(n);
free(enc);
if (g_total_mismatches) {
+16 -8
View File
@@ -883,7 +883,9 @@ by both this package's decoder and libbrotlidec.
`script/manual-test-brotli-float.c` checks the encoder's package-private
float fast paths (`encode_float.wuffs`, which spare the hot loops the
cross-package `std/softfloat` calls) against the C compiler's hardware float
and double arithmetic, for operands biased towards their edge cases.
and double arithmetic, for operands biased towards their edge cases, and
checks the integer bit helpers (`log2_floor_non_zero` for every 32-bit value,
and `ctz64`).
`script/manual-test-brotli-encoder-parity.c` runs a larger grid over every
file in `test/data/brotli`, some larger files in `test/data` (up to about
400 KiB) and some synthetic inputs (up to 1 MiB). Its `-q=LO-HI` and
@@ -1131,13 +1133,19 @@ metablock) for as long as it needs the results.
- `workbuf_len` is a worst case (or, at qualities 10 and 11, a practical
bound). A future option could take the input length (like the C code's
size hint) to give a tighter bound.
- Speed. Compressing the 3 MiB `test/data/harvesters.bmp` in one call (on
x86-64, lgwin 22), the Wuffs encoder takes about 1.2 times libbrotlienc's
time at quality 5, 1.7 times at quality 9, 4 times at quality 10 and 3.5
times at quality 11 (on text, 2 to 3 times at qualities 10 and 11), mostly
because of the soft float arithmetic in the Zopfli cost model, the block
splitter and histogram clustering. Decoding takes about 1.2 times
libbrotlidec's time.
- Speed. On a mix of text, JSON, map data, an image and an ISO image (on
x86-64, lgwin 22, one call each, reusing the workbuf), the Wuffs encoder
takes about 1.2 to 1.8 times libbrotlienc's time at qualities 0 to 8 (about
1.8 times at the qualities 4 to 6 that web servers use for on-the-fly
compression) and about the same time at quality 9. At qualities 10 and 11
it takes about 2 times libbrotlienc's time on text and 3.5 to 4 times on
images (such as the 3 MiB `test/data/harvesters.bmp`), mostly because of
the soft float arithmetic in the Zopfli cost model, the block splitter and
histogram clustering. Decoding takes about 1.2 to 1.6 times libbrotlidec's
time. The workbuf is sized for the window (about 48 to 62 MiB at lgwin 22
and qualities 5 to 9), so callers that compress many small inputs should
reuse it: allocating and first touching a fresh one for each call costs
more than the compression of a small input.
- Serialized (shared) dictionaries, which only an experimental
(`BROTLI_EXPERIMENTAL`) libbrotlienc build supports.
- Changing parameters mid-stream.
+87 -14
View File
@@ -738,6 +738,13 @@ pri func decoder.process_commands_fast!(src: base.io_reader, workbuf: slice base
var dhi : base.u64
var slo : base.u64
var shi : base.u64
var cm_slice : base.u32[..= 16320]
var wmask : base.u32
var pos : base.u32
var lit_n : base.u32
var lit_i : base.u32
var wi : base.u64
var copied16 : base.bool
if this.n_bits >= 8 {
return ok
@@ -829,26 +836,38 @@ pri func decoder.process_commands_fast!(src: base.io_reader, workbuf: slice base
}
p1 = this.get_byte(workbuf: args.workbuf, distance: 1) as base.u32
p2 = this.get_byte(workbuf: args.workbuf, distance: 2) as base.u32
while this.insert_length > 0 {
if (this.block_length[0] == 0) or ((this.pos ~mod+ 1) >= this.rb_size) {
break.outer
mode_offset = this.context_mode << 9
cm_slice = this.context_map_slice
wmask = this.window_size ~mod- 1
pos = this.pos
// lit_n is how many literals the loop below decodes before this
// command's literals, this block's literals or the ring buffer
// (before the next ring_event) run out. The loop keeps pos, the
// counts and the symbol in local variables, and writes them back
// to this.etc fields before every break.
lit_n = this.insert_length.min(no_more_than: this.block_length[0])
if (pos ~mod+ 1) >= this.rb_size {
lit_n = 0
} else {
lit_n = lit_n.min(no_more_than: (this.rb_size ~mod- pos) ~mod- 1)
}
lit_i = 0
while lit_i < lit_n {
if n_bits < 24 {
if args.src.length() < 8 {
break.outer
break
}
bits |= args.src.peek_u64le() ~mod<< n_bits
args.src.skip_u32_fast!(actual: (63 - n_bits) >> 3, worst_case: 8)
n_bits |= 56
}
mode_offset = this.context_mode << 9
ctx = (CONTEXT_LOOKUP[mode_offset | p1] |
CONTEXT_LOOKUP[mode_offset | 0x100 | p2]) as base.u32
tree = this.context_maps[this.context_map_slice + ctx] as base.u32
tree = this.context_maps[cm_slice + ctx] as base.u32
i = (LIT_HUFFS_BASE + (tree * LIT_HUFFS_STRIDE)) ~mod+ ((bits & 0xFF) as base.u32)
if i >= HUFFS_LEN {
ret = "#internal error: inconsistent Huffman table"
break.outer
break
}
e = this.huffs[i] as base.u32
e_bits = e & 15
@@ -856,23 +875,34 @@ pri func decoder.process_commands_fast!(src: base.io_reader, workbuf: slice base
i = ((LIT_HUFFS_BASE + (tree * LIT_HUFFS_STRIDE)) ~mod+ (e >> 4)) ~mod+ ((((bits >> 8) & 0x7FFF) as base.u32) & (((1 as base.u32) << (e_bits - 8)) - 1))
if i >= HUFFS_LEN {
ret = "#internal error: inconsistent Huffman table"
break.outer
break
}
e = this.huffs[i] as base.u32
e_bits = (e & 15) + 8
}
if n_bits < e_bits {
ret = "#internal error: inconsistent bit reader"
break.outer
break
}
bits >>= e_bits
n_bits -= e_bits
this.symbol = e >> 4
this.put_byte!(workbuf: args.workbuf, b: (this.symbol & 0xFF) as base.u8)
p2 = p1
p1 = this.symbol & 0xFF
this.block_length[0] ~mod-= 1
this.insert_length ~mod-= 1
p1 = (e >> 4) & 0xFF
wi = (pos & wmask) as base.u64
if wi < args.workbuf.length() {
args.workbuf[wi] = p1 as base.u8
}
pos ~mod+= 1
lit_i ~mod+= 1
}
if lit_i > 0 {
this.symbol = p1
this.pos = pos
this.block_length[0] ~mod-= lit_i
this.insert_length ~mod-= lit_i
}
if (ret.is_error()) or (this.insert_length > 0) {
break.outer
}
this.cmd_state = 2
if (this.mb_remaining == 0) or (this.mb_remaining >= 0x8000_0000) {
@@ -971,6 +1001,18 @@ pri func decoder.process_commands_fast!(src: base.io_reader, workbuf: slice base
this.mb_remaining ~mod-= n
// Copy in chunks of at most distance bytes, like process_commands.
k64 = 0
if (n <= 16) and (distance >= 16) and
((dst_i ~mod+ 16) <= (this.rb_size as base.u64)) and
((src_i ~mod+ 16) <= (this.window_size as base.u64)) {
// Like the C code, copy 16 bytes, overwriting up to 16 - n bytes
// after the copy's end. Those bytes are more than the maximum
// backward distance (BROTLI_WINDOW_GAP is 16) away from any later
// position, so they are never read before being rewritten.
copied16 = this.copy16!(workbuf: args.workbuf, d: dst_i, s: src_i)
if copied16 {
k64 = n as base.u64
}
}
while k64 < (n as base.u64) {
n64 = (n as base.u64) ~mod- k64
n64 = n64.min(no_more_than: distance as base.u64)
@@ -1098,3 +1140,34 @@ pri func decoder.take_distance_from_ring_buffer!(code: base.u32[..= 15]) {
this.distance_code = v - 3
}
}
// copy16 copies the 16 bytes at workbuf[s ..] to workbuf[d ..], if both are
// in bounds (and returns whether it did). The two ranges must not overlap.
pri func decoder.copy16!(workbuf: slice base.u8, d: base.u64, s: base.u64) base.bool {
var lo : base.u64
var hi : base.u64
var q : slice base.u8
var r : slice base.u8
if (args.s > args.workbuf.length()) or (args.d > args.workbuf.length()) {
return false
}
q = args.workbuf[args.s ..]
if q.length() >= 16 {
lo = q.peek_u64le()
q = q[8 ..]
if q.length() >= 8 {
hi = q.peek_u64le()
q = args.workbuf[args.d ..]
if q.length() >= 16 {
r = q[8 ..]
q.poke_u64le!(a: lo)
if r.length() >= 8 {
r.poke_u64le!(a: hi)
return true
}
}
}
}
return false
}
+24 -38
View File
@@ -125,33 +125,27 @@ pri func encoder.store_meta_block_header!(len: base.u64, is_uncompressed: base.b
}
}
// LOG2_DEBRUIJN32 maps (v * 0x07C4_ACDD) >> 27, for v = (1 << (k + 1)) - 1,
// to k.
pri const LOG2_DEBRUIJN32 : roarray[32] base.u32[..= 31] = [
0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30,
8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31,
]
// Mirrors Log2FloorNonZero in fast_log.h, which (with BROTLI_BSR32) truncates
// its argument to 32 bits. It returns 0 for an argument of 0.
pri func encoder.log2_floor_non_zero(n: base.u64) base.u32[..= 31] {
var x : base.u32
var r : base.u32
// Smear the highest set bit of the low 32 bits of n rightwards, then
// look up its position.
x = (args.n & 0xFFFF_FFFF) as base.u32
if x >= 0x1_0000 {
x >>= 16
r = 16
}
if x >= 0x100 {
x >>= 8
r ~mod+= 8
}
if x >= 0x10 {
x >>= 4
r ~mod+= 4
}
if x >= 4 {
x >>= 2
r ~mod+= 2
}
if x >= 2 {
r ~mod+= 1
}
return r & 31
x |= x >> 1
x |= x >> 2
x |= x >> 4
x |= x >> 8
x |= x >> 16
return LOG2_DEBRUIJN32[(x ~mod* 0x07C4_ACDD) >> 27]
}
// Mirrors FastLog2 in fast_log.h. The result is the bit pattern of an IEEE
@@ -193,12 +187,10 @@ pri func encoder.bits_entropy_of_histogram() base.u64 {
// code's BROTLI_UNALIGNED_LOAD64LE. The C code never reads out of bounds, so
// the zero result for an out of bounds read should never be used.
pri func encoder.load64le(s: roslice base.u8, i: base.u64) base.u64 {
var q : roslice base.u8
if args.i < args.s.length() {
q = args.s[args.i ..]
if q.length() >= 8 {
return q.peek_u64le()
if args.i < 0xFFFF_FFFF_FFFF_0000 {
if (args.i + 8) <= args.s.length() {
assert args.i <= (args.i + 8) via "a <= (a + b): 0 <= b"(b: 8)
return args.s[args.i .. args.i + 8].peek_u64le()
}
}
return 0
@@ -206,12 +198,10 @@ pri func encoder.load64le(s: roslice base.u8, i: base.u64) base.u64 {
// load32le is like load64le but for 4 bytes.
pri func encoder.load32le(s: roslice base.u8, i: base.u64) base.u32 {
var q : roslice base.u8
if args.i < args.s.length() {
q = args.s[args.i ..]
if q.length() >= 4 {
return q.peek_u32le()
if args.i < 0xFFFF_FFFF_FFFF_0000 {
if (args.i + 4) <= args.s.length() {
assert args.i <= (args.i + 4) via "a <= (a + b): 0 <= b"(b: 4)
return args.s[args.i .. args.i + 4].peek_u32le()
}
}
return 0
@@ -235,11 +225,7 @@ pri func encoder.find_match_length_with_limit(s: roslice base.u8, i1: base.u64,
while (n >= 8) and (a.length() >= 8) and (b.length() >= 8) {
x = a.peek_u64le() ^ b.peek_u64le()
if x <> 0 {
while (x & 0xFF) == 0 {
matched ~mod+= 1
x >>= 8
}
return matched
return matched ~mod+ (this.ctz64(x: x) >> 3)
}
matched ~mod+= 8
n -= 8
+158 -94
View File
@@ -400,7 +400,6 @@ pri func encoder.hasher_store!(wb: slice base.u8, ix: base.u64) {
var bsize : base.u64
var off : base.u64
var num : base.u16
var hash : base.u32
var bank : base.u64
var idx : base.u64
var delta : base.u64
@@ -408,6 +407,10 @@ pri func encoder.hasher_store!(wb: slice base.u8, ix: base.u64) {
var fc_bits : base.u32[..= 16]
htype = this.hasher_type
if (htype == 58) or (htype == 68) {
this.hasher_store_h58!(wb: args.wb, ix: args.ix)
return nothing
}
if this.rb_buf_off > args.wb.length() {
return nothing
}
@@ -427,16 +430,6 @@ pri func encoder.hasher_store!(wb: slice base.u8, ix: base.u64) {
this.poke_u16!(s: args.wb, i: this.h_off0 ~mod+ (2 ~mod* (key as base.u64)), v: num ~mod+ 1)
this.poke_u32!(s: args.wb, i: this.h_off1 ~mod+ (4 ~mod* off), v: (args.ix & 0xFFFF_FFFF) as base.u32)
} else if (htype == 58) or (htype == 68) {
hash = this.hash_longest(data: data, i: args.ix & (this.rb_mask as base.u64))
key = hash >> 8
num = this.peek_u16(s: args.wb, i: this.h_off0 ~mod+ (2 ~mod* (key as base.u64)))
off = ((num as base.u64) & (((1 as base.u64) << this.h_block_bits) - 1)) ~mod+
((key as base.u64) << this.h_block_bits)
this.poke_u16!(s: args.wb, i: this.h_off0 ~mod+ (2 ~mod* (key as base.u64)), v: num ~mod- 1)
this.poke_u32!(s: args.wb, i: this.h_off2 ~mod+ (4 ~mod* off), v: (args.ix & 0xFFFF_FFFF) as base.u32)
this.set_byte!(s: args.wb, i: this.h_off1 ~mod+ off, v: (hash & 0xFF) as base.u8)
} else if (htype == 40) or (htype == 41) or (htype == 42) {
key = this.hash_forgetful(data: data, i: args.ix & (this.rb_mask as base.u64))
fc_bits = this.fc_bank_bits()
@@ -457,6 +450,36 @@ pri func encoder.hasher_store!(wb: slice base.u8, ix: base.u64) {
}
}
// hasher_store_h58 is FN(Store) for H58 and H68.
pri func encoder.hasher_store_h58!(wb: slice base.u8, ix: base.u64) {
var hash : base.u32
var key : base.u64
var koff : base.u64
var num : base.u16
var off : base.u64
hash = this.hash_h58(wb: args.wb, ix: args.ix)
key = (hash >> 8) as base.u64
koff = this.h_off0 ~mod+ (2 ~mod* key)
num = this.peek_u16(s: args.wb, i: koff)
off = ((num as base.u64) & (((1 as base.u64) << this.h_block_bits) - 1)) ~mod+ (key << this.h_block_bits)
this.poke_u16!(s: args.wb, i: koff, v: num ~mod- 1)
this.poke_u32!(s: args.wb, i: this.h_off2 ~mod+ (4 ~mod* off), v: (args.ix & 0xFFFF_FFFF) as base.u32)
this.set_byte!(s: args.wb, i: this.h_off1 ~mod+ off, v: (hash & 0xFF) as base.u8)
}
// hash_h58 is hash_longest for H58 and H68, of the ring buffer bytes at
// position ix (masked).
pri func encoder.hash_h58(wb: roslice base.u8, ix: base.u64) base.u32 {
var i : base.u64
i = this.rb_buf_off ~mod+ (args.ix & (this.rb_mask as base.u64))
if this.hasher_type == 58 {
return (this.load32le(s: args.wb, i: i) ~mod* HASH_MUL_32) >> ((24 - this.h_bucket_bits) & 31)
}
return ((this.load64le(s: args.wb, i: i) ~mod* ((HASH_MUL_64 << 24) & 0xFFFF_FFFF_FFFF_FFFF)) >> 41) as base.u32
}
// Mirrors FN(StoreRange) of every hasher.
pri func encoder.hasher_store_range!(wb: slice base.u8, ix_start: base.u64, ix_end: base.u64) {
var i : base.u64
@@ -803,6 +826,8 @@ pri func encoder.find_last_distances!(data: roslice base.u8, cur_ix: base.u64, m
var prev_ix : base.u64
var len : base.u64
var score : base.u64
var cur_byte : base.u8
var n : base.u32
mask = this.rb_mask as base.u64
cur_ix_masked = args.cur_ix & mask
@@ -811,15 +836,14 @@ pri func encoder.find_last_distances!(data: roslice base.u8, cur_ix: base.u64, m
this.sr_len = 0
this.sr_len_code_delta = 0
// cur_byte is data[cur_ix_masked + best_len], updated with best_len.
cur_byte = this.byte_at(s: args.data, i: cur_ix_masked ~mod+ best_len)
n = this.h_num_last_distances
i = 0
while i < this.h_num_last_distances {
while i < n {
backward = this.dist_cache_u64(i: i)
prev_ix = args.cur_ix ~mod- backward
if prev_ix >= args.cur_ix {
i ~mod+= 1
continue
}
if backward > args.max_backward {
if (prev_ix >= args.cur_ix) or (backward > args.max_backward) {
i ~mod+= 1
continue
}
@@ -828,7 +852,7 @@ pri func encoder.find_last_distances!(data: roslice base.u8, cur_ix: base.u64, m
break
}
if ((prev_ix ~mod+ best_len) > mask) or
(this.byte_at(s: args.data, i: cur_ix_masked ~mod+ best_len) <> this.byte_at(s: args.data, i: prev_ix ~mod+ best_len)) {
(cur_byte <> this.byte_at(s: args.data, i: prev_ix ~mod+ best_len)) {
i ~mod+= 1
continue
}
@@ -845,6 +869,7 @@ pri func encoder.find_last_distances!(data: roslice base.u8, cur_ix: base.u64, m
this.sr_len = best_len
this.sr_distance = backward
this.sr_score = best_score
cur_byte = this.byte_at(s: args.data, i: cur_ix_masked ~mod+ best_len)
}
}
}
@@ -853,55 +878,6 @@ pri func encoder.find_last_distances!(data: roslice base.u8, cur_ix: base.u64, m
return best_len
}
// try_bucket_entry is the body of the bucket loop shared by the H5, H6, H58
// and H68 FN(FindLongestMatch) functions, for one prev_ix candidate. It
// returns 0 to continue, 1 to break, or 2 + the new best_len.
pri func encoder.try_bucket_entry!(data: roslice base.u8, cur_ix: base.u64, prev_ix_in: base.u64, best_len: base.u64, max_length: base.u64, max_backward: base.u64) base.u64 {
var mask : base.u64
var cur_ix_masked : base.u64
var prev_ix : base.u64
var backward : base.u64
var len : base.u64
var score : base.u64
mask = this.rb_mask as base.u64
cur_ix_masked = args.cur_ix & mask
backward = args.cur_ix ~mod- args.prev_ix_in
if backward > args.max_backward {
return 1
}
prev_ix = args.prev_ix_in & mask
if (cur_ix_masked ~mod+ args.best_len) > mask {
return 1
}
if ((prev_ix ~mod+ args.best_len) > mask) or
(this.load32le(s: args.data, i: (cur_ix_masked ~mod+ args.best_len) ~mod- 3) <>
this.load32le(s: args.data, i: (prev_ix ~mod+ args.best_len) ~mod- 3)) {
return 0
}
if (this.hasher_type == 6) or (this.hasher_type == 68) {
if this.load32le(s: args.data, i: cur_ix_masked) <> this.load32le(s: args.data, i: prev_ix) {
return 0
}
len = this.find_match_length_with_limit(s: args.data, i1: prev_ix ~mod+ 4, i2: cur_ix_masked ~mod+ 4,
limit: args.max_length ~mod- 4) ~mod+ 4
} else {
len = this.find_match_length_with_limit(s: args.data, i1: prev_ix, i2: cur_ix_masked, limit: args.max_length)
if len < 4 {
return 0
}
}
score = this.backward_reference_score(copy_length: len, backward_reference_offset: backward)
if this.sr_score < score {
// The C code's best_score is always equal to out->score here.
this.sr_len = len
this.sr_distance = backward
this.sr_score = score
return 2 ~mod+ len
}
return 0
}
// Mirrors FN(FindLongestMatch) in hash_longest_match_inc.h (H5) and
// hash_longest_match64_inc.h (H6).
pri func encoder.find_longest_match_h5!(wb: slice base.u8, cur_ix: base.u64, max_length: base.u64, max_backward: base.u64, dictionary_distance: base.u64, max_distance: base.u64) {
@@ -916,7 +892,13 @@ pri func encoder.find_longest_match_h5!(wb: slice base.u8, cur_ix: base.u64, max
var bucket : base.u64
var down : base.u64
var i : base.u64
var r : base.u64
var mask : base.u64
var is64 : base.bool
var cur4 : base.u32
var prev_ix : base.u64
var backward : base.u64
var len : base.u64
var score : base.u64
if this.rb_buf_off > args.wb.length() {
return nothing
@@ -938,20 +920,46 @@ pri func encoder.find_longest_match_h5!(wb: slice base.u8, cur_ix: base.u64, max
if (num as base.u64) > block_size {
down = (num as base.u64) ~mod- block_size
}
// Try the bucket's entries, newest first. Like the C code, this
// compares 4 bytes ending at best_len + 1 before the full match.
mask = this.rb_mask as base.u64
is64 = this.hasher_type == 6
cur4 = this.load32le(s: data, i: (cur_ix_masked ~mod+ best_len) ~mod- 3)
i = num as base.u64
while i > down {
i ~mod-= 1
r = this.try_bucket_entry!(
data: data,
cur_ix: args.cur_ix,
prev_ix_in: this.load32le(s: args.wb, i: bucket ~mod+ (4 ~mod* (i & block_mask))) as base.u64,
best_len: best_len,
max_length: args.max_length,
max_backward: args.max_backward)
if r == 1 {
prev_ix = this.load32le(s: args.wb, i: bucket ~mod+ (4 ~mod* (i & block_mask))) as base.u64
backward = args.cur_ix ~mod- prev_ix
if backward > args.max_backward {
break
} else if r >= 2 {
best_len = r ~mod- 2
}
prev_ix &= mask
if (cur_ix_masked ~mod+ best_len) > mask {
break
}
if ((prev_ix ~mod+ best_len) > mask) or
(cur4 <> this.load32le(s: data, i: (prev_ix ~mod+ best_len) ~mod- 3)) {
continue
}
if is64 {
if this.load32le(s: data, i: cur_ix_masked) <> this.load32le(s: data, i: prev_ix) {
continue
}
len = this.find_match_length_with_limit(s: data, i1: prev_ix ~mod+ 4, i2: cur_ix_masked ~mod+ 4,
limit: args.max_length ~mod- 4) ~mod+ 4
} else {
len = this.find_match_length_with_limit(s: data, i1: prev_ix, i2: cur_ix_masked, limit: args.max_length)
if len < 4 {
continue
}
}
score = this.backward_reference_score(copy_length: len, backward_reference_offset: backward)
if this.sr_score < score {
this.sr_len = len
this.sr_distance = backward
this.sr_score = score
best_len = len
cur4 = this.load32le(s: data, i: (cur_ix_masked ~mod+ best_len) ~mod- 3)
}
}
this.poke_u32!(s: args.wb, i: bucket ~mod+ (4 ~mod* ((num as base.u64) & block_mask)), v: (args.cur_ix & 0xFFFF_FFFF) as base.u32)
@@ -984,7 +992,17 @@ pri func encoder.find_longest_match_h58!(wb: slice base.u8, cur_ix: base.u64, ma
var limit : base.u64
var k : base.u64
var rb_index : base.u64
var r : base.u64
var splat : base.u64
var matches : base.u64
var chunk : base.u64
var j : base.u64
var mask : base.u64
var is64 : base.bool
var cur4 : base.u32
var prev_ix : base.u64
var backward : base.u64
var len : base.u64
var score : base.u64
if this.rb_buf_off > args.wb.length() {
return nothing
@@ -992,7 +1010,7 @@ pri func encoder.find_longest_match_h58!(wb: slice base.u8, cur_ix: base.u64, ma
data = args.wb[this.rb_buf_off ..]
cur_ix_masked = args.cur_ix & (this.rb_mask as base.u64)
min_score = this.sr_score
hash = this.hash_longest(data: data, i: cur_ix_masked)
hash = this.hash_h58(wb: args.wb, ix: args.cur_ix)
key = (hash >> 8) as base.u64
block_size = (1 as base.u64) << this.h_block_bits
block_mask = block_size ~mod- 1
@@ -1014,24 +1032,70 @@ pri func encoder.find_longest_match_h58!(wb: slice base.u8, cur_ix: base.u64, ma
if block_size > (n as base.u64) {
limit = n as base.u64
}
k = 0
while k < limit {
// Like GetMatchingTagMask's non-SSE2 code: bit j of matches is whether
// the tag of entry j matches, 8 entries at a time.
splat = (tag as base.u64) ~mod* 0x0101_0101_0101_0101
matches = 0
j = 0
while j < block_size {
chunk = this.load64le(s: args.wb, i: tag_bucket ~mod+ j) ^ splat
// The 0x80 bit of each byte is set when that byte is non-zero.
chunk = (((chunk | 0x8080_8080_8080_8080) ~mod- 0x0101_0101_0101_0101) | chunk) & 0x8080_8080_8080_8080
chunk = (chunk ^ 0x8080_8080_8080_8080) >> 7
matches |= ((chunk ~mod* 0x0102_0408_1020_4080) >> 56) ~mod<< (j & 63)
j ~mod+= 8
}
// Rotate so that bit k is entry (head + k) & block_mask.
if head > 0 {
matches = ((matches >> (head & 63)) | (matches ~mod<< ((block_size ~mod- head) & 63))) &
((0xFFFF_FFFF_FFFF_FFFF as base.u64) >> ((64 ~mod- block_size) & 63))
}
if limit < 64 {
matches &= ((1 as base.u64) << (limit & 63)) ~mod- 1
}
// Try the entries whose tags match, in the C code's order (from head).
// Like the C code, this compares 4 bytes ending at best_len + 1 before
// the full match.
mask = this.rb_mask as base.u64
is64 = this.hasher_type == 68
cur4 = this.load32le(s: data, i: (cur_ix_masked ~mod+ best_len) ~mod- 3)
while matches <> 0 {
k = this.ctz64(x: matches)
matches &= matches ~mod- 1
rb_index = (head ~mod+ k) & block_mask
if this.byte_at(s: args.wb, i: tag_bucket ~mod+ rb_index) == tag {
r = this.try_bucket_entry!(
data: data,
cur_ix: args.cur_ix,
prev_ix_in: this.load32le(s: args.wb, i: bucket ~mod+ (4 ~mod* rb_index)) as base.u64,
best_len: best_len,
max_length: args.max_length,
max_backward: args.max_backward)
if r == 1 {
prev_ix = this.load32le(s: args.wb, i: bucket ~mod+ (4 ~mod* rb_index)) as base.u64
backward = args.cur_ix ~mod- prev_ix
if backward > args.max_backward {
break
} else if r >= 2 {
best_len = r ~mod- 2
}
prev_ix &= mask
if (cur_ix_masked ~mod+ best_len) > mask {
break
}
if ((prev_ix ~mod+ best_len) > mask) or
(cur4 <> this.load32le(s: data, i: (prev_ix ~mod+ best_len) ~mod- 3)) {
continue
}
if is64 {
if this.load32le(s: data, i: cur_ix_masked) <> this.load32le(s: data, i: prev_ix) {
continue
}
len = this.find_match_length_with_limit(s: data, i1: prev_ix ~mod+ 4, i2: cur_ix_masked ~mod+ 4,
limit: args.max_length ~mod- 4) ~mod+ 4
} else {
len = this.find_match_length_with_limit(s: data, i1: prev_ix, i2: cur_ix_masked, limit: args.max_length)
if len < 4 {
continue
}
}
k ~mod+= 1
score = this.backward_reference_score(copy_length: len, backward_reference_offset: backward)
if this.sr_score < score {
this.sr_len = len
this.sr_distance = backward
this.sr_score = score
best_len = len
cur4 = this.load32le(s: data, i: (cur_ix_masked ~mod+ best_len) ~mod- 3)
}
}
this.poke_u32!(s: args.wb, i: bucket ~mod+ (4 ~mod* ((num as base.u64) & block_mask)), v: (args.cur_ix & 0xFFFF_FFFF) as base.u32)
this.set_byte!(s: args.wb, i: tag_bucket ~mod+ ((num as base.u64) & block_mask), v: tag)
+8 -8
View File
@@ -283,16 +283,16 @@ pri func metablock_builder.h_population_cost!(wb: roslice base.u8, h: base.u64,
// Mirrors Log2FloorNonZero in fast_log.h (which truncates n to 32 bits).
pri func metablock_builder.log2_floor_non_zero(n: base.u64) base.u32[..= 31] {
var x : base.u32
var r : base.u32[..= 31]
// Smear the highest set bit of the low 32 bits of n rightwards, then
// look up its position.
x = (args.n & 0xFFFF_FFFF) as base.u32
while x > 1 {
x >>= 1
if r < 31 {
r += 1
}
}
return r
x |= x >> 1
x |= x >> 2
x |= x >> 4
x |= x >> 8
x |= x >> 16
return LOG2_DEBRUIJN32[(x ~mod* 0x07C4_ACDD) >> 27]
}
// Mirrors PrefixEncodeCopyDistance in prefix.h. It returns the code (the
+27 -18
View File
@@ -18,6 +18,21 @@
// out of bounds branches here (which do nothing, or return 0) should never be
// taken.
// CTZ64_DEBRUIJN maps ((x & -x) * 0x03F7_9D71_B4CB_0A89) >> 58 to the
// number of trailing zero bits of the non-zero x.
pri const CTZ64_DEBRUIJN : roarray[64] base.u8[..= 63] = [
0, 1, 48, 2, 57, 49, 28, 3, 61, 58, 50, 42, 38, 29, 17, 4,
62, 55, 59, 36, 53, 51, 43, 22, 45, 39, 33, 30, 24, 18, 12, 5,
63, 47, 56, 27, 60, 41, 37, 16, 54, 35, 52, 21, 44, 32, 23, 11,
46, 26, 40, 15, 34, 20, 31, 10, 25, 14, 19, 9, 13, 8, 7, 6,
]
// ctz64 returns the number of trailing zero bits of x, which must be
// non-zero (for x == 0 it returns 0).
pri func encoder.ctz64(x: base.u64) base.u64[..= 63] {
return CTZ64_DEBRUIJN[((args.x & (0 ~mod- args.x)) ~mod* 0x03F7_9D71_B4CB_0A89) >> 58] as base.u64
}
// byte_at returns s[i], or 0 if i is out of bounds.
pri func encoder.byte_at(s: roslice base.u8, i: base.u64) base.u8 {
if args.i < args.s.length() {
@@ -28,12 +43,10 @@ pri func encoder.byte_at(s: roslice base.u8, i: base.u64) base.u8 {
// peek_u16 returns the 2 bytes (little-endian) of s starting at i.
pri func encoder.peek_u16(s: roslice base.u8, i: base.u64) base.u16 {
var q : roslice base.u8
if args.i < args.s.length() {
q = args.s[args.i ..]
if q.length() >= 2 {
return q.peek_u16le()
if args.i < 0xFFFF_FFFF_FFFF_0000 {
if (args.i + 2) <= args.s.length() {
assert args.i <= (args.i + 2) via "a <= (a + b): 0 <= b"(b: 2)
return args.s[args.i .. args.i + 2].peek_u16le()
}
}
return 0
@@ -41,24 +54,20 @@ pri func encoder.peek_u16(s: roslice base.u8, i: base.u64) base.u16 {
// poke_u16 sets the 2 bytes (little-endian) of s starting at i.
pri func encoder.poke_u16!(s: slice base.u8, i: base.u64, v: base.u16) {
var q : slice base.u8
if args.i < args.s.length() {
q = args.s[args.i ..]
if q.length() >= 2 {
q.poke_u16le!(a: args.v)
if args.i < 0xFFFF_FFFF_FFFF_0000 {
if (args.i + 2) <= args.s.length() {
assert args.i <= (args.i + 2) via "a <= (a + b): 0 <= b"(b: 2)
args.s[args.i .. args.i + 2].poke_u16le!(a: args.v)
}
}
}
// poke_u32 sets the 4 bytes (little-endian) of s starting at i.
pri func encoder.poke_u32!(s: slice base.u8, i: base.u64, v: base.u32) {
var q : slice base.u8
if args.i < args.s.length() {
q = args.s[args.i ..]
if q.length() >= 4 {
q.poke_u32le!(a: args.v)
if args.i < 0xFFFF_FFFF_FFFF_0000 {
if (args.i + 4) <= args.s.length() {
assert args.i <= (args.i + 4) via "a <= (a + b): 0 <= b"(b: 4)
args.s[args.i .. args.i + 4].poke_u32le!(a: args.v)
}
}
}