From 044437a8ec165af046dd6a8a8f338ea19b6f19f9 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 23:21:31 +0000 Subject: [PATCH 1/2] std/brotli: speed up the encoder at qualities 10 and 11, byte-exactly - encode_float.wuffs: package-private fast paths for the f32/f64 add, sub, mul, compare and int conversion calls in the encoder's hot loops. Calling std/softfloat is a cross-package public call (a receiver check, never inlined). Each fast path returns exactly the softfloat result and falls back to it for the uncommon cases. script/manual-test-brotli-float.c cross-checks them against hardware float and double arithmetic. - Zopfli update_nodes: compute a length's cost only when its copy code changes, hoist the distance cache values and the continuation byte, and use a table for small int to f32 conversions. - FindBlocks: memoize (block_switch_bitcost + insert_cost) for capped costs and skip the subtraction for costs at or above a per-byte threshold (S(c) = c - min_cost is monotone, so such costs would be capped anyway). - BrotliPopulationCost: per-call memo of each count's term and depth, and skip the zero terms of the entropy sums. - std/softfloat: a 64-bit f32_add_mags. Output is unchanged (mimic tests on gcc and clang, parity sweeps). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UZ978BikvkegL2JUkqoB1e --- release/c/wuffs-unsupported-snapshot.c | 1333 ++++++++++++++++++--- script/manual-test-brotli-float.c | 370 ++++++ std/brotli/README.md | 17 +- std/brotli/encode_bit_cost.wuffs | 105 +- std/brotli/encode_bit_writer.wuffs | 12 +- std/brotli/encode_block_splitter.wuffs | 120 +- std/brotli/encode_cluster.wuffs | 28 +- std/brotli/encode_data.wuffs | 64 +- std/brotli/encode_float.wuffs | 742 ++++++++++++ std/brotli/encode_fragment_fast.wuffs | 20 +- std/brotli/encode_fragment_two_pass.wuffs | 14 +- std/brotli/encode_literal_cost.wuffs | 24 +- std/brotli/encode_metablock.wuffs | 12 +- std/brotli/encode_metablock_greedy.wuffs | 14 +- std/brotli/encode_zopfli.wuffs | 104 +- std/softfloat/softfloat_f32.wuffs | 71 +- 16 files changed, 2650 insertions(+), 400 deletions(-) create mode 100644 script/manual-test-brotli-float.c create mode 100644 std/brotli/encode_float.wuffs diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 4a26316b..87d7f819 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -9912,10 +9912,14 @@ struct wuffs_brotli__bit_cost__struct { wuffs_base__vtable null_vtable; uint64_t f_log2_valid[1024]; + uint32_t f_pc_call; + uint32_t f_pc_valid[256]; } private_impl; struct { wuffs_softfloat__arith f_sf; + uint64_t f_pc_term[256]; + uint8_t f_pc_depth[256]; uint64_t f_log2_cache[65536]; } private_data; @@ -9963,6 +9967,7 @@ struct wuffs_brotli__metablock_builder__struct { wuffs_brotli__bit_cost f_bc; uint64_t f_cost[256]; uint64_t f_insert_cost[35200]; + uint64_t f_capped_plus[35200]; uint16_t f_new_id[256]; uint8_t f_good_for_rle[704]; } private_data; @@ -42966,47 +42971,32 @@ wuffs_softfloat__arith__f32_add_mags( const wuffs_softfloat__arith* self, uint32_t a_a, uint32_t a_b) { - uint32_t v_ea = 0; - uint32_t v_eb = 0; - uint32_t v_ma = 0; - uint32_t v_mb = 0; + uint32_t v_x = 0; + uint32_t v_y = 0; + uint32_t v_ex = 0; uint32_t v_d = 0; + uint64_t v_s = 0; uint32_t v_z = 0; - uint32_t v_rb = 0; - v_ea = ((a_a >> 23u) & 255u); - v_eb = ((a_b >> 23u) & 255u); - v_ma = (((a_a & 8388607u) | 8388608u) << 6u); - v_mb = (((a_b & 8388607u) | 8388608u) << 6u); - if (v_ea < v_eb) { - v_d = (((uint32_t)(v_eb - v_ea)) & 255u); - v_ea = v_eb; - v_z = v_ma; - v_ma = v_mb; - v_mb = v_z; + v_x = (a_a & 2147483647u); + v_y = (a_b & 2147483647u); + if (v_y > v_x) { + v_x = (a_b & 2147483647u); + v_y = (a_a & 2147483647u); + } + v_ex = ((v_x >> 23u) & 255u); + v_d = ((uint32_t)(v_ex - ((v_y >> 23u) & 255u))); + if (v_d >= 25u) { + return ((a_a & 2147483648u) | v_x); + } + v_s = ((uint64_t)((((uint64_t)(((v_x & 8388607u) | 8388608u))) << 39u) + ((((uint64_t)(((v_y & 8388607u) | 8388608u))) << 39u) >> (v_d & 31u)))); + if (v_s >= 9223372036854775808u) { + v_s = (((uint64_t)(v_s + ((uint64_t)(549755813887u + ((v_s >> 40u) & 1u))))) >> 40u); + v_z = ((uint32_t)(((uint32_t)(v_ex << 23u)) + ((uint32_t)((v_s & 33554431u))))); } else { - v_d = (((uint32_t)(v_ea - v_eb)) & 255u); + v_s = (((uint64_t)(v_s + ((uint64_t)(274877906943u + ((v_s >> 39u) & 1u))))) >> 39u); + v_z = ((uint32_t)(((uint32_t)(((uint32_t)(v_ex - 1u)) << 23u)) + ((uint32_t)((v_s & 33554431u))))); } - if (v_d >= 31u) { - v_mb = 1u; - } else if (v_d > 0u) { - if (((uint32_t)(v_mb << (32u - v_d))) != 0u) { - v_mb = ((v_mb >> v_d) | 1u); - } else { - v_mb = (v_mb >> v_d); - } - } - v_z = ((v_ma & 1073741823u) + (v_mb & 1073741823u)); - if (v_z >= 1073741824u) { - v_ea += 1u; - v_z = ((v_z >> 1u) | (v_z & 1u)); - } - v_rb = (v_z & 63u); - v_z = (((uint32_t)(v_z + 32u)) >> 6u); - if (v_rb == 32u) { - v_z &= 4294967294u; - } - v_z = ((uint32_t)(((uint32_t)(((uint32_t)(v_ea - 1u)) << 23u)) + v_z)); if (v_z >= 2139095040u) { v_z = 2139095040u; } @@ -77257,6 +77247,14 @@ WUFFS_BROTLI__DICTIONARY[122784] WUFFS_BASE__POTENTIALLY_UNUSED = { #define WUFFS_BROTLI__TREE_NONE 65535u +static const uint32_t +WUFFS_BROTLI__F32_OF_SMALL_INT[32] WUFFS_BASE__POTENTIALLY_UNUSED = { + 0u, 1065353216u, 1073741824u, 1077936128u, 1082130432u, 1084227584u, 1086324736u, 1088421888u, + 1090519040u, 1091567616u, 1092616192u, 1093664768u, 1094713344u, 1095761920u, 1096810496u, 1097859072u, + 1098907648u, 1099431936u, 1099956224u, 1100480512u, 1101004800u, 1101529088u, 1102053376u, 1102577664u, + 1103101952u, 1103626240u, 1104150528u, 1104674816u, 1105199104u, 1105723392u, 1106247680u, 1106771968u, +}; + #define WUFFS_BROTLI__FRAGMENT_MAX_DISTANCE 262128u #define WUFFS_BROTLI__HASH_MUL32 506832829u @@ -79011,6 +79009,195 @@ wuffs_brotli__encoder__copy_tmp_bits_to_cmd_bits( uint32_t a_src, uint32_t a_n); +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__bit_cost__clz64( + const wuffs_brotli__bit_cost* self, + uint64_t a_a); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__bit_cost__f32_add( + const wuffs_brotli__bit_cost* self, + uint32_t a_a, + uint32_t a_b); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__bit_cost__f32_sub( + const wuffs_brotli__bit_cost* self, + uint32_t a_a, + uint32_t a_b); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__bit_cost__f32_add_any( + const wuffs_brotli__bit_cost* self, + uint32_t a_a, + uint32_t a_b); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__bit_cost__f32_sub_any( + const wuffs_brotli__bit_cost* self, + uint32_t a_a, + uint32_t a_b); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__bit_cost__f32_key( + const wuffs_brotli__bit_cost* self, + uint32_t a_a); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__bit_cost__f32_lt( + const wuffs_brotli__bit_cost* self, + uint32_t a_a, + uint32_t a_b); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__bit_cost__f32_lt_any( + const wuffs_brotli__bit_cost* self, + uint32_t a_a, + uint32_t a_b); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__bit_cost__f32_le( + const wuffs_brotli__bit_cost* self, + uint32_t a_a, + uint32_t a_b); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__bit_cost__f32_le_any( + const wuffs_brotli__bit_cost* self, + uint32_t a_a, + uint32_t a_b); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__bit_cost__f32_gt( + const wuffs_brotli__bit_cost* self, + uint32_t a_a, + uint32_t a_b); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__bit_cost__f32_gt_any( + const wuffs_brotli__bit_cost* self, + uint32_t a_a, + uint32_t a_b); + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__bit_cost__f64_add( + const wuffs_brotli__bit_cost* self, + uint64_t a_a, + uint64_t a_b); + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__bit_cost__f64_sub( + const wuffs_brotli__bit_cost* self, + uint64_t a_a, + uint64_t a_b); + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__bit_cost__f64_add_any( + const wuffs_brotli__bit_cost* self, + uint64_t a_a, + uint64_t a_b); + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__bit_cost__f64_sub_any( + const wuffs_brotli__bit_cost* self, + uint64_t a_a, + uint64_t a_b); + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__bit_cost__f64_key( + const wuffs_brotli__bit_cost* self, + uint64_t a_a); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__bit_cost__f64_lt( + const wuffs_brotli__bit_cost* self, + uint64_t a_a, + uint64_t a_b); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__bit_cost__f64_lt_any( + const wuffs_brotli__bit_cost* self, + uint64_t a_a, + uint64_t a_b); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__bit_cost__f64_gt( + const wuffs_brotli__bit_cost* self, + uint64_t a_a, + uint64_t a_b); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__bit_cost__f64_gt_any( + const wuffs_brotli__bit_cost* self, + uint64_t a_a, + uint64_t a_b); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__bit_cost__f64_ge( + const wuffs_brotli__bit_cost* self, + uint64_t a_a, + uint64_t a_b); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__bit_cost__f64_ge_any( + const wuffs_brotli__bit_cost* self, + uint64_t a_a, + uint64_t a_b); + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__bit_cost__f64_mul( + const wuffs_brotli__bit_cost* self, + uint64_t a_a, + uint64_t a_b); + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__bit_cost__f64_from_u64( + const wuffs_brotli__bit_cost* self, + uint64_t a_a); + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__bit_cost__f64_from_u32( + const wuffs_brotli__bit_cost* self, + uint32_t a_a); + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__bit_cost__f64_to_u64_trunc( + const wuffs_brotli__bit_cost* self, + uint64_t a_a); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__bit_cost__f32_from_u32( + const wuffs_brotli__bit_cost* self, + uint32_t a_a); + WUFFS_BASE__GENERATED_C_CODE static uint32_t wuffs_brotli__encoder__hash_fast( @@ -86390,15 +86577,18 @@ wuffs_brotli__bit_cost__bits_entropy( v_q = wuffs_base__slice_u8__subslice_i(v_q, 4u); } v_sum += v_p; + if (v_p <= 1u) { + continue; + } v_lp = wuffs_brotli__bit_cost__fast_log2_memo(self, v_p); - v_retval = wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, v_retval, wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_p), v_lp)); + v_retval = wuffs_brotli__bit_cost__f64_sub(self, v_retval, wuffs_brotli__bit_cost__f64_mul(self, wuffs_brotli__bit_cost__f64_from_u64(self, v_p), v_lp)); } if (v_sum != 0u) { v_lp = wuffs_brotli__bit_cost__fast_log2_memo(self, v_sum); - v_retval = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_retval, wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_sum), v_lp)); + v_retval = wuffs_brotli__bit_cost__f64_add(self, v_retval, wuffs_brotli__bit_cost__f64_mul(self, wuffs_brotli__bit_cost__f64_from_u64(self, v_sum), v_lp)); } - if (wuffs_softfloat__arith__f64_lt(&self->private_data.f_sf, v_retval, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_sum)) != 0u) { - v_retval = wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_sum); + if (wuffs_brotli__bit_cost__f64_lt(self, v_retval, wuffs_brotli__bit_cost__f64_from_u64(self, v_sum)) != 0u) { + v_retval = wuffs_brotli__bit_cost__f64_from_u64(self, v_sum); } return v_retval; } @@ -86435,6 +86625,7 @@ wuffs_brotli__bit_cost__population_cost( uint64_t v_sum = 0; uint64_t v_retval = 0; uint64_t v_p = 0; + uint64_t v_term = 0; v_total = wuffs_brotli__bit_cost__ld64(self, a_histogram, (((uint64_t)(a_data_size)) * 4u)); if (v_total == 0u) { @@ -86457,14 +86648,14 @@ wuffs_brotli__bit_cost__population_cost( return 4622945017495814144u; } if (v_count == 2u) { - return wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, 4626322717216342016u, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_total)); + return wuffs_brotli__bit_cost__f64_add(self, 4626322717216342016u, wuffs_brotli__bit_cost__f64_from_u64(self, v_total)); } if (v_count == 3u) { v_h0 = wuffs_brotli__bit_cost__ld32(self, a_histogram, (((uint64_t)(v_s[0u])) * 4u)); v_h1 = wuffs_brotli__bit_cost__ld32(self, a_histogram, (((uint64_t)(v_s[1u])) * 4u)); v_h2 = wuffs_brotli__bit_cost__ld32(self, a_histogram, (((uint64_t)(v_s[2u])) * 4u)); v_histomax = wuffs_base__u32__max(v_h0, wuffs_base__u32__max(v_h1, v_h2)); - return wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, 4628574517030027264u, wuffs_softfloat__arith__f64_from_u32(&self->private_data.f_sf, ((uint32_t)(2u * ((uint32_t)(((uint32_t)(v_h0 + v_h1)) + v_h2)))))), wuffs_softfloat__arith__f64_from_u32(&self->private_data.f_sf, v_histomax)); + return wuffs_brotli__bit_cost__f64_sub(self, wuffs_brotli__bit_cost__f64_add(self, 4628574517030027264u, wuffs_brotli__bit_cost__f64_from_u32(self, ((uint32_t)(2u * ((uint32_t)(((uint32_t)(v_h0 + v_h1)) + v_h2)))))), wuffs_brotli__bit_cost__f64_from_u32(self, v_histomax)); } if (v_count == 4u) { v_i = 0u; @@ -86487,18 +86678,37 @@ wuffs_brotli__bit_cost__population_cost( } v_h23 = ((uint32_t)(v_histo[2u] + v_histo[3u])); v_histomax = wuffs_base__u32__max(v_h23, v_histo[0u]); - return wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, 4630404104378646528u, wuffs_softfloat__arith__f64_from_u32(&self->private_data.f_sf, ((uint32_t)(3u * v_h23)))), wuffs_softfloat__arith__f64_from_u32(&self->private_data.f_sf, ((uint32_t)(2u * ((uint32_t)(v_histo[0u] + v_histo[1u])))))), wuffs_softfloat__arith__f64_from_u32(&self->private_data.f_sf, v_histomax)); + return wuffs_brotli__bit_cost__f64_sub(self, wuffs_brotli__bit_cost__f64_add(self, wuffs_brotli__bit_cost__f64_add(self, 4630404104378646528u, wuffs_brotli__bit_cost__f64_from_u32(self, ((uint32_t)(3u * v_h23)))), wuffs_brotli__bit_cost__f64_from_u32(self, ((uint32_t)(2u * ((uint32_t)(v_histo[0u] + v_histo[1u])))))), wuffs_brotli__bit_cost__f64_from_u32(self, v_histomax)); } v_max_depth = 1u; v_log2total = wuffs_brotli__bit_cost__fast_log2_memo(self, v_total); + self->private_impl.f_pc_call += 1u; + if (self->private_impl.f_pc_call == 0u) { + wuffs_private_impl__bulk_memset(&self->private_impl.f_pc_valid[0], 256u * (size_t)4u, 0u); + self->private_impl.f_pc_call = 1u; + } v_i = 0u; while (v_i < a_data_size) { v_c = wuffs_brotli__bit_cost__ld32(self, a_histogram, (((uint64_t)(v_i)) * 4u)); if (v_c > 0u) { - v_lp = wuffs_brotli__bit_cost__fast_log2_memo(self, ((uint64_t)(v_c))); - v_log2p = wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, v_log2total, v_lp); - v_depth = wuffs_softfloat__arith__f64_to_u64_trunc(&self->private_data.f_sf, wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_log2p, 4602678819172646912u)); - v_bits = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_bits, wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u32(&self->private_data.f_sf, v_c), v_log2p)); + if ((v_c < 256u) && (self->private_impl.f_pc_valid[(v_c & 255u)] == self->private_impl.f_pc_call)) { + v_term = self->private_data.f_pc_term[(v_c & 255u)]; + v_depth = ((uint64_t)(self->private_data.f_pc_depth[(v_c & 255u)])); + } else { + v_lp = wuffs_brotli__bit_cost__fast_log2_memo(self, ((uint64_t)(v_c))); + v_log2p = wuffs_brotli__bit_cost__f64_sub(self, v_log2total, v_lp); + v_depth = wuffs_brotli__bit_cost__f64_to_u64_trunc(self, wuffs_brotli__bit_cost__f64_add(self, v_log2p, 4602678819172646912u)); + v_term = wuffs_brotli__bit_cost__f64_mul(self, wuffs_brotli__bit_cost__f64_from_u32(self, v_c), v_log2p); + if (v_depth > 15u) { + v_depth = 15u; + } + if (v_c < 256u) { + self->private_impl.f_pc_valid[(v_c & 255u)] = self->private_impl.f_pc_call; + self->private_data.f_pc_term[(v_c & 255u)] = v_term; + self->private_data.f_pc_depth[(v_c & 255u)] = ((uint8_t)((v_depth & 15u))); + } + } + v_bits = wuffs_brotli__bit_cost__f64_add(self, v_bits, v_term); if (v_depth > 15u) { v_depth = 15u; } @@ -86527,29 +86737,32 @@ wuffs_brotli__bit_cost__population_cost( v_reps -= 2u; while (v_reps > 0u) { v_depth_histo[17u] += 1u; - v_bits = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_bits, 4613937818241073152u); + v_bits = wuffs_brotli__bit_cost__f64_add(self, v_bits, 4613937818241073152u); v_reps >>= 3u; } } } } - v_bits = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_bits, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, (18u + (2u * v_max_depth)))); + v_bits = wuffs_brotli__bit_cost__f64_add(self, v_bits, wuffs_brotli__bit_cost__f64_from_u64(self, (18u + (2u * v_max_depth)))); v_i = 0u; while (v_i < 18u) { v_p = ((uint64_t)(v_depth_histo[v_i])); v_sum += v_p; - v_lp = wuffs_brotli__bit_cost__fast_log2_memo(self, v_p); - v_retval = wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, v_retval, wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_p), v_lp)); v_i += 1u; + if (v_p <= 1u) { + continue; + } + v_lp = wuffs_brotli__bit_cost__fast_log2_memo(self, v_p); + v_retval = wuffs_brotli__bit_cost__f64_sub(self, v_retval, wuffs_brotli__bit_cost__f64_mul(self, wuffs_brotli__bit_cost__f64_from_u64(self, v_p), v_lp)); } if (v_sum != 0u) { v_lp = wuffs_brotli__bit_cost__fast_log2_memo(self, v_sum); - v_retval = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_retval, wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_sum), v_lp)); + v_retval = wuffs_brotli__bit_cost__f64_add(self, v_retval, wuffs_brotli__bit_cost__f64_mul(self, wuffs_brotli__bit_cost__f64_from_u64(self, v_sum), v_lp)); } - if (wuffs_softfloat__arith__f64_lt(&self->private_data.f_sf, v_retval, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_sum)) != 0u) { - v_retval = wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_sum); + if (wuffs_brotli__bit_cost__f64_lt(self, v_retval, wuffs_brotli__bit_cost__f64_from_u64(self, v_sum)) != 0u) { + v_retval = wuffs_brotli__bit_cost__f64_from_u64(self, v_sum); } - return wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_bits, v_retval); + return wuffs_brotli__bit_cost__f64_add(self, v_bits, v_retval); } // -------- func brotli.encoder.rb_byte @@ -88046,14 +88259,14 @@ wuffs_brotli__encoder__bits_entropy_of_histogram( while (v_i < 256u) { v_p = ((uint64_t)(self->private_data.f_histogram[v_i])); v_sum += v_p; - v_retval = wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, v_retval, wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_p), wuffs_brotli__encoder__fast_log2(self, v_p))); + v_retval = wuffs_brotli__bit_cost__f64_sub(&self->private_data.f_bc, v_retval, wuffs_brotli__bit_cost__f64_mul(&self->private_data.f_bc, wuffs_brotli__bit_cost__f64_from_u64(&self->private_data.f_bc, v_p), wuffs_brotli__encoder__fast_log2(self, v_p))); v_i += 1u; } if (v_sum != 0u) { - v_retval = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_retval, wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_sum), wuffs_brotli__encoder__fast_log2(self, v_sum))); + v_retval = wuffs_brotli__bit_cost__f64_add(&self->private_data.f_bc, v_retval, wuffs_brotli__bit_cost__f64_mul(&self->private_data.f_bc, wuffs_brotli__bit_cost__f64_from_u64(&self->private_data.f_bc, v_sum), wuffs_brotli__encoder__fast_log2(self, v_sum))); } - if (wuffs_softfloat__arith__f64_lt(&self->private_data.f_sf, v_retval, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_sum)) != 0u) { - v_retval = wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_sum); + if (wuffs_brotli__bit_cost__f64_lt(&self->private_data.f_bc, v_retval, wuffs_brotli__bit_cost__f64_from_u64(&self->private_data.f_bc, v_sum)) != 0u) { + v_retval = wuffs_brotli__bit_cost__f64_from_u64(&self->private_data.f_bc, v_sum); } return v_retval; } @@ -88528,6 +88741,13 @@ wuffs_brotli__metablock_builder__find_blocks( uint64_t v_c = 0; uint8_t v_cur_id = 0; uint8_t v_b = 0; + uint64_t v_nh = 0; + uint64_t v_icn = 0; + uint64_t v_best_k = 0; + bool v_use_capped = false; + uint64_t v_skip_t = 0; + uint64_t v_t = 0; + uint32_t v_n = 0; v_alphabet_size = ((uint64_t)(a_ds)); v_bitmap_len = (((uint64_t)(a_num_histograms + 7u)) >> 3u); @@ -88555,13 +88775,14 @@ wuffs_brotli__metablock_builder__find_blocks( v_k = ((uint64_t)(((uint64_t)(v_i * a_num_histograms)) + v_j)); if (v_k < 35200u) { v_c = wuffs_brotli__metablock_builder__bit_cost(self, ((uint64_t)(wuffs_brotli__metablock_builder__ld32(self, a_wb, ((uint64_t)(((uint64_t)(a_histograms + ((uint64_t)(v_j * v_hs)))) + ((uint64_t)(v_i * 4u)))))))); - self->private_data.f_insert_cost[v_k] = wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, self->private_data.f_insert_cost[(v_j & 255u)], v_c); + self->private_data.f_insert_cost[v_k] = wuffs_brotli__bit_cost__f64_sub(&self->private_data.f_bc, self->private_data.f_insert_cost[(v_j & 255u)], v_c); } v_j += 1u; } } wuffs_private_impl__bulk_memset(&self->private_data.f_cost[0], 256u * (size_t)8u, 0u); wuffs_brotli__metablock_builder__zero(self, a_wb, a_switch_signal, ((uint64_t)(a_length * v_bitmap_len))); + v_nh = wuffs_base__u64__min(a_num_histograms, 256u); v_byte_ix = 0u; while (v_byte_ix < a_length) { v_ix = ((uint64_t)(v_byte_ix * v_bitmap_len)); @@ -88569,31 +88790,79 @@ wuffs_brotli__metablock_builder__find_blocks( a_wb, a_data, v_byte_ix, - a_width) * a_num_histograms)); + a_width) * v_nh)); v_min_cost = 6088095589093318446u; v_block_switch_cost = a_block_switch_bitcost; + v_icn = 0u; + if (v_insert_cost_ix <= 35200u) { + v_icn = wuffs_base__u64__min(v_nh, (35200u - v_insert_cost_ix)); + } + if ((v_byte_ix == 2000u) && (((uint64_t)(a_length - 2000u)) > v_alphabet_size) && (v_nh >= 4u)) { + v_t = ((uint64_t)(v_alphabet_size * v_nh)); + v_t = wuffs_base__u64__min(v_t, 35200u); + v_i = 0u; + while (v_i < v_t) { + self->private_data.f_capped_plus[wuffs_base__u64__min(v_i, 35199u)] = wuffs_brotli__bit_cost__f64_add(&self->private_data.f_bc, a_block_switch_bitcost, self->private_data.f_insert_cost[wuffs_base__u64__min(v_i, 35199u)]); + v_i += 1u; + } + v_use_capped = true; + } + v_best_k = 18446744073709551615u; v_k = 0u; - while (v_k < a_num_histograms) { - v_c = 0u; - if (((uint64_t)(v_insert_cost_ix + v_k)) < 35200u) { - v_c = self->private_data.f_insert_cost[(((uint64_t)(v_insert_cost_ix + v_k)) % 35200u)]; + while (v_k < v_nh) { + if (v_k < v_icn) { + v_j = ((uint64_t)(v_insert_cost_ix + v_k)); + v_j = wuffs_base__u64__min(v_j, 35199u); + if (v_use_capped && (self->private_data.f_cost[(v_k & 255u)] == a_block_switch_bitcost)) { + v_c = self->private_data.f_capped_plus[v_j]; + } else { + v_c = wuffs_brotli__bit_cost__f64_add(&self->private_data.f_bc, self->private_data.f_cost[(v_k & 255u)], self->private_data.f_insert_cost[v_j]); + } + } else { + v_c = wuffs_brotli__bit_cost__f64_add(&self->private_data.f_bc, self->private_data.f_cost[(v_k & 255u)], 0u); } - v_c = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, self->private_data.f_cost[(v_k & 255u)], v_c); self->private_data.f_cost[(v_k & 255u)] = v_c; - if (wuffs_softfloat__arith__f64_lt(&self->private_data.f_sf, v_c, v_min_cost) != 0u) { + if (wuffs_brotli__bit_cost__f64_lt(&self->private_data.f_bc, v_c, v_min_cost) != 0u) { v_min_cost = v_c; - wuffs_brotli__metablock_builder__st8(self, a_wb, ((uint64_t)(a_block_id + v_byte_ix)), ((uint8_t)(v_k))); + v_best_k = v_k; } v_k += 1u; } + if (v_best_k <= 255u) { + wuffs_brotli__metablock_builder__st8(self, a_wb, ((uint64_t)(a_block_id + v_byte_ix)), ((uint8_t)(v_best_k))); + } if (v_byte_ix < 2000u) { - v_block_switch_cost = wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, v_block_switch_cost, wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, 4605110762971426980u, wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, 4540289913102728147u, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_byte_ix)))); + v_block_switch_cost = wuffs_brotli__bit_cost__f64_mul(&self->private_data.f_bc, v_block_switch_cost, wuffs_brotli__bit_cost__f64_add(&self->private_data.f_bc, 4605110762971426980u, wuffs_brotli__bit_cost__f64_mul(&self->private_data.f_bc, 4540289913102728147u, wuffs_brotli__bit_cost__f64_from_u64(&self->private_data.f_bc, v_byte_ix)))); + } + v_skip_t = 18446744073709551615u; + if ((v_nh >= 8u) && (v_min_cost < 9218868437227405312u) && (((uint64_t)(v_block_switch_cost - 1u)) < 9218868437227405311u)) { + v_t = wuffs_brotli__bit_cost__f64_add(&self->private_data.f_bc, v_block_switch_cost, v_min_cost); + v_n = 0u; + while ((v_n < 3u) && (v_t < 9218868437227405312u) && (wuffs_brotli__bit_cost__f64_ge(&self->private_data.f_bc, wuffs_brotli__bit_cost__f64_sub(&self->private_data.f_bc, v_t, v_min_cost), v_block_switch_cost) == 0u)) { + v_t += 1u; + v_n += 1u; + } + if ((v_t < 9218868437227405312u) && (wuffs_brotli__bit_cost__f64_ge(&self->private_data.f_bc, wuffs_brotli__bit_cost__f64_sub(&self->private_data.f_bc, v_t, v_min_cost), v_block_switch_cost) != 0u)) { + v_n = 0u; + while ((v_n < 3u) && (v_t > 0u) && (wuffs_brotli__bit_cost__f64_ge(&self->private_data.f_bc, wuffs_brotli__bit_cost__f64_sub(&self->private_data.f_bc, ((uint64_t)(v_t - 1u)), v_min_cost), v_block_switch_cost) != 0u)) { + v_t -= 1u; + v_n += 1u; + } + if ((v_t == 0u) || (wuffs_brotli__bit_cost__f64_ge(&self->private_data.f_bc, wuffs_brotli__bit_cost__f64_sub(&self->private_data.f_bc, ((uint64_t)(v_t - 1u)), v_min_cost), v_block_switch_cost) == 0u)) { + v_skip_t = v_t; + } + } } v_k = 0u; - while (v_k < a_num_histograms) { - v_c = wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, self->private_data.f_cost[(v_k & 255u)], v_min_cost); + while (v_k < v_nh) { + v_c = self->private_data.f_cost[(v_k & 255u)]; + if ((v_c <= 9218868437227405312u) && (v_c >= v_skip_t)) { + v_c = v_block_switch_cost; + } else { + v_c = wuffs_brotli__bit_cost__f64_sub(&self->private_data.f_bc, v_c, v_min_cost); + } self->private_data.f_cost[(v_k & 255u)] = v_c; - if (wuffs_softfloat__arith__f64_ge(&self->private_data.f_sf, v_c, v_block_switch_cost) != 0u) { + if (wuffs_brotli__bit_cost__f64_ge(&self->private_data.f_bc, v_c, v_block_switch_cost) != 0u) { self->private_data.f_cost[(v_k & 255u)] = v_block_switch_cost; v_j = ((uint64_t)(v_ix + (v_k >> 3u))); v_b = wuffs_brotli__metablock_builder__ld8(self, a_wb, ((uint64_t)(a_switch_signal + v_j))); @@ -88938,7 +89207,7 @@ wuffs_brotli__metablock_builder__cluster_blocks( v_tmp, ((uint64_t)(v_all_histograms + ((uint64_t)(((uint64_t)(v_s)) * v_hs)))), ((uint64_t)(v_tmp + v_hs))); - if (wuffs_softfloat__arith__f64_lt(&self->private_data.f_sf, v_cur_bits, v_best_bits) != 0u) { + if (wuffs_brotli__bit_cost__f64_lt(&self->private_data.f_bc, v_cur_bits, v_best_bits) != 0u) { v_best_bits = v_cur_bits; v_best_out = v_s; } @@ -90167,7 +90436,7 @@ wuffs_brotli__metablock_builder__histogram_pair_is_less( uint32_t a_p2_idx1, uint32_t a_p2_idx2) { if (wuffs_softfloat__arith__f64_ne(&self->private_data.f_sf, a_p1_cost_diff, a_p2_cost_diff) != 0u) { - return (wuffs_softfloat__arith__f64_gt(&self->private_data.f_sf, a_p1_cost_diff, a_p2_cost_diff) != 0u); + return (wuffs_brotli__bit_cost__f64_gt(&self->private_data.f_bc, a_p1_cost_diff, a_p2_cost_diff) != 0u); } return (((uint32_t)(a_p1_idx2 - a_p1_idx1)) > ((uint32_t)(a_p2_idx2 - a_p2_idx1))); } @@ -90183,7 +90452,7 @@ wuffs_brotli__metablock_builder__cluster_cost_diff( uint64_t v_size_c = 0; v_size_c = ((uint64_t)(a_size_a + a_size_b)); - return wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, a_size_a), wuffs_brotli__bit_cost__fast_log2(&self->private_data.f_bc, a_size_a)), wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, a_size_b), wuffs_brotli__bit_cost__fast_log2(&self->private_data.f_bc, a_size_b))), wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_size_c), wuffs_brotli__bit_cost__fast_log2(&self->private_data.f_bc, v_size_c))); + return wuffs_brotli__bit_cost__f64_sub(&self->private_data.f_bc, wuffs_brotli__bit_cost__f64_add(&self->private_data.f_bc, wuffs_brotli__bit_cost__f64_mul(&self->private_data.f_bc, wuffs_brotli__bit_cost__f64_from_u64(&self->private_data.f_bc, a_size_a), wuffs_brotli__bit_cost__fast_log2(&self->private_data.f_bc, a_size_a)), wuffs_brotli__bit_cost__f64_mul(&self->private_data.f_bc, wuffs_brotli__bit_cost__f64_from_u64(&self->private_data.f_bc, a_size_b), wuffs_brotli__bit_cost__fast_log2(&self->private_data.f_bc, a_size_b))), wuffs_brotli__bit_cost__f64_mul(&self->private_data.f_bc, wuffs_brotli__bit_cost__f64_from_u64(&self->private_data.f_bc, v_size_c), wuffs_brotli__bit_cost__fast_log2(&self->private_data.f_bc, v_size_c))); } // -------- func brotli.metablock_builder.pair_copy @@ -90260,9 +90529,9 @@ wuffs_brotli__metablock_builder__compare_and_push_to_queue( } v_h1 = ((uint64_t)(a_out + ((uint64_t)(((uint64_t)(v_idx1)) * v_hs)))); v_h2 = ((uint64_t)(a_out + ((uint64_t)(((uint64_t)(v_idx2)) * v_hs)))); - v_cost_diff = wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, 4602678819172646912u, wuffs_brotli__metablock_builder__cluster_cost_diff(self, ((uint64_t)(wuffs_brotli__metablock_builder__ld32(self, a_wb, ((uint64_t)(a_cluster_size + ((uint64_t)(((uint64_t)(v_idx1)) * 4u))))))), ((uint64_t)(wuffs_brotli__metablock_builder__ld32(self, a_wb, ((uint64_t)(a_cluster_size + ((uint64_t)(((uint64_t)(v_idx2)) * 4u))))))))); - v_cost_diff = wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, v_cost_diff, wuffs_brotli__metablock_builder__h_bit_cost(self, a_wb, v_h1, a_ds)); - v_cost_diff = wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, v_cost_diff, wuffs_brotli__metablock_builder__h_bit_cost(self, a_wb, v_h2, a_ds)); + v_cost_diff = wuffs_brotli__bit_cost__f64_mul(&self->private_data.f_bc, 4602678819172646912u, wuffs_brotli__metablock_builder__cluster_cost_diff(self, ((uint64_t)(wuffs_brotli__metablock_builder__ld32(self, a_wb, ((uint64_t)(a_cluster_size + ((uint64_t)(((uint64_t)(v_idx1)) * 4u))))))), ((uint64_t)(wuffs_brotli__metablock_builder__ld32(self, a_wb, ((uint64_t)(a_cluster_size + ((uint64_t)(((uint64_t)(v_idx2)) * 4u))))))))); + v_cost_diff = wuffs_brotli__bit_cost__f64_sub(&self->private_data.f_bc, v_cost_diff, wuffs_brotli__metablock_builder__h_bit_cost(self, a_wb, v_h1, a_ds)); + v_cost_diff = wuffs_brotli__bit_cost__f64_sub(&self->private_data.f_bc, v_cost_diff, wuffs_brotli__metablock_builder__h_bit_cost(self, a_wb, v_h2, a_ds)); if (wuffs_brotli__metablock_builder__h_total(self, a_wb, v_h1, a_ds) == 0u) { v_cost_combo = wuffs_brotli__metablock_builder__h_bit_cost(self, a_wb, v_h2, a_ds); v_is_good_pair = true; @@ -90286,13 +90555,13 @@ wuffs_brotli__metablock_builder__compare_and_push_to_queue( v_h2, a_ds); v_c = wuffs_brotli__metablock_builder__h_population_cost(self, a_wb, a_tmp, a_ds); - if (wuffs_softfloat__arith__f64_lt(&self->private_data.f_sf, v_c, wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, v_threshold, v_cost_diff)) != 0u) { + if (wuffs_brotli__bit_cost__f64_lt(&self->private_data.f_bc, v_c, wuffs_brotli__bit_cost__f64_sub(&self->private_data.f_bc, v_threshold, v_cost_diff)) != 0u) { v_cost_combo = v_c; v_is_good_pair = true; } } if (v_is_good_pair) { - v_cost_diff = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_cost_diff, v_cost_combo); + v_cost_diff = wuffs_brotli__bit_cost__f64_add(&self->private_data.f_bc, v_cost_diff, v_cost_combo); if ((self->private_impl.f_num_pairs > 0u) && wuffs_brotli__metablock_builder__histogram_pair_is_less(self, wuffs_brotli__metablock_builder__ld64(self, a_wb, ((uint64_t)(a_pairs + 16u))), wuffs_brotli__metablock_builder__ld32(self, a_wb, a_pairs), @@ -90384,7 +90653,7 @@ wuffs_brotli__metablock_builder__histogram_combine( v_idx1 += 1u; } while (v_num_clusters > v_min_cluster_size) { - if (wuffs_softfloat__arith__f64_ge(&self->private_data.f_sf, wuffs_brotli__metablock_builder__ld64(self, a_wb, ((uint64_t)(a_pairs + 16u))), v_cost_diff_threshold) != 0u) { + if (wuffs_brotli__bit_cost__f64_ge(&self->private_data.f_bc, wuffs_brotli__metablock_builder__ld64(self, a_wb, ((uint64_t)(a_pairs + 16u))), v_cost_diff_threshold) != 0u) { v_cost_diff_threshold = 6088095589093318446u; v_min_cluster_size = a_max_clusters; continue; @@ -90510,7 +90779,7 @@ wuffs_brotli__metablock_builder__histogram_bit_cost_distance( a_candidate, a_ds); v_c = wuffs_brotli__metablock_builder__h_population_cost(self, a_wb, a_tmp, a_ds); - return wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, v_c, wuffs_brotli__metablock_builder__h_bit_cost(self, a_wb, a_candidate, a_ds)); + return wuffs_brotli__bit_cost__f64_sub(&self->private_data.f_bc, v_c, wuffs_brotli__metablock_builder__h_bit_cost(self, a_wb, a_candidate, a_ds)); } // -------- func brotli.metablock_builder.histogram_remap @@ -90559,7 +90828,7 @@ wuffs_brotli__metablock_builder__histogram_remap( ((uint64_t)(a_in_histos + ((uint64_t)(v_i * v_hs)))), ((uint64_t)(a_out + ((uint64_t)(((uint64_t)(v_c)) * v_hs)))), a_tmp); - if (wuffs_softfloat__arith__f64_lt(&self->private_data.f_sf, v_cur_bits, v_best_bits) != 0u) { + if (wuffs_brotli__bit_cost__f64_lt(&self->private_data.f_bc, v_cur_bits, v_best_bits) != 0u) { v_best_bits = v_cur_bits; v_best_out = v_c; } @@ -91413,10 +91682,10 @@ wuffs_brotli__encoder__estimate_entropy( while (v_i < a_size) { v_p = self->private_data.f_ctx_histo[(((uint32_t)(a_off + v_i)) & 511u)]; v_total += ((uint64_t)(v_p)); - v_result = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_result, wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u32(&self->private_data.f_sf, v_p), wuffs_brotli__bit_cost__fast_log2(&self->private_data.f_bc, ((uint64_t)(v_p))))); + v_result = wuffs_brotli__bit_cost__f64_add(&self->private_data.f_bc, v_result, wuffs_brotli__bit_cost__f64_mul(&self->private_data.f_bc, wuffs_brotli__bit_cost__f64_from_u32(&self->private_data.f_bc, v_p), wuffs_brotli__bit_cost__fast_log2(&self->private_data.f_bc, ((uint64_t)(v_p))))); v_i += 1u; } - return wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_total), wuffs_brotli__bit_cost__fast_log2(&self->private_data.f_bc, v_total)), v_result); + return wuffs_brotli__bit_cost__f64_sub(&self->private_data.f_bc, wuffs_brotli__bit_cost__f64_mul(&self->private_data.f_bc, wuffs_brotli__bit_cost__f64_from_u64(&self->private_data.f_bc, v_total), wuffs_brotli__bit_cost__fast_log2(&self->private_data.f_bc, v_total)), v_result); } // -------- func brotli.encoder.choose_context_map @@ -91444,24 +91713,24 @@ wuffs_brotli__encoder__choose_context_map( v_i += 1u; } v_e1 = wuffs_brotli__encoder__estimate_entropy(self, 16u, 3u); - v_e2 = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, wuffs_brotli__encoder__estimate_entropy(self, 24u, 3u), wuffs_brotli__encoder__estimate_entropy(self, 27u, 3u)); + v_e2 = wuffs_brotli__bit_cost__f64_add(&self->private_data.f_bc, wuffs_brotli__encoder__estimate_entropy(self, 24u, 3u), wuffs_brotli__encoder__estimate_entropy(self, 27u, 3u)); v_e3 = 0u; v_i = 0u; while (v_i < 3u) { - v_e3 = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_e3, wuffs_brotli__encoder__estimate_entropy(self, (3u * v_i), 3u)); + v_e3 = wuffs_brotli__bit_cost__f64_add(&self->private_data.f_bc, v_e3, wuffs_brotli__encoder__estimate_entropy(self, (3u * v_i), 3u)); v_i += 1u; } v_total = ((uint32_t)(((uint32_t)(self->private_data.f_ctx_histo[16u] + self->private_data.f_ctx_histo[17u])) + self->private_data.f_ctx_histo[18u])); - v_e0 = wuffs_softfloat__arith__f64_div(&self->private_data.f_sf, 4607182418800017408u, wuffs_softfloat__arith__f64_from_u32(&self->private_data.f_sf, v_total)); - v_e1 = wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, v_e1, v_e0); - v_e2 = wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, v_e2, v_e0); - v_e3 = wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, v_e3, v_e0); + v_e0 = wuffs_softfloat__arith__f64_div(&self->private_data.f_sf, 4607182418800017408u, wuffs_brotli__bit_cost__f64_from_u32(&self->private_data.f_bc, v_total)); + v_e1 = wuffs_brotli__bit_cost__f64_mul(&self->private_data.f_bc, v_e1, v_e0); + v_e2 = wuffs_brotli__bit_cost__f64_mul(&self->private_data.f_bc, v_e2, v_e0); + v_e3 = wuffs_brotli__bit_cost__f64_mul(&self->private_data.f_bc, v_e3, v_e0); if (self->private_impl.f_quality < 7u) { - v_e3 = wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, v_e1, 4621819117588971520u); + v_e3 = wuffs_brotli__bit_cost__f64_mul(&self->private_data.f_bc, v_e1, 4621819117588971520u); } - if ((wuffs_softfloat__arith__f64_lt(&self->private_data.f_sf, wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, v_e1, v_e2), 4596373779694328218u) != 0u) && (wuffs_softfloat__arith__f64_lt(&self->private_data.f_sf, wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, v_e1, v_e3), 4596373779694328218u) != 0u)) { + if ((wuffs_brotli__bit_cost__f64_lt(&self->private_data.f_bc, wuffs_brotli__bit_cost__f64_sub(&self->private_data.f_bc, v_e1, v_e2), 4596373779694328218u) != 0u) && (wuffs_brotli__bit_cost__f64_lt(&self->private_data.f_bc, wuffs_brotli__bit_cost__f64_sub(&self->private_data.f_bc, v_e1, v_e3), 4596373779694328218u) != 0u)) { return 1u; - } else if (wuffs_softfloat__arith__f64_lt(&self->private_data.f_sf, wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, v_e2, v_e3), 4581421828931458171u) != 0u) { + } else if (wuffs_brotli__bit_cost__f64_lt(&self->private_data.f_bc, wuffs_brotli__bit_cost__f64_sub(&self->private_data.f_bc, v_e2, v_e3), 4581421828931458171u) != 0u) { return 2u; } return 3u; @@ -91521,13 +91790,13 @@ wuffs_brotli__encoder__should_use_complex_static_context_map( v_e2 = 0u; v_i = 0u; while (v_i < 13u) { - v_e2 = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_e2, wuffs_brotli__encoder__estimate_entropy(self, (32u + (v_i << 5u)), 32u)); + v_e2 = wuffs_brotli__bit_cost__f64_add(&self->private_data.f_bc, v_e2, wuffs_brotli__encoder__estimate_entropy(self, (32u + (v_i << 5u)), 32u)); v_i += 1u; } - v_e0 = wuffs_softfloat__arith__f64_div(&self->private_data.f_sf, 4607182418800017408u, wuffs_softfloat__arith__f64_from_u32(&self->private_data.f_sf, v_total)); - v_e1 = wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, v_e1, v_e0); - v_e2 = wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, v_e2, v_e0); - if ((wuffs_softfloat__arith__f64_gt(&self->private_data.f_sf, v_e2, 4613937818241073152u) != 0u) || (wuffs_softfloat__arith__f64_lt(&self->private_data.f_sf, wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, v_e1, v_e2), 4596373779694328218u) != 0u)) { + v_e0 = wuffs_softfloat__arith__f64_div(&self->private_data.f_sf, 4607182418800017408u, wuffs_brotli__bit_cost__f64_from_u32(&self->private_data.f_bc, v_total)); + v_e1 = wuffs_brotli__bit_cost__f64_mul(&self->private_data.f_bc, v_e1, v_e0); + v_e2 = wuffs_brotli__bit_cost__f64_mul(&self->private_data.f_bc, v_e2, v_e0); + if ((wuffs_brotli__bit_cost__f64_gt(&self->private_data.f_bc, v_e2, 4613937818241073152u) != 0u) || (wuffs_brotli__bit_cost__f64_lt(&self->private_data.f_bc, wuffs_brotli__bit_cost__f64_sub(&self->private_data.f_bc, v_e1, v_e2), 4596373779694328218u) != 0u)) { return 0u; } return 13u; @@ -91615,13 +91884,13 @@ wuffs_brotli__encoder__should_compress( return false; } if (a_num_commands < ((a_bytes >> 8u) + 2u)) { - if (wuffs_softfloat__arith__f64_gt(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, a_num_literals), wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, 4607092346807469998u, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, a_bytes))) != 0u) { + if (wuffs_brotli__bit_cost__f64_gt(&self->private_data.f_bc, wuffs_brotli__bit_cost__f64_from_u64(&self->private_data.f_bc, a_num_literals), wuffs_brotli__bit_cost__f64_mul(&self->private_data.f_bc, 4607092346807469998u, wuffs_brotli__bit_cost__f64_from_u64(&self->private_data.f_bc, a_bytes))) != 0u) { v_i = 0u; while (v_i < 256u) { self->private_data.f_ctx_histo[v_i] = 0u; v_i += 1u; } - v_bit_cost_threshold = wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, a_bytes), 4620603145689581486u), 4590207312512236308u); + v_bit_cost_threshold = wuffs_brotli__bit_cost__f64_mul(&self->private_data.f_bc, wuffs_brotli__bit_cost__f64_mul(&self->private_data.f_bc, wuffs_brotli__bit_cost__f64_from_u64(&self->private_data.f_bc, a_bytes), 4620603145689581486u), 4590207312512236308u); v_t = (((uint64_t)(a_bytes + 12u)) / 13u); v_pos = ((uint32_t)(a_last_flush_pos)); v_i = 0u; @@ -91630,7 +91899,7 @@ wuffs_brotli__encoder__should_compress( v_pos += 13u; v_i += 1u; } - if (wuffs_softfloat__arith__f64_gt(&self->private_data.f_sf, wuffs_brotli__encoder__ctx_histo_bits_entropy(self), v_bit_cost_threshold) != 0u) { + if (wuffs_brotli__bit_cost__f64_gt(&self->private_data.f_bc, wuffs_brotli__encoder__ctx_histo_bits_entropy(self), v_bit_cost_threshold) != 0u) { return false; } } @@ -91653,14 +91922,14 @@ wuffs_brotli__encoder__ctx_histo_bits_entropy( while (v_i < 256u) { v_p = ((uint64_t)(self->private_data.f_ctx_histo[v_i])); v_sum += v_p; - v_retval = wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, v_retval, wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_p), wuffs_brotli__bit_cost__fast_log2(&self->private_data.f_bc, v_p))); + v_retval = wuffs_brotli__bit_cost__f64_sub(&self->private_data.f_bc, v_retval, wuffs_brotli__bit_cost__f64_mul(&self->private_data.f_bc, wuffs_brotli__bit_cost__f64_from_u64(&self->private_data.f_bc, v_p), wuffs_brotli__bit_cost__fast_log2(&self->private_data.f_bc, v_p))); v_i += 1u; } if (v_sum != 0u) { - v_retval = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_retval, wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_sum), wuffs_brotli__bit_cost__fast_log2(&self->private_data.f_bc, v_sum))); + v_retval = wuffs_brotli__bit_cost__f64_add(&self->private_data.f_bc, v_retval, wuffs_brotli__bit_cost__f64_mul(&self->private_data.f_bc, wuffs_brotli__bit_cost__f64_from_u64(&self->private_data.f_bc, v_sum), wuffs_brotli__bit_cost__fast_log2(&self->private_data.f_bc, v_sum))); } - if (wuffs_softfloat__arith__f64_lt(&self->private_data.f_sf, v_retval, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_sum)) != 0u) { - v_retval = wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_sum); + if (wuffs_brotli__bit_cost__f64_lt(&self->private_data.f_bc, v_retval, wuffs_brotli__bit_cost__f64_from_u64(&self->private_data.f_bc, v_sum)) != 0u) { + v_retval = wuffs_brotli__bit_cost__f64_from_u64(&self->private_data.f_bc, v_sum); } return v_retval; } @@ -93539,6 +93808,723 @@ wuffs_brotli__encoder__copy_tmp_bits_to_cmd_bits( return wuffs_base__make_empty_struct(); } +// -------- func brotli.bit_cost.clz64 + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__bit_cost__clz64( + const wuffs_brotli__bit_cost* self, + uint64_t a_a) { + uint64_t v_x = 0; + uint32_t v_n = 0; + + v_x = a_a; + if ((v_x >> 32u) == 0u) { + v_x = ((uint64_t)(v_x << 32u)); + v_n = 32u; + } + if ((v_x >> 48u) == 0u) { + v_x = ((uint64_t)(v_x << 16u)); + v_n = ((v_n + 16u) & 63u); + } + if ((v_x >> 56u) == 0u) { + v_x = ((uint64_t)(v_x << 8u)); + v_n = ((v_n + 8u) & 63u); + } + if ((v_x >> 60u) == 0u) { + v_x = ((uint64_t)(v_x << 4u)); + v_n = ((v_n + 4u) & 63u); + } + if ((v_x >> 62u) == 0u) { + v_x = ((uint64_t)(v_x << 2u)); + v_n = ((v_n + 2u) & 63u); + } + if ((v_x >> 63u) == 0u) { + v_n = ((v_n + 1u) & 63u); + } + return v_n; +} + +// -------- func brotli.bit_cost.f32_add + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__bit_cost__f32_add( + const wuffs_brotli__bit_cost* self, + uint32_t a_a, + uint32_t a_b) { + uint32_t v_x = 0; + uint32_t v_y = 0; + uint32_t v_ex = 0; + uint32_t v_d = 0; + uint64_t v_s = 0; + uint32_t v_z = 0; + + if ((((uint32_t)(a_a - 8388608u)) < 2130706432u) && (((uint32_t)(a_b - 8388608u)) < 2130706432u)) { + v_x = a_a; + v_y = a_b; + if (v_y > v_x) { + v_x = a_b; + v_y = a_a; + } + v_ex = ((v_x >> 23u) & 255u); + v_d = ((uint32_t)(v_ex - ((v_y >> 23u) & 255u))); + if (v_d >= 25u) { + return v_x; + } + v_s = ((uint64_t)((((uint64_t)(((v_x & 8388607u) | 8388608u))) << 39u) + ((((uint64_t)(((v_y & 8388607u) | 8388608u))) << 39u) >> (v_d & 31u)))); + if (v_s >= 9223372036854775808u) { + v_s = (((uint64_t)(v_s + ((uint64_t)(549755813887u + ((v_s >> 40u) & 1u))))) >> 40u); + v_z = ((uint32_t)(((uint32_t)(v_ex << 23u)) + ((uint32_t)((v_s & 33554431u))))); + if (v_z >= 2139095040u) { + return 2139095040u; + } + return v_z; + } + v_s = (((uint64_t)(v_s + ((uint64_t)(274877906943u + ((v_s >> 39u) & 1u))))) >> 39u); + return ((uint32_t)(((uint32_t)(((uint32_t)(v_ex - 1u)) << 23u)) + ((uint32_t)((v_s & 33554431u))))); + } else if ((a_a == 0u) && ((a_b & 2147483647u) <= 2139095040u) && (a_b != 2147483648u)) { + return a_b; + } else if ((a_b == 0u) && ((a_a & 2147483647u) <= 2139095040u) && (a_a != 2147483648u)) { + return a_a; + } + return wuffs_brotli__bit_cost__f32_add_any(self, a_a, a_b); +} + +// -------- func brotli.bit_cost.f32_sub + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__bit_cost__f32_sub( + const wuffs_brotli__bit_cost* self, + uint32_t a_a, + uint32_t a_b) { + uint32_t v_ex = 0; + uint32_t v_d = 0; + uint64_t v_s = 0; + + if ((((uint32_t)(a_b - 8388608u)) < 2130706432u) && (a_a > a_b) && (a_a < 2139095040u)) { + v_ex = ((a_a >> 23u) & 255u); + v_d = ((uint32_t)(v_ex - ((a_b >> 23u) & 255u))); + if (v_d >= 26u) { + return a_a; + } else if (v_d >= 2u) { + v_s = ((uint64_t)((((uint64_t)(((a_a & 8388607u) | 8388608u))) << 39u) - ((((uint64_t)(((a_b & 8388607u) | 8388608u))) << 39u) >> (v_d & 31u)))); + if (v_s < 4611686018427387904u) { + v_s <<= 1u; + v_ex = (((uint32_t)(v_ex - 1u)) & 255u); + } + v_s = (((uint64_t)(v_s + ((uint64_t)(274877906943u + ((v_s >> 39u) & 1u))))) >> 39u); + return ((uint32_t)(((uint32_t)(((uint32_t)(v_ex - 1u)) << 23u)) + ((uint32_t)((v_s & 33554431u))))); + } + } else if ((a_b == 0u) && ((a_a & 2147483647u) <= 2139095040u)) { + return a_a; + } else if ((a_a == a_b) && (a_a < 2139095040u)) { + return 0u; + } + return wuffs_brotli__bit_cost__f32_sub_any(self, a_a, a_b); +} + +// -------- func brotli.bit_cost.f32_add_any + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__bit_cost__f32_add_any( + const wuffs_brotli__bit_cost* self, + uint32_t a_a, + uint32_t a_b) { + uint32_t v_x = 0; + uint32_t v_y = 0; + uint32_t v_ex = 0; + uint32_t v_d = 0; + uint64_t v_mx = 0; + uint64_t v_my = 0; + uint64_t v_s = 0; + uint32_t v_lz = 0; + uint32_t v_z = 0; + + if ((((uint32_t)(((a_a >> 23u) & 255u) - 1u)) < 254u) && (((uint32_t)(((a_b >> 23u) & 255u) - 1u)) < 254u)) { + v_x = a_a; + v_y = a_b; + if ((v_y & 2147483647u) > (v_x & 2147483647u)) { + v_x = a_b; + v_y = a_a; + } + v_ex = ((v_x >> 23u) & 255u); + v_d = ((uint32_t)(v_ex - ((v_y >> 23u) & 255u))); + if (v_d >= 26u) { + return v_x; + } + v_mx = (((uint64_t)(((v_x & 8388607u) | 8388608u))) << 39u); + v_my = ((((uint64_t)(((v_y & 8388607u) | 8388608u))) << 39u) >> (v_d & 31u)); + if (((v_x ^ v_y) >> 31u) == 0u) { + v_s = ((uint64_t)(v_mx + v_my)); + if (v_s >= 9223372036854775808u) { + v_s = (((uint64_t)(v_s + ((uint64_t)(549755813887u + ((v_s >> 40u) & 1u))))) >> 40u); + v_z = ((uint32_t)(((uint32_t)(v_ex << 23u)) + ((uint32_t)((v_s & 33554431u))))); + if (v_z >= 2139095040u) { + v_z = 2139095040u; + } + return ((v_x & 2147483648u) | v_z); + } + } else { + v_s = ((uint64_t)(v_mx - v_my)); + if (v_s == 0u) { + return 0u; + } else if (v_s < 4611686018427387904u) { + v_lz = (((uint32_t)(wuffs_brotli__bit_cost__clz64(self, v_s) - 1u)) & 63u); + if (v_lz >= v_ex) { + return wuffs_softfloat__arith__f32_add(&self->private_data.f_sf, a_a, a_b); + } + v_ex = (((uint32_t)(v_ex - v_lz)) & 255u); + v_s = ((uint64_t)(v_s << v_lz)); + } + } + v_s = (((uint64_t)(v_s + ((uint64_t)(274877906943u + ((v_s >> 39u) & 1u))))) >> 39u); + return ((v_x & 2147483648u) | ((uint32_t)(((uint32_t)(((uint32_t)(v_ex - 1u)) << 23u)) + ((uint32_t)((v_s & 33554431u)))))); + } + return wuffs_softfloat__arith__f32_add(&self->private_data.f_sf, a_a, a_b); +} + +// -------- func brotli.bit_cost.f32_sub_any + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__bit_cost__f32_sub_any( + const wuffs_brotli__bit_cost* self, + uint32_t a_a, + uint32_t a_b) { + if (((uint32_t)(((a_b >> 23u) & 255u) - 1u)) < 254u) { + return wuffs_brotli__bit_cost__f32_add_any(self, a_a, (a_b ^ 2147483648u)); + } + return wuffs_softfloat__arith__f32_sub(&self->private_data.f_sf, a_a, a_b); +} + +// -------- func brotli.bit_cost.f32_key + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__bit_cost__f32_key( + const wuffs_brotli__bit_cost* self, + uint32_t a_a) { + if ((a_a >> 31u) != 0u) { + return (4294967295u ^ a_a); + } + return (a_a | 2147483648u); +} + +// -------- func brotli.bit_cost.f32_lt + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__bit_cost__f32_lt( + const wuffs_brotli__bit_cost* self, + uint32_t a_a, + uint32_t a_b) { + if ((a_a | a_b) <= 2139095040u) { + if (a_a < a_b) { + return 1u; + } + return 0u; + } + return wuffs_brotli__bit_cost__f32_lt_any(self, a_a, a_b); +} + +// -------- func brotli.bit_cost.f32_lt_any + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__bit_cost__f32_lt_any( + const wuffs_brotli__bit_cost* self, + uint32_t a_a, + uint32_t a_b) { + if (((a_a & 2147483647u) <= 2139095040u) && ((a_b & 2147483647u) <= 2139095040u)) { + if ((wuffs_brotli__bit_cost__f32_key(self, a_a) < wuffs_brotli__bit_cost__f32_key(self, a_b)) && (((a_a | a_b) & 2147483647u) != 0u)) { + return 1u; + } + return 0u; + } + return wuffs_softfloat__arith__f32_lt(&self->private_data.f_sf, a_a, a_b); +} + +// -------- func brotli.bit_cost.f32_le + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__bit_cost__f32_le( + const wuffs_brotli__bit_cost* self, + uint32_t a_a, + uint32_t a_b) { + if ((a_a | a_b) <= 2139095040u) { + if (a_a <= a_b) { + return 1u; + } + return 0u; + } + return wuffs_brotli__bit_cost__f32_le_any(self, a_a, a_b); +} + +// -------- func brotli.bit_cost.f32_le_any + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__bit_cost__f32_le_any( + const wuffs_brotli__bit_cost* self, + uint32_t a_a, + uint32_t a_b) { + if (((a_a & 2147483647u) <= 2139095040u) && ((a_b & 2147483647u) <= 2139095040u)) { + if ((wuffs_brotli__bit_cost__f32_key(self, a_a) <= wuffs_brotli__bit_cost__f32_key(self, a_b)) || (((a_a | a_b) & 2147483647u) == 0u)) { + return 1u; + } + return 0u; + } + return wuffs_softfloat__arith__f32_le(&self->private_data.f_sf, a_a, a_b); +} + +// -------- func brotli.bit_cost.f32_gt + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__bit_cost__f32_gt( + const wuffs_brotli__bit_cost* self, + uint32_t a_a, + uint32_t a_b) { + if ((a_a | a_b) <= 2139095040u) { + if (a_a > a_b) { + return 1u; + } + return 0u; + } + return wuffs_brotli__bit_cost__f32_gt_any(self, a_a, a_b); +} + +// -------- func brotli.bit_cost.f32_gt_any + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__bit_cost__f32_gt_any( + const wuffs_brotli__bit_cost* self, + uint32_t a_a, + uint32_t a_b) { + return wuffs_brotli__bit_cost__f32_lt_any(self, a_b, a_a); +} + +// -------- func brotli.bit_cost.f64_add + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__bit_cost__f64_add( + const wuffs_brotli__bit_cost* self, + uint64_t a_a, + uint64_t a_b) { + uint64_t v_x = 0; + uint64_t v_y = 0; + uint64_t v_ex = 0; + uint64_t v_d = 0; + uint64_t v_my = 0; + uint64_t v_s = 0; + + if ((((uint64_t)(a_a - 4503599627370496u)) < 9214364837600034816u) && (((uint64_t)(a_b - 4503599627370496u)) < 9214364837600034816u)) { + v_x = a_a; + v_y = a_b; + if (v_y > v_x) { + v_x = a_b; + v_y = a_a; + } + v_ex = ((v_x >> 52u) & 2047u); + v_d = ((uint64_t)(v_ex - ((v_y >> 52u) & 2047u))); + if (v_d >= 54u) { + return v_x; + } + v_my = (((v_y & 4503599627370495u) | 4503599627370496u) << 10u); + if ((v_my & ((uint64_t)((((uint64_t)(1u)) << (v_d & 63u)) - 1u))) != 0u) { + v_my = ((v_my >> (v_d & 63u)) | 1u); + } else { + v_my = (v_my >> (v_d & 63u)); + } + v_s = ((uint64_t)((((v_x & 4503599627370495u) | 4503599627370496u) << 10u) + v_my)); + if (v_s >= 9223372036854775808u) { + v_ex = (((uint64_t)(v_ex + 1u)) & 2047u); + v_s = ((v_s >> 1u) | (v_s & 1u)); + } + v_s = (((uint64_t)(v_s + ((uint64_t)(511u + ((v_s >> 10u) & 1u))))) >> 10u); + v_s = ((uint64_t)(((uint64_t)(((uint64_t)(v_ex - 1u)) << 52u)) + v_s)); + if (v_s >= 9218868437227405312u) { + return 9218868437227405312u; + } + return v_s; + } else if ((a_a == 0u) && ((a_b & 9223372036854775807u) <= 9218868437227405312u) && (a_b != 9223372036854775808u)) { + return a_b; + } else if ((a_b == 0u) && ((a_a & 9223372036854775807u) <= 9218868437227405312u) && (a_a != 9223372036854775808u)) { + return a_a; + } + return wuffs_brotli__bit_cost__f64_add_any(self, a_a, a_b); +} + +// -------- func brotli.bit_cost.f64_sub + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__bit_cost__f64_sub( + const wuffs_brotli__bit_cost* self, + uint64_t a_a, + uint64_t a_b) { + uint64_t v_ex = 0; + uint64_t v_d = 0; + uint64_t v_my = 0; + uint64_t v_s = 0; + uint64_t v_lz = 0; + + if ((((uint64_t)(a_b - 4503599627370496u)) < 9214364837600034816u) && (a_a > a_b) && (a_a < 9218868437227405312u)) { + v_ex = ((a_a >> 52u) & 2047u); + v_d = ((uint64_t)(v_ex - ((a_b >> 52u) & 2047u))); + if (v_d >= 55u) { + return a_a; + } else if (v_d >= 2u) { + v_my = (((a_b & 4503599627370495u) | 4503599627370496u) << 10u); + if ((v_my & ((uint64_t)((((uint64_t)(1u)) << (v_d & 63u)) - 1u))) != 0u) { + v_my = ((v_my >> (v_d & 63u)) | 1u); + } else { + v_my = (v_my >> (v_d & 63u)); + } + v_s = ((uint64_t)((((a_a & 4503599627370495u) | 4503599627370496u) << 10u) - v_my)); + if (v_s < 4611686018427387904u) { + v_s <<= 1u; + v_ex = (((uint64_t)(v_ex - 1u)) & 2047u); + } + v_s = (((uint64_t)(v_s + ((uint64_t)(511u + ((v_s >> 10u) & 1u))))) >> 10u); + return ((uint64_t)(((uint64_t)(((uint64_t)(v_ex - 1u)) << 52u)) + v_s)); + } + v_s = ((uint64_t)((((a_a & 4503599627370495u) | 4503599627370496u) << 10u) - ((((a_b & 4503599627370495u) | 4503599627370496u) << 10u) >> (v_d & 1u)))); + v_lz = ((uint64_t)(((uint64_t)(wuffs_brotli__bit_cost__clz64(self, v_s))) - 1u)); + if (v_lz < v_ex) { + v_ex = (((uint64_t)(v_ex - v_lz)) & 2047u); + v_s = ((uint64_t)(v_s << (v_lz & 63u))); + v_s = (((uint64_t)(v_s + ((uint64_t)(511u + ((v_s >> 10u) & 1u))))) >> 10u); + return ((uint64_t)(((uint64_t)(((uint64_t)(v_ex - 1u)) << 52u)) + v_s)); + } + } else if ((a_b == 0u) && ((a_a & 9223372036854775807u) <= 9218868437227405312u)) { + return a_a; + } else if ((a_a == a_b) && (a_a < 9218868437227405312u)) { + return 0u; + } + return wuffs_brotli__bit_cost__f64_sub_any(self, a_a, a_b); +} + +// -------- func brotli.bit_cost.f64_add_any + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__bit_cost__f64_add_any( + const wuffs_brotli__bit_cost* self, + uint64_t a_a, + uint64_t a_b) { + uint64_t v_x = 0; + uint64_t v_y = 0; + uint64_t v_ex = 0; + uint64_t v_d = 0; + uint64_t v_mx = 0; + uint64_t v_my = 0; + uint64_t v_s = 0; + uint64_t v_lz = 0; + + if ((((uint64_t)(((a_a >> 52u) & 2047u) - 1u)) < 2046u) && (((uint64_t)(((a_b >> 52u) & 2047u) - 1u)) < 2046u)) { + v_x = a_a; + v_y = a_b; + if ((v_y & 9223372036854775807u) > (v_x & 9223372036854775807u)) { + v_x = a_b; + v_y = a_a; + } + v_ex = ((v_x >> 52u) & 2047u); + v_d = ((uint64_t)(v_ex - ((v_y >> 52u) & 2047u))); + if (v_d >= 55u) { + return v_x; + } + v_mx = (((v_x & 4503599627370495u) | 4503599627370496u) << 10u); + v_my = (((v_y & 4503599627370495u) | 4503599627370496u) << 10u); + if ((v_my & ((uint64_t)((((uint64_t)(1u)) << (v_d & 63u)) - 1u))) != 0u) { + v_my = ((v_my >> (v_d & 63u)) | 1u); + } else { + v_my = (v_my >> (v_d & 63u)); + } + if (((v_x ^ v_y) >> 63u) == 0u) { + v_s = ((uint64_t)(v_mx + v_my)); + if (v_s >= 9223372036854775808u) { + v_ex = (((uint64_t)(v_ex + 1u)) & 2047u); + v_s = ((v_s >> 1u) | (v_s & 1u)); + } + } else { + v_s = ((uint64_t)(v_mx - v_my)); + if (v_s == 0u) { + return 0u; + } else if (v_s < 4611686018427387904u) { + v_lz = (((uint64_t)(((uint64_t)(wuffs_brotli__bit_cost__clz64(self, v_s))) - 1u)) & 63u); + if (v_lz >= v_ex) { + return wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, a_a, a_b); + } + v_ex = (((uint64_t)(v_ex - v_lz)) & 2047u); + v_s = ((uint64_t)(v_s << v_lz)); + } + } + v_s = (((uint64_t)(v_s + ((uint64_t)(511u + ((v_s >> 10u) & 1u))))) >> 10u); + v_s = ((uint64_t)(((uint64_t)(((uint64_t)(v_ex - 1u)) << 52u)) + v_s)); + if (v_s >= 9218868437227405312u) { + v_s = 9218868437227405312u; + } + return ((v_x & 9223372036854775808u) | v_s); + } + return wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, a_a, a_b); +} + +// -------- func brotli.bit_cost.f64_sub_any + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__bit_cost__f64_sub_any( + const wuffs_brotli__bit_cost* self, + uint64_t a_a, + uint64_t a_b) { + if (((uint64_t)(((a_b >> 52u) & 2047u) - 1u)) < 2046u) { + return wuffs_brotli__bit_cost__f64_add_any(self, a_a, (a_b ^ 9223372036854775808u)); + } + return wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, a_a, a_b); +} + +// -------- func brotli.bit_cost.f64_key + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__bit_cost__f64_key( + const wuffs_brotli__bit_cost* self, + uint64_t a_a) { + if ((a_a >> 63u) != 0u) { + return (18446744073709551615u ^ a_a); + } + return (a_a | 9223372036854775808u); +} + +// -------- func brotli.bit_cost.f64_lt + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__bit_cost__f64_lt( + const wuffs_brotli__bit_cost* self, + uint64_t a_a, + uint64_t a_b) { + if ((a_a | a_b) <= 9218868437227405312u) { + if (a_a < a_b) { + return 1u; + } + return 0u; + } + return wuffs_brotli__bit_cost__f64_lt_any(self, a_a, a_b); +} + +// -------- func brotli.bit_cost.f64_lt_any + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__bit_cost__f64_lt_any( + const wuffs_brotli__bit_cost* self, + uint64_t a_a, + uint64_t a_b) { + if (((a_a & 9223372036854775807u) <= 9218868437227405312u) && ((a_b & 9223372036854775807u) <= 9218868437227405312u)) { + if ((wuffs_brotli__bit_cost__f64_key(self, a_a) < wuffs_brotli__bit_cost__f64_key(self, a_b)) && (((a_a | a_b) & 9223372036854775807u) != 0u)) { + return 1u; + } + return 0u; + } + return wuffs_softfloat__arith__f64_lt(&self->private_data.f_sf, a_a, a_b); +} + +// -------- func brotli.bit_cost.f64_gt + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__bit_cost__f64_gt( + const wuffs_brotli__bit_cost* self, + uint64_t a_a, + uint64_t a_b) { + if ((a_a | a_b) <= 9218868437227405312u) { + if (a_a > a_b) { + return 1u; + } + return 0u; + } + return wuffs_brotli__bit_cost__f64_gt_any(self, a_a, a_b); +} + +// -------- func brotli.bit_cost.f64_gt_any + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__bit_cost__f64_gt_any( + const wuffs_brotli__bit_cost* self, + uint64_t a_a, + uint64_t a_b) { + return wuffs_brotli__bit_cost__f64_lt_any(self, a_b, a_a); +} + +// -------- func brotli.bit_cost.f64_ge + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__bit_cost__f64_ge( + const wuffs_brotli__bit_cost* self, + uint64_t a_a, + uint64_t a_b) { + if ((a_a | a_b) <= 9218868437227405312u) { + if (a_a >= a_b) { + return 1u; + } + return 0u; + } + return wuffs_brotli__bit_cost__f64_ge_any(self, a_a, a_b); +} + +// -------- func brotli.bit_cost.f64_ge_any + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__bit_cost__f64_ge_any( + const wuffs_brotli__bit_cost* self, + uint64_t a_a, + uint64_t a_b) { + if (((a_a & 9223372036854775807u) <= 9218868437227405312u) && ((a_b & 9223372036854775807u) <= 9218868437227405312u)) { + if ((wuffs_brotli__bit_cost__f64_key(self, a_a) >= wuffs_brotli__bit_cost__f64_key(self, a_b)) || (((a_a | a_b) & 9223372036854775807u) == 0u)) { + return 1u; + } + return 0u; + } + return wuffs_softfloat__arith__f64_ge(&self->private_data.f_sf, a_a, a_b); +} + +// -------- func brotli.bit_cost.f64_mul + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__bit_cost__f64_mul( + const wuffs_brotli__bit_cost* self, + uint64_t a_a, + uint64_t a_b) { + uint64_t v_ea = 0; + uint64_t v_eb = 0; + uint64_t v_e = 0; + uint64_t v_ma = 0; + uint64_t v_mb = 0; + uint64_t v_a0 = 0; + uint64_t v_a1 = 0; + uint64_t v_b0 = 0; + uint64_t v_b1 = 0; + uint64_t v_p00 = 0; + uint64_t v_p01 = 0; + uint64_t v_p10 = 0; + uint64_t v_mid = 0; + uint64_t v_s = 0; + uint64_t v_rb = 0; + + v_ea = ((a_a >> 52u) & 2047u); + v_eb = ((a_b >> 52u) & 2047u); + if ((((uint64_t)(v_ea - 1u)) < 2046u) && (((uint64_t)(v_eb - 1u)) < 2046u)) { + v_e = ((uint64_t)(((uint64_t)(v_ea + v_eb)) - 1023u)); + v_ma = (((a_a & 4503599627370495u) | 4503599627370496u) << 10u); + v_mb = (((a_b & 4503599627370495u) | 4503599627370496u) << 11u); + v_a0 = (v_ma & 4294967295u); + v_a1 = (v_ma >> 32u); + v_b0 = (v_mb & 4294967295u); + v_b1 = (v_mb >> 32u); + v_p00 = (v_a0 * v_b0); + v_p01 = (v_a0 * v_b1); + v_p10 = (v_a1 * v_b0); + v_mid = ((v_p00 >> 32u) + (v_p01 & 4294967295u) + (v_p10 & 4294967295u)); + v_s = ((uint64_t)(((uint64_t)(((uint64_t)((v_a1 * v_b1) + (v_p01 >> 32u))) + (v_p10 >> 32u))) + (v_mid >> 32u))); + if (((uint64_t)(v_ma * v_mb)) != 0u) { + v_s |= 1u; + } + if (v_s < 4611686018427387904u) { + v_e -= 1u; + v_s <<= 1u; + } + if (v_e < 2045u) { + v_rb = (v_s & 1023u); + v_s = (((uint64_t)(v_s + 512u)) >> 10u); + if (v_rb == 512u) { + v_s &= 18446744073709551614u; + } + return (((a_a ^ a_b) & 9223372036854775808u) | ((uint64_t)(((uint64_t)(v_e << 52u)) + v_s))); + } + } + return wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, a_a, a_b); +} + +// -------- func brotli.bit_cost.f64_from_u64 + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__bit_cost__f64_from_u64( + const wuffs_brotli__bit_cost* self, + uint64_t a_a) { + uint32_t v_lz = 0; + + if (a_a == 0u) { + return 0u; + } else if (a_a < 9007199254740992u) { + v_lz = wuffs_brotli__bit_cost__clz64(self, a_a); + if (v_lz >= 11u) { + return ((uint64_t)(((uint64_t)((((uint64_t)(1085u - ((uint64_t)(v_lz)))) & 2047u) << 52u)) + ((uint64_t)(a_a << (v_lz - 11u))))); + } + } + return wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, a_a); +} + +// -------- func brotli.bit_cost.f64_from_u32 + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__bit_cost__f64_from_u32( + const wuffs_brotli__bit_cost* self, + uint32_t a_a) { + return wuffs_brotli__bit_cost__f64_from_u64(self, ((uint64_t)(a_a))); +} + +// -------- func brotli.bit_cost.f64_to_u64_trunc + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__bit_cost__f64_to_u64_trunc( + const wuffs_brotli__bit_cost* self, + uint64_t a_a) { + uint64_t v_exp = 0; + + v_exp = (a_a >> 52u); + if (v_exp < 1023u) { + return 0u; + } else if (v_exp <= 1075u) { + return (((a_a & 4503599627370495u) | 4503599627370496u) >> ((1075u - v_exp) & 63u)); + } + return wuffs_softfloat__arith__f64_to_u64_trunc(&self->private_data.f_sf, a_a); +} + +// -------- func brotli.bit_cost.f32_from_u32 + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__bit_cost__f32_from_u32( + const wuffs_brotli__bit_cost* self, + uint32_t a_a) { + uint32_t v_lz = 0; + + if (a_a == 0u) { + return 0u; + } else if (a_a < 16777216u) { + v_lz = wuffs_brotli__bit_cost__clz64(self, ((uint64_t)(a_a))); + if (v_lz >= 40u) { + return ((uint32_t)(((uint32_t)(((uint32_t)(189u - v_lz)) << 23u)) + ((uint32_t)(a_a << (v_lz - 40u))))); + } + } + return wuffs_softfloat__arith__f32_from_u32(&self->private_data.f_sf, a_a); +} + // -------- func brotli.encoder.hash_fast WUFFS_BASE__GENERATED_C_CODE @@ -93923,14 +94909,14 @@ wuffs_brotli__encoder__should_merge_block( v_i += 43u; } v_total = (((uint64_t)(a_len + 42u)) / 43u); - v_r = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, wuffs_brotli__encoder__fast_log2(self, v_total), 4602678819172646912u), wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_total)), 4641240890982006784u); + v_r = wuffs_brotli__bit_cost__f64_add(&self->private_data.f_bc, wuffs_brotli__bit_cost__f64_mul(&self->private_data.f_bc, wuffs_brotli__bit_cost__f64_add(&self->private_data.f_bc, wuffs_brotli__encoder__fast_log2(self, v_total), 4602678819172646912u), wuffs_brotli__bit_cost__f64_from_u64(&self->private_data.f_bc, v_total)), 4641240890982006784u); v_j = 0u; while (v_j < 256u) { v_h = ((uint64_t)(self->private_data.f_histogram[v_j])); - v_r = wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, v_r, wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_h), wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u32(&self->private_data.f_sf, ((uint32_t)(self->private_data.f_lit_depth[v_j]))), wuffs_brotli__encoder__fast_log2(self, v_h)))); + v_r = wuffs_brotli__bit_cost__f64_sub(&self->private_data.f_bc, v_r, wuffs_brotli__bit_cost__f64_mul(&self->private_data.f_bc, wuffs_brotli__bit_cost__f64_from_u64(&self->private_data.f_bc, v_h), wuffs_brotli__bit_cost__f64_add(&self->private_data.f_bc, wuffs_brotli__bit_cost__f64_from_u32(&self->private_data.f_bc, ((uint32_t)(self->private_data.f_lit_depth[v_j]))), wuffs_brotli__encoder__fast_log2(self, v_h)))); v_j += 1u; } - return (wuffs_softfloat__arith__f64_ge(&self->private_data.f_sf, v_r, 0u) != 0u); + return (wuffs_brotli__bit_cost__f64_ge(&self->private_data.f_bc, v_r, 0u) != 0u); } // -------- func brotli.encoder.should_use_uncompressed_mode @@ -94926,11 +95912,11 @@ wuffs_brotli__encoder__should_compress_two_pass( uint64_t v_max_total_bit_cost = 0; uint64_t v_i = 0; - v_corpus_size = wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, a_input_size); - if (wuffs_softfloat__arith__f64_lt(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, a_num_literals), wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, 4607002274814922588u, v_corpus_size)) != 0u) { + v_corpus_size = wuffs_brotli__bit_cost__f64_from_u64(&self->private_data.f_bc, a_input_size); + if (wuffs_brotli__bit_cost__f64_lt(&self->private_data.f_bc, wuffs_brotli__bit_cost__f64_from_u64(&self->private_data.f_bc, a_num_literals), wuffs_brotli__bit_cost__f64_mul(&self->private_data.f_bc, 4607002274814922588u, v_corpus_size)) != 0u) { return true; } - v_max_total_bit_cost = wuffs_softfloat__arith__f64_div(&self->private_data.f_sf, wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, v_corpus_size, 4620693217682128896u), 4607002274814922588u), 4631248529308778496u); + v_max_total_bit_cost = wuffs_softfloat__arith__f64_div(&self->private_data.f_sf, wuffs_brotli__bit_cost__f64_mul(&self->private_data.f_bc, wuffs_brotli__bit_cost__f64_mul(&self->private_data.f_bc, v_corpus_size, 4620693217682128896u), 4607002274814922588u), 4631248529308778496u); wuffs_private_impl__bulk_memset(&self->private_data.f_histogram[0], 256u * (size_t)4u, 0u); v_i = 0u; while (v_i < a_input_size) { @@ -94939,7 +95925,7 @@ wuffs_brotli__encoder__should_compress_two_pass( } v_i += 43u; } - return (wuffs_softfloat__arith__f64_lt(&self->private_data.f_sf, wuffs_brotli__encoder__bits_entropy_of_histogram(self), v_max_total_bit_cost) != 0u); + return (wuffs_brotli__bit_cost__f64_lt(&self->private_data.f_bc, wuffs_brotli__encoder__bits_entropy_of_histogram(self), v_max_total_bit_cost) != 0u); } // -------- func brotli.encoder.emit_uncompressed_meta_block_two_pass @@ -97373,7 +98359,7 @@ wuffs_brotli__encoder__is_mostly_utf8( v_size_utf8 += v_bytes_read; } } - return (wuffs_softfloat__arith__f64_gt(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_size_utf8), wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, 4604930618986332160u, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, a_length))) != 0u); + return (wuffs_brotli__bit_cost__f64_gt(&self->private_data.f_bc, wuffs_brotli__bit_cost__f64_from_u64(&self->private_data.f_bc, v_size_utf8), wuffs_brotli__bit_cost__f64_mul(&self->private_data.f_bc, 4604930618986332160u, wuffs_brotli__bit_cost__f64_from_u64(&self->private_data.f_bc, a_length))) != 0u); } // -------- func brotli.encoder.utf8_position @@ -97447,12 +98433,12 @@ wuffs_brotli__encoder__lc_finish_cost( uint64_t v_lit_cost = 0; v_lit_cost = a_lit_cost; - if (wuffs_softfloat__arith__f64_lt(&self->private_data.f_sf, v_lit_cost, 4607182418800017408u) != 0u) { - v_lit_cost = wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, v_lit_cost, 4602678819172646912u); - v_lit_cost = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_lit_cost, 4602678819172646912u); + if (wuffs_brotli__bit_cost__f64_lt(&self->private_data.f_bc, v_lit_cost, 4607182418800017408u) != 0u) { + v_lit_cost = wuffs_brotli__bit_cost__f64_mul(&self->private_data.f_bc, v_lit_cost, 4602678819172646912u); + v_lit_cost = wuffs_brotli__bit_cost__f64_add(&self->private_data.f_bc, v_lit_cost, 4602678819172646912u); } if (a_utf8 && (a_i < 2000u)) { - v_lit_cost = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_lit_cost, wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, 4599976659396224614u, wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, 4550588384442628807u, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, a_i)))); + v_lit_cost = wuffs_brotli__bit_cost__f64_add(&self->private_data.f_bc, v_lit_cost, wuffs_brotli__bit_cost__f64_add(&self->private_data.f_bc, 4599976659396224614u, wuffs_brotli__bit_cost__f64_mul(&self->private_data.f_bc, 4550588384442628807u, wuffs_brotli__bit_cost__f64_from_u64(&self->private_data.f_bc, a_i)))); } wuffs_brotli__encoder__poke_u32(self, a_wb, ((uint64_t)(a_cost_off + ((uint64_t)(a_i * 4u)))), wuffs_softfloat__arith__f64_to_f32(&self->private_data.f_sf, v_lit_cost)); return wuffs_base__make_empty_struct(); @@ -97567,8 +98553,8 @@ wuffs_brotli__encoder__estimate_bit_costs_for_literals_utf8( v_iw = v_iwu2; } v_lit_cost = wuffs_brotli__bit_cost__fast_log2_memo(&self->private_data.f_bc, v_iw); - v_lit_cost = wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, v_lit_cost, wuffs_brotli__bit_cost__fast_log2(&self->private_data.f_bc, v_histo)); - v_lit_cost = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_lit_cost, 4584030313835631162u); + v_lit_cost = wuffs_brotli__bit_cost__f64_sub(&self->private_data.f_bc, v_lit_cost, wuffs_brotli__bit_cost__fast_log2(&self->private_data.f_bc, v_histo)); + v_lit_cost = wuffs_brotli__bit_cost__f64_add(&self->private_data.f_bc, v_lit_cost, 4584030313835631162u); wuffs_brotli__encoder__lc_finish_cost(self, a_wb, a_cost_off, @@ -97639,8 +98625,8 @@ wuffs_brotli__encoder__estimate_bit_costs_for_literals( v_histo = 1u; } v_lit_cost = wuffs_brotli__bit_cost__fast_log2_memo(&self->private_data.f_bc, v_in_window); - v_lit_cost = wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, v_lit_cost, wuffs_brotli__bit_cost__fast_log2(&self->private_data.f_bc, v_histo)); - v_lit_cost = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_lit_cost, 4584015902316823577u); + v_lit_cost = wuffs_brotli__bit_cost__f64_sub(&self->private_data.f_bc, v_lit_cost, wuffs_brotli__bit_cost__fast_log2(&self->private_data.f_bc, v_histo)); + v_lit_cost = wuffs_brotli__bit_cost__f64_add(&self->private_data.f_bc, v_lit_cost, 4584015902316823577u); wuffs_brotli__encoder__lc_finish_cost(self, a_wb, a_cost_off, @@ -98229,12 +99215,12 @@ wuffs_brotli__metablock_builder__compute_distance_cost( a_tmp, 544u, ((uint64_t)((v_dist_prefix & 1023u)))); - v_extra_bits = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_extra_bits, wuffs_softfloat__arith__f64_from_u32(&self->private_data.f_sf, (v_dist_prefix >> 10u))); + v_extra_bits = wuffs_brotli__bit_cost__f64_add(&self->private_data.f_bc, v_extra_bits, wuffs_brotli__bit_cost__f64_from_u32(&self->private_data.f_bc, (v_dist_prefix >> 10u))); } v_i += 1u; } self->private_impl.f_dist_cost = wuffs_brotli__metablock_builder__h_population_cost(self, a_wb, a_tmp, 544u); - self->private_impl.f_dist_cost = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, self->private_impl.f_dist_cost, v_extra_bits); + self->private_impl.f_dist_cost = wuffs_brotli__bit_cost__f64_add(&self->private_data.f_bc, self->private_impl.f_dist_cost, v_extra_bits); return true; } @@ -98388,7 +99374,7 @@ wuffs_brotli__metablock_builder__build_meta_block( v_ndirect, self->private_impl.f_dist_max_distance, v_tmp); - if ( ! v_success || (wuffs_softfloat__arith__f64_gt(&self->private_data.f_sf, self->private_impl.f_dist_cost, v_best_dist_cost) != 0u)) { + if ( ! v_success || (wuffs_brotli__bit_cost__f64_gt(&self->private_data.f_bc, self->private_impl.f_dist_cost, v_best_dist_cost) != 0u)) { break; } v_best_dist_cost = self->private_impl.f_dist_cost; @@ -98416,7 +99402,7 @@ wuffs_brotli__metablock_builder__build_meta_block( v_orig_ndirect, v_orig_md, v_tmp); - if (wuffs_softfloat__arith__f64_lt(&self->private_data.f_sf, self->private_impl.f_dist_cost, v_best_dist_cost) != 0u) { + if (wuffs_brotli__bit_cost__f64_lt(&self->private_data.f_bc, self->private_impl.f_dist_cost, v_best_dist_cost) != 0u) { v_best_npostfix = v_orig_npostfix; v_best_ndirect = v_orig_ndirect; v_best_asm = v_orig_asm; @@ -99102,25 +100088,25 @@ wuffs_brotli__encoder__gbs_finish_block( v_data_size); } self->private_data.f_gbs_comb_entropy[(v_jx & 31u)] = wuffs_brotli__encoder__gbs_entropy(self, a_wb, v_tmp, v_alphabet_size); - v_d = wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, self->private_data.f_gbs_comb_entropy[(v_jx & 31u)], v_e), self->private_data.f_gbs_last_entropy[(((uint64_t)(((uint64_t)(v_le_base)) + v_jx)) & 127u)]); + v_d = wuffs_brotli__bit_cost__f64_sub(&self->private_data.f_bc, wuffs_brotli__bit_cost__f64_sub(&self->private_data.f_bc, self->private_data.f_gbs_comb_entropy[(v_jx & 31u)], v_e), self->private_data.f_gbs_last_entropy[(((uint64_t)(((uint64_t)(v_le_base)) + v_jx)) & 127u)]); if (v_j == 0u) { if (v_nc == 1u) { v_diff0 = v_d; } else { - v_diff0 = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_diff0, v_d); + v_diff0 = wuffs_brotli__bit_cost__f64_add(&self->private_data.f_bc, v_diff0, v_d); } } else { if (v_nc == 1u) { v_diff1 = v_d; } else { - v_diff1 = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_diff1, v_d); + v_diff1 = wuffs_brotli__bit_cost__f64_add(&self->private_data.f_bc, v_diff1, v_d); } } v_j += 1u; } v_i += 1u; } - if ((self->private_data.f_gbs_num_types[a_c] < self->private_data.f_gbs_max_block_types[a_c]) && (wuffs_softfloat__arith__f64_gt(&self->private_data.f_sf, v_diff0, v_threshold) != 0u) && (wuffs_softfloat__arith__f64_gt(&self->private_data.f_sf, v_diff1, v_threshold) != 0u)) { + if ((self->private_data.f_gbs_num_types[a_c] < self->private_data.f_gbs_max_block_types[a_c]) && (wuffs_brotli__bit_cost__f64_gt(&self->private_data.f_bc, v_diff0, v_threshold) != 0u) && (wuffs_brotli__bit_cost__f64_gt(&self->private_data.f_bc, v_diff1, v_threshold) != 0u)) { wuffs_brotli__encoder__poke_u32(self, a_wb, ((uint64_t)(v_lengths_off + ((uint64_t)(v_num_blocks * 4u)))), ((uint32_t)(self->private_data.f_gbs_block_size[a_c]))); wuffs_brotli__encoder__set_byte(self, a_wb, ((uint64_t)(v_types_off + v_num_blocks)), ((uint8_t)(self->private_data.f_gbs_num_types[a_c]))); self->private_data.f_gbs_last_histogram_ix[((a_c * 2u) + 1u)] = v_last0; @@ -99141,7 +100127,7 @@ wuffs_brotli__encoder__gbs_finish_block( self->private_data.f_gbs_block_size[a_c] = 0u; self->private_data.f_gbs_merge_last_count[a_c] = 0u; self->private_data.f_gbs_target_block_size[a_c] = ((uint64_t)(self->private_data.f_gbs_min_block_size[a_c])); - } else if (wuffs_softfloat__arith__f64_lt(&self->private_data.f_sf, v_diff1, wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, v_diff0, 4626322717216342016u)) != 0u) { + } else if (wuffs_brotli__bit_cost__f64_lt(&self->private_data.f_bc, v_diff1, wuffs_brotli__bit_cost__f64_sub(&self->private_data.f_bc, v_diff0, 4626322717216342016u)) != 0u) { wuffs_brotli__encoder__poke_u32(self, a_wb, ((uint64_t)(v_lengths_off + ((uint64_t)(v_num_blocks * 4u)))), ((uint32_t)(self->private_data.f_gbs_block_size[a_c]))); wuffs_brotli__encoder__set_byte(self, a_wb, ((uint64_t)(v_types_off + v_num_blocks)), wuffs_brotli__encoder__byte_at(self, a_wb, ((uint64_t)(v_types_off + ((uint64_t)(v_num_blocks - 2u)))))); self->private_data.f_gbs_last_histogram_ix[(a_c * 2u)] = v_last1; @@ -101307,15 +102293,15 @@ wuffs_brotli__encoder__z_set_cost_from_histogram( v_i += 1u; } } - v_missing_symbol_cost = wuffs_softfloat__arith__f32_add(&self->private_data.f_sf, wuffs_softfloat__arith__f64_to_f32(&self->private_data.f_sf, wuffs_brotli__bit_cost__fast_log2(&self->private_data.f_bc, v_missing_symbol_sum)), 1073741824u); + v_missing_symbol_cost = wuffs_brotli__bit_cost__f32_add(&self->private_data.f_bc, wuffs_softfloat__arith__f64_to_f32(&self->private_data.f_sf, wuffs_brotli__bit_cost__fast_log2(&self->private_data.f_bc, v_missing_symbol_sum)), 1073741824u); v_i = 0u; while (v_i < a_histogram_size) { v_h = wuffs_brotli__encoder__z_hist(self, a_which, v_i); if (v_h == 0u) { wuffs_brotli__encoder__z_set_cost(self, a_which, v_i, v_missing_symbol_cost); } else { - v_cost = wuffs_softfloat__arith__f32_sub(&self->private_data.f_sf, v_log2sum, wuffs_softfloat__arith__f64_to_f32(&self->private_data.f_sf, wuffs_brotli__bit_cost__fast_log2(&self->private_data.f_bc, ((uint64_t)(v_h))))); - if (wuffs_softfloat__arith__f32_lt(&self->private_data.f_sf, v_cost, 1065353216u) != 0u) { + v_cost = wuffs_brotli__bit_cost__f32_sub(&self->private_data.f_bc, v_log2sum, wuffs_softfloat__arith__f64_to_f32(&self->private_data.f_sf, wuffs_brotli__bit_cost__fast_log2(&self->private_data.f_bc, ((uint64_t)(v_h))))); + if (wuffs_brotli__bit_cost__f32_lt(&self->private_data.f_bc, v_cost, 1065353216u) != 0u) { v_cost = 1065353216u; } wuffs_brotli__encoder__z_set_cost(self, a_which, v_i, v_cost); @@ -101398,10 +102384,10 @@ wuffs_brotli__encoder__zopfli_cost_model_set_from_commands( v_i = 0u; while (v_i < self->private_impl.f_z_num_bytes) { v_b = wuffs_brotli__encoder__rb_byte(self, a_wb, ((uint64_t)(a_position + v_i))); - v_literal_carry = wuffs_softfloat__arith__f32_add(&self->private_data.f_sf, v_literal_carry, self->private_data.f_z_cost_literal[v_b]); - v_lc_i1 = wuffs_softfloat__arith__f32_add(&self->private_data.f_sf, v_lc_i, v_literal_carry); + v_literal_carry = wuffs_brotli__bit_cost__f32_add(&self->private_data.f_bc, v_literal_carry, self->private_data.f_z_cost_literal[v_b]); + v_lc_i1 = wuffs_brotli__bit_cost__f32_add(&self->private_data.f_bc, v_lc_i, v_literal_carry); wuffs_brotli__encoder__poke_u32(self, a_wb, ((uint64_t)(self->private_impl.f_z_lit_off + ((uint64_t)(((uint64_t)(v_i + 1u)) * 4u)))), v_lc_i1); - v_literal_carry = wuffs_softfloat__arith__f32_sub(&self->private_data.f_sf, v_literal_carry, wuffs_softfloat__arith__f32_sub(&self->private_data.f_sf, v_lc_i1, v_lc_i)); + v_literal_carry = wuffs_brotli__bit_cost__f32_sub(&self->private_data.f_bc, v_literal_carry, wuffs_brotli__bit_cost__f32_sub(&self->private_data.f_bc, v_lc_i1, v_lc_i)); v_lc_i = v_lc_i1; v_i += 1u; } @@ -101432,10 +102418,10 @@ wuffs_brotli__encoder__zopfli_cost_model_set_from_literal_costs( v_lc_i = 0u; v_i = 0u; while (v_i < self->private_impl.f_z_num_bytes) { - v_literal_carry = wuffs_softfloat__arith__f32_add(&self->private_data.f_sf, v_literal_carry, wuffs_brotli__encoder__lit_cost_at(self, a_wb, ((uint64_t)(v_i + 1u)))); - v_lc_i1 = wuffs_softfloat__arith__f32_add(&self->private_data.f_sf, v_lc_i, v_literal_carry); + v_literal_carry = wuffs_brotli__bit_cost__f32_add(&self->private_data.f_bc, v_literal_carry, wuffs_brotli__encoder__lit_cost_at(self, a_wb, ((uint64_t)(v_i + 1u)))); + v_lc_i1 = wuffs_brotli__bit_cost__f32_add(&self->private_data.f_bc, v_lc_i, v_literal_carry); wuffs_brotli__encoder__poke_u32(self, a_wb, ((uint64_t)(self->private_impl.f_z_lit_off + ((uint64_t)(((uint64_t)(v_i + 1u)) * 4u)))), v_lc_i1); - v_literal_carry = wuffs_softfloat__arith__f32_sub(&self->private_data.f_sf, v_literal_carry, wuffs_softfloat__arith__f32_sub(&self->private_data.f_sf, v_lc_i1, v_lc_i)); + v_literal_carry = wuffs_brotli__bit_cost__f32_sub(&self->private_data.f_bc, v_literal_carry, wuffs_brotli__bit_cost__f32_sub(&self->private_data.f_bc, v_lc_i1, v_lc_i)); v_lc_i = v_lc_i1; v_i += 1u; } @@ -101462,7 +102448,7 @@ wuffs_brotli__encoder__z_literal_costs( wuffs_base__slice_u8 a_wb, uint64_t a_from, uint64_t a_to) { - return wuffs_softfloat__arith__f32_sub(&self->private_data.f_sf, wuffs_brotli__encoder__lit_cost_at(self, a_wb, a_to), wuffs_brotli__encoder__lit_cost_at(self, a_wb, a_from)); + return wuffs_brotli__bit_cost__f32_sub(&self->private_data.f_bc, wuffs_brotli__encoder__lit_cost_at(self, a_wb, a_to), wuffs_brotli__encoder__lit_cost_at(self, a_wb, a_from)); } // -------- func brotli.encoder.update_zopfli_node @@ -101527,7 +102513,7 @@ wuffs_brotli__encoder__start_pos_queue_push( while (v_i < v_len) { v_a = ((uint32_t)((v_offset & 7u))); v_b = ((uint32_t)((((uint64_t)(v_offset + 1u)) & 7u))); - if (wuffs_softfloat__arith__f32_gt(&self->private_data.f_sf, self->private_data.f_zq_costdiff[v_a], self->private_data.f_zq_costdiff[v_b]) != 0u) { + if (wuffs_brotli__bit_cost__f32_gt(&self->private_data.f_bc, self->private_data.f_zq_costdiff[v_a], self->private_data.f_zq_costdiff[v_b]) != 0u) { wuffs_brotli__encoder__start_pos_queue_swap(self, v_a, v_b); } v_offset += 1u; @@ -101596,10 +102582,10 @@ wuffs_brotli__encoder__compute_minimum_copy_length( v_len = 2u; v_next_len_bucket = 4u; v_next_len_offset = 10u; - while ((((uint64_t)(a_pos + v_len)) <= a_num_bytes) && (wuffs_softfloat__arith__f32_le(&self->private_data.f_sf, wuffs_brotli__encoder__zn_u(self, a_wb, ((uint64_t)(a_pos + v_len))), v_min_cost) != 0u)) { + while ((((uint64_t)(a_pos + v_len)) <= a_num_bytes) && (wuffs_brotli__bit_cost__f32_le(&self->private_data.f_bc, wuffs_brotli__encoder__zn_u(self, a_wb, ((uint64_t)(a_pos + v_len))), v_min_cost) != 0u)) { v_len += 1u; if (v_len == v_next_len_offset) { - v_min_cost = wuffs_softfloat__arith__f32_add(&self->private_data.f_sf, v_min_cost, 1065353216u); + v_min_cost = wuffs_brotli__bit_cost__f32_add(&self->private_data.f_bc, v_min_cost, 1065353216u); v_next_len_offset += v_next_len_bucket; v_next_len_bucket *= 2u; } @@ -101684,10 +102670,10 @@ wuffs_brotli__encoder__evaluate_node( a_pos, a_max_backward_limit)); v_lc = wuffs_brotli__encoder__z_literal_costs(self, a_wb, 0u, a_pos); - if (wuffs_softfloat__arith__f32_le(&self->private_data.f_sf, v_node_cost, v_lc) != 0u) { + if (wuffs_brotli__bit_cost__f32_le(&self->private_data.f_bc, v_node_cost, v_lc) != 0u) { self->private_impl.f_zp_pos = a_pos; self->private_impl.f_zp_cost = v_node_cost; - self->private_impl.f_zp_costdiff = wuffs_softfloat__arith__f32_sub(&self->private_data.f_sf, v_node_cost, v_lc); + self->private_impl.f_zp_costdiff = wuffs_brotli__bit_cost__f32_sub(&self->private_data.f_bc, v_node_cost, v_lc); wuffs_brotli__encoder__compute_distance_cache(self, a_wb, a_pos); wuffs_brotli__encoder__start_pos_queue_push(self); } @@ -101727,7 +102713,6 @@ wuffs_brotli__encoder__update_nodes( uint32_t v_base_cost = 0; uint64_t v_best_len = 0; uint64_t v_j = 0; - uint32_t v_idx = 0; uint64_t v_backward = 0; uint32_t v_bw32 = 0; uint64_t v_prev_ix = 0; @@ -101748,6 +102733,8 @@ wuffs_brotli__encoder__update_nodes( uint64_t v_max_match_len = 0; uint64_t v_len_code = 0; uint64_t v_gap = 0; + uint32_t v_dcs[4] = {0}; + uint32_t v_prev_copycode = 0; v_mask = ((uint64_t)(self->private_impl.f_rb_mask)); v_gap = self->private_impl.f_dict_total_size; @@ -101768,7 +102755,7 @@ wuffs_brotli__encoder__update_nodes( a_pos, a_max_backward_limit); v_q = wuffs_brotli__encoder__start_pos_queue_at(self, 0u); - v_min_cost = wuffs_softfloat__arith__f32_add(&self->private_data.f_sf, wuffs_softfloat__arith__f32_add(&self->private_data.f_sf, self->private_data.f_zq_cost[v_q], self->private_impl.f_z_min_cost_cmd), wuffs_brotli__encoder__z_literal_costs(self, a_wb, self->private_data.f_zq_pos[v_q], a_pos)); + v_min_cost = wuffs_brotli__bit_cost__f32_add(&self->private_data.f_bc, wuffs_brotli__bit_cost__f32_add(&self->private_data.f_bc, self->private_data.f_zq_cost[v_q], self->private_impl.f_z_min_cost_cmd), wuffs_brotli__encoder__z_literal_costs(self, a_wb, self->private_data.f_zq_pos[v_q], a_pos)); v_min_len = wuffs_brotli__encoder__compute_minimum_copy_length(self, a_wb, v_min_cost, @@ -101780,22 +102767,25 @@ wuffs_brotli__encoder__update_nodes( v_start = self->private_data.f_zq_pos[v_q]; v_inscode = wuffs_brotli__encoder__get_insert_length_code(self, ((uint64_t)(a_pos - v_start))); v_start_costdiff = self->private_data.f_zq_costdiff[v_q]; - v_base_cost = wuffs_softfloat__arith__f32_add(&self->private_data.f_sf, wuffs_softfloat__arith__f32_add(&self->private_data.f_sf, v_start_costdiff, wuffs_softfloat__arith__f32_from_u32(&self->private_data.f_sf, WUFFS_BROTLI__INS_EXTRA[wuffs_base__u32__min(v_inscode, 23u)])), wuffs_brotli__encoder__z_literal_costs(self, a_wb, 0u, a_pos)); + v_base_cost = wuffs_brotli__bit_cost__f32_add(&self->private_data.f_bc, wuffs_brotli__bit_cost__f32_add(&self->private_data.f_bc, v_start_costdiff, WUFFS_BROTLI__F32_OF_SMALL_INT[WUFFS_BROTLI__INS_EXTRA[wuffs_base__u32__min(v_inscode, 23u)]]), wuffs_brotli__encoder__z_literal_costs(self, a_wb, 0u, a_pos)); v_best_len = ((uint64_t)(v_min_len - 1u)); + v_dcs[0u] = self->private_data.f_zq_dc[(((uint32_t)(v_q)) << 2u)]; + v_dcs[1u] = self->private_data.f_zq_dc[((((uint32_t)(v_q)) << 2u) | 1u)]; + v_dcs[2u] = self->private_data.f_zq_dc[((((uint32_t)(v_q)) << 2u) | 2u)]; + v_dcs[3u] = self->private_data.f_zq_dc[((((uint32_t)(v_q)) << 2u) | 3u)]; + v_continuation = wuffs_brotli__encoder__rb_raw(self, a_wb, ((uint64_t)(v_cur_ix_masked + v_best_len)), 0u); v_j = 0u; while ((v_j < 16u) && (v_best_len < v_max_len)) { - v_idx = WUFFS_BROTLI__Z_DISTANCE_CACHE_INDEX[v_j]; - v_bw32 = ((uint32_t)(self->private_data.f_zq_dc[((((uint32_t)(v_q)) << 2u) | v_idx)] + WUFFS_BROTLI__Z_DISTANCE_CACHE_OFFSET[v_j])); + if (((uint64_t)(v_cur_ix_masked + v_best_len)) > v_mask) { + break; + } + v_bw32 = ((uint32_t)(v_dcs[WUFFS_BROTLI__Z_DISTANCE_CACHE_INDEX[v_j]] + WUFFS_BROTLI__Z_DISTANCE_CACHE_OFFSET[v_j])); v_backward = ((uint64_t)(v_bw32)); if (v_bw32 >= 2147483648u) { v_backward |= 18446744069414584320u; } v_prev_ix = ((uint64_t)(v_cur_ix - v_backward)); v_len = 0u; - v_continuation = wuffs_brotli__encoder__rb_raw(self, a_wb, ((uint64_t)(v_cur_ix_masked + v_best_len)), 0u); - if (((uint64_t)(v_cur_ix_masked + v_best_len)) > v_mask) { - break; - } if (v_backward > ((uint64_t)(v_dictionary_start + v_gap))) { v_j += 1u; continue; @@ -101831,18 +102821,22 @@ wuffs_brotli__encoder__update_nodes( v_j += 1u; continue; } - v_dist_cost = wuffs_softfloat__arith__f32_add(&self->private_data.f_sf, v_base_cost, self->private_data.f_z_cost_dist[v_j]); + v_dist_cost = wuffs_brotli__bit_cost__f32_add(&self->private_data.f_bc, v_base_cost, self->private_data.f_z_cost_dist[v_j]); v_l = ((uint64_t)(v_best_len + 1u)); + v_prev_copycode = 4294967295u; while (v_l <= v_len) { v_copycode = wuffs_brotli__encoder__get_copy_length_code(self, v_l); - v_cmdcode = wuffs_brotli__encoder__combine_length_codes(self, v_inscode, v_copycode, (v_j == 0u)); - if (v_cmdcode < 128u) { - v_cost = v_base_cost; - } else { - v_cost = v_dist_cost; + if (v_copycode != v_prev_copycode) { + v_prev_copycode = v_copycode; + v_cmdcode = wuffs_brotli__encoder__combine_length_codes(self, v_inscode, v_copycode, (v_j == 0u)); + if (v_cmdcode < 128u) { + v_cost = v_base_cost; + } else { + v_cost = v_dist_cost; + } + v_cost = wuffs_brotli__bit_cost__f32_add(&self->private_data.f_bc, wuffs_brotli__bit_cost__f32_add(&self->private_data.f_bc, v_cost, WUFFS_BROTLI__F32_OF_SMALL_INT[WUFFS_BROTLI__COPY_EXTRA[wuffs_base__u32__min(v_copycode, 23u)]]), self->private_data.f_z_cost_cmd[wuffs_base__u16__min(v_cmdcode, 703u)]); } - v_cost = wuffs_softfloat__arith__f32_add(&self->private_data.f_sf, wuffs_softfloat__arith__f32_add(&self->private_data.f_sf, v_cost, wuffs_softfloat__arith__f32_from_u32(&self->private_data.f_sf, WUFFS_BROTLI__COPY_EXTRA[wuffs_base__u32__min(v_copycode, 23u)])), self->private_data.f_z_cost_cmd[wuffs_base__u16__min(v_cmdcode, 703u)]); - if (wuffs_softfloat__arith__f32_lt(&self->private_data.f_sf, v_cost, wuffs_brotli__encoder__zn_u(self, a_wb, ((uint64_t)(a_pos + v_l)))) != 0u) { + if (wuffs_brotli__bit_cost__f32_lt(&self->private_data.f_bc, v_cost, wuffs_brotli__encoder__zn_u(self, a_wb, ((uint64_t)(a_pos + v_l)))) != 0u) { wuffs_brotli__encoder__update_zopfli_node(self, a_wb, a_pos, @@ -101857,6 +102851,7 @@ wuffs_brotli__encoder__update_nodes( v_best_len = v_l; v_l += 1u; } + v_continuation = wuffs_brotli__encoder__rb_raw(self, a_wb, ((uint64_t)(v_cur_ix_masked + v_best_len)), 0u); v_j += 1u; } if (v_k >= 2u) { @@ -101873,20 +102868,24 @@ wuffs_brotli__encoder__update_nodes( v_pe = wuffs_brotli__metablock_builder__prefix_encode_copy_distance(&self->private_data.f_mb, v_dist_code, ((uint64_t)(self->private_impl.f_num_direct_distance_codes)), self->private_impl.f_distance_postfix_bits); v_dist_symbol = ((uint32_t)((v_pe & 65535u))); v_distnumextra = (v_dist_symbol >> 10u); - v_dist_cost = wuffs_softfloat__arith__f32_add(&self->private_data.f_sf, wuffs_softfloat__arith__f32_add(&self->private_data.f_sf, v_base_cost, wuffs_softfloat__arith__f32_from_u32(&self->private_data.f_sf, v_distnumextra)), self->private_data.f_z_cost_dist[wuffs_brotli__encoder__min32_543(self, (v_dist_symbol & 1023u))]); + v_dist_cost = wuffs_brotli__bit_cost__f32_add(&self->private_data.f_bc, wuffs_brotli__bit_cost__f32_add(&self->private_data.f_bc, v_base_cost, wuffs_brotli__bit_cost__f32_from_u32(&self->private_data.f_bc, v_distnumextra)), self->private_data.f_z_cost_dist[wuffs_brotli__encoder__min32_543(self, (v_dist_symbol & 1023u))]); v_max_match_len = wuffs_brotli__encoder__bm_length(self, a_wb, v_moff); if ((v_len < v_max_match_len) && (v_is_dictionary_match || (v_max_match_len > v_max_zopfli_len))) { v_len = v_max_match_len; } + v_prev_copycode = 4294967295u; while (v_len <= v_max_match_len) { v_len_code = v_len; if (v_is_dictionary_match) { v_len_code = wuffs_brotli__encoder__bm_length_code(self, a_wb, v_moff); } v_copycode = wuffs_brotli__encoder__get_copy_length_code(self, v_len_code); - v_cmdcode = wuffs_brotli__encoder__combine_length_codes(self, v_inscode, v_copycode, false); - v_cost = wuffs_softfloat__arith__f32_add(&self->private_data.f_sf, wuffs_softfloat__arith__f32_add(&self->private_data.f_sf, v_dist_cost, wuffs_softfloat__arith__f32_from_u32(&self->private_data.f_sf, WUFFS_BROTLI__COPY_EXTRA[wuffs_base__u32__min(v_copycode, 23u)])), self->private_data.f_z_cost_cmd[wuffs_base__u16__min(v_cmdcode, 703u)]); - if (wuffs_softfloat__arith__f32_lt(&self->private_data.f_sf, v_cost, wuffs_brotli__encoder__zn_u(self, a_wb, ((uint64_t)(a_pos + v_len)))) != 0u) { + if (v_copycode != v_prev_copycode) { + v_prev_copycode = v_copycode; + v_cmdcode = wuffs_brotli__encoder__combine_length_codes(self, v_inscode, v_copycode, false); + v_cost = wuffs_brotli__bit_cost__f32_add(&self->private_data.f_bc, wuffs_brotli__bit_cost__f32_add(&self->private_data.f_bc, v_dist_cost, WUFFS_BROTLI__F32_OF_SMALL_INT[WUFFS_BROTLI__COPY_EXTRA[wuffs_base__u32__min(v_copycode, 23u)]]), self->private_data.f_z_cost_cmd[wuffs_base__u16__min(v_cmdcode, 703u)]); + } + if (wuffs_brotli__bit_cost__f32_lt(&self->private_data.f_bc, v_cost, wuffs_brotli__encoder__zn_u(self, a_wb, ((uint64_t)(a_pos + v_len)))) != 0u) { wuffs_brotli__encoder__update_zopfli_node(self, a_wb, a_pos, diff --git a/script/manual-test-brotli-float.c b/script/manual-test-brotli-float.c new file mode 100644 index 00000000..22953fc2 --- /dev/null +++ b/script/manual-test-brotli-float.c @@ -0,0 +1,370 @@ +// Copyright 2026 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. +// +// SPDX-License-Identifier: Apache-2.0 OR MIT + +// ---------------- + +// manual-test-brotli-float.c compares std/brotli's package-private softfloat +// fast paths (the bit_cost methods in std/brotli/encode_float.wuffs) against +// the C compiler's native (hardware) float and double arithmetic, for +// pseudo-random operands that are biased towards the fast paths' edge cases: +// every exponent difference, near-ties, cancellation, results near the +// subnormal and overflow thresholds, zeroes, subnormals, infinities and NaNs. +// Being static functions, they are called directly (this program #include's +// the Wuffs C file with WUFFS_CONFIG__STATIC_FUNCTIONS). +// +// Build it WITHOUT -ffast-math and WITHOUT -mfma / -march=native: +// gcc -O2 -std=c99 manual-test-brotli-float.c -o /tmp/brfloat +// /tmp/brfloat # Default: 100 million operations per op. +// /tmp/brfloat 1000000 # Or pass the number of operations per op. +// It prints one line per op and the first few mismatches (if any), and exits +// with a non-zero status if there were any mismatches. NaN results only need +// to be NaN (on both sides). + +#include +#include +#include +#include +#include +#include + +#define WUFFS_IMPLEMENTATION +#define WUFFS_CONFIG__STATIC_FUNCTIONS +#define WUFFS_CONFIG__MODULES +#define WUFFS_CONFIG__MODULE__BASE +#define WUFFS_CONFIG__MODULE__BROTLI +#define WUFFS_CONFIG__MODULE__SOFTFLOAT +#include "../release/c/wuffs-unsupported-snapshot.c" + +#if defined(__FAST_MATH__) || defined(__FMA__) +#error "Build without -ffast-math and without -mfma / -march=native" +#endif + +static wuffs_brotli__bit_cost* g_bc; + +static uint64_t g_rng = 0x123456789ABCDEF1ull; + +static uint64_t // +rnd(void) { + // xorshift64*. + g_rng ^= g_rng >> 12; + g_rng ^= g_rng << 25; + g_rng ^= g_rng >> 27; + return g_rng * 0x2545F4914F6CDD1Dull; +} + +static double // +d_of(uint64_t u) { + double d; + memcpy(&d, &u, 8); + return d; +} + +static uint64_t // +u_of_d(double d) { + uint64_t u; + memcpy(&u, &d, 8); + return u; +} + +static float // +f_of(uint32_t u) { + float f; + memcpy(&f, &u, 4); + return f; +} + +static uint32_t // +u_of_f(float f) { + uint32_t u; + memcpy(&u, &f, 4); + return u; +} + +// rnd_f32 returns a random f32 bit pattern. Usually it is a normal number +// whose exponent is near ref's (so that every exponent difference and lots +// of cancellation is exercised) and whose low fraction bits are often all +// zeroes or all ones (giving near-ties). +static uint32_t // +rnd_f32(uint32_t ref) { + uint64_t r = rnd(); + uint32_t sign = (uint32_t)(r & 1) << 31; + uint32_t frac = (uint32_t)(r >> 8) & 0x7FFFFF; + switch ((r >> 1) & 7) { + case 0: + frac &= ~0u << ((r >> 32) % 24); + break; + case 1: + frac |= ~(~0u << ((r >> 32) % 24)) & 0x7FFFFF; + break; + } + uint32_t exp; + switch ((r >> 4) & 15) { + case 0: + return (uint32_t)rnd(); // Anything, including NaNs. + case 1: + return sign | (frac & ((r >> 40) & 1 ? 0x7FFFFF : 0)); // Subnormal. + case 2: + return sign | 0x7F800000 | ((r >> 41) & 1 ? frac : 0); // Inf or NaN. + case 3: + exp = 1 + ((r >> 42) % 4); // Near the subnormal threshold. + break; + case 4: + exp = 250 + ((r >> 42) % 5); // Near the overflow threshold. + break; + case 5: + exp = 1 + ((r >> 42) % 254); + break; + default: { + int32_t e = (int32_t)((ref >> 23) & 0xFF) - 30 + (int32_t)((r >> 42) % 61); + exp = (e < 1) ? 1 : (e > 254) ? 254 : (uint32_t)e; + break; + } + } + return sign | (exp << 23) | frac; +} + +// rnd_f64 is like rnd_f32 but for f64. +static uint64_t // +rnd_f64(uint64_t ref) { + uint64_t r = rnd(); + uint64_t r2 = rnd(); + uint64_t sign = (r & 1) << 63; + uint64_t frac = r2 & 0xFFFFFFFFFFFFFull; + switch ((r >> 1) & 7) { + case 0: + frac &= ~0ull << ((r >> 32) % 53); + break; + case 1: + frac |= ~(~0ull << ((r >> 32) % 53)) & 0xFFFFFFFFFFFFFull; + break; + } + uint64_t exp; + switch ((r >> 4) & 15) { + case 0: + return rnd(); // Anything, including NaNs. + case 1: + return sign | (frac & ((r >> 40) & 1 ? 0xFFFFFFFFFFFFFull : 0)); + case 2: + return sign | 0x7FF0000000000000ull | ((r >> 41) & 1 ? frac : 0); + case 3: + exp = 1 + ((r >> 42) % 4); + break; + case 4: + exp = 2042 + ((r >> 42) % 5); + break; + case 5: + exp = 1 + ((r >> 42) % 2046); + break; + default: { + int64_t e = (int64_t)((ref >> 52) & 0x7FF) - 60 + (int64_t)((r >> 42) % 121); + exp = (e < 1) ? 1 : (e > 2046) ? 2046 : (uint64_t)e; + break; + } + } + return sign | (exp << 52) | frac; +} + +static uint64_t g_total_mismatches; + +#define CHECK32(name, have_expr, want_expr, is_cmp) \ + static void check_##name(uint64_t n) { \ + uint64_t mismatches = 0; \ + for (uint64_t i = 0; i < n; i++) { \ + uint32_t a = rnd_f32((uint32_t)rnd()); \ + uint32_t b = (rnd() & 3) ? rnd_f32(a) : a ^ ((rnd() & 1) << 31); \ + if (rnd() & 1) { \ + uint32_t t = a; \ + a = b; \ + b = t; \ + } \ + uint32_t have = (have_expr); \ + uint32_t want = (want_expr); \ + bool ok = (have == want) || \ + (!is_cmp && ((have & 0x7FFFFFFF) > 0x7F800000) && \ + ((want & 0x7FFFFFFF) > 0x7F800000)); \ + if (!ok && (mismatches++ < 5)) { \ + printf("%s(0x%08" PRIX32 ", 0x%08" PRIX32 "): have 0x%08" PRIX32 \ + ", want 0x%08" PRIX32 "\n", \ + #name, a, b, have, want); \ + } \ + } \ + printf("%-8s %12" PRIu64 " iterations, %8" PRIu64 " mismatches\n", \ + #name, n, mismatches); \ + g_total_mismatches += mismatches; \ + } + +#define CHECK64(name, have_expr, want_expr, is_cmp) \ + static void check_##name(uint64_t n) { \ + uint64_t mismatches = 0; \ + for (uint64_t i = 0; i < n; i++) { \ + uint64_t a = rnd_f64(rnd()); \ + uint64_t b = (rnd() & 3) ? rnd_f64(a) : a ^ ((rnd() & 1) << 63); \ + if (rnd() & 1) { \ + uint64_t t = a; \ + a = b; \ + b = t; \ + } \ + uint64_t have = (have_expr); \ + uint64_t want = (want_expr); \ + bool ok = (have == want) || \ + (!is_cmp && \ + ((have & 0x7FFFFFFFFFFFFFFFull) > 0x7FF0000000000000ull) && \ + ((want & 0x7FFFFFFFFFFFFFFFull) > 0x7FF0000000000000ull)); \ + if (!ok && (mismatches++ < 5)) { \ + printf("%s(0x%016" PRIX64 ", 0x%016" PRIX64 "): have 0x%016" PRIX64 \ + ", want 0x%016" PRIX64 "\n", \ + #name, a, b, have, want); \ + } \ + } \ + printf("%-8s %12" PRIu64 " iterations, %8" PRIu64 " mismatches\n", \ + #name, n, mismatches); \ + g_total_mismatches += mismatches; \ + } + +CHECK32(f32_add, + wuffs_brotli__bit_cost__f32_add(g_bc, a, b), + u_of_f(f_of(a) + f_of(b)), + false) +CHECK32(f32_sub, + wuffs_brotli__bit_cost__f32_sub(g_bc, a, b), + u_of_f(f_of(a) - f_of(b)), + false) +CHECK32(f32_lt, + wuffs_brotli__bit_cost__f32_lt(g_bc, a, b), + f_of(a) < f_of(b), + true) +CHECK32(f32_le, + wuffs_brotli__bit_cost__f32_le(g_bc, a, b), + f_of(a) <= f_of(b), + true) +CHECK32(f32_gt, + wuffs_brotli__bit_cost__f32_gt(g_bc, a, b), + f_of(a) > f_of(b), + true) +CHECK64(f64_add, + wuffs_brotli__bit_cost__f64_add(g_bc, a, b), + u_of_d(d_of(a) + d_of(b)), + false) +CHECK64(f64_sub, + wuffs_brotli__bit_cost__f64_sub(g_bc, a, b), + u_of_d(d_of(a) - d_of(b)), + false) +CHECK64(f64_mul, + wuffs_brotli__bit_cost__f64_mul(g_bc, a, b), + u_of_d(d_of(a) * d_of(b)), + false) +CHECK64(f64_lt, + wuffs_brotli__bit_cost__f64_lt(g_bc, a, b), + d_of(a) < d_of(b), + true) +CHECK64(f64_gt, + wuffs_brotli__bit_cost__f64_gt(g_bc, a, b), + d_of(a) > d_of(b), + true) +CHECK64(f64_ge, + wuffs_brotli__bit_cost__f64_ge(g_bc, a, b), + d_of(a) >= d_of(b), + true) + +// rnd_int returns a random integer, with a random number of significant +// bits (and often with many trailing zeroes). +static uint64_t // +rnd_int(void) { + uint64_t r = rnd(); + uint64_t x = rnd() >> (r % 64); + if (r & 0x100) { + x &= ~0ull << ((r >> 9) % 64); + } + return x; +} + +static void // +check_conversions(uint64_t n) { + uint64_t mismatches = 0; + for (uint64_t i = 0; i < n; i++) { + uint64_t x = rnd_int(); + uint32_t x32 = (uint32_t)x; + uint64_t have = wuffs_brotli__bit_cost__f64_from_u64(g_bc, x); + uint64_t want = u_of_d((double)x); + if ((have != want) && (mismatches++ < 5)) { + printf("f64_from_u64(0x%016" PRIX64 "): have 0x%016" PRIX64 + ", want 0x%016" PRIX64 "\n", x, have, want); + } + have = wuffs_brotli__bit_cost__f64_from_u32(g_bc, x32); + want = u_of_d((double)x32); + if ((have != want) && (mismatches++ < 5)) { + printf("f64_from_u32(0x%08" PRIX32 "): have 0x%016" PRIX64 + ", want 0x%016" PRIX64 "\n", x32, have, want); + } + have = wuffs_brotli__bit_cost__f32_from_u32(g_bc, x32); + want = u_of_f((float)x32); + if ((have != want) && (mismatches++ < 5)) { + printf("f32_from_u32(0x%08" PRIX32 "): have 0x%08" PRIX32 + ", want 0x%08" PRIX32 "\n", x32, (uint32_t)have, (uint32_t)want); + } + // f64_to_u64_trunc, for (non-negative) values that fit in a u64. + uint64_t a = rnd_f64(rnd()) & 0x7FFFFFFFFFFFFFFFull; + if ((rnd() & 3) == 0) { + a = u_of_d((double)x) + (rnd() % 3) - 1; + } + if (d_of(a) < 18446744073709551616.0) { + have = wuffs_brotli__bit_cost__f64_to_u64_trunc(g_bc, a); + want = (uint64_t)d_of(a); + if ((have != want) && (mismatches++ < 5)) { + printf("f64_to_u64_trunc(0x%016" PRIX64 "): have 0x%016" PRIX64 + ", want 0x%016" PRIX64 "\n", a, have, want); + } + } + } + for (uint32_t v = 0; v < 32; v++) { + if ((WUFFS_BROTLI__F32_OF_SMALL_INT[v] != u_of_f((float)v)) && + (mismatches++ < 5)) { + printf("F32_OF_SMALL_INT[%" PRIu32 "] is wrong\n", v); + } + } + printf("%-8s %12" PRIu64 " iterations, %8" PRIu64 " mismatches\n", "convert", + n, mismatches); + g_total_mismatches += mismatches; +} + +int // +main(int argc, char** argv) { + uint64_t n = 100000000; + if (argc > 1) { + n = strtoull(argv[1], NULL, 10); + } + wuffs_brotli__encoder* enc = wuffs_brotli__encoder__alloc(); + if (!enc) { + printf("alloc failed\n"); + return 1; + } + g_bc = &enc->private_data.f_bc; + + check_f32_add(n); + check_f32_sub(n); + check_f32_lt(n); + check_f32_le(n); + check_f32_gt(n); + check_f64_add(n); + check_f64_sub(n); + check_f64_mul(n); + check_f64_lt(n); + check_f64_gt(n); + check_f64_ge(n); + check_conversions(n); + + free(enc); + if (g_total_mismatches) { + printf("FAIL: %" PRIu64 " mismatches in total\n", g_total_mismatches); + return 1; + } + printf("PASS: 0 mismatches in total\n"); + return 0; +} diff --git a/std/brotli/README.md b/std/brotli/README.md index 5b7e1509..ba407481 100644 --- a/std/brotli/README.md +++ b/std/brotli/README.md @@ -880,6 +880,10 @@ round trips through the decoder. Its large-window tests cover qualities 3 to 9 with lgwin 18 to 30 (and the H35, H55 and H65 hashers) on small files and on a 46 MiB input with repeats at distances of about 20 and 40 MiB, decoded by both this package's decoder and libbrotlidec. +`script/manual-test-brotli-float.c` checks the encoder's package-private +float fast paths (`encode_float.wuffs`, which spare the hot loops the +cross-package `std/softfloat` calls) against the C compiler's hardware float +and double arithmetic, for operands biased towards their edge cases. `script/manual-test-brotli-encoder-parity.c` runs a larger grid over every file in `test/data/brotli`, some larger files in `test/data` (up to about 400 KiB) and some synthetic inputs (up to 1 MiB). Its `-q=LO-HI` and @@ -1127,12 +1131,13 @@ metablock) for as long as it needs the results. - `workbuf_len` is a worst case (or, at qualities 10 and 11, a practical bound). A future option could take the input length (like the C code's size hint) to give a tighter bound. -- Speed. Compressing the 3 MiB `test/data/harvesters.bmp` (with - `example/brcat` on x86-64, lgwin 22), the Wuffs encoder takes about 1.2 - times libbrotlienc's time at quality 5, 1.7 times at quality 9 and 6 times - at quality 11, mostly because of the soft float arithmetic in the Zopfli - cost model, the block splitter and histogram clustering. Decoding takes - about 1.2 times libbrotlidec's time. +- Speed. Compressing the 3 MiB `test/data/harvesters.bmp` in one call (on + x86-64, lgwin 22), the Wuffs encoder takes about 1.2 times libbrotlienc's + time at quality 5, 1.7 times at quality 9, 4 times at quality 10 and 3.5 + times at quality 11 (on text, 2 to 3 times at qualities 10 and 11), mostly + because of the soft float arithmetic in the Zopfli cost model, the block + splitter and histogram clustering. Decoding takes about 1.2 times + libbrotlidec's time. - Serialized (shared) dictionaries, which only an experimental (`BROTLI_EXPERIMENTAL`) libbrotlienc build supports. - Changing parameters mid-stream. diff --git a/std/brotli/encode_bit_cost.wuffs b/std/brotli/encode_bit_cost.wuffs index 81b157d4..9d97b710 100644 --- a/std/brotli/encode_bit_cost.wuffs +++ b/std/brotli/encode_bit_cost.wuffs @@ -54,8 +54,18 @@ pri struct bit_cost?( // (but log2_cache need not be), so the cache starts empty. log2_valid : array[1024] base.u64, + // population_cost's per-call memo of its per-symbol values, for + // symbol counts less than 256: pc_term[c] and pc_depth[c] are valid + // during the call numbered pc_call if pc_valid[c] == pc_call. Call + // numbers start at 1, so the zeroed pc_valid starts empty. + pc_call : base.u32, + pc_valid : array[256] base.u32, + sf : softfloat.arith, ) + ( + pc_term : array[256] base.u64, + pc_depth : array[256] base.u8, + // log2_cache[v] is FastLog2(v), for 256 <= v < 65536, filled lazily // (see fast_log2_memo). It is only a speed optimization (glibc's log2 // is expensive to emulate): the values are computed by the same @@ -166,17 +176,22 @@ pri func bit_cost.bits_entropy!(population: roslice base.u8, size: base.u64) bas q = q[4 ..] } sum ~mod+= p + if p <= 1 { + // FastLog2(p) is +0, so (p * FastLog2(p)) is +0 and subtracting + // it leaves retval (which is never a NaN) unchanged. + continue + } lp = this.fast_log2_memo!(v: p) - retval = this.sf.f64_sub(a: retval, b: this.sf.f64_mul( - a: this.sf.f64_from_u64(a: p), b: lp)) + retval = this.f64_sub(a: retval, b: this.f64_mul( + a: this.f64_from_u64(a: p), b: lp)) } if sum <> 0 { lp = this.fast_log2_memo!(v: sum) - retval = this.sf.f64_add(a: retval, b: this.sf.f64_mul( - a: this.sf.f64_from_u64(a: sum), b: lp)) + retval = this.f64_add(a: retval, b: this.f64_mul( + a: this.f64_from_u64(a: sum), b: lp)) } - if this.sf.f64_lt(a: retval, b: this.sf.f64_from_u64(a: sum)) <> 0 { - retval = this.sf.f64_from_u64(a: sum) + if this.f64_lt(a: retval, b: this.f64_from_u64(a: sum)) <> 0 { + retval = this.f64_from_u64(a: sum) } return retval } @@ -184,6 +199,7 @@ pri func bit_cost.bits_entropy!(population: roslice base.u8, size: base.u64) bas // Mirrors BrotliPopulationCostLiteral, BrotliPopulationCostCommand and // BrotliPopulationCostDistance (FN(BrotliPopulationCost) in bit_cost_inc.h). // histogram is a histogram of data_size symbols (see the layout above). + pri func bit_cost.population_cost!(histogram: roslice base.u8, data_size: base.u32[..= 704]) base.u64 { var total : base.u64 var count : base.u32 @@ -209,6 +225,7 @@ pri func bit_cost.population_cost!(histogram: roslice base.u8, data_size: base.u var sum : base.u64 var retval : base.u64 var p : base.u64 + var term : base.u64 total = this.ld64(s: args.histogram, i: (args.data_size as base.u64) * 4) if total == 0 { @@ -234,7 +251,7 @@ pri func bit_cost.population_cost!(histogram: roslice base.u8, data_size: base.u } if count == 2 { // kTwoSymbolHistogramCost + (double)total_count_. - return this.sf.f64_add(a: 0x4034_0000_0000_0000, b: this.sf.f64_from_u64(a: total)) + return this.f64_add(a: 0x4034_0000_0000_0000, b: this.f64_from_u64(a: total)) } if count == 3 { h0 = this.ld32(s: args.histogram, i: (s[0] as base.u64) * 4) @@ -243,11 +260,11 @@ pri func bit_cost.population_cost!(histogram: roslice base.u8, data_size: base.u histomax = h0.max(no_less_than: h1.max(no_less_than: h2)) // kThreeSymbolHistogramCost + 2 * (histo0 + histo1 + histo2) - // histomax, where the integer expression is uint32_t arithmetic. - return this.sf.f64_sub( - a: this.sf.f64_add( + return this.f64_sub( + a: this.f64_add( a: 0x403C_0000_0000_0000, - b: this.sf.f64_from_u32(a: 2 ~mod* ((h0 ~mod+ h1) ~mod+ h2))), - b: this.sf.f64_from_u32(a: histomax)) + b: this.f64_from_u32(a: 2 ~mod* ((h0 ~mod+ h1) ~mod+ h2))), + b: this.f64_from_u32(a: histomax)) } if count == 4 { i = 0 @@ -273,28 +290,48 @@ pri func bit_cost.population_cost!(histogram: roslice base.u8, data_size: base.u histomax = h23.max(no_less_than: histo[0]) // kFourSymbolHistogramCost + 3 * h23 + 2 * (histo[0] + histo[1]) - // histomax, evaluated left to right. - return this.sf.f64_sub( - a: this.sf.f64_add( - a: this.sf.f64_add( + return this.f64_sub( + a: this.f64_add( + a: this.f64_add( a: 0x4042_8000_0000_0000, - b: this.sf.f64_from_u32(a: 3 ~mod* h23)), - b: this.sf.f64_from_u32(a: 2 ~mod* (histo[0] ~mod+ histo[1]))), - b: this.sf.f64_from_u32(a: histomax)) + b: this.f64_from_u32(a: 3 ~mod* h23)), + b: this.f64_from_u32(a: 2 ~mod* (histo[0] ~mod+ histo[1]))), + b: this.f64_from_u32(a: histomax)) } // Compute the entropy of the histogram and simultaneously build a // simplified histogram of the code length codes. max_depth = 1 log2total = this.fast_log2_memo!(v: total) + this.pc_call ~mod+= 1 + if this.pc_call == 0 { + this.pc_valid[.. 256].bulk_memset!(byte_value: 0) + this.pc_call = 1 + } i = 0 while i < args.data_size { c = this.ld32(s: args.histogram, i: (i as base.u64) * 4) if c > 0 { - lp = this.fast_log2_memo!(v: c as base.u64) - log2p = this.sf.f64_sub(a: log2total, b: lp) - depth = this.sf.f64_to_u64_trunc(a: this.sf.f64_add(a: log2p, b: 0x3FE0_0000_0000_0000)) - bits = this.sf.f64_add(a: bits, b: this.sf.f64_mul( - a: this.sf.f64_from_u32(a: c), b: log2p)) + // term is (c * log2p) and depth is (size_t)(log2p + 0.5) (capped + // at 15), which only depend on c (and the fixed total). + if (c < 256) and (this.pc_valid[c & 255] == this.pc_call) { + term = this.pc_term[c & 255] + depth = this.pc_depth[c & 255] as base.u64 + } else { + lp = this.fast_log2_memo!(v: c as base.u64) + log2p = this.f64_sub(a: log2total, b: lp) + depth = this.f64_to_u64_trunc(a: this.f64_add(a: log2p, b: 0x3FE0_0000_0000_0000)) + term = this.f64_mul(a: this.f64_from_u32(a: c), b: log2p) + if depth > 15 { + depth = 15 + } + if c < 256 { + this.pc_valid[c & 255] = this.pc_call + this.pc_term[c & 255] = term + this.pc_depth[c & 255] = (depth & 15) as base.u8 + } + } + bits = this.f64_add(a: bits, b: term) if depth > 15 { depth = 15 } @@ -326,14 +363,14 @@ pri func bit_cost.population_cost!(histogram: roslice base.u8, data_size: base.u while reps > 0 { depth_histo[17] ~mod+= 1 // Add the 3 extra bits for the 17 code length code. - bits = this.sf.f64_add(a: bits, b: 0x4008_0000_0000_0000) + bits = this.f64_add(a: bits, b: 0x4008_0000_0000_0000) reps >>= 3 } } } } // Add the estimated encoding cost of the code length code histogram. - bits = this.sf.f64_add(a: bits, b: this.sf.f64_from_u64(a: 18 + (2 * max_depth))) + bits = this.f64_add(a: bits, b: this.f64_from_u64(a: 18 + (2 * max_depth))) // Add the entropy of the code length code histogram. This is // BrotliBitsEntropy(depth_histo, BROTLI_CODE_LENGTH_CODES), inlined @@ -342,18 +379,22 @@ pri func bit_cost.population_cost!(histogram: roslice base.u8, data_size: base.u while i < 18 { p = depth_histo[i] as base.u64 sum ~mod+= p - lp = this.fast_log2_memo!(v: p) - retval = this.sf.f64_sub(a: retval, b: this.sf.f64_mul( - a: this.sf.f64_from_u64(a: p), b: lp)) i ~mod+= 1 + if p <= 1 { + // As in bits_entropy, this would leave retval unchanged. + continue + } + lp = this.fast_log2_memo!(v: p) + retval = this.f64_sub(a: retval, b: this.f64_mul( + a: this.f64_from_u64(a: p), b: lp)) } if sum <> 0 { lp = this.fast_log2_memo!(v: sum) - retval = this.sf.f64_add(a: retval, b: this.sf.f64_mul( - a: this.sf.f64_from_u64(a: sum), b: lp)) + retval = this.f64_add(a: retval, b: this.f64_mul( + a: this.f64_from_u64(a: sum), b: lp)) } - if this.sf.f64_lt(a: retval, b: this.sf.f64_from_u64(a: sum)) <> 0 { - retval = this.sf.f64_from_u64(a: sum) + if this.f64_lt(a: retval, b: this.f64_from_u64(a: sum)) <> 0 { + retval = this.f64_from_u64(a: sum) } - return this.sf.f64_add(a: bits, b: retval) + return this.f64_add(a: bits, b: retval) } diff --git a/std/brotli/encode_bit_writer.wuffs b/std/brotli/encode_bit_writer.wuffs index 79fdead4..d6acd49f 100644 --- a/std/brotli/encode_bit_writer.wuffs +++ b/std/brotli/encode_bit_writer.wuffs @@ -175,16 +175,16 @@ pri func encoder.bits_entropy_of_histogram() base.u64 { while i < 256 { p = this.histogram[i] as base.u64 sum ~mod+= p - retval = this.sf.f64_sub(a: retval, b: this.sf.f64_mul( - a: this.sf.f64_from_u64(a: p), b: this.fast_log2(v: p))) + retval = this.bc.f64_sub(a: retval, b: this.bc.f64_mul( + a: this.bc.f64_from_u64(a: p), b: this.fast_log2(v: p))) i += 1 } if sum <> 0 { - retval = this.sf.f64_add(a: retval, b: this.sf.f64_mul( - a: this.sf.f64_from_u64(a: sum), b: this.fast_log2(v: sum))) + retval = this.bc.f64_add(a: retval, b: this.bc.f64_mul( + a: this.bc.f64_from_u64(a: sum), b: this.fast_log2(v: sum))) } - if this.sf.f64_lt(a: retval, b: this.sf.f64_from_u64(a: sum)) <> 0 { - retval = this.sf.f64_from_u64(a: sum) + if this.bc.f64_lt(a: retval, b: this.bc.f64_from_u64(a: sum)) <> 0 { + retval = this.bc.f64_from_u64(a: sum) } return retval } diff --git a/std/brotli/encode_block_splitter.wuffs b/std/brotli/encode_block_splitter.wuffs index b13173e7..f93446af 100644 --- a/std/brotli/encode_block_splitter.wuffs +++ b/std/brotli/encode_block_splitter.wuffs @@ -295,6 +295,13 @@ pri func metablock_builder.find_blocks!(wb: slice base.u8, data: base.u64, lengt var c : base.u64 var cur_id : base.u8 var b : base.u8 + var nh : base.u64[..= 256] + var icn : base.u64 + var best_k : base.u64 + var use_capped : base.bool + var skip_t : base.u64 + var t : base.u64 + var n : base.u32 alphabet_size = args.ds as base.u64 bitmap_len = (args.num_histograms ~mod+ 7) >> 3 @@ -330,7 +337,7 @@ pri func metablock_builder.find_blocks!(wb: slice base.u8, data: base.u64, lengt k = (i ~mod* args.num_histograms) ~mod+ j if k < 35200 { c = this.bit_cost!(count: this.ld32(wb: args.wb, i: (args.histograms ~mod+ (j ~mod* hs)) ~mod+ (i ~mod* 4)) as base.u64) - this.insert_cost[k] = this.sf.f64_sub(a: this.insert_cost[j & 255], b: c) + this.insert_cost[k] = this.bc.f64_sub(a: this.insert_cost[j & 255], b: c) } j ~mod+= 1 } @@ -344,48 +351,121 @@ pri func metablock_builder.find_blocks!(wb: slice base.u8, data: base.u64, lengt // last position, we need to switch here. this.cost[.. 256].bulk_memset!(byte_value: 0) this.zero!(wb: args.wb, i: args.switch_signal, n: args.length ~mod* bitmap_len) + nh = args.num_histograms.min(no_more_than: 256) byte_ix = 0 while byte_ix < args.length { ix = byte_ix ~mod* bitmap_len - insert_cost_ix = this.sym(wb: args.wb, base: args.data, i: byte_ix, width: args.width) ~mod* args.num_histograms + insert_cost_ix = this.sym(wb: args.wb, base: args.data, i: byte_ix, width: args.width) ~mod* nh min_cost = F64_1E99 block_switch_cost = args.block_switch_bitcost + // insert_cost_ix is (symbol * nh) for a symbol less than + // alphabet_size, so (insert_cost_ix + nh) is at most 35200. + // Otherwise (which never happens), like the C code, the symbol's + // insert costs would be out of bounds (and here, they are zero). + icn = 0 + if insert_cost_ix <= 35200 { + icn = nh.min(no_more_than: 35200 - insert_cost_ix) + } + // From byte 2000 on, block_switch_cost is block_switch_bitcost, so + // the many cost[k] values that the previous byte capped at it all add + // the same (block_switch_bitcost + insert_cost[i]) values. Memoize + // those sums (the same f64_add, on the same operands) in + // capped_plus, if there are enough remaining bytes to pay for it. + if (byte_ix == 2000) and ((args.length ~mod- 2000) > alphabet_size) and (nh >= 4) { + t = alphabet_size ~mod* nh + t = t.min(no_more_than: 35200) + i = 0 + while i < t { + this.capped_plus[i.min(no_more_than: 35199)] = this.bc.f64_add( + a: args.block_switch_bitcost, b: this.insert_cost[i.min(no_more_than: 35199)]) + i ~mod+= 1 + } + use_capped = true + } + best_k = 0xFFFF_FFFF_FFFF_FFFF k = 0 - while k < args.num_histograms { + while k < nh { // We are coding the symbol with entropy code k. - c = 0 - if (insert_cost_ix ~mod+ k) < 35200 { - c = this.insert_cost[(insert_cost_ix ~mod+ k) % 35200] + if k < icn { + j = insert_cost_ix ~mod+ k + j = j.min(no_more_than: 35199) + if use_capped and (this.cost[k & 255] == args.block_switch_bitcost) { + c = this.capped_plus[j] + } else { + c = this.bc.f64_add(a: this.cost[k & 255], b: this.insert_cost[j]) + } + } else { + c = this.bc.f64_add(a: this.cost[k & 255], b: 0) } - c = this.sf.f64_add(a: this.cost[k & 255], b: c) this.cost[k & 255] = c - if this.sf.f64_lt(a: c, b: min_cost) <> 0 { + if this.bc.f64_lt(a: c, b: min_cost) <> 0 { min_cost = c - this.st8!(wb: args.wb, i: args.block_id ~mod+ byte_ix, v: (k & 0xFF) as base.u8) + best_k = k } k ~mod+= 1 } + if best_k <= 0xFF { + this.st8!(wb: args.wb, i: args.block_id ~mod+ byte_ix, v: (best_k & 0xFF) as base.u8) + } // More blocks for the beginning. if byte_ix < 2000 { // block_switch_cost *= 0.77 + multiplier * (double)byte_ix, with // multiplier = 0.07 / 2000. - block_switch_cost = this.sf.f64_mul( + block_switch_cost = this.bc.f64_mul( a: block_switch_cost, - b: this.sf.f64_add( + b: this.bc.f64_add( a: 0x3FE8_A3D7_0A3D_70A4, - b: this.sf.f64_mul(a: 0x3F02_599E_D7C6_FBD3, b: this.sf.f64_from_u64(a: byte_ix)))) + b: this.bc.f64_mul(a: 0x3F02_599E_D7C6_FBD3, b: this.bc.f64_from_u64(a: byte_ix)))) + } + // S(c) = (c - min_cost), rounded, is monotone non-decreasing in c, so + // the c values whose S(c) is at least block_switch_cost (the cost[k] + // values that the loop below caps) are those at least some threshold + // T. Find T by stepping (by one ULP) from (block_switch_cost + + // min_cost) and, if that takes only a few steps, skip the + // subtraction for every c >= T. skip_t holds T, or 0xFFFF_FFFF_FFFF_FFFF + // (which no non-negative, non-NaN c is at least) to skip nothing. + skip_t = 0xFFFF_FFFF_FFFF_FFFF + if (nh >= 8) and (min_cost < 0x7FF0_0000_0000_0000) and + ((block_switch_cost ~mod- 1) < 0x7FEF_FFFF_FFFF_FFFF) { + // min_cost is non-negative and finite, and block_switch_cost is + // positive and finite. + t = this.bc.f64_add(a: block_switch_cost, b: min_cost) + n = 0 + while (n < 3) and (t < 0x7FF0_0000_0000_0000) and + (this.bc.f64_ge(a: this.bc.f64_sub(a: t, b: min_cost), b: block_switch_cost) == 0) { + t ~mod+= 1 + n += 1 + } + if (t < 0x7FF0_0000_0000_0000) and + (this.bc.f64_ge(a: this.bc.f64_sub(a: t, b: min_cost), b: block_switch_cost) <> 0) { + // S(t) >= block_switch_cost. Step down while S(t - 1) is too. + n = 0 + while (n < 3) and (t > 0) and + (this.bc.f64_ge(a: this.bc.f64_sub(a: t ~mod- 1, b: min_cost), b: block_switch_cost) <> 0) { + t ~mod-= 1 + n += 1 + } + if (t == 0) or + (this.bc.f64_ge(a: this.bc.f64_sub(a: t ~mod- 1, b: min_cost), b: block_switch_cost) == 0) { + skip_t = t + } + } } k = 0 - while k < args.num_histograms { - c = this.sf.f64_sub(a: this.cost[k & 255], b: min_cost) + while k < nh { + c = this.cost[k & 255] + if (c <= 0x7FF0_0000_0000_0000) and (c >= skip_t) { + // S(c) >= S(T) >= block_switch_cost. + c = block_switch_cost + } else { + c = this.bc.f64_sub(a: c, b: min_cost) + } this.cost[k & 255] = c - if this.sf.f64_ge(a: c, b: block_switch_cost) <> 0 { + if this.bc.f64_ge(a: c, b: block_switch_cost) <> 0 { this.cost[k & 255] = block_switch_cost j = ix ~mod+ (k >> 3) - if true { - b = this.ld8(wb: args.wb, i: args.switch_signal ~mod+ j) - this.st8!(wb: args.wb, i: args.switch_signal ~mod+ j, v: b | ((1 as base.u8) << (k & 7))) - } + b = this.ld8(wb: args.wb, i: args.switch_signal ~mod+ j) + this.st8!(wb: args.wb, i: args.switch_signal ~mod+ j, v: b | ((1 as base.u8) << (k & 7))) } k ~mod+= 1 } @@ -648,7 +728,7 @@ pri func metablock_builder.cluster_blocks!(wb: slice base.u8, cat: base.u32[..= s = this.ld32(wb: args.wb, i: clusters ~mod+ (j ~mod* 4)) cur_bits = this.histogram_bit_cost_distance!(wb: args.wb, ds: args.ds, histogram: tmp, candidate: all_histograms ~mod+ ((s as base.u64) ~mod* hs), tmp: tmp ~mod+ hs) - if this.sf.f64_lt(a: cur_bits, b: best_bits) <> 0 { + if this.bc.f64_lt(a: cur_bits, b: best_bits) <> 0 { best_bits = cur_bits best_out = s } diff --git a/std/brotli/encode_cluster.wuffs b/std/brotli/encode_cluster.wuffs index 805bd04b..f649e75e 100644 --- a/std/brotli/encode_cluster.wuffs +++ b/std/brotli/encode_cluster.wuffs @@ -26,7 +26,7 @@ pri func metablock_builder.histogram_pair_is_less( p1_cost_diff: base.u64, p1_idx1: base.u32, p1_idx2: base.u32, p2_cost_diff: base.u64, p2_idx1: base.u32, p2_idx2: base.u32) base.bool { if this.sf.f64_ne(a: args.p1_cost_diff, b: args.p2_cost_diff) <> 0 { - return this.sf.f64_gt(a: args.p1_cost_diff, b: args.p2_cost_diff) <> 0 + return this.bc.f64_gt(a: args.p1_cost_diff, b: args.p2_cost_diff) <> 0 } return (args.p1_idx2 ~mod- args.p1_idx1) > (args.p2_idx2 ~mod- args.p2_idx1) } @@ -36,11 +36,11 @@ pri func metablock_builder.cluster_cost_diff(size_a: base.u64, size_b: base.u64) var size_c : base.u64 size_c = args.size_a ~mod+ args.size_b - return this.sf.f64_sub( - a: this.sf.f64_add( - a: this.sf.f64_mul(a: this.sf.f64_from_u64(a: args.size_a), b: this.bc.fast_log2(v: args.size_a)), - b: this.sf.f64_mul(a: this.sf.f64_from_u64(a: args.size_b), b: this.bc.fast_log2(v: args.size_b))), - b: this.sf.f64_mul(a: this.sf.f64_from_u64(a: size_c), b: this.bc.fast_log2(v: size_c))) + return this.bc.f64_sub( + a: this.bc.f64_add( + a: this.bc.f64_mul(a: this.bc.f64_from_u64(a: args.size_a), b: this.bc.fast_log2(v: args.size_a)), + b: this.bc.f64_mul(a: this.bc.f64_from_u64(a: args.size_b), b: this.bc.fast_log2(v: args.size_b))), + b: this.bc.f64_mul(a: this.bc.f64_from_u64(a: size_c), b: this.bc.fast_log2(v: size_c))) } // pair_copy is the struct assignment pairs[dst] = pairs[src], where dst and @@ -88,11 +88,11 @@ pri func metablock_builder.compare_and_push_to_queue!(wb: slice base.u8, ds: bas } h1 = args.out ~mod+ ((idx1 as base.u64) ~mod* hs) h2 = args.out ~mod+ ((idx2 as base.u64) ~mod* hs) - cost_diff = this.sf.f64_mul(a: 0x3FE0_0000_0000_0000, b: this.cluster_cost_diff( + cost_diff = this.bc.f64_mul(a: 0x3FE0_0000_0000_0000, b: this.cluster_cost_diff( size_a: this.ld32(wb: args.wb, i: args.cluster_size ~mod+ ((idx1 as base.u64) ~mod* 4)) as base.u64, size_b: this.ld32(wb: args.wb, i: args.cluster_size ~mod+ ((idx2 as base.u64) ~mod* 4)) as base.u64)) - cost_diff = this.sf.f64_sub(a: cost_diff, b: this.h_bit_cost(wb: args.wb, h: h1, ds: args.ds)) - cost_diff = this.sf.f64_sub(a: cost_diff, b: this.h_bit_cost(wb: args.wb, h: h2, ds: args.ds)) + cost_diff = this.bc.f64_sub(a: cost_diff, b: this.h_bit_cost(wb: args.wb, h: h1, ds: args.ds)) + cost_diff = this.bc.f64_sub(a: cost_diff, b: this.h_bit_cost(wb: args.wb, h: h2, ds: args.ds)) if this.h_total(wb: args.wb, h: h1, ds: args.ds) == 0 { cost_combo = this.h_bit_cost(wb: args.wb, h: h2, ds: args.ds) @@ -109,13 +109,13 @@ pri func metablock_builder.compare_and_push_to_queue!(wb: slice base.u8, ds: bas this.h_copy!(wb: args.wb, h: args.tmp, v: h1, ds: args.ds) this.h_add_histogram!(wb: args.wb, h: args.tmp, v: h2, ds: args.ds) c = this.h_population_cost!(wb: args.wb, h: args.tmp, ds: args.ds) - if this.sf.f64_lt(a: c, b: this.sf.f64_sub(a: threshold, b: cost_diff)) <> 0 { + if this.bc.f64_lt(a: c, b: this.bc.f64_sub(a: threshold, b: cost_diff)) <> 0 { cost_combo = c is_good_pair = true } } if is_good_pair { - cost_diff = this.sf.f64_add(a: cost_diff, b: cost_combo) + cost_diff = this.bc.f64_add(a: cost_diff, b: cost_combo) if (this.num_pairs > 0) and this.histogram_pair_is_less( p1_cost_diff: this.ld64(wb: args.wb, i: args.pairs ~mod+ 16), p1_idx1: this.ld32(wb: args.wb, i: args.pairs), @@ -182,7 +182,7 @@ pri func metablock_builder.histogram_combine!(wb: slice base.u8, ds: base.u32[.. } while num_clusters > min_cluster_size { - if this.sf.f64_ge(a: this.ld64(wb: args.wb, i: args.pairs ~mod+ 16), b: cost_diff_threshold) <> 0 { + if this.bc.f64_ge(a: this.ld64(wb: args.wb, i: args.pairs ~mod+ 16), b: cost_diff_threshold) <> 0 { cost_diff_threshold = F64_1E99 min_cluster_size = args.max_clusters continue @@ -279,7 +279,7 @@ pri func metablock_builder.histogram_bit_cost_distance!(wb: slice base.u8, ds: b this.h_copy!(wb: args.wb, h: args.tmp, v: args.histogram, ds: args.ds) this.h_add_histogram!(wb: args.wb, h: args.tmp, v: args.candidate, ds: args.ds) c = this.h_population_cost!(wb: args.wb, h: args.tmp, ds: args.ds) - return this.sf.f64_sub(a: c, b: this.h_bit_cost(wb: args.wb, h: args.candidate, ds: args.ds)) + return this.bc.f64_sub(a: c, b: this.h_bit_cost(wb: args.wb, h: args.candidate, ds: args.ds)) } // Mirrors FN(BrotliHistogramRemap) in cluster_inc.h. @@ -310,7 +310,7 @@ pri func metablock_builder.histogram_remap!(wb: slice base.u8, ds: base.u32[..= cur_bits = this.histogram_bit_cost_distance!(wb: args.wb, ds: args.ds, histogram: args.in_histos ~mod+ (i ~mod* hs), candidate: args.out ~mod+ ((c as base.u64) ~mod* hs), tmp: args.tmp) - if this.sf.f64_lt(a: cur_bits, b: best_bits) <> 0 { + if this.bc.f64_lt(a: cur_bits, b: best_bits) <> 0 { best_bits = cur_bits best_out = c } diff --git a/std/brotli/encode_data.wuffs b/std/brotli/encode_data.wuffs index 5f7f61b2..a5ff22a2 100644 --- a/std/brotli/encode_data.wuffs +++ b/std/brotli/encode_data.wuffs @@ -553,12 +553,12 @@ pri func encoder.estimate_entropy(off: base.u32, size: base.u32) base.u64 { while i < args.size { p = this.ctx_histo[(args.off ~mod+ i) & 511] total ~mod+= p as base.u64 - result = this.sf.f64_add(a: result, b: this.sf.f64_mul( - a: this.sf.f64_from_u32(a: p), b: this.bc.fast_log2(v: p as base.u64))) + result = this.bc.f64_add(a: result, b: this.bc.f64_mul( + a: this.bc.f64_from_u32(a: p), b: this.bc.fast_log2(v: p as base.u64))) i ~mod+= 1 } - return this.sf.f64_sub(a: this.sf.f64_mul( - a: this.sf.f64_from_u64(a: total), b: this.bc.fast_log2(v: total)), b: result) + return this.bc.f64_sub(a: this.bc.f64_mul( + a: this.bc.f64_from_u64(a: total), b: this.bc.fast_log2(v: total)), b: result) } // Mirrors ChooseContextMap in encode.c. The bigram_histo is @@ -585,30 +585,30 @@ pri func encoder.choose_context_map!() base.u32 { i ~mod+= 1 } e1 = this.estimate_entropy(off: 16, size: 3) - e2 = this.sf.f64_add(a: this.estimate_entropy(off: 24, size: 3), b: this.estimate_entropy(off: 27, size: 3)) + e2 = this.bc.f64_add(a: this.estimate_entropy(off: 24, size: 3), b: this.estimate_entropy(off: 27, size: 3)) e3 = 0 i = 0 while i < 3 { - e3 = this.sf.f64_add(a: e3, b: this.estimate_entropy(off: 3 * i, size: 3)) + e3 = this.bc.f64_add(a: e3, b: this.estimate_entropy(off: 3 * i, size: 3)) i ~mod+= 1 } total = (this.ctx_histo[16] ~mod+ this.ctx_histo[17]) ~mod+ this.ctx_histo[18] - e0 = this.sf.f64_div(a: F64_1, b: this.sf.f64_from_u32(a: total)) - e1 = this.sf.f64_mul(a: e1, b: e0) - e2 = this.sf.f64_mul(a: e2, b: e0) - e3 = this.sf.f64_mul(a: e3, b: e0) + e0 = this.sf.f64_div(a: F64_1, b: this.bc.f64_from_u32(a: total)) + e1 = this.bc.f64_mul(a: e1, b: e0) + e2 = this.bc.f64_mul(a: e2, b: e0) + e3 = this.bc.f64_mul(a: e3, b: e0) if this.quality < 7 { // 3 context models is a bit slower, don't use it at lower qualities. - e3 = this.sf.f64_mul(a: e1, b: F64_10) + e3 = this.bc.f64_mul(a: e1, b: F64_10) } // If expected savings by symbol are less than 0.2 bits, skip the context // modeling -- in exchange for faster decoding speed. - if (this.sf.f64_lt(a: this.sf.f64_sub(a: e1, b: e2), b: F64_0_2) <> 0) and - (this.sf.f64_lt(a: this.sf.f64_sub(a: e1, b: e3), b: F64_0_2) <> 0) { + if (this.bc.f64_lt(a: this.bc.f64_sub(a: e1, b: e2), b: F64_0_2) <> 0) and + (this.bc.f64_lt(a: this.bc.f64_sub(a: e1, b: e3), b: F64_0_2) <> 0) { return 1 - } else if this.sf.f64_lt(a: this.sf.f64_sub(a: e2, b: e3), b: F64_0_02) <> 0 { + } else if this.bc.f64_lt(a: this.bc.f64_sub(a: e2, b: e3), b: F64_0_02) <> 0 { return 2 } return 3 @@ -668,16 +668,16 @@ pri func encoder.should_use_complex_static_context_map!(wb: roslice base.u8, sta e2 = 0 i = 0 while i < 13 { - e2 = this.sf.f64_add(a: e2, b: this.estimate_entropy(off: 32 + (i << 5), size: 32)) + e2 = this.bc.f64_add(a: e2, b: this.estimate_entropy(off: 32 + (i << 5), size: 32)) i ~mod+= 1 } - e0 = this.sf.f64_div(a: F64_1, b: this.sf.f64_from_u32(a: total)) - e1 = this.sf.f64_mul(a: e1, b: e0) - e2 = this.sf.f64_mul(a: e2, b: e0) + e0 = this.sf.f64_div(a: F64_1, b: this.bc.f64_from_u32(a: total)) + e1 = this.bc.f64_mul(a: e1, b: e0) + e2 = this.bc.f64_mul(a: e2, b: e0) // The triggering heuristics below were tuned by compressing the // individual files of the silesia corpus. - if (this.sf.f64_gt(a: e2, b: F64_3) <> 0) or - (this.sf.f64_lt(a: this.sf.f64_sub(a: e1, b: e2), b: F64_0_2) <> 0) { + if (this.bc.f64_gt(a: e2, b: F64_3) <> 0) or + (this.bc.f64_lt(a: this.bc.f64_sub(a: e1, b: e2), b: F64_0_2) <> 0) { return 0 } return 13 @@ -748,16 +748,16 @@ pri func encoder.should_compress!(wb: roslice base.u8, last_flush_pos: base.u64, return false } if args.num_commands < ((args.bytes >> 8) + 2) { - if this.sf.f64_gt( - a: this.sf.f64_from_u64(a: args.num_literals), - b: this.sf.f64_mul(a: F64_0_99, b: this.sf.f64_from_u64(a: args.bytes))) <> 0 { + if this.bc.f64_gt( + a: this.bc.f64_from_u64(a: args.num_literals), + b: this.bc.f64_mul(a: F64_0_99, b: this.bc.f64_from_u64(a: args.bytes))) <> 0 { i = 0 while i < 256 { this.ctx_histo[i] = 0 i ~mod+= 1 } - bit_cost_threshold = this.sf.f64_mul( - a: this.sf.f64_mul(a: this.sf.f64_from_u64(a: args.bytes), b: F64_7_92), + bit_cost_threshold = this.bc.f64_mul( + a: this.bc.f64_mul(a: this.bc.f64_from_u64(a: args.bytes), b: F64_7_92), b: F64_INV_13) t = (args.bytes ~mod+ 12) / 13 pos = (args.last_flush_pos & 0xFFFF_FFFF) as base.u32 @@ -767,7 +767,7 @@ pri func encoder.should_compress!(wb: roslice base.u8, last_flush_pos: base.u64, pos ~mod+= 13 i ~mod+= 1 } - if this.sf.f64_gt(a: this.ctx_histo_bits_entropy(), b: bit_cost_threshold) <> 0 { + if this.bc.f64_gt(a: this.ctx_histo_bits_entropy(), b: bit_cost_threshold) <> 0 { return false } } @@ -787,16 +787,16 @@ pri func encoder.ctx_histo_bits_entropy() base.u64 { while i < 256 { p = this.ctx_histo[i] as base.u64 sum ~mod+= p - retval = this.sf.f64_sub(a: retval, b: this.sf.f64_mul( - a: this.sf.f64_from_u64(a: p), b: this.bc.fast_log2(v: p))) + retval = this.bc.f64_sub(a: retval, b: this.bc.f64_mul( + a: this.bc.f64_from_u64(a: p), b: this.bc.fast_log2(v: p))) i ~mod+= 1 } if sum <> 0 { - retval = this.sf.f64_add(a: retval, b: this.sf.f64_mul( - a: this.sf.f64_from_u64(a: sum), b: this.bc.fast_log2(v: sum))) + retval = this.bc.f64_add(a: retval, b: this.bc.f64_mul( + a: this.bc.f64_from_u64(a: sum), b: this.bc.fast_log2(v: sum))) } - if this.sf.f64_lt(a: retval, b: this.sf.f64_from_u64(a: sum)) <> 0 { - retval = this.sf.f64_from_u64(a: sum) + if this.bc.f64_lt(a: retval, b: this.bc.f64_from_u64(a: sum)) <> 0 { + retval = this.bc.f64_from_u64(a: sum) } return retval } diff --git a/std/brotli/encode_float.wuffs b/std/brotli/encode_float.wuffs new file mode 100644 index 00000000..d5f2b1dc --- /dev/null +++ b/std/brotli/encode_float.wuffs @@ -0,0 +1,742 @@ +// Copyright 2026 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. +// +// SPDX-License-Identifier: Apache-2.0 OR MIT + +// -------- + +// This file holds package-private fast paths for the std/softfloat +// operations that the encoder's hot loops use (the Zopfli cost model, the +// block splitter and histogram clustering). +// +// Each softfloat.arith method is another package's public function: every +// call checks its receiver and, being an exported function, is not inlined +// by the C compiler. These bit_cost methods instead handle the common cases +// (finite, normal and, for addition and subtraction, mostly non-negative +// operands with normal results) directly, so that they can be inlined, and +// otherwise call the softfloat.arith method of the same name. Either way, +// they return exactly the same bits as that method (the IEEE 754 result, +// rounding to nearest, ties to even), so they are only a speed optimization. +// +// Operands are IEEE 754 binary32 (base.u32) or binary64 (base.u64) bit +// patterns. The fast paths' conditions on positive normal values use that +// such bit patterns are in [0x0080_0000, 0x7F80_0000) (or, for f64, in +// [0x0010_0000_0000_0000, 0x7FF0_0000_0000_0000)) and that, there, the bit +// patterns sort like the values. + +// -------- Helpers. + +// clz64 returns the number of leading zero bits of a, which must be non-zero +// (if a is zero, it returns 63). +pri func bit_cost.clz64(a: base.u64) base.u32[..= 63] { + var x : base.u64 + var n : base.u32[..= 63] + + x = args.a + if (x >> 32) == 0 { + x = x ~mod<< 32 + n = 32 + } + if (x >> 48) == 0 { + x = x ~mod<< 16 + n = (n + 16) & 63 + } + if (x >> 56) == 0 { + x = x ~mod<< 8 + n = (n + 8) & 63 + } + if (x >> 60) == 0 { + x = x ~mod<< 4 + n = (n + 4) & 63 + } + if (x >> 62) == 0 { + x = x ~mod<< 2 + n = (n + 2) & 63 + } + if (x >> 63) == 0 { + n = (n + 1) & 63 + } + return n +} + +// -------- f32. + +// f32_add returns (a + b), like softfloat.arith.f32_add. Its fast path is +// for the most common case (both are positive and normal) and the others +// take f32_add_any. +pri func bit_cost.f32_add(a: base.u32, b: base.u32) base.u32 { + var x : base.u32 + var y : base.u32 + var ex : base.u32[..= 0xFF] + var d : base.u32 + var s : base.u64 + var z : base.u32 + + if ((args.a ~mod- 0x80_0000) < 0x7F00_0000) and ((args.b ~mod- 0x80_0000) < 0x7F00_0000) { + // Both are positive and normal. Order them so that x >= y. + x = args.a + y = args.b + if y > x { + x = args.b + y = args.a + } + ex = (x >> 23) & 0xFF + d = ex ~mod- ((y >> 23) & 0xFF) + if d >= 25 { + // y is less than half an ULP of x, so the sum rounds to x. + return x + } + // The significands, with their implicit bit at bit 62. Shifting y's + // right by at most 24 is exact, and the sum is less than (1 << 64). + s = ((((x & 0x7F_FFFF) | 0x80_0000) as base.u64) << 39) ~mod+ + (((((y & 0x7F_FFFF) | 0x80_0000) as base.u64) << 39) >> (d & 31)) + // Round to nearest, ties to even: add just under half an ULP, plus + // one if the truncated result is odd. The rounded significand is in + // [1 << 23 ..= 1 << 24] and adding it (whose bit 23 is the implicit + // bit) to the shifted exponent carries any rounding overflow into the + // exponent field. + if s >= 0x8000_0000_0000_0000 { + s = (s ~mod+ (0x7F_FFFF_FFFF ~mod+ ((s >> 40) & 1))) >> 40 + z = (ex ~mod<< 23) ~mod+ ((s & 0x1FF_FFFF) as base.u32) + if z >= 0x7F80_0000 { + return 0x7F80_0000 + } + return z + } + s = (s ~mod+ (0x3F_FFFF_FFFF ~mod+ ((s >> 39) & 1))) >> 39 + return ((ex ~mod- 1) ~mod<< 23) ~mod+ ((s & 0x1FF_FFFF) as base.u32) + } else if (args.a == 0) and ((args.b & 0x7FFF_FFFF) <= 0x7F80_0000) and (args.b <> 0x8000_0000) { + // (+0 + b) is b, for any b other than a NaN or -0. + return args.b + } else if (args.b == 0) and ((args.a & 0x7FFF_FFFF) <= 0x7F80_0000) and (args.a <> 0x8000_0000) { + return args.a + } + return this.f32_add_any(a: args.a, b: args.b) +} + +// f32_sub returns (a - b), like softfloat.arith.f32_sub. Its fast path is +// for the most common case (both are positive and normal, a > b and there is +// little cancellation) and the others take f32_sub_any. +pri func bit_cost.f32_sub(a: base.u32, b: base.u32) base.u32 { + var ex : base.u32[..= 0xFF] + var d : base.u32 + var s : base.u64 + + if ((args.b ~mod- 0x80_0000) < 0x7F00_0000) and (args.a > args.b) and (args.a < 0x7F80_0000) { + // Both are positive and normal, and a > b. + ex = (args.a >> 23) & 0xFF + d = ex ~mod- ((args.b >> 23) & 0xFF) + if d >= 26 { + // b is less than half of the spacing of the f32 values just + // below a, so the difference rounds to a. + return args.a + } else if d >= 2 { + // The significands, with their implicit bit at bit 62. Shifting + // b's right by at most 25 is exact. As (d >= 2), the difference + // is at least (1 << 61), so it needs at most a 1 bit shift to + // normalize (which is exact too). + s = ((((args.a & 0x7F_FFFF) | 0x80_0000) as base.u64) << 39) ~mod- + (((((args.b & 0x7F_FFFF) | 0x80_0000) as base.u64) << 39) >> (d & 31)) + if s < 0x4000_0000_0000_0000 { + s ~mod<<= 1 + ex = (ex ~mod- 1) & 0xFF + } + // Round to nearest, ties to even (see f32_add). The result is + // normal, as ex is at least 2 after the shift (a's biased + // exponent is at least (d + 1)). + s = (s ~mod+ (0x3F_FFFF_FFFF ~mod+ ((s >> 39) & 1))) >> 39 + return ((ex ~mod- 1) ~mod<< 23) ~mod+ ((s & 0x1FF_FFFF) as base.u32) + } + } else if (args.b == 0) and ((args.a & 0x7FFF_FFFF) <= 0x7F80_0000) { + // (a - +0) is a, for any non-NaN a. + return args.a + } else if (args.a == args.b) and (args.a < 0x7F80_0000) { + // (a - a) is +0, for finite non-negative a. + return 0 + } + return this.f32_sub_any(a: args.a, b: args.b) +} + +// f32_add_any returns (a + b), like softfloat.arith.f32_add. It is the +// general (but slower) fast path, for operands of any sign. +pri func bit_cost.f32_add_any(a: base.u32, b: base.u32) base.u32 { + var x : base.u32 + var y : base.u32 + var ex : base.u32[..= 0xFF] + var d : base.u32 + var mx : base.u64 + var my : base.u64 + var s : base.u64 + var lz : base.u32[..= 63] + var z : base.u32 + + if ((((args.a >> 23) & 0xFF) ~mod- 1) < 0xFE) and ((((args.b >> 23) & 0xFF) ~mod- 1) < 0xFE) { + // Both are normal. Order them so that |x| >= |y|. The result has x's + // sign (or is +0). + x = args.a + y = args.b + if (y & 0x7FFF_FFFF) > (x & 0x7FFF_FFFF) { + x = args.b + y = args.a + } + ex = (x >> 23) & 0xFF + d = ex ~mod- ((y >> 23) & 0xFF) + if d >= 26 { + // |y| is less than half an ULP of |x| (and less than half of the + // spacing of the f32 values just below |x|), so the result + // rounds to x. + return x + } + // The significands, with their implicit bit at bit 62. Shifting y's + // right by at most 25 is exact, so the sum or difference is exact + // (and less than (1 << 64)). + mx = (((x & 0x7F_FFFF) | 0x80_0000) as base.u64) << 39 + my = ((((y & 0x7F_FFFF) | 0x80_0000) as base.u64) << 39) >> (d & 31) + if ((x ^ y) >> 31) == 0 { + s = mx ~mod+ my + if s >= 0x8000_0000_0000_0000 { + // Round to nearest, ties to even: add just under half an ULP, + // plus one if the truncated result is odd. The rounded + // significand is in [1 << 23 ..= 1 << 24] and adding it + // (whose bit 23 is the implicit bit) to the shifted exponent + // carries any rounding overflow into the exponent field. + s = (s ~mod+ (0x7F_FFFF_FFFF ~mod+ ((s >> 40) & 1))) >> 40 + z = (ex ~mod<< 23) ~mod+ ((s & 0x1FF_FFFF) as base.u32) + if z >= 0x7F80_0000 { + z = 0x7F80_0000 + } + return (x & 0x8000_0000) | z + } + } else { + s = mx ~mod- my + if s == 0 { + // Exact cancellation gives +0 when rounding to nearest. + return 0 + } else if s < 0x4000_0000_0000_0000 { + lz = (this.clz64(a: s) ~mod- 1) & 63 + if lz >= ex { + // The result is subnormal. + return this.sf.f32_add(a: args.a, b: args.b) + } + ex = (ex ~mod- lz) & 0xFF + s = s ~mod<< lz + } + } + // s is in [1 << 62, 1 << 63) and ex in [1 ..= 254]. + s = (s ~mod+ (0x3F_FFFF_FFFF ~mod+ ((s >> 39) & 1))) >> 39 + return (x & 0x8000_0000) | (((ex ~mod- 1) ~mod<< 23) ~mod+ ((s & 0x1FF_FFFF) as base.u32)) + } + return this.sf.f32_add(a: args.a, b: args.b) +} + +// f32_sub_any returns (a - b), like softfloat.arith.f32_sub. +pri func bit_cost.f32_sub_any(a: base.u32, b: base.u32) base.u32 { + if (((args.b >> 23) & 0xFF) ~mod- 1) < 0xFE { + // b is normal (not a NaN), so (a - b) is exactly (a + -b), including + // the sign of a zero result. + return this.f32_add_any(a: args.a, b: args.b ^ 0x8000_0000) + } + return this.sf.f32_sub(a: args.a, b: args.b) +} + +// f32_key maps a non-NaN f32 to a u32 that sorts in the same order as the +// f32 values (except that -0 sorts just below +0). +pri func bit_cost.f32_key(a: base.u32) base.u32 { + if (args.a >> 31) <> 0 { + return 0xFFFF_FFFF ^ args.a + } + return args.a | 0x8000_0000 +} + +// f32_lt returns (a < b), like softfloat.arith.f32_lt. +pri func bit_cost.f32_lt(a: base.u32, b: base.u32) base.u32 { + if (args.a | args.b) <= 0x7F80_0000 { + // Neither is negative or a NaN, so the bit patterns sort like the + // values. + if args.a < args.b { + return 1 + } + return 0 + } + return this.f32_lt_any(a: args.a, b: args.b) +} + +// f32_lt_any returns (a < b), like softfloat.arith.f32_lt. +pri func bit_cost.f32_lt_any(a: base.u32, b: base.u32) base.u32 { + if ((args.a & 0x7FFF_FFFF) <= 0x7F80_0000) and ((args.b & 0x7FFF_FFFF) <= 0x7F80_0000) { + // Neither is a NaN. + if (this.f32_key(a: args.a) < this.f32_key(a: args.b)) and + (((args.a | args.b) & 0x7FFF_FFFF) <> 0) { + return 1 + } + return 0 + } + return this.sf.f32_lt(a: args.a, b: args.b) +} + +// f32_le returns (a <= b), like softfloat.arith.f32_le. +pri func bit_cost.f32_le(a: base.u32, b: base.u32) base.u32 { + if (args.a | args.b) <= 0x7F80_0000 { + // Neither is negative or a NaN, so the bit patterns sort like the + // values. + if args.a <= args.b { + return 1 + } + return 0 + } + return this.f32_le_any(a: args.a, b: args.b) +} + +// f32_le_any returns (a <= b), like softfloat.arith.f32_le. +pri func bit_cost.f32_le_any(a: base.u32, b: base.u32) base.u32 { + if ((args.a & 0x7FFF_FFFF) <= 0x7F80_0000) and ((args.b & 0x7FFF_FFFF) <= 0x7F80_0000) { + if (this.f32_key(a: args.a) <= this.f32_key(a: args.b)) or + (((args.a | args.b) & 0x7FFF_FFFF) == 0) { + return 1 + } + return 0 + } + return this.sf.f32_le(a: args.a, b: args.b) +} + +// f32_gt returns (a > b), like softfloat.arith.f32_gt. +pri func bit_cost.f32_gt(a: base.u32, b: base.u32) base.u32 { + if (args.a | args.b) <= 0x7F80_0000 { + // Neither is negative or a NaN, so the bit patterns sort like the + // values. + if args.a > args.b { + return 1 + } + return 0 + } + return this.f32_gt_any(a: args.a, b: args.b) +} + +// f32_gt_any returns (a > b), like softfloat.arith.f32_gt. +pri func bit_cost.f32_gt_any(a: base.u32, b: base.u32) base.u32 { + return this.f32_lt_any(a: args.b, b: args.a) +} + +// -------- f64. + +// f64_add returns (a + b), like softfloat.arith.f64_add. Its fast path is +// for the most common case (both are positive and normal) and the others +// take f64_add_any. +pri func bit_cost.f64_add(a: base.u64, b: base.u64) base.u64 { + var x : base.u64 + var y : base.u64 + var ex : base.u64[..= 0x7FF] + var d : base.u64 + var my : base.u64 + var s : base.u64 + + if ((args.a ~mod- 0x0010_0000_0000_0000) < 0x7FE0_0000_0000_0000) and + ((args.b ~mod- 0x0010_0000_0000_0000) < 0x7FE0_0000_0000_0000) { + // Both are positive and normal. Order them so that x >= y. + x = args.a + y = args.b + if y > x { + x = args.b + y = args.a + } + ex = (x >> 52) & 0x7FF + d = ex ~mod- ((y >> 52) & 0x7FF) + if d >= 54 { + // y is less than half an ULP of x, so the sum rounds to x. + return x + } + // The significands, with their implicit bit at bit 62 and 10 extra + // bits of precision at the bottom. Align y's with x's, keeping a + // sticky bit. + my = ((y & 0x000F_FFFF_FFFF_FFFF) | 0x0010_0000_0000_0000) << 10 + if (my & (((1 as base.u64) << (d & 63)) ~mod- 1)) <> 0 { + my = (my >> (d & 63)) | 1 + } else { + my = my >> (d & 63) + } + s = (((x & 0x000F_FFFF_FFFF_FFFF) | 0x0010_0000_0000_0000) << 10) ~mod+ my + if s >= 0x8000_0000_0000_0000 { + ex = (ex ~mod+ 1) & 0x7FF + s = (s >> 1) | (s & 1) + } + // Round to nearest, ties to even (see f32_add). The rounded + // significand is in [1 << 52 ..= 1 << 53]. + s = (s ~mod+ (0x1FF ~mod+ ((s >> 10) & 1))) >> 10 + s = ((ex ~mod- 1) ~mod<< 52) ~mod+ s + if s >= 0x7FF0_0000_0000_0000 { + return 0x7FF0_0000_0000_0000 + } + return s + } else if (args.a == 0) and ((args.b & 0x7FFF_FFFF_FFFF_FFFF) <= 0x7FF0_0000_0000_0000) and + (args.b <> 0x8000_0000_0000_0000) { + // (+0 + b) is b, for any b other than a NaN or -0. + return args.b + } else if (args.b == 0) and ((args.a & 0x7FFF_FFFF_FFFF_FFFF) <= 0x7FF0_0000_0000_0000) and + (args.a <> 0x8000_0000_0000_0000) { + return args.a + } + return this.f64_add_any(a: args.a, b: args.b) +} + +// f64_sub returns (a - b), like softfloat.arith.f64_sub. Its fast path is +// for the most common case (both are positive and normal, a > b and there is +// little cancellation) and the others take f64_sub_any. +pri func bit_cost.f64_sub(a: base.u64, b: base.u64) base.u64 { + var ex : base.u64[..= 0x7FF] + var d : base.u64 + var my : base.u64 + var s : base.u64 + var lz : base.u64 + + if ((args.b ~mod- 0x0010_0000_0000_0000) < 0x7FE0_0000_0000_0000) and + (args.a > args.b) and (args.a < 0x7FF0_0000_0000_0000) { + // Both are positive and normal, and a > b. + ex = (args.a >> 52) & 0x7FF + d = ex ~mod- ((args.b >> 52) & 0x7FF) + if d >= 55 { + // b is less than half of the spacing of the f64 values just + // below a, so the difference rounds to a. + return args.a + } else if d >= 2 { + // The significands, with their implicit bit at bit 62 and 10 + // extra bits of precision at the bottom. Align b's with a's, + // keeping a sticky bit. As (d >= 2), the difference is at least + // (1 << 61), so it needs at most a 1 bit shift to normalize. + my = ((args.b & 0x000F_FFFF_FFFF_FFFF) | 0x0010_0000_0000_0000) << 10 + if (my & (((1 as base.u64) << (d & 63)) ~mod- 1)) <> 0 { + my = (my >> (d & 63)) | 1 + } else { + my = my >> (d & 63) + } + s = (((args.a & 0x000F_FFFF_FFFF_FFFF) | 0x0010_0000_0000_0000) << 10) ~mod- my + if s < 0x4000_0000_0000_0000 { + s ~mod<<= 1 + ex = (ex ~mod- 1) & 0x7FF + } + // Round to nearest, ties to even (see f32_add). The result is + // normal, as ex is at least 2 after the shift. + s = (s ~mod+ (0x1FF ~mod+ ((s >> 10) & 1))) >> 10 + return ((ex ~mod- 1) ~mod<< 52) ~mod+ s + } + // d is 0 or 1, so shifting b's significand right by d (with 10 + // extra bits) is exact, and so is the difference (which is non-zero, + // as a > b) and normalizing it. + s = (((args.a & 0x000F_FFFF_FFFF_FFFF) | 0x0010_0000_0000_0000) << 10) ~mod- + ((((args.b & 0x000F_FFFF_FFFF_FFFF) | 0x0010_0000_0000_0000) << 10) >> (d & 1)) + lz = (this.clz64(a: s) as base.u64) ~mod- 1 + if lz < ex { + ex = (ex ~mod- lz) & 0x7FF + s = s ~mod<< (lz & 63) + s = (s ~mod+ (0x1FF ~mod+ ((s >> 10) & 1))) >> 10 + return ((ex ~mod- 1) ~mod<< 52) ~mod+ s + } + } else if (args.b == 0) and ((args.a & 0x7FFF_FFFF_FFFF_FFFF) <= 0x7FF0_0000_0000_0000) { + // (a - +0) is a, for any non-NaN a. + return args.a + } else if (args.a == args.b) and (args.a < 0x7FF0_0000_0000_0000) { + // (a - a) is +0, for finite non-negative a. + return 0 + } + return this.f64_sub_any(a: args.a, b: args.b) +} + +// f64_add_any returns (a + b), like softfloat.arith.f64_add. It is the +// general (but slower) fast path, for operands of any sign. +pri func bit_cost.f64_add_any(a: base.u64, b: base.u64) base.u64 { + var x : base.u64 + var y : base.u64 + var ex : base.u64[..= 0x7FF] + var d : base.u64 + var mx : base.u64 + var my : base.u64 + var s : base.u64 + var lz : base.u64[..= 63] + + if ((((args.a >> 52) & 0x7FF) ~mod- 1) < 0x7FE) and ((((args.b >> 52) & 0x7FF) ~mod- 1) < 0x7FE) { + // Both are normal. Order them so that |x| >= |y|. The result has x's + // sign (or is +0). + x = args.a + y = args.b + if (y & 0x7FFF_FFFF_FFFF_FFFF) > (x & 0x7FFF_FFFF_FFFF_FFFF) { + x = args.b + y = args.a + } + ex = (x >> 52) & 0x7FF + d = ex ~mod- ((y >> 52) & 0x7FF) + if d >= 55 { + // |y| is less than half an ULP of |x| (and less than half of the + // spacing of the f64 values just below |x|), so the result + // rounds to x. + return x + } + // The significands, with their implicit bit at bit 62 and 10 extra + // bits of precision at the bottom. Align my with mx, keeping a + // sticky bit (which only happens if (d > 10)). + mx = ((x & 0x000F_FFFF_FFFF_FFFF) | 0x0010_0000_0000_0000) << 10 + my = ((y & 0x000F_FFFF_FFFF_FFFF) | 0x0010_0000_0000_0000) << 10 + if (my & (((1 as base.u64) << (d & 63)) ~mod- 1)) <> 0 { + my = (my >> (d & 63)) | 1 + } else { + my = my >> (d & 63) + } + if ((x ^ y) >> 63) == 0 { + s = mx ~mod+ my + if s >= 0x8000_0000_0000_0000 { + ex = (ex ~mod+ 1) & 0x7FF + s = (s >> 1) | (s & 1) + } + } else { + s = mx ~mod- my + if s == 0 { + // Exact cancellation gives +0 when rounding to nearest. + return 0 + } else if s < 0x4000_0000_0000_0000 { + // If (d >= 2) then s >= (1 << 61) and the normalization shift + // is 1. Otherwise, no bits were shifted out of my. + lz = ((this.clz64(a: s) as base.u64) ~mod- 1) & 63 + if lz >= ex { + // The result is subnormal. + return this.sf.f64_add(a: args.a, b: args.b) + } + ex = (ex ~mod- lz) & 0x7FF + s = s ~mod<< lz + } + } + // Round to nearest, ties to even (see f32_add). s is in + // [1 << 62, 1 << 63) and ex in [1 ..= 0x7FF]. The rounded + // significand is in [1 << 52 ..= 1 << 53]. + s = (s ~mod+ (0x1FF ~mod+ ((s >> 10) & 1))) >> 10 + s = ((ex ~mod- 1) ~mod<< 52) ~mod+ s + if s >= 0x7FF0_0000_0000_0000 { + s = 0x7FF0_0000_0000_0000 + } + return (x & 0x8000_0000_0000_0000) | s + } + return this.sf.f64_add(a: args.a, b: args.b) +} + +// f64_sub_any returns (a - b), like softfloat.arith.f64_sub. +pri func bit_cost.f64_sub_any(a: base.u64, b: base.u64) base.u64 { + if (((args.b >> 52) & 0x7FF) ~mod- 1) < 0x7FE { + // b is normal (not a NaN), so (a - b) is exactly (a + -b), including + // the sign of a zero result. + return this.f64_add_any(a: args.a, b: args.b ^ 0x8000_0000_0000_0000) + } + return this.sf.f64_sub(a: args.a, b: args.b) +} + +// f64_key maps a non-NaN f64 to a u64 that sorts in the same order as the +// f64 values (except that -0 sorts just below +0). +pri func bit_cost.f64_key(a: base.u64) base.u64 { + if (args.a >> 63) <> 0 { + return 0xFFFF_FFFF_FFFF_FFFF ^ args.a + } + return args.a | 0x8000_0000_0000_0000 +} + +// f64_lt returns (a < b), like softfloat.arith.f64_lt. +pri func bit_cost.f64_lt(a: base.u64, b: base.u64) base.u32 { + if (args.a | args.b) <= 0x7FF0_0000_0000_0000 { + // Neither is negative or a NaN, so the bit patterns sort like the + // values. + if args.a < args.b { + return 1 + } + return 0 + } + return this.f64_lt_any(a: args.a, b: args.b) +} + +// f64_lt_any returns (a < b), like softfloat.arith.f64_lt. +pri func bit_cost.f64_lt_any(a: base.u64, b: base.u64) base.u32 { + if ((args.a & 0x7FFF_FFFF_FFFF_FFFF) <= 0x7FF0_0000_0000_0000) and + ((args.b & 0x7FFF_FFFF_FFFF_FFFF) <= 0x7FF0_0000_0000_0000) { + // Neither is a NaN. + if (this.f64_key(a: args.a) < this.f64_key(a: args.b)) and + (((args.a | args.b) & 0x7FFF_FFFF_FFFF_FFFF) <> 0) { + return 1 + } + return 0 + } + return this.sf.f64_lt(a: args.a, b: args.b) +} + +// f64_gt returns (a > b), like softfloat.arith.f64_gt. +pri func bit_cost.f64_gt(a: base.u64, b: base.u64) base.u32 { + if (args.a | args.b) <= 0x7FF0_0000_0000_0000 { + // Neither is negative or a NaN, so the bit patterns sort like the + // values. + if args.a > args.b { + return 1 + } + return 0 + } + return this.f64_gt_any(a: args.a, b: args.b) +} + +// f64_gt_any returns (a > b), like softfloat.arith.f64_gt. +pri func bit_cost.f64_gt_any(a: base.u64, b: base.u64) base.u32 { + return this.f64_lt_any(a: args.b, b: args.a) +} + +// f64_ge returns (a >= b), like softfloat.arith.f64_ge. +pri func bit_cost.f64_ge(a: base.u64, b: base.u64) base.u32 { + if (args.a | args.b) <= 0x7FF0_0000_0000_0000 { + // Neither is negative or a NaN, so the bit patterns sort like the + // values. + if args.a >= args.b { + return 1 + } + return 0 + } + return this.f64_ge_any(a: args.a, b: args.b) +} + +// f64_ge_any returns (a >= b), like softfloat.arith.f64_ge. +pri func bit_cost.f64_ge_any(a: base.u64, b: base.u64) base.u32 { + if ((args.a & 0x7FFF_FFFF_FFFF_FFFF) <= 0x7FF0_0000_0000_0000) and + ((args.b & 0x7FFF_FFFF_FFFF_FFFF) <= 0x7FF0_0000_0000_0000) { + if (this.f64_key(a: args.a) >= this.f64_key(a: args.b)) or + (((args.a | args.b) & 0x7FFF_FFFF_FFFF_FFFF) == 0) { + return 1 + } + return 0 + } + return this.sf.f64_ge(a: args.a, b: args.b) +} + +// f64_mul returns (a * b), like softfloat.arith.f64_mul. Its fast path is +// for normal operands with a normal (and finite) result. +pri func bit_cost.f64_mul(a: base.u64, b: base.u64) base.u64 { + var ea : base.u64 + var eb : base.u64 + var e : base.u64 + var ma : base.u64 + var mb : base.u64 + var a0 : base.u64[..= 0xFFFF_FFFF] + var a1 : base.u64[..= 0xFFFF_FFFF] + var b0 : base.u64[..= 0xFFFF_FFFF] + var b1 : base.u64[..= 0xFFFF_FFFF] + var p00 : base.u64 + var p01 : base.u64 + var p10 : base.u64 + var mid : base.u64 + var s : base.u64 + var rb : base.u64 + + ea = (args.a >> 52) & 0x7FF + eb = (args.b >> 52) & 0x7FF + if ((ea ~mod- 1) < 0x7FE) and ((eb ~mod- 1) < 0x7FE) { + // Both are normal. e is one less than the result's biased exponent + // (before normalization), as a two's complement integer. The + // significands have their implicit bit at bit 62 and bit 63, so that + // the high 64 bits of their 128-bit product have the implicit bit at + // bit 61 or 62. + e = (ea ~mod+ eb) ~mod- 0x3FF + ma = ((args.a & 0x000F_FFFF_FFFF_FFFF) | 0x0010_0000_0000_0000) << 10 + mb = ((args.b & 0x000F_FFFF_FFFF_FFFF) | 0x0010_0000_0000_0000) << 11 + a0 = ma & 0xFFFF_FFFF + a1 = ma >> 32 + b0 = mb & 0xFFFF_FFFF + b1 = mb >> 32 + p00 = a0 * b0 + p01 = a0 * b1 + p10 = a1 * b0 + mid = (p00 >> 32) + (p01 & 0xFFFF_FFFF) + (p10 & 0xFFFF_FFFF) + s = (((a1 * b1) ~mod+ (p01 >> 32)) ~mod+ (p10 >> 32)) ~mod+ (mid >> 32) + // The low 64 bits of the product, (ma * mb), give the sticky bit. + if (ma ~mod* mb) <> 0 { + s |= 1 + } + if s < 0x4000_0000_0000_0000 { + e ~mod-= 1 + s ~mod<<= 1 + } + if e < 0x7FD { + // The result is normal and finite (if e is 0x7FC and the + // significand rounds up to (1 << 53), the exponent field is 0x7FE). + // Round to nearest, ties to even. + rb = s & 0x3FF + s = (s ~mod+ 0x200) >> 10 + if rb == 0x200 { + s &= 0xFFFF_FFFF_FFFF_FFFE + } + return ((args.a ^ args.b) & 0x8000_0000_0000_0000) | ((e ~mod<< 52) ~mod+ s) + } + } + return this.sf.f64_mul(a: args.a, b: args.b) +} + +// f64_from_u64 returns the f64 nearest to a (C's (double)a), like +// softfloat.arith.f64_from_u64. Its fast path is for a < (1 << 53), whose +// conversion is exact. +pri func bit_cost.f64_from_u64(a: base.u64) base.u64 { + var lz : base.u32[..= 63] + + if args.a == 0 { + return 0 + } else if args.a < 0x0020_0000_0000_0000 { + // Shift a's top bit to bit 52 (the implicit bit, which is then added + // to the exponent field). + lz = this.clz64(a: args.a) + if lz >= 11 { + return (((0x43D ~mod- (lz as base.u64)) & 0x7FF) ~mod<< 52) ~mod+ (args.a ~mod<< (lz - 11)) + } + } + return this.sf.f64_from_u64(a: args.a) +} + +// f64_from_u32 returns the f64 equal to a (C's (double)a), like +// softfloat.arith.f64_from_u32. +pri func bit_cost.f64_from_u32(a: base.u32) base.u64 { + return this.f64_from_u64(a: args.a as base.u64) +} + +// f64_to_u64_trunc returns a, truncated towards zero, like +// softfloat.arith.f64_to_u64_trunc. Its fast path is for 0 <= a < 2**63. +pri func bit_cost.f64_to_u64_trunc(a: base.u64) base.u64 { + var exp : base.u64 + + exp = args.a >> 52 + if exp < 0x3FF { + // a is in [+0 .. 1), so the result is 0. + return 0 + } else if exp <= 0x433 { + return ((args.a & 0x000F_FFFF_FFFF_FFFF) | 0x0010_0000_0000_0000) >> ((0x433 - exp) & 63) + } + return this.sf.f64_to_u64_trunc(a: args.a) +} + +// f32_from_u32 returns the f32 nearest to a (C's (float)a), like +// softfloat.arith.f32_from_u32. Its fast path is for a < (1 << 24), whose +// conversion is exact. +pri func bit_cost.f32_from_u32(a: base.u32) base.u32 { + var lz : base.u32[..= 63] + + if args.a == 0 { + return 0 + } else if args.a < 0x100_0000 { + // Shift a's top bit to bit 23 (the implicit bit, which is then added + // to the exponent field). + lz = this.clz64(a: args.a as base.u64) + if lz >= 40 { + return ((189 ~mod- lz) ~mod<< 23) ~mod+ (args.a ~mod<< (lz - 40)) + } + } + return this.sf.f32_from_u32(a: args.a) +} + +// F32_OF_SMALL_INT[i] is the f32 equal to i (C's (float)i). +pri const F32_OF_SMALL_INT : roarray[32] base.u32 = [ + 0x0000_0000, 0x3F80_0000, 0x4000_0000, 0x4040_0000, 0x4080_0000, 0x40A0_0000, + 0x40C0_0000, 0x40E0_0000, 0x4100_0000, 0x4110_0000, 0x4120_0000, 0x4130_0000, + 0x4140_0000, 0x4150_0000, 0x4160_0000, 0x4170_0000, 0x4180_0000, 0x4188_0000, + 0x4190_0000, 0x4198_0000, 0x41A0_0000, 0x41A8_0000, 0x41B0_0000, 0x41B8_0000, + 0x41C0_0000, 0x41C8_0000, 0x41D0_0000, 0x41D8_0000, 0x41E0_0000, 0x41E8_0000, + 0x41F0_0000, 0x41F8_0000, +] diff --git a/std/brotli/encode_fragment_fast.wuffs b/std/brotli/encode_fragment_fast.wuffs index 185fb69b..62ce5ea6 100644 --- a/std/brotli/encode_fragment_fast.wuffs +++ b/std/brotli/encode_fragment_fast.wuffs @@ -332,25 +332,25 @@ pri func encoder.should_merge_block!(input: roslice base.u8, from: base.u64, len total = (args.len ~mod+ 42) / 43 // r = (FastLog2(total) + 0.5) * (double)total + 200. - r = this.sf.f64_add( - a: this.sf.f64_mul( - a: this.sf.f64_add(a: this.fast_log2(v: total), b: 0x3FE0_0000_0000_0000), - b: this.sf.f64_from_u64(a: total)), + r = this.bc.f64_add( + a: this.bc.f64_mul( + a: this.bc.f64_add(a: this.fast_log2(v: total), b: 0x3FE0_0000_0000_0000), + b: this.bc.f64_from_u64(a: total)), b: 0x4069_0000_0000_0000) j = 0 while j < 256 { // r -= (double)histo[i] * (depths[i] + FastLog2(histo[i])). h = this.histogram[j] as base.u64 - r = this.sf.f64_sub( + r = this.bc.f64_sub( a: r, - b: this.sf.f64_mul( - a: this.sf.f64_from_u64(a: h), - b: this.sf.f64_add( - a: this.sf.f64_from_u32(a: this.lit_depth[j] as base.u32), + b: this.bc.f64_mul( + a: this.bc.f64_from_u64(a: h), + b: this.bc.f64_add( + a: this.bc.f64_from_u32(a: this.lit_depth[j] as base.u32), b: this.fast_log2(v: h)))) j ~mod+= 1 } - return this.sf.f64_ge(a: r, b: 0) <> 0 + return this.bc.f64_ge(a: r, b: 0) <> 0 } // Mirrors ShouldUseUncompressedMode in compress_fragment.c. diff --git a/std/brotli/encode_fragment_two_pass.wuffs b/std/brotli/encode_fragment_two_pass.wuffs index e41e3754..c39dde1e 100644 --- a/std/brotli/encode_fragment_two_pass.wuffs +++ b/std/brotli/encode_fragment_two_pass.wuffs @@ -472,18 +472,18 @@ pri func encoder.should_compress_two_pass!(input: roslice base.u8, from: base.u6 var max_total_bit_cost : base.u64 var i : base.u64 - corpus_size = this.sf.f64_from_u64(a: args.input_size) + corpus_size = this.bc.f64_from_u64(a: args.input_size) // MIN_RATIO is 0.98 (0x3FEF_5C28_F5C2_8F5C). - if this.sf.f64_lt( - a: this.sf.f64_from_u64(a: args.num_literals), - b: this.sf.f64_mul(a: 0x3FEF_5C28_F5C2_8F5C, b: corpus_size)) <> 0 { + if this.bc.f64_lt( + a: this.bc.f64_from_u64(a: args.num_literals), + b: this.bc.f64_mul(a: 0x3FEF_5C28_F5C2_8F5C, b: corpus_size)) <> 0 { return true } // max_total_bit_cost = corpus_size * 8 * MIN_RATIO / SAMPLE_RATE, where // SAMPLE_RATE is 43 and 8.0 is 0x4020_0000_0000_0000. max_total_bit_cost = this.sf.f64_div( - a: this.sf.f64_mul( - a: this.sf.f64_mul(a: corpus_size, b: 0x4020_0000_0000_0000), + a: this.bc.f64_mul( + a: this.bc.f64_mul(a: corpus_size, b: 0x4020_0000_0000_0000), b: 0x3FEF_5C28_F5C2_8F5C), b: 0x4045_8000_0000_0000) this.histogram[.. 256].bulk_memset!(byte_value: 0) @@ -494,7 +494,7 @@ pri func encoder.should_compress_two_pass!(input: roslice base.u8, from: base.u6 } i ~mod+= 43 } - return this.sf.f64_lt(a: this.bits_entropy_of_histogram(), b: max_total_bit_cost) <> 0 + return this.bc.f64_lt(a: this.bits_entropy_of_histogram(), b: max_total_bit_cost) <> 0 } // Mirrors EmitUncompressedMetaBlock in compress_fragment_two_pass.c (which, diff --git a/std/brotli/encode_literal_cost.wuffs b/std/brotli/encode_literal_cost.wuffs index 3b0846cf..c357aa1b 100644 --- a/std/brotli/encode_literal_cost.wuffs +++ b/std/brotli/encode_literal_cost.wuffs @@ -93,9 +93,9 @@ pri func encoder.is_mostly_utf8!(wb: roslice base.u8, pos: base.u64, length: bas size_utf8 ~mod+= bytes_read } } - return this.sf.f64_gt( - a: this.sf.f64_from_u64(a: size_utf8), - b: this.sf.f64_mul(a: LC_F64_UTF8_FRAC, b: this.sf.f64_from_u64(a: args.length))) <> 0 + return this.bc.f64_gt( + a: this.bc.f64_from_u64(a: size_utf8), + b: this.bc.f64_mul(a: LC_F64_UTF8_FRAC, b: this.bc.f64_from_u64(a: args.length))) <> 0 } // Mirrors UTF8Position in literal_cost.c. @@ -154,17 +154,17 @@ pri func encoder.lc_finish_cost!(wb: slice base.u8, cost_off: base.u64, i: base. var lit_cost : base.u64 lit_cost = args.lit_cost - if this.sf.f64_lt(a: lit_cost, b: LC_F64_1) <> 0 { - lit_cost = this.sf.f64_mul(a: lit_cost, b: LC_F64_0_5) - lit_cost = this.sf.f64_add(a: lit_cost, b: LC_F64_0_5) + if this.bc.f64_lt(a: lit_cost, b: LC_F64_1) <> 0 { + lit_cost = this.bc.f64_mul(a: lit_cost, b: LC_F64_0_5) + lit_cost = this.bc.f64_add(a: lit_cost, b: LC_F64_0_5) } // Make the first bytes more expensive -- seems to help, not sure why. // Perhaps because the entropy source is changing its properties rapidly // in the beginning of the file, perhaps because the beginning of the data // is a statistical "anomaly". if args.utf8 and (args.i < 2000) { - lit_cost = this.sf.f64_add(a: lit_cost, b: this.sf.f64_add(a: LC_F64_0_35, - b: this.sf.f64_mul(a: LC_F64_MULT, b: this.sf.f64_from_u64(a: args.i)))) + lit_cost = this.bc.f64_add(a: lit_cost, b: this.bc.f64_add(a: LC_F64_0_35, + b: this.bc.f64_mul(a: LC_F64_MULT, b: this.bc.f64_from_u64(a: args.i)))) } this.poke_u32!(s: args.wb, i: args.cost_off ~mod+ (args.i ~mod* 4), v: this.sf.f64_to_f32(a: lit_cost)) } @@ -278,8 +278,8 @@ pri func encoder.estimate_bit_costs_for_literals_utf8!(wb: slice base.u8, pos: b iw = iwu2 } lit_cost = this.bc.fast_log2_memo!(v: iw) - lit_cost = this.sf.f64_sub(a: lit_cost, b: this.bc.fast_log2(v: histo)) - lit_cost = this.sf.f64_add(a: lit_cost, b: LC_F64_0_02905) + lit_cost = this.bc.f64_sub(a: lit_cost, b: this.bc.fast_log2(v: histo)) + lit_cost = this.bc.f64_add(a: lit_cost, b: LC_F64_0_02905) this.lc_finish_cost!(wb: args.wb, cost_off: args.cost_off, i: i, lit_cost: lit_cost, utf8: true) i ~mod+= 1 } @@ -339,8 +339,8 @@ pri func encoder.estimate_bit_costs_for_literals!(wb: slice base.u8, pos: base.u histo = 1 } lit_cost = this.bc.fast_log2_memo!(v: in_window) - lit_cost = this.sf.f64_sub(a: lit_cost, b: this.bc.fast_log2(v: histo)) - lit_cost = this.sf.f64_add(a: lit_cost, b: LC_F64_0_029) + lit_cost = this.bc.f64_sub(a: lit_cost, b: this.bc.fast_log2(v: histo)) + lit_cost = this.bc.f64_add(a: lit_cost, b: LC_F64_0_029) this.lc_finish_cost!(wb: args.wb, cost_off: args.cost_off, i: i, lit_cost: lit_cost, utf8: false) i ~mod+= 1 } diff --git a/std/brotli/encode_metablock.wuffs b/std/brotli/encode_metablock.wuffs index 9da3c388..fbc42351 100644 --- a/std/brotli/encode_metablock.wuffs +++ b/std/brotli/encode_metablock.wuffs @@ -103,6 +103,10 @@ pri struct metablock_builder?( cost : array[256] base.u64, insert_cost : array[35200] base.u64, + // FindBlocks' memoized (block_switch_bitcost + insert_cost[i]) sums + // (see find_blocks). + capped_plus : array[35200] base.u64, + // RemapBlockIds' new_id array. new_id : array[256] base.u16, @@ -481,12 +485,12 @@ pri func metablock_builder.compute_distance_cost!(wb: slice base.u8, cmds: rosli postfix_bits: args.new_npostfix) & 0xFFFF) as base.u32 } this.h_add!(wb: args.wb, h: args.tmp, ds: HISTOGRAM_DISTANCE_DATA_SIZE, val: (dist_prefix & 0x3FF) as base.u64) - extra_bits = this.sf.f64_add(a: extra_bits, b: this.sf.f64_from_u32(a: dist_prefix >> 10)) + extra_bits = this.bc.f64_add(a: extra_bits, b: this.bc.f64_from_u32(a: dist_prefix >> 10)) } i ~mod+= 1 } this.dist_cost = this.h_population_cost!(wb: args.wb, h: args.tmp, ds: HISTOGRAM_DISTANCE_DATA_SIZE) - this.dist_cost = this.sf.f64_add(a: this.dist_cost, b: extra_bits) + this.dist_cost = this.bc.f64_add(a: this.dist_cost, b: extra_bits) return true } @@ -624,7 +628,7 @@ pri func metablock_builder.build_meta_block!(wb: slice base.u8, ringbuffer: rosl success = this.compute_distance_cost!(wb: args.wb, cmds: args.cmds, num_commands: args.num_commands, orig_npostfix: orig_npostfix, orig_ndirect: orig_ndirect, new_npostfix: npostfix, new_ndirect: ndirect, new_max_distance: this.dist_max_distance, tmp: tmp) - if (not success) or (this.sf.f64_gt(a: this.dist_cost, b: best_dist_cost) <> 0) { + if (not success) or (this.bc.f64_gt(a: this.dist_cost, b: best_dist_cost) <> 0) { break } best_dist_cost = this.dist_cost @@ -645,7 +649,7 @@ pri func metablock_builder.build_meta_block!(wb: slice base.u8, ringbuffer: rosl success = this.compute_distance_cost!(wb: args.wb, cmds: args.cmds, num_commands: args.num_commands, orig_npostfix: orig_npostfix, orig_ndirect: orig_ndirect, new_npostfix: orig_npostfix, new_ndirect: orig_ndirect, new_max_distance: orig_md, tmp: tmp) - if this.sf.f64_lt(a: this.dist_cost, b: best_dist_cost) <> 0 { + if this.bc.f64_lt(a: this.dist_cost, b: best_dist_cost) <> 0 { best_npostfix = orig_npostfix best_ndirect = orig_ndirect best_asm = orig_asm diff --git a/std/brotli/encode_metablock_greedy.wuffs b/std/brotli/encode_metablock_greedy.wuffs index 103d6bbf..1d691110 100644 --- a/std/brotli/encode_metablock_greedy.wuffs +++ b/std/brotli/encode_metablock_greedy.wuffs @@ -201,20 +201,20 @@ pri func encoder.gbs_finish_block!(wb: slice base.u8, c: base.u32[..= 3], is_fin this.gbs_add_histogram!(wb: args.wb, dst: tmp, src: this.gbs_histo_off(c: args.c, ix: last1 ~mod+ i), n: data_size) } this.gbs_comb_entropy[jx & 31] = this.gbs_entropy!(wb: args.wb, off: tmp, size: alphabet_size) - d = this.sf.f64_sub( - a: this.sf.f64_sub(a: this.gbs_comb_entropy[jx & 31], b: e), + d = this.bc.f64_sub( + a: this.bc.f64_sub(a: this.gbs_comb_entropy[jx & 31], b: e), b: this.gbs_last_entropy[((le_base as base.u64) ~mod+ jx) & 127]) if j == 0 { if nc == 1 { diff0 = d } else { - diff0 = this.sf.f64_add(a: diff0, b: d) + diff0 = this.bc.f64_add(a: diff0, b: d) } } else { if nc == 1 { diff1 = d } else { - diff1 = this.sf.f64_add(a: diff1, b: d) + diff1 = this.bc.f64_add(a: diff1, b: d) } } j ~mod+= 1 @@ -223,8 +223,8 @@ pri func encoder.gbs_finish_block!(wb: slice base.u8, c: base.u32[..= 3], is_fin } if (this.gbs_num_types[args.c] < this.gbs_max_block_types[args.c]) and - (this.sf.f64_gt(a: diff0, b: threshold) <> 0) and - (this.sf.f64_gt(a: diff1, b: threshold) <> 0) { + (this.bc.f64_gt(a: diff0, b: threshold) <> 0) and + (this.bc.f64_gt(a: diff1, b: threshold) <> 0) { // Create new block. this.poke_u32!(s: args.wb, i: lengths_off ~mod+ (num_blocks ~mod* 4), v: (this.gbs_block_size[args.c] & 0xFFFF_FFFF) as base.u32) @@ -250,7 +250,7 @@ pri func encoder.gbs_finish_block!(wb: slice base.u8, c: base.u32[..= 3], is_fin this.gbs_merge_last_count[args.c] = 0 this.gbs_target_block_size[args.c] = this.gbs_min_block_size[args.c] as base.u64 - } else if this.sf.f64_lt(a: diff1, b: this.sf.f64_sub(a: diff0, b: GBS_F64_20)) <> 0 { + } else if this.bc.f64_lt(a: diff1, b: this.bc.f64_sub(a: diff0, b: GBS_F64_20)) <> 0 { // Combine this block with second last block. this.poke_u32!(s: args.wb, i: lengths_off ~mod+ (num_blocks ~mod* 4), v: (this.gbs_block_size[args.c] & 0xFFFF_FFFF) as base.u32) diff --git a/std/brotli/encode_zopfli.wuffs b/std/brotli/encode_zopfli.wuffs index 3a1c3a94..ecacb41d 100644 --- a/std/brotli/encode_zopfli.wuffs +++ b/std/brotli/encode_zopfli.wuffs @@ -232,7 +232,7 @@ pri func encoder.z_set_cost_from_histogram!(which: base.u32, histogram_size: bas i ~mod+= 1 } } - missing_symbol_cost = this.sf.f32_add(a: this.sf.f64_to_f32(a: this.bc.fast_log2(v: missing_symbol_sum)), b: Z_F32_2) + missing_symbol_cost = this.bc.f32_add(a: this.sf.f64_to_f32(a: this.bc.fast_log2(v: missing_symbol_sum)), b: Z_F32_2) i = 0 while i < args.histogram_size { h = this.z_hist(which: args.which, i: i) @@ -240,9 +240,9 @@ pri func encoder.z_set_cost_from_histogram!(which: base.u32, histogram_size: bas this.z_set_cost!(which: args.which, i: i, v: missing_symbol_cost) } else { // Shannon bits for this symbol. - cost = this.sf.f32_sub(a: log2sum, b: this.sf.f64_to_f32(a: this.bc.fast_log2(v: h as base.u64))) + cost = this.bc.f32_sub(a: log2sum, b: this.sf.f64_to_f32(a: this.bc.fast_log2(v: h as base.u64))) // Cannot be coded with less than 1 bit. - if this.sf.f32_lt(a: cost, b: Z_F32_1) <> 0 { + if this.bc.f32_lt(a: cost, b: Z_F32_1) <> 0 { cost = Z_F32_1 } this.z_set_cost!(which: args.which, i: i, v: cost) @@ -322,10 +322,10 @@ pri func encoder.zopfli_cost_model_set_from_commands!(wb: slice base.u8, positio i = 0 while i < this.z_num_bytes { b = this.rb_byte(wb: args.wb, pos: args.position ~mod+ i) - literal_carry = this.sf.f32_add(a: literal_carry, b: this.z_cost_literal[b]) - lc_i1 = this.sf.f32_add(a: lc_i, b: literal_carry) + literal_carry = this.bc.f32_add(a: literal_carry, b: this.z_cost_literal[b]) + lc_i1 = this.bc.f32_add(a: lc_i, b: literal_carry) this.poke_u32!(s: args.wb, i: this.z_lit_off ~mod+ ((i ~mod+ 1) ~mod* 4), v: lc_i1) - literal_carry = this.sf.f32_sub(a: literal_carry, b: this.sf.f32_sub(a: lc_i1, b: lc_i)) + literal_carry = this.bc.f32_sub(a: literal_carry, b: this.bc.f32_sub(a: lc_i1, b: lc_i)) lc_i = lc_i1 i ~mod+= 1 } @@ -346,10 +346,10 @@ pri func encoder.zopfli_cost_model_set_from_literal_costs!(wb: slice base.u8, po lc_i = 0 i = 0 while i < this.z_num_bytes { - literal_carry = this.sf.f32_add(a: literal_carry, b: this.lit_cost_at(wb: args.wb, i: i ~mod+ 1)) - lc_i1 = this.sf.f32_add(a: lc_i, b: literal_carry) + literal_carry = this.bc.f32_add(a: literal_carry, b: this.lit_cost_at(wb: args.wb, i: i ~mod+ 1)) + lc_i1 = this.bc.f32_add(a: lc_i, b: literal_carry) this.poke_u32!(s: args.wb, i: this.z_lit_off ~mod+ ((i ~mod+ 1) ~mod* 4), v: lc_i1) - literal_carry = this.sf.f32_sub(a: literal_carry, b: this.sf.f32_sub(a: lc_i1, b: lc_i)) + literal_carry = this.bc.f32_sub(a: literal_carry, b: this.bc.f32_sub(a: lc_i1, b: lc_i)) lc_i = lc_i1 i ~mod+= 1 } @@ -368,7 +368,7 @@ pri func encoder.zopfli_cost_model_set_from_literal_costs!(wb: slice base.u8, po // Mirrors ZopfliCostModelGetLiteralCosts in backward_references_hq.c. pri func encoder.z_literal_costs(wb: roslice base.u8, from: base.u64, to: base.u64) base.u32 { - return this.sf.f32_sub(a: this.lit_cost_at(wb: args.wb, i: args.to), b: this.lit_cost_at(wb: args.wb, i: args.from)) + return this.bc.f32_sub(a: this.lit_cost_at(wb: args.wb, i: args.to), b: this.lit_cost_at(wb: args.wb, i: args.from)) } // Mirrors UpdateZopfliNode in backward_references_hq.c. @@ -420,7 +420,7 @@ pri func encoder.start_pos_queue_push!() { while i < len { a = (offset & 7) as base.u32 b = ((offset ~mod+ 1) & 7) as base.u32 - if this.sf.f32_gt(a: this.zq_costdiff[a], b: this.zq_costdiff[b]) <> 0 { + if this.bc.f32_gt(a: this.zq_costdiff[a], b: this.zq_costdiff[b]) <> 0 { this.start_pos_queue_swap!(a: a, b: b) } offset ~mod+= 1 @@ -473,7 +473,7 @@ pri func encoder.compute_minimum_copy_length(wb: roslice base.u8, start_cost: ba next_len_bucket = 4 next_len_offset = 10 while ((args.pos ~mod+ len) <= args.num_bytes) and - (this.sf.f32_le(a: this.zn_u(wb: args.wb, i: args.pos ~mod+ len), b: min_cost) <> 0) { + (this.bc.f32_le(a: this.zn_u(wb: args.wb, i: args.pos ~mod+ len), b: min_cost) <> 0) { // We already reached (pos + len) with no more cost than the minimum // possible cost of reaching anything from this pos, so there is no // point in looking for lengths <= len. @@ -481,7 +481,7 @@ pri func encoder.compute_minimum_copy_length(wb: roslice base.u8, start_cost: ba if len == next_len_offset { // We reached the next copy length code bucket, so we add one // more extra bit to the minimum cost. - min_cost = this.sf.f32_add(a: min_cost, b: Z_F32_1) + min_cost = this.bc.f32_add(a: min_cost, b: Z_F32_1) next_len_offset ~mod+= next_len_bucket next_len_bucket ~mod*= 2 } @@ -553,10 +553,10 @@ pri func encoder.evaluate_node!(wb: slice base.u8, block_start: base.u64, pos: b this.zn_set_u!(wb: args.wb, i: args.pos, v: this.compute_distance_shortcut( wb: args.wb, block_start: args.block_start, pos: args.pos, max_backward_limit: args.max_backward_limit)) lc = this.z_literal_costs(wb: args.wb, from: 0, to: args.pos) - if this.sf.f32_le(a: node_cost, b: lc) <> 0 { + if this.bc.f32_le(a: node_cost, b: lc) <> 0 { this.zp_pos = args.pos this.zp_cost = node_cost - this.zp_costdiff = this.sf.f32_sub(a: node_cost, b: lc) + this.zp_costdiff = this.bc.f32_sub(a: node_cost, b: lc) this.compute_distance_cache!(wb: args.wb, pos: args.pos) this.start_pos_queue_push!() } @@ -586,7 +586,6 @@ pri func encoder.update_nodes!(wb: slice base.u8, dwb: roslice base.u8, num_byte var base_cost : base.u32 var best_len : base.u64 var j : base.u64 - var idx : base.u32[..= 3] var backward : base.u64 var bw32 : base.u32 var prev_ix : base.u64 @@ -607,6 +606,8 @@ pri func encoder.update_nodes!(wb: slice base.u8, dwb: roslice base.u8, num_byte var max_match_len : base.u64 var len_code : base.u64 var gap : base.u64 + var dcs : array[4] base.u32 + var prev_copycode : base.u32 mask = this.rb_mask as base.u64 gap = this.dict_total_size @@ -626,8 +627,8 @@ pri func encoder.update_nodes!(wb: slice base.u8, dwb: roslice base.u8, num_byte pos: args.pos, max_backward_limit: args.max_backward_limit) q = this.start_pos_queue_at(k: 0) - min_cost = this.sf.f32_add( - a: this.sf.f32_add(a: this.zq_cost[q], b: this.z_min_cost_cmd), + min_cost = this.bc.f32_add( + a: this.bc.f32_add(a: this.zq_cost[q], b: this.z_min_cost_cmd), b: this.z_literal_costs(wb: args.wb, from: this.zq_pos[q], to: args.pos)) min_len = this.compute_minimum_copy_length(wb: args.wb, start_cost: min_cost, num_bytes: args.num_bytes, pos: args.pos) @@ -639,27 +640,32 @@ pri func encoder.update_nodes!(wb: slice base.u8, dwb: roslice base.u8, num_byte start = this.zq_pos[q] inscode = this.get_insert_length_code(insertlen: args.pos ~mod- start) start_costdiff = this.zq_costdiff[q] - base_cost = this.sf.f32_add( - a: this.sf.f32_add(a: start_costdiff, b: this.sf.f32_from_u32(a: INS_EXTRA[inscode.min(no_more_than: 23)])), + base_cost = this.bc.f32_add( + a: this.bc.f32_add(a: start_costdiff, b: F32_OF_SMALL_INT[INS_EXTRA[inscode.min(no_more_than: 23)]]), b: this.z_literal_costs(wb: args.wb, from: 0, to: args.pos)) // Look for last distance matches using the distance cache from this // starting position. best_len = min_len ~mod- 1 + dcs[0] = this.zq_dc[((q as base.u32) << 2) | 0] + dcs[1] = this.zq_dc[((q as base.u32) << 2) | 1] + dcs[2] = this.zq_dc[((q as base.u32) << 2) | 2] + dcs[3] = this.zq_dc[((q as base.u32) << 2) | 3] + // continuation and the break condition only depend on best_len, so + // they are computed again only when best_len changes. + continuation = this.rb_raw(wb: args.wb, mpos: cur_ix_masked ~mod+ best_len, k: 0) j = 0 while (j < 16) and (best_len < max_len) { - idx = Z_DISTANCE_CACHE_INDEX[j] - bw32 = this.zq_dc[((q as base.u32) << 2) | idx] ~mod+ Z_DISTANCE_CACHE_OFFSET[j] + if (cur_ix_masked ~mod+ best_len) > mask { + break + } + bw32 = dcs[Z_DISTANCE_CACHE_INDEX[j]] ~mod+ Z_DISTANCE_CACHE_OFFSET[j] backward = bw32 as base.u64 if bw32 >= 0x8000_0000 { backward |= 0xFFFF_FFFF_0000_0000 } prev_ix = cur_ix ~mod- backward len = 0 - continuation = this.rb_raw(wb: args.wb, mpos: cur_ix_masked ~mod+ best_len, k: 0) - if (cur_ix_masked ~mod+ best_len) > mask { - break - } if backward > (dictionary_start ~mod+ gap) { // Word dictionary -> ignore. j ~mod+= 1 @@ -691,20 +697,26 @@ pri func encoder.update_nodes!(wb: slice base.u8, dwb: roslice base.u8, num_byte j ~mod+= 1 continue } - dist_cost = this.sf.f32_add(a: base_cost, b: this.z_cost_dist[j]) + dist_cost = this.bc.f32_add(a: base_cost, b: this.z_cost_dist[j]) l = best_len ~mod+ 1 + // The cost only depends on copycode (inscode and j are fixed), so + // it is computed again only when copycode changes. + prev_copycode = 0xFFFF_FFFF while l <= len { copycode = this.get_copy_length_code(copylen: l) - cmdcode = this.combine_length_codes(inscode: inscode, copycode: copycode, use_last_distance: j == 0) - if cmdcode < 128 { - cost = base_cost - } else { - cost = dist_cost + if copycode <> prev_copycode { + prev_copycode = copycode + cmdcode = this.combine_length_codes(inscode: inscode, copycode: copycode, use_last_distance: j == 0) + if cmdcode < 128 { + cost = base_cost + } else { + cost = dist_cost + } + cost = this.bc.f32_add( + a: this.bc.f32_add(a: cost, b: F32_OF_SMALL_INT[COPY_EXTRA[copycode.min(no_more_than: 23)]]), + b: this.z_cost_cmd[cmdcode.min(no_more_than: 703)]) } - cost = this.sf.f32_add( - a: this.sf.f32_add(a: cost, b: this.sf.f32_from_u32(a: COPY_EXTRA[copycode.min(no_more_than: 23)])), - b: this.z_cost_cmd[cmdcode.min(no_more_than: 703)]) - if this.sf.f32_lt(a: cost, b: this.zn_u(wb: args.wb, i: args.pos ~mod+ l)) <> 0 { + if this.bc.f32_lt(a: cost, b: this.zn_u(wb: args.wb, i: args.pos ~mod+ l)) <> 0 { this.update_zopfli_node!(wb: args.wb, pos: args.pos, start_pos: start, len: l, len_code: l, dist: backward, short_code: j ~mod+ 1, cost: cost) result = result.max(no_less_than: l) @@ -712,6 +724,7 @@ pri func encoder.update_nodes!(wb: slice base.u8, dwb: roslice base.u8, num_byte best_len = l l ~mod+= 1 } + continuation = this.rb_raw(wb: args.wb, mpos: cur_ix_masked ~mod+ best_len, k: 0) j ~mod+= 1 } @@ -739,8 +752,8 @@ pri func encoder.update_nodes!(wb: slice base.u8, dwb: roslice base.u8, num_byte postfix_bits: this.distance_postfix_bits) dist_symbol = (pe & 0xFFFF) as base.u32 distnumextra = dist_symbol >> 10 - dist_cost = this.sf.f32_add( - a: this.sf.f32_add(a: base_cost, b: this.sf.f32_from_u32(a: distnumextra)), + dist_cost = this.bc.f32_add( + a: this.bc.f32_add(a: base_cost, b: this.bc.f32_from_u32(a: distnumextra)), b: this.z_cost_dist[this.min32_543(a: dist_symbol & 0x3FF)]) // Try all copy lengths up until the maximum copy length @@ -751,17 +764,22 @@ pri func encoder.update_nodes!(wb: slice base.u8, dwb: roslice base.u8, num_byte if (len < max_match_len) and (is_dictionary_match or (max_match_len > max_zopfli_len)) { len = max_match_len } + // As above, the cost only depends on copycode. + prev_copycode = 0xFFFF_FFFF while len <= max_match_len { len_code = len if is_dictionary_match { len_code = this.bm_length_code(wb: args.wb, off: moff) } copycode = this.get_copy_length_code(copylen: len_code) - cmdcode = this.combine_length_codes(inscode: inscode, copycode: copycode, use_last_distance: false) - cost = this.sf.f32_add( - a: this.sf.f32_add(a: dist_cost, b: this.sf.f32_from_u32(a: COPY_EXTRA[copycode.min(no_more_than: 23)])), - b: this.z_cost_cmd[cmdcode.min(no_more_than: 703)]) - if this.sf.f32_lt(a: cost, b: this.zn_u(wb: args.wb, i: args.pos ~mod+ len)) <> 0 { + if copycode <> prev_copycode { + prev_copycode = copycode + cmdcode = this.combine_length_codes(inscode: inscode, copycode: copycode, use_last_distance: false) + cost = this.bc.f32_add( + a: this.bc.f32_add(a: dist_cost, b: F32_OF_SMALL_INT[COPY_EXTRA[copycode.min(no_more_than: 23)]]), + b: this.z_cost_cmd[cmdcode.min(no_more_than: 703)]) + } + if this.bc.f32_lt(a: cost, b: this.zn_u(wb: args.wb, i: args.pos ~mod+ len)) <> 0 { this.update_zopfli_node!(wb: args.wb, pos: args.pos, start_pos: start, len: len, len_code: len_code, dist: dist, short_code: 0, cost: cost) result = result.max(no_less_than: len) diff --git a/std/softfloat/softfloat_f32.wuffs b/std/softfloat/softfloat_f32.wuffs index c49d64e9..7cd43e44 100644 --- a/std/softfloat/softfloat_f32.wuffs +++ b/std/softfloat/softfloat_f32.wuffs @@ -172,50 +172,41 @@ pub func arith.f32_mul(a: base.u32, b: base.u32) base.u32 { // f32_add_mags returns (sign(a) * (|a| + |b|)), for normal a and b. pri func arith.f32_add_mags(a: base.u32, b: base.u32) base.u32 { - var ea : base.u32[..= 0x100] - var eb : base.u32[..= 0xFF] - var ma : base.u32[..= 0x3FFF_FFC0] - var mb : base.u32[..= 0x3FFF_FFC0] - var d : base.u32[..= 0xFF] + var x : base.u32 + var y : base.u32 + var ex : base.u32[..= 0xFF] + var d : base.u32 + var s : base.u64 var z : base.u32 - var rb : base.u32 - ea = (args.a >> 23) & 0xFF - eb = (args.b >> 23) & 0xFF - // The significands, with their implicit bit at bit 29 and 6 extra bits - // of precision at the bottom. - ma = ((args.a & 0x7F_FFFF) | 0x80_0000) << 6 - mb = ((args.b & 0x7F_FFFF) | 0x80_0000) << 6 - if ea < eb { - d = (eb ~mod- ea) & 0xFF - ea = eb - z = ma - ma = mb - mb = z + // Order the magnitudes so that |x| >= |y|. + x = args.a & F32_ABS + y = args.b & F32_ABS + if y > x { + x = args.b & F32_ABS + y = args.a & F32_ABS + } + ex = (x >> 23) & 0xFF + d = ex ~mod- ((y >> 23) & 0xFF) + if d >= 25 { + // |y| is less than half an ULP of |x|, so the sum rounds to x. + return (args.a & F32_SIGN) | x + } + // The significands, with their implicit bit at bit 62. Shifting y's + // right by at most 24 is exact, and the sum is less than (1 << 64). + s = ((((x & 0x7F_FFFF) | 0x80_0000) as base.u64) << 39) ~mod+ + (((((y & 0x7F_FFFF) | 0x80_0000) as base.u64) << 39) >> (d & 31)) + // Round to nearest, ties to even: add just under half an ULP, plus one + // if the truncated result is odd. The result is in [1 << 23 ..= 1 << 24] + // and adding it (whose bit 23 is the implicit bit) to the shifted + // exponent carries any rounding overflow into the exponent field. + if s >= 0x8000_0000_0000_0000 { + s = (s ~mod+ (0x7F_FFFF_FFFF ~mod+ ((s >> 40) & 1))) >> 40 + z = (ex ~mod<< 23) ~mod+ ((s & 0x1FF_FFFF) as base.u32) } else { - d = (ea ~mod- eb) & 0xFF + s = (s ~mod+ (0x3F_FFFF_FFFF ~mod+ ((s >> 39) & 1))) >> 39 + z = ((ex ~mod- 1) ~mod<< 23) ~mod+ ((s & 0x1FF_FFFF) as base.u32) } - if d >= 31 { - mb = 1 - } else if d > 0 { - if (mb ~mod<< (32 - d)) <> 0 { - mb = (mb >> d) | 1 - } else { - mb = mb >> d - } - } - z = (ma & 0x3FFF_FFFF) + (mb & 0x3FFF_FFFF) - if z >= 0x4000_0000 { - ea += 1 - z = (z >> 1) | (z & 1) - } - rb = z & 0x3F - z = (z ~mod+ 0x20) >> 6 - if rb == 0x20 { - z &= 0xFFFF_FFFE - } - // ea is in [1 ..= 255] and z is in [1 << 23 ..= 1 << 24]. - z = ((ea ~mod- 1) ~mod<< 23) ~mod+ z if z >= F32_INF { z = F32_INF } From fbae955a74584e2b4051d62a197ca0cb08d30b78 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 08:50:15 +0000 Subject: [PATCH 2/2] std/brotli: speed up qualities 0 to 9 and the decoder Encoder (output stays byte-exact with libbrotlienc v1.2.1): - H58/H68 (q5-6): match the tags 8 at a time (SWAR, like GetMatchingTagMask's non-SSE2 code) and visit the matches with a de Bruijn ctz; inline the bucket entry check into the H5/H6 and H58/H68 loops, caching the 4 bytes compared at best_len; a dedicated H58/H68 store and hash. - find_last_distances caches the current byte at best_len. - ctz64 and a branch-free log2_floor_non_zero (de Bruijn tables), and find_match_length_with_limit uses ctz. - load32le, load64le, peek_u16, poke_u16 and poke_u32 use one bounds check each. Decoder (same output and statuses): - The literal fast path keeps pos and the counts in local variables. - Copies of at most 16 bytes, with a distance of at least 16, copy 16 bytes (like the C code; the window gap keeps the extra bytes unread). script/manual-test-brotli-float.c also checks log2_floor_non_zero (every u32) and ctz64. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UZ978BikvkegL2JUkqoB1e --- release/c/wuffs-unsupported-snapshot.c | 548 ++++++++++++++++--------- script/manual-test-brotli-float.c | 41 ++ std/brotli/README.md | 24 +- std/brotli/decode_metablock.wuffs | 103 ++++- std/brotli/encode_bit_writer.wuffs | 62 ++- std/brotli/encode_hasher.wuffs | 254 +++++++----- std/brotli/encode_metablock.wuffs | 16 +- std/brotli/encode_util.wuffs | 45 +- 8 files changed, 716 insertions(+), 377 deletions(-) diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 87d7f819..6001456e 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -77153,6 +77153,14 @@ WUFFS_BROTLI__DICTIONARY[122784] WUFFS_BASE__POTENTIALLY_UNUSED = { #define WUFFS_BROTLI__F64_1E99 6088095589093318446u +static const uint32_t +WUFFS_BROTLI__LOG2_DEBRUIJN32[32] WUFFS_BASE__POTENTIALLY_UNUSED = { + 0u, 9u, 1u, 10u, 13u, 21u, 2u, 29u, + 11u, 14u, 16u, 18u, 22u, 25u, 3u, 30u, + 8u, 12u, 20u, 28u, 15u, 17u, 24u, 7u, + 19u, 27u, 23u, 6u, 26u, 5u, 4u, 31u, +}; + #define WUFFS_BROTLI__LITERAL_BLOCK_SWITCH_COST 4628602664527698330u #define WUFFS_BROTLI__COMMAND_BLOCK_SWITCH_COST 4623789442425946112u @@ -77339,6 +77347,18 @@ WUFFS_BROTLI__F32_OF_SMALL_INT[32] WUFFS_BASE__POTENTIALLY_UNUSED = { #define WUFFS_BROTLI__HQ_SHADOW_MATCHES 256u +static const uint8_t +WUFFS_BROTLI__CTZ64_DEBRUIJN[64] WUFFS_BASE__POTENTIALLY_UNUSED = { + 0u, 1u, 48u, 2u, 57u, 49u, 28u, 3u, + 61u, 58u, 50u, 42u, 38u, 29u, 17u, 4u, + 62u, 55u, 59u, 36u, 53u, 51u, 43u, 22u, + 45u, 39u, 33u, 30u, 24u, 18u, 12u, 5u, + 63u, 47u, 56u, 27u, 60u, 41u, 37u, 16u, + 54u, 35u, 52u, 21u, 44u, 32u, 23u, 11u, + 46u, 26u, 40u, 15u, 34u, 20u, 31u, 10u, + 25u, 14u, 19u, 9u, 13u, 8u, 7u, 6u, +}; + #define WUFFS_BROTLI__Z_INFINITY 2130692510u #define WUFFS_BROTLI__Z_F32_1 1065353216u @@ -77759,6 +77779,14 @@ wuffs_brotli__decoder__take_distance_from_ring_buffer( wuffs_brotli__decoder* self, uint32_t a_code); +WUFFS_BASE__GENERATED_C_CODE +static bool +wuffs_brotli__decoder__copy16( + wuffs_brotli__decoder* self, + wuffs_base__slice_u8 a_workbuf, + uint64_t a_d, + uint64_t a_s); + WUFFS_BASE__GENERATED_C_CODE static uint32_t wuffs_brotli__encoder__cmd_insert_len( @@ -79652,6 +79680,20 @@ wuffs_brotli__encoder__hasher_store( wuffs_base__slice_u8 a_wb, uint64_t a_ix); +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__empty_struct +wuffs_brotli__encoder__hasher_store_h58( + wuffs_brotli__encoder* self, + wuffs_base__slice_u8 a_wb, + uint64_t a_ix); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__encoder__hash_h58( + const wuffs_brotli__encoder* self, + wuffs_base__slice_u8 a_wb, + uint64_t a_ix); + WUFFS_BASE__GENERATED_C_CODE static wuffs_base__empty_struct wuffs_brotli__encoder__hasher_store_range( @@ -79755,17 +79797,6 @@ wuffs_brotli__encoder__find_last_distances( uint64_t a_max_length, uint64_t a_max_backward); -WUFFS_BASE__GENERATED_C_CODE -static uint64_t -wuffs_brotli__encoder__try_bucket_entry( - wuffs_brotli__encoder* self, - wuffs_base__slice_u8 a_data, - uint64_t a_cur_ix, - uint64_t a_prev_ix_in, - uint64_t a_best_len, - uint64_t a_max_length, - uint64_t a_max_backward); - WUFFS_BASE__GENERATED_C_CODE static wuffs_base__empty_struct wuffs_brotli__encoder__find_longest_match_h5( @@ -80547,6 +80578,12 @@ wuffs_brotli__encoder__hq_extend_var_region( wuffs_brotli__encoder* self, uint64_t a_wb_len); +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__encoder__ctz64( + const wuffs_brotli__encoder* self, + uint64_t a_x); + WUFFS_BASE__GENERATED_C_CODE static uint8_t wuffs_brotli__encoder__byte_at( @@ -85534,6 +85571,13 @@ wuffs_brotli__decoder__process_commands_fast( uint64_t v_dhi = 0; uint64_t v_slo = 0; uint64_t v_shi = 0; + uint32_t v_cm_slice = 0; + uint32_t v_wmask = 0; + uint32_t v_pos = 0; + uint32_t v_lit_n = 0; + uint32_t v_lit_i = 0; + uint64_t v_wi = 0; + bool v_copied16 = false; const uint8_t* iop_a_src = NULL; const uint8_t* io0_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; @@ -85630,25 +85674,32 @@ wuffs_brotli__decoder__process_commands_fast( } v_p1 = ((uint32_t)(wuffs_brotli__decoder__get_byte(self, a_workbuf, 1u))); v_p2 = ((uint32_t)(wuffs_brotli__decoder__get_byte(self, a_workbuf, 2u))); - while (self->private_impl.f_insert_length > 0u) { - if ((self->private_impl.f_block_length[0u] == 0u) || (((uint32_t)(self->private_impl.f_pos + 1u)) >= self->private_impl.f_rb_size)) { - goto label__outer__break; - } + v_mode_offset = (self->private_impl.f_context_mode << 9u); + v_cm_slice = self->private_impl.f_context_map_slice; + v_wmask = ((uint32_t)(self->private_impl.f_window_size - 1u)); + v_pos = self->private_impl.f_pos; + v_lit_n = wuffs_base__u32__min(self->private_impl.f_insert_length, self->private_impl.f_block_length[0u]); + if (((uint32_t)(v_pos + 1u)) >= self->private_impl.f_rb_size) { + v_lit_n = 0u; + } else { + v_lit_n = wuffs_base__u32__min(v_lit_n, ((uint32_t)(((uint32_t)(self->private_impl.f_rb_size - v_pos)) - 1u))); + } + v_lit_i = 0u; + while (v_lit_i < v_lit_n) { if (v_n_bits < 24u) { if (((uint64_t)(io2_a_src - iop_a_src)) < 8u) { - goto label__outer__break; + break; } v_bits |= ((uint64_t)(wuffs_base__peek_u64le__no_bounds_check(iop_a_src) << v_n_bits)); iop_a_src += ((63u - v_n_bits) >> 3u); v_n_bits |= 56u; } - v_mode_offset = (self->private_impl.f_context_mode << 9u); v_ctx = ((uint32_t)(((uint8_t)(WUFFS_BROTLI__CONTEXT_LOOKUP[(v_mode_offset | v_p1)] | WUFFS_BROTLI__CONTEXT_LOOKUP[(v_mode_offset | 256u | v_p2)])))); - v_tree = ((uint32_t)(self->private_data.f_context_maps[(self->private_impl.f_context_map_slice + v_ctx)])); + v_tree = ((uint32_t)(self->private_data.f_context_maps[(v_cm_slice + v_ctx)])); v_i = ((uint32_t)((v_tree * 632u) + ((uint32_t)((v_bits & 255u))))); if (v_i >= 678034u) { v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); - goto label__outer__break; + break; } v_e = ((uint32_t)(self->private_data.f_huffs[v_i])); v_e_bits = (v_e & 15u); @@ -85656,23 +85707,34 @@ wuffs_brotli__decoder__process_commands_fast( v_i = ((uint32_t)(((uint32_t)((v_tree * 632u) + (v_e >> 4u))) + (((uint32_t)(((v_bits >> 8u) & 32767u))) & ((((uint32_t)(1u)) << (v_e_bits - 8u)) - 1u)))); if (v_i >= 678034u) { v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); - goto label__outer__break; + break; } v_e = ((uint32_t)(self->private_data.f_huffs[v_i])); v_e_bits = ((v_e & 15u) + 8u); } if (v_n_bits < v_e_bits) { v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_bit_reader); - goto label__outer__break; + break; } v_bits >>= v_e_bits; v_n_bits -= v_e_bits; - self->private_impl.f_symbol = (v_e >> 4u); - wuffs_brotli__decoder__put_byte(self, a_workbuf, ((uint8_t)(self->private_impl.f_symbol))); v_p2 = v_p1; - v_p1 = (self->private_impl.f_symbol & 255u); - self->private_impl.f_block_length[0u] -= 1u; - self->private_impl.f_insert_length -= 1u; + v_p1 = ((v_e >> 4u) & 255u); + v_wi = ((uint64_t)((v_pos & v_wmask))); + if (v_wi < ((uint64_t)(a_workbuf.len))) { + a_workbuf.ptr[v_wi] = ((uint8_t)(v_p1)); + } + v_pos += 1u; + v_lit_i += 1u; + } + if (v_lit_i > 0u) { + self->private_impl.f_symbol = v_p1; + self->private_impl.f_pos = v_pos; + self->private_impl.f_block_length[0u] -= v_lit_i; + self->private_impl.f_insert_length -= v_lit_i; + } + if (wuffs_base__status__is_error(&v_ret) || (self->private_impl.f_insert_length > 0u)) { + break; } self->private_impl.f_cmd_state = 2u; if ((self->private_impl.f_mb_remaining == 0u) || (self->private_impl.f_mb_remaining >= 2147483648u)) { @@ -85758,6 +85820,15 @@ wuffs_brotli__decoder__process_commands_fast( self->private_impl.f_dist_rb_idx += 1u; self->private_impl.f_mb_remaining -= v_n; v_k64 = 0u; + if ((v_n <= 16u) && + (v_distance >= 16u) && + (((uint64_t)(v_dst_i + 16u)) <= ((uint64_t)(self->private_impl.f_rb_size))) && + (((uint64_t)(v_src_i + 16u)) <= ((uint64_t)(self->private_impl.f_window_size)))) { + v_copied16 = wuffs_brotli__decoder__copy16(self, a_workbuf, v_dst_i, v_src_i); + if (v_copied16) { + v_k64 = ((uint64_t)(v_n)); + } + } while (v_k64 < ((uint64_t)(v_n))) { v_n64 = ((uint64_t)(((uint64_t)(v_n)) - v_k64)); v_n64 = wuffs_base__u64__min(v_n64, ((uint64_t)(v_distance))); @@ -85787,7 +85858,6 @@ wuffs_brotli__decoder__process_commands_fast( break; } } - label__outer__break:; while (v_n_bits >= 8u) { if ( ! (iop_a_src > io1_a_src)) { status = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_bit_reader); @@ -85938,6 +86008,43 @@ wuffs_brotli__decoder__take_distance_from_ring_buffer( return wuffs_base__make_empty_struct(); } +// -------- func brotli.decoder.copy16 + +WUFFS_BASE__GENERATED_C_CODE +static bool +wuffs_brotli__decoder__copy16( + wuffs_brotli__decoder* self, + wuffs_base__slice_u8 a_workbuf, + uint64_t a_d, + uint64_t a_s) { + uint64_t v_lo = 0; + uint64_t v_hi = 0; + wuffs_base__slice_u8 v_q = {0}; + wuffs_base__slice_u8 v_r = {0}; + + if ((a_s > ((uint64_t)(a_workbuf.len))) || (a_d > ((uint64_t)(a_workbuf.len)))) { + return false; + } + v_q = wuffs_base__slice_u8__subslice_i(a_workbuf, a_s); + if (((uint64_t)(v_q.len)) >= 16u) { + v_lo = wuffs_base__peek_u64le__no_bounds_check(v_q.ptr); + v_q = wuffs_base__slice_u8__subslice_i(v_q, 8u); + if (((uint64_t)(v_q.len)) >= 8u) { + v_hi = wuffs_base__peek_u64le__no_bounds_check(v_q.ptr); + v_q = wuffs_base__slice_u8__subslice_i(a_workbuf, a_d); + if (((uint64_t)(v_q.len)) >= 16u) { + v_r = wuffs_base__slice_u8__subslice_i(v_q, 8u); + wuffs_base__poke_u64le__no_bounds_check(v_q.ptr, v_lo); + if (((uint64_t)(v_r.len)) >= 8u) { + wuffs_base__poke_u64le__no_bounds_check(v_r.ptr, v_hi); + return true; + } + } + } + } + return false; +} + // -------- func brotli.encoder.cmd_insert_len WUFFS_BASE__GENERATED_C_CODE @@ -88206,29 +88313,14 @@ wuffs_brotli__encoder__log2_floor_non_zero( const wuffs_brotli__encoder* self, uint64_t a_n) { uint32_t v_x = 0; - uint32_t v_r = 0; v_x = ((uint32_t)(a_n)); - if (v_x >= 65536u) { - v_x >>= 16u; - v_r = 16u; - } - if (v_x >= 256u) { - v_x >>= 8u; - v_r += 8u; - } - if (v_x >= 16u) { - v_x >>= 4u; - v_r += 4u; - } - if (v_x >= 4u) { - v_x >>= 2u; - v_r += 2u; - } - if (v_x >= 2u) { - v_r += 1u; - } - return (v_r & 31u); + v_x |= (v_x >> 1u); + v_x |= (v_x >> 2u); + v_x |= (v_x >> 4u); + v_x |= (v_x >> 8u); + v_x |= (v_x >> 16u); + return WUFFS_BROTLI__LOG2_DEBRUIJN32[(((uint32_t)(v_x * 130329821u)) >> 27u)]; } // -------- func brotli.encoder.fast_log2 @@ -88279,12 +88371,11 @@ wuffs_brotli__encoder__load64le( const wuffs_brotli__encoder* self, wuffs_base__slice_u8 a_s, uint64_t a_i) { - wuffs_base__slice_u8 v_q = {0}; - - if (a_i < ((uint64_t)(a_s.len))) { - v_q = wuffs_base__slice_u8__subslice_i(a_s, a_i); - if (((uint64_t)(v_q.len)) >= 8u) { - return wuffs_base__peek_u64le__no_bounds_check(v_q.ptr); + if (a_i < 18446744073709486080u) { + if ((a_i + 8u) <= ((uint64_t)(a_s.len))) { + return wuffs_base__peek_u64le__no_bounds_check(wuffs_base__slice_u8__subslice_ij(a_s, + a_i, + (a_i + 8u)).ptr); } } return 0u; @@ -88298,12 +88389,11 @@ wuffs_brotli__encoder__load32le( const wuffs_brotli__encoder* self, wuffs_base__slice_u8 a_s, uint64_t a_i) { - wuffs_base__slice_u8 v_q = {0}; - - if (a_i < ((uint64_t)(a_s.len))) { - v_q = wuffs_base__slice_u8__subslice_i(a_s, a_i); - if (((uint64_t)(v_q.len)) >= 4u) { - return wuffs_base__peek_u32le__no_bounds_check(v_q.ptr); + if (a_i < 18446744073709486080u) { + if ((a_i + 4u) <= ((uint64_t)(a_s.len))) { + return wuffs_base__peek_u32le__no_bounds_check(wuffs_base__slice_u8__subslice_ij(a_s, + a_i, + (a_i + 4u)).ptr); } } return 0u; @@ -88334,11 +88424,7 @@ wuffs_brotli__encoder__find_match_length_with_limit( while ((v_n >= 8u) && (((uint64_t)(v_a.len)) >= 8u) && (((uint64_t)(v_b.len)) >= 8u)) { v_x = (wuffs_base__peek_u64le__no_bounds_check(v_a.ptr) ^ wuffs_base__peek_u64le__no_bounds_check(v_b.ptr)); if (v_x != 0u) { - while ((v_x & 255u) == 0u) { - v_matched += 1u; - v_x >>= 8u; - } - return v_matched; + return ((uint64_t)(v_matched + (wuffs_brotli__encoder__ctz64(self, v_x) >> 3u))); } v_matched += 8u; v_n -= 8u; @@ -96842,7 +96928,6 @@ wuffs_brotli__encoder__hasher_store( uint64_t v_bsize = 0; uint64_t v_off = 0; uint16_t v_num = 0; - uint32_t v_hash = 0; uint64_t v_bank = 0; uint64_t v_idx = 0; uint64_t v_delta = 0; @@ -96850,6 +96935,10 @@ wuffs_brotli__encoder__hasher_store( uint32_t v_fc_bits = 0; v_htype = self->private_impl.f_hasher_type; + if ((v_htype == 58u) || (v_htype == 68u)) { + wuffs_brotli__encoder__hasher_store_h58(self, a_wb, a_ix); + return wuffs_base__make_empty_struct(); + } if (self->private_impl.f_rb_buf_off > ((uint64_t)(a_wb.len))) { return wuffs_base__make_empty_struct(); } @@ -96868,14 +96957,6 @@ wuffs_brotli__encoder__hasher_store( v_off = ((uint64_t)((((uint64_t)(v_num)) & ((((uint64_t)(1u)) << self->private_impl.f_h_block_bits) - 1u)) + (((uint64_t)(v_key)) << self->private_impl.f_h_block_bits))); wuffs_brotli__encoder__poke_u16(self, a_wb, ((uint64_t)(self->private_impl.f_h_off0 + ((uint64_t)(2u * ((uint64_t)(v_key)))))), ((uint16_t)(v_num + 1u))); wuffs_brotli__encoder__poke_u32(self, a_wb, ((uint64_t)(self->private_impl.f_h_off1 + ((uint64_t)(4u * v_off)))), ((uint32_t)(a_ix))); - } else if ((v_htype == 58u) || (v_htype == 68u)) { - v_hash = wuffs_brotli__encoder__hash_longest(self, v_data, (a_ix & ((uint64_t)(self->private_impl.f_rb_mask)))); - v_key = (v_hash >> 8u); - v_num = wuffs_brotli__encoder__peek_u16(self, a_wb, ((uint64_t)(self->private_impl.f_h_off0 + ((uint64_t)(2u * ((uint64_t)(v_key))))))); - v_off = ((uint64_t)((((uint64_t)(v_num)) & ((((uint64_t)(1u)) << self->private_impl.f_h_block_bits) - 1u)) + (((uint64_t)(v_key)) << self->private_impl.f_h_block_bits))); - wuffs_brotli__encoder__poke_u16(self, a_wb, ((uint64_t)(self->private_impl.f_h_off0 + ((uint64_t)(2u * ((uint64_t)(v_key)))))), ((uint16_t)(v_num - 1u))); - wuffs_brotli__encoder__poke_u32(self, a_wb, ((uint64_t)(self->private_impl.f_h_off2 + ((uint64_t)(4u * v_off)))), ((uint32_t)(a_ix))); - wuffs_brotli__encoder__set_byte(self, a_wb, ((uint64_t)(self->private_impl.f_h_off1 + v_off)), ((uint8_t)(v_hash))); } else if ((v_htype == 40u) || (v_htype == 41u) || (v_htype == 42u)) { v_key = wuffs_brotli__encoder__hash_forgetful(self, v_data, (a_ix & ((uint64_t)(self->private_impl.f_rb_mask)))); v_fc_bits = wuffs_brotli__encoder__fc_bank_bits(self); @@ -96904,6 +96985,48 @@ wuffs_brotli__encoder__hasher_store( return wuffs_base__make_empty_struct(); } +// -------- func brotli.encoder.hasher_store_h58 + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__empty_struct +wuffs_brotli__encoder__hasher_store_h58( + wuffs_brotli__encoder* self, + wuffs_base__slice_u8 a_wb, + uint64_t a_ix) { + uint32_t v_hash = 0; + uint64_t v_key = 0; + uint64_t v_koff = 0; + uint16_t v_num = 0; + uint64_t v_off = 0; + + v_hash = wuffs_brotli__encoder__hash_h58(self, a_wb, a_ix); + v_key = ((uint64_t)((v_hash >> 8u))); + v_koff = ((uint64_t)(self->private_impl.f_h_off0 + ((uint64_t)(2u * v_key)))); + v_num = wuffs_brotli__encoder__peek_u16(self, a_wb, v_koff); + v_off = ((uint64_t)((((uint64_t)(v_num)) & ((((uint64_t)(1u)) << self->private_impl.f_h_block_bits) - 1u)) + (v_key << self->private_impl.f_h_block_bits))); + wuffs_brotli__encoder__poke_u16(self, a_wb, v_koff, ((uint16_t)(v_num - 1u))); + wuffs_brotli__encoder__poke_u32(self, a_wb, ((uint64_t)(self->private_impl.f_h_off2 + ((uint64_t)(4u * v_off)))), ((uint32_t)(a_ix))); + wuffs_brotli__encoder__set_byte(self, a_wb, ((uint64_t)(self->private_impl.f_h_off1 + v_off)), ((uint8_t)(v_hash))); + return wuffs_base__make_empty_struct(); +} + +// -------- func brotli.encoder.hash_h58 + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__encoder__hash_h58( + const wuffs_brotli__encoder* self, + wuffs_base__slice_u8 a_wb, + uint64_t a_ix) { + uint64_t v_i = 0; + + v_i = ((uint64_t)(self->private_impl.f_rb_buf_off + (a_ix & ((uint64_t)(self->private_impl.f_rb_mask))))); + if (self->private_impl.f_hasher_type == 58u) { + return (((uint32_t)(wuffs_brotli__encoder__load32le(self, a_wb, v_i) * 506832829u)) >> ((24u - self->private_impl.f_h_bucket_bits) & 31u)); + } + return ((uint32_t)((((uint64_t)(wuffs_brotli__encoder__load64le(self, a_wb, v_i) * 8922571613522624512u)) >> 41u))); +} + // -------- func brotli.encoder.hasher_store_range WUFFS_BASE__GENERATED_C_CODE @@ -97384,6 +97507,8 @@ wuffs_brotli__encoder__find_last_distances( uint64_t v_prev_ix = 0; uint64_t v_len = 0; uint64_t v_score = 0; + uint8_t v_cur_byte = 0; + uint32_t v_n = 0; v_mask = ((uint64_t)(self->private_impl.f_rb_mask)); v_cur_ix_masked = (a_cur_ix & v_mask); @@ -97391,15 +97516,13 @@ wuffs_brotli__encoder__find_last_distances( v_best_len = self->private_impl.f_sr_len; self->private_impl.f_sr_len = 0u; self->private_impl.f_sr_len_code_delta = 0u; + v_cur_byte = wuffs_brotli__encoder__byte_at(self, a_data, ((uint64_t)(v_cur_ix_masked + v_best_len))); + v_n = self->private_impl.f_h_num_last_distances; v_i = 0u; - while (v_i < self->private_impl.f_h_num_last_distances) { + while (v_i < v_n) { v_backward = wuffs_brotli__encoder__dist_cache_u64(self, v_i); v_prev_ix = ((uint64_t)(a_cur_ix - v_backward)); - if (v_prev_ix >= a_cur_ix) { - v_i += 1u; - continue; - } - if (v_backward > a_max_backward) { + if ((v_prev_ix >= a_cur_ix) || (v_backward > a_max_backward)) { v_i += 1u; continue; } @@ -97407,7 +97530,7 @@ wuffs_brotli__encoder__find_last_distances( if (((uint64_t)(v_cur_ix_masked + v_best_len)) > v_mask) { break; } - if ((((uint64_t)(v_prev_ix + v_best_len)) > v_mask) || (wuffs_brotli__encoder__byte_at(self, a_data, ((uint64_t)(v_cur_ix_masked + v_best_len))) != wuffs_brotli__encoder__byte_at(self, a_data, ((uint64_t)(v_prev_ix + v_best_len))))) { + if ((((uint64_t)(v_prev_ix + v_best_len)) > v_mask) || (v_cur_byte != wuffs_brotli__encoder__byte_at(self, a_data, ((uint64_t)(v_prev_ix + v_best_len))))) { v_i += 1u; continue; } @@ -97428,6 +97551,7 @@ wuffs_brotli__encoder__find_last_distances( self->private_impl.f_sr_len = v_best_len; self->private_impl.f_sr_distance = v_backward; self->private_impl.f_sr_score = v_best_score; + v_cur_byte = wuffs_brotli__encoder__byte_at(self, a_data, ((uint64_t)(v_cur_ix_masked + v_best_len))); } } } @@ -97436,67 +97560,6 @@ wuffs_brotli__encoder__find_last_distances( return v_best_len; } -// -------- func brotli.encoder.try_bucket_entry - -WUFFS_BASE__GENERATED_C_CODE -static uint64_t -wuffs_brotli__encoder__try_bucket_entry( - wuffs_brotli__encoder* self, - wuffs_base__slice_u8 a_data, - uint64_t a_cur_ix, - uint64_t a_prev_ix_in, - uint64_t a_best_len, - uint64_t a_max_length, - uint64_t a_max_backward) { - uint64_t v_mask = 0; - uint64_t v_cur_ix_masked = 0; - uint64_t v_prev_ix = 0; - uint64_t v_backward = 0; - uint64_t v_len = 0; - uint64_t v_score = 0; - - v_mask = ((uint64_t)(self->private_impl.f_rb_mask)); - v_cur_ix_masked = (a_cur_ix & v_mask); - v_backward = ((uint64_t)(a_cur_ix - a_prev_ix_in)); - if (v_backward > a_max_backward) { - return 1u; - } - v_prev_ix = (a_prev_ix_in & v_mask); - if (((uint64_t)(v_cur_ix_masked + a_best_len)) > v_mask) { - return 1u; - } - if ((((uint64_t)(v_prev_ix + a_best_len)) > v_mask) || (wuffs_brotli__encoder__load32le(self, a_data, ((uint64_t)(((uint64_t)(v_cur_ix_masked + a_best_len)) - 3u))) != wuffs_brotli__encoder__load32le(self, a_data, ((uint64_t)(((uint64_t)(v_prev_ix + a_best_len)) - 3u))))) { - return 0u; - } - if ((self->private_impl.f_hasher_type == 6u) || (self->private_impl.f_hasher_type == 68u)) { - if (wuffs_brotli__encoder__load32le(self, a_data, v_cur_ix_masked) != wuffs_brotli__encoder__load32le(self, a_data, v_prev_ix)) { - return 0u; - } - v_len = ((uint64_t)(wuffs_brotli__encoder__find_match_length_with_limit(self, - a_data, - ((uint64_t)(v_prev_ix + 4u)), - ((uint64_t)(v_cur_ix_masked + 4u)), - ((uint64_t)(a_max_length - 4u))) + 4u)); - } else { - v_len = wuffs_brotli__encoder__find_match_length_with_limit(self, - a_data, - v_prev_ix, - v_cur_ix_masked, - a_max_length); - if (v_len < 4u) { - return 0u; - } - } - v_score = wuffs_brotli__encoder__backward_reference_score(self, v_len, v_backward); - if (self->private_impl.f_sr_score < v_score) { - self->private_impl.f_sr_len = v_len; - self->private_impl.f_sr_distance = v_backward; - self->private_impl.f_sr_score = v_score; - return ((uint64_t)(2u + v_len)); - } - return 0u; -} - // -------- func brotli.encoder.find_longest_match_h5 WUFFS_BASE__GENERATED_C_CODE @@ -97520,7 +97583,13 @@ wuffs_brotli__encoder__find_longest_match_h5( uint64_t v_bucket = 0; uint64_t v_down = 0; uint64_t v_i = 0; - uint64_t v_r = 0; + uint64_t v_mask = 0; + bool v_is64 = false; + uint32_t v_cur4 = 0; + uint64_t v_prev_ix = 0; + uint64_t v_backward = 0; + uint64_t v_len = 0; + uint64_t v_score = 0; if (self->private_impl.f_rb_buf_off > ((uint64_t)(a_wb.len))) { return wuffs_base__make_empty_struct(); @@ -97545,20 +97614,50 @@ wuffs_brotli__encoder__find_longest_match_h5( if (((uint64_t)(v_num)) > v_block_size) { v_down = ((uint64_t)(((uint64_t)(v_num)) - v_block_size)); } + v_mask = ((uint64_t)(self->private_impl.f_rb_mask)); + v_is64 = (self->private_impl.f_hasher_type == 6u); + v_cur4 = wuffs_brotli__encoder__load32le(self, v_data, ((uint64_t)(((uint64_t)(v_cur_ix_masked + v_best_len)) - 3u))); v_i = ((uint64_t)(v_num)); while (v_i > v_down) { v_i -= 1u; - v_r = wuffs_brotli__encoder__try_bucket_entry(self, - v_data, - a_cur_ix, - ((uint64_t)(wuffs_brotli__encoder__load32le(self, a_wb, ((uint64_t)(v_bucket + ((uint64_t)(4u * (v_i & v_block_mask)))))))), - v_best_len, - a_max_length, - a_max_backward); - if (v_r == 1u) { + v_prev_ix = ((uint64_t)(wuffs_brotli__encoder__load32le(self, a_wb, ((uint64_t)(v_bucket + ((uint64_t)(4u * (v_i & v_block_mask)))))))); + v_backward = ((uint64_t)(a_cur_ix - v_prev_ix)); + if (v_backward > a_max_backward) { break; - } else if (v_r >= 2u) { - v_best_len = ((uint64_t)(v_r - 2u)); + } + v_prev_ix &= v_mask; + if (((uint64_t)(v_cur_ix_masked + v_best_len)) > v_mask) { + break; + } + if ((((uint64_t)(v_prev_ix + v_best_len)) > v_mask) || (v_cur4 != wuffs_brotli__encoder__load32le(self, v_data, ((uint64_t)(((uint64_t)(v_prev_ix + v_best_len)) - 3u))))) { + continue; + } + if (v_is64) { + if (wuffs_brotli__encoder__load32le(self, v_data, v_cur_ix_masked) != wuffs_brotli__encoder__load32le(self, v_data, v_prev_ix)) { + continue; + } + v_len = ((uint64_t)(wuffs_brotli__encoder__find_match_length_with_limit(self, + v_data, + ((uint64_t)(v_prev_ix + 4u)), + ((uint64_t)(v_cur_ix_masked + 4u)), + ((uint64_t)(a_max_length - 4u))) + 4u)); + } else { + v_len = wuffs_brotli__encoder__find_match_length_with_limit(self, + v_data, + v_prev_ix, + v_cur_ix_masked, + a_max_length); + if (v_len < 4u) { + continue; + } + } + v_score = wuffs_brotli__encoder__backward_reference_score(self, v_len, v_backward); + if (self->private_impl.f_sr_score < v_score) { + self->private_impl.f_sr_len = v_len; + self->private_impl.f_sr_distance = v_backward; + self->private_impl.f_sr_score = v_score; + v_best_len = v_len; + v_cur4 = wuffs_brotli__encoder__load32le(self, v_data, ((uint64_t)(((uint64_t)(v_cur_ix_masked + v_best_len)) - 3u))); } } wuffs_brotli__encoder__poke_u32(self, a_wb, ((uint64_t)(v_bucket + ((uint64_t)(4u * (((uint64_t)(v_num)) & v_block_mask))))), ((uint32_t)(a_cur_ix))); @@ -97604,7 +97703,17 @@ wuffs_brotli__encoder__find_longest_match_h58( uint64_t v_limit = 0; uint64_t v_k = 0; uint64_t v_rb_index = 0; - uint64_t v_r = 0; + uint64_t v_splat = 0; + uint64_t v_matches = 0; + uint64_t v_chunk = 0; + uint64_t v_j = 0; + uint64_t v_mask = 0; + bool v_is64 = false; + uint32_t v_cur4 = 0; + uint64_t v_prev_ix = 0; + uint64_t v_backward = 0; + uint64_t v_len = 0; + uint64_t v_score = 0; if (self->private_impl.f_rb_buf_off > ((uint64_t)(a_wb.len))) { return wuffs_base__make_empty_struct(); @@ -97612,7 +97721,7 @@ wuffs_brotli__encoder__find_longest_match_h58( v_data = wuffs_base__slice_u8__subslice_i(a_wb, self->private_impl.f_rb_buf_off); v_cur_ix_masked = (a_cur_ix & ((uint64_t)(self->private_impl.f_rb_mask))); v_min_score = self->private_impl.f_sr_score; - v_hash = wuffs_brotli__encoder__hash_longest(self, v_data, v_cur_ix_masked); + v_hash = wuffs_brotli__encoder__hash_h58(self, a_wb, a_cur_ix); v_key = ((uint64_t)((v_hash >> 8u))); v_block_size = (((uint64_t)(1u)) << self->private_impl.f_h_block_bits); v_block_mask = ((uint64_t)(v_block_size - 1u)); @@ -97634,24 +97743,68 @@ wuffs_brotli__encoder__find_longest_match_h58( if (v_block_size > ((uint64_t)(v_n))) { v_limit = ((uint64_t)(v_n)); } - v_k = 0u; - while (v_k < v_limit) { + v_splat = ((uint64_t)(((uint64_t)(v_tag)) * 72340172838076673u)); + v_matches = 0u; + v_j = 0u; + while (v_j < v_block_size) { + v_chunk = (wuffs_brotli__encoder__load64le(self, a_wb, ((uint64_t)(v_tag_bucket + v_j))) ^ v_splat); + v_chunk = ((((uint64_t)((v_chunk | 9259542123273814144u) - 72340172838076673u)) | v_chunk) & 9259542123273814144u); + v_chunk = ((v_chunk ^ 9259542123273814144u) >> 7u); + v_matches |= ((uint64_t)((((uint64_t)(v_chunk * 72624976668147840u)) >> 56u) << (v_j & 63u))); + v_j += 8u; + } + if (v_head > 0u) { + v_matches = (((v_matches >> (v_head & 63u)) | ((uint64_t)(v_matches << (((uint64_t)(v_block_size - v_head)) & 63u)))) & (((uint64_t)(18446744073709551615u)) >> (((uint64_t)(64u - v_block_size)) & 63u))); + } + if (v_limit < 64u) { + v_matches &= ((uint64_t)((((uint64_t)(1u)) << (v_limit & 63u)) - 1u)); + } + v_mask = ((uint64_t)(self->private_impl.f_rb_mask)); + v_is64 = (self->private_impl.f_hasher_type == 68u); + v_cur4 = wuffs_brotli__encoder__load32le(self, v_data, ((uint64_t)(((uint64_t)(v_cur_ix_masked + v_best_len)) - 3u))); + while (v_matches != 0u) { + v_k = wuffs_brotli__encoder__ctz64(self, v_matches); + v_matches &= ((uint64_t)(v_matches - 1u)); v_rb_index = (((uint64_t)(v_head + v_k)) & v_block_mask); - if (wuffs_brotli__encoder__byte_at(self, a_wb, ((uint64_t)(v_tag_bucket + v_rb_index))) == v_tag) { - v_r = wuffs_brotli__encoder__try_bucket_entry(self, + v_prev_ix = ((uint64_t)(wuffs_brotli__encoder__load32le(self, a_wb, ((uint64_t)(v_bucket + ((uint64_t)(4u * v_rb_index))))))); + v_backward = ((uint64_t)(a_cur_ix - v_prev_ix)); + if (v_backward > a_max_backward) { + break; + } + v_prev_ix &= v_mask; + if (((uint64_t)(v_cur_ix_masked + v_best_len)) > v_mask) { + break; + } + if ((((uint64_t)(v_prev_ix + v_best_len)) > v_mask) || (v_cur4 != wuffs_brotli__encoder__load32le(self, v_data, ((uint64_t)(((uint64_t)(v_prev_ix + v_best_len)) - 3u))))) { + continue; + } + if (v_is64) { + if (wuffs_brotli__encoder__load32le(self, v_data, v_cur_ix_masked) != wuffs_brotli__encoder__load32le(self, v_data, v_prev_ix)) { + continue; + } + v_len = ((uint64_t)(wuffs_brotli__encoder__find_match_length_with_limit(self, v_data, - a_cur_ix, - ((uint64_t)(wuffs_brotli__encoder__load32le(self, a_wb, ((uint64_t)(v_bucket + ((uint64_t)(4u * v_rb_index))))))), - v_best_len, - a_max_length, - a_max_backward); - if (v_r == 1u) { - break; - } else if (v_r >= 2u) { - v_best_len = ((uint64_t)(v_r - 2u)); + ((uint64_t)(v_prev_ix + 4u)), + ((uint64_t)(v_cur_ix_masked + 4u)), + ((uint64_t)(a_max_length - 4u))) + 4u)); + } else { + v_len = wuffs_brotli__encoder__find_match_length_with_limit(self, + v_data, + v_prev_ix, + v_cur_ix_masked, + a_max_length); + if (v_len < 4u) { + continue; } } - v_k += 1u; + v_score = wuffs_brotli__encoder__backward_reference_score(self, v_len, v_backward); + if (self->private_impl.f_sr_score < v_score) { + self->private_impl.f_sr_len = v_len; + self->private_impl.f_sr_distance = v_backward; + self->private_impl.f_sr_score = v_score; + v_best_len = v_len; + v_cur4 = wuffs_brotli__encoder__load32le(self, v_data, ((uint64_t)(((uint64_t)(v_cur_ix_masked + v_best_len)) - 3u))); + } } wuffs_brotli__encoder__poke_u32(self, a_wb, ((uint64_t)(v_bucket + ((uint64_t)(4u * (((uint64_t)(v_num)) & v_block_mask))))), ((uint32_t)(a_cur_ix))); wuffs_brotli__encoder__set_byte(self, a_wb, ((uint64_t)(v_tag_bucket + (((uint64_t)(v_num)) & v_block_mask))), v_tag); @@ -98976,16 +99129,14 @@ wuffs_brotli__metablock_builder__log2_floor_non_zero( const wuffs_brotli__metablock_builder* self, uint64_t a_n) { uint32_t v_x = 0; - uint32_t v_r = 0; v_x = ((uint32_t)(a_n)); - while (v_x > 1u) { - v_x >>= 1u; - if (v_r < 31u) { - v_r += 1u; - } - } - return v_r; + v_x |= (v_x >> 1u); + v_x |= (v_x >> 2u); + v_x |= (v_x >> 4u); + v_x |= (v_x >> 8u); + v_x |= (v_x >> 16u); + return WUFFS_BROTLI__LOG2_DEBRUIJN32[(((uint32_t)(v_x * 130329821u)) >> 27u)]; } // -------- func brotli.metablock_builder.prefix_encode_copy_distance @@ -101839,6 +101990,16 @@ wuffs_brotli__metablock_tester__get( return 0u; } +// -------- func brotli.encoder.ctz64 + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__encoder__ctz64( + const wuffs_brotli__encoder* self, + uint64_t a_x) { + return ((uint64_t)(WUFFS_BROTLI__CTZ64_DEBRUIJN[(((uint64_t)((a_x & ((uint64_t)(0u - a_x))) * 285870213051386505u)) >> 58u)])); +} + // -------- func brotli.encoder.byte_at WUFFS_BASE__GENERATED_C_CODE @@ -101861,12 +102022,11 @@ wuffs_brotli__encoder__peek_u16( const wuffs_brotli__encoder* self, wuffs_base__slice_u8 a_s, uint64_t a_i) { - wuffs_base__slice_u8 v_q = {0}; - - if (a_i < ((uint64_t)(a_s.len))) { - v_q = wuffs_base__slice_u8__subslice_i(a_s, a_i); - if (((uint64_t)(v_q.len)) >= 2u) { - return wuffs_base__peek_u16le__no_bounds_check(v_q.ptr); + if (a_i < 18446744073709486080u) { + if ((a_i + 2u) <= ((uint64_t)(a_s.len))) { + return wuffs_base__peek_u16le__no_bounds_check(wuffs_base__slice_u8__subslice_ij(a_s, + a_i, + (a_i + 2u)).ptr); } } return 0u; @@ -101881,12 +102041,11 @@ wuffs_brotli__encoder__poke_u16( wuffs_base__slice_u8 a_s, uint64_t a_i, uint16_t a_v) { - wuffs_base__slice_u8 v_q = {0}; - - if (a_i < ((uint64_t)(a_s.len))) { - v_q = wuffs_base__slice_u8__subslice_i(a_s, a_i); - if (((uint64_t)(v_q.len)) >= 2u) { - wuffs_base__poke_u16le__no_bounds_check(v_q.ptr, a_v); + if (a_i < 18446744073709486080u) { + if ((a_i + 2u) <= ((uint64_t)(a_s.len))) { + wuffs_base__poke_u16le__no_bounds_check(wuffs_base__slice_u8__subslice_ij(a_s, + a_i, + (a_i + 2u)).ptr, a_v); } } return wuffs_base__make_empty_struct(); @@ -101901,12 +102060,11 @@ wuffs_brotli__encoder__poke_u32( wuffs_base__slice_u8 a_s, uint64_t a_i, uint32_t a_v) { - wuffs_base__slice_u8 v_q = {0}; - - if (a_i < ((uint64_t)(a_s.len))) { - v_q = wuffs_base__slice_u8__subslice_i(a_s, a_i); - if (((uint64_t)(v_q.len)) >= 4u) { - wuffs_base__poke_u32le__no_bounds_check(v_q.ptr, a_v); + if (a_i < 18446744073709486080u) { + if ((a_i + 4u) <= ((uint64_t)(a_s.len))) { + wuffs_base__poke_u32le__no_bounds_check(wuffs_base__slice_u8__subslice_ij(a_s, + a_i, + (a_i + 4u)).ptr, a_v); } } return wuffs_base__make_empty_struct(); diff --git a/script/manual-test-brotli-float.c b/script/manual-test-brotli-float.c index 22953fc2..561bb28d 100644 --- a/script/manual-test-brotli-float.c +++ b/script/manual-test-brotli-float.c @@ -23,6 +23,9 @@ // gcc -O2 -std=c99 manual-test-brotli-float.c -o /tmp/brfloat // /tmp/brfloat # Default: 100 million operations per op. // /tmp/brfloat 1000000 # Or pass the number of operations per op. +// It also checks the integer bit helpers (log2_floor_non_zero, for every u32, +// and ctz64) against the compiler's __builtin_clz and __builtin_ctzll. +// // It prints one line per op and the first few mismatches (if any), and exits // with a non-zero status if there were any mismatches. NaN results only need // to be NaN (on both sides). @@ -46,6 +49,7 @@ #error "Build without -ffast-math and without -mfma / -march=native" #endif +static wuffs_brotli__encoder* g_enc; static wuffs_brotli__bit_cost* g_bc; static uint64_t g_rng = 0x123456789ABCDEF1ull; @@ -334,6 +338,41 @@ check_conversions(uint64_t n) { g_total_mismatches += mismatches; } +static void // +check_bit_helpers(uint64_t n) { + uint64_t mismatches = 0; + wuffs_brotli__encoder* enc = g_enc; + // log2_floor_non_zero truncates its argument to 32 bits (and returns 0 for + // 0), so check every u32 value, with random high bits. + uint32_t v = 0; + do { + uint64_t x = ((uint64_t)(rnd() & 0xFFFFFFFF00000000ull)) | v; + uint32_t want = v ? (31 - (uint32_t)__builtin_clz(v)) : 0; + uint32_t have0 = wuffs_brotli__encoder__log2_floor_non_zero(enc, x); + uint32_t have1 = wuffs_brotli__metablock_builder__log2_floor_non_zero( + &enc->private_data.f_mb, x); + if (((have0 != want) || (have1 != want)) && (mismatches++ < 5)) { + printf("log2_floor_non_zero(0x%016" PRIX64 "): have %" PRIu32 + " and %" PRIu32 ", want %" PRIu32 "\n", x, have0, have1, want); + } + } while (++v != 0); + for (uint64_t i = 0; i < n; i++) { + uint64_t x = (i < 64) ? (1ull << i) : rnd_int(); + if (x == 0) { + continue; + } + uint64_t want = (uint64_t)__builtin_ctzll(x); + uint64_t have = wuffs_brotli__encoder__ctz64(enc, x); + if ((have != want) && (mismatches++ < 5)) { + printf("ctz64(0x%016" PRIX64 "): have %" PRIu64 ", want %" PRIu64 "\n", + x, have, want); + } + } + printf("%-8s %12" PRIu64 " iterations, %8" PRIu64 " mismatches\n", "bits", + n + (uint64_t)0x100000000ull, mismatches); + g_total_mismatches += mismatches; +} + int // main(int argc, char** argv) { uint64_t n = 100000000; @@ -345,6 +384,7 @@ main(int argc, char** argv) { printf("alloc failed\n"); return 1; } + g_enc = enc; g_bc = &enc->private_data.f_bc; check_f32_add(n); @@ -359,6 +399,7 @@ main(int argc, char** argv) { check_f64_gt(n); check_f64_ge(n); check_conversions(n); + check_bit_helpers(n); free(enc); if (g_total_mismatches) { diff --git a/std/brotli/README.md b/std/brotli/README.md index ba407481..4e3c9f9f 100644 --- a/std/brotli/README.md +++ b/std/brotli/README.md @@ -883,7 +883,9 @@ by both this package's decoder and libbrotlidec. `script/manual-test-brotli-float.c` checks the encoder's package-private float fast paths (`encode_float.wuffs`, which spare the hot loops the cross-package `std/softfloat` calls) against the C compiler's hardware float -and double arithmetic, for operands biased towards their edge cases. +and double arithmetic, for operands biased towards their edge cases, and +checks the integer bit helpers (`log2_floor_non_zero` for every 32-bit value, +and `ctz64`). `script/manual-test-brotli-encoder-parity.c` runs a larger grid over every file in `test/data/brotli`, some larger files in `test/data` (up to about 400 KiB) and some synthetic inputs (up to 1 MiB). Its `-q=LO-HI` and @@ -1131,13 +1133,19 @@ metablock) for as long as it needs the results. - `workbuf_len` is a worst case (or, at qualities 10 and 11, a practical bound). A future option could take the input length (like the C code's size hint) to give a tighter bound. -- Speed. Compressing the 3 MiB `test/data/harvesters.bmp` in one call (on - x86-64, lgwin 22), the Wuffs encoder takes about 1.2 times libbrotlienc's - time at quality 5, 1.7 times at quality 9, 4 times at quality 10 and 3.5 - times at quality 11 (on text, 2 to 3 times at qualities 10 and 11), mostly - because of the soft float arithmetic in the Zopfli cost model, the block - splitter and histogram clustering. Decoding takes about 1.2 times - libbrotlidec's time. +- Speed. On a mix of text, JSON, map data, an image and an ISO image (on + x86-64, lgwin 22, one call each, reusing the workbuf), the Wuffs encoder + takes about 1.2 to 1.8 times libbrotlienc's time at qualities 0 to 8 (about + 1.8 times at the qualities 4 to 6 that web servers use for on-the-fly + compression) and about the same time at quality 9. At qualities 10 and 11 + it takes about 2 times libbrotlienc's time on text and 3.5 to 4 times on + images (such as the 3 MiB `test/data/harvesters.bmp`), mostly because of + the soft float arithmetic in the Zopfli cost model, the block splitter and + histogram clustering. Decoding takes about 1.2 to 1.6 times libbrotlidec's + time. The workbuf is sized for the window (about 48 to 62 MiB at lgwin 22 + and qualities 5 to 9), so callers that compress many small inputs should + reuse it: allocating and first touching a fresh one for each call costs + more than the compression of a small input. - Serialized (shared) dictionaries, which only an experimental (`BROTLI_EXPERIMENTAL`) libbrotlienc build supports. - Changing parameters mid-stream. diff --git a/std/brotli/decode_metablock.wuffs b/std/brotli/decode_metablock.wuffs index ae301fec..26787df2 100644 --- a/std/brotli/decode_metablock.wuffs +++ b/std/brotli/decode_metablock.wuffs @@ -738,6 +738,13 @@ pri func decoder.process_commands_fast!(src: base.io_reader, workbuf: slice base var dhi : base.u64 var slo : base.u64 var shi : base.u64 + var cm_slice : base.u32[..= 16320] + var wmask : base.u32 + var pos : base.u32 + var lit_n : base.u32 + var lit_i : base.u32 + var wi : base.u64 + var copied16 : base.bool if this.n_bits >= 8 { return ok @@ -829,26 +836,38 @@ pri func decoder.process_commands_fast!(src: base.io_reader, workbuf: slice base } p1 = this.get_byte(workbuf: args.workbuf, distance: 1) as base.u32 p2 = this.get_byte(workbuf: args.workbuf, distance: 2) as base.u32 - while this.insert_length > 0 { - if (this.block_length[0] == 0) or ((this.pos ~mod+ 1) >= this.rb_size) { - break.outer - } + mode_offset = this.context_mode << 9 + cm_slice = this.context_map_slice + wmask = this.window_size ~mod- 1 + pos = this.pos + // lit_n is how many literals the loop below decodes before this + // command's literals, this block's literals or the ring buffer + // (before the next ring_event) run out. The loop keeps pos, the + // counts and the symbol in local variables, and writes them back + // to this.etc fields before every break. + lit_n = this.insert_length.min(no_more_than: this.block_length[0]) + if (pos ~mod+ 1) >= this.rb_size { + lit_n = 0 + } else { + lit_n = lit_n.min(no_more_than: (this.rb_size ~mod- pos) ~mod- 1) + } + lit_i = 0 + while lit_i < lit_n { if n_bits < 24 { if args.src.length() < 8 { - break.outer + break } bits |= args.src.peek_u64le() ~mod<< n_bits args.src.skip_u32_fast!(actual: (63 - n_bits) >> 3, worst_case: 8) n_bits |= 56 } - mode_offset = this.context_mode << 9 ctx = (CONTEXT_LOOKUP[mode_offset | p1] | CONTEXT_LOOKUP[mode_offset | 0x100 | p2]) as base.u32 - tree = this.context_maps[this.context_map_slice + ctx] as base.u32 + tree = this.context_maps[cm_slice + ctx] as base.u32 i = (LIT_HUFFS_BASE + (tree * LIT_HUFFS_STRIDE)) ~mod+ ((bits & 0xFF) as base.u32) if i >= HUFFS_LEN { ret = "#internal error: inconsistent Huffman table" - break.outer + break } e = this.huffs[i] as base.u32 e_bits = e & 15 @@ -856,23 +875,34 @@ pri func decoder.process_commands_fast!(src: base.io_reader, workbuf: slice base i = ((LIT_HUFFS_BASE + (tree * LIT_HUFFS_STRIDE)) ~mod+ (e >> 4)) ~mod+ ((((bits >> 8) & 0x7FFF) as base.u32) & (((1 as base.u32) << (e_bits - 8)) - 1)) if i >= HUFFS_LEN { ret = "#internal error: inconsistent Huffman table" - break.outer + break } e = this.huffs[i] as base.u32 e_bits = (e & 15) + 8 } if n_bits < e_bits { ret = "#internal error: inconsistent bit reader" - break.outer + break } bits >>= e_bits n_bits -= e_bits - this.symbol = e >> 4 - this.put_byte!(workbuf: args.workbuf, b: (this.symbol & 0xFF) as base.u8) p2 = p1 - p1 = this.symbol & 0xFF - this.block_length[0] ~mod-= 1 - this.insert_length ~mod-= 1 + p1 = (e >> 4) & 0xFF + wi = (pos & wmask) as base.u64 + if wi < args.workbuf.length() { + args.workbuf[wi] = p1 as base.u8 + } + pos ~mod+= 1 + lit_i ~mod+= 1 + } + if lit_i > 0 { + this.symbol = p1 + this.pos = pos + this.block_length[0] ~mod-= lit_i + this.insert_length ~mod-= lit_i + } + if (ret.is_error()) or (this.insert_length > 0) { + break.outer } this.cmd_state = 2 if (this.mb_remaining == 0) or (this.mb_remaining >= 0x8000_0000) { @@ -971,6 +1001,18 @@ pri func decoder.process_commands_fast!(src: base.io_reader, workbuf: slice base this.mb_remaining ~mod-= n // Copy in chunks of at most distance bytes, like process_commands. k64 = 0 + if (n <= 16) and (distance >= 16) and + ((dst_i ~mod+ 16) <= (this.rb_size as base.u64)) and + ((src_i ~mod+ 16) <= (this.window_size as base.u64)) { + // Like the C code, copy 16 bytes, overwriting up to 16 - n bytes + // after the copy's end. Those bytes are more than the maximum + // backward distance (BROTLI_WINDOW_GAP is 16) away from any later + // position, so they are never read before being rewritten. + copied16 = this.copy16!(workbuf: args.workbuf, d: dst_i, s: src_i) + if copied16 { + k64 = n as base.u64 + } + } while k64 < (n as base.u64) { n64 = (n as base.u64) ~mod- k64 n64 = n64.min(no_more_than: distance as base.u64) @@ -1098,3 +1140,34 @@ pri func decoder.take_distance_from_ring_buffer!(code: base.u32[..= 15]) { this.distance_code = v - 3 } } + +// copy16 copies the 16 bytes at workbuf[s ..] to workbuf[d ..], if both are +// in bounds (and returns whether it did). The two ranges must not overlap. +pri func decoder.copy16!(workbuf: slice base.u8, d: base.u64, s: base.u64) base.bool { + var lo : base.u64 + var hi : base.u64 + var q : slice base.u8 + var r : slice base.u8 + + if (args.s > args.workbuf.length()) or (args.d > args.workbuf.length()) { + return false + } + q = args.workbuf[args.s ..] + if q.length() >= 16 { + lo = q.peek_u64le() + q = q[8 ..] + if q.length() >= 8 { + hi = q.peek_u64le() + q = args.workbuf[args.d ..] + if q.length() >= 16 { + r = q[8 ..] + q.poke_u64le!(a: lo) + if r.length() >= 8 { + r.poke_u64le!(a: hi) + return true + } + } + } + } + return false +} diff --git a/std/brotli/encode_bit_writer.wuffs b/std/brotli/encode_bit_writer.wuffs index d6acd49f..9abf7020 100644 --- a/std/brotli/encode_bit_writer.wuffs +++ b/std/brotli/encode_bit_writer.wuffs @@ -125,33 +125,27 @@ pri func encoder.store_meta_block_header!(len: base.u64, is_uncompressed: base.b } } +// LOG2_DEBRUIJN32 maps (v * 0x07C4_ACDD) >> 27, for v = (1 << (k + 1)) - 1, +// to k. +pri const LOG2_DEBRUIJN32 : roarray[32] base.u32[..= 31] = [ + 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, + 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31, +] + // Mirrors Log2FloorNonZero in fast_log.h, which (with BROTLI_BSR32) truncates // its argument to 32 bits. It returns 0 for an argument of 0. pri func encoder.log2_floor_non_zero(n: base.u64) base.u32[..= 31] { var x : base.u32 - var r : base.u32 + // Smear the highest set bit of the low 32 bits of n rightwards, then + // look up its position. x = (args.n & 0xFFFF_FFFF) as base.u32 - if x >= 0x1_0000 { - x >>= 16 - r = 16 - } - if x >= 0x100 { - x >>= 8 - r ~mod+= 8 - } - if x >= 0x10 { - x >>= 4 - r ~mod+= 4 - } - if x >= 4 { - x >>= 2 - r ~mod+= 2 - } - if x >= 2 { - r ~mod+= 1 - } - return r & 31 + x |= x >> 1 + x |= x >> 2 + x |= x >> 4 + x |= x >> 8 + x |= x >> 16 + return LOG2_DEBRUIJN32[(x ~mod* 0x07C4_ACDD) >> 27] } // Mirrors FastLog2 in fast_log.h. The result is the bit pattern of an IEEE @@ -193,12 +187,10 @@ pri func encoder.bits_entropy_of_histogram() base.u64 { // code's BROTLI_UNALIGNED_LOAD64LE. The C code never reads out of bounds, so // the zero result for an out of bounds read should never be used. pri func encoder.load64le(s: roslice base.u8, i: base.u64) base.u64 { - var q : roslice base.u8 - - if args.i < args.s.length() { - q = args.s[args.i ..] - if q.length() >= 8 { - return q.peek_u64le() + if args.i < 0xFFFF_FFFF_FFFF_0000 { + if (args.i + 8) <= args.s.length() { + assert args.i <= (args.i + 8) via "a <= (a + b): 0 <= b"(b: 8) + return args.s[args.i .. args.i + 8].peek_u64le() } } return 0 @@ -206,12 +198,10 @@ pri func encoder.load64le(s: roslice base.u8, i: base.u64) base.u64 { // load32le is like load64le but for 4 bytes. pri func encoder.load32le(s: roslice base.u8, i: base.u64) base.u32 { - var q : roslice base.u8 - - if args.i < args.s.length() { - q = args.s[args.i ..] - if q.length() >= 4 { - return q.peek_u32le() + if args.i < 0xFFFF_FFFF_FFFF_0000 { + if (args.i + 4) <= args.s.length() { + assert args.i <= (args.i + 4) via "a <= (a + b): 0 <= b"(b: 4) + return args.s[args.i .. args.i + 4].peek_u32le() } } return 0 @@ -235,11 +225,7 @@ pri func encoder.find_match_length_with_limit(s: roslice base.u8, i1: base.u64, while (n >= 8) and (a.length() >= 8) and (b.length() >= 8) { x = a.peek_u64le() ^ b.peek_u64le() if x <> 0 { - while (x & 0xFF) == 0 { - matched ~mod+= 1 - x >>= 8 - } - return matched + return matched ~mod+ (this.ctz64(x: x) >> 3) } matched ~mod+= 8 n -= 8 diff --git a/std/brotli/encode_hasher.wuffs b/std/brotli/encode_hasher.wuffs index b51e3b5e..1e64c3dd 100644 --- a/std/brotli/encode_hasher.wuffs +++ b/std/brotli/encode_hasher.wuffs @@ -400,7 +400,6 @@ pri func encoder.hasher_store!(wb: slice base.u8, ix: base.u64) { var bsize : base.u64 var off : base.u64 var num : base.u16 - var hash : base.u32 var bank : base.u64 var idx : base.u64 var delta : base.u64 @@ -408,6 +407,10 @@ pri func encoder.hasher_store!(wb: slice base.u8, ix: base.u64) { var fc_bits : base.u32[..= 16] htype = this.hasher_type + if (htype == 58) or (htype == 68) { + this.hasher_store_h58!(wb: args.wb, ix: args.ix) + return nothing + } if this.rb_buf_off > args.wb.length() { return nothing } @@ -427,16 +430,6 @@ pri func encoder.hasher_store!(wb: slice base.u8, ix: base.u64) { this.poke_u16!(s: args.wb, i: this.h_off0 ~mod+ (2 ~mod* (key as base.u64)), v: num ~mod+ 1) this.poke_u32!(s: args.wb, i: this.h_off1 ~mod+ (4 ~mod* off), v: (args.ix & 0xFFFF_FFFF) as base.u32) - } else if (htype == 58) or (htype == 68) { - hash = this.hash_longest(data: data, i: args.ix & (this.rb_mask as base.u64)) - key = hash >> 8 - num = this.peek_u16(s: args.wb, i: this.h_off0 ~mod+ (2 ~mod* (key as base.u64))) - off = ((num as base.u64) & (((1 as base.u64) << this.h_block_bits) - 1)) ~mod+ - ((key as base.u64) << this.h_block_bits) - this.poke_u16!(s: args.wb, i: this.h_off0 ~mod+ (2 ~mod* (key as base.u64)), v: num ~mod- 1) - this.poke_u32!(s: args.wb, i: this.h_off2 ~mod+ (4 ~mod* off), v: (args.ix & 0xFFFF_FFFF) as base.u32) - this.set_byte!(s: args.wb, i: this.h_off1 ~mod+ off, v: (hash & 0xFF) as base.u8) - } else if (htype == 40) or (htype == 41) or (htype == 42) { key = this.hash_forgetful(data: data, i: args.ix & (this.rb_mask as base.u64)) fc_bits = this.fc_bank_bits() @@ -457,6 +450,36 @@ pri func encoder.hasher_store!(wb: slice base.u8, ix: base.u64) { } } +// hasher_store_h58 is FN(Store) for H58 and H68. +pri func encoder.hasher_store_h58!(wb: slice base.u8, ix: base.u64) { + var hash : base.u32 + var key : base.u64 + var koff : base.u64 + var num : base.u16 + var off : base.u64 + + hash = this.hash_h58(wb: args.wb, ix: args.ix) + key = (hash >> 8) as base.u64 + koff = this.h_off0 ~mod+ (2 ~mod* key) + num = this.peek_u16(s: args.wb, i: koff) + off = ((num as base.u64) & (((1 as base.u64) << this.h_block_bits) - 1)) ~mod+ (key << this.h_block_bits) + this.poke_u16!(s: args.wb, i: koff, v: num ~mod- 1) + this.poke_u32!(s: args.wb, i: this.h_off2 ~mod+ (4 ~mod* off), v: (args.ix & 0xFFFF_FFFF) as base.u32) + this.set_byte!(s: args.wb, i: this.h_off1 ~mod+ off, v: (hash & 0xFF) as base.u8) +} + +// hash_h58 is hash_longest for H58 and H68, of the ring buffer bytes at +// position ix (masked). +pri func encoder.hash_h58(wb: roslice base.u8, ix: base.u64) base.u32 { + var i : base.u64 + + i = this.rb_buf_off ~mod+ (args.ix & (this.rb_mask as base.u64)) + if this.hasher_type == 58 { + return (this.load32le(s: args.wb, i: i) ~mod* HASH_MUL_32) >> ((24 - this.h_bucket_bits) & 31) + } + return ((this.load64le(s: args.wb, i: i) ~mod* ((HASH_MUL_64 << 24) & 0xFFFF_FFFF_FFFF_FFFF)) >> 41) as base.u32 +} + // Mirrors FN(StoreRange) of every hasher. pri func encoder.hasher_store_range!(wb: slice base.u8, ix_start: base.u64, ix_end: base.u64) { var i : base.u64 @@ -803,6 +826,8 @@ pri func encoder.find_last_distances!(data: roslice base.u8, cur_ix: base.u64, m var prev_ix : base.u64 var len : base.u64 var score : base.u64 + var cur_byte : base.u8 + var n : base.u32 mask = this.rb_mask as base.u64 cur_ix_masked = args.cur_ix & mask @@ -811,15 +836,14 @@ pri func encoder.find_last_distances!(data: roslice base.u8, cur_ix: base.u64, m this.sr_len = 0 this.sr_len_code_delta = 0 + // cur_byte is data[cur_ix_masked + best_len], updated with best_len. + cur_byte = this.byte_at(s: args.data, i: cur_ix_masked ~mod+ best_len) + n = this.h_num_last_distances i = 0 - while i < this.h_num_last_distances { + while i < n { backward = this.dist_cache_u64(i: i) prev_ix = args.cur_ix ~mod- backward - if prev_ix >= args.cur_ix { - i ~mod+= 1 - continue - } - if backward > args.max_backward { + if (prev_ix >= args.cur_ix) or (backward > args.max_backward) { i ~mod+= 1 continue } @@ -828,7 +852,7 @@ pri func encoder.find_last_distances!(data: roslice base.u8, cur_ix: base.u64, m break } if ((prev_ix ~mod+ best_len) > mask) or - (this.byte_at(s: args.data, i: cur_ix_masked ~mod+ best_len) <> this.byte_at(s: args.data, i: prev_ix ~mod+ best_len)) { + (cur_byte <> this.byte_at(s: args.data, i: prev_ix ~mod+ best_len)) { i ~mod+= 1 continue } @@ -845,6 +869,7 @@ pri func encoder.find_last_distances!(data: roslice base.u8, cur_ix: base.u64, m this.sr_len = best_len this.sr_distance = backward this.sr_score = best_score + cur_byte = this.byte_at(s: args.data, i: cur_ix_masked ~mod+ best_len) } } } @@ -853,55 +878,6 @@ pri func encoder.find_last_distances!(data: roslice base.u8, cur_ix: base.u64, m return best_len } -// try_bucket_entry is the body of the bucket loop shared by the H5, H6, H58 -// and H68 FN(FindLongestMatch) functions, for one prev_ix candidate. It -// returns 0 to continue, 1 to break, or 2 + the new best_len. -pri func encoder.try_bucket_entry!(data: roslice base.u8, cur_ix: base.u64, prev_ix_in: base.u64, best_len: base.u64, max_length: base.u64, max_backward: base.u64) base.u64 { - var mask : base.u64 - var cur_ix_masked : base.u64 - var prev_ix : base.u64 - var backward : base.u64 - var len : base.u64 - var score : base.u64 - - mask = this.rb_mask as base.u64 - cur_ix_masked = args.cur_ix & mask - backward = args.cur_ix ~mod- args.prev_ix_in - if backward > args.max_backward { - return 1 - } - prev_ix = args.prev_ix_in & mask - if (cur_ix_masked ~mod+ args.best_len) > mask { - return 1 - } - if ((prev_ix ~mod+ args.best_len) > mask) or - (this.load32le(s: args.data, i: (cur_ix_masked ~mod+ args.best_len) ~mod- 3) <> - this.load32le(s: args.data, i: (prev_ix ~mod+ args.best_len) ~mod- 3)) { - return 0 - } - if (this.hasher_type == 6) or (this.hasher_type == 68) { - if this.load32le(s: args.data, i: cur_ix_masked) <> this.load32le(s: args.data, i: prev_ix) { - return 0 - } - len = this.find_match_length_with_limit(s: args.data, i1: prev_ix ~mod+ 4, i2: cur_ix_masked ~mod+ 4, - limit: args.max_length ~mod- 4) ~mod+ 4 - } else { - len = this.find_match_length_with_limit(s: args.data, i1: prev_ix, i2: cur_ix_masked, limit: args.max_length) - if len < 4 { - return 0 - } - } - score = this.backward_reference_score(copy_length: len, backward_reference_offset: backward) - if this.sr_score < score { - // The C code's best_score is always equal to out->score here. - this.sr_len = len - this.sr_distance = backward - this.sr_score = score - return 2 ~mod+ len - } - return 0 -} - // Mirrors FN(FindLongestMatch) in hash_longest_match_inc.h (H5) and // hash_longest_match64_inc.h (H6). pri func encoder.find_longest_match_h5!(wb: slice base.u8, cur_ix: base.u64, max_length: base.u64, max_backward: base.u64, dictionary_distance: base.u64, max_distance: base.u64) { @@ -916,7 +892,13 @@ pri func encoder.find_longest_match_h5!(wb: slice base.u8, cur_ix: base.u64, max var bucket : base.u64 var down : base.u64 var i : base.u64 - var r : base.u64 + var mask : base.u64 + var is64 : base.bool + var cur4 : base.u32 + var prev_ix : base.u64 + var backward : base.u64 + var len : base.u64 + var score : base.u64 if this.rb_buf_off > args.wb.length() { return nothing @@ -938,20 +920,46 @@ pri func encoder.find_longest_match_h5!(wb: slice base.u8, cur_ix: base.u64, max if (num as base.u64) > block_size { down = (num as base.u64) ~mod- block_size } + // Try the bucket's entries, newest first. Like the C code, this + // compares 4 bytes ending at best_len + 1 before the full match. + mask = this.rb_mask as base.u64 + is64 = this.hasher_type == 6 + cur4 = this.load32le(s: data, i: (cur_ix_masked ~mod+ best_len) ~mod- 3) i = num as base.u64 while i > down { i ~mod-= 1 - r = this.try_bucket_entry!( - data: data, - cur_ix: args.cur_ix, - prev_ix_in: this.load32le(s: args.wb, i: bucket ~mod+ (4 ~mod* (i & block_mask))) as base.u64, - best_len: best_len, - max_length: args.max_length, - max_backward: args.max_backward) - if r == 1 { + prev_ix = this.load32le(s: args.wb, i: bucket ~mod+ (4 ~mod* (i & block_mask))) as base.u64 + backward = args.cur_ix ~mod- prev_ix + if backward > args.max_backward { break - } else if r >= 2 { - best_len = r ~mod- 2 + } + prev_ix &= mask + if (cur_ix_masked ~mod+ best_len) > mask { + break + } + if ((prev_ix ~mod+ best_len) > mask) or + (cur4 <> this.load32le(s: data, i: (prev_ix ~mod+ best_len) ~mod- 3)) { + continue + } + if is64 { + if this.load32le(s: data, i: cur_ix_masked) <> this.load32le(s: data, i: prev_ix) { + continue + } + len = this.find_match_length_with_limit(s: data, i1: prev_ix ~mod+ 4, i2: cur_ix_masked ~mod+ 4, + limit: args.max_length ~mod- 4) ~mod+ 4 + } else { + len = this.find_match_length_with_limit(s: data, i1: prev_ix, i2: cur_ix_masked, limit: args.max_length) + if len < 4 { + continue + } + } + score = this.backward_reference_score(copy_length: len, backward_reference_offset: backward) + if this.sr_score < score { + this.sr_len = len + this.sr_distance = backward + this.sr_score = score + best_len = len + cur4 = this.load32le(s: data, i: (cur_ix_masked ~mod+ best_len) ~mod- 3) } } this.poke_u32!(s: args.wb, i: bucket ~mod+ (4 ~mod* ((num as base.u64) & block_mask)), v: (args.cur_ix & 0xFFFF_FFFF) as base.u32) @@ -984,7 +992,17 @@ pri func encoder.find_longest_match_h58!(wb: slice base.u8, cur_ix: base.u64, ma var limit : base.u64 var k : base.u64 var rb_index : base.u64 - var r : base.u64 + var splat : base.u64 + var matches : base.u64 + var chunk : base.u64 + var j : base.u64 + var mask : base.u64 + var is64 : base.bool + var cur4 : base.u32 + var prev_ix : base.u64 + var backward : base.u64 + var len : base.u64 + var score : base.u64 if this.rb_buf_off > args.wb.length() { return nothing @@ -992,7 +1010,7 @@ pri func encoder.find_longest_match_h58!(wb: slice base.u8, cur_ix: base.u64, ma data = args.wb[this.rb_buf_off ..] cur_ix_masked = args.cur_ix & (this.rb_mask as base.u64) min_score = this.sr_score - hash = this.hash_longest(data: data, i: cur_ix_masked) + hash = this.hash_h58(wb: args.wb, ix: args.cur_ix) key = (hash >> 8) as base.u64 block_size = (1 as base.u64) << this.h_block_bits block_mask = block_size ~mod- 1 @@ -1014,24 +1032,70 @@ pri func encoder.find_longest_match_h58!(wb: slice base.u8, cur_ix: base.u64, ma if block_size > (n as base.u64) { limit = n as base.u64 } - k = 0 - while k < limit { + // Like GetMatchingTagMask's non-SSE2 code: bit j of matches is whether + // the tag of entry j matches, 8 entries at a time. + splat = (tag as base.u64) ~mod* 0x0101_0101_0101_0101 + matches = 0 + j = 0 + while j < block_size { + chunk = this.load64le(s: args.wb, i: tag_bucket ~mod+ j) ^ splat + // The 0x80 bit of each byte is set when that byte is non-zero. + chunk = (((chunk | 0x8080_8080_8080_8080) ~mod- 0x0101_0101_0101_0101) | chunk) & 0x8080_8080_8080_8080 + chunk = (chunk ^ 0x8080_8080_8080_8080) >> 7 + matches |= ((chunk ~mod* 0x0102_0408_1020_4080) >> 56) ~mod<< (j & 63) + j ~mod+= 8 + } + // Rotate so that bit k is entry (head + k) & block_mask. + if head > 0 { + matches = ((matches >> (head & 63)) | (matches ~mod<< ((block_size ~mod- head) & 63))) & + ((0xFFFF_FFFF_FFFF_FFFF as base.u64) >> ((64 ~mod- block_size) & 63)) + } + if limit < 64 { + matches &= ((1 as base.u64) << (limit & 63)) ~mod- 1 + } + // Try the entries whose tags match, in the C code's order (from head). + // Like the C code, this compares 4 bytes ending at best_len + 1 before + // the full match. + mask = this.rb_mask as base.u64 + is64 = this.hasher_type == 68 + cur4 = this.load32le(s: data, i: (cur_ix_masked ~mod+ best_len) ~mod- 3) + while matches <> 0 { + k = this.ctz64(x: matches) + matches &= matches ~mod- 1 rb_index = (head ~mod+ k) & block_mask - if this.byte_at(s: args.wb, i: tag_bucket ~mod+ rb_index) == tag { - r = this.try_bucket_entry!( - data: data, - cur_ix: args.cur_ix, - prev_ix_in: this.load32le(s: args.wb, i: bucket ~mod+ (4 ~mod* rb_index)) as base.u64, - best_len: best_len, - max_length: args.max_length, - max_backward: args.max_backward) - if r == 1 { - break - } else if r >= 2 { - best_len = r ~mod- 2 + prev_ix = this.load32le(s: args.wb, i: bucket ~mod+ (4 ~mod* rb_index)) as base.u64 + backward = args.cur_ix ~mod- prev_ix + if backward > args.max_backward { + break + } + prev_ix &= mask + if (cur_ix_masked ~mod+ best_len) > mask { + break + } + if ((prev_ix ~mod+ best_len) > mask) or + (cur4 <> this.load32le(s: data, i: (prev_ix ~mod+ best_len) ~mod- 3)) { + continue + } + if is64 { + if this.load32le(s: data, i: cur_ix_masked) <> this.load32le(s: data, i: prev_ix) { + continue + } + len = this.find_match_length_with_limit(s: data, i1: prev_ix ~mod+ 4, i2: cur_ix_masked ~mod+ 4, + limit: args.max_length ~mod- 4) ~mod+ 4 + } else { + len = this.find_match_length_with_limit(s: data, i1: prev_ix, i2: cur_ix_masked, limit: args.max_length) + if len < 4 { + continue } } - k ~mod+= 1 + score = this.backward_reference_score(copy_length: len, backward_reference_offset: backward) + if this.sr_score < score { + this.sr_len = len + this.sr_distance = backward + this.sr_score = score + best_len = len + cur4 = this.load32le(s: data, i: (cur_ix_masked ~mod+ best_len) ~mod- 3) + } } this.poke_u32!(s: args.wb, i: bucket ~mod+ (4 ~mod* ((num as base.u64) & block_mask)), v: (args.cur_ix & 0xFFFF_FFFF) as base.u32) this.set_byte!(s: args.wb, i: tag_bucket ~mod+ ((num as base.u64) & block_mask), v: tag) diff --git a/std/brotli/encode_metablock.wuffs b/std/brotli/encode_metablock.wuffs index fbc42351..4f352e31 100644 --- a/std/brotli/encode_metablock.wuffs +++ b/std/brotli/encode_metablock.wuffs @@ -283,16 +283,16 @@ pri func metablock_builder.h_population_cost!(wb: roslice base.u8, h: base.u64, // Mirrors Log2FloorNonZero in fast_log.h (which truncates n to 32 bits). pri func metablock_builder.log2_floor_non_zero(n: base.u64) base.u32[..= 31] { var x : base.u32 - var r : base.u32[..= 31] + // Smear the highest set bit of the low 32 bits of n rightwards, then + // look up its position. x = (args.n & 0xFFFF_FFFF) as base.u32 - while x > 1 { - x >>= 1 - if r < 31 { - r += 1 - } - } - return r + x |= x >> 1 + x |= x >> 2 + x |= x >> 4 + x |= x >> 8 + x |= x >> 16 + return LOG2_DEBRUIJN32[(x ~mod* 0x07C4_ACDD) >> 27] } // Mirrors PrefixEncodeCopyDistance in prefix.h. It returns the code (the diff --git a/std/brotli/encode_util.wuffs b/std/brotli/encode_util.wuffs index 2d10a076..73271f1f 100644 --- a/std/brotli/encode_util.wuffs +++ b/std/brotli/encode_util.wuffs @@ -18,6 +18,21 @@ // out of bounds branches here (which do nothing, or return 0) should never be // taken. +// CTZ64_DEBRUIJN maps ((x & -x) * 0x03F7_9D71_B4CB_0A89) >> 58 to the +// number of trailing zero bits of the non-zero x. +pri const CTZ64_DEBRUIJN : roarray[64] base.u8[..= 63] = [ + 0, 1, 48, 2, 57, 49, 28, 3, 61, 58, 50, 42, 38, 29, 17, 4, + 62, 55, 59, 36, 53, 51, 43, 22, 45, 39, 33, 30, 24, 18, 12, 5, + 63, 47, 56, 27, 60, 41, 37, 16, 54, 35, 52, 21, 44, 32, 23, 11, + 46, 26, 40, 15, 34, 20, 31, 10, 25, 14, 19, 9, 13, 8, 7, 6, +] + +// ctz64 returns the number of trailing zero bits of x, which must be +// non-zero (for x == 0 it returns 0). +pri func encoder.ctz64(x: base.u64) base.u64[..= 63] { + return CTZ64_DEBRUIJN[((args.x & (0 ~mod- args.x)) ~mod* 0x03F7_9D71_B4CB_0A89) >> 58] as base.u64 +} + // byte_at returns s[i], or 0 if i is out of bounds. pri func encoder.byte_at(s: roslice base.u8, i: base.u64) base.u8 { if args.i < args.s.length() { @@ -28,12 +43,10 @@ pri func encoder.byte_at(s: roslice base.u8, i: base.u64) base.u8 { // peek_u16 returns the 2 bytes (little-endian) of s starting at i. pri func encoder.peek_u16(s: roslice base.u8, i: base.u64) base.u16 { - var q : roslice base.u8 - - if args.i < args.s.length() { - q = args.s[args.i ..] - if q.length() >= 2 { - return q.peek_u16le() + if args.i < 0xFFFF_FFFF_FFFF_0000 { + if (args.i + 2) <= args.s.length() { + assert args.i <= (args.i + 2) via "a <= (a + b): 0 <= b"(b: 2) + return args.s[args.i .. args.i + 2].peek_u16le() } } return 0 @@ -41,24 +54,20 @@ pri func encoder.peek_u16(s: roslice base.u8, i: base.u64) base.u16 { // poke_u16 sets the 2 bytes (little-endian) of s starting at i. pri func encoder.poke_u16!(s: slice base.u8, i: base.u64, v: base.u16) { - var q : slice base.u8 - - if args.i < args.s.length() { - q = args.s[args.i ..] - if q.length() >= 2 { - q.poke_u16le!(a: args.v) + if args.i < 0xFFFF_FFFF_FFFF_0000 { + if (args.i + 2) <= args.s.length() { + assert args.i <= (args.i + 2) via "a <= (a + b): 0 <= b"(b: 2) + args.s[args.i .. args.i + 2].poke_u16le!(a: args.v) } } } // poke_u32 sets the 4 bytes (little-endian) of s starting at i. pri func encoder.poke_u32!(s: slice base.u8, i: base.u64, v: base.u32) { - var q : slice base.u8 - - if args.i < args.s.length() { - q = args.s[args.i ..] - if q.length() >= 4 { - q.poke_u32le!(a: args.v) + if args.i < 0xFFFF_FFFF_FFFF_0000 { + if (args.i + 4) <= args.s.length() { + assert args.i <= (args.i + 4) via "a <= (a + b): 0 <= b"(b: 4) + args.s[args.i .. args.i + 4].poke_u32le!(a: args.v) } } }