diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index f5acc1a0..9ae3b834 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -9474,14 +9474,14 @@ wuffs_brotli__encoder__add_dictionary_workbuf_len( WUFFS_BASE__GENERATED_C_CODE WUFFS_BASE__MAYBE_STATIC uint64_t wuffs_brotli__metablock_tester__bits_entropy( - const wuffs_brotli__metablock_tester* self, + wuffs_brotli__metablock_tester* self, wuffs_base__slice_u8 a_population, uint64_t a_size); WUFFS_BASE__GENERATED_C_CODE WUFFS_BASE__MAYBE_STATIC uint64_t wuffs_brotli__metablock_tester__population_cost( - const wuffs_brotli__metablock_tester* self, + wuffs_brotli__metablock_tester* self, wuffs_base__slice_u8 a_histogram, uint32_t a_data_size); @@ -9609,6 +9609,7 @@ struct wuffs_brotli__decoder__struct { bool f_implicit_zero_distance; uint32_t f_insert_length; uint32_t f_copy_length; + uint32_t f_cmd_state; bool f_quirk_large_window; bool f_large_window; bool f_started; @@ -9762,7 +9763,6 @@ struct wuffs_brotli__decoder__struct { } s_decode_block_switch; struct { uint32_t v_cmd_code; - uint32_t v_insert_length; uint32_t v_tree; uint32_t v_distance; uint32_t v_i; @@ -9773,6 +9773,7 @@ struct wuffs_brotli__decoder__struct { } s_process_commands; struct { uint32_t v_code; + uint32_t v_nbits; } s_read_distance; } private_data; @@ -9893,7 +9894,7 @@ struct wuffs_brotli__bit_cost__struct { uint32_t active_coroutine; wuffs_base__vtable null_vtable; - bool f_log2_cache_ready; + uint64_t f_log2_valid[1024]; } private_impl; struct { @@ -10399,14 +10400,14 @@ struct wuffs_brotli__metablock_tester__struct { inline uint64_t bits_entropy( wuffs_base__slice_u8 a_population, - uint64_t a_size) const { + uint64_t a_size) { return wuffs_brotli__metablock_tester__bits_entropy(this, a_population, a_size); } inline uint64_t population_cost( wuffs_base__slice_u8 a_histogram, - uint32_t a_data_size) const { + uint32_t a_data_size) { return wuffs_brotli__metablock_tester__population_cost(this, a_histogram, a_data_size); } @@ -41132,6 +41133,12 @@ wuffs_softfloat__arith__clz64( const wuffs_softfloat__arith* self, uint64_t a_a); +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_softfloat__arith__clz32( + const wuffs_softfloat__arith* self, + uint32_t a_a); + WUFFS_BASE__GENERATED_C_CODE static uint64_t wuffs_softfloat__arith__shift_right_jam64( @@ -41197,6 +41204,20 @@ wuffs_softfloat__arith__f64_propagate_nan( uint64_t a_a, uint64_t a_b); +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_softfloat__arith__f64_cmp_slow( + const wuffs_softfloat__arith* self, + uint64_t a_a, + uint64_t a_b); + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_softfloat__arith__f64_add_normal( + const wuffs_softfloat__arith* self, + uint64_t a_a, + uint64_t a_b); + WUFFS_BASE__GENERATED_C_CODE static uint64_t wuffs_softfloat__arith__f64_add_mags( @@ -41221,6 +41242,48 @@ wuffs_softfloat__arith__round_pack_f32( uint32_t a_exp, uint32_t a_sig); +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_softfloat__arith__f32_add_mags( + const wuffs_softfloat__arith* self, + uint32_t a_a, + uint32_t a_b); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_softfloat__arith__f32_sub_mags( + const wuffs_softfloat__arith* self, + uint32_t a_a, + uint32_t a_b); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_softfloat__arith__slow_f32_add( + const wuffs_softfloat__arith* self, + uint32_t a_a, + uint32_t a_b); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_softfloat__arith__slow_f32_sub( + const wuffs_softfloat__arith* self, + uint32_t a_a, + uint32_t a_b); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_softfloat__arith__slow_f32_mul( + const wuffs_softfloat__arith* self, + uint32_t a_a, + uint32_t a_b); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_softfloat__arith__f32_cmp_slow( + const wuffs_softfloat__arith* self, + uint32_t a_a, + uint32_t a_b); + // ---------------- VTables // ---------------- Initializer Implementations @@ -41321,6 +41384,29 @@ wuffs_softfloat__arith__clz64( (((uint32_t)(WUFFS_SOFTFLOAT__CLZ8[(v_x >> 56u)])) & 7u)) & 63u); } +// -------- func softfloat.arith.clz32 + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_softfloat__arith__clz32( + const wuffs_softfloat__arith* self, + uint32_t a_a) { + uint32_t v_x = 0; + uint32_t v_n16 = 0; + uint32_t v_n8 = 0; + + v_x = a_a; + if ((v_x >> 16u) == 0u) { + v_x = ((uint32_t)(v_x << 16u)); + v_n16 = 16u; + } + if ((v_x >> 24u) == 0u) { + v_x = ((uint32_t)(v_x << 8u)); + v_n8 = 8u; + } + return ((v_n16 + v_n8 + (((uint32_t)(WUFFS_SOFTFLOAT__CLZ8[(v_x >> 24u)])) & 7u)) & 31u); +} + // -------- func softfloat.arith.shift_right_jam64 WUFFS_BASE__GENERATED_C_CODE @@ -41596,6 +41682,25 @@ wuffs_softfloat__arith__f64_cmp( return 0; } + if ((a_a | a_b) <= 9218868437227405312u) { + if (a_a < a_b) { + return 0u; + } else if (a_a == a_b) { + return 1u; + } + return 2u; + } + return wuffs_softfloat__arith__f64_cmp_slow(self, a_a, a_b); +} + +// -------- func softfloat.arith.f64_cmp_slow + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_softfloat__arith__f64_cmp_slow( + const wuffs_softfloat__arith* self, + uint64_t a_a, + uint64_t a_b) { uint64_t v_ka = 0; uint64_t v_kb = 0; @@ -41636,7 +41741,13 @@ wuffs_softfloat__arith__f64_eq( return 0; } - if (wuffs_softfloat__arith__f64_cmp(self, a_a, a_b) == 1u) { + if ((a_a | a_b) <= 9218868437227405312u) { + if (a_a == a_b) { + return 1u; + } + return 0u; + } + if (wuffs_softfloat__arith__f64_cmp_slow(self, a_a, a_b) == 1u) { return 1u; } return 0u; @@ -41658,7 +41769,13 @@ wuffs_softfloat__arith__f64_ne( return 0; } - if (wuffs_softfloat__arith__f64_cmp(self, a_a, a_b) == 1u) { + if ((a_a | a_b) <= 9218868437227405312u) { + if (a_a != a_b) { + return 1u; + } + return 0u; + } + if (wuffs_softfloat__arith__f64_cmp_slow(self, a_a, a_b) == 1u) { return 0u; } return 1u; @@ -41680,7 +41797,13 @@ wuffs_softfloat__arith__f64_lt( return 0; } - if (wuffs_softfloat__arith__f64_cmp(self, a_a, a_b) == 0u) { + if ((a_a | a_b) <= 9218868437227405312u) { + if (a_a < a_b) { + return 1u; + } + return 0u; + } + if (wuffs_softfloat__arith__f64_cmp_slow(self, a_a, a_b) == 0u) { return 1u; } return 0u; @@ -41702,7 +41825,13 @@ wuffs_softfloat__arith__f64_le( return 0; } - if (wuffs_softfloat__arith__f64_cmp(self, a_a, a_b) <= 1u) { + if ((a_a | a_b) <= 9218868437227405312u) { + if (a_a <= a_b) { + return 1u; + } + return 0u; + } + if (wuffs_softfloat__arith__f64_cmp_slow(self, a_a, a_b) <= 1u) { return 1u; } return 0u; @@ -41724,7 +41853,13 @@ wuffs_softfloat__arith__f64_gt( return 0; } - if (wuffs_softfloat__arith__f64_cmp(self, a_a, a_b) == 2u) { + if ((a_a | a_b) <= 9218868437227405312u) { + if (a_a > a_b) { + return 1u; + } + return 0u; + } + if (wuffs_softfloat__arith__f64_cmp_slow(self, a_a, a_b) == 2u) { return 1u; } return 0u; @@ -41748,7 +41883,13 @@ wuffs_softfloat__arith__f64_ge( uint32_t v_c = 0; - v_c = wuffs_softfloat__arith__f64_cmp(self, a_a, a_b); + if ((a_a | a_b) <= 9218868437227405312u) { + if (a_a >= a_b) { + return 1u; + } + return 0u; + } + v_c = wuffs_softfloat__arith__f64_cmp_slow(self, a_a, a_b); if ((v_c == 1u) || (v_c == 2u)) { return 1u; } @@ -41771,7 +41912,13 @@ wuffs_softfloat__arith__f64_min( return 0; } - if (wuffs_softfloat__arith__f64_cmp(self, a_a, a_b) == 0u) { + if ((a_a | a_b) <= 9218868437227405312u) { + if (a_a < a_b) { + return a_a; + } + return a_b; + } + if (wuffs_softfloat__arith__f64_cmp_slow(self, a_a, a_b) == 0u) { return a_a; } return a_b; @@ -41793,7 +41940,13 @@ wuffs_softfloat__arith__f64_max( return 0; } - if (wuffs_softfloat__arith__f64_cmp(self, a_a, a_b) == 2u) { + if ((a_a | a_b) <= 9218868437227405312u) { + if (a_a > a_b) { + return a_a; + } + return a_b; + } + if (wuffs_softfloat__arith__f64_cmp_slow(self, a_a, a_b) == 2u) { return a_a; } return a_b; @@ -41815,6 +41968,18 @@ wuffs_softfloat__arith__f64_add( return 0; } + uint64_t v_z = 0; + + if ((((uint64_t)(((a_a >> 52u) & 2047u) - 1u)) < 2046u) && (((uint64_t)(((a_b >> 52u) & 2047u) - 1u)) < 2046u)) { + v_z = wuffs_softfloat__arith__f64_add_normal(self, a_a, a_b); + if (v_z != 18446744073709551615u) { + return v_z; + } + } else if (((a_b & 9223372036854775807u) == 0u) && (((uint64_t)((a_a & 9223372036854775807u) - 1u)) < 9218868437227405311u)) { + return a_a; + } else if (((a_a & 9223372036854775807u) == 0u) && (((uint64_t)((a_b & 9223372036854775807u) - 1u)) < 9218868437227405311u)) { + return a_b; + } if (((a_a ^ a_b) >> 63u) == 0u) { return wuffs_softfloat__arith__f64_add_mags(self, a_a, a_b, (a_a & 9223372036854775808u)); } @@ -41837,12 +42002,91 @@ wuffs_softfloat__arith__f64_sub( return 0; } + uint64_t v_z = 0; + + if ((((uint64_t)(((a_a >> 52u) & 2047u) - 1u)) < 2046u) && (((uint64_t)(((a_b >> 52u) & 2047u) - 1u)) < 2046u)) { + v_z = wuffs_softfloat__arith__f64_add_normal(self, a_a, (a_b ^ 9223372036854775808u)); + if (v_z != 18446744073709551615u) { + return v_z; + } + } else if (((a_b & 9223372036854775807u) == 0u) && (((uint64_t)((a_a & 9223372036854775807u) - 1u)) < 9218868437227405311u)) { + return a_a; + } else if (((a_a & 9223372036854775807u) == 0u) && (((uint64_t)((a_b & 9223372036854775807u) - 1u)) < 9218868437227405311u)) { + return (a_b ^ 9223372036854775808u); + } if (((a_a ^ a_b) >> 63u) == 0u) { return wuffs_softfloat__arith__f64_sub_mags(self, a_a, a_b, (a_a & 9223372036854775808u)); } return wuffs_softfloat__arith__f64_add_mags(self, a_a, a_b, (a_a & 9223372036854775808u)); } +// -------- func softfloat.arith.f64_add_normal + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_softfloat__arith__f64_add_normal( + const wuffs_softfloat__arith* self, + uint64_t a_a, + uint64_t a_b) { + uint64_t v_x = 0; + uint64_t v_y = 0; + uint32_t v_ex = 0; + uint32_t v_d = 0; + uint64_t v_mx = 0; + uint64_t v_my = 0; + uint64_t v_z = 0; + uint32_t v_s = 0; + uint64_t v_rb = 0; + + v_x = a_a; + v_y = a_b; + if ((v_y & 9223372036854775807u) > (v_x & 9223372036854775807u)) { + v_x = a_b; + v_y = a_a; + } + v_ex = ((uint32_t)(((v_x >> 52u) & 2047u))); + v_d = ((uint32_t)(v_ex - ((uint32_t)(((v_y >> 52u) & 2047u))))); + if (v_d > 63u) { + v_d = 63u; + } + v_mx = (((v_x & 4503599627370495u) | 4503599627370496u) << 10u); + v_my = (((v_y & 4503599627370495u) | 4503599627370496u) << 10u); + if ((v_my & ((uint64_t)((((uint64_t)(1u)) << (v_d & 63u)) - 1u))) != 0u) { + v_my = ((v_my >> (v_d & 63u)) | 1u); + } else { + v_my = (v_my >> (v_d & 63u)); + } + if (((v_x ^ v_y) >> 63u) == 0u) { + v_z = (v_mx + v_my); + if (v_z >= 9223372036854775808u) { + v_ex += 1u; + v_z = ((v_z >> 1u) | (v_z & 1u)); + } + } else { + v_z = ((uint64_t)(v_mx - v_my)); + if (v_z == 0u) { + return 0u; + } else if (v_z < 4611686018427387904u) { + v_s = (((uint32_t)(wuffs_softfloat__arith__clz64(self, v_z) - 1u)) & 63u); + if (v_s >= v_ex) { + return 18446744073709551615u; + } + v_ex = (((uint32_t)(v_ex - v_s)) & 2047u); + v_z = ((uint64_t)(v_z << v_s)); + } + } + v_rb = (v_z & 1023u); + v_z = (((uint64_t)(v_z + 512u)) >> 10u); + if (v_rb == 512u) { + v_z &= 18446744073709551614u; + } + v_z = ((uint64_t)(((uint64_t)(((uint64_t)(((uint32_t)(v_ex - 1u)))) << 52u)) + v_z)); + if (v_z >= 9218868437227405312u) { + v_z = 9218868437227405312u; + } + return ((v_x & 9223372036854775808u) | v_z); +} + // -------- func softfloat.arith.f64_add_mags WUFFS_BASE__GENERATED_C_CODE @@ -42592,7 +42836,22 @@ wuffs_softfloat__arith__f32_add( return 0; } - return wuffs_softfloat__arith__f64_to_f32(self, wuffs_softfloat__arith__f64_add(self, wuffs_softfloat__arith__f32_to_f64(self, a_a), wuffs_softfloat__arith__f32_to_f64(self, a_b))); + uint32_t v_z = 0; + + if ((((uint32_t)(((a_a >> 23u) & 255u) - 1u)) < 254u) && (((uint32_t)(((a_b >> 23u) & 255u) - 1u)) < 254u)) { + if (((a_a ^ a_b) >> 31u) == 0u) { + return wuffs_softfloat__arith__f32_add_mags(self, a_a, a_b); + } + v_z = wuffs_softfloat__arith__f32_sub_mags(self, a_a, a_b); + if (v_z != 4294967295u) { + return v_z; + } + } else if (((a_b & 2147483647u) == 0u) && (((uint32_t)((a_a & 2147483647u) - 1u)) < 2139095039u)) { + return a_a; + } else if (((a_a & 2147483647u) == 0u) && (((uint32_t)((a_b & 2147483647u) - 1u)) < 2139095039u)) { + return a_b; + } + return wuffs_softfloat__arith__slow_f32_add(self, a_a, a_b); } // -------- func softfloat.arith.f32_sub @@ -42611,7 +42870,22 @@ wuffs_softfloat__arith__f32_sub( return 0; } - return wuffs_softfloat__arith__f64_to_f32(self, wuffs_softfloat__arith__f64_sub(self, wuffs_softfloat__arith__f32_to_f64(self, a_a), wuffs_softfloat__arith__f32_to_f64(self, a_b))); + uint32_t v_z = 0; + + if ((((uint32_t)(((a_a >> 23u) & 255u) - 1u)) < 254u) && (((uint32_t)(((a_b >> 23u) & 255u) - 1u)) < 254u)) { + if (((a_a ^ a_b) >> 31u) != 0u) { + return wuffs_softfloat__arith__f32_add_mags(self, a_a, a_b); + } + v_z = wuffs_softfloat__arith__f32_sub_mags(self, a_a, a_b); + if (v_z != 4294967295u) { + return v_z; + } + } else if (((a_b & 2147483647u) == 0u) && (((uint32_t)((a_a & 2147483647u) - 1u)) < 2139095039u)) { + return a_a; + } else if (((a_a & 2147483647u) == 0u) && (((uint32_t)((a_b & 2147483647u) - 1u)) < 2139095039u)) { + return (a_b ^ 2147483648u); + } + return wuffs_softfloat__arith__slow_f32_sub(self, a_a, a_b); } // -------- func softfloat.arith.f32_mul @@ -42630,6 +42904,202 @@ wuffs_softfloat__arith__f32_mul( return 0; } + uint32_t v_ea = 0; + uint32_t v_eb = 0; + uint32_t v_e = 0; + uint64_t v_p = 0; + uint32_t v_m = 0; + uint32_t v_r = 0; + uint32_t v_z = 0; + + v_ea = ((a_a >> 23u) & 255u); + v_eb = ((a_b >> 23u) & 255u); + if ((((uint32_t)(v_ea - 1u)) < 254u) && (((uint32_t)(v_eb - 1u)) < 254u)) { + v_p = (((uint64_t)(((a_a & 8388607u) | 8388608u))) * ((uint64_t)(((a_b & 8388607u) | 8388608u)))); + v_e = (v_ea + v_eb); + if (v_p >= 140737488355328u) { + v_e += 1u; + v_m = ((uint32_t)(((v_p >> 24u) & 16777215u))); + v_r = ((uint32_t)(((uint32_t)((v_p & 16777215u))) - 8388608u)); + } else { + v_m = ((uint32_t)(((v_p >> 23u) & 16777215u))); + v_r = ((uint32_t)(((uint32_t)((v_p & 8388607u))) - 4194304u)); + } + if ((v_r != 0u) && (v_r < 2147483648u)) { + v_m += 1u; + } else if (v_r == 0u) { + v_m += (v_m & 1u); + } + if ((v_e >= 128u) && (v_e <= 381u)) { + v_z = ((uint32_t)(((v_e - 128u) << 23u) + v_m)); + if (v_z >= 2139095040u) { + v_z = 2139095040u; + } + return (((a_a ^ a_b) & 2147483648u) | v_z); + } + } + return wuffs_softfloat__arith__slow_f32_mul(self, a_a, a_b); +} + +// -------- func softfloat.arith.f32_add_mags + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_softfloat__arith__f32_add_mags( + const wuffs_softfloat__arith* self, + uint32_t a_a, + uint32_t a_b) { + uint32_t v_ea = 0; + uint32_t v_eb = 0; + uint32_t v_ma = 0; + uint32_t v_mb = 0; + uint32_t v_d = 0; + uint32_t v_z = 0; + uint32_t v_rb = 0; + + v_ea = ((a_a >> 23u) & 255u); + v_eb = ((a_b >> 23u) & 255u); + v_ma = (((a_a & 8388607u) | 8388608u) << 6u); + v_mb = (((a_b & 8388607u) | 8388608u) << 6u); + if (v_ea < v_eb) { + v_d = (((uint32_t)(v_eb - v_ea)) & 255u); + v_ea = v_eb; + v_z = v_ma; + v_ma = v_mb; + v_mb = v_z; + } else { + v_d = (((uint32_t)(v_ea - v_eb)) & 255u); + } + if (v_d >= 31u) { + v_mb = 1u; + } else if (v_d > 0u) { + if (((uint32_t)(v_mb << (32u - v_d))) != 0u) { + v_mb = ((v_mb >> v_d) | 1u); + } else { + v_mb = (v_mb >> v_d); + } + } + v_z = ((v_ma & 1073741823u) + (v_mb & 1073741823u)); + if (v_z >= 1073741824u) { + v_ea += 1u; + v_z = ((v_z >> 1u) | (v_z & 1u)); + } + v_rb = (v_z & 63u); + v_z = (((uint32_t)(v_z + 32u)) >> 6u); + if (v_rb == 32u) { + v_z &= 4294967294u; + } + v_z = ((uint32_t)(((uint32_t)(((uint32_t)(v_ea - 1u)) << 23u)) + v_z)); + if (v_z >= 2139095040u) { + v_z = 2139095040u; + } + return ((a_a & 2147483648u) | v_z); +} + +// -------- func softfloat.arith.f32_sub_mags + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_softfloat__arith__f32_sub_mags( + const wuffs_softfloat__arith* self, + uint32_t a_a, + uint32_t a_b) { + uint32_t v_sign = 0; + uint32_t v_ea = 0; + uint32_t v_eb = 0; + uint32_t v_fa = 0; + uint32_t v_fb = 0; + uint32_t v_ma = 0; + uint32_t v_mb = 0; + uint32_t v_d = 0; + uint32_t v_s = 0; + uint32_t v_z = 0; + uint32_t v_rb = 0; + + v_sign = (a_a & 2147483648u); + v_ea = ((a_a >> 23u) & 255u); + v_eb = ((a_b >> 23u) & 255u); + v_fa = (a_a & 8388607u); + v_fb = (a_b & 8388607u); + if (v_ea == v_eb) { + if (v_fa == v_fb) { + return 0u; + } else if (v_fa < v_fb) { + v_sign ^= 2147483648u; + v_z = ((uint32_t)(v_fb - v_fa)); + } else { + v_z = ((uint32_t)(v_fa - v_fb)); + } + v_s = (((uint32_t)(wuffs_softfloat__arith__clz32(self, v_z) - 8u)) & 31u); + if (v_s >= v_ea) { + return 4294967295u; + } + return (v_sign | ((uint32_t)(((uint32_t)(((uint32_t)(((uint32_t)(v_ea - v_s)) - 1u)) << 23u)) + ((uint32_t)(v_z << v_s))))); + } + v_ma = ((v_fa | 8388608u) << 7u); + v_mb = ((v_fb | 8388608u) << 7u); + if (v_ea < v_eb) { + v_sign ^= 2147483648u; + v_d = (((uint32_t)(v_eb - v_ea)) & 255u); + v_ea = v_eb; + v_z = v_ma; + v_ma = v_mb; + v_mb = v_z; + } else { + v_d = (((uint32_t)(v_ea - v_eb)) & 255u); + } + if (v_d >= 31u) { + v_mb = 1u; + } else if (((uint32_t)(v_mb << ((32u - v_d) & 31u))) != 0u) { + v_mb = ((v_mb >> v_d) | 1u); + } else { + v_mb = (v_mb >> v_d); + } + v_z = ((uint32_t)(v_ma - v_mb)); + v_s = (((uint32_t)(wuffs_softfloat__arith__clz32(self, v_z) - 1u)) & 31u); + if (v_s >= v_ea) { + return 4294967295u; + } + v_ea = (((uint32_t)(v_ea - v_s)) & 255u); + v_z = ((uint32_t)(v_z << v_s)); + v_rb = (v_z & 127u); + v_z = (((uint32_t)(v_z + 64u)) >> 7u); + if (v_rb == 64u) { + v_z &= 4294967294u; + } + return (v_sign | ((uint32_t)(((uint32_t)(((uint32_t)(v_ea - 1u)) << 23u)) + v_z))); +} + +// -------- func softfloat.arith.slow_f32_add + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_softfloat__arith__slow_f32_add( + const wuffs_softfloat__arith* self, + uint32_t a_a, + uint32_t a_b) { + return wuffs_softfloat__arith__f64_to_f32(self, wuffs_softfloat__arith__f64_add(self, wuffs_softfloat__arith__f32_to_f64(self, a_a), wuffs_softfloat__arith__f32_to_f64(self, a_b))); +} + +// -------- func softfloat.arith.slow_f32_sub + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_softfloat__arith__slow_f32_sub( + const wuffs_softfloat__arith* self, + uint32_t a_a, + uint32_t a_b) { + return wuffs_softfloat__arith__f64_to_f32(self, wuffs_softfloat__arith__f64_sub(self, wuffs_softfloat__arith__f32_to_f64(self, a_a), wuffs_softfloat__arith__f32_to_f64(self, a_b))); +} + +// -------- func softfloat.arith.slow_f32_mul + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_softfloat__arith__slow_f32_mul( + const wuffs_softfloat__arith* self, + uint32_t a_a, + uint32_t a_b) { return wuffs_softfloat__arith__f64_to_f32(self, wuffs_softfloat__arith__f64_mul(self, wuffs_softfloat__arith__f32_to_f64(self, a_a), wuffs_softfloat__arith__f32_to_f64(self, a_b))); } @@ -42668,6 +43138,25 @@ wuffs_softfloat__arith__f32_cmp( return 0; } + if ((a_a | a_b) <= 2139095040u) { + if (a_a < a_b) { + return 0u; + } else if (a_a == a_b) { + return 1u; + } + return 2u; + } + return wuffs_softfloat__arith__f32_cmp_slow(self, a_a, a_b); +} + +// -------- func softfloat.arith.f32_cmp_slow + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_softfloat__arith__f32_cmp_slow( + const wuffs_softfloat__arith* self, + uint32_t a_a, + uint32_t a_b) { uint32_t v_ka = 0; uint32_t v_kb = 0; @@ -42708,7 +43197,13 @@ wuffs_softfloat__arith__f32_eq( return 0; } - if (wuffs_softfloat__arith__f32_cmp(self, a_a, a_b) == 1u) { + if ((a_a | a_b) <= 2139095040u) { + if (a_a == a_b) { + return 1u; + } + return 0u; + } + if (wuffs_softfloat__arith__f32_cmp_slow(self, a_a, a_b) == 1u) { return 1u; } return 0u; @@ -42730,7 +43225,13 @@ wuffs_softfloat__arith__f32_ne( return 0; } - if (wuffs_softfloat__arith__f32_cmp(self, a_a, a_b) == 1u) { + if ((a_a | a_b) <= 2139095040u) { + if (a_a != a_b) { + return 1u; + } + return 0u; + } + if (wuffs_softfloat__arith__f32_cmp_slow(self, a_a, a_b) == 1u) { return 0u; } return 1u; @@ -42752,7 +43253,13 @@ wuffs_softfloat__arith__f32_lt( return 0; } - if (wuffs_softfloat__arith__f32_cmp(self, a_a, a_b) == 0u) { + if ((a_a | a_b) <= 2139095040u) { + if (a_a < a_b) { + return 1u; + } + return 0u; + } + if (wuffs_softfloat__arith__f32_cmp_slow(self, a_a, a_b) == 0u) { return 1u; } return 0u; @@ -42774,7 +43281,13 @@ wuffs_softfloat__arith__f32_le( return 0; } - if (wuffs_softfloat__arith__f32_cmp(self, a_a, a_b) <= 1u) { + if ((a_a | a_b) <= 2139095040u) { + if (a_a <= a_b) { + return 1u; + } + return 0u; + } + if (wuffs_softfloat__arith__f32_cmp_slow(self, a_a, a_b) <= 1u) { return 1u; } return 0u; @@ -42796,7 +43309,13 @@ wuffs_softfloat__arith__f32_gt( return 0; } - if (wuffs_softfloat__arith__f32_cmp(self, a_a, a_b) == 2u) { + if ((a_a | a_b) <= 2139095040u) { + if (a_a > a_b) { + return 1u; + } + return 0u; + } + if (wuffs_softfloat__arith__f32_cmp_slow(self, a_a, a_b) == 2u) { return 1u; } return 0u; @@ -42820,7 +43339,13 @@ wuffs_softfloat__arith__f32_ge( uint32_t v_c = 0; - v_c = wuffs_softfloat__arith__f32_cmp(self, a_a, a_b); + if ((a_a | a_b) <= 2139095040u) { + if (a_a >= a_b) { + return 1u; + } + return 0u; + } + v_c = wuffs_softfloat__arith__f32_cmp_slow(self, a_a, a_b); if ((v_c == 1u) || (v_c == 2u)) { return 1u; } @@ -42843,7 +43368,13 @@ wuffs_softfloat__arith__f32_min( return 0; } - if (wuffs_softfloat__arith__f32_cmp(self, a_a, a_b) == 0u) { + if ((a_a | a_b) <= 2139095040u) { + if (a_a < a_b) { + return a_a; + } + return a_b; + } + if (wuffs_softfloat__arith__f32_cmp_slow(self, a_a, a_b) == 0u) { return a_a; } return a_b; @@ -42865,7 +43396,13 @@ wuffs_softfloat__arith__f32_max( return 0; } - if (wuffs_softfloat__arith__f32_cmp(self, a_a, a_b) == 2u) { + if ((a_a | a_b) <= 2139095040u) { + if (a_a > a_b) { + return a_a; + } + return a_b; + } + if (wuffs_softfloat__arith__f32_cmp_slow(self, a_a, a_b) == 2u) { return a_a; } return a_b; @@ -42886,6 +43423,16 @@ wuffs_softfloat__arith__f32_from_u32( return 0; } + uint32_t v_c = 0; + + if (a_a == 0u) { + return 0u; + } else if (a_a < 16777216u) { + v_c = wuffs_softfloat__arith__clz32(self, a_a); + if (v_c >= 8u) { + return ((uint32_t)(((uint32_t)(((uint32_t)(157u - v_c)) << 23u)) + ((uint32_t)(a_a << (v_c - 8u))))); + } + } return wuffs_softfloat__arith__f64_to_f32(self, wuffs_softfloat__arith__f64_from_u32(self, a_a)); } @@ -76599,8 +77146,6 @@ WUFFS_BROTLI__DICTIONARY[122784] WUFFS_BASE__POTENTIALLY_UNUSED = { #define WUFFS_BROTLI__F64_1E99 6088095589093318446u -#define WUFFS_BROTLI__LOG2_CACHE_SIZE 65536u - #define WUFFS_BROTLI__LITERAL_BLOCK_SWITCH_COST 4628602664527698330u #define WUFFS_BROTLI__COMMAND_BLOCK_SWITCH_COST 4623789442425946112u @@ -76874,6 +77419,20 @@ wuffs_brotli__decoder__read_symbol( wuffs_base__io_buffer* a_src, uint32_t a_hoff); +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__decoder__read_bits_fast( + wuffs_brotli__decoder* self, + wuffs_base__io_buffer* a_src, + uint32_t a_n); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__decoder__read_symbol_fast( + wuffs_brotli__decoder* self, + wuffs_base__io_buffer* a_src, + uint32_t a_hoff); + WUFFS_BASE__GENERATED_C_CODE static bool wuffs_brotli__decoder__jump_to_byte_boundary( @@ -77153,6 +77712,13 @@ wuffs_brotli__decoder__process_commands( wuffs_base__slice_u8 a_workbuf, wuffs_base__slice_u8 a_dictbuf); +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__status +wuffs_brotli__decoder__process_commands_fast( + wuffs_brotli__decoder* self, + wuffs_base__io_buffer* a_src, + wuffs_base__slice_u8 a_workbuf); + WUFFS_BASE__GENERATED_C_CODE static wuffs_base__empty_struct wuffs_brotli__decoder__update_dist_htree_index( @@ -77307,11 +77873,6 @@ wuffs_brotli__encoder__create_backward_references( uint64_t a_num_bytes, uint64_t a_position); -WUFFS_BASE__GENERATED_C_CODE -static wuffs_base__empty_struct -wuffs_brotli__bit_cost__init_log2_cache( - wuffs_brotli__bit_cost* self); - WUFFS_BASE__GENERATED_C_CODE static uint32_t wuffs_brotli__bit_cost__ld32( @@ -77348,17 +77909,23 @@ wuffs_brotli__bit_cost__fast_log2( const wuffs_brotli__bit_cost* self, uint64_t a_v); +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__bit_cost__fast_log2_memo( + wuffs_brotli__bit_cost* self, + uint64_t a_v); + WUFFS_BASE__GENERATED_C_CODE static uint64_t wuffs_brotli__bit_cost__bits_entropy( - const wuffs_brotli__bit_cost* self, + wuffs_brotli__bit_cost* self, wuffs_base__slice_u8 a_population, uint64_t a_size); WUFFS_BASE__GENERATED_C_CODE static uint64_t wuffs_brotli__bit_cost__population_cost( - const wuffs_brotli__bit_cost* self, + wuffs_brotli__bit_cost* self, wuffs_base__slice_u8 a_histogram, uint32_t a_data_size); @@ -77671,7 +78238,7 @@ wuffs_brotli__encoder__find_match_length_with_limit( WUFFS_BASE__GENERATED_C_CODE static uint64_t wuffs_brotli__metablock_builder__bit_cost( - const wuffs_brotli__metablock_builder* self, + wuffs_brotli__metablock_builder* self, uint64_t a_count); WUFFS_BASE__GENERATED_C_CODE @@ -79369,7 +79936,7 @@ wuffs_brotli__metablock_builder__h_set_bit_cost( WUFFS_BASE__GENERATED_C_CODE static uint64_t wuffs_brotli__metablock_builder__h_population_cost( - const wuffs_brotli__metablock_builder* self, + wuffs_brotli__metablock_builder* self, wuffs_base__slice_u8 a_wb, uint64_t a_h, uint32_t a_ds); @@ -79536,7 +80103,7 @@ wuffs_brotli__encoder__gbs_histo_off( WUFFS_BASE__GENERATED_C_CODE static uint64_t wuffs_brotli__encoder__gbs_entropy( - const wuffs_brotli__encoder* self, + wuffs_brotli__encoder* self, wuffs_base__slice_u8 a_wb, uint64_t a_off, uint64_t a_size); @@ -81245,6 +81812,132 @@ wuffs_brotli__decoder__read_symbol( return status; } +// -------- func brotli.decoder.read_bits_fast + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__decoder__read_bits_fast( + wuffs_brotli__decoder* self, + wuffs_base__io_buffer* a_src, + uint32_t a_n) { + uint32_t v_n = 0; + uint32_t v_bits = 0; + uint32_t v_n_bits = 0; + + const uint8_t* iop_a_src = NULL; + const uint8_t* io0_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + const uint8_t* io1_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + const uint8_t* io2_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + if (a_src && a_src->data.ptr) { + io0_a_src = a_src->data.ptr; + io1_a_src = io0_a_src + a_src->meta.ri; + iop_a_src = io1_a_src; + io2_a_src = io0_a_src + a_src->meta.wi; + } + + v_n = a_n; + v_bits = self->private_impl.f_bits; + v_n_bits = self->private_impl.f_n_bits; + while (v_n_bits < v_n) { + if (((uint64_t)(io2_a_src - iop_a_src)) <= 0u) { + self->private_impl.f_bits = v_bits; + self->private_impl.f_n_bits = v_n_bits; + if (a_src && a_src->data.ptr) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + return 1u; + } + v_bits |= (((uint32_t)(wuffs_base__peek_u8be__no_bounds_check(iop_a_src))) << v_n_bits); + iop_a_src += 1u; + v_n_bits += 8u; + } + self->private_impl.f_bits_value = (v_bits & ((((uint32_t)(1u)) << v_n) - 1u)); + self->private_impl.f_bits = (v_bits >> v_n); + self->private_impl.f_n_bits = (v_n_bits - v_n); + if (a_src && a_src->data.ptr) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + return 0u; +} + +// -------- func brotli.decoder.read_symbol_fast + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__decoder__read_symbol_fast( + wuffs_brotli__decoder* self, + wuffs_base__io_buffer* a_src, + uint32_t a_hoff) { + uint32_t v_hoff = 0; + uint32_t v_bits = 0; + uint32_t v_n_bits = 0; + uint32_t v_i = 0; + uint32_t v_e = 0; + uint32_t v_e_bits = 0; + + const uint8_t* iop_a_src = NULL; + const uint8_t* io0_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + const uint8_t* io1_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + const uint8_t* io2_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + if (a_src && a_src->data.ptr) { + io0_a_src = a_src->data.ptr; + io1_a_src = io0_a_src + a_src->meta.ri; + iop_a_src = io1_a_src; + io2_a_src = io0_a_src + a_src->meta.wi; + } + + v_hoff = a_hoff; + v_bits = self->private_impl.f_bits; + v_n_bits = self->private_impl.f_n_bits; + while (true) { + v_i = ((uint32_t)(v_hoff + (v_bits & 255u))); + if (v_i >= 678034u) { + break; + } + v_e = ((uint32_t)(self->private_data.f_huffs[v_i])); + v_e_bits = (v_e & 15u); + if (v_e_bits <= 8u) { + if (v_n_bits >= v_e_bits) { + self->private_impl.f_symbol = (v_e >> 4u); + self->private_impl.f_bits = (v_bits >> v_e_bits); + self->private_impl.f_n_bits = (v_n_bits - v_e_bits); + if (a_src && a_src->data.ptr) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + return 0u; + } + } else if (v_n_bits > 8u) { + v_i = ((uint32_t)(((uint32_t)(v_hoff + (v_e >> 4u))) + ((v_bits >> 8u) & ((((uint32_t)(1u)) << (v_e_bits - 8u)) - 1u)))); + if (v_i >= 678034u) { + break; + } + v_e = ((uint32_t)(self->private_data.f_huffs[v_i])); + v_e_bits = ((v_e & 15u) + 8u); + if (v_n_bits >= v_e_bits) { + self->private_impl.f_symbol = (v_e >> 4u); + self->private_impl.f_bits = (v_bits >> v_e_bits); + self->private_impl.f_n_bits = (v_n_bits - v_e_bits); + if (a_src && a_src->data.ptr) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + return 0u; + } + } + if ((v_n_bits >= 24u) || (((uint64_t)(io2_a_src - iop_a_src)) <= 0u)) { + break; + } + v_bits |= (((uint32_t)(wuffs_base__peek_u8be__no_bounds_check(iop_a_src))) << v_n_bits); + iop_a_src += 1u; + v_n_bits += 8u; + } + self->private_impl.f_bits = v_bits; + self->private_impl.f_n_bits = v_n_bits; + if (a_src && a_src->data.ptr) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + return 1u; +} + // -------- func brotli.decoder.jump_to_byte_boundary WUFFS_BASE__GENERATED_C_CODE @@ -84033,7 +84726,6 @@ wuffs_brotli__decoder__process_commands( wuffs_base__status status = wuffs_base__make_status(NULL); uint32_t v_cmd_code = 0; - uint32_t v_insert_length = 0; uint32_t v_p1 = 0; uint32_t v_p2 = 0; uint32_t v_mode_offset = 0; @@ -84049,11 +84741,31 @@ wuffs_brotli__decoder__process_commands( uint32_t v_address = 0; uint32_t v_chunk = 0; uint64_t v_j = 0; + uint32_t v_fr = 0; + uint64_t v_dst_i = 0; + uint64_t v_src_i = 0; + uint64_t v_k64 = 0; + uint64_t v_n64 = 0; + uint64_t v_dlo = 0; + uint64_t v_dhi = 0; + uint64_t v_slo = 0; + uint64_t v_shi = 0; + wuffs_base__status v_fs = wuffs_base__make_status(NULL); + + const uint8_t* iop_a_src = NULL; + const uint8_t* io0_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + const uint8_t* io1_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + const uint8_t* io2_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + if (a_src && a_src->data.ptr) { + io0_a_src = a_src->data.ptr; + io1_a_src = io0_a_src + a_src->meta.ri; + iop_a_src = io1_a_src; + io2_a_src = io0_a_src + a_src->meta.wi; + } uint32_t coro_susp_point = self->private_impl.p_process_commands; if (coro_susp_point) { v_cmd_code = self->private_data.s_process_commands.v_cmd_code; - v_insert_length = self->private_data.s_process_commands.v_insert_length; v_tree = self->private_data.s_process_commands.v_tree; v_distance = self->private_data.s_process_commands.v_distance; v_i = self->private_data.s_process_commands.v_i; @@ -84066,47 +84778,134 @@ wuffs_brotli__decoder__process_commands( WUFFS_BASE__COROUTINE_SUSPENSION_POINT_0; while (true) { - if (self->private_impl.f_block_length[1u] == 0u) { - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(1); - status = wuffs_brotli__decoder__decode_block_switch(self, a_src, 1u); - if (status.repr) { - goto suspend; + if ((self->private_impl.f_cmd_state == 0u) && (((uint64_t)(io2_a_src - iop_a_src)) >= 16u)) { + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + v_fs = wuffs_brotli__decoder__process_commands_fast(self, a_src, a_workbuf); + if (a_src) { + iop_a_src = a_src->data.ptr + a_src->meta.ri; + } + if ( ! wuffs_base__status__is_ok(&v_fs)) { + status = v_fs; + if (wuffs_base__status__is_error(&status)) { + goto exit; + } else if (wuffs_base__status__is_suspension(&status)) { + status = wuffs_base__make_status(wuffs_base__error__cannot_return_a_suspension); + goto exit; + } + goto ok; + } else if (self->private_impl.f_cmd_state == 4u) { + self->private_impl.f_cmd_state = 0u; + status = wuffs_base__make_status(NULL); + goto ok; } - self->private_impl.f_htree_command = (self->private_impl.f_block_type_rb[3u] & 255u); - continue; } - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(2); - status = wuffs_brotli__decoder__read_symbol(self, a_src, (161792u + (self->private_impl.f_htree_command * 1080u))); - if (status.repr) { - goto suspend; + if (self->private_impl.f_cmd_state == 0u) { + if (self->private_impl.f_block_length[1u] == 0u) { + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(1); + status = wuffs_brotli__decoder__decode_block_switch(self, a_src, 1u); + if (a_src) { + iop_a_src = a_src->data.ptr + a_src->meta.ri; + } + if (status.repr) { + goto suspend; + } + self->private_impl.f_htree_command = (self->private_impl.f_block_type_rb[3u] & 255u); + continue; + } + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + v_fr = wuffs_brotli__decoder__read_symbol_fast(self, a_src, (161792u + (self->private_impl.f_htree_command * 1080u))); + if (a_src) { + iop_a_src = a_src->data.ptr + a_src->meta.ri; + } + if (v_fr != 0u) { + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(2); + status = wuffs_brotli__decoder__read_symbol(self, a_src, (161792u + (self->private_impl.f_htree_command * 1080u))); + if (a_src) { + iop_a_src = a_src->data.ptr + a_src->meta.ri; + } + if (status.repr) { + goto suspend; + } + } + if (self->private_impl.f_symbol >= 704u) { + status = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); + goto exit; + } + v_cmd_code = wuffs_base__u32__min(self->private_impl.f_symbol, 703u); + self->private_impl.f_implicit_zero_distance = (WUFFS_BROTLI__CMD_IMPLICIT_ZERO_DISTANCE[v_cmd_code] != 0u); + self->private_impl.f_distance_context = ((uint32_t)(WUFFS_BROTLI__CMD_DISTANCE_CONTEXT[v_cmd_code])); + wuffs_brotli__decoder__update_dist_htree_index(self); + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + v_fr = wuffs_brotli__decoder__read_bits_fast(self, a_src, ((uint32_t)(WUFFS_BROTLI__CMD_INSERT_LEN_EXTRA_BITS[v_cmd_code]))); + if (a_src) { + iop_a_src = a_src->data.ptr + a_src->meta.ri; + } + if (v_fr != 0u) { + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(3); + status = wuffs_brotli__decoder__read_bits(self, a_src, ((uint32_t)(WUFFS_BROTLI__CMD_INSERT_LEN_EXTRA_BITS[v_cmd_code]))); + if (a_src) { + iop_a_src = a_src->data.ptr + a_src->meta.ri; + } + if (status.repr) { + goto suspend; + } + } + self->private_impl.f_insert_length = ((uint32_t)(WUFFS_BROTLI__CMD_INSERT_LEN_OFFSET[v_cmd_code] + self->private_impl.f_bits_value)); + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + v_fr = wuffs_brotli__decoder__read_bits_fast(self, a_src, ((uint32_t)(WUFFS_BROTLI__CMD_COPY_LEN_EXTRA_BITS[v_cmd_code]))); + if (a_src) { + iop_a_src = a_src->data.ptr + a_src->meta.ri; + } + if (v_fr != 0u) { + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(4); + status = wuffs_brotli__decoder__read_bits(self, a_src, ((uint32_t)(WUFFS_BROTLI__CMD_COPY_LEN_EXTRA_BITS[v_cmd_code]))); + if (a_src) { + iop_a_src = a_src->data.ptr + a_src->meta.ri; + } + if (status.repr) { + goto suspend; + } + } + self->private_impl.f_copy_length = ((uint32_t)(WUFFS_BROTLI__CMD_COPY_LEN_OFFSET[v_cmd_code] + self->private_impl.f_bits_value)); + self->private_impl.f_block_length[1u] -= 1u; + if (self->private_impl.f_insert_length > 0u) { + self->private_impl.f_mb_remaining -= self->private_impl.f_insert_length; + self->private_impl.f_cmd_state = 1u; + } else { + self->private_impl.f_cmd_state = 2u; + } } - if (self->private_impl.f_symbol >= 704u) { - status = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); - goto exit; - } - v_cmd_code = wuffs_base__u32__min(self->private_impl.f_symbol, 703u); - self->private_impl.f_implicit_zero_distance = (WUFFS_BROTLI__CMD_IMPLICIT_ZERO_DISTANCE[v_cmd_code] != 0u); - self->private_impl.f_distance_context = ((uint32_t)(WUFFS_BROTLI__CMD_DISTANCE_CONTEXT[v_cmd_code])); - wuffs_brotli__decoder__update_dist_htree_index(self); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(3); - status = wuffs_brotli__decoder__read_bits(self, a_src, ((uint32_t)(WUFFS_BROTLI__CMD_INSERT_LEN_EXTRA_BITS[v_cmd_code]))); - if (status.repr) { - goto suspend; - } - v_insert_length = ((uint32_t)(WUFFS_BROTLI__CMD_INSERT_LEN_OFFSET[v_cmd_code] + self->private_impl.f_bits_value)); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(4); - status = wuffs_brotli__decoder__read_bits(self, a_src, ((uint32_t)(WUFFS_BROTLI__CMD_COPY_LEN_EXTRA_BITS[v_cmd_code]))); - if (status.repr) { - goto suspend; - } - self->private_impl.f_copy_length = ((uint32_t)(WUFFS_BROTLI__CMD_COPY_LEN_OFFSET[v_cmd_code] + self->private_impl.f_bits_value)); - self->private_impl.f_block_length[1u] -= 1u; - if (v_insert_length > 0u) { - self->private_impl.f_mb_remaining -= v_insert_length; - while (v_insert_length > 0u) { + if (self->private_impl.f_cmd_state == 1u) { + while (self->private_impl.f_insert_length > 0u) { if (self->private_impl.f_block_length[0u] == 0u) { + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } WUFFS_BASE__COROUTINE_SUSPENSION_POINT(5); status = wuffs_brotli__decoder__decode_block_switch(self, a_src, 0u); + if (a_src) { + iop_a_src = a_src->data.ptr + a_src->meta.ri; + } if (status.repr) { goto suspend; } @@ -84118,14 +84917,29 @@ wuffs_brotli__decoder__process_commands( v_mode_offset = (self->private_impl.f_context_mode << 9u); v_ctx = ((uint32_t)(((uint8_t)(WUFFS_BROTLI__CONTEXT_LOOKUP[(v_mode_offset | v_p1)] | WUFFS_BROTLI__CONTEXT_LOOKUP[(v_mode_offset | 256u | v_p2)])))); v_tree = ((uint32_t)(self->private_data.f_context_maps[(self->private_impl.f_context_map_slice + v_ctx)])); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(6); - status = wuffs_brotli__decoder__read_symbol(self, a_src, (v_tree * 632u)); - if (status.repr) { - goto suspend; + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + v_fr = wuffs_brotli__decoder__read_symbol_fast(self, a_src, (v_tree * 632u)); + if (a_src) { + iop_a_src = a_src->data.ptr + a_src->meta.ri; + } + if (v_fr != 0u) { + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(6); + status = wuffs_brotli__decoder__read_symbol(self, a_src, (v_tree * 632u)); + if (a_src) { + iop_a_src = a_src->data.ptr + a_src->meta.ri; + } + if (status.repr) { + goto suspend; + } } wuffs_brotli__decoder__put_byte(self, a_workbuf, ((uint8_t)(self->private_impl.f_symbol))); self->private_impl.f_block_length[0u] -= 1u; - v_insert_length -= 1u; + self->private_impl.f_insert_length -= 1u; if (self->private_impl.f_pos == self->private_impl.f_rb_size) { WUFFS_BASE__COROUTINE_SUSPENSION_POINT(7); status = wuffs_brotli__decoder__ring_event(self); @@ -84134,30 +84948,47 @@ wuffs_brotli__decoder__process_commands( } } } + self->private_impl.f_cmd_state = 2u; if ((self->private_impl.f_mb_remaining == 0u) || (self->private_impl.f_mb_remaining >= 2147483648u)) { + self->private_impl.f_cmd_state = 0u; status = wuffs_base__make_status(NULL); goto ok; } } - if (self->private_impl.f_implicit_zero_distance) { - self->private_impl.f_distance_context = 1u; - self->private_impl.f_dist_rb_idx -= 1u; - self->private_impl.f_distance_code = self->private_impl.f_dist_rb[(self->private_impl.f_dist_rb_idx & 3u)]; - } else { - if (self->private_impl.f_block_length[2u] == 0u) { - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(8); - status = wuffs_brotli__decoder__decode_block_switch(self, a_src, 2u); + if (self->private_impl.f_cmd_state == 2u) { + if (self->private_impl.f_implicit_zero_distance) { + self->private_impl.f_distance_context = 1u; + self->private_impl.f_dist_rb_idx -= 1u; + self->private_impl.f_distance_code = self->private_impl.f_dist_rb[(self->private_impl.f_dist_rb_idx & 3u)]; + } else { + if (self->private_impl.f_block_length[2u] == 0u) { + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(8); + status = wuffs_brotli__decoder__decode_block_switch(self, a_src, 2u); + if (a_src) { + iop_a_src = a_src->data.ptr + a_src->meta.ri; + } + if (status.repr) { + goto suspend; + } + self->private_impl.f_dist_context_map_slice = ((self->private_impl.f_block_type_rb[5u] & 255u) << 2u); + wuffs_brotli__decoder__update_dist_htree_index(self); + } + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(9); + status = wuffs_brotli__decoder__read_distance(self, a_src); + if (a_src) { + iop_a_src = a_src->data.ptr + a_src->meta.ri; + } if (status.repr) { goto suspend; } - self->private_impl.f_dist_context_map_slice = ((self->private_impl.f_block_type_rb[5u] & 255u) << 2u); - wuffs_brotli__decoder__update_dist_htree_index(self); - } - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(9); - status = wuffs_brotli__decoder__read_distance(self, a_src); - if (status.repr) { - goto suspend; } + self->private_impl.f_cmd_state = 3u; } if (self->private_impl.f_max_distance != self->private_impl.f_max_backward_distance) { self->private_impl.f_max_distance = wuffs_base__u32__min(self->private_impl.f_pos, self->private_impl.f_max_backward_distance); @@ -84247,6 +85078,33 @@ wuffs_brotli__decoder__process_commands( self->private_impl.f_dist_rb[(self->private_impl.f_dist_rb_idx & 3u)] = v_distance; self->private_impl.f_dist_rb_idx += 1u; self->private_impl.f_mb_remaining -= v_i; + v_dst_i = ((uint64_t)(self->private_impl.f_pos)); + v_src_i = ((uint64_t)((((uint32_t)(self->private_impl.f_pos - v_distance)) & ((uint32_t)(self->private_impl.f_window_size - 1u))))); + if (((v_dst_i + ((uint64_t)(v_i))) < ((uint64_t)(self->private_impl.f_rb_size))) && + ((v_src_i + ((uint64_t)(v_i))) <= ((uint64_t)(self->private_impl.f_window_size))) && + (((uint64_t)(self->private_impl.f_window_size)) <= ((uint64_t)(a_workbuf.len))) && + (v_distance > 0u)) { + v_k64 = 0u; + while (v_k64 < ((uint64_t)(v_i))) { + v_n64 = ((uint64_t)(((uint64_t)(v_i)) - v_k64)); + v_n64 = wuffs_base__u64__min(v_n64, ((uint64_t)(v_distance))); + v_dlo = ((uint64_t)(v_dst_i + v_k64)); + v_dhi = ((uint64_t)(v_dlo + v_n64)); + v_slo = ((uint64_t)(v_src_i + v_k64)); + v_shi = ((uint64_t)(v_slo + v_n64)); + if ((v_n64 == 0u) || + (v_dlo > v_dhi) || + (v_dhi > ((uint64_t)(a_workbuf.len))) || + (v_slo > v_shi) || + (v_shi > ((uint64_t)(a_workbuf.len)))) { + break; + } + wuffs_private_impl__slice_u8__copy_from_slice(wuffs_base__slice_u8__subslice_ij(a_workbuf, v_dlo, v_dhi), wuffs_base__slice_u8__subslice_ij(a_workbuf, v_slo, v_shi)); + v_k64 += v_n64; + } + self->private_impl.f_pos += ((uint32_t)(v_k64)); + v_i -= ((uint32_t)(v_k64)); + } while (v_i > 0u) { v_b = wuffs_brotli__decoder__get_byte(self, a_workbuf, v_distance); wuffs_brotli__decoder__put_byte(self, a_workbuf, v_b); @@ -84260,6 +85118,7 @@ wuffs_brotli__decoder__process_commands( } } } + self->private_impl.f_cmd_state = 0u; if ((self->private_impl.f_mb_remaining == 0u) || (self->private_impl.f_mb_remaining >= 2147483648u)) { status = wuffs_base__make_status(NULL); goto ok; @@ -84275,7 +85134,6 @@ wuffs_brotli__decoder__process_commands( suspend: self->private_impl.p_process_commands = wuffs_base__status__is_suspension(&status) ? coro_susp_point : 0; self->private_data.s_process_commands.v_cmd_code = v_cmd_code; - self->private_data.s_process_commands.v_insert_length = v_insert_length; self->private_data.s_process_commands.v_tree = v_tree; self->private_data.s_process_commands.v_distance = v_distance; self->private_data.s_process_commands.v_i = v_i; @@ -84286,6 +85144,330 @@ wuffs_brotli__decoder__process_commands( goto exit; exit: + if (a_src && a_src->data.ptr) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + + return status; +} + +// -------- func brotli.decoder.process_commands_fast + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__status +wuffs_brotli__decoder__process_commands_fast( + wuffs_brotli__decoder* self, + wuffs_base__io_buffer* a_src, + wuffs_base__slice_u8 a_workbuf) { + wuffs_base__status status = wuffs_base__make_status(NULL); + + wuffs_base__status v_ret = wuffs_base__make_status(NULL); + uint64_t v_bits = 0; + uint32_t v_n_bits = 0; + uint32_t v_i = 0; + uint32_t v_e = 0; + uint32_t v_e_bits = 0; + uint32_t v_cmd_code = 0; + uint32_t v_nb = 0; + uint32_t v_p1 = 0; + uint32_t v_p2 = 0; + uint32_t v_mode_offset = 0; + uint32_t v_ctx = 0; + uint32_t v_tree = 0; + uint32_t v_code = 0; + uint32_t v_nbits = 0; + uint32_t v_lo = 0; + uint32_t v_max_distance = 0; + uint32_t v_distance = 0; + uint32_t v_n = 0; + uint64_t v_dst_i = 0; + uint64_t v_src_i = 0; + uint64_t v_k64 = 0; + uint64_t v_n64 = 0; + uint64_t v_dlo = 0; + uint64_t v_dhi = 0; + uint64_t v_slo = 0; + uint64_t v_shi = 0; + + const uint8_t* iop_a_src = NULL; + const uint8_t* io0_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + const uint8_t* io1_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + const uint8_t* io2_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + if (a_src && a_src->data.ptr) { + io0_a_src = a_src->data.ptr; + io1_a_src = io0_a_src + a_src->meta.ri; + iop_a_src = io1_a_src; + io2_a_src = io0_a_src + a_src->meta.wi; + } + + if (self->private_impl.f_n_bits >= 8u) { + status = wuffs_base__make_status(NULL); + goto ok; + } + v_n_bits = (self->private_impl.f_n_bits & 7u); + v_bits = (((uint64_t)(self->private_impl.f_bits)) & ((((uint64_t)(1u)) << v_n_bits) - 1u)); + while (true) { + if (self->private_impl.f_cmd_state == 0u) { + if ((self->private_impl.f_block_length[1u] == 0u) || (((uint64_t)(io2_a_src - iop_a_src)) < 16u)) { + break; + } + v_bits |= ((uint64_t)(wuffs_base__peek_u64le__no_bounds_check(iop_a_src) << v_n_bits)); + iop_a_src += ((63u - v_n_bits) >> 3u); + v_n_bits |= 56u; + v_i = ((uint32_t)((161792u + (self->private_impl.f_htree_command * 1080u)) + ((uint32_t)((v_bits & 255u))))); + if (v_i >= 678034u) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); + break; + } + v_e = ((uint32_t)(self->private_data.f_huffs[v_i])); + v_e_bits = (v_e & 15u); + if (v_e_bits > 8u) { + v_i = ((uint32_t)(((uint32_t)((161792u + (self->private_impl.f_htree_command * 1080u)) + (v_e >> 4u))) + (((uint32_t)(((v_bits >> 8u) & 32767u))) & ((((uint32_t)(1u)) << (v_e_bits - 8u)) - 1u)))); + if (v_i >= 678034u) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); + break; + } + v_e = ((uint32_t)(self->private_data.f_huffs[v_i])); + v_e_bits = ((v_e & 15u) + 8u); + } + if (v_n_bits < v_e_bits) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_bit_reader); + break; + } + v_bits >>= v_e_bits; + v_n_bits -= v_e_bits; + if ((v_e >> 4u) >= 704u) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); + break; + } + self->private_impl.f_symbol = (v_e >> 4u); + v_cmd_code = wuffs_base__u32__min(self->private_impl.f_symbol, 703u); + self->private_impl.f_implicit_zero_distance = (WUFFS_BROTLI__CMD_IMPLICIT_ZERO_DISTANCE[v_cmd_code] != 0u); + self->private_impl.f_distance_context = ((uint32_t)(WUFFS_BROTLI__CMD_DISTANCE_CONTEXT[v_cmd_code])); + wuffs_brotli__decoder__update_dist_htree_index(self); + v_nb = ((uint32_t)(WUFFS_BROTLI__CMD_INSERT_LEN_EXTRA_BITS[v_cmd_code])); + if (v_n_bits < v_nb) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_bit_reader); + break; + } + self->private_impl.f_bits_value = ((uint32_t)((v_bits & ((((uint64_t)(1u)) << v_nb) - 1u)))); + v_bits >>= v_nb; + v_n_bits -= v_nb; + self->private_impl.f_insert_length = ((uint32_t)(WUFFS_BROTLI__CMD_INSERT_LEN_OFFSET[v_cmd_code] + self->private_impl.f_bits_value)); + if (((uint64_t)(io2_a_src - iop_a_src)) < 8u) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_bit_reader); + break; + } + v_bits |= ((uint64_t)(wuffs_base__peek_u64le__no_bounds_check(iop_a_src) << v_n_bits)); + iop_a_src += ((63u - v_n_bits) >> 3u); + v_n_bits |= 56u; + v_nb = ((uint32_t)(WUFFS_BROTLI__CMD_COPY_LEN_EXTRA_BITS[v_cmd_code])); + if (v_n_bits < v_nb) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_bit_reader); + break; + } + self->private_impl.f_bits_value = ((uint32_t)((v_bits & ((((uint64_t)(1u)) << v_nb) - 1u)))); + v_bits >>= v_nb; + v_n_bits -= v_nb; + self->private_impl.f_copy_length = ((uint32_t)(WUFFS_BROTLI__CMD_COPY_LEN_OFFSET[v_cmd_code] + self->private_impl.f_bits_value)); + self->private_impl.f_block_length[1u] -= 1u; + if (self->private_impl.f_insert_length > 0u) { + self->private_impl.f_mb_remaining -= self->private_impl.f_insert_length; + self->private_impl.f_cmd_state = 1u; + } else { + self->private_impl.f_cmd_state = 2u; + } + } + if (self->private_impl.f_cmd_state == 1u) { + if (((uint64_t)(self->private_impl.f_window_size)) > ((uint64_t)(a_workbuf.len))) { + break; + } + v_p1 = ((uint32_t)(wuffs_brotli__decoder__get_byte(self, a_workbuf, 1u))); + v_p2 = ((uint32_t)(wuffs_brotli__decoder__get_byte(self, a_workbuf, 2u))); + while (self->private_impl.f_insert_length > 0u) { + if ((self->private_impl.f_block_length[0u] == 0u) || (((uint32_t)(self->private_impl.f_pos + 1u)) >= self->private_impl.f_rb_size)) { + goto label__outer__break; + } + if (v_n_bits < 24u) { + if (((uint64_t)(io2_a_src - iop_a_src)) < 8u) { + goto label__outer__break; + } + v_bits |= ((uint64_t)(wuffs_base__peek_u64le__no_bounds_check(iop_a_src) << v_n_bits)); + iop_a_src += ((63u - v_n_bits) >> 3u); + v_n_bits |= 56u; + } + v_mode_offset = (self->private_impl.f_context_mode << 9u); + v_ctx = ((uint32_t)(((uint8_t)(WUFFS_BROTLI__CONTEXT_LOOKUP[(v_mode_offset | v_p1)] | WUFFS_BROTLI__CONTEXT_LOOKUP[(v_mode_offset | 256u | v_p2)])))); + v_tree = ((uint32_t)(self->private_data.f_context_maps[(self->private_impl.f_context_map_slice + v_ctx)])); + v_i = ((uint32_t)((v_tree * 632u) + ((uint32_t)((v_bits & 255u))))); + if (v_i >= 678034u) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); + goto label__outer__break; + } + v_e = ((uint32_t)(self->private_data.f_huffs[v_i])); + v_e_bits = (v_e & 15u); + if (v_e_bits > 8u) { + v_i = ((uint32_t)(((uint32_t)((v_tree * 632u) + (v_e >> 4u))) + (((uint32_t)(((v_bits >> 8u) & 32767u))) & ((((uint32_t)(1u)) << (v_e_bits - 8u)) - 1u)))); + if (v_i >= 678034u) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); + goto label__outer__break; + } + v_e = ((uint32_t)(self->private_data.f_huffs[v_i])); + v_e_bits = ((v_e & 15u) + 8u); + } + if (v_n_bits < v_e_bits) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_bit_reader); + goto label__outer__break; + } + v_bits >>= v_e_bits; + v_n_bits -= v_e_bits; + self->private_impl.f_symbol = (v_e >> 4u); + wuffs_brotli__decoder__put_byte(self, a_workbuf, ((uint8_t)(self->private_impl.f_symbol))); + v_p2 = v_p1; + v_p1 = (self->private_impl.f_symbol & 255u); + self->private_impl.f_block_length[0u] -= 1u; + self->private_impl.f_insert_length -= 1u; + } + self->private_impl.f_cmd_state = 2u; + if ((self->private_impl.f_mb_remaining == 0u) || (self->private_impl.f_mb_remaining >= 2147483648u)) { + self->private_impl.f_cmd_state = 4u; + break; + } + } + if (self->private_impl.f_cmd_state == 2u) { + if (self->private_impl.f_implicit_zero_distance) { + self->private_impl.f_distance_context = 1u; + self->private_impl.f_dist_rb_idx -= 1u; + self->private_impl.f_distance_code = self->private_impl.f_dist_rb[(self->private_impl.f_dist_rb_idx & 3u)]; + } else { + if ((self->private_impl.f_block_length[2u] == 0u) || (((uint64_t)(io2_a_src - iop_a_src)) < 8u)) { + break; + } + v_bits |= ((uint64_t)(wuffs_base__peek_u64le__no_bounds_check(iop_a_src) << v_n_bits)); + iop_a_src += ((63u - v_n_bits) >> 3u); + v_n_bits |= 56u; + v_i = ((uint32_t)((438272u + (self->private_impl.f_dist_htree_index * 920u)) + ((uint32_t)((v_bits & 255u))))); + if (v_i >= 678034u) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); + break; + } + v_e = ((uint32_t)(self->private_data.f_huffs[v_i])); + v_e_bits = (v_e & 15u); + if (v_e_bits > 8u) { + v_i = ((uint32_t)(((uint32_t)((438272u + (self->private_impl.f_dist_htree_index * 920u)) + (v_e >> 4u))) + (((uint32_t)(((v_bits >> 8u) & 32767u))) & ((((uint32_t)(1u)) << (v_e_bits - 8u)) - 1u)))); + if (v_i >= 678034u) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); + break; + } + v_e = ((uint32_t)(self->private_data.f_huffs[v_i])); + v_e_bits = ((v_e & 15u) + 8u); + } + if (v_n_bits < v_e_bits) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_bit_reader); + break; + } + v_bits >>= v_e_bits; + v_n_bits -= v_e_bits; + self->private_impl.f_symbol = (v_e >> 4u); + v_code = self->private_impl.f_symbol; + self->private_impl.f_block_length[2u] -= 1u; + self->private_impl.f_distance_context = 0u; + if (v_code < 16u) { + wuffs_brotli__decoder__take_distance_from_ring_buffer(self, v_code); + } else { + if ((v_code >= self->private_impl.f_distance_alphabet_size) || (v_code >= 1128u)) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); + break; + } + v_nbits = ((uint32_t)(wuffs_base__u8__min(self->private_data.f_dist_extra_bits[v_code], 32u))); + if (v_n_bits < v_nbits) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_bit_reader); + break; + } + v_lo = ((uint32_t)((v_bits & ((((uint64_t)(1u)) << v_nbits) - 1u)))); + self->private_impl.f_bits_value = v_lo; + v_bits >>= v_nbits; + v_n_bits -= v_nbits; + self->private_impl.f_distance_code = ((uint32_t)(((uint64_t)(((uint64_t)(self->private_data.f_dist_offset[v_code])) + ((uint64_t)(((uint64_t)(self->private_impl.f_bits_value)) << self->private_impl.f_distance_postfix_bits)))))); + } + } + self->private_impl.f_cmd_state = 3u; + } + v_max_distance = self->private_impl.f_max_distance; + if (v_max_distance != self->private_impl.f_max_backward_distance) { + v_max_distance = wuffs_base__u32__min(self->private_impl.f_pos, self->private_impl.f_max_backward_distance); + } + v_distance = self->private_impl.f_distance_code; + v_n = self->private_impl.f_copy_length; + if ((v_distance > v_max_distance) || (v_distance == 0u)) { + break; + } + v_dst_i = ((uint64_t)(self->private_impl.f_pos)); + v_src_i = ((uint64_t)((((uint32_t)(self->private_impl.f_pos - v_distance)) & ((uint32_t)(self->private_impl.f_window_size - 1u))))); + if (((v_dst_i + ((uint64_t)(v_n))) >= ((uint64_t)(self->private_impl.f_rb_size))) || ((v_src_i + ((uint64_t)(v_n))) > ((uint64_t)(self->private_impl.f_window_size))) || (((uint64_t)(self->private_impl.f_window_size)) > ((uint64_t)(a_workbuf.len)))) { + break; + } + self->private_impl.f_max_distance = v_max_distance; + self->private_impl.f_dist_rb[(self->private_impl.f_dist_rb_idx & 3u)] = v_distance; + self->private_impl.f_dist_rb_idx += 1u; + self->private_impl.f_mb_remaining -= v_n; + v_k64 = 0u; + while (v_k64 < ((uint64_t)(v_n))) { + v_n64 = ((uint64_t)(((uint64_t)(v_n)) - v_k64)); + v_n64 = wuffs_base__u64__min(v_n64, ((uint64_t)(v_distance))); + v_dlo = ((uint64_t)(v_dst_i + v_k64)); + v_dhi = ((uint64_t)(v_dlo + v_n64)); + v_slo = ((uint64_t)(v_src_i + v_k64)); + v_shi = ((uint64_t)(v_slo + v_n64)); + if ((v_n64 == 0u) || + (v_dlo > v_dhi) || + (v_dhi > ((uint64_t)(a_workbuf.len))) || + (v_slo > v_shi) || + (v_shi > ((uint64_t)(a_workbuf.len)))) { + break; + } + wuffs_private_impl__slice_u8__copy_from_slice(wuffs_base__slice_u8__subslice_ij(a_workbuf, v_dlo, v_dhi), wuffs_base__slice_u8__subslice_ij(a_workbuf, v_slo, v_shi)); + v_k64 += v_n64; + } + self->private_impl.f_pos += ((uint32_t)(v_k64)); + v_n -= ((uint32_t)(v_k64)); + while (v_n > 0u) { + wuffs_brotli__decoder__put_byte(self, a_workbuf, wuffs_brotli__decoder__get_byte(self, a_workbuf, v_distance)); + v_n -= 1u; + } + self->private_impl.f_cmd_state = 0u; + if ((self->private_impl.f_mb_remaining == 0u) || (self->private_impl.f_mb_remaining >= 2147483648u)) { + self->private_impl.f_cmd_state = 4u; + break; + } + } + label__outer__break:; + while (v_n_bits >= 8u) { + if ( ! (iop_a_src > io1_a_src)) { + status = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_bit_reader); + goto exit; + } + iop_a_src--; + v_n_bits -= 8u; + } + self->private_impl.f_bits = ((uint32_t)((v_bits & ((((uint64_t)(1u)) << v_n_bits) - 1u)))); + self->private_impl.f_n_bits = v_n_bits; + status = v_ret; + if (wuffs_base__status__is_error(&status)) { + goto exit; + } else if (wuffs_base__status__is_suspension(&status)) { + status = wuffs_base__make_status(wuffs_base__error__cannot_return_a_suspension); + goto exit; + } + goto ok; + + ok: + goto exit; + exit: + if (a_src && a_src->data.ptr) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + return status; } @@ -84309,18 +85491,24 @@ wuffs_brotli__decoder__read_distance( wuffs_base__status status = wuffs_base__make_status(NULL); uint32_t v_code = 0; + uint32_t v_fr = 0; + uint32_t v_nbits = 0; uint32_t coro_susp_point = self->private_impl.p_read_distance; if (coro_susp_point) { v_code = self->private_data.s_read_distance.v_code; + v_nbits = self->private_data.s_read_distance.v_nbits; } switch (coro_susp_point) { WUFFS_BASE__COROUTINE_SUSPENSION_POINT_0; - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(1); - status = wuffs_brotli__decoder__read_symbol(self, a_src, (438272u + (self->private_impl.f_dist_htree_index * 920u))); - if (status.repr) { - goto suspend; + v_fr = wuffs_brotli__decoder__read_symbol_fast(self, a_src, (438272u + (self->private_impl.f_dist_htree_index * 920u))); + if (v_fr != 0u) { + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(1); + status = wuffs_brotli__decoder__read_symbol(self, a_src, (438272u + (self->private_impl.f_dist_htree_index * 920u))); + if (status.repr) { + goto suspend; + } } v_code = self->private_impl.f_symbol; self->private_impl.f_block_length[2u] -= 1u; @@ -84334,10 +85522,17 @@ wuffs_brotli__decoder__read_distance( status = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); goto exit; } - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(2); - status = wuffs_brotli__decoder__read_bits32(self, a_src, ((uint32_t)(wuffs_base__u8__min(self->private_data.f_dist_extra_bits[v_code], 32u)))); - if (status.repr) { - goto suspend; + v_nbits = ((uint32_t)(wuffs_base__u8__min(self->private_data.f_dist_extra_bits[v_code], 32u))); + v_fr = 1u; + if (v_nbits <= 24u) { + v_fr = wuffs_brotli__decoder__read_bits_fast(self, a_src, v_nbits); + } + if (v_fr != 0u) { + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(2); + status = wuffs_brotli__decoder__read_bits32(self, a_src, v_nbits); + if (status.repr) { + goto suspend; + } } if (v_code >= 1128u) { status = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); @@ -84354,6 +85549,7 @@ wuffs_brotli__decoder__read_distance( suspend: self->private_impl.p_read_distance = wuffs_base__status__is_suspension(&status) ? coro_susp_point : 0; self->private_data.s_read_distance.v_code = v_code; + self->private_data.s_read_distance.v_nbits = v_nbits; goto exit; exit: @@ -84897,30 +86093,6 @@ wuffs_brotli__encoder__create_backward_references( return wuffs_base__make_status(NULL); } -// -------- func brotli.bit_cost.init_log2_cache - -WUFFS_BASE__GENERATED_C_CODE -static wuffs_base__empty_struct -wuffs_brotli__bit_cost__init_log2_cache( - wuffs_brotli__bit_cost* self) { - uint64_t v_v = 0; - - if (self->private_impl.f_log2_cache_ready) { - return wuffs_base__make_empty_struct(); - } - v_v = 0u; - while (v_v < 256u) { - self->private_data.f_log2_cache[v_v] = WUFFS_BROTLI__LOG2_TABLE[v_v]; - v_v += 1u; - } - while (v_v < 65536u) { - self->private_data.f_log2_cache[v_v] = wuffs_softfloat__arith__log2_u64(&self->private_data.f_sf, v_v); - v_v += 1u; - } - self->private_impl.f_log2_cache_ready = true; - return wuffs_base__make_empty_struct(); -} - // -------- func brotli.bit_cost.ld32 WUFFS_BASE__GENERATED_C_CODE @@ -85008,23 +86180,44 @@ wuffs_brotli__bit_cost__fast_log2( uint64_t a_v) { if (a_v < 256u) { return WUFFS_BROTLI__LOG2_TABLE[a_v]; - } else if (self->private_impl.f_log2_cache_ready && (a_v < 65536u)) { - return self->private_data.f_log2_cache[a_v]; + } else if (a_v < 65536u) { + if (((self->private_impl.f_log2_valid[(a_v >> 6u)] >> (a_v & 63u)) & 1u) != 0u) { + return self->private_data.f_log2_cache[a_v]; + } } return wuffs_softfloat__arith__log2_u64(&self->private_data.f_sf, a_v); } +// -------- func brotli.bit_cost.fast_log2_memo + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__bit_cost__fast_log2_memo( + wuffs_brotli__bit_cost* self, + uint64_t a_v) { + if (a_v < 256u) { + return WUFFS_BROTLI__LOG2_TABLE[a_v]; + } else if (a_v >= 65536u) { + return wuffs_softfloat__arith__log2_u64(&self->private_data.f_sf, a_v); + } else if (((self->private_impl.f_log2_valid[(a_v >> 6u)] >> (a_v & 63u)) & 1u) == 0u) { + self->private_data.f_log2_cache[a_v] = wuffs_softfloat__arith__log2_u64(&self->private_data.f_sf, a_v); + self->private_impl.f_log2_valid[(a_v >> 6u)] |= (((uint64_t)(1u)) << (a_v & 63u)); + } + return self->private_data.f_log2_cache[a_v]; +} + // -------- func brotli.bit_cost.bits_entropy WUFFS_BASE__GENERATED_C_CODE static uint64_t wuffs_brotli__bit_cost__bits_entropy( - const wuffs_brotli__bit_cost* self, + wuffs_brotli__bit_cost* self, wuffs_base__slice_u8 a_population, uint64_t a_size) { uint64_t v_sum = 0; uint64_t v_retval = 0; uint64_t v_p = 0; + uint64_t v_lp = 0; wuffs_base__slice_u8 v_q = {0}; uint64_t v_n = 0; @@ -85038,10 +86231,12 @@ wuffs_brotli__bit_cost__bits_entropy( v_q = wuffs_base__slice_u8__subslice_i(v_q, 4u); } v_sum += v_p; - v_retval = wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, v_retval, wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_p), wuffs_brotli__bit_cost__fast_log2(self, v_p))); + v_lp = wuffs_brotli__bit_cost__fast_log2_memo(self, v_p); + v_retval = wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, v_retval, wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_p), v_lp)); } if (v_sum != 0u) { - v_retval = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_retval, wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_sum), wuffs_brotli__bit_cost__fast_log2(self, v_sum))); + v_lp = wuffs_brotli__bit_cost__fast_log2_memo(self, v_sum); + v_retval = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_retval, wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_sum), v_lp)); } if (wuffs_softfloat__arith__f64_lt(&self->private_data.f_sf, v_retval, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_sum)) != 0u) { v_retval = wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_sum); @@ -85054,7 +86249,7 @@ wuffs_brotli__bit_cost__bits_entropy( WUFFS_BASE__GENERATED_C_CODE static uint64_t wuffs_brotli__bit_cost__population_cost( - const wuffs_brotli__bit_cost* self, + wuffs_brotli__bit_cost* self, wuffs_base__slice_u8 a_histogram, uint32_t a_data_size) { uint64_t v_total = 0; @@ -85075,6 +86270,7 @@ wuffs_brotli__bit_cost__population_cost( uint32_t v_depth_histo[18] = {0}; uint64_t v_log2total = 0; uint64_t v_log2p = 0; + uint64_t v_lp = 0; uint64_t v_depth = 0; uint32_t v_reps = 0; uint64_t v_sum = 0; @@ -85135,12 +86331,13 @@ wuffs_brotli__bit_cost__population_cost( return wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, 4630404104378646528u, wuffs_softfloat__arith__f64_from_u32(&self->private_data.f_sf, ((uint32_t)(3u * v_h23)))), wuffs_softfloat__arith__f64_from_u32(&self->private_data.f_sf, ((uint32_t)(2u * ((uint32_t)(v_histo[0u] + v_histo[1u])))))), wuffs_softfloat__arith__f64_from_u32(&self->private_data.f_sf, v_histomax)); } v_max_depth = 1u; - v_log2total = wuffs_brotli__bit_cost__fast_log2(self, v_total); + v_log2total = wuffs_brotli__bit_cost__fast_log2_memo(self, v_total); v_i = 0u; while (v_i < a_data_size) { v_c = wuffs_brotli__bit_cost__ld32(self, a_histogram, (((uint64_t)(v_i)) * 4u)); if (v_c > 0u) { - v_log2p = wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, v_log2total, wuffs_brotli__bit_cost__fast_log2(self, ((uint64_t)(v_c)))); + v_lp = wuffs_brotli__bit_cost__fast_log2_memo(self, ((uint64_t)(v_c))); + v_log2p = wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, v_log2total, v_lp); v_depth = wuffs_softfloat__arith__f64_to_u64_trunc(&self->private_data.f_sf, wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_log2p, 4602678819172646912u)); v_bits = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_bits, wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u32(&self->private_data.f_sf, v_c), v_log2p)); if (v_depth > 15u) { @@ -85182,11 +86379,13 @@ wuffs_brotli__bit_cost__population_cost( while (v_i < 18u) { v_p = ((uint64_t)(v_depth_histo[v_i])); v_sum += v_p; - v_retval = wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, v_retval, wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_p), wuffs_brotli__bit_cost__fast_log2(self, v_p))); + v_lp = wuffs_brotli__bit_cost__fast_log2_memo(self, v_p); + v_retval = wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, v_retval, wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_p), v_lp)); v_i += 1u; } if (v_sum != 0u) { - v_retval = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_retval, wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_sum), wuffs_brotli__bit_cost__fast_log2(self, v_sum))); + v_lp = wuffs_brotli__bit_cost__fast_log2_memo(self, v_sum); + v_retval = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_retval, wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_sum), v_lp)); } if (wuffs_softfloat__arith__f64_lt(&self->private_data.f_sf, v_retval, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_sum)) != 0u) { v_retval = wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_sum); @@ -86791,12 +87990,15 @@ wuffs_brotli__encoder__find_match_length_with_limit( WUFFS_BASE__GENERATED_C_CODE static uint64_t wuffs_brotli__metablock_builder__bit_cost( - const wuffs_brotli__metablock_builder* self, + wuffs_brotli__metablock_builder* self, uint64_t a_count) { + uint64_t v_c = 0; + if (a_count == 0u) { return 13835058055282163712u; } - return wuffs_brotli__bit_cost__fast_log2(&self->private_data.f_bc, a_count); + v_c = wuffs_brotli__bit_cost__fast_log2_memo(&self->private_data.f_bc, a_count); + return v_c; } // -------- func brotli.metablock_builder.histogram_add_vector @@ -87183,7 +88385,7 @@ wuffs_brotli__metablock_builder__find_blocks( wuffs_private_impl__bulk_memset(&self->private_data.f_insert_cost[0], 35200u * (size_t)8u, 0u); v_i = 0u; while (v_i < a_num_histograms) { - self->private_data.f_insert_cost[(v_i & 255u)] = wuffs_brotli__bit_cost__fast_log2(&self->private_data.f_bc, (wuffs_brotli__metablock_builder__h_total(self, a_wb, ((uint64_t)(a_histograms + ((uint64_t)(v_i * v_hs)))), a_ds) & 4294967295u)); + self->private_data.f_insert_cost[(v_i & 255u)] = wuffs_brotli__bit_cost__fast_log2_memo(&self->private_data.f_bc, (wuffs_brotli__metablock_builder__h_total(self, a_wb, ((uint64_t)(a_histograms + ((uint64_t)(v_i * v_hs)))), a_ds) & 4294967295u)); v_i += 1u; } v_i = v_alphabet_size; @@ -87193,7 +88395,8 @@ wuffs_brotli__metablock_builder__find_blocks( while (v_j < a_num_histograms) { v_k = ((uint64_t)(((uint64_t)(v_i * a_num_histograms)) + v_j)); if (v_k < 35200u) { - self->private_data.f_insert_cost[v_k] = wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, self->private_data.f_insert_cost[(v_j & 255u)], wuffs_brotli__metablock_builder__bit_cost(self, ((uint64_t)(wuffs_brotli__metablock_builder__ld32(self, a_wb, ((uint64_t)(((uint64_t)(a_histograms + ((uint64_t)(v_j * v_hs)))) + ((uint64_t)(v_i * 4u))))))))); + v_c = wuffs_brotli__metablock_builder__bit_cost(self, ((uint64_t)(wuffs_brotli__metablock_builder__ld32(self, a_wb, ((uint64_t)(((uint64_t)(a_histograms + ((uint64_t)(v_j * v_hs)))) + ((uint64_t)(v_i * 4u)))))))); + self->private_data.f_insert_cost[v_k] = wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, self->private_data.f_insert_cost[(v_j & 255u)], v_c); } v_j += 1u; } @@ -87363,6 +88566,7 @@ wuffs_brotli__metablock_builder__cluster_blocks( uint64_t a_num_blocks, uint64_t a_block_ids, uint32_t a_ds) { + uint64_t v_c = 0; uint64_t v_hs = 0; uint64_t v_histogram_symbols = 0; uint64_t v_u32s = 0; @@ -87452,11 +88656,12 @@ wuffs_brotli__metablock_builder__cluster_blocks( v_pos += 1u; v_k += 1u; } + v_c = wuffs_brotli__metablock_builder__h_population_cost(self, a_wb, ((uint64_t)(v_histograms + ((uint64_t)(v_j * v_hs)))), a_ds); wuffs_brotli__metablock_builder__h_set_bit_cost(self, a_wb, ((uint64_t)(v_histograms + ((uint64_t)(v_j * v_hs)))), a_ds, - wuffs_brotli__metablock_builder__h_population_cost(self, a_wb, ((uint64_t)(v_histograms + ((uint64_t)(v_j * v_hs)))), a_ds)); + v_c); wuffs_brotli__metablock_builder__st32(self, a_wb, ((uint64_t)(v_new_clusters + ((uint64_t)(v_j * 4u)))), ((uint32_t)(v_j))); wuffs_brotli__metablock_builder__st32(self, a_wb, ((uint64_t)(v_symbols + ((uint64_t)(v_j * 4u)))), ((uint32_t)(v_j))); wuffs_brotli__metablock_builder__st32(self, a_wb, ((uint64_t)(v_sizes + ((uint64_t)(v_j * 4u)))), 1u); @@ -89130,6 +90335,8 @@ wuffs_brotli__metablock_builder__histogram_bit_cost_distance( uint64_t a_histogram, uint64_t a_candidate, uint64_t a_tmp) { + uint64_t v_c = 0; + if (wuffs_brotli__metablock_builder__h_total(self, a_wb, a_histogram, a_ds) == 0u) { return 0u; } @@ -89143,7 +90350,8 @@ wuffs_brotli__metablock_builder__histogram_bit_cost_distance( a_tmp, a_candidate, a_ds); - return wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, wuffs_brotli__metablock_builder__h_population_cost(self, a_wb, a_tmp, a_ds), wuffs_brotli__metablock_builder__h_bit_cost(self, a_wb, a_candidate, a_ds)); + v_c = wuffs_brotli__metablock_builder__h_population_cost(self, a_wb, a_tmp, a_ds); + return wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, v_c, wuffs_brotli__metablock_builder__h_bit_cost(self, a_wb, a_candidate, a_ds)); } // -------- func brotli.metablock_builder.histogram_remap @@ -89307,6 +90515,7 @@ wuffs_brotli__metablock_builder__cluster_histograms( uint64_t a_max_histograms, uint64_t a_out_histos, uint64_t a_histogram_symbols) { + uint64_t v_c = 0; uint64_t v_hs = 0; uint64_t v_mark = 0; uint64_t v_cluster_size = 0; @@ -89344,11 +90553,12 @@ wuffs_brotli__metablock_builder__cluster_histograms( ((uint64_t)(a_out_histos + ((uint64_t)(v_i * v_hs)))), ((uint64_t)(a_in_histos + ((uint64_t)(v_i * v_hs)))), a_ds); + v_c = wuffs_brotli__metablock_builder__h_population_cost(self, a_wb, ((uint64_t)(a_in_histos + ((uint64_t)(v_i * v_hs)))), a_ds); wuffs_brotli__metablock_builder__h_set_bit_cost(self, a_wb, ((uint64_t)(a_out_histos + ((uint64_t)(v_i * v_hs)))), a_ds, - wuffs_brotli__metablock_builder__h_population_cost(self, a_wb, ((uint64_t)(a_in_histos + ((uint64_t)(v_i * v_hs)))), a_ds)); + v_c); wuffs_brotli__metablock_builder__st32(self, a_wb, ((uint64_t)(a_histogram_symbols + ((uint64_t)(v_i * 4u)))), ((uint32_t)(v_i))); v_i += 1u; } @@ -96197,7 +97407,8 @@ wuffs_brotli__encoder__estimate_bit_costs_for_literals_utf8( } else if (v_utf8_pos == 2u) { v_iw = v_iwu2; } - v_lit_cost = wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, wuffs_brotli__bit_cost__fast_log2(&self->private_data.f_bc, v_iw), wuffs_brotli__bit_cost__fast_log2(&self->private_data.f_bc, v_histo)); + v_lit_cost = wuffs_brotli__bit_cost__fast_log2_memo(&self->private_data.f_bc, v_iw); + v_lit_cost = wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, v_lit_cost, wuffs_brotli__bit_cost__fast_log2(&self->private_data.f_bc, v_histo)); v_lit_cost = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_lit_cost, 4584030313835631162u); wuffs_brotli__encoder__lc_finish_cost(self, a_wb, @@ -96268,7 +97479,8 @@ wuffs_brotli__encoder__estimate_bit_costs_for_literals( if (v_histo == 0u) { v_histo = 1u; } - v_lit_cost = wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, wuffs_brotli__bit_cost__fast_log2(&self->private_data.f_bc, v_in_window), wuffs_brotli__bit_cost__fast_log2(&self->private_data.f_bc, v_histo)); + v_lit_cost = wuffs_brotli__bit_cost__fast_log2_memo(&self->private_data.f_bc, v_in_window); + v_lit_cost = wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, v_lit_cost, wuffs_brotli__bit_cost__fast_log2(&self->private_data.f_bc, v_histo)); v_lit_cost = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_lit_cost, 4584015902316823577u); wuffs_brotli__encoder__lc_finish_cost(self, a_wb, @@ -96598,12 +97810,15 @@ wuffs_brotli__metablock_builder__h_set_bit_cost( WUFFS_BASE__GENERATED_C_CODE static uint64_t wuffs_brotli__metablock_builder__h_population_cost( - const wuffs_brotli__metablock_builder* self, + wuffs_brotli__metablock_builder* self, wuffs_base__slice_u8 a_wb, uint64_t a_h, uint32_t a_ds) { + uint64_t v_c = 0; + if (a_h <= ((uint64_t)(a_wb.len))) { - return wuffs_brotli__bit_cost__population_cost(&self->private_data.f_bc, wuffs_base__slice_u8__subslice_i(a_wb, a_h), a_ds); + v_c = wuffs_brotli__bit_cost__population_cost(&self->private_data.f_bc, wuffs_base__slice_u8__subslice_i(a_wb, a_h), a_ds); + return v_c; } return 0u; } @@ -96859,7 +98074,8 @@ wuffs_brotli__metablock_builder__compute_distance_cost( } v_i += 1u; } - self->private_impl.f_dist_cost = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, wuffs_brotli__metablock_builder__h_population_cost(self, a_wb, a_tmp, 544u), v_extra_bits); + self->private_impl.f_dist_cost = wuffs_brotli__metablock_builder__h_population_cost(self, a_wb, a_tmp, 544u); + self->private_impl.f_dist_cost = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, self->private_impl.f_dist_cost, v_extra_bits); return true; } @@ -96979,7 +98195,6 @@ wuffs_brotli__metablock_builder__build_meta_block( uint32_t v_v = 0; wuffs_brotli__metablock_builder__reset_arena(self, ((uint64_t)(a_wb.len))); - wuffs_brotli__bit_cost__init_log2_cache(&self->private_data.f_bc); v_orig_npostfix = self->private_impl.f_dist_postfix_bits; v_orig_ndirect = self->private_impl.f_dist_num_direct_codes; v_orig_asm = self->private_impl.f_dist_alphabet_size_max; @@ -97528,12 +98743,15 @@ wuffs_brotli__encoder__gbs_histo_off( WUFFS_BASE__GENERATED_C_CODE static uint64_t wuffs_brotli__encoder__gbs_entropy( - const wuffs_brotli__encoder* self, + wuffs_brotli__encoder* self, wuffs_base__slice_u8 a_wb, uint64_t a_off, uint64_t a_size) { + uint64_t v_e = 0; + if (a_off <= ((uint64_t)(a_wb.len))) { - return wuffs_brotli__bit_cost__bits_entropy(&self->private_data.f_bc, wuffs_base__slice_u8__subslice_i(a_wb, a_off), a_size); + v_e = wuffs_brotli__bit_cost__bits_entropy(&self->private_data.f_bc, wuffs_base__slice_u8__subslice_i(a_wb, a_off), a_size); + return v_e; } return 0u; } @@ -99178,18 +100396,20 @@ wuffs_brotli__encoder__hq_extend_var_region( WUFFS_BASE__GENERATED_C_CODE WUFFS_BASE__MAYBE_STATIC uint64_t wuffs_brotli__metablock_tester__bits_entropy( - const wuffs_brotli__metablock_tester* self, + wuffs_brotli__metablock_tester* self, wuffs_base__slice_u8 a_population, uint64_t a_size) { if (!self) { return 0; } - if ((self->private_impl.magic != WUFFS_BASE__MAGIC) && - (self->private_impl.magic != WUFFS_BASE__DISABLED)) { + if (self->private_impl.magic != WUFFS_BASE__MAGIC) { return 0; } - return wuffs_brotli__bit_cost__bits_entropy(&self->private_data.f_bc, a_population, a_size); + uint64_t v_c = 0; + + v_c = wuffs_brotli__bit_cost__bits_entropy(&self->private_data.f_bc, a_population, a_size); + return v_c; } // -------- func brotli.metablock_tester.population_cost @@ -99197,23 +100417,27 @@ wuffs_brotli__metablock_tester__bits_entropy( WUFFS_BASE__GENERATED_C_CODE WUFFS_BASE__MAYBE_STATIC uint64_t wuffs_brotli__metablock_tester__population_cost( - const wuffs_brotli__metablock_tester* self, + wuffs_brotli__metablock_tester* self, wuffs_base__slice_u8 a_histogram, uint32_t a_data_size) { if (!self) { return 0; } - if ((self->private_impl.magic != WUFFS_BASE__MAGIC) && - (self->private_impl.magic != WUFFS_BASE__DISABLED)) { + if (self->private_impl.magic != WUFFS_BASE__MAGIC) { return 0; } + uint64_t v_c = 0; + if (a_data_size == 256u) { - return wuffs_brotli__bit_cost__population_cost(&self->private_data.f_bc, a_histogram, 256u); + v_c = wuffs_brotli__bit_cost__population_cost(&self->private_data.f_bc, a_histogram, 256u); + return v_c; } else if (a_data_size == 704u) { - return wuffs_brotli__bit_cost__population_cost(&self->private_data.f_bc, a_histogram, 704u); + v_c = wuffs_brotli__bit_cost__population_cost(&self->private_data.f_bc, a_histogram, 704u); + return v_c; } else if (a_data_size == 544u) { - return wuffs_brotli__bit_cost__population_cost(&self->private_data.f_bc, a_histogram, 544u); + v_c = wuffs_brotli__bit_cost__population_cost(&self->private_data.f_bc, a_histogram, 544u); + return v_c; } return 0u; } @@ -99263,7 +100487,6 @@ wuffs_brotli__metablock_tester__split_block( wuffs_brotli__metablock_builder__set_params(&self->private_data.f_mb, a_quality, false, false); wuffs_brotli__metablock_builder__reset_arena(&self->private_data.f_mb, ((uint64_t)(a_workbuf.len))); - wuffs_brotli__bit_cost__init_log2_cache(&self->private_data.f_mb.private_data.f_bc); wuffs_brotli__metablock_builder__alloc_block_splits(&self->private_data.f_mb, a_cmds, a_num_commands); if (self->private_data.f_mb.private_impl.f_oom) { return wuffs_base__make_status(wuffs_base__error__bad_workbuf_length); @@ -99317,7 +100540,6 @@ wuffs_brotli__metablock_tester__cluster_histograms( return 18446744073709551615u; } v_hs = wuffs_brotli__metablock_builder__hsize(&self->private_data.f_mb, v_ds); - wuffs_brotli__bit_cost__init_log2_cache(&self->private_data.f_mb.private_data.f_bc); wuffs_brotli__metablock_builder__reset_arena(&self->private_data.f_mb, ((uint64_t)(a_workbuf.len))); v_n = ((uint64_t)(a_in_size * v_hs)); v_in_off = wuffs_brotli__metablock_builder__alloc(&self->private_data.f_mb, v_n); diff --git a/script/manual-test-softfloat-sweep.c b/script/manual-test-softfloat-sweep.c new file mode 100644 index 00000000..b47b3301 --- /dev/null +++ b/script/manual-test-softfloat-sweep.c @@ -0,0 +1,687 @@ +// Copyright 2026 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. +// +// SPDX-License-Identifier: Apache-2.0 OR MIT + +// ---------------- + +// manual-test-softfloat-sweep.c systematically checks Wuffs' std/softfloat +// binary32 operations (and the f64 to / from f32 conversions) over every +// combination of operand exponents, signs and mantissa patterns, plus +// exhaustive sweeps of the integer conversions. Each f32 result is compared +// against: +// +// - the C compiler's native (hardware) float arithmetic: the bits must be +// identical, except that a NaN only has to match a NaN, and +// - the "via f64" definition, f64_to_f32(f64_op(f32_to_f64(a), +// f32_to_f64(b))), computed with the package's own (public) f64 functions: +// here the bits must be identical even for NaNs, so that the NaN payloads +// and signs produced by the f32 fast paths' slow path fallbacks are +// exactly those of the via-f64 implementation. +// +// The operands for the binary operations are every (exponent_a, exponent_b) +// pair in [0 ..= 255] x [0 ..= 255], times both signs for each operand, times +// NUM_MANTISSAS x NUM_MANTISSAS mantissa patterns (zero, one, all ones, only +// top / bottom bits, half-way patterns and pseudo-random ones, so that exact +// results, rounding ties, carries, cancellation, subnormal results, overflow +// to infinity and NaN payloads are all covered). +// +// The f64 add, sub, mul and div operations are checked the same way (every +// exponent pair in [0 ..= 2047] x [0 ..= 2047], both signs, NUM_MANTISSAS64 +// x NUM_MANTISSAS64 mantissa patterns) against the hardware, with NaN +// results checked bit-for-bit against the package's documented NaN rule: +// the first NaN operand, quietened, else the default NaN 0xFFF8_0000_0000_0000. +// +// Build it WITHOUT -ffast-math and WITHOUT -mfma / -march=native: +// +// gcc -O2 -std=c99 script/manual-test-softfloat-sweep.c -lm -lpthread +// -o /tmp/sfsweep +// +// (as one command line), then run /tmp/sfsweep (or /tmp/sfsweep OPNAME to run +// only one op: f32_add, f32_sub, f32_mul, f32_div, f64_add, f64_sub, f64_mul, +// f64_div, f64_to_f32, f32_to_f64, f32_from_u32, f32_from_i32, f32_from_u64, +// f32_compare or f64_compare). It exits with a non-zero status if there were +// any mismatches. +// +// The f32_compare and f64_compare checks run the cmp, eq, ne, lt, le, gt, ge, +// min and max functions on every pair of a set of values (both signs, zero, +// subnormal, normal and maximal exponents, boundary mantissas and their +// neighbors, infinities and NaNs), against the hardware's comparison +// operators. For min and max, the result must be the bits of +// ((a < b) ? a : b) and ((a > b) ? a : b). + +#define _POSIX_C_SOURCE 200809L + +#include +#include +#include +#include +#include +#include +#include +#include + +#define WUFFS_IMPLEMENTATION +#define WUFFS_CONFIG__STATIC_FUNCTIONS +#define WUFFS_CONFIG__MODULES +#define WUFFS_CONFIG__MODULE__BASE +#define WUFFS_CONFIG__MODULE__SOFTFLOAT +#include "../release/c/wuffs-unsupported-snapshot.c" + +#if defined(__FAST_MATH__) || defined(__FMA__) +#error "Build without -ffast-math and without -mfma / -march=native" +#endif + +#define NUM_THREADS 4 + +static wuffs_softfloat__arith g_sf; + +static float // +f_of(uint32_t u) { + float f; + memcpy(&f, &u, 4); + return f; +} + +static uint32_t // +u_of_f(float f) { + uint32_t u; + memcpy(&u, &f, 4); + return u; +} + +static double // +d_of(uint64_t u) { + double d; + memcpy(&d, &u, 8); + return d; +} + +static uint64_t // +u_of_d(double d) { + uint64_t u; + memcpy(&u, &d, 8); + return u; +} + +static bool // +is_nan32(uint32_t u) { + return (u & 0x7FFFFFFFu) > 0x7F800000u; +} + +static bool // +is_nan64(uint64_t u) { + return (u & 0x7FFFFFFFFFFFFFFFull) > 0x7FF0000000000000ull; +} + +// ---------------- Mantissa patterns. + +#define NUM_MANTISSAS 40 + +static uint32_t g_mantissas[NUM_MANTISSAS]; + +static void // +init_mantissas() { + static const uint32_t fixed[] = { + 0x000000, 0x000001, 0x000002, 0x000003, 0x7FFFFF, 0x7FFFFE, 0x7FFFFD, + 0x400000, 0x400001, 0x3FFFFF, 0x200000, 0x600000, 0x000100, 0x0000FF, + 0x100001, 0x080000, 0x7C0000, 0x000040, 0x00003F, 0x555555, 0x2AAAAA, + 0x7FFF80, 0x000080, 0x400080, 0x3FFF80, + }; + size_t n = sizeof(fixed) / sizeof(fixed[0]); + memcpy(g_mantissas, fixed, sizeof(fixed)); + uint64_t x = 0x9E3779B97F4A7C15ull; + for (size_t i = n; i < NUM_MANTISSAS; i++) { + x ^= x >> 12; + x ^= x << 25; + x ^= x >> 27; + uint32_t r = (uint32_t)((x * 0x2545F4914F6CDD1Dull) >> 41) & 0x7FFFFF; + if (i & 1) { + r &= 0x7FF000 | (r & 0xF); // Sparse. + } + g_mantissas[i] = r; + } +} + +#define NUM_MANTISSAS64 9 + +static uint64_t g_mantissas64[NUM_MANTISSAS64]; + +static void // +init_mantissas64() { + static const uint64_t fixed[] = { + 0x0000000000000ull, 0x0000000000001ull, 0xFFFFFFFFFFFFFull, + 0x8000000000000ull, 0x8000000000001ull, 0x0000000000200ull, + }; + size_t n = sizeof(fixed) / sizeof(fixed[0]); + memcpy(g_mantissas64, fixed, sizeof(fixed)); + uint64_t x = 0x0123456789ABCDEFull; + for (size_t i = n; i < NUM_MANTISSAS64; i++) { + x ^= x >> 12; + x ^= x << 25; + x ^= x >> 27; + uint64_t r = (x * 0x2545F4914F6CDD1Dull) >> 12; + if (i & 1) { + r &= 0xFFF00000000FFull; // Sparse. + } + g_mantissas64[i] = r; + } +} + +// ---------------- Binary ops. + +typedef uint32_t (*wuffs_op)(const wuffs_softfloat__arith*, uint32_t, uint32_t); +typedef uint64_t (*wuffs_op64)(const wuffs_softfloat__arith*, + uint64_t, + uint64_t); + +static float // +hw_add(float a, float b) { + return a + b; +} +static float // +hw_sub(float a, float b) { + return a - b; +} +static float // +hw_mul(float a, float b) { + return a * b; +} +static float // +hw_div(float a, float b) { + return a / b; +} + +static struct { + const char* name; + wuffs_op op; + wuffs_op64 op64; + float (*hw)(float, float); +} g_binops[] = { + {"f32_add", wuffs_softfloat__arith__f32_add, wuffs_softfloat__arith__f64_add, + hw_add}, + {"f32_sub", wuffs_softfloat__arith__f32_sub, wuffs_softfloat__arith__f64_sub, + hw_sub}, + {"f32_mul", wuffs_softfloat__arith__f32_mul, wuffs_softfloat__arith__f64_mul, + hw_mul}, + {"f32_div", wuffs_softfloat__arith__f32_div, wuffs_softfloat__arith__f64_div, + hw_div}, +}; + +static double // +hw_add64(double a, double b) { + return a + b; +} +static double // +hw_sub64(double a, double b) { + return a - b; +} +static double // +hw_mul64(double a, double b) { + return a * b; +} +static double // +hw_div64(double a, double b) { + return a / b; +} + +static struct { + const char* name; + wuffs_op64 op64; + double (*hw)(double, double); +} g_binops64[] = { + {"f64_add", wuffs_softfloat__arith__f64_add, hw_add64}, + {"f64_sub", wuffs_softfloat__arith__f64_sub, hw_sub64}, + {"f64_mul", wuffs_softfloat__arith__f64_mul, hw_mul64}, + {"f64_div", wuffs_softfloat__arith__f64_div, hw_div64}, +}; + +typedef struct { + int which; + uint32_t ea_lo; + uint32_t ea_hi; + uint64_t count; + uint64_t mismatches; +} job; + +static pthread_mutex_t g_print_mutex = PTHREAD_MUTEX_INITIALIZER; + +static void // +report(job* j, const char* name, uint32_t a, uint32_t b, uint32_t have, + uint32_t want_hw, uint32_t want_via) { + j->mismatches++; + if (j->mismatches <= 5) { + pthread_mutex_lock(&g_print_mutex); + printf("MISMATCH %s(0x%08" PRIX32 ", 0x%08" PRIX32 "): have 0x%08" PRIX32 + ", hardware 0x%08" PRIX32 ", via f64 0x%08" PRIX32 "\n", + name, a, b, have, want_hw, want_via); + pthread_mutex_unlock(&g_print_mutex); + } +} + +static void* // +run_binop(void* arg) { + job* j = (job*)arg; + wuffs_op op = g_binops[j->which].op; + wuffs_op64 op64 = g_binops[j->which].op64; + float (*hw)(float, float) = g_binops[j->which].hw; + const char* name = g_binops[j->which].name; + for (uint32_t ea = j->ea_lo; ea < j->ea_hi; ea++) { + for (uint32_t eb = 0; eb < 256; eb++) { + for (uint32_t ma = 0; ma < NUM_MANTISSAS; ma++) { + for (uint32_t mb = 0; mb < NUM_MANTISSAS; mb++) { + for (uint32_t signs = 0; signs < 4; signs++) { + uint32_t a = ((signs & 1) << 31) | (ea << 23) | g_mantissas[ma]; + uint32_t b = ((signs >> 1) << 31) | (eb << 23) | g_mantissas[mb]; + uint32_t have = op(&g_sf, a, b); + uint32_t want_hw = u_of_f(hw(f_of(a), f_of(b))); + uint32_t want_via = wuffs_softfloat__arith__f64_to_f32( + &g_sf, op64(&g_sf, wuffs_softfloat__arith__f32_to_f64(&g_sf, a), + wuffs_softfloat__arith__f32_to_f64(&g_sf, b))); + j->count++; + if (have != want_via) { + report(j, name, a, b, have, want_hw, want_via); + } else if (is_nan32(have) ? !is_nan32(want_hw) + : (have != want_hw)) { + report(j, name, a, b, have, want_hw, want_via); + } + } + } + } + } + } + return NULL; +} + +static void* // +run_binop64(void* arg) { + job* j = (job*)arg; + wuffs_op64 op64 = g_binops64[j->which].op64; + double (*hw)(double, double) = g_binops64[j->which].hw; + const char* name = g_binops64[j->which].name; + for (uint64_t ea = j->ea_lo; ea < j->ea_hi; ea++) { + for (uint64_t eb = 0; eb < 2048; eb++) { + for (uint32_t ma = 0; ma < NUM_MANTISSAS64; ma++) { + for (uint32_t mb = 0; mb < NUM_MANTISSAS64; mb++) { + for (uint64_t signs = 0; signs < 4; signs++) { + uint64_t a = ((signs & 1) << 63) | (ea << 52) | g_mantissas64[ma]; + uint64_t b = ((signs >> 1) << 63) | (eb << 52) | g_mantissas64[mb]; + uint64_t have = op64(&g_sf, a, b); + uint64_t want = u_of_d(hw(d_of(a), d_of(b))); + j->count++; + if (is_nan64(want)) { + want = is_nan64(a) ? (a | 0x0008000000000000ull) + : is_nan64(b) ? (b | 0x0008000000000000ull) + : 0xFFF8000000000000ull; + } + if (have != want) { + j->mismatches++; + if (j->mismatches <= 5) { + pthread_mutex_lock(&g_print_mutex); + printf("MISMATCH %s(0x%016" PRIX64 ", 0x%016" PRIX64 + "): have 0x%016" PRIX64 ", want 0x%016" PRIX64 "\n", + name, a, b, have, want); + pthread_mutex_unlock(&g_print_mutex); + } + } + } + } + } + } + } + return NULL; +} + +// ---------------- Conversions. + +typedef struct { + int which; + uint64_t lo; + uint64_t hi; + uint64_t count; + uint64_t mismatches; +} cjob; + +static void // +creport(cjob* j, const char* name, uint64_t a, uint64_t have, uint64_t want) { + j->mismatches++; + if (j->mismatches <= 5) { + pthread_mutex_lock(&g_print_mutex); + printf("MISMATCH %s(0x%016" PRIX64 "): have 0x%016" PRIX64 + ", want 0x%016" PRIX64 "\n", + name, a, have, want); + pthread_mutex_unlock(&g_print_mutex); + } +} + +// f32_from_u64 operands: (1 << top) plus, for top <= 26, every pattern of +// the bits below it or, for larger top, every pattern of the 13 bits just +// below it (where the rounding happens) and of the 13 bottom bits (sticky). +static uint64_t // +u64_operand(uint64_t i) { + uint32_t top = (uint32_t)(i >> 26) & 63; + uint64_t low = i & 0x3FFFFFF; + uint64_t below = low; + if (top > 26) { + below = ((low >> 13) << (top - 13)) | (low & 0x1FFF); + } + below &= (((uint64_t)1) << top) - 1; + return (((uint64_t)1) << top) | below; +} + +// f64_to_f32_exponent maps [0 ..= 325] to the f64 biased exponents 0, 1, 2, +// 0x340 ..= 0x47F (the f32 normal and subnormal range, and a little more on +// either side) and 0x7FD, 0x7FE and 0x7FF. +static uint64_t // +f64_to_f32_exponent(uint32_t i) { + if (i < 3) { + return i; + } else if (i < 323) { + return 0x340 + (i - 3); + } + return 0x7FD + (i - 323); +} + +static void* // +run_conv(void* arg) { + cjob* j = (cjob*)arg; + for (uint64_t i = j->lo; i < j->hi; i++) { + j->count++; + switch (j->which) { + case 0: { // f64_to_f32: exponents near the f32 range, many fractions. + uint64_t e = f64_to_f32_exponent((uint32_t)(i >> 20)); + uint64_t f = i & 0xFFFFF; + // 10 bits at the top of the fraction, 10 around the rounding point + // (bit 29) and 6 (repeated) at the bottom. + uint64_t frac = ((f >> 10) << 42) | ((f & 0x3FF) << 24) | + ((f & 0x3F) * 0x41); + for (int s = 0; s < 2; s++) { + uint64_t a = ((uint64_t)s << 63) | (e << 52) | frac; + uint32_t have = wuffs_softfloat__arith__f64_to_f32(&g_sf, a); + uint32_t want = u_of_f((float)d_of(a)); + if (is_nan32(have) ? !is_nan32(want) : (have != want)) { + creport(j, "f64_to_f32", a, have, want); + } else if (is_nan32(have) && + (have != (((uint32_t)(a >> 32) & 0x80000000u) | + 0x7FC00000u | + (uint32_t)((a & 0xFFFFFFFFFFFFFull) >> 29)))) { + creport(j, "f64_to_f32 (NaN payload)", a, have, want); + } + } + break; + } + case 1: { // f32_to_f64: every f32. + uint32_t a = (uint32_t)i; + uint64_t have = wuffs_softfloat__arith__f32_to_f64(&g_sf, a); + uint64_t want = u_of_d((double)f_of(a)); + if (is_nan64(have) ? !is_nan64(want) : (have != want)) { + creport(j, "f32_to_f64", a, have, want); + } else if (is_nan64(have) && + (have != (((uint64_t)(a >> 31) << 63) | + 0x7FF8000000000000ull | + ((uint64_t)(a & 0x7FFFFF) << 29)))) { + creport(j, "f32_to_f64 (NaN payload)", a, have, want); + } + break; + } + case 2: { // f32_from_u32: every u32. + uint32_t a = (uint32_t)i; + uint32_t have = wuffs_softfloat__arith__f32_from_u32(&g_sf, a); + uint32_t want = u_of_f((float)a); + if (have != want) { + creport(j, "f32_from_u32", a, have, want); + } + break; + } + case 3: { // f32_from_i32: every i32. + int32_t a = (int32_t)(uint32_t)i; + uint32_t have = wuffs_softfloat__arith__f32_from_i32(&g_sf, a); + uint32_t want = u_of_f((float)a); + if (have != want) { + creport(j, "f32_from_i32", (uint32_t)a, have, want); + } + break; + } + case 4: { // f32_from_u64. + uint64_t a = u64_operand(i); + uint32_t have = wuffs_softfloat__arith__f32_from_u64(&g_sf, a); + uint32_t want = u_of_f((float)a); + if (have != want) { + creport(j, "f32_from_u64", a, have, want); + } + break; + } + } + } + return NULL; +} + +static const struct { + const char* name; + uint64_t n; +} g_convs[] = { + {"f64_to_f32", ((uint64_t)326) << 20}, + {"f32_to_f64", ((uint64_t)1) << 32}, + {"f32_from_u32", ((uint64_t)1) << 32}, + {"f32_from_i32", ((uint64_t)1) << 32}, + {"f32_from_u64", ((uint64_t)64) << 26}, +}; + +static uint64_t // +check_compare32() { + static const uint32_t exps[] = {0, 1, 2, 126, 127, 128, 253, 254, 255}; + static const uint32_t mans[] = {0x000000, 0x000001, 0x000002, 0x000003, + 0x3FFFFF, 0x400000, 0x400001, 0x7FFFFE, + 0x7FFFFF, 0x123456, 0x654321}; + static uint32_t v[2 * 9 * 11 * 3]; + uint32_t n = 0; + for (uint32_t s = 0; s < 2; s++) { + for (uint32_t e = 0; e < 9; e++) { + for (uint32_t m = 0; m < 11; m++) { + uint32_t x = (s << 31) | (exps[e] << 23) | mans[m]; + v[n++] = x; + v[n++] = x + 1; + v[n++] = (x & 0x7FFFFFFF) ? (x - 1) : x; + } + } + } + uint64_t mismatches = 0; + for (uint32_t i = 0; i < n; i++) { + for (uint32_t j = 0; j < n; j++) { + uint32_t a = v[i]; + uint32_t b = v[j]; + float x = f_of(a); + float y = f_of(b); + uint32_t c = ((x != x) || (y != y)) ? 3 : (x < y) ? 0 : (x == y) ? 1 : 2; + bool ok = (wuffs_softfloat__arith__f32_cmp(&g_sf, a, b) == c) && + (wuffs_softfloat__arith__f32_eq(&g_sf, a, b) == (x == y)) && + (wuffs_softfloat__arith__f32_ne(&g_sf, a, b) == (x != y)) && + (wuffs_softfloat__arith__f32_lt(&g_sf, a, b) == (x < y)) && + (wuffs_softfloat__arith__f32_le(&g_sf, a, b) == (x <= y)) && + (wuffs_softfloat__arith__f32_gt(&g_sf, a, b) == (x > y)) && + (wuffs_softfloat__arith__f32_ge(&g_sf, a, b) == (x >= y)) && + (wuffs_softfloat__arith__f32_min(&g_sf, a, b) == + ((x < y) ? a : b)) && + (wuffs_softfloat__arith__f32_max(&g_sf, a, b) == + ((x > y) ? a : b)); + if (!ok && (mismatches++ < 5)) { + printf("MISMATCH f32_compare(0x%08" PRIX32 ", 0x%08" PRIX32 ")\n", a, + b); + } + } + } + return mismatches; +} + +static uint64_t // +check_compare64() { + static const uint64_t exps[] = {0, 1, 2, 1022, 1023, 1024, 2045, 2046, 2047}; + static const uint64_t mans[] = { + 0x0000000000000, 0x0000000000001, 0x0000000000002, 0x0000000000003, + 0x7FFFFFFFFFFFF, 0x8000000000000, 0x8000000000001, 0xFFFFFFFFFFFFE, + 0xFFFFFFFFFFFFF, 0x0123456789ABC, 0xFEDCBA9876543}; + static uint64_t v[2 * 9 * 11 * 3]; + uint32_t n = 0; + for (uint64_t s = 0; s < 2; s++) { + for (uint32_t e = 0; e < 9; e++) { + for (uint32_t m = 0; m < 11; m++) { + uint64_t x = (s << 63) | (exps[e] << 52) | mans[m]; + v[n++] = x; + v[n++] = x + 1; + v[n++] = (x & 0x7FFFFFFFFFFFFFFFull) ? (x - 1) : x; + } + } + } + uint64_t mismatches = 0; + for (uint32_t i = 0; i < n; i++) { + for (uint32_t j = 0; j < n; j++) { + uint64_t a = v[i]; + uint64_t b = v[j]; + double x = d_of(a); + double y = d_of(b); + uint32_t c = ((x != x) || (y != y)) ? 3 : (x < y) ? 0 : (x == y) ? 1 : 2; + bool ok = (wuffs_softfloat__arith__f64_cmp(&g_sf, a, b) == c) && + (wuffs_softfloat__arith__f64_eq(&g_sf, a, b) == (x == y)) && + (wuffs_softfloat__arith__f64_ne(&g_sf, a, b) == (x != y)) && + (wuffs_softfloat__arith__f64_lt(&g_sf, a, b) == (x < y)) && + (wuffs_softfloat__arith__f64_le(&g_sf, a, b) == (x <= y)) && + (wuffs_softfloat__arith__f64_gt(&g_sf, a, b) == (x > y)) && + (wuffs_softfloat__arith__f64_ge(&g_sf, a, b) == (x >= y)) && + (wuffs_softfloat__arith__f64_min(&g_sf, a, b) == + ((x < y) ? a : b)) && + (wuffs_softfloat__arith__f64_max(&g_sf, a, b) == + ((x > y) ? a : b)); + if (!ok && (mismatches++ < 5)) { + printf("MISMATCH f64_compare(0x%016" PRIX64 ", 0x%016" PRIX64 ")\n", a, + b); + } + } + } + return mismatches; +} + +static double // +now() { + struct timespec t; + clock_gettime(CLOCK_MONOTONIC, &t); + return (double)t.tv_sec + (1e-9 * (double)t.tv_nsec); +} + +int // +main(int argc, char** argv) { + const char* only = (argc > 1) ? argv[1] : NULL; + wuffs_base__status status = wuffs_softfloat__arith__initialize( + &g_sf, sizeof g_sf, WUFFS_VERSION, 0); + if (status.repr) { + printf("initialize: %s\n", status.repr); + return 1; + } + init_mantissas(); + init_mantissas64(); + uint64_t total_mismatches = 0; + double t_start = now(); + + for (int w = 0; w < (int)(sizeof(g_binops) / sizeof(g_binops[0])); w++) { + if (only && strcmp(only, g_binops[w].name)) { + continue; + } + double t0 = now(); + job jobs[NUM_THREADS]; + pthread_t threads[NUM_THREADS]; + for (int t = 0; t < NUM_THREADS; t++) { + jobs[t] = (job){w, (uint32_t)(t * 256 / NUM_THREADS), + (uint32_t)((t + 1) * 256 / NUM_THREADS), 0, 0}; + pthread_create(&threads[t], NULL, run_binop, &jobs[t]); + } + uint64_t count = 0, mismatches = 0; + for (int t = 0; t < NUM_THREADS; t++) { + pthread_join(threads[t], NULL); + count += jobs[t].count; + mismatches += jobs[t].mismatches; + } + total_mismatches += mismatches; + printf("%-14s %12" PRIu64 " operand pairs, %8" PRIu64 + " mismatches, %6.1f seconds\n", + g_binops[w].name, count, mismatches, now() - t0); + fflush(stdout); + } + + for (int w = 0; w < (int)(sizeof(g_binops64) / sizeof(g_binops64[0])); + w++) { + if (only && strcmp(only, g_binops64[w].name)) { + continue; + } + double t0 = now(); + job jobs[NUM_THREADS]; + pthread_t threads[NUM_THREADS]; + for (int t = 0; t < NUM_THREADS; t++) { + jobs[t] = (job){w, (uint32_t)(t * 2048 / NUM_THREADS), + (uint32_t)((t + 1) * 2048 / NUM_THREADS), 0, 0}; + pthread_create(&threads[t], NULL, run_binop64, &jobs[t]); + } + uint64_t count = 0, mismatches = 0; + for (int t = 0; t < NUM_THREADS; t++) { + pthread_join(threads[t], NULL); + count += jobs[t].count; + mismatches += jobs[t].mismatches; + } + total_mismatches += mismatches; + printf("%-14s %12" PRIu64 " operand pairs, %8" PRIu64 + " mismatches, %6.1f seconds\n", + g_binops64[w].name, count, mismatches, now() - t0); + fflush(stdout); + } + + for (int w = 0; w < (int)(sizeof(g_convs) / sizeof(g_convs[0])); w++) { + if (only && strcmp(only, g_convs[w].name)) { + continue; + } + double t0 = now(); + cjob jobs[NUM_THREADS]; + pthread_t threads[NUM_THREADS]; + uint64_t n = g_convs[w].n; + for (int t = 0; t < NUM_THREADS; t++) { + jobs[t] = (cjob){w, (n / NUM_THREADS) * (uint64_t)t, + (t == NUM_THREADS - 1) ? n + : (n / NUM_THREADS) * (t + 1), + 0, 0}; + pthread_create(&threads[t], NULL, run_conv, &jobs[t]); + } + uint64_t count = 0, mismatches = 0; + for (int t = 0; t < NUM_THREADS; t++) { + pthread_join(threads[t], NULL); + count += jobs[t].count; + mismatches += jobs[t].mismatches; + } + total_mismatches += mismatches; + printf("%-14s %12" PRIu64 " operands, %8" PRIu64 + " mismatches, %6.1f seconds\n", + g_convs[w].name, count, mismatches, now() - t0); + fflush(stdout); + } + + for (int w = 0; w < 2; w++) { + const char* name = w ? "f64_compare" : "f32_compare"; + if (only && strcmp(only, name)) { + continue; + } + double t0 = now(); + uint64_t mismatches = w ? check_compare64() : check_compare32(); + total_mismatches += mismatches; + printf("%-14s %12" PRIu64 " operand pairs, %8" PRIu64 + " mismatches, %6.1f seconds\n", + name, (uint64_t)(2 * 9 * 11 * 3) * (2 * 9 * 11 * 3), mismatches, + now() - t0); + fflush(stdout); + } + + printf("%s: %" PRIu64 " mismatches in total (%.0f seconds)\n", + total_mismatches ? "FAIL" : "PASS", total_mismatches, + now() - t_start); + return total_mismatches ? 1 : 0; +} diff --git a/std/brotli/README.md b/std/brotli/README.md index 115c6318..56b50b32 100644 --- a/std/brotli/README.md +++ b/std/brotli/README.md @@ -55,8 +55,9 @@ Without the large-window quirk and without attached dictionaries (see the window size plus some slack. The slack mirrors the C code's `kRingBufferWriteAheadSlack`, which its fast paths use to write (and discard) a few bytes past the ring buffer's end. This -implementation does not yet have such fast paths, but it still reserves the -slack, so that adding them later will not change the `workbuf_len` API. A +implementation's fast paths (see below) do not write past the ring buffer's +end, but it still reserves the slack, so that adding such fast paths later +will not change the `workbuf_len` API. A workbuf shorter than the stream's window size plus 542 bytes is rejected with `base."#bad workbuf length"`. @@ -158,12 +159,25 @@ invariants for memory safety but Wuffs requires explicit bounds checks. - The bit reader reads input bytes only when it needs them (like the C code's "safe" `BrotliSafeReadBits` and `SafeReadSymbol` functions), so that - `transform_io` can suspend at any point. There is no "fast path" (like the C - code's `ProcessCommands`, as opposed to `SafeProcessCommands`). Both paths - produce the same output and errors. -- LZ77 copies and dictionary words are written to the ring buffer one byte at - a time, with ring buffer events (the C code's `WriteRingBuffer` and - `WrapRingBuffer` calls) at the same positions as the C code. + `transform_io` can suspend at any point. Between reads, fewer than 8 bits + are buffered. The fast path, `process_commands_fast` (like the C code's + `ProcessCommands`, as opposed to `SafeProcessCommands`), is not a coroutine. + It reads ahead (up to 8 bytes at a time) into a 64-bit bit buffer while the + input has enough bytes, and when it returns to `process_commands` (at a + block switch, a ring buffer event, a dictionary reference or when the input + runs low), it gives back the whole bytes left in that buffer, which restores + the "fewer than 8 bits" state exactly. `process_commands` tracks where it is + within a command (`cmd_state`), so that either can pick up where the other + left off. Within `process_commands`, `read_symbol_fast` and `read_bits_fast` + (which are not coroutines either) read the same bytes at the same times as + the coroutines `read_symbol` and `read_bits`, which finish the job when the + input runs out. All paths produce the same output and errors. +- Dictionary words, and LZ77 copies that wrap around the physical ring buffer + or reach the C code's (virtual) ring buffer size, are written to the ring + buffer one byte at a time, with ring buffer events (the C code's + `WriteRingBuffer` and `WrapRingBuffer` calls) at the same positions as the + C code. Other LZ77 copies are written in chunks of at most the copy distance + (so that each chunk's source bytes are final), which is equivalent. - On error, the Wuffs decoder may have already written more output to `dst` than libbrotlidec would have (libbrotlidec does not flush on error), but what both have written agrees. @@ -173,7 +187,8 @@ invariants for memory safety but Wuffs requires explicit bounds checks. - Metadata callbacks (metadata blocks are currently skipped). See the "Metadata Blocks" section below. -- Fast paths. +- More fast paths, for example for dictionary words, for copies that wrap + around the ring buffer and for block switches. ## Encoder @@ -969,7 +984,12 @@ uses it at qualities 10 and 11 (and its smaller helpers at lower qualities). Its fixed-size state (about 0.8 MiB: FindBlocks' cost arrays and a 64Ki-entry FastLog2 cache in its `bit_cost`) lives in the struct; everything that the C -code allocates on the heap lives in a caller-supplied workbuf. +code allocates on the heap lives in a caller-supplied workbuf. The FastLog2 +cache (like the encoder's own `bit_cost` one) is filled lazily, one entry at +a time, when `fast_log2_memo` first needs that entry (a bitmap, zeroed when the +struct is initialized, records which entries are valid): typically only a few +thousand of the 65536 entries are ever used, and each one costs a (slow, exact) +software `log2`. Data formats (all little-endian, matching the C structs' memory layout on a 64-bit little-endian machine, so that results can be compared with `memcmp`): diff --git a/std/brotli/decode_brotli.wuffs b/std/brotli/decode_brotli.wuffs index eccf6bd2..495e7c55 100644 --- a/std/brotli/decode_brotli.wuffs +++ b/std/brotli/decode_brotli.wuffs @@ -230,6 +230,13 @@ pub struct decoder? implements base.io_transformer( insert_length : base.u32, copy_length : base.u32, + // cmd_state is where process_commands (and process_commands_fast) + // are within a command: 0 at its start, 1 when reading its literals + // (insert_length is how many are left), 2 when reading its distance + // and 3 when applying its copy. process_commands_fast sets it to 4 + // when the metablock is done. + cmd_state : base.u32[..= 4], + // quirk_large_window is the QUIRK_LARGE_WINDOW quirk (the C code's // BROTLI_DECODER_PARAM_LARGE_WINDOW parameter). large_window is // whether the stream is a large-window stream (the C code's @@ -679,6 +686,93 @@ pri func decoder.read_symbol?(src: base.io_reader, hoff: base.u32) { this.n_bits = n_bits } +// read_bits_fast is like read_bits but is not a coroutine. It reads input +// bytes exactly when read_bits would, for as long as args.src has them. It +// returns 0 when it has set this.bits_value, or 1 when read_bits must finish +// the job (this.bits and this.n_bits hold the bits read so far). +pri func decoder.read_bits_fast!(src: base.io_reader, n: base.u32[..= 24]) base.u32 { + var n : base.u32[..= 24] + var bits : base.u32 + var n_bits : base.u32 + + n = args.n + bits = this.bits + n_bits = this.n_bits + while n_bits < n, + post n_bits >= n, + { + if args.src.length() <= 0 { + this.bits = bits + this.n_bits = n_bits + return 1 + } + assert n_bits < 24 via "a < b: a < c; c <= b"(c: n) + bits |= args.src.peek_u8_as_u32() << n_bits + args.src.skip_u32_fast!(actual: 1, worst_case: 1) + n_bits += 8 + } + this.bits_value = bits & (((1 as base.u32) << n) - 1) + this.bits = bits >> n + this.n_bits = n_bits - n + return 0 +} + +// read_symbol_fast is like read_symbol but is not a coroutine. It reads +// input bytes exactly when read_symbol would, for as long as args.src has +// them. It returns 0 when it has set this.symbol, or 1 when read_symbol must +// finish the job (this.bits and this.n_bits hold the bits read so far). In +// particular, read_symbol (not read_symbol_fast) returns any error. +pri func decoder.read_symbol_fast!(src: base.io_reader, hoff: base.u32) base.u32 { + var hoff : base.u32 + var bits : base.u32 + var n_bits : base.u32 + var i : base.u32 + var e : base.u32 + var e_bits : base.u32[..= 23] + + hoff = args.hoff + bits = this.bits + n_bits = this.n_bits + while true { + i = hoff ~mod+ (bits & 0xFF) + if i >= HUFFS_LEN { + break + } + e = this.huffs[i] as base.u32 + e_bits = e & 15 + if e_bits <= 8 { + if n_bits >= e_bits { + this.symbol = e >> 4 + this.bits = bits >> e_bits + this.n_bits = n_bits - e_bits + return 0 + } + } else if n_bits > 8 { + i = (hoff ~mod+ (e >> 4)) ~mod+ ((bits >> 8) & (((1 as base.u32) << (e_bits - 8)) - 1)) + if i >= HUFFS_LEN { + break + } + e = this.huffs[i] as base.u32 + e_bits = (e & 15) + 8 + if n_bits >= e_bits { + this.symbol = e >> 4 + this.bits = bits >> e_bits + this.n_bits = n_bits - e_bits + return 0 + } + } + if (n_bits >= 24) or (args.src.length() <= 0) { + break + } + bits |= args.src.peek_u8_as_u32() << n_bits + args.src.skip_u32_fast!(actual: 1, worst_case: 1) + n_bits += 8 + } + this.bits = bits + this.n_bits = n_bits + return 1 +} + // jump_to_byte_boundary drops the bits up to the next byte boundary and // returns whether they were all zero. Mirrors BrotliJumpToByteBoundary in // bit_reader.h. diff --git a/std/brotli/decode_metablock.wuffs b/std/brotli/decode_metablock.wuffs index f3d29d85..ae301fec 100644 --- a/std/brotli/decode_metablock.wuffs +++ b/std/brotli/decode_metablock.wuffs @@ -418,7 +418,6 @@ pri func decoder.calculate_distance_lut!() { // Mirrors ProcessCommandsInternal in decode.c. pri func decoder.process_commands?(src: base.io_reader, workbuf: slice base.u8, dictbuf: roslice base.u8) { var cmd_code : base.u32[..= 703] - var insert_length : base.u32 var p1 : base.u32[..= 0xFF] var p2 : base.u32[..= 0xFF] var mode_offset : base.u32[..= 0x600] @@ -434,37 +433,72 @@ pri func decoder.process_commands?(src: base.io_reader, workbuf: slice base.u8, var address : base.u32 var chunk : base.u32[..= 14] var j : base.u64 + var fr : base.u32 + var dst_i : base.u64 + var src_i : base.u64 + var k64 : base.u64 + var n64 : base.u64 + var dlo : base.u64 + var dhi : base.u64 + var slo : base.u64 + var shi : base.u64 + var fs : base.status while true { - // CommandBegin. - if this.block_length[1] == 0 { - // Mirrors DecodeCommandBlockSwitch. - this.decode_block_switch?(src: args.src, t: 1) - this.htree_command = this.block_type_rb[3] & 0xFF - continue + if (this.cmd_state == 0) and (args.src.length() >= 16) { + fs = this.process_commands_fast!(src: args.src, workbuf: args.workbuf) + if not fs.is_ok() { + return fs + } else if this.cmd_state == 4 { + this.cmd_state = 0 + return ok + } } - // Mirrors ReadCommandInternal. - this.read_symbol?(src: args.src, - hoff: CMD_HUFFS_BASE + (this.htree_command * CMD_HUFFS_STRIDE)) - if this.symbol >= 704 { - return "#internal error: inconsistent Huffman table" + if this.cmd_state == 0 { + // CommandBegin. + if this.block_length[1] == 0 { + // Mirrors DecodeCommandBlockSwitch. + this.decode_block_switch?(src: args.src, t: 1) + this.htree_command = this.block_type_rb[3] & 0xFF + continue + } + + // Mirrors ReadCommandInternal. + fr = this.read_symbol_fast!(src: args.src, hoff: CMD_HUFFS_BASE + (this.htree_command * CMD_HUFFS_STRIDE)) + if fr <> 0 { + this.read_symbol?(src: args.src, + hoff: CMD_HUFFS_BASE + (this.htree_command * CMD_HUFFS_STRIDE)) + } + if this.symbol >= 704 { + return "#internal error: inconsistent Huffman table" + } + cmd_code = this.symbol.min(no_more_than: 703) + this.implicit_zero_distance = CMD_IMPLICIT_ZERO_DISTANCE[cmd_code] <> 0 + this.distance_context = CMD_DISTANCE_CONTEXT[cmd_code] as base.u32 + this.update_dist_htree_index!() + fr = this.read_bits_fast!(src: args.src, n: CMD_INSERT_LEN_EXTRA_BITS[cmd_code] as base.u32) + if fr <> 0 { + this.read_bits?(src: args.src, n: CMD_INSERT_LEN_EXTRA_BITS[cmd_code] as base.u32) + } + this.insert_length = CMD_INSERT_LEN_OFFSET[cmd_code] ~mod+ this.bits_value + fr = this.read_bits_fast!(src: args.src, n: CMD_COPY_LEN_EXTRA_BITS[cmd_code] as base.u32) + if fr <> 0 { + this.read_bits?(src: args.src, n: CMD_COPY_LEN_EXTRA_BITS[cmd_code] as base.u32) + } + this.copy_length = CMD_COPY_LEN_OFFSET[cmd_code] ~mod+ this.bits_value + this.block_length[1] ~mod-= 1 + if this.insert_length > 0 { + this.mb_remaining ~mod-= this.insert_length + this.cmd_state = 1 + } else { + this.cmd_state = 2 + } } - cmd_code = this.symbol.min(no_more_than: 703) - this.implicit_zero_distance = CMD_IMPLICIT_ZERO_DISTANCE[cmd_code] <> 0 - this.distance_context = CMD_DISTANCE_CONTEXT[cmd_code] as base.u32 - this.update_dist_htree_index!() - this.read_bits?(src: args.src, n: CMD_INSERT_LEN_EXTRA_BITS[cmd_code] as base.u32) - insert_length = CMD_INSERT_LEN_OFFSET[cmd_code] ~mod+ this.bits_value - this.read_bits?(src: args.src, n: CMD_COPY_LEN_EXTRA_BITS[cmd_code] as base.u32) - this.copy_length = CMD_COPY_LEN_OFFSET[cmd_code] ~mod+ this.bits_value - this.block_length[1] ~mod-= 1 - - if insert_length > 0 { - this.mb_remaining ~mod-= insert_length + if this.cmd_state == 1 { // CommandInner. Read the literals in the command. - while insert_length > 0 { + while this.insert_length > 0 { if this.block_length[0] == 0 { // Mirrors DecodeLiteralBlockSwitch. this.decode_block_switch?(src: args.src, t: 0) @@ -478,39 +512,48 @@ pri func decoder.process_commands?(src: base.io_reader, workbuf: slice base.u8, ctx = (CONTEXT_LOOKUP[mode_offset | p1] | CONTEXT_LOOKUP[mode_offset | 0x100 | p2]) as base.u32 tree = this.context_maps[this.context_map_slice + ctx] as base.u32 - this.read_symbol?(src: args.src, hoff: LIT_HUFFS_BASE + (tree * LIT_HUFFS_STRIDE)) + fr = this.read_symbol_fast!(src: args.src, hoff: LIT_HUFFS_BASE + (tree * LIT_HUFFS_STRIDE)) + if fr <> 0 { + this.read_symbol?(src: args.src, hoff: LIT_HUFFS_BASE + (tree * LIT_HUFFS_STRIDE)) + } this.put_byte!(workbuf: args.workbuf, b: (this.symbol & 0xFF) as base.u8) this.block_length[0] ~mod-= 1 - insert_length -= 1 + this.insert_length ~mod-= 1 if this.pos == this.rb_size { this.ring_event?() } } + this.cmd_state = 2 if (this.mb_remaining == 0) or (this.mb_remaining >= 0x8000_0000) { // BROTLI_STATE_METABLOCK_DONE. + this.cmd_state = 0 return ok } } - // CommandPostDecodeLiterals. - if this.implicit_zero_distance { - // Implicit distance case. - this.distance_context = 1 - this.dist_rb_idx ~mod-= 1 - this.distance_code = this.dist_rb[this.dist_rb_idx & 3] - } else { - // Read distance code in the command, unless it was implicitly - // zero. - if this.block_length[2] == 0 { - // Mirrors DecodeDistanceBlockSwitch. - this.decode_block_switch?(src: args.src, t: 2) - this.dist_context_map_slice = (this.block_type_rb[5] & 0xFF) << 2 - this.update_dist_htree_index!() + if this.cmd_state == 2 { + // CommandPostDecodeLiterals. + if this.implicit_zero_distance { + // Implicit distance case. + this.distance_context = 1 + this.dist_rb_idx ~mod-= 1 + this.distance_code = this.dist_rb[this.dist_rb_idx & 3] + } else { + // Read distance code in the command, unless it was implicitly + // zero. + if this.block_length[2] == 0 { + // Mirrors DecodeDistanceBlockSwitch. + this.decode_block_switch?(src: args.src, t: 2) + this.dist_context_map_slice = (this.block_type_rb[5] & 0xFF) << 2 + this.update_dist_htree_index!() + } + this.read_distance?(src: args.src) } - this.read_distance?(src: args.src) + this.cmd_state = 3 } + // this.cmd_state == 3. if this.max_distance <> this.max_backward_distance { this.max_distance = this.pos.min(no_more_than: this.max_backward_distance) } @@ -604,6 +647,37 @@ pri func decoder.process_commands?(src: base.io_reader, workbuf: slice base.u8, this.dist_rb[this.dist_rb_idx & 3] = distance this.dist_rb_idx ~mod+= 1 this.mb_remaining ~mod-= i + // Fast path: when the copy neither wraps around the physical ring + // buffer (at either end) nor makes this.pos reach this.rb_size, + // copy the bytes in chunks of at most distance bytes. Each chunk's + // source bytes are final (and do not overlap the chunk's + // destination), so this matches the byte by byte loop below. + dst_i = this.pos as base.u64 + src_i = ((this.pos ~mod- distance) & (this.window_size ~mod- 1)) as base.u64 + if ((dst_i + (i as base.u64)) < (this.rb_size as base.u64)) and + ((src_i + (i as base.u64)) <= (this.window_size as base.u64)) and + ((this.window_size as base.u64) <= args.workbuf.length()) and + (distance > 0) { + k64 = 0 + while k64 < (i as base.u64) { + n64 = (i as base.u64) ~mod- k64 + n64 = n64.min(no_more_than: distance as base.u64) + dlo = dst_i ~mod+ k64 + dhi = dlo ~mod+ n64 + slo = src_i ~mod+ k64 + shi = slo ~mod+ n64 + if (n64 == 0) or (dlo > dhi) or (dhi > args.workbuf.length()) or + (slo > shi) or (shi > args.workbuf.length()) { + break + } + args.workbuf[dlo .. dhi].copy_from_slice!(s: args.workbuf[slo .. shi]) + k64 ~mod+= n64 + } + // k64 == i, unless the (unreachable) break above was taken, + // in which case the loop below copies the remaining bytes. + this.pos ~mod+= (k64 & 0xFFFF_FFFF) as base.u32 + i ~mod-= (k64 & 0xFFFF_FFFF) as base.u32 + } while i > 0 { b = this.get_byte(workbuf: args.workbuf, distance: distance) this.put_byte!(workbuf: args.workbuf, b: b) @@ -614,6 +688,7 @@ pri func decoder.process_commands?(src: base.io_reader, workbuf: slice base.u8, } } + this.cmd_state = 0 if (this.mb_remaining == 0) or (this.mb_remaining >= 0x8000_0000) { // BROTLI_STATE_METABLOCK_DONE. return ok @@ -621,6 +696,323 @@ pri func decoder.process_commands?(src: base.io_reader, workbuf: slice base.u8, } } +// process_commands_fast is a fast path for process_commands. It is not a +// coroutine. It handles commands, or parts of commands (see +// decoder.cmd_state), for as long as args.src holds enough bytes and the work +// needs no block switch, ring buffer event or dictionary reference. Then it +// returns, and process_commands picks up where it left off. +// +// It reads ahead, up to 8 bytes at a time, into a 64-bit bit buffer. When it +// returns, it gives back (undoes) the whole bytes left in that buffer. The +// coroutines read_symbol and read_bits read an input byte only when they +// need more bits, so that between reads, fewer than 8 bits are buffered. +// Giving back the whole bytes restores that state exactly, so this function +// reads the same bits and leaves the decoder in the same state as +// process_commands would, and it writes the same bytes and returns the same +// errors. It sets this.cmd_state to 4 when the metablock is done. +pri func decoder.process_commands_fast!(src: base.io_reader, workbuf: slice base.u8) base.status { + var ret : base.status + var bits : base.u64 + var n_bits : base.u32[..= 63] + var i : base.u32 + var e : base.u32 + var e_bits : base.u32[..= 23] + var cmd_code : base.u32[..= 703] + var nb : base.u32[..= 24] + var p1 : base.u32[..= 0xFF] + var p2 : base.u32[..= 0xFF] + var mode_offset : base.u32[..= 0x600] + var ctx : base.u32[..= 63] + var tree : base.u32[..= 0xFF] + var code : base.u32 + var nbits : base.u32[..= 32] + var lo : base.u32 + var max_distance : base.u32 + var distance : base.u32 + var n : base.u32 + var dst_i : base.u64 + var src_i : base.u64 + var k64 : base.u64 + var n64 : base.u64 + var dlo : base.u64 + var dhi : base.u64 + var slo : base.u64 + var shi : base.u64 + + if this.n_bits >= 8 { + return ok + } + n_bits = this.n_bits & 7 + bits = (this.bits as base.u64) & (((1 as base.u64) << n_bits) - 1) + + while.outer true { + if this.cmd_state == 0 { + // Two refills, of at most 8 bytes each. + if (this.block_length[1] == 0) or (args.src.length() < 16) { + break.outer + } + bits |= args.src.peek_u64le() ~mod<< n_bits + args.src.skip_u32_fast!(actual: (63 - n_bits) >> 3, worst_case: 8) + n_bits |= 56 + // Mirrors ReadCommandInternal. + i = (CMD_HUFFS_BASE + (this.htree_command * CMD_HUFFS_STRIDE)) ~mod+ ((bits & 0xFF) as base.u32) + if i >= HUFFS_LEN { + ret = "#internal error: inconsistent Huffman table" + break.outer + } + e = this.huffs[i] as base.u32 + e_bits = e & 15 + if e_bits > 8 { + i = ((CMD_HUFFS_BASE + (this.htree_command * CMD_HUFFS_STRIDE)) ~mod+ (e >> 4)) ~mod+ ((((bits >> 8) & 0x7FFF) as base.u32) & (((1 as base.u32) << (e_bits - 8)) - 1)) + if i >= HUFFS_LEN { + ret = "#internal error: inconsistent Huffman table" + break.outer + } + e = this.huffs[i] as base.u32 + e_bits = (e & 15) + 8 + } + if n_bits < e_bits { + ret = "#internal error: inconsistent bit reader" + break.outer + } + bits >>= e_bits + n_bits -= e_bits + if (e >> 4) >= 704 { + ret = "#internal error: inconsistent Huffman table" + break.outer + } + this.symbol = e >> 4 + cmd_code = this.symbol.min(no_more_than: 703) + this.implicit_zero_distance = CMD_IMPLICIT_ZERO_DISTANCE[cmd_code] <> 0 + this.distance_context = CMD_DISTANCE_CONTEXT[cmd_code] as base.u32 + this.update_dist_htree_index!() + nb = CMD_INSERT_LEN_EXTRA_BITS[cmd_code] as base.u32 + if n_bits < nb { + ret = "#internal error: inconsistent bit reader" + break.outer + } + this.bits_value = ((bits & (((1 as base.u64) << nb) - 1)) & 0xFFFF_FFFF) as base.u32 + bits >>= nb + n_bits -= nb + this.insert_length = CMD_INSERT_LEN_OFFSET[cmd_code] ~mod+ this.bits_value + if args.src.length() < 8 { + ret = "#internal error: inconsistent bit reader" + break.outer + } + bits |= args.src.peek_u64le() ~mod<< n_bits + args.src.skip_u32_fast!(actual: (63 - n_bits) >> 3, worst_case: 8) + n_bits |= 56 + nb = CMD_COPY_LEN_EXTRA_BITS[cmd_code] as base.u32 + if n_bits < nb { + ret = "#internal error: inconsistent bit reader" + break.outer + } + this.bits_value = ((bits & (((1 as base.u64) << nb) - 1)) & 0xFFFF_FFFF) as base.u32 + bits >>= nb + n_bits -= nb + this.copy_length = CMD_COPY_LEN_OFFSET[cmd_code] ~mod+ this.bits_value + this.block_length[1] ~mod-= 1 + if this.insert_length > 0 { + this.mb_remaining ~mod-= this.insert_length + this.cmd_state = 1 + } else { + this.cmd_state = 2 + } + } + + if this.cmd_state == 1 { + // CommandInner. Read the literals in the command. p1 and p2 are + // the last two bytes written to the ring buffer (put_byte always + // writes when the physical ring buffer fits in args.workbuf). + if (this.window_size as base.u64) > args.workbuf.length() { + break.outer + } + p1 = this.get_byte(workbuf: args.workbuf, distance: 1) as base.u32 + p2 = this.get_byte(workbuf: args.workbuf, distance: 2) as base.u32 + while this.insert_length > 0 { + if (this.block_length[0] == 0) or ((this.pos ~mod+ 1) >= this.rb_size) { + break.outer + } + if n_bits < 24 { + if args.src.length() < 8 { + break.outer + } + bits |= args.src.peek_u64le() ~mod<< n_bits + args.src.skip_u32_fast!(actual: (63 - n_bits) >> 3, worst_case: 8) + n_bits |= 56 + } + mode_offset = this.context_mode << 9 + ctx = (CONTEXT_LOOKUP[mode_offset | p1] | + CONTEXT_LOOKUP[mode_offset | 0x100 | p2]) as base.u32 + tree = this.context_maps[this.context_map_slice + ctx] as base.u32 + i = (LIT_HUFFS_BASE + (tree * LIT_HUFFS_STRIDE)) ~mod+ ((bits & 0xFF) as base.u32) + if i >= HUFFS_LEN { + ret = "#internal error: inconsistent Huffman table" + break.outer + } + e = this.huffs[i] as base.u32 + e_bits = e & 15 + if e_bits > 8 { + i = ((LIT_HUFFS_BASE + (tree * LIT_HUFFS_STRIDE)) ~mod+ (e >> 4)) ~mod+ ((((bits >> 8) & 0x7FFF) as base.u32) & (((1 as base.u32) << (e_bits - 8)) - 1)) + if i >= HUFFS_LEN { + ret = "#internal error: inconsistent Huffman table" + break.outer + } + e = this.huffs[i] as base.u32 + e_bits = (e & 15) + 8 + } + if n_bits < e_bits { + ret = "#internal error: inconsistent bit reader" + break.outer + } + bits >>= e_bits + n_bits -= e_bits + this.symbol = e >> 4 + this.put_byte!(workbuf: args.workbuf, b: (this.symbol & 0xFF) as base.u8) + p2 = p1 + p1 = this.symbol & 0xFF + this.block_length[0] ~mod-= 1 + this.insert_length ~mod-= 1 + } + this.cmd_state = 2 + if (this.mb_remaining == 0) or (this.mb_remaining >= 0x8000_0000) { + this.cmd_state = 4 + break.outer + } + } + + if this.cmd_state == 2 { + // CommandPostDecodeLiterals. + if this.implicit_zero_distance { + this.distance_context = 1 + this.dist_rb_idx ~mod-= 1 + this.distance_code = this.dist_rb[this.dist_rb_idx & 3] + } else { + // One refill: 56 bits suffice for a symbol and 32 extra bits. + if (this.block_length[2] == 0) or (args.src.length() < 8) { + break.outer + } + bits |= args.src.peek_u64le() ~mod<< n_bits + args.src.skip_u32_fast!(actual: (63 - n_bits) >> 3, worst_case: 8) + n_bits |= 56 + // Mirrors read_distance. + i = (DIST_HUFFS_BASE + (this.dist_htree_index * DIST_HUFFS_STRIDE)) ~mod+ ((bits & 0xFF) as base.u32) + if i >= HUFFS_LEN { + ret = "#internal error: inconsistent Huffman table" + break.outer + } + e = this.huffs[i] as base.u32 + e_bits = e & 15 + if e_bits > 8 { + i = ((DIST_HUFFS_BASE + (this.dist_htree_index * DIST_HUFFS_STRIDE)) ~mod+ (e >> 4)) ~mod+ ((((bits >> 8) & 0x7FFF) as base.u32) & (((1 as base.u32) << (e_bits - 8)) - 1)) + if i >= HUFFS_LEN { + ret = "#internal error: inconsistent Huffman table" + break.outer + } + e = this.huffs[i] as base.u32 + e_bits = (e & 15) + 8 + } + if n_bits < e_bits { + ret = "#internal error: inconsistent bit reader" + break.outer + } + bits >>= e_bits + n_bits -= e_bits + this.symbol = e >> 4 + code = this.symbol + this.block_length[2] ~mod-= 1 + this.distance_context = 0 + if code < 16 { + this.take_distance_from_ring_buffer!(code: code) + } else { + if (code >= this.distance_alphabet_size) or (code >= 1128) { + ret = "#internal error: inconsistent Huffman table" + break.outer + } + nbits = this.dist_extra_bits[code].min(no_more_than: 32) as base.u32 + if n_bits < nbits { + ret = "#internal error: inconsistent bit reader" + break.outer + } + // Like read_bits32, this is (lo | (hi << 16)) when nbits + // is more than 24. + lo = ((bits & (((1 as base.u64) << nbits) - 1)) & 0xFFFF_FFFF) as base.u32 + this.bits_value = lo + bits >>= nbits + n_bits -= nbits + this.distance_code = (((this.dist_offset[code] as base.u64) ~mod+ + ((this.bits_value as base.u64) ~mod<< this.distance_postfix_bits)) & 0xFFFF_FFFF) as base.u32 + } + } + this.cmd_state = 3 + } + + // this.cmd_state == 3. Only handle LZ77 copies that need no ring + // buffer event and do not wrap around the physical ring buffer. + max_distance = this.max_distance + if max_distance <> this.max_backward_distance { + max_distance = this.pos.min(no_more_than: this.max_backward_distance) + } + distance = this.distance_code + n = this.copy_length + if (distance > max_distance) or (distance == 0) { + break.outer + } + dst_i = this.pos as base.u64 + src_i = ((this.pos ~mod- distance) & (this.window_size ~mod- 1)) as base.u64 + if ((dst_i + (n as base.u64)) >= (this.rb_size as base.u64)) or + ((src_i + (n as base.u64)) > (this.window_size as base.u64)) or + ((this.window_size as base.u64) > args.workbuf.length()) { + break.outer + } + this.max_distance = max_distance + this.dist_rb[this.dist_rb_idx & 3] = distance + this.dist_rb_idx ~mod+= 1 + this.mb_remaining ~mod-= n + // Copy in chunks of at most distance bytes, like process_commands. + k64 = 0 + while k64 < (n as base.u64) { + n64 = (n as base.u64) ~mod- k64 + n64 = n64.min(no_more_than: distance as base.u64) + dlo = dst_i ~mod+ k64 + dhi = dlo ~mod+ n64 + slo = src_i ~mod+ k64 + shi = slo ~mod+ n64 + if (n64 == 0) or (dlo > dhi) or (dhi > args.workbuf.length()) or + (slo > shi) or (shi > args.workbuf.length()) { + break + } + args.workbuf[dlo .. dhi].copy_from_slice!(s: args.workbuf[slo .. shi]) + k64 ~mod+= n64 + } + this.pos ~mod+= (k64 & 0xFFFF_FFFF) as base.u32 + n ~mod-= (k64 & 0xFFFF_FFFF) as base.u32 + // n == 0, unless the (unreachable) break above was taken. This.pos + // stays below this.rb_size, so there is no ring buffer event. + while n > 0 { + this.put_byte!(workbuf: args.workbuf, b: this.get_byte(workbuf: args.workbuf, distance: distance)) + n -= 1 + } + this.cmd_state = 0 + if (this.mb_remaining == 0) or (this.mb_remaining >= 0x8000_0000) { + this.cmd_state = 4 + break.outer + } + }.outer + + // Give back the whole bytes left in the bit buffer. + while n_bits >= 8 { + if not args.src.can_undo_byte() { + return "#internal error: inconsistent bit reader" + } + args.src.undo_byte!() + n_bits -= 8 + } + this.bits = ((bits & (((1 as base.u64) << n_bits) - 1)) & 0xFFFF_FFFF) as base.u32 + this.n_bits = n_bits + return ret +} + // update_dist_htree_index sets this.dist_htree_index from the distance // context map. Mirrors part of ReadCommandInternal and // DecodeDistanceBlockSwitch in decode.c. @@ -631,10 +1023,15 @@ pri func decoder.update_dist_htree_index!() { // Mirrors ReadDistanceInternal in decode.c. pri func decoder.read_distance?(src: base.io_reader) { - var code : base.u32 + var code : base.u32 + var fr : base.u32 + var nbits : base.u32[..= 32] - this.read_symbol?(src: args.src, - hoff: DIST_HUFFS_BASE + (this.dist_htree_index * DIST_HUFFS_STRIDE)) + fr = this.read_symbol_fast!(src: args.src, hoff: DIST_HUFFS_BASE + (this.dist_htree_index * DIST_HUFFS_STRIDE)) + if fr <> 0 { + this.read_symbol?(src: args.src, + hoff: DIST_HUFFS_BASE + (this.dist_htree_index * DIST_HUFFS_STRIDE)) + } code = this.symbol this.block_length[2] ~mod-= 1 // Convert the distance code to the actual distance by possibly looking up @@ -649,7 +1046,14 @@ pri func decoder.read_distance?(src: base.io_reader) { } // The C code uses BrotliSafeReadBits32, as large-window brotli can have // more than 24 extra bits. - this.read_bits32?(src: args.src, n: this.dist_extra_bits[code].min(no_more_than: 32) as base.u32) + nbits = this.dist_extra_bits[code].min(no_more_than: 32) as base.u32 + fr = 1 + if nbits <= 24 { + fr = this.read_bits_fast!(src: args.src, n: nbits) + } + if fr <> 0 { + this.read_bits32?(src: args.src, n: nbits) + } if code >= 1128 { return "#internal error: inconsistent Huffman table" } diff --git a/std/brotli/encode_bit_cost.wuffs b/std/brotli/encode_bit_cost.wuffs index f000465c..81b157d4 100644 --- a/std/brotli/encode_bit_cost.wuffs +++ b/std/brotli/encode_bit_cost.wuffs @@ -12,8 +12,8 @@ // This file mirrors bit_cost.c, bit_cost_inc.h and fast_log.h. The functions // are methods of the bit_cost struct (whose only state is the stateless -// softfloat arith), so that other structs (the encoder, the -// metablock_builder) can embed one with a single field. +// softfloat arith and a lazily filled FastLog2 cache), so that other structs +// (the encoder, the metablock_builder) can embed one with a single field. // // Doubles are IEEE 754 binary64 bit patterns (base.u64). A uint32_t array // (e.g. a histogram's data_) is a slice of little-endian bytes, because @@ -48,43 +48,23 @@ pri const F64_HUGE_VAL : base.u64 = 0x7FF0_0000_0000_0000 // F64_1E99 is the C code's 1e99. pri const F64_1E99 : base.u64 = 0x547D_42AE_A287_9F2E -// LOG2_CACHE_SIZE is the size of the bit_cost.log2_cache table. -pri const LOG2_CACHE_SIZE : base.u64 = 65536 - pri struct bit_cost?( - // log2_cache_ready is whether log2_cache has been filled (by - // init_log2_cache). - log2_cache_ready : base.bool, + // Bit (v & 63) of log2_valid[v >> 6] is whether log2_cache[v] holds + // FastLog2(v). This bitmap is zeroed when the struct is initialized + // (but log2_cache need not be), so the cache starts empty. + log2_valid : array[1024] base.u64, sf : softfloat.arith, ) + ( - // log2_cache[v] is FastLog2(v), for v < LOG2_CACHE_SIZE. It is only a - // speed optimization (glibc's log2 is expensive to emulate): the - // values are computed by the same functions, so the results are - // identical with or without it. + // log2_cache[v] is FastLog2(v), for 256 <= v < 65536, filled lazily + // (see fast_log2_memo). It is only a speed optimization (glibc's log2 + // is expensive to emulate): the values are computed by the same + // functions, so the results are identical with or without it. Few of + // the 65536 entries are typically needed, so filling it lazily is + // much cheaper than filling it all up front. log2_cache : array[65536] base.u64, ) -// init_log2_cache fills the log2_cache table (if it is not already filled). -// Calling it is optional, but it makes fast_log2 faster. -pri func bit_cost.init_log2_cache!() { - var v : base.u64 - - if this.log2_cache_ready { - return nothing - } - v = 0 - while v < 256 { - this.log2_cache[v] = LOG2_TABLE[v] - v += 1 - } - while v < 65536 { - this.log2_cache[v] = this.sf.log2_u64(v: v) - v += 1 - } - this.log2_cache_ready = true -} - // ld32 returns the u32le at s[i ..], or 0 if that is out of bounds (which // the callers never do). pri func bit_cost.ld32(s: roslice base.u8, i: base.u64) base.u32 { @@ -138,23 +118,41 @@ pri func bit_cost.st64!(s: slice base.u8, i: base.u64, v: base.u64) { } // Mirrors FastLog2 in fast_log.h. The result is the bit pattern of an IEEE -// 754 binary64 value. +// 754 binary64 value. It uses (but does not fill) log2_cache: see +// fast_log2_memo. pri func bit_cost.fast_log2(v: base.u64) base.u64 { if args.v < 256 { return LOG2_TABLE[args.v] - } else if this.log2_cache_ready and (args.v < 65536) { - return this.log2_cache[args.v] + } else if args.v < 65536 { + if ((this.log2_valid[args.v >> 6] >> (args.v & 63)) & 1) <> 0 { + return this.log2_cache[args.v] + } } return this.sf.log2_u64(v: args.v) } +// fast_log2_memo is like fast_log2 (it also mirrors FastLog2) but it fills +// the log2_cache entry for v (if v is in the cache's range). +pri func bit_cost.fast_log2_memo!(v: base.u64) base.u64 { + if args.v < 256 { + return LOG2_TABLE[args.v] + } else if args.v >= 65536 { + return this.sf.log2_u64(v: args.v) + } else if ((this.log2_valid[args.v >> 6] >> (args.v & 63)) & 1) == 0 { + this.log2_cache[args.v] = this.sf.log2_u64(v: args.v) + this.log2_valid[args.v >> 6] |= (1 as base.u64) << (args.v & 63) + } + return this.log2_cache[args.v] +} + // Mirrors BrotliBitsEntropy in bit_cost.c. population holds (at least) size // u32le values. The C code's "goto odd_number_of_elements_left" only changes // the loop's unrolling, not the order of the operations. -pri func bit_cost.bits_entropy(population: roslice base.u8, size: base.u64) base.u64 { +pri func bit_cost.bits_entropy!(population: roslice base.u8, size: base.u64) base.u64 { var sum : base.u64 var retval : base.u64 var p : base.u64 + var lp : base.u64 var q : roslice base.u8 var n : base.u64 @@ -168,12 +166,14 @@ pri func bit_cost.bits_entropy(population: roslice base.u8, size: base.u64) base q = q[4 ..] } sum ~mod+= p + lp = this.fast_log2_memo!(v: p) retval = this.sf.f64_sub(a: retval, b: this.sf.f64_mul( - a: this.sf.f64_from_u64(a: p), b: this.fast_log2(v: p))) + a: this.sf.f64_from_u64(a: p), b: lp)) } if sum <> 0 { + lp = this.fast_log2_memo!(v: sum) retval = this.sf.f64_add(a: retval, b: this.sf.f64_mul( - a: this.sf.f64_from_u64(a: sum), b: this.fast_log2(v: sum))) + a: this.sf.f64_from_u64(a: sum), b: lp)) } if this.sf.f64_lt(a: retval, b: this.sf.f64_from_u64(a: sum)) <> 0 { retval = this.sf.f64_from_u64(a: sum) @@ -184,7 +184,7 @@ pri func bit_cost.bits_entropy(population: roslice base.u8, size: base.u64) base // Mirrors BrotliPopulationCostLiteral, BrotliPopulationCostCommand and // BrotliPopulationCostDistance (FN(BrotliPopulationCost) in bit_cost_inc.h). // histogram is a histogram of data_size symbols (see the layout above). -pri func bit_cost.population_cost(histogram: roslice base.u8, data_size: base.u32[..= 704]) base.u64 { +pri func bit_cost.population_cost!(histogram: roslice base.u8, data_size: base.u32[..= 704]) base.u64 { var total : base.u64 var count : base.u32 var s : array[4] base.u32 @@ -203,6 +203,7 @@ pri func bit_cost.population_cost(histogram: roslice base.u8, data_size: base.u3 var depth_histo : array[18] base.u32 var log2total : base.u64 var log2p : base.u64 + var lp : base.u64 var depth : base.u64 var reps : base.u32 var sum : base.u64 @@ -284,12 +285,13 @@ pri func bit_cost.population_cost(histogram: roslice base.u8, data_size: base.u3 // Compute the entropy of the histogram and simultaneously build a // simplified histogram of the code length codes. max_depth = 1 - log2total = this.fast_log2(v: total) + log2total = this.fast_log2_memo!(v: total) i = 0 while i < args.data_size { c = this.ld32(s: args.histogram, i: (i as base.u64) * 4) if c > 0 { - log2p = this.sf.f64_sub(a: log2total, b: this.fast_log2(v: c as base.u64)) + lp = this.fast_log2_memo!(v: c as base.u64) + log2p = this.sf.f64_sub(a: log2total, b: lp) depth = this.sf.f64_to_u64_trunc(a: this.sf.f64_add(a: log2p, b: 0x3FE0_0000_0000_0000)) bits = this.sf.f64_add(a: bits, b: this.sf.f64_mul( a: this.sf.f64_from_u32(a: c), b: log2p)) @@ -340,13 +342,15 @@ pri func bit_cost.population_cost(histogram: roslice base.u8, data_size: base.u3 while i < 18 { p = depth_histo[i] as base.u64 sum ~mod+= p + lp = this.fast_log2_memo!(v: p) retval = this.sf.f64_sub(a: retval, b: this.sf.f64_mul( - a: this.sf.f64_from_u64(a: p), b: this.fast_log2(v: p))) + a: this.sf.f64_from_u64(a: p), b: lp)) i ~mod+= 1 } if sum <> 0 { + lp = this.fast_log2_memo!(v: sum) retval = this.sf.f64_add(a: retval, b: this.sf.f64_mul( - a: this.sf.f64_from_u64(a: sum), b: this.fast_log2(v: sum))) + a: this.sf.f64_from_u64(a: sum), b: lp)) } if this.sf.f64_lt(a: retval, b: this.sf.f64_from_u64(a: sum)) <> 0 { retval = this.sf.f64_from_u64(a: sum) diff --git a/std/brotli/encode_block_splitter.wuffs b/std/brotli/encode_block_splitter.wuffs index e95b44e2..b13173e7 100644 --- a/std/brotli/encode_block_splitter.wuffs +++ b/std/brotli/encode_block_splitter.wuffs @@ -31,12 +31,15 @@ pri const HISTOGRAMS_PER_BATCH : base.u64 = 64 pri const CLUSTERS_PER_BATCH : base.u64 = 16 // Mirrors BitCost in block_splitter.c. -pri func metablock_builder.bit_cost(count: base.u64) base.u64 { +pri func metablock_builder.bit_cost!(count: base.u64) base.u64 { + var c : base.u64 + if args.count == 0 { // -2.0 return 0xC000_0000_0000_0000 } - return this.bc.fast_log2(v: args.count) + c = this.bc.fast_log2_memo!(v: args.count) + return c } // Mirrors FN(HistogramAddVector) in histogram_inc.h, adding the n symbols at @@ -314,7 +317,7 @@ pri func metablock_builder.find_blocks!(wb: slice base.u8, data: base.u64, lengt i = 0 while i < args.num_histograms { // FastLog2((uint32_t)histograms[i].total_count_). - this.insert_cost[i & 255] = this.bc.fast_log2( + this.insert_cost[i & 255] = this.bc.fast_log2_memo!( v: this.h_total(wb: args.wb, h: args.histograms ~mod+ (i ~mod* hs), ds: args.ds) & 0xFFFF_FFFF) i ~mod+= 1 } @@ -326,9 +329,8 @@ pri func metablock_builder.find_blocks!(wb: slice base.u8, data: base.u64, lengt while j < args.num_histograms { k = (i ~mod* args.num_histograms) ~mod+ j if k < 35200 { - this.insert_cost[k] = this.sf.f64_sub( - a: this.insert_cost[j & 255], - b: this.bit_cost(count: this.ld32(wb: args.wb, i: (args.histograms ~mod+ (j ~mod* hs)) ~mod+ (i ~mod* 4)) as base.u64)) + c = this.bit_cost!(count: this.ld32(wb: args.wb, i: (args.histograms ~mod+ (j ~mod* hs)) ~mod+ (i ~mod* 4)) as base.u64) + this.insert_cost[k] = this.sf.f64_sub(a: this.insert_cost[j & 255], b: c) } j ~mod+= 1 } @@ -471,6 +473,7 @@ pri func metablock_builder.build_block_histograms!(wb: slice base.u8, data: base // this.split_etc[cat]. pri func metablock_builder.cluster_blocks!(wb: slice base.u8, cat: base.u32[..= 2], data: base.u64, length: base.u64, width: base.u32, num_blocks: base.u64, block_ids: base.u64, ds: base.u32[..= 704]) { + var c : base.u64 var hs : base.u64 var histogram_symbols : base.u64 var u32s : base.u64 @@ -562,8 +565,8 @@ pri func metablock_builder.cluster_blocks!(wb: slice base.u8, cat: base.u32[..= pos ~mod+= 1 k ~mod+= 1 } - this.h_set_bit_cost!(wb: args.wb, h: histograms ~mod+ (j ~mod* hs), ds: args.ds, - bit_cost: this.h_population_cost(wb: args.wb, h: histograms ~mod+ (j ~mod* hs), ds: args.ds)) + c = this.h_population_cost!(wb: args.wb, h: histograms ~mod+ (j ~mod* hs), ds: args.ds) + this.h_set_bit_cost!(wb: args.wb, h: histograms ~mod+ (j ~mod* hs), ds: args.ds, bit_cost: c) this.st32!(wb: args.wb, i: new_clusters ~mod+ (j ~mod* 4), v: (j & 0xFFFF_FFFF) as base.u32) this.st32!(wb: args.wb, i: symbols ~mod+ (j ~mod* 4), v: (j & 0xFFFF_FFFF) as base.u32) this.st32!(wb: args.wb, i: sizes ~mod+ (j ~mod* 4), v: 1) diff --git a/std/brotli/encode_cluster.wuffs b/std/brotli/encode_cluster.wuffs index b8a24408..805bd04b 100644 --- a/std/brotli/encode_cluster.wuffs +++ b/std/brotli/encode_cluster.wuffs @@ -108,7 +108,7 @@ pri func metablock_builder.compare_and_push_to_queue!(wb: slice base.u8, ds: bas } this.h_copy!(wb: args.wb, h: args.tmp, v: h1, ds: args.ds) this.h_add_histogram!(wb: args.wb, h: args.tmp, v: h2, ds: args.ds) - c = this.h_population_cost(wb: args.wb, h: args.tmp, ds: args.ds) + c = this.h_population_cost!(wb: args.wb, h: args.tmp, ds: args.ds) if this.sf.f64_lt(a: c, b: this.sf.f64_sub(a: threshold, b: cost_diff)) <> 0 { cost_combo = c is_good_pair = true @@ -271,14 +271,15 @@ pri func metablock_builder.histogram_combine!(wb: slice base.u8, ds: base.u32[.. // Mirrors FN(BrotliHistogramBitCostDistance) in cluster_inc.h. pri func metablock_builder.histogram_bit_cost_distance!(wb: slice base.u8, ds: base.u32[..= 704], histogram: base.u64, candidate: base.u64, tmp: base.u64) base.u64 { + var c : base.u64 + if this.h_total(wb: args.wb, h: args.histogram, ds: args.ds) == 0 { return 0 } this.h_copy!(wb: args.wb, h: args.tmp, v: args.histogram, ds: args.ds) this.h_add_histogram!(wb: args.wb, h: args.tmp, v: args.candidate, ds: args.ds) - return this.sf.f64_sub( - a: this.h_population_cost(wb: args.wb, h: args.tmp, ds: args.ds), - b: this.h_bit_cost(wb: args.wb, h: args.candidate, ds: args.ds)) + c = this.h_population_cost!(wb: args.wb, h: args.tmp, ds: args.ds) + return this.sf.f64_sub(a: c, b: this.h_bit_cost(wb: args.wb, h: args.candidate, ds: args.ds)) } // Mirrors FN(BrotliHistogramRemap) in cluster_inc.h. @@ -395,6 +396,7 @@ pri func metablock_builder.histogram_reindex!(wb: slice base.u8, ds: base.u32[.. // histogram_symbols. It returns the C code's *out_size. pri func metablock_builder.cluster_histograms!(wb: slice base.u8, ds: base.u32[..= 704], in_histos: base.u64, in_size: base.u64, max_histograms: base.u64, out_histos: base.u64, histogram_symbols: base.u64) base.u64 { + var c : base.u64 var hs : base.u64 var mark : base.u64 var cluster_size : base.u64 @@ -426,8 +428,8 @@ pri func metablock_builder.cluster_histograms!(wb: slice base.u8, ds: base.u32[. i = 0 while i < args.in_size { this.h_copy!(wb: args.wb, h: args.out_histos ~mod+ (i ~mod* hs), v: args.in_histos ~mod+ (i ~mod* hs), ds: args.ds) - this.h_set_bit_cost!(wb: args.wb, h: args.out_histos ~mod+ (i ~mod* hs), ds: args.ds, - bit_cost: this.h_population_cost(wb: args.wb, h: args.in_histos ~mod+ (i ~mod* hs), ds: args.ds)) + c = this.h_population_cost!(wb: args.wb, h: args.in_histos ~mod+ (i ~mod* hs), ds: args.ds) + this.h_set_bit_cost!(wb: args.wb, h: args.out_histos ~mod+ (i ~mod* hs), ds: args.ds, bit_cost: c) this.st32!(wb: args.wb, i: args.histogram_symbols ~mod+ (i ~mod* 4), v: (i & 0xFFFF_FFFF) as base.u32) i ~mod+= 1 } diff --git a/std/brotli/encode_literal_cost.wuffs b/std/brotli/encode_literal_cost.wuffs index 7fabc9c8..3b0846cf 100644 --- a/std/brotli/encode_literal_cost.wuffs +++ b/std/brotli/encode_literal_cost.wuffs @@ -277,7 +277,8 @@ pri func encoder.estimate_bit_costs_for_literals_utf8!(wb: slice base.u8, pos: b } else if utf8_pos == 2 { iw = iwu2 } - lit_cost = this.sf.f64_sub(a: this.bc.fast_log2(v: iw), b: this.bc.fast_log2(v: histo)) + lit_cost = this.bc.fast_log2_memo!(v: iw) + lit_cost = this.sf.f64_sub(a: lit_cost, b: this.bc.fast_log2(v: histo)) lit_cost = this.sf.f64_add(a: lit_cost, b: LC_F64_0_02905) this.lc_finish_cost!(wb: args.wb, cost_off: args.cost_off, i: i, lit_cost: lit_cost, utf8: true) i ~mod+= 1 @@ -337,7 +338,8 @@ pri func encoder.estimate_bit_costs_for_literals!(wb: slice base.u8, pos: base.u if histo == 0 { histo = 1 } - lit_cost = this.sf.f64_sub(a: this.bc.fast_log2(v: in_window), b: this.bc.fast_log2(v: histo)) + lit_cost = this.bc.fast_log2_memo!(v: in_window) + lit_cost = this.sf.f64_sub(a: lit_cost, b: this.bc.fast_log2(v: histo)) lit_cost = this.sf.f64_add(a: lit_cost, b: LC_F64_0_029) this.lc_finish_cost!(wb: args.wb, cost_off: args.cost_off, i: i, lit_cost: lit_cost, utf8: false) i ~mod+= 1 diff --git a/std/brotli/encode_metablock.wuffs b/std/brotli/encode_metablock.wuffs index fbbd15b4..9da3c388 100644 --- a/std/brotli/encode_metablock.wuffs +++ b/std/brotli/encode_metablock.wuffs @@ -264,9 +264,12 @@ pri func metablock_builder.h_set_bit_cost!(wb: slice base.u8, h: base.u64, ds: b this.st64!(wb: args.wb, i: args.h ~mod+ (((args.ds as base.u64) * 4) + 8), v: args.bit_cost) } -pri func metablock_builder.h_population_cost(wb: roslice base.u8, h: base.u64, ds: base.u32[..= 704]) base.u64 { +pri func metablock_builder.h_population_cost!(wb: roslice base.u8, h: base.u64, ds: base.u32[..= 704]) base.u64 { + var c : base.u64 + if args.h <= args.wb.length() { - return this.bc.population_cost(histogram: args.wb[args.h ..], data_size: args.ds) + c = this.bc.population_cost!(histogram: args.wb[args.h ..], data_size: args.ds) + return c } return 0 } @@ -482,9 +485,8 @@ pri func metablock_builder.compute_distance_cost!(wb: slice base.u8, cmds: rosli } i ~mod+= 1 } - this.dist_cost = this.sf.f64_add( - a: this.h_population_cost(wb: args.wb, h: args.tmp, ds: HISTOGRAM_DISTANCE_DATA_SIZE), - b: extra_bits) + this.dist_cost = this.h_population_cost!(wb: args.wb, h: args.tmp, ds: HISTOGRAM_DISTANCE_DATA_SIZE) + this.dist_cost = this.sf.f64_add(a: this.dist_cost, b: extra_bits) return true } @@ -592,7 +594,6 @@ pri func metablock_builder.build_meta_block!(wb: slice base.u8, ringbuffer: rosl var v : base.u32 this.reset_arena!(wb_len: args.wb.length()) - this.bc.init_log2_cache!() orig_npostfix = this.dist_postfix_bits orig_ndirect = this.dist_num_direct_codes diff --git a/std/brotli/encode_metablock_greedy.wuffs b/std/brotli/encode_metablock_greedy.wuffs index eaec8b21..103d6bbf 100644 --- a/std/brotli/encode_metablock_greedy.wuffs +++ b/std/brotli/encode_metablock_greedy.wuffs @@ -47,9 +47,12 @@ pri func encoder.gbs_histo_off(c: base.u32[..= 3], ix: base.u64) base.u64 { // gbs_entropy returns BrotliBitsEntropy of the size u32le values at // args.wb[off ..]. -pri func encoder.gbs_entropy(wb: roslice base.u8, off: base.u64, size: base.u64) base.u64 { +pri func encoder.gbs_entropy!(wb: roslice base.u8, off: base.u64, size: base.u64) base.u64 { + var e : base.u64 + if args.off <= args.wb.length() { - return this.bc.bits_entropy(population: args.wb[args.off ..], size: args.size) + e = this.bc.bits_entropy!(population: args.wb[args.off ..], size: args.size) + return e } return 0 } @@ -162,7 +165,7 @@ pri func encoder.gbs_finish_block!(wb: slice base.u8, c: base.u32[..= 3], is_fin this.set_byte!(s: args.wb, i: types_off, v: 0) i = 0 while i < nc { - e = this.gbs_entropy(wb: args.wb, off: this.gbs_histo_off(c: args.c, ix: i), size: alphabet_size) + e = this.gbs_entropy!(wb: args.wb, off: this.gbs_histo_off(c: args.c, ix: i), size: alphabet_size) this.gbs_last_entropy[((le_base as base.u64) ~mod+ i) & 127] = e this.gbs_last_entropy[(((le_base as base.u64) ~mod+ nc) ~mod+ i) & 127] = e i ~mod+= 1 @@ -185,7 +188,7 @@ pri func encoder.gbs_finish_block!(wb: slice base.u8, c: base.u32[..= 3], is_fin last1 = this.gbs_last_histogram_ix[(args.c * 2) + 1] i = 0 while i < nc { - e = this.gbs_entropy(wb: args.wb, off: this.gbs_histo_off(c: args.c, ix: curr ~mod+ i), size: alphabet_size) + e = this.gbs_entropy!(wb: args.wb, off: this.gbs_histo_off(c: args.c, ix: curr ~mod+ i), size: alphabet_size) this.gbs_entropy_tmp[i & 15] = e j = 0 while j < 2 { @@ -197,7 +200,7 @@ pri func encoder.gbs_finish_block!(wb: slice base.u8, c: base.u32[..= 3], is_fin } else { this.gbs_add_histogram!(wb: args.wb, dst: tmp, src: this.gbs_histo_off(c: args.c, ix: last1 ~mod+ i), n: data_size) } - this.gbs_comb_entropy[jx & 31] = this.gbs_entropy(wb: args.wb, off: tmp, size: alphabet_size) + this.gbs_comb_entropy[jx & 31] = this.gbs_entropy!(wb: args.wb, off: tmp, size: alphabet_size) d = this.sf.f64_sub( a: this.sf.f64_sub(a: this.gbs_comb_entropy[jx & 31], b: e), b: this.gbs_last_entropy[((le_base as base.u64) ~mod+ jx) & 127]) diff --git a/std/brotli/encode_tester.wuffs b/std/brotli/encode_tester.wuffs index 71aa8c45..855921c4 100644 --- a/std/brotli/encode_tester.wuffs +++ b/std/brotli/encode_tester.wuffs @@ -29,8 +29,11 @@ pub struct metablock_tester?( ) // bits_entropy mirrors BrotliBitsEntropy. population holds size u32le values. -pub func metablock_tester.bits_entropy(population: roslice base.u8, size: base.u64) base.u64 { - return this.bc.bits_entropy(population: args.population, size: args.size) +pub func metablock_tester.bits_entropy!(population: roslice base.u8, size: base.u64) base.u64 { + var c : base.u64 + + c = this.bc.bits_entropy!(population: args.population, size: args.size) + return c } // population_cost mirrors BrotliPopulationCostLiteral (data_size 256), @@ -38,13 +41,18 @@ pub func metablock_tester.bits_entropy(population: roslice base.u8, size: base.u // histogram has the C struct's layout: data_size u32le counts, then the // total_count_ u64le (and then the bit_cost_, which is ignored). It returns 0 // for any other data_size. -pub func metablock_tester.population_cost(histogram: roslice base.u8, data_size: base.u32) base.u64 { +pub func metablock_tester.population_cost!(histogram: roslice base.u8, data_size: base.u32) base.u64 { + var c : base.u64 + if args.data_size == 256 { - return this.bc.population_cost(histogram: args.histogram, data_size: 256) + c = this.bc.population_cost!(histogram: args.histogram, data_size: 256) + return c } else if args.data_size == 704 { - return this.bc.population_cost(histogram: args.histogram, data_size: 704) + c = this.bc.population_cost!(histogram: args.histogram, data_size: 704) + return c } else if args.data_size == 544 { - return this.bc.population_cost(histogram: args.histogram, data_size: 544) + c = this.bc.population_cost!(histogram: args.histogram, data_size: 544) + return c } return 0 } @@ -62,7 +70,6 @@ pub func metablock_tester.split_block!(cmds: roslice base.u8, num_commands: base data: roslice base.u8, pos: base.u64, mask: base.u64, quality: base.u32, workbuf: slice base.u8) base.status { this.mb.set_params!(quality: args.quality, large_window: false, disable_literal_context_modeling: false) this.mb.reset_arena!(wb_len: args.workbuf.length()) - this.mb.bc.init_log2_cache!() this.mb.alloc_block_splits!(cmds: args.cmds, num_commands: args.num_commands) if this.mb.oom { return base."#bad workbuf length" @@ -102,7 +109,6 @@ pub func metablock_tester.cluster_histograms!(data_size: base.u32, in_histos: ro return 0xFFFF_FFFF_FFFF_FFFF } hs = this.mb.hsize(ds: ds) - this.mb.bc.init_log2_cache!() this.mb.reset_arena!(wb_len: args.workbuf.length()) n = args.in_size ~mod* hs in_off = this.mb.alloc!(n: n) diff --git a/std/softfloat/README.md b/std/softfloat/README.md index 9b7d8a22..e2294330 100644 --- a/std/softfloat/README.md +++ b/std/softfloat/README.md @@ -96,12 +96,32 @@ integer long division, 11 quotient bits per (64-bit integer) division step, with the final remainder as the sticky bit. Leading zero counts use a 256-entry table. -The `f32` add, sub, mul and div operations convert their operands exactly to -`f64`, perform the `f64` operation and then round to `f32`. This double -rounding is innocuous: binary64's 53 bits of precision are more than `(2 * 24) -+ 2`, which suffices for `+`, `-`, `*` and `/` (S. A. Figueroa, "When is double -rounding innocuous?", ACM SIGNUM Newsletter, 1995). The tests confirm this -against hardware, including for subnormal `f32` results. +`f64_add` and `f64_sub` have a fast path for the common case where both +operands are normal (`f64_add_normal`): it orders the operands by magnitude, +aligns the significands with a sticky bit, adds or subtracts them and rounds +inline. Only a result that is subnormal falls back to the general code +(`f64_add_mags`, `f64_sub_mags`). Adding (or subtracting) a zero and a finite +non-zero value also returns early. + +Likewise, `f32_add`, `f32_sub` and `f32_mul` compute the result directly in +binary32 (with 32-bit significands for add and sub and an exact 48-bit +product for mul) when both operands are normal and the result is normal (or, +for add and mul, overflows to infinity), and add and sub return early for a +zero plus a finite non-zero value. `f32_from_u32` converts integers below +`1 << 24` (which are exact) directly. Otherwise (zeroes, subnormal +operands or results, infinities and NaNs), they, and `f32_div` always, +convert their operands exactly to `f64`, perform the `f64` operation and then +round to `f32`. This double rounding is innocuous: binary64's 53 bits of +precision are more than `(2 * 24) + 2`, which suffices for `+`, `-`, `*` and +`/` (S. A. Figueroa, "When is double rounding innocuous?", ACM SIGNUM +Newsletter, 1995). The tests confirm this against hardware, including for +subnormal `f32` results. The fast paths never produce a NaN, so NaN results +(and their payloads) are exactly those of the "via `f64`" computation. + +The comparison functions (`f64_cmp`, `f64_lt`, `f64_min`, etc. and their `f32` +counterparts) compare the bit patterns directly when neither operand is +negative or a NaN (when `a | b` is at most the bit pattern of `+inf`), and +otherwise call `f64_cmp_slow` or `f32_cmp_slow`. ## log2 @@ -187,3 +207,38 @@ threads): (6 minutes 20 seconds wall clock in total) No exception table is needed. + +`script/manual-test-softfloat-sweep.c` systematically checks the `f32` add, +sub, mul and div operations over every pair of operand exponents (`[0 ..= +255]` squared), both signs and 40 x 40 mantissa patterns (exact results, ties, +carries, cancellation, subnormal results, overflow, NaN payloads): each result +must equal the hardware's (any NaN matching any NaN) and, bit-for-bit +including NaNs, the "via `f64`" composition `f64_to_f32(f64_op(f32_to_f64(a), +f32_to_f64(b)))`. It does the same for the `f64` add, sub, mul and div +operations (every pair of exponents in `[0 ..= 2047]`, 9 x 9 mantissa +patterns, NaN results checked bit-for-bit against the NaN rule above), and +checks `f64_to_f32` (326 exponents times 2²⁰ fraction patterns times both +signs), `f32_to_f64` (every `f32`), `f32_from_u32` and `f32_from_i32` (every +32-bit integer) and `f32_from_u64` (2³² structured values). It also checks +the `f32` and `f64` cmp, eq, ne, lt, le, gt, ge, min and max functions on +every pair of 594 values (both signs, zero, subnormal, normal and maximal +exponents, boundary mantissas and their neighbors, infinities and NaNs) +against the hardware's comparison operators. Result (x86-64, gcc 13 `-O2`, 4 +threads): + + f32_add 419430400 operand pairs, 0 mismatches + f32_sub 419430400 operand pairs, 0 mismatches + f32_mul 419430400 operand pairs, 0 mismatches + f32_div 419430400 operand pairs, 0 mismatches + f64_add 1358954496 operand pairs, 0 mismatches + f64_sub 1358954496 operand pairs, 0 mismatches + f64_mul 1358954496 operand pairs, 0 mismatches + f64_div 1358954496 operand pairs, 0 mismatches + f64_to_f32 341835776 operands, 0 mismatches + f32_to_f64 4294967296 operands, 0 mismatches + f32_from_u32 4294967296 operands, 0 mismatches + f32_from_i32 4294967296 operands, 0 mismatches + f32_from_u64 4294967296 operands, 0 mismatches + f32_compare 352836 operand pairs, 0 mismatches + f64_compare 352836 operand pairs, 0 mismatches + PASS: 0 mismatches in total diff --git a/std/softfloat/softfloat.wuffs b/std/softfloat/softfloat.wuffs index aeb9540f..65bd77e5 100644 --- a/std/softfloat/softfloat.wuffs +++ b/std/softfloat/softfloat.wuffs @@ -61,6 +61,24 @@ pri func arith.clz64(a: base.u64) base.u32[..= 63] { return (n32 + n16 + n8 + ((CLZ8[x >> 56] as base.u32) & 7)) & 63 } +// clz32 returns the number of leading zero bits of a, which must be non-zero. +pri func arith.clz32(a: base.u32) base.u32[..= 31] { + var x : base.u32 + var n16 : base.u32[..= 16] + var n8 : base.u32[..= 8] + + x = args.a + if (x >> 16) == 0 { + x = x ~mod<< 16 + n16 = 16 + } + if (x >> 24) == 0 { + x = x ~mod<< 8 + n8 = 8 + } + return (n16 + n8 + ((CLZ8[x >> 24] as base.u32) & 7)) & 31 +} + // shift_right_jam64 returns (a >> dist), except that if any non-zero bits // were shifted out then the least significant bit of the result is set. This // "sticky bit" is needed to round correctly. @@ -247,6 +265,21 @@ pub func arith.f64_abs(a: base.u64) base.u64 { // f64_cmp compares a and b, returning 0 if (a < b), 1 if (a == b), 2 if (a // > b) and 3 if they are unordered (either is a NaN). -0 and +0 compare equal. pub func arith.f64_cmp(a: base.u64, b: base.u64) base.u32 { + if (args.a | args.b) <= F64_INF { + // Fast path: neither is negative (the sign bits are clear) or a NaN + // (both are at most +inf), so the bit patterns sort like the values. + if args.a < args.b { + return 0 + } else if args.a == args.b { + return 1 + } + return 2 + } + return this.f64_cmp_slow(a: args.a, b: args.b) +} + +// f64_cmp_slow is f64_cmp without the fast path. +pri func arith.f64_cmp_slow(a: base.u64, b: base.u64) base.u32 { var ka : base.u64 var kb : base.u64 @@ -279,35 +312,65 @@ pub func arith.f64_cmp(a: base.u64, b: base.u64) base.u32 { // true), and -0 equals +0. pub func arith.f64_eq(a: base.u64, b: base.u64) base.u32 { - if this.f64_cmp(a: args.a, b: args.b) == 1 { + if (args.a | args.b) <= F64_INF { + if args.a == args.b { + return 1 + } + return 0 + } + if this.f64_cmp_slow(a: args.a, b: args.b) == 1 { return 1 } return 0 } pub func arith.f64_ne(a: base.u64, b: base.u64) base.u32 { - if this.f64_cmp(a: args.a, b: args.b) == 1 { + if (args.a | args.b) <= F64_INF { + if args.a <> args.b { + return 1 + } + return 0 + } + if this.f64_cmp_slow(a: args.a, b: args.b) == 1 { return 0 } return 1 } pub func arith.f64_lt(a: base.u64, b: base.u64) base.u32 { - if this.f64_cmp(a: args.a, b: args.b) == 0 { + if (args.a | args.b) <= F64_INF { + if args.a < args.b { + return 1 + } + return 0 + } + if this.f64_cmp_slow(a: args.a, b: args.b) == 0 { return 1 } return 0 } pub func arith.f64_le(a: base.u64, b: base.u64) base.u32 { - if this.f64_cmp(a: args.a, b: args.b) <= 1 { + if (args.a | args.b) <= F64_INF { + if args.a <= args.b { + return 1 + } + return 0 + } + if this.f64_cmp_slow(a: args.a, b: args.b) <= 1 { return 1 } return 0 } pub func arith.f64_gt(a: base.u64, b: base.u64) base.u32 { - if this.f64_cmp(a: args.a, b: args.b) == 2 { + if (args.a | args.b) <= F64_INF { + if args.a > args.b { + return 1 + } + return 0 + } + if this.f64_cmp_slow(a: args.a, b: args.b) == 2 { return 1 } return 0 @@ -316,7 +379,13 @@ pub func arith.f64_gt(a: base.u64, b: base.u64) base.u32 { pub func arith.f64_ge(a: base.u64, b: base.u64) base.u32 { var c : base.u32 - c = this.f64_cmp(a: args.a, b: args.b) + if (args.a | args.b) <= F64_INF { + if args.a >= args.b { + return 1 + } + return 0 + } + c = this.f64_cmp_slow(a: args.a, b: args.b) if (c == 1) or (c == 2) { return 1 } @@ -327,7 +396,13 @@ pub func arith.f64_ge(a: base.u64, b: base.u64) base.u32 { // Note that this is not IEEE 754 minNum: f64_min(NaN, x) and f64_min(x, NaN) // both return the second argument, and f64_min(-0, +0) returns +0. pub func arith.f64_min(a: base.u64, b: base.u64) base.u64 { - if this.f64_cmp(a: args.a, b: args.b) == 0 { + if (args.a | args.b) <= F64_INF { + if args.a < args.b { + return args.a + } + return args.b + } + if this.f64_cmp_slow(a: args.a, b: args.b) == 0 { return args.a } return args.b @@ -335,7 +410,13 @@ pub func arith.f64_min(a: base.u64, b: base.u64) base.u64 { // f64_max returns ((a > b) ? a : b), like brotli's BROTLI_MAX(double, a, b). pub func arith.f64_max(a: base.u64, b: base.u64) base.u64 { - if this.f64_cmp(a: args.a, b: args.b) == 2 { + if (args.a | args.b) <= F64_INF { + if args.a > args.b { + return args.a + } + return args.b + } + if this.f64_cmp_slow(a: args.a, b: args.b) == 2 { return args.a } return args.b @@ -345,6 +426,21 @@ pub func arith.f64_max(a: base.u64, b: base.u64) base.u64 { // f64_add returns (a + b). pub func arith.f64_add(a: base.u64, b: base.u64) base.u64 { + var z : base.u64 + + if ((((args.a >> 52) & 0x7FF) ~mod- 1) < 0x7FE) and ((((args.b >> 52) & 0x7FF) ~mod- 1) < 0x7FE) { + // Both operands are normal: take the fast path. + z = this.f64_add_normal(a: args.a, b: args.b) + if z <> 0xFFFF_FFFF_FFFF_FFFF { + return z + } + } else if ((args.b & F64_ABS) == 0) and (((args.a & F64_ABS) ~mod- 1) < (F64_INF - 1)) { + // b is a zero and a is finite and non-zero, so (a + b) is a. + return args.a + } else if ((args.a & F64_ABS) == 0) and (((args.b & F64_ABS) ~mod- 1) < (F64_INF - 1)) { + // a is a zero and b is finite and non-zero, so (a + b) is b. + return args.b + } if ((args.a ^ args.b) >> 63) == 0 { return this.f64_add_mags(a: args.a, b: args.b, sign: args.a & F64_SIGN) } @@ -353,12 +449,105 @@ pub func arith.f64_add(a: base.u64, b: base.u64) base.u64 { // f64_sub returns (a - b). pub func arith.f64_sub(a: base.u64, b: base.u64) base.u64 { + var z : base.u64 + + if ((((args.a >> 52) & 0x7FF) ~mod- 1) < 0x7FE) and ((((args.b >> 52) & 0x7FF) ~mod- 1) < 0x7FE) { + // Both operands are normal: take the fast path. For non-NaN + // operands, (a - b) is exactly (a + -b), including the sign of a + // zero result. + z = this.f64_add_normal(a: args.a, b: args.b ^ F64_SIGN) + if z <> 0xFFFF_FFFF_FFFF_FFFF { + return z + } + } else if ((args.b & F64_ABS) == 0) and (((args.a & F64_ABS) ~mod- 1) < (F64_INF - 1)) { + // b is a zero and a is finite and non-zero, so (a - b) is a. + return args.a + } else if ((args.a & F64_ABS) == 0) and (((args.b & F64_ABS) ~mod- 1) < (F64_INF - 1)) { + // a is a zero and b is finite and non-zero, so (a - b) is -b. + return args.b ^ F64_SIGN + } if ((args.a ^ args.b) >> 63) == 0 { return this.f64_sub_mags(a: args.a, b: args.b, sign: args.a & F64_SIGN) } return this.f64_add_mags(a: args.a, b: args.b, sign: args.a & F64_SIGN) } +// f64_add_normal returns (a + b), for normal a and b, unless the result is +// subnormal, in which case it returns 0xFFFF_FFFF_FFFF_FFFF (a NaN, which it +// never otherwise returns) and the caller must take its slow path. It is a +// faster (but less general) version of f64_add_mags and f64_sub_mags. +pri func arith.f64_add_normal(a: base.u64, b: base.u64) base.u64 { + var x : base.u64 + var y : base.u64 + var ex : base.u32[..= 0x800] + var d : base.u32 + var mx : base.u64[..= 0x7FFF_FFFF_FFFF_FFFF] + var my : base.u64[..= 0x7FFF_FFFF_FFFF_FFFF] + var z : base.u64 + var s : base.u32[..= 63] + var rb : base.u64 + + // Order the operands so that |x| >= |y|. The result has x's sign. + x = args.a + y = args.b + if (y & F64_ABS) > (x & F64_ABS) { + x = args.b + y = args.a + } + ex = ((x >> 52) & 0x7FF) as base.u32 + d = ex ~mod- (((y >> 52) & 0x7FF) as base.u32) + if d > 63 { + d = 63 + } + // The significands, with their implicit bit at bit 62 and 10 extra bits + // of precision at the bottom. Align my with mx, keeping a sticky bit. + mx = ((x & F64_FRAC) | F64_HIDDEN) << 10 + my = ((y & F64_FRAC) | F64_HIDDEN) << 10 + if (my & (((1 as base.u64) << (d & 63)) ~mod- 1)) <> 0 { + my = (my >> (d & 63)) | 1 + } else { + my = my >> (d & 63) + } + + if ((x ^ y) >> 63) == 0 { + z = mx + my + if z >= 0x8000_0000_0000_0000 { + ex += 1 + z = (z >> 1) | (z & 1) + } + } else { + // mx >= my, since |x| >= |y|. + z = mx ~mod- my + if z == 0 { + // Exact cancellation gives +0 when rounding to nearest. + return 0 + } else if z < 0x4000_0000_0000_0000 { + // If (d >= 2) then z >= (1 << 61) and the normalization shift + // is 1. Otherwise, no bits were shifted out of my. + s = (this.clz64(a: z) ~mod- 1) & 63 + if s >= ex { + return 0xFFFF_FFFF_FFFF_FFFF + } + ex = (ex ~mod- s) & 0x7FF + z = z ~mod<< s + } + } + + // Round to nearest, ties to even. z is now in [1 << 52 ..= 1 << 53] and + // ex in [1 ..= 0x7FF]. Adding z (whose bit 52 is the implicit bit) to the + // shifted exponent carries any rounding overflow into the exponent field. + rb = z & 0x3FF + z = (z ~mod+ 0x200) >> 10 + if rb == 0x200 { + z &= 0xFFFF_FFFF_FFFF_FFFE + } + z = (((ex ~mod- 1) as base.u64) ~mod<< 52) ~mod+ z + if z >= F64_INF { + z = F64_INF + } + return (x & F64_SIGN) | z +} + // f64_add_mags returns (sign * (|a| + |b|)). pri func arith.f64_add_mags(a: base.u64, b: base.u64, sign: base.u64) base.u64 { var exp_a : base.u32 diff --git a/std/softfloat/softfloat_f32.wuffs b/std/softfloat/softfloat_f32.wuffs index f604d093..c49d64e9 100644 --- a/std/softfloat/softfloat_f32.wuffs +++ b/std/softfloat/softfloat_f32.wuffs @@ -10,12 +10,19 @@ // -------- f32 (binary32) operations. // -// The f32 add, sub, mul and div functions convert their (exactly -// representable) operands to f64, perform the f64 operation and round the -// result to f32. Double rounding is innocuous here: binary64 has more than -// (2 * 24) + 2 bits of precision, which suffices for +, -, * and / (Figueroa, -// "When is double rounding innocuous?", 1995). This is also exhaustively -// cross-checked against hardware by the tests. +// When both operands are normal (and, unless it overflows to infinity, the +// result is normal too), the f32 add, sub and mul functions compute the +// result directly in binary32 (with 32-bit or 64-bit integer significands). +// This is the common case. Adding or subtracting a zero and a finite +// non-zero operand is also handled directly. Otherwise (zeroes, subnormals, +// infinities, NaNs and subnormal results), they and f32_div convert their +// (exactly representable) operands to f64, perform the f64 operation and +// round the result to f32 (the slow_etc functions). Double rounding is +// innocuous here: binary64 has more than (2 * 24) + 2 bits of precision, +// which suffices for +, -, * and / (Figueroa, "When is double rounding +// innocuous?", 1995). This is also exhaustively cross-checked against +// hardware by the tests. The fast paths never return a NaN, so NaN results +// (and their payloads) always come from the slow paths. pri const F32_SIGN : base.u32 = 0x8000_0000 pri const F32_ABS : base.u32 = 0x7FFF_FFFF @@ -74,20 +81,244 @@ pub func arith.f32_abs(a: base.u32) base.u32 { // f32_add returns (a + b). pub func arith.f32_add(a: base.u32, b: base.u32) base.u32 { + var z : base.u32 + + if ((((args.a >> 23) & 0xFF) ~mod- 1) < 0xFE) and ((((args.b >> 23) & 0xFF) ~mod- 1) < 0xFE) { + if ((args.a ^ args.b) >> 31) == 0 { + return this.f32_add_mags(a: args.a, b: args.b) + } + z = this.f32_sub_mags(a: args.a, b: args.b) + if z <> 0xFFFF_FFFF { + return z + } + } else if ((args.b & F32_ABS) == 0) and (((args.a & F32_ABS) ~mod- 1) < (F32_INF - 1)) { + // b is a zero and a is finite and non-zero, so (a + b) is a. + return args.a + } else if ((args.a & F32_ABS) == 0) and (((args.b & F32_ABS) ~mod- 1) < (F32_INF - 1)) { + // a is a zero and b is finite and non-zero, so (a + b) is b. + return args.b + } + return this.slow_f32_add(a: args.a, b: args.b) +} + +// f32_sub returns (a - b). +pub func arith.f32_sub(a: base.u32, b: base.u32) base.u32 { + var z : base.u32 + + if ((((args.a >> 23) & 0xFF) ~mod- 1) < 0xFE) and ((((args.b >> 23) & 0xFF) ~mod- 1) < 0xFE) { + if ((args.a ^ args.b) >> 31) <> 0 { + return this.f32_add_mags(a: args.a, b: args.b) + } + z = this.f32_sub_mags(a: args.a, b: args.b) + if z <> 0xFFFF_FFFF { + return z + } + } else if ((args.b & F32_ABS) == 0) and (((args.a & F32_ABS) ~mod- 1) < (F32_INF - 1)) { + // b is a zero and a is finite and non-zero, so (a - b) is a. + return args.a + } else if ((args.a & F32_ABS) == 0) and (((args.b & F32_ABS) ~mod- 1) < (F32_INF - 1)) { + // a is a zero and b is finite and non-zero, so (a - b) is -b. + return args.b ^ F32_SIGN + } + return this.slow_f32_sub(a: args.a, b: args.b) +} + +// f32_mul returns (a * b). +pub func arith.f32_mul(a: base.u32, b: base.u32) base.u32 { + var ea : base.u32[..= 0xFF] + var eb : base.u32[..= 0xFF] + var e : base.u32[..= 0x1FF] + var p : base.u64 + var m : base.u32 + var r : base.u32 + var z : base.u32 + + ea = (args.a >> 23) & 0xFF + eb = (args.b >> 23) & 0xFF + if ((ea ~mod- 1) < 0xFE) and ((eb ~mod- 1) < 0xFE) { + // Both operands are normal. The exact product of the two 24-bit + // significands, p, is in [1 << 46, 1 << 48). The biased exponent of + // the result, e, is (ea + eb - 127) (plus 1 if p >= (1 << 47)). + p = (((args.a & 0x7F_FFFF) | 0x80_0000) as base.u64) * + (((args.b & 0x7F_FFFF) | 0x80_0000) as base.u64) + e = ea + eb + if p >= 0x8000_0000_0000 { + e += 1 + m = ((p >> 24) & 0xFF_FFFF) as base.u32 + r = ((p & 0xFF_FFFF) as base.u32) ~mod- 0x80_0000 + } else { + m = ((p >> 23) & 0xFF_FFFF) as base.u32 + r = ((p & 0x7F_FFFF) as base.u32) ~mod- 0x40_0000 + } + // r is the remainder minus half an ULP, as a two's complement + // integer. Round to nearest, ties to even. + if (r <> 0) and (r < 0x8000_0000) { + m ~mod+= 1 + } else if r == 0 { + m ~mod+= m & 1 + } + if (e >= 128) and (e <= 381) { + // Adding m (whose bit 23 is the implicit bit) carries any + // rounding overflow into the exponent field. + z = ((e - 128) << 23) ~mod+ m + if z >= F32_INF { + z = F32_INF + } + return ((args.a ^ args.b) & F32_SIGN) | z + } + } + return this.slow_f32_mul(a: args.a, b: args.b) +} + +// f32_add_mags returns (sign(a) * (|a| + |b|)), for normal a and b. +pri func arith.f32_add_mags(a: base.u32, b: base.u32) base.u32 { + var ea : base.u32[..= 0x100] + var eb : base.u32[..= 0xFF] + var ma : base.u32[..= 0x3FFF_FFC0] + var mb : base.u32[..= 0x3FFF_FFC0] + var d : base.u32[..= 0xFF] + var z : base.u32 + var rb : base.u32 + + ea = (args.a >> 23) & 0xFF + eb = (args.b >> 23) & 0xFF + // The significands, with their implicit bit at bit 29 and 6 extra bits + // of precision at the bottom. + ma = ((args.a & 0x7F_FFFF) | 0x80_0000) << 6 + mb = ((args.b & 0x7F_FFFF) | 0x80_0000) << 6 + if ea < eb { + d = (eb ~mod- ea) & 0xFF + ea = eb + z = ma + ma = mb + mb = z + } else { + d = (ea ~mod- eb) & 0xFF + } + if d >= 31 { + mb = 1 + } else if d > 0 { + if (mb ~mod<< (32 - d)) <> 0 { + mb = (mb >> d) | 1 + } else { + mb = mb >> d + } + } + z = (ma & 0x3FFF_FFFF) + (mb & 0x3FFF_FFFF) + if z >= 0x4000_0000 { + ea += 1 + z = (z >> 1) | (z & 1) + } + rb = z & 0x3F + z = (z ~mod+ 0x20) >> 6 + if rb == 0x20 { + z &= 0xFFFF_FFFE + } + // ea is in [1 ..= 255] and z is in [1 << 23 ..= 1 << 24]. + z = ((ea ~mod- 1) ~mod<< 23) ~mod+ z + if z >= F32_INF { + z = F32_INF + } + return (args.a & F32_SIGN) | z +} + +// f32_sub_mags returns (sign(a) * (|a| - |b|)), for normal a and b, unless +// the result is subnormal, in which case it returns 0xFFFF_FFFF (a NaN, which +// it never otherwise returns) and the caller must take its slow path. +pri func arith.f32_sub_mags(a: base.u32, b: base.u32) base.u32 { + var sign : base.u32 + var ea : base.u32[..= 0xFF] + var eb : base.u32[..= 0xFF] + var fa : base.u32[..= 0x7F_FFFF] + var fb : base.u32[..= 0x7F_FFFF] + var ma : base.u32 + var mb : base.u32 + var d : base.u32[..= 0xFF] + var s : base.u32[..= 31] + var z : base.u32 + var rb : base.u32 + + sign = args.a & F32_SIGN + ea = (args.a >> 23) & 0xFF + eb = (args.b >> 23) & 0xFF + fa = args.a & 0x7F_FFFF + fb = args.b & 0x7F_FFFF + + if ea == eb { + // The difference is exact. + if fa == fb { + // Exact cancellation gives +0 when rounding to nearest. + return 0 + } else if fa < fb { + sign ^= F32_SIGN + z = fb ~mod- fa + } else { + z = fa ~mod- fb + } + // z is in [1 ..= 0x7F_FFFF]. Shift its top bit to bit 23. + s = (this.clz32(a: z) ~mod- 8) & 31 + if s >= ea { + return 0xFFFF_FFFF + } + return sign | ((((ea ~mod- s) ~mod- 1) ~mod<< 23) ~mod+ (z ~mod<< s)) + } + + // The significands, with their implicit bit at bit 30 and 7 extra bits + // of precision at the bottom. + ma = (fa | 0x80_0000) << 7 + mb = (fb | 0x80_0000) << 7 + if ea < eb { + sign ^= F32_SIGN + d = (eb ~mod- ea) & 0xFF + ea = eb + z = ma + ma = mb + mb = z + } else { + d = (ea ~mod- eb) & 0xFF + } + if d >= 31 { + mb = 1 + } else if (mb ~mod<< ((32 - d) & 31)) <> 0 { + mb = (mb >> d) | 1 + } else { + mb = mb >> d + } + // ma > mb, so z is non-zero. If (d >= 2) then z >= (1 << 29) and the + // normalization shift is at most 1. Otherwise, no bits were shifted out. + z = ma ~mod- mb + s = (this.clz32(a: z) ~mod- 1) & 31 + if s >= ea { + return 0xFFFF_FFFF + } + ea = (ea ~mod- s) & 0xFF + z = z ~mod<< s + rb = z & 0x7F + z = (z ~mod+ 0x40) >> 7 + if rb == 0x40 { + z &= 0xFFFF_FFFE + } + // ea is in [1 ..= 254] and z is in [1 << 23 ..= 1 << 24], so this does + // not overflow. + return sign | (((ea ~mod- 1) ~mod<< 23) ~mod+ z) +} + +// slow_f32_add returns (a + b), computed via f64. +pri func arith.slow_f32_add(a: base.u32, b: base.u32) base.u32 { return this.f64_to_f32(a: this.f64_add( a: this.f32_to_f64(a: args.a), b: this.f32_to_f64(a: args.b))) } -// f32_sub returns (a - b). -pub func arith.f32_sub(a: base.u32, b: base.u32) base.u32 { +// slow_f32_sub returns (a - b), computed via f64. +pri func arith.slow_f32_sub(a: base.u32, b: base.u32) base.u32 { return this.f64_to_f32(a: this.f64_sub( a: this.f32_to_f64(a: args.a), b: this.f32_to_f64(a: args.b))) } -// f32_mul returns (a * b). -pub func arith.f32_mul(a: base.u32, b: base.u32) base.u32 { +// slow_f32_mul returns (a * b), computed via f64. +pri func arith.slow_f32_mul(a: base.u32, b: base.u32) base.u32 { return this.f64_to_f32(a: this.f64_mul( a: this.f32_to_f64(a: args.a), b: this.f32_to_f64(a: args.b))) @@ -103,6 +334,21 @@ pub func arith.f32_div(a: base.u32, b: base.u32) base.u32 { // f32_cmp compares a and b, returning 0 if (a < b), 1 if (a == b), 2 if (a // > b) and 3 if they are unordered (either is a NaN). -0 and +0 compare equal. pub func arith.f32_cmp(a: base.u32, b: base.u32) base.u32 { + if (args.a | args.b) <= F32_INF { + // Fast path: neither is negative (the sign bits are clear) or a NaN + // (both are at most +inf), so the bit patterns sort like the values. + if args.a < args.b { + return 0 + } else if args.a == args.b { + return 1 + } + return 2 + } + return this.f32_cmp_slow(a: args.a, b: args.b) +} + +// f32_cmp_slow is f32_cmp without the fast path. +pri func arith.f32_cmp_slow(a: base.u32, b: base.u32) base.u32 { var ka : base.u32 var kb : base.u32 @@ -133,35 +379,65 @@ pub func arith.f32_cmp(a: base.u32, b: base.u32) base.u32 { // true), and -0 equals +0. pub func arith.f32_eq(a: base.u32, b: base.u32) base.u32 { - if this.f32_cmp(a: args.a, b: args.b) == 1 { + if (args.a | args.b) <= F32_INF { + if args.a == args.b { + return 1 + } + return 0 + } + if this.f32_cmp_slow(a: args.a, b: args.b) == 1 { return 1 } return 0 } pub func arith.f32_ne(a: base.u32, b: base.u32) base.u32 { - if this.f32_cmp(a: args.a, b: args.b) == 1 { + if (args.a | args.b) <= F32_INF { + if args.a <> args.b { + return 1 + } + return 0 + } + if this.f32_cmp_slow(a: args.a, b: args.b) == 1 { return 0 } return 1 } pub func arith.f32_lt(a: base.u32, b: base.u32) base.u32 { - if this.f32_cmp(a: args.a, b: args.b) == 0 { + if (args.a | args.b) <= F32_INF { + if args.a < args.b { + return 1 + } + return 0 + } + if this.f32_cmp_slow(a: args.a, b: args.b) == 0 { return 1 } return 0 } pub func arith.f32_le(a: base.u32, b: base.u32) base.u32 { - if this.f32_cmp(a: args.a, b: args.b) <= 1 { + if (args.a | args.b) <= F32_INF { + if args.a <= args.b { + return 1 + } + return 0 + } + if this.f32_cmp_slow(a: args.a, b: args.b) <= 1 { return 1 } return 0 } pub func arith.f32_gt(a: base.u32, b: base.u32) base.u32 { - if this.f32_cmp(a: args.a, b: args.b) == 2 { + if (args.a | args.b) <= F32_INF { + if args.a > args.b { + return 1 + } + return 0 + } + if this.f32_cmp_slow(a: args.a, b: args.b) == 2 { return 1 } return 0 @@ -170,7 +446,13 @@ pub func arith.f32_gt(a: base.u32, b: base.u32) base.u32 { pub func arith.f32_ge(a: base.u32, b: base.u32) base.u32 { var c : base.u32 - c = this.f32_cmp(a: args.a, b: args.b) + if (args.a | args.b) <= F32_INF { + if args.a >= args.b { + return 1 + } + return 0 + } + c = this.f32_cmp_slow(a: args.a, b: args.b) if (c == 1) or (c == 2) { return 1 } @@ -181,7 +463,13 @@ pub func arith.f32_ge(a: base.u32, b: base.u32) base.u32 { // Note that this is not IEEE 754 minNum: f32_min(NaN, x) and f32_min(x, NaN) // both return the second argument, and f32_min(-0, +0) returns +0. pub func arith.f32_min(a: base.u32, b: base.u32) base.u32 { - if this.f32_cmp(a: args.a, b: args.b) == 0 { + if (args.a | args.b) <= F32_INF { + if args.a < args.b { + return args.a + } + return args.b + } + if this.f32_cmp_slow(a: args.a, b: args.b) == 0 { return args.a } return args.b @@ -189,7 +477,13 @@ pub func arith.f32_min(a: base.u32, b: base.u32) base.u32 { // f32_max returns ((a > b) ? a : b), like brotli's BROTLI_MAX(float, a, b). pub func arith.f32_max(a: base.u32, b: base.u32) base.u32 { - if this.f32_cmp(a: args.a, b: args.b) == 2 { + if (args.a | args.b) <= F32_INF { + if args.a > args.b { + return args.a + } + return args.b + } + if this.f32_cmp_slow(a: args.a, b: args.b) == 2 { return args.a } return args.b @@ -197,6 +491,18 @@ pub func arith.f32_max(a: base.u32, b: base.u32) base.u32 { // f32_from_u32 returns the f32 nearest to the integer a (C's (float)a). pub func arith.f32_from_u32(a: base.u32) base.u32 { + var c : base.u32[..= 31] + + if args.a == 0 { + return 0 + } else if args.a < 0x100_0000 { + // a is exactly representable. Shift its top bit to bit 23 (the + // implicit bit, which is then added to the exponent field). + c = this.clz32(a: args.a) + if c >= 8 { + return ((157 ~mod- c) ~mod<< 23) ~mod+ (args.a ~mod<< (c - 8)) + } + } // The u32 to f64 conversion is exact, so there is only one rounding. return this.f64_to_f32(a: this.f64_from_u32(a: args.a)) }