From 1d1c338981ba3220e656de08639f77a1f71bedd6 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 09:26:09 +0000 Subject: [PATCH 1/6] std/softfloat: add fast paths for f32 and f64 add, sub and mul f32_add, f32_sub and f32_mul now compute the result directly in binary32 when both operands are normal and the result is normal (or overflows), instead of always going through f64. f64_add and f64_sub get a similar fast path for normal operands. Every other case takes the previous code path, so results (including NaN payloads) are unchanged. Add script/manual-test-softfloat-sweep.c, which checks the f32 and f64 add, sub, mul and div operations over every pair of operand exponents with many mantissa patterns, and the f32 conversions, against the hardware and (bit-for-bit, NaNs included) the via-f64 definitions. --- release/c/wuffs-unsupported-snapshot.c | 451 ++++++++++++++++++- script/manual-test-softfloat-sweep.c | 571 +++++++++++++++++++++++++ std/softfloat/README.md | 53 ++- std/softfloat/softfloat.wuffs | 179 ++++++++ std/softfloat/softfloat_f32.wuffs | 238 ++++++++++- 5 files changed, 1474 insertions(+), 18 deletions(-) create mode 100644 script/manual-test-softfloat-sweep.c diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 16e642f7..2d724b30 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -41133,6 +41133,12 @@ wuffs_softfloat__arith__clz64( const wuffs_softfloat__arith* self, uint64_t a_a); +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_softfloat__arith__clz32( + const wuffs_softfloat__arith* self, + uint32_t a_a); + WUFFS_BASE__GENERATED_C_CODE static uint64_t wuffs_softfloat__arith__shift_right_jam64( @@ -41198,6 +41204,20 @@ wuffs_softfloat__arith__f64_propagate_nan( uint64_t a_a, uint64_t a_b); +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_softfloat__arith__f64_add_normal_mags( + const wuffs_softfloat__arith* self, + uint64_t a_a, + uint64_t a_b); + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_softfloat__arith__f64_sub_normal_mags( + const wuffs_softfloat__arith* self, + uint64_t a_a, + uint64_t a_b); + WUFFS_BASE__GENERATED_C_CODE static uint64_t wuffs_softfloat__arith__f64_add_mags( @@ -41222,6 +41242,41 @@ wuffs_softfloat__arith__round_pack_f32( uint32_t a_exp, uint32_t a_sig); +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_softfloat__arith__f32_add_mags( + const wuffs_softfloat__arith* self, + uint32_t a_a, + uint32_t a_b); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_softfloat__arith__f32_sub_mags( + const wuffs_softfloat__arith* self, + uint32_t a_a, + uint32_t a_b); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_softfloat__arith__slow_f32_add( + const wuffs_softfloat__arith* self, + uint32_t a_a, + uint32_t a_b); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_softfloat__arith__slow_f32_sub( + const wuffs_softfloat__arith* self, + uint32_t a_a, + uint32_t a_b); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_softfloat__arith__slow_f32_mul( + const wuffs_softfloat__arith* self, + uint32_t a_a, + uint32_t a_b); + // ---------------- VTables // ---------------- Initializer Implementations @@ -41322,6 +41377,29 @@ wuffs_softfloat__arith__clz64( (((uint32_t)(WUFFS_SOFTFLOAT__CLZ8[(v_x >> 56u)])) & 7u)) & 63u); } +// -------- func softfloat.arith.clz32 + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_softfloat__arith__clz32( + const wuffs_softfloat__arith* self, + uint32_t a_a) { + uint32_t v_x = 0; + uint32_t v_n16 = 0; + uint32_t v_n8 = 0; + + v_x = a_a; + if ((v_x >> 16u) == 0u) { + v_x = ((uint32_t)(v_x << 16u)); + v_n16 = 16u; + } + if ((v_x >> 24u) == 0u) { + v_x = ((uint32_t)(v_x << 8u)); + v_n8 = 8u; + } + return ((v_n16 + v_n8 + (((uint32_t)(WUFFS_SOFTFLOAT__CLZ8[(v_x >> 24u)])) & 7u)) & 31u); +} + // -------- func softfloat.arith.shift_right_jam64 WUFFS_BASE__GENERATED_C_CODE @@ -41816,6 +41894,17 @@ wuffs_softfloat__arith__f64_add( return 0; } + uint64_t v_z = 0; + + if ((((uint64_t)(((a_a >> 52u) & 2047u) - 1u)) < 2046u) && (((uint64_t)(((a_b >> 52u) & 2047u) - 1u)) < 2046u)) { + if (((a_a ^ a_b) >> 63u) == 0u) { + return wuffs_softfloat__arith__f64_add_normal_mags(self, a_a, a_b); + } + v_z = wuffs_softfloat__arith__f64_sub_normal_mags(self, a_a, a_b); + if (v_z != 18446744073709551615u) { + return v_z; + } + } if (((a_a ^ a_b) >> 63u) == 0u) { return wuffs_softfloat__arith__f64_add_mags(self, a_a, a_b, (a_a & 9223372036854775808u)); } @@ -41838,12 +41927,152 @@ wuffs_softfloat__arith__f64_sub( return 0; } + uint64_t v_z = 0; + + if ((((uint64_t)(((a_a >> 52u) & 2047u) - 1u)) < 2046u) && (((uint64_t)(((a_b >> 52u) & 2047u) - 1u)) < 2046u)) { + if (((a_a ^ a_b) >> 63u) != 0u) { + return wuffs_softfloat__arith__f64_add_normal_mags(self, a_a, a_b); + } + v_z = wuffs_softfloat__arith__f64_sub_normal_mags(self, a_a, a_b); + if (v_z != 18446744073709551615u) { + return v_z; + } + } if (((a_a ^ a_b) >> 63u) == 0u) { return wuffs_softfloat__arith__f64_sub_mags(self, a_a, a_b, (a_a & 9223372036854775808u)); } return wuffs_softfloat__arith__f64_add_mags(self, a_a, a_b, (a_a & 9223372036854775808u)); } +// -------- func softfloat.arith.f64_add_normal_mags + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_softfloat__arith__f64_add_normal_mags( + const wuffs_softfloat__arith* self, + uint64_t a_a, + uint64_t a_b) { + uint32_t v_ea = 0; + uint32_t v_eb = 0; + uint64_t v_ma = 0; + uint64_t v_mb = 0; + uint32_t v_d = 0; + uint64_t v_z = 0; + uint64_t v_rb = 0; + + v_ea = ((uint32_t)(((a_a >> 52u) & 2047u))); + v_eb = ((uint32_t)(((a_b >> 52u) & 2047u))); + v_ma = (((a_a & 4503599627370495u) | 4503599627370496u) << 9u); + v_mb = (((a_b & 4503599627370495u) | 4503599627370496u) << 9u); + if (v_ea < v_eb) { + v_d = (((uint32_t)(v_eb - v_ea)) & 2047u); + v_ea = v_eb; + v_z = v_ma; + v_ma = v_mb; + v_mb = v_z; + } else { + v_d = (((uint32_t)(v_ea - v_eb)) & 2047u); + } + if (v_d >= 63u) { + v_mb = 1u; + } else if (v_d > 0u) { + if (((uint64_t)(v_mb << (64u - v_d))) != 0u) { + v_mb = ((v_mb >> v_d) | 1u); + } else { + v_mb = (v_mb >> v_d); + } + } + v_z = (v_ma + v_mb); + if (v_z >= 4611686018427387904u) { + v_ea += 1u; + v_z = ((v_z >> 1u) | (v_z & 1u)); + } + v_rb = (v_z & 511u); + v_z = (((uint64_t)(v_z + 256u)) >> 9u); + if (v_rb == 256u) { + v_z &= 18446744073709551614u; + } + v_z = ((uint64_t)(((uint64_t)(((uint64_t)(((uint32_t)(v_ea - 1u)))) << 52u)) + v_z)); + if (v_z >= 9218868437227405312u) { + v_z = 9218868437227405312u; + } + return ((a_a & 9223372036854775808u) | v_z); +} + +// -------- func softfloat.arith.f64_sub_normal_mags + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_softfloat__arith__f64_sub_normal_mags( + const wuffs_softfloat__arith* self, + uint64_t a_a, + uint64_t a_b) { + uint64_t v_sign = 0; + uint32_t v_ea = 0; + uint32_t v_eb = 0; + uint64_t v_fa = 0; + uint64_t v_fb = 0; + uint64_t v_ma = 0; + uint64_t v_mb = 0; + uint32_t v_d = 0; + uint32_t v_s = 0; + uint64_t v_z = 0; + uint64_t v_rb = 0; + + v_sign = (a_a & 9223372036854775808u); + v_ea = ((uint32_t)(((a_a >> 52u) & 2047u))); + v_eb = ((uint32_t)(((a_b >> 52u) & 2047u))); + v_fa = (a_a & 4503599627370495u); + v_fb = (a_b & 4503599627370495u); + if (v_ea == v_eb) { + if (v_fa == v_fb) { + return 0u; + } else if (v_fa < v_fb) { + v_sign ^= 9223372036854775808u; + v_z = ((uint64_t)(v_fb - v_fa)); + } else { + v_z = ((uint64_t)(v_fa - v_fb)); + } + v_s = (((uint32_t)(wuffs_softfloat__arith__clz64(self, v_z) - 11u)) & 63u); + if (v_s >= v_ea) { + return 18446744073709551615u; + } + return (v_sign | ((uint64_t)(((uint64_t)(((uint64_t)(((uint32_t)(((uint32_t)(v_ea - v_s)) - 1u)))) << 52u)) + ((uint64_t)(v_z << v_s))))); + } + v_ma = ((v_fa | 4503599627370496u) << 10u); + v_mb = ((v_fb | 4503599627370496u) << 10u); + if (v_ea < v_eb) { + v_sign ^= 9223372036854775808u; + v_d = (((uint32_t)(v_eb - v_ea)) & 2047u); + v_ea = v_eb; + v_z = v_ma; + v_ma = v_mb; + v_mb = v_z; + } else { + v_d = (((uint32_t)(v_ea - v_eb)) & 2047u); + } + if (v_d >= 63u) { + v_mb = 1u; + } else if (((uint64_t)(v_mb << ((64u - v_d) & 63u))) != 0u) { + v_mb = ((v_mb >> v_d) | 1u); + } else { + v_mb = (v_mb >> v_d); + } + v_z = ((uint64_t)(v_ma - v_mb)); + v_s = (((uint32_t)(wuffs_softfloat__arith__clz64(self, v_z) - 1u)) & 63u); + if (v_s >= v_ea) { + return 18446744073709551615u; + } + v_ea = (((uint32_t)(v_ea - v_s)) & 2047u); + v_z = ((uint64_t)(v_z << v_s)); + v_rb = (v_z & 1023u); + v_z = (((uint64_t)(v_z + 512u)) >> 10u); + if (v_rb == 512u) { + v_z &= 18446744073709551614u; + } + return (v_sign | ((uint64_t)(((uint64_t)(((uint64_t)(((uint32_t)(v_ea - 1u)))) << 52u)) + v_z))); +} + // -------- func softfloat.arith.f64_add_mags WUFFS_BASE__GENERATED_C_CODE @@ -42593,7 +42822,18 @@ wuffs_softfloat__arith__f32_add( return 0; } - return wuffs_softfloat__arith__f64_to_f32(self, wuffs_softfloat__arith__f64_add(self, wuffs_softfloat__arith__f32_to_f64(self, a_a), wuffs_softfloat__arith__f32_to_f64(self, a_b))); + uint32_t v_z = 0; + + if ((((uint32_t)(((a_a >> 23u) & 255u) - 1u)) < 254u) && (((uint32_t)(((a_b >> 23u) & 255u) - 1u)) < 254u)) { + if (((a_a ^ a_b) >> 31u) == 0u) { + return wuffs_softfloat__arith__f32_add_mags(self, a_a, a_b); + } + v_z = wuffs_softfloat__arith__f32_sub_mags(self, a_a, a_b); + if (v_z != 4294967295u) { + return v_z; + } + } + return wuffs_softfloat__arith__slow_f32_add(self, a_a, a_b); } // -------- func softfloat.arith.f32_sub @@ -42612,7 +42852,18 @@ wuffs_softfloat__arith__f32_sub( return 0; } - return wuffs_softfloat__arith__f64_to_f32(self, wuffs_softfloat__arith__f64_sub(self, wuffs_softfloat__arith__f32_to_f64(self, a_a), wuffs_softfloat__arith__f32_to_f64(self, a_b))); + uint32_t v_z = 0; + + if ((((uint32_t)(((a_a >> 23u) & 255u) - 1u)) < 254u) && (((uint32_t)(((a_b >> 23u) & 255u) - 1u)) < 254u)) { + if (((a_a ^ a_b) >> 31u) != 0u) { + return wuffs_softfloat__arith__f32_add_mags(self, a_a, a_b); + } + v_z = wuffs_softfloat__arith__f32_sub_mags(self, a_a, a_b); + if (v_z != 4294967295u) { + return v_z; + } + } + return wuffs_softfloat__arith__slow_f32_sub(self, a_a, a_b); } // -------- func softfloat.arith.f32_mul @@ -42631,6 +42882,202 @@ wuffs_softfloat__arith__f32_mul( return 0; } + uint32_t v_ea = 0; + uint32_t v_eb = 0; + uint32_t v_e = 0; + uint64_t v_p = 0; + uint32_t v_m = 0; + uint32_t v_r = 0; + uint32_t v_z = 0; + + v_ea = ((a_a >> 23u) & 255u); + v_eb = ((a_b >> 23u) & 255u); + if ((((uint32_t)(v_ea - 1u)) < 254u) && (((uint32_t)(v_eb - 1u)) < 254u)) { + v_p = (((uint64_t)(((a_a & 8388607u) | 8388608u))) * ((uint64_t)(((a_b & 8388607u) | 8388608u)))); + v_e = (v_ea + v_eb); + if (v_p >= 140737488355328u) { + v_e += 1u; + v_m = ((uint32_t)(((v_p >> 24u) & 16777215u))); + v_r = ((uint32_t)(((uint32_t)((v_p & 16777215u))) - 8388608u)); + } else { + v_m = ((uint32_t)(((v_p >> 23u) & 16777215u))); + v_r = ((uint32_t)(((uint32_t)((v_p & 8388607u))) - 4194304u)); + } + if ((v_r != 0u) && (v_r < 2147483648u)) { + v_m += 1u; + } else if (v_r == 0u) { + v_m += (v_m & 1u); + } + if ((v_e >= 128u) && (v_e <= 381u)) { + v_z = ((uint32_t)(((v_e - 128u) << 23u) + v_m)); + if (v_z >= 2139095040u) { + v_z = 2139095040u; + } + return (((a_a ^ a_b) & 2147483648u) | v_z); + } + } + return wuffs_softfloat__arith__slow_f32_mul(self, a_a, a_b); +} + +// -------- func softfloat.arith.f32_add_mags + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_softfloat__arith__f32_add_mags( + const wuffs_softfloat__arith* self, + uint32_t a_a, + uint32_t a_b) { + uint32_t v_ea = 0; + uint32_t v_eb = 0; + uint32_t v_ma = 0; + uint32_t v_mb = 0; + uint32_t v_d = 0; + uint32_t v_z = 0; + uint32_t v_rb = 0; + + v_ea = ((a_a >> 23u) & 255u); + v_eb = ((a_b >> 23u) & 255u); + v_ma = (((a_a & 8388607u) | 8388608u) << 6u); + v_mb = (((a_b & 8388607u) | 8388608u) << 6u); + if (v_ea < v_eb) { + v_d = (((uint32_t)(v_eb - v_ea)) & 255u); + v_ea = v_eb; + v_z = v_ma; + v_ma = v_mb; + v_mb = v_z; + } else { + v_d = (((uint32_t)(v_ea - v_eb)) & 255u); + } + if (v_d >= 31u) { + v_mb = 1u; + } else if (v_d > 0u) { + if (((uint32_t)(v_mb << (32u - v_d))) != 0u) { + v_mb = ((v_mb >> v_d) | 1u); + } else { + v_mb = (v_mb >> v_d); + } + } + v_z = ((v_ma & 1073741823u) + (v_mb & 1073741823u)); + if (v_z >= 1073741824u) { + v_ea += 1u; + v_z = ((v_z >> 1u) | (v_z & 1u)); + } + v_rb = (v_z & 63u); + v_z = (((uint32_t)(v_z + 32u)) >> 6u); + if (v_rb == 32u) { + v_z &= 4294967294u; + } + v_z = ((uint32_t)(((uint32_t)(((uint32_t)(v_ea - 1u)) << 23u)) + v_z)); + if (v_z >= 2139095040u) { + v_z = 2139095040u; + } + return ((a_a & 2147483648u) | v_z); +} + +// -------- func softfloat.arith.f32_sub_mags + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_softfloat__arith__f32_sub_mags( + const wuffs_softfloat__arith* self, + uint32_t a_a, + uint32_t a_b) { + uint32_t v_sign = 0; + uint32_t v_ea = 0; + uint32_t v_eb = 0; + uint32_t v_fa = 0; + uint32_t v_fb = 0; + uint32_t v_ma = 0; + uint32_t v_mb = 0; + uint32_t v_d = 0; + uint32_t v_s = 0; + uint32_t v_z = 0; + uint32_t v_rb = 0; + + v_sign = (a_a & 2147483648u); + v_ea = ((a_a >> 23u) & 255u); + v_eb = ((a_b >> 23u) & 255u); + v_fa = (a_a & 8388607u); + v_fb = (a_b & 8388607u); + if (v_ea == v_eb) { + if (v_fa == v_fb) { + return 0u; + } else if (v_fa < v_fb) { + v_sign ^= 2147483648u; + v_z = ((uint32_t)(v_fb - v_fa)); + } else { + v_z = ((uint32_t)(v_fa - v_fb)); + } + v_s = (((uint32_t)(wuffs_softfloat__arith__clz32(self, v_z) - 8u)) & 31u); + if (v_s >= v_ea) { + return 4294967295u; + } + return (v_sign | ((uint32_t)(((uint32_t)(((uint32_t)(((uint32_t)(v_ea - v_s)) - 1u)) << 23u)) + ((uint32_t)(v_z << v_s))))); + } + v_ma = ((v_fa | 8388608u) << 7u); + v_mb = ((v_fb | 8388608u) << 7u); + if (v_ea < v_eb) { + v_sign ^= 2147483648u; + v_d = (((uint32_t)(v_eb - v_ea)) & 255u); + v_ea = v_eb; + v_z = v_ma; + v_ma = v_mb; + v_mb = v_z; + } else { + v_d = (((uint32_t)(v_ea - v_eb)) & 255u); + } + if (v_d >= 31u) { + v_mb = 1u; + } else if (((uint32_t)(v_mb << ((32u - v_d) & 31u))) != 0u) { + v_mb = ((v_mb >> v_d) | 1u); + } else { + v_mb = (v_mb >> v_d); + } + v_z = ((uint32_t)(v_ma - v_mb)); + v_s = (((uint32_t)(wuffs_softfloat__arith__clz32(self, v_z) - 1u)) & 31u); + if (v_s >= v_ea) { + return 4294967295u; + } + v_ea = (((uint32_t)(v_ea - v_s)) & 255u); + v_z = ((uint32_t)(v_z << v_s)); + v_rb = (v_z & 127u); + v_z = (((uint32_t)(v_z + 64u)) >> 7u); + if (v_rb == 64u) { + v_z &= 4294967294u; + } + return (v_sign | ((uint32_t)(((uint32_t)(((uint32_t)(v_ea - 1u)) << 23u)) + v_z))); +} + +// -------- func softfloat.arith.slow_f32_add + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_softfloat__arith__slow_f32_add( + const wuffs_softfloat__arith* self, + uint32_t a_a, + uint32_t a_b) { + return wuffs_softfloat__arith__f64_to_f32(self, wuffs_softfloat__arith__f64_add(self, wuffs_softfloat__arith__f32_to_f64(self, a_a), wuffs_softfloat__arith__f32_to_f64(self, a_b))); +} + +// -------- func softfloat.arith.slow_f32_sub + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_softfloat__arith__slow_f32_sub( + const wuffs_softfloat__arith* self, + uint32_t a_a, + uint32_t a_b) { + return wuffs_softfloat__arith__f64_to_f32(self, wuffs_softfloat__arith__f64_sub(self, wuffs_softfloat__arith__f32_to_f64(self, a_a), wuffs_softfloat__arith__f32_to_f64(self, a_b))); +} + +// -------- func softfloat.arith.slow_f32_mul + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_softfloat__arith__slow_f32_mul( + const wuffs_softfloat__arith* self, + uint32_t a_a, + uint32_t a_b) { return wuffs_softfloat__arith__f64_to_f32(self, wuffs_softfloat__arith__f64_mul(self, wuffs_softfloat__arith__f32_to_f64(self, a_a), wuffs_softfloat__arith__f32_to_f64(self, a_b))); } diff --git a/script/manual-test-softfloat-sweep.c b/script/manual-test-softfloat-sweep.c new file mode 100644 index 00000000..f7b5cfc5 --- /dev/null +++ b/script/manual-test-softfloat-sweep.c @@ -0,0 +1,571 @@ +// Copyright 2026 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. +// +// SPDX-License-Identifier: Apache-2.0 OR MIT + +// ---------------- + +// manual-test-softfloat-sweep.c systematically checks Wuffs' std/softfloat +// binary32 operations (and the f64 to / from f32 conversions) over every +// combination of operand exponents, signs and mantissa patterns, plus +// exhaustive sweeps of the integer conversions. Each f32 result is compared +// against: +// +// - the C compiler's native (hardware) float arithmetic: the bits must be +// identical, except that a NaN only has to match a NaN, and +// - the "via f64" definition, f64_to_f32(f64_op(f32_to_f64(a), +// f32_to_f64(b))), computed with the package's own (public) f64 functions: +// here the bits must be identical even for NaNs, so that the NaN payloads +// and signs produced by the f32 fast paths' slow path fallbacks are +// exactly those of the via-f64 implementation. +// +// The operands for the binary operations are every (exponent_a, exponent_b) +// pair in [0 ..= 255] x [0 ..= 255], times both signs for each operand, times +// NUM_MANTISSAS x NUM_MANTISSAS mantissa patterns (zero, one, all ones, only +// top / bottom bits, half-way patterns and pseudo-random ones, so that exact +// results, rounding ties, carries, cancellation, subnormal results, overflow +// to infinity and NaN payloads are all covered). +// +// The f64 add, sub, mul and div operations are checked the same way (every +// exponent pair in [0 ..= 2047] x [0 ..= 2047], both signs, NUM_MANTISSAS64 +// x NUM_MANTISSAS64 mantissa patterns) against the hardware, with NaN +// results checked bit-for-bit against the package's documented NaN rule: +// the first NaN operand, quietened, else the default NaN 0xFFF8_0000_0000_0000. +// +// Build it WITHOUT -ffast-math and WITHOUT -mfma / -march=native: +// +// gcc -O2 -std=c99 script/manual-test-softfloat-sweep.c -lm -lpthread +// -o /tmp/sfsweep +// +// (as one command line), then run /tmp/sfsweep (or /tmp/sfsweep OPNAME to run +// only one op: f32_add, f32_sub, f32_mul, f32_div, f64_to_f32, f32_to_f64, +// f32_from_u32, f32_from_i32 or f32_from_u64). It exits with a non-zero +// status if there were any mismatches. + +#define _POSIX_C_SOURCE 200809L + +#include +#include +#include +#include +#include +#include +#include +#include + +#define WUFFS_IMPLEMENTATION +#define WUFFS_CONFIG__STATIC_FUNCTIONS +#define WUFFS_CONFIG__MODULES +#define WUFFS_CONFIG__MODULE__BASE +#define WUFFS_CONFIG__MODULE__SOFTFLOAT +#include "../release/c/wuffs-unsupported-snapshot.c" + +#if defined(__FAST_MATH__) || defined(__FMA__) +#error "Build without -ffast-math and without -mfma / -march=native" +#endif + +#define NUM_THREADS 4 + +static wuffs_softfloat__arith g_sf; + +static float // +f_of(uint32_t u) { + float f; + memcpy(&f, &u, 4); + return f; +} + +static uint32_t // +u_of_f(float f) { + uint32_t u; + memcpy(&u, &f, 4); + return u; +} + +static double // +d_of(uint64_t u) { + double d; + memcpy(&d, &u, 8); + return d; +} + +static uint64_t // +u_of_d(double d) { + uint64_t u; + memcpy(&u, &d, 8); + return u; +} + +static bool // +is_nan32(uint32_t u) { + return (u & 0x7FFFFFFFu) > 0x7F800000u; +} + +static bool // +is_nan64(uint64_t u) { + return (u & 0x7FFFFFFFFFFFFFFFull) > 0x7FF0000000000000ull; +} + +// ---------------- Mantissa patterns. + +#define NUM_MANTISSAS 40 + +static uint32_t g_mantissas[NUM_MANTISSAS]; + +static void // +init_mantissas() { + static const uint32_t fixed[] = { + 0x000000, 0x000001, 0x000002, 0x000003, 0x7FFFFF, 0x7FFFFE, 0x7FFFFD, + 0x400000, 0x400001, 0x3FFFFF, 0x200000, 0x600000, 0x000100, 0x0000FF, + 0x100001, 0x080000, 0x7C0000, 0x000040, 0x00003F, 0x555555, 0x2AAAAA, + 0x7FFF80, 0x000080, 0x400080, 0x3FFF80, + }; + size_t n = sizeof(fixed) / sizeof(fixed[0]); + memcpy(g_mantissas, fixed, sizeof(fixed)); + uint64_t x = 0x9E3779B97F4A7C15ull; + for (size_t i = n; i < NUM_MANTISSAS; i++) { + x ^= x >> 12; + x ^= x << 25; + x ^= x >> 27; + uint32_t r = (uint32_t)((x * 0x2545F4914F6CDD1Dull) >> 41) & 0x7FFFFF; + if (i & 1) { + r &= 0x7FF000 | (r & 0xF); // Sparse. + } + g_mantissas[i] = r; + } +} + +#define NUM_MANTISSAS64 9 + +static uint64_t g_mantissas64[NUM_MANTISSAS64]; + +static void // +init_mantissas64() { + static const uint64_t fixed[] = { + 0x0000000000000ull, 0x0000000000001ull, 0xFFFFFFFFFFFFFull, + 0x8000000000000ull, 0x8000000000001ull, 0x0000000000200ull, + }; + size_t n = sizeof(fixed) / sizeof(fixed[0]); + memcpy(g_mantissas64, fixed, sizeof(fixed)); + uint64_t x = 0x0123456789ABCDEFull; + for (size_t i = n; i < NUM_MANTISSAS64; i++) { + x ^= x >> 12; + x ^= x << 25; + x ^= x >> 27; + uint64_t r = (x * 0x2545F4914F6CDD1Dull) >> 12; + if (i & 1) { + r &= 0xFFF00000000FFull; // Sparse. + } + g_mantissas64[i] = r; + } +} + +// ---------------- Binary ops. + +typedef uint32_t (*wuffs_op)(const wuffs_softfloat__arith*, uint32_t, uint32_t); +typedef uint64_t (*wuffs_op64)(const wuffs_softfloat__arith*, + uint64_t, + uint64_t); + +static float // +hw_add(float a, float b) { + return a + b; +} +static float // +hw_sub(float a, float b) { + return a - b; +} +static float // +hw_mul(float a, float b) { + return a * b; +} +static float // +hw_div(float a, float b) { + return a / b; +} + +static struct { + const char* name; + wuffs_op op; + wuffs_op64 op64; + float (*hw)(float, float); +} g_binops[] = { + {"f32_add", wuffs_softfloat__arith__f32_add, wuffs_softfloat__arith__f64_add, + hw_add}, + {"f32_sub", wuffs_softfloat__arith__f32_sub, wuffs_softfloat__arith__f64_sub, + hw_sub}, + {"f32_mul", wuffs_softfloat__arith__f32_mul, wuffs_softfloat__arith__f64_mul, + hw_mul}, + {"f32_div", wuffs_softfloat__arith__f32_div, wuffs_softfloat__arith__f64_div, + hw_div}, +}; + +static double // +hw_add64(double a, double b) { + return a + b; +} +static double // +hw_sub64(double a, double b) { + return a - b; +} +static double // +hw_mul64(double a, double b) { + return a * b; +} +static double // +hw_div64(double a, double b) { + return a / b; +} + +static struct { + const char* name; + wuffs_op64 op64; + double (*hw)(double, double); +} g_binops64[] = { + {"f64_add", wuffs_softfloat__arith__f64_add, hw_add64}, + {"f64_sub", wuffs_softfloat__arith__f64_sub, hw_sub64}, + {"f64_mul", wuffs_softfloat__arith__f64_mul, hw_mul64}, + {"f64_div", wuffs_softfloat__arith__f64_div, hw_div64}, +}; + +typedef struct { + int which; + uint32_t ea_lo; + uint32_t ea_hi; + uint64_t count; + uint64_t mismatches; +} job; + +static pthread_mutex_t g_print_mutex = PTHREAD_MUTEX_INITIALIZER; + +static void // +report(job* j, const char* name, uint32_t a, uint32_t b, uint32_t have, + uint32_t want_hw, uint32_t want_via) { + j->mismatches++; + if (j->mismatches <= 5) { + pthread_mutex_lock(&g_print_mutex); + printf("MISMATCH %s(0x%08" PRIX32 ", 0x%08" PRIX32 "): have 0x%08" PRIX32 + ", hardware 0x%08" PRIX32 ", via f64 0x%08" PRIX32 "\n", + name, a, b, have, want_hw, want_via); + pthread_mutex_unlock(&g_print_mutex); + } +} + +static void* // +run_binop(void* arg) { + job* j = (job*)arg; + wuffs_op op = g_binops[j->which].op; + wuffs_op64 op64 = g_binops[j->which].op64; + float (*hw)(float, float) = g_binops[j->which].hw; + const char* name = g_binops[j->which].name; + for (uint32_t ea = j->ea_lo; ea < j->ea_hi; ea++) { + for (uint32_t eb = 0; eb < 256; eb++) { + for (uint32_t ma = 0; ma < NUM_MANTISSAS; ma++) { + for (uint32_t mb = 0; mb < NUM_MANTISSAS; mb++) { + for (uint32_t signs = 0; signs < 4; signs++) { + uint32_t a = ((signs & 1) << 31) | (ea << 23) | g_mantissas[ma]; + uint32_t b = ((signs >> 1) << 31) | (eb << 23) | g_mantissas[mb]; + uint32_t have = op(&g_sf, a, b); + uint32_t want_hw = u_of_f(hw(f_of(a), f_of(b))); + uint32_t want_via = wuffs_softfloat__arith__f64_to_f32( + &g_sf, op64(&g_sf, wuffs_softfloat__arith__f32_to_f64(&g_sf, a), + wuffs_softfloat__arith__f32_to_f64(&g_sf, b))); + j->count++; + if (have != want_via) { + report(j, name, a, b, have, want_hw, want_via); + } else if (is_nan32(have) ? !is_nan32(want_hw) + : (have != want_hw)) { + report(j, name, a, b, have, want_hw, want_via); + } + } + } + } + } + } + return NULL; +} + +static void* // +run_binop64(void* arg) { + job* j = (job*)arg; + wuffs_op64 op64 = g_binops64[j->which].op64; + double (*hw)(double, double) = g_binops64[j->which].hw; + const char* name = g_binops64[j->which].name; + for (uint64_t ea = j->ea_lo; ea < j->ea_hi; ea++) { + for (uint64_t eb = 0; eb < 2048; eb++) { + for (uint32_t ma = 0; ma < NUM_MANTISSAS64; ma++) { + for (uint32_t mb = 0; mb < NUM_MANTISSAS64; mb++) { + for (uint64_t signs = 0; signs < 4; signs++) { + uint64_t a = ((signs & 1) << 63) | (ea << 52) | g_mantissas64[ma]; + uint64_t b = ((signs >> 1) << 63) | (eb << 52) | g_mantissas64[mb]; + uint64_t have = op64(&g_sf, a, b); + uint64_t want = u_of_d(hw(d_of(a), d_of(b))); + j->count++; + if (is_nan64(want)) { + want = is_nan64(a) ? (a | 0x0008000000000000ull) + : is_nan64(b) ? (b | 0x0008000000000000ull) + : 0xFFF8000000000000ull; + } + if (have != want) { + j->mismatches++; + if (j->mismatches <= 5) { + pthread_mutex_lock(&g_print_mutex); + printf("MISMATCH %s(0x%016" PRIX64 ", 0x%016" PRIX64 + "): have 0x%016" PRIX64 ", want 0x%016" PRIX64 "\n", + name, a, b, have, want); + pthread_mutex_unlock(&g_print_mutex); + } + } + } + } + } + } + } + return NULL; +} + +// ---------------- Conversions. + +typedef struct { + int which; + uint64_t lo; + uint64_t hi; + uint64_t count; + uint64_t mismatches; +} cjob; + +static void // +creport(cjob* j, const char* name, uint64_t a, uint64_t have, uint64_t want) { + j->mismatches++; + if (j->mismatches <= 5) { + pthread_mutex_lock(&g_print_mutex); + printf("MISMATCH %s(0x%016" PRIX64 "): have 0x%016" PRIX64 + ", want 0x%016" PRIX64 "\n", + name, a, have, want); + pthread_mutex_unlock(&g_print_mutex); + } +} + +// f32_from_u64 operands: (1 << top) plus, for top <= 26, every pattern of +// the bits below it or, for larger top, every pattern of the 13 bits just +// below it (where the rounding happens) and of the 13 bottom bits (sticky). +static uint64_t // +u64_operand(uint64_t i) { + uint32_t top = (uint32_t)(i >> 26) & 63; + uint64_t low = i & 0x3FFFFFF; + uint64_t below = low; + if (top > 26) { + below = ((low >> 13) << (top - 13)) | (low & 0x1FFF); + } + below &= (((uint64_t)1) << top) - 1; + return (((uint64_t)1) << top) | below; +} + +// f64_to_f32_exponent maps [0 ..= 325] to the f64 biased exponents 0, 1, 2, +// 0x340 ..= 0x47F (the f32 normal and subnormal range, and a little more on +// either side) and 0x7FD, 0x7FE and 0x7FF. +static uint64_t // +f64_to_f32_exponent(uint32_t i) { + if (i < 3) { + return i; + } else if (i < 323) { + return 0x340 + (i - 3); + } + return 0x7FD + (i - 323); +} + +static void* // +run_conv(void* arg) { + cjob* j = (cjob*)arg; + for (uint64_t i = j->lo; i < j->hi; i++) { + j->count++; + switch (j->which) { + case 0: { // f64_to_f32: exponents near the f32 range, many fractions. + uint64_t e = f64_to_f32_exponent((uint32_t)(i >> 20)); + uint64_t f = i & 0xFFFFF; + // 10 bits at the top of the fraction, 10 around the rounding point + // (bit 29) and 6 (repeated) at the bottom. + uint64_t frac = ((f >> 10) << 42) | ((f & 0x3FF) << 24) | + ((f & 0x3F) * 0x41); + for (int s = 0; s < 2; s++) { + uint64_t a = ((uint64_t)s << 63) | (e << 52) | frac; + uint32_t have = wuffs_softfloat__arith__f64_to_f32(&g_sf, a); + uint32_t want = u_of_f((float)d_of(a)); + if (is_nan32(have) ? !is_nan32(want) : (have != want)) { + creport(j, "f64_to_f32", a, have, want); + } else if (is_nan32(have) && + (have != (((uint32_t)(a >> 32) & 0x80000000u) | + 0x7FC00000u | + (uint32_t)((a & 0xFFFFFFFFFFFFFull) >> 29)))) { + creport(j, "f64_to_f32 (NaN payload)", a, have, want); + } + } + break; + } + case 1: { // f32_to_f64: every f32. + uint32_t a = (uint32_t)i; + uint64_t have = wuffs_softfloat__arith__f32_to_f64(&g_sf, a); + uint64_t want = u_of_d((double)f_of(a)); + if (is_nan64(have) ? !is_nan64(want) : (have != want)) { + creport(j, "f32_to_f64", a, have, want); + } else if (is_nan64(have) && + (have != (((uint64_t)(a >> 31) << 63) | + 0x7FF8000000000000ull | + ((uint64_t)(a & 0x7FFFFF) << 29)))) { + creport(j, "f32_to_f64 (NaN payload)", a, have, want); + } + break; + } + case 2: { // f32_from_u32: every u32. + uint32_t a = (uint32_t)i; + uint32_t have = wuffs_softfloat__arith__f32_from_u32(&g_sf, a); + uint32_t want = u_of_f((float)a); + if (have != want) { + creport(j, "f32_from_u32", a, have, want); + } + break; + } + case 3: { // f32_from_i32: every i32. + int32_t a = (int32_t)(uint32_t)i; + uint32_t have = wuffs_softfloat__arith__f32_from_i32(&g_sf, a); + uint32_t want = u_of_f((float)a); + if (have != want) { + creport(j, "f32_from_i32", (uint32_t)a, have, want); + } + break; + } + case 4: { // f32_from_u64. + uint64_t a = u64_operand(i); + uint32_t have = wuffs_softfloat__arith__f32_from_u64(&g_sf, a); + uint32_t want = u_of_f((float)a); + if (have != want) { + creport(j, "f32_from_u64", a, have, want); + } + break; + } + } + } + return NULL; +} + +static const struct { + const char* name; + uint64_t n; +} g_convs[] = { + {"f64_to_f32", ((uint64_t)326) << 20}, + {"f32_to_f64", ((uint64_t)1) << 32}, + {"f32_from_u32", ((uint64_t)1) << 32}, + {"f32_from_i32", ((uint64_t)1) << 32}, + {"f32_from_u64", ((uint64_t)64) << 26}, +}; + +static double // +now() { + struct timespec t; + clock_gettime(CLOCK_MONOTONIC, &t); + return (double)t.tv_sec + (1e-9 * (double)t.tv_nsec); +} + +int // +main(int argc, char** argv) { + const char* only = (argc > 1) ? argv[1] : NULL; + wuffs_base__status status = wuffs_softfloat__arith__initialize( + &g_sf, sizeof g_sf, WUFFS_VERSION, 0); + if (status.repr) { + printf("initialize: %s\n", status.repr); + return 1; + } + init_mantissas(); + init_mantissas64(); + uint64_t total_mismatches = 0; + double t_start = now(); + + for (int w = 0; w < (int)(sizeof(g_binops) / sizeof(g_binops[0])); w++) { + if (only && strcmp(only, g_binops[w].name)) { + continue; + } + double t0 = now(); + job jobs[NUM_THREADS]; + pthread_t threads[NUM_THREADS]; + for (int t = 0; t < NUM_THREADS; t++) { + jobs[t] = (job){w, (uint32_t)(t * 256 / NUM_THREADS), + (uint32_t)((t + 1) * 256 / NUM_THREADS), 0, 0}; + pthread_create(&threads[t], NULL, run_binop, &jobs[t]); + } + uint64_t count = 0, mismatches = 0; + for (int t = 0; t < NUM_THREADS; t++) { + pthread_join(threads[t], NULL); + count += jobs[t].count; + mismatches += jobs[t].mismatches; + } + total_mismatches += mismatches; + printf("%-14s %12" PRIu64 " operand pairs, %8" PRIu64 + " mismatches, %6.1f seconds\n", + g_binops[w].name, count, mismatches, now() - t0); + fflush(stdout); + } + + for (int w = 0; w < (int)(sizeof(g_binops64) / sizeof(g_binops64[0])); + w++) { + if (only && strcmp(only, g_binops64[w].name)) { + continue; + } + double t0 = now(); + job jobs[NUM_THREADS]; + pthread_t threads[NUM_THREADS]; + for (int t = 0; t < NUM_THREADS; t++) { + jobs[t] = (job){w, (uint32_t)(t * 2048 / NUM_THREADS), + (uint32_t)((t + 1) * 2048 / NUM_THREADS), 0, 0}; + pthread_create(&threads[t], NULL, run_binop64, &jobs[t]); + } + uint64_t count = 0, mismatches = 0; + for (int t = 0; t < NUM_THREADS; t++) { + pthread_join(threads[t], NULL); + count += jobs[t].count; + mismatches += jobs[t].mismatches; + } + total_mismatches += mismatches; + printf("%-14s %12" PRIu64 " operand pairs, %8" PRIu64 + " mismatches, %6.1f seconds\n", + g_binops64[w].name, count, mismatches, now() - t0); + fflush(stdout); + } + + for (int w = 0; w < (int)(sizeof(g_convs) / sizeof(g_convs[0])); w++) { + if (only && strcmp(only, g_convs[w].name)) { + continue; + } + double t0 = now(); + cjob jobs[NUM_THREADS]; + pthread_t threads[NUM_THREADS]; + uint64_t n = g_convs[w].n; + for (int t = 0; t < NUM_THREADS; t++) { + jobs[t] = (cjob){w, (n / NUM_THREADS) * (uint64_t)t, + (t == NUM_THREADS - 1) ? n + : (n / NUM_THREADS) * (t + 1), + 0, 0}; + pthread_create(&threads[t], NULL, run_conv, &jobs[t]); + } + uint64_t count = 0, mismatches = 0; + for (int t = 0; t < NUM_THREADS; t++) { + pthread_join(threads[t], NULL); + count += jobs[t].count; + mismatches += jobs[t].mismatches; + } + total_mismatches += mismatches; + printf("%-14s %12" PRIu64 " operands, %8" PRIu64 + " mismatches, %6.1f seconds\n", + g_convs[w].name, count, mismatches, now() - t0); + fflush(stdout); + } + + printf("%s: %" PRIu64 " mismatches in total (%.0f seconds)\n", + total_mismatches ? "FAIL" : "PASS", total_mismatches, + now() - t_start); + return total_mismatches ? 1 : 0; +} diff --git a/std/softfloat/README.md b/std/softfloat/README.md index 9b7d8a22..23c560bd 100644 --- a/std/softfloat/README.md +++ b/std/softfloat/README.md @@ -96,12 +96,24 @@ integer long division, 11 quotient bits per (64-bit integer) division step, with the final remainder as the sticky bit. Leading zero counts use a 256-entry table. -The `f32` add, sub, mul and div operations convert their operands exactly to -`f64`, perform the `f64` operation and then round to `f32`. This double -rounding is innocuous: binary64's 53 bits of precision are more than `(2 * 24) -+ 2`, which suffices for `+`, `-`, `*` and `/` (S. A. Figueroa, "When is double -rounding innocuous?", ACM SIGNUM Newsletter, 1995). The tests confirm this -against hardware, including for subnormal `f32` results. +`f64_add` and `f64_sub` have a fast path for the common case where both +operands are normal (`f64_add_normal_mags` and `f64_sub_normal_mags`): it +skips the special-case handling, aligns the significands with a sticky bit, +adds or subtracts them and rounds inline. Only a subtraction whose result is +subnormal falls back to the general code (`f64_add_mags`, `f64_sub_mags`). + +Likewise, `f32_add`, `f32_sub` and `f32_mul` compute the result directly in +binary32 (with 32-bit significands for add and sub and an exact 48-bit +product for mul) when both operands are normal and the result is normal (or, +for add and mul, overflows to infinity). Otherwise (zeroes, subnormal +operands or results, infinities and NaNs), they, and `f32_div` always, +convert their operands exactly to `f64`, perform the `f64` operation and then +round to `f32`. This double rounding is innocuous: binary64's 53 bits of +precision are more than `(2 * 24) + 2`, which suffices for `+`, `-`, `*` and +`/` (S. A. Figueroa, "When is double rounding innocuous?", ACM SIGNUM +Newsletter, 1995). The tests confirm this against hardware, including for +subnormal `f32` results. The fast paths never produce a NaN, so NaN results +(and their payloads) are exactly those of the "via `f64`" computation. ## log2 @@ -187,3 +199,32 @@ threads): (6 minutes 20 seconds wall clock in total) No exception table is needed. + +`script/manual-test-softfloat-sweep.c` systematically checks the `f32` add, +sub, mul and div operations over every pair of operand exponents (`[0 ..= +255]` squared), both signs and 40 x 40 mantissa patterns (exact results, ties, +carries, cancellation, subnormal results, overflow, NaN payloads): each result +must equal the hardware's (any NaN matching any NaN) and, bit-for-bit +including NaNs, the "via `f64`" composition `f64_to_f32(f64_op(f32_to_f64(a), +f32_to_f64(b)))`. It does the same for the `f64` add, sub, mul and div +operations (every pair of exponents in `[0 ..= 2047]`, 9 x 9 mantissa +patterns, NaN results checked bit-for-bit against the NaN rule above), and +checks `f64_to_f32` (326 exponents times 2²⁰ fraction patterns times both +signs), `f32_to_f64` (every `f32`), `f32_from_u32` and `f32_from_i32` (every +32-bit integer) and `f32_from_u64` (2³² structured values). Result (x86-64, +gcc 13 `-O2`, 4 threads): + + f32_add 419430400 operand pairs, 0 mismatches + f32_sub 419430400 operand pairs, 0 mismatches + f32_mul 419430400 operand pairs, 0 mismatches + f32_div 419430400 operand pairs, 0 mismatches + f64_add 1358954496 operand pairs, 0 mismatches + f64_sub 1358954496 operand pairs, 0 mismatches + f64_mul 1358954496 operand pairs, 0 mismatches + f64_div 1358954496 operand pairs, 0 mismatches + f64_to_f32 341835776 operands, 0 mismatches + f32_to_f64 4294967296 operands, 0 mismatches + f32_from_u32 4294967296 operands, 0 mismatches + f32_from_i32 4294967296 operands, 0 mismatches + f32_from_u64 4294967296 operands, 0 mismatches + PASS: 0 mismatches in total diff --git a/std/softfloat/softfloat.wuffs b/std/softfloat/softfloat.wuffs index aeb9540f..c1392b67 100644 --- a/std/softfloat/softfloat.wuffs +++ b/std/softfloat/softfloat.wuffs @@ -61,6 +61,24 @@ pri func arith.clz64(a: base.u64) base.u32[..= 63] { return (n32 + n16 + n8 + ((CLZ8[x >> 56] as base.u32) & 7)) & 63 } +// clz32 returns the number of leading zero bits of a, which must be non-zero. +pri func arith.clz32(a: base.u32) base.u32[..= 31] { + var x : base.u32 + var n16 : base.u32[..= 16] + var n8 : base.u32[..= 8] + + x = args.a + if (x >> 16) == 0 { + x = x ~mod<< 16 + n16 = 16 + } + if (x >> 24) == 0 { + x = x ~mod<< 8 + n8 = 8 + } + return (n16 + n8 + ((CLZ8[x >> 24] as base.u32) & 7)) & 31 +} + // shift_right_jam64 returns (a >> dist), except that if any non-zero bits // were shifted out then the least significant bit of the result is set. This // "sticky bit" is needed to round correctly. @@ -345,6 +363,18 @@ pub func arith.f64_max(a: base.u64, b: base.u64) base.u64 { // f64_add returns (a + b). pub func arith.f64_add(a: base.u64, b: base.u64) base.u64 { + var z : base.u64 + + if ((((args.a >> 52) & 0x7FF) ~mod- 1) < 0x7FE) and ((((args.b >> 52) & 0x7FF) ~mod- 1) < 0x7FE) { + // Both operands are normal: take the fast path. + if ((args.a ^ args.b) >> 63) == 0 { + return this.f64_add_normal_mags(a: args.a, b: args.b) + } + z = this.f64_sub_normal_mags(a: args.a, b: args.b) + if z <> 0xFFFF_FFFF_FFFF_FFFF { + return z + } + } if ((args.a ^ args.b) >> 63) == 0 { return this.f64_add_mags(a: args.a, b: args.b, sign: args.a & F64_SIGN) } @@ -353,12 +383,161 @@ pub func arith.f64_add(a: base.u64, b: base.u64) base.u64 { // f64_sub returns (a - b). pub func arith.f64_sub(a: base.u64, b: base.u64) base.u64 { + var z : base.u64 + + if ((((args.a >> 52) & 0x7FF) ~mod- 1) < 0x7FE) and ((((args.b >> 52) & 0x7FF) ~mod- 1) < 0x7FE) { + // Both operands are normal: take the fast path. + if ((args.a ^ args.b) >> 63) <> 0 { + return this.f64_add_normal_mags(a: args.a, b: args.b) + } + z = this.f64_sub_normal_mags(a: args.a, b: args.b) + if z <> 0xFFFF_FFFF_FFFF_FFFF { + return z + } + } if ((args.a ^ args.b) >> 63) == 0 { return this.f64_sub_mags(a: args.a, b: args.b, sign: args.a & F64_SIGN) } return this.f64_add_mags(a: args.a, b: args.b, sign: args.a & F64_SIGN) } +// f64_add_normal_mags returns (sign(a) * (|a| + |b|)), for normal a and b. +// It is a faster version of f64_add_mags. +pri func arith.f64_add_normal_mags(a: base.u64, b: base.u64) base.u64 { + var ea : base.u32[..= 0x800] + var eb : base.u32[..= 0x7FF] + var ma : base.u64[..= 0x3FFF_FFFF_FFFF_FE00] + var mb : base.u64[..= 0x3FFF_FFFF_FFFF_FE00] + var d : base.u32[..= 0x7FF] + var z : base.u64 + var rb : base.u64 + + ea = ((args.a >> 52) & 0x7FF) as base.u32 + eb = ((args.b >> 52) & 0x7FF) as base.u32 + // The significands, with their implicit bit at bit 61 and 9 extra bits + // of precision at the bottom. + ma = ((args.a & F64_FRAC) | F64_HIDDEN) << 9 + mb = ((args.b & F64_FRAC) | F64_HIDDEN) << 9 + if ea < eb { + d = (eb ~mod- ea) & 0x7FF + ea = eb + z = ma + ma = mb + mb = z + } else { + d = (ea ~mod- eb) & 0x7FF + } + if d >= 63 { + mb = 1 + } else if d > 0 { + if (mb ~mod<< (64 - d)) <> 0 { + mb = (mb >> d) | 1 + } else { + mb = mb >> d + } + } + z = ma + mb + if z >= 0x4000_0000_0000_0000 { + ea += 1 + z = (z >> 1) | (z & 1) + } + rb = z & 0x1FF + z = (z ~mod+ 0x100) >> 9 + if rb == 0x100 { + z &= 0xFFFF_FFFF_FFFF_FFFE + } + // ea is in [1 ..= 0x7FF] and z is in [1 << 52 ..= 1 << 53]. Adding z + // (whose bit 52 is the implicit bit) carries any rounding overflow into + // the exponent field. + z = (((ea ~mod- 1) as base.u64) ~mod<< 52) ~mod+ z + if z >= F64_INF { + z = F64_INF + } + return (args.a & F64_SIGN) | z +} + +// f64_sub_normal_mags returns (sign(a) * (|a| - |b|)), for normal a and b, +// unless the result is subnormal, in which case it returns +// 0xFFFF_FFFF_FFFF_FFFF (a NaN, which it never otherwise returns) and the +// caller must take its slow path. It is a faster version of f64_sub_mags. +pri func arith.f64_sub_normal_mags(a: base.u64, b: base.u64) base.u64 { + var sign : base.u64 + var ea : base.u32[..= 0x7FF] + var eb : base.u32[..= 0x7FF] + var fa : base.u64[..= 0x000F_FFFF_FFFF_FFFF] + var fb : base.u64[..= 0x000F_FFFF_FFFF_FFFF] + var ma : base.u64 + var mb : base.u64 + var d : base.u32[..= 0x7FF] + var s : base.u32[..= 63] + var z : base.u64 + var rb : base.u64 + + sign = args.a & F64_SIGN + ea = ((args.a >> 52) & 0x7FF) as base.u32 + eb = ((args.b >> 52) & 0x7FF) as base.u32 + fa = args.a & F64_FRAC + fb = args.b & F64_FRAC + + if ea == eb { + // The difference is exact. + if fa == fb { + // Exact cancellation gives +0 when rounding to nearest. + return 0 + } else if fa < fb { + sign ^= F64_SIGN + z = fb ~mod- fa + } else { + z = fa ~mod- fb + } + // z is in [1 ..= F64_FRAC]. Shift its top bit to bit 52. + s = (this.clz64(a: z) ~mod- 11) & 63 + if s >= ea { + return 0xFFFF_FFFF_FFFF_FFFF + } + return sign | (((((ea ~mod- s) ~mod- 1) as base.u64) ~mod<< 52) ~mod+ (z ~mod<< s)) + } + + // The significands, with their implicit bit at bit 62 and 10 extra bits + // of precision at the bottom. + ma = (fa | F64_HIDDEN) << 10 + mb = (fb | F64_HIDDEN) << 10 + if ea < eb { + sign ^= F64_SIGN + d = (eb ~mod- ea) & 0x7FF + ea = eb + z = ma + ma = mb + mb = z + } else { + d = (ea ~mod- eb) & 0x7FF + } + if d >= 63 { + mb = 1 + } else if (mb ~mod<< ((64 - d) & 63)) <> 0 { + mb = (mb >> d) | 1 + } else { + mb = mb >> d + } + // ma > mb, so z is non-zero. If (d >= 2) then z >= (1 << 61) and the + // normalization shift is at most 1. Otherwise, no bits were shifted out. + z = ma ~mod- mb + s = (this.clz64(a: z) ~mod- 1) & 63 + if s >= ea { + return 0xFFFF_FFFF_FFFF_FFFF + } + ea = (ea ~mod- s) & 0x7FF + z = z ~mod<< s + rb = z & 0x3FF + z = (z ~mod+ 0x200) >> 10 + if rb == 0x200 { + z &= 0xFFFF_FFFF_FFFF_FFFE + } + // ea is in [1 ..= 0x7FE] and z is in [1 << 52 ..= 1 << 53], so this does + // not overflow. + return sign | ((((ea ~mod- 1) as base.u64) ~mod<< 52) ~mod+ z) +} + // f64_add_mags returns (sign * (|a| + |b|)). pri func arith.f64_add_mags(a: base.u64, b: base.u64, sign: base.u64) base.u64 { var exp_a : base.u32 diff --git a/std/softfloat/softfloat_f32.wuffs b/std/softfloat/softfloat_f32.wuffs index f604d093..ce616e11 100644 --- a/std/softfloat/softfloat_f32.wuffs +++ b/std/softfloat/softfloat_f32.wuffs @@ -10,12 +10,18 @@ // -------- f32 (binary32) operations. // -// The f32 add, sub, mul and div functions convert their (exactly -// representable) operands to f64, perform the f64 operation and round the -// result to f32. Double rounding is innocuous here: binary64 has more than -// (2 * 24) + 2 bits of precision, which suffices for +, -, * and / (Figueroa, -// "When is double rounding innocuous?", 1995). This is also exhaustively -// cross-checked against hardware by the tests. +// When both operands are normal (and, for sub_mags and mul, when the result +// is normal too), the f32 add, sub and mul functions compute the result +// directly in binary32 (with 32-bit or 64-bit integer significands). This is +// the common case. Otherwise (zeroes, subnormals, infinities, NaNs and +// subnormal results), they and f32_div convert their (exactly representable) +// operands to f64, perform the f64 operation and round the result to f32 +// (the slow_etc functions). Double rounding is innocuous here: binary64 has +// more than (2 * 24) + 2 bits of precision, which suffices for +, -, * and / +// (Figueroa, "When is double rounding innocuous?", 1995). This is also +// exhaustively cross-checked against hardware by the tests. The fast paths +// never return a NaN, so NaN results (and their payloads) always come from +// the slow paths. pri const F32_SIGN : base.u32 = 0x8000_0000 pri const F32_ABS : base.u32 = 0x7FFF_FFFF @@ -74,20 +80,232 @@ pub func arith.f32_abs(a: base.u32) base.u32 { // f32_add returns (a + b). pub func arith.f32_add(a: base.u32, b: base.u32) base.u32 { + var z : base.u32 + + if ((((args.a >> 23) & 0xFF) ~mod- 1) < 0xFE) and ((((args.b >> 23) & 0xFF) ~mod- 1) < 0xFE) { + if ((args.a ^ args.b) >> 31) == 0 { + return this.f32_add_mags(a: args.a, b: args.b) + } + z = this.f32_sub_mags(a: args.a, b: args.b) + if z <> 0xFFFF_FFFF { + return z + } + } + return this.slow_f32_add(a: args.a, b: args.b) +} + +// f32_sub returns (a - b). +pub func arith.f32_sub(a: base.u32, b: base.u32) base.u32 { + var z : base.u32 + + if ((((args.a >> 23) & 0xFF) ~mod- 1) < 0xFE) and ((((args.b >> 23) & 0xFF) ~mod- 1) < 0xFE) { + if ((args.a ^ args.b) >> 31) <> 0 { + return this.f32_add_mags(a: args.a, b: args.b) + } + z = this.f32_sub_mags(a: args.a, b: args.b) + if z <> 0xFFFF_FFFF { + return z + } + } + return this.slow_f32_sub(a: args.a, b: args.b) +} + +// f32_mul returns (a * b). +pub func arith.f32_mul(a: base.u32, b: base.u32) base.u32 { + var ea : base.u32[..= 0xFF] + var eb : base.u32[..= 0xFF] + var e : base.u32[..= 0x1FF] + var p : base.u64 + var m : base.u32 + var r : base.u32 + var z : base.u32 + + ea = (args.a >> 23) & 0xFF + eb = (args.b >> 23) & 0xFF + if ((ea ~mod- 1) < 0xFE) and ((eb ~mod- 1) < 0xFE) { + // Both operands are normal. The exact product of the two 24-bit + // significands, p, is in [1 << 46, 1 << 48). The biased exponent of + // the result, e, is (ea + eb - 127) (plus 1 if p >= (1 << 47)). + p = (((args.a & 0x7F_FFFF) | 0x80_0000) as base.u64) * + (((args.b & 0x7F_FFFF) | 0x80_0000) as base.u64) + e = ea + eb + if p >= 0x8000_0000_0000 { + e += 1 + m = ((p >> 24) & 0xFF_FFFF) as base.u32 + r = ((p & 0xFF_FFFF) as base.u32) ~mod- 0x80_0000 + } else { + m = ((p >> 23) & 0xFF_FFFF) as base.u32 + r = ((p & 0x7F_FFFF) as base.u32) ~mod- 0x40_0000 + } + // r is the remainder minus half an ULP, as a two's complement + // integer. Round to nearest, ties to even. + if (r <> 0) and (r < 0x8000_0000) { + m ~mod+= 1 + } else if r == 0 { + m ~mod+= m & 1 + } + if (e >= 128) and (e <= 381) { + // Adding m (whose bit 23 is the implicit bit) carries any + // rounding overflow into the exponent field. + z = ((e - 128) << 23) ~mod+ m + if z >= F32_INF { + z = F32_INF + } + return ((args.a ^ args.b) & F32_SIGN) | z + } + } + return this.slow_f32_mul(a: args.a, b: args.b) +} + +// f32_add_mags returns (sign(a) * (|a| + |b|)), for normal a and b. +pri func arith.f32_add_mags(a: base.u32, b: base.u32) base.u32 { + var ea : base.u32[..= 0x100] + var eb : base.u32[..= 0xFF] + var ma : base.u32[..= 0x3FFF_FFC0] + var mb : base.u32[..= 0x3FFF_FFC0] + var d : base.u32[..= 0xFF] + var z : base.u32 + var rb : base.u32 + + ea = (args.a >> 23) & 0xFF + eb = (args.b >> 23) & 0xFF + // The significands, with their implicit bit at bit 29 and 6 extra bits + // of precision at the bottom. + ma = ((args.a & 0x7F_FFFF) | 0x80_0000) << 6 + mb = ((args.b & 0x7F_FFFF) | 0x80_0000) << 6 + if ea < eb { + d = (eb ~mod- ea) & 0xFF + ea = eb + z = ma + ma = mb + mb = z + } else { + d = (ea ~mod- eb) & 0xFF + } + if d >= 31 { + mb = 1 + } else if d > 0 { + if (mb ~mod<< (32 - d)) <> 0 { + mb = (mb >> d) | 1 + } else { + mb = mb >> d + } + } + z = (ma & 0x3FFF_FFFF) + (mb & 0x3FFF_FFFF) + if z >= 0x4000_0000 { + ea += 1 + z = (z >> 1) | (z & 1) + } + rb = z & 0x3F + z = (z ~mod+ 0x20) >> 6 + if rb == 0x20 { + z &= 0xFFFF_FFFE + } + // ea is in [1 ..= 255] and z is in [1 << 23 ..= 1 << 24]. + z = ((ea ~mod- 1) ~mod<< 23) ~mod+ z + if z >= F32_INF { + z = F32_INF + } + return (args.a & F32_SIGN) | z +} + +// f32_sub_mags returns (sign(a) * (|a| - |b|)), for normal a and b, unless +// the result is subnormal, in which case it returns 0xFFFF_FFFF (a NaN, which +// it never otherwise returns) and the caller must take its slow path. +pri func arith.f32_sub_mags(a: base.u32, b: base.u32) base.u32 { + var sign : base.u32 + var ea : base.u32[..= 0xFF] + var eb : base.u32[..= 0xFF] + var fa : base.u32[..= 0x7F_FFFF] + var fb : base.u32[..= 0x7F_FFFF] + var ma : base.u32 + var mb : base.u32 + var d : base.u32[..= 0xFF] + var s : base.u32[..= 31] + var z : base.u32 + var rb : base.u32 + + sign = args.a & F32_SIGN + ea = (args.a >> 23) & 0xFF + eb = (args.b >> 23) & 0xFF + fa = args.a & 0x7F_FFFF + fb = args.b & 0x7F_FFFF + + if ea == eb { + // The difference is exact. + if fa == fb { + // Exact cancellation gives +0 when rounding to nearest. + return 0 + } else if fa < fb { + sign ^= F32_SIGN + z = fb ~mod- fa + } else { + z = fa ~mod- fb + } + // z is in [1 ..= 0x7F_FFFF]. Shift its top bit to bit 23. + s = (this.clz32(a: z) ~mod- 8) & 31 + if s >= ea { + return 0xFFFF_FFFF + } + return sign | ((((ea ~mod- s) ~mod- 1) ~mod<< 23) ~mod+ (z ~mod<< s)) + } + + // The significands, with their implicit bit at bit 30 and 7 extra bits + // of precision at the bottom. + ma = (fa | 0x80_0000) << 7 + mb = (fb | 0x80_0000) << 7 + if ea < eb { + sign ^= F32_SIGN + d = (eb ~mod- ea) & 0xFF + ea = eb + z = ma + ma = mb + mb = z + } else { + d = (ea ~mod- eb) & 0xFF + } + if d >= 31 { + mb = 1 + } else if (mb ~mod<< ((32 - d) & 31)) <> 0 { + mb = (mb >> d) | 1 + } else { + mb = mb >> d + } + // ma > mb, so z is non-zero. If (d >= 2) then z >= (1 << 29) and the + // normalization shift is at most 1. Otherwise, no bits were shifted out. + z = ma ~mod- mb + s = (this.clz32(a: z) ~mod- 1) & 31 + if s >= ea { + return 0xFFFF_FFFF + } + ea = (ea ~mod- s) & 0xFF + z = z ~mod<< s + rb = z & 0x7F + z = (z ~mod+ 0x40) >> 7 + if rb == 0x40 { + z &= 0xFFFF_FFFE + } + // ea is in [1 ..= 254] and z is in [1 << 23 ..= 1 << 24], so this does + // not overflow. + return sign | (((ea ~mod- 1) ~mod<< 23) ~mod+ z) +} + +// slow_f32_add returns (a + b), computed via f64. +pri func arith.slow_f32_add(a: base.u32, b: base.u32) base.u32 { return this.f64_to_f32(a: this.f64_add( a: this.f32_to_f64(a: args.a), b: this.f32_to_f64(a: args.b))) } -// f32_sub returns (a - b). -pub func arith.f32_sub(a: base.u32, b: base.u32) base.u32 { +// slow_f32_sub returns (a - b), computed via f64. +pri func arith.slow_f32_sub(a: base.u32, b: base.u32) base.u32 { return this.f64_to_f32(a: this.f64_sub( a: this.f32_to_f64(a: args.a), b: this.f32_to_f64(a: args.b))) } -// f32_mul returns (a * b). -pub func arith.f32_mul(a: base.u32, b: base.u32) base.u32 { +// slow_f32_mul returns (a * b), computed via f64. +pri func arith.slow_f32_mul(a: base.u32, b: base.u32) base.u32 { return this.f64_to_f32(a: this.f64_mul( a: this.f32_to_f64(a: args.a), b: this.f32_to_f64(a: args.b))) From 49356e06e4ba28f296a76e339c7b73ecc4132a62 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 09:57:01 +0000 Subject: [PATCH 2/6] std/brotli: fill the FastLog2 caches lazily Each bit_cost (one in the encoder, one in its metablock_builder) has a 64Ki-entry FastLog2 cache. The metablock_builder filled all of it before building each quality 10 or 11 metablock (about 20 ms per encoder, since each entry is an exact software log2), and the encoder's own cache was never filled, so the quality 10 and 11 literal cost estimates computed one software log2 per input byte. Instead, a bitmap (zeroed when the struct is initialized) records which cache entries are valid, and the new fast_log2_memo fills an entry the first time it is needed. Typically only a few thousand entries are used. bits_entropy, population_cost and their callers become impure (!) to call it. The values are computed by the same functions, so the results are unchanged. On alice29 at quality 10, the encoder executes about 20% fewer instructions, and a 1000-byte input takes 6 ms instead of 21 ms. --- release/c/wuffs-unsupported-snapshot.c | 181 +++++++++++++---------- std/brotli/README.md | 7 +- std/brotli/encode_bit_cost.wuffs | 90 +++++------ std/brotli/encode_block_splitter.wuffs | 19 ++- std/brotli/encode_cluster.wuffs | 14 +- std/brotli/encode_literal_cost.wuffs | 6 +- std/brotli/encode_metablock.wuffs | 13 +- std/brotli/encode_metablock_greedy.wuffs | 13 +- std/brotli/encode_tester.wuffs | 22 ++- 9 files changed, 208 insertions(+), 157 deletions(-) diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 2d724b30..6cfc619f 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -9475,14 +9475,14 @@ wuffs_brotli__encoder__add_dictionary_workbuf_len( WUFFS_BASE__GENERATED_C_CODE WUFFS_BASE__MAYBE_STATIC uint64_t wuffs_brotli__metablock_tester__bits_entropy( - const wuffs_brotli__metablock_tester* self, + wuffs_brotli__metablock_tester* self, wuffs_base__slice_u8 a_population, uint64_t a_size); WUFFS_BASE__GENERATED_C_CODE WUFFS_BASE__MAYBE_STATIC uint64_t wuffs_brotli__metablock_tester__population_cost( - const wuffs_brotli__metablock_tester* self, + wuffs_brotli__metablock_tester* self, wuffs_base__slice_u8 a_histogram, uint32_t a_data_size); @@ -9894,7 +9894,7 @@ struct wuffs_brotli__bit_cost__struct { uint32_t active_coroutine; wuffs_base__vtable null_vtable; - bool f_log2_cache_ready; + uint64_t f_log2_valid[1024]; } private_impl; struct { @@ -10400,14 +10400,14 @@ struct wuffs_brotli__metablock_tester__struct { inline uint64_t bits_entropy( wuffs_base__slice_u8 a_population, - uint64_t a_size) const { + uint64_t a_size) { return wuffs_brotli__metablock_tester__bits_entropy(this, a_population, a_size); } inline uint64_t population_cost( wuffs_base__slice_u8 a_histogram, - uint32_t a_data_size) const { + uint32_t a_data_size) { return wuffs_brotli__metablock_tester__population_cost(this, a_histogram, a_data_size); } @@ -77048,8 +77048,6 @@ WUFFS_BROTLI__DICTIONARY[122784] WUFFS_BASE__POTENTIALLY_UNUSED = { #define WUFFS_BROTLI__F64_1E99 6088095589093318446u -#define WUFFS_BROTLI__LOG2_CACHE_SIZE 65536u - #define WUFFS_BROTLI__LITERAL_BLOCK_SWITCH_COST 4628602664527698330u #define WUFFS_BROTLI__COMMAND_BLOCK_SWITCH_COST 4623789442425946112u @@ -77752,11 +77750,6 @@ wuffs_brotli__encoder__create_backward_references( uint64_t a_num_bytes, uint64_t a_position); -WUFFS_BASE__GENERATED_C_CODE -static wuffs_base__empty_struct -wuffs_brotli__bit_cost__init_log2_cache( - wuffs_brotli__bit_cost* self); - WUFFS_BASE__GENERATED_C_CODE static uint32_t wuffs_brotli__bit_cost__ld32( @@ -77793,17 +77786,23 @@ wuffs_brotli__bit_cost__fast_log2( const wuffs_brotli__bit_cost* self, uint64_t a_v); +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__bit_cost__fast_log2_memo( + wuffs_brotli__bit_cost* self, + uint64_t a_v); + WUFFS_BASE__GENERATED_C_CODE static uint64_t wuffs_brotli__bit_cost__bits_entropy( - const wuffs_brotli__bit_cost* self, + wuffs_brotli__bit_cost* self, wuffs_base__slice_u8 a_population, uint64_t a_size); WUFFS_BASE__GENERATED_C_CODE static uint64_t wuffs_brotli__bit_cost__population_cost( - const wuffs_brotli__bit_cost* self, + wuffs_brotli__bit_cost* self, wuffs_base__slice_u8 a_histogram, uint32_t a_data_size); @@ -78116,7 +78115,7 @@ wuffs_brotli__encoder__find_match_length_with_limit( WUFFS_BASE__GENERATED_C_CODE static uint64_t wuffs_brotli__metablock_builder__bit_cost( - const wuffs_brotli__metablock_builder* self, + wuffs_brotli__metablock_builder* self, uint64_t a_count); WUFFS_BASE__GENERATED_C_CODE @@ -79785,7 +79784,7 @@ wuffs_brotli__metablock_builder__h_set_bit_cost( WUFFS_BASE__GENERATED_C_CODE static uint64_t wuffs_brotli__metablock_builder__h_population_cost( - const wuffs_brotli__metablock_builder* self, + wuffs_brotli__metablock_builder* self, wuffs_base__slice_u8 a_wb, uint64_t a_h, uint32_t a_ds); @@ -79952,7 +79951,7 @@ wuffs_brotli__encoder__gbs_histo_off( WUFFS_BASE__GENERATED_C_CODE static uint64_t wuffs_brotli__encoder__gbs_entropy( - const wuffs_brotli__encoder* self, + wuffs_brotli__encoder* self, wuffs_base__slice_u8 a_wb, uint64_t a_off, uint64_t a_size); @@ -85284,30 +85283,6 @@ wuffs_brotli__encoder__create_backward_references( return wuffs_base__make_status(NULL); } -// -------- func brotli.bit_cost.init_log2_cache - -WUFFS_BASE__GENERATED_C_CODE -static wuffs_base__empty_struct -wuffs_brotli__bit_cost__init_log2_cache( - wuffs_brotli__bit_cost* self) { - uint64_t v_v = 0; - - if (self->private_impl.f_log2_cache_ready) { - return wuffs_base__make_empty_struct(); - } - v_v = 0u; - while (v_v < 256u) { - self->private_data.f_log2_cache[v_v] = WUFFS_BROTLI__LOG2_TABLE[v_v]; - v_v += 1u; - } - while (v_v < 65536u) { - self->private_data.f_log2_cache[v_v] = wuffs_softfloat__arith__log2_u64(&self->private_data.f_sf, v_v); - v_v += 1u; - } - self->private_impl.f_log2_cache_ready = true; - return wuffs_base__make_empty_struct(); -} - // -------- func brotli.bit_cost.ld32 WUFFS_BASE__GENERATED_C_CODE @@ -85395,23 +85370,44 @@ wuffs_brotli__bit_cost__fast_log2( uint64_t a_v) { if (a_v < 256u) { return WUFFS_BROTLI__LOG2_TABLE[a_v]; - } else if (self->private_impl.f_log2_cache_ready && (a_v < 65536u)) { - return self->private_data.f_log2_cache[a_v]; + } else if (a_v < 65536u) { + if (((self->private_impl.f_log2_valid[(a_v >> 6u)] >> (a_v & 63u)) & 1u) != 0u) { + return self->private_data.f_log2_cache[a_v]; + } } return wuffs_softfloat__arith__log2_u64(&self->private_data.f_sf, a_v); } +// -------- func brotli.bit_cost.fast_log2_memo + +WUFFS_BASE__GENERATED_C_CODE +static uint64_t +wuffs_brotli__bit_cost__fast_log2_memo( + wuffs_brotli__bit_cost* self, + uint64_t a_v) { + if (a_v < 256u) { + return WUFFS_BROTLI__LOG2_TABLE[a_v]; + } else if (a_v >= 65536u) { + return wuffs_softfloat__arith__log2_u64(&self->private_data.f_sf, a_v); + } else if (((self->private_impl.f_log2_valid[(a_v >> 6u)] >> (a_v & 63u)) & 1u) == 0u) { + self->private_data.f_log2_cache[a_v] = wuffs_softfloat__arith__log2_u64(&self->private_data.f_sf, a_v); + self->private_impl.f_log2_valid[(a_v >> 6u)] |= (((uint64_t)(1u)) << (a_v & 63u)); + } + return self->private_data.f_log2_cache[a_v]; +} + // -------- func brotli.bit_cost.bits_entropy WUFFS_BASE__GENERATED_C_CODE static uint64_t wuffs_brotli__bit_cost__bits_entropy( - const wuffs_brotli__bit_cost* self, + wuffs_brotli__bit_cost* self, wuffs_base__slice_u8 a_population, uint64_t a_size) { uint64_t v_sum = 0; uint64_t v_retval = 0; uint64_t v_p = 0; + uint64_t v_lp = 0; wuffs_base__slice_u8 v_q = {0}; uint64_t v_n = 0; @@ -85425,10 +85421,12 @@ wuffs_brotli__bit_cost__bits_entropy( v_q = wuffs_base__slice_u8__subslice_i(v_q, 4u); } v_sum += v_p; - v_retval = wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, v_retval, wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_p), wuffs_brotli__bit_cost__fast_log2(self, v_p))); + v_lp = wuffs_brotli__bit_cost__fast_log2_memo(self, v_p); + v_retval = wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, v_retval, wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_p), v_lp)); } if (v_sum != 0u) { - v_retval = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_retval, wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_sum), wuffs_brotli__bit_cost__fast_log2(self, v_sum))); + v_lp = wuffs_brotli__bit_cost__fast_log2_memo(self, v_sum); + v_retval = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_retval, wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_sum), v_lp)); } if (wuffs_softfloat__arith__f64_lt(&self->private_data.f_sf, v_retval, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_sum)) != 0u) { v_retval = wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_sum); @@ -85441,7 +85439,7 @@ wuffs_brotli__bit_cost__bits_entropy( WUFFS_BASE__GENERATED_C_CODE static uint64_t wuffs_brotli__bit_cost__population_cost( - const wuffs_brotli__bit_cost* self, + wuffs_brotli__bit_cost* self, wuffs_base__slice_u8 a_histogram, uint32_t a_data_size) { uint64_t v_total = 0; @@ -85462,6 +85460,7 @@ wuffs_brotli__bit_cost__population_cost( uint32_t v_depth_histo[18] = {0}; uint64_t v_log2total = 0; uint64_t v_log2p = 0; + uint64_t v_lp = 0; uint64_t v_depth = 0; uint32_t v_reps = 0; uint64_t v_sum = 0; @@ -85522,12 +85521,13 @@ wuffs_brotli__bit_cost__population_cost( return wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, 4630404104378646528u, wuffs_softfloat__arith__f64_from_u32(&self->private_data.f_sf, ((uint32_t)(3u * v_h23)))), wuffs_softfloat__arith__f64_from_u32(&self->private_data.f_sf, ((uint32_t)(2u * ((uint32_t)(v_histo[0u] + v_histo[1u])))))), wuffs_softfloat__arith__f64_from_u32(&self->private_data.f_sf, v_histomax)); } v_max_depth = 1u; - v_log2total = wuffs_brotli__bit_cost__fast_log2(self, v_total); + v_log2total = wuffs_brotli__bit_cost__fast_log2_memo(self, v_total); v_i = 0u; while (v_i < a_data_size) { v_c = wuffs_brotli__bit_cost__ld32(self, a_histogram, (((uint64_t)(v_i)) * 4u)); if (v_c > 0u) { - v_log2p = wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, v_log2total, wuffs_brotli__bit_cost__fast_log2(self, ((uint64_t)(v_c)))); + v_lp = wuffs_brotli__bit_cost__fast_log2_memo(self, ((uint64_t)(v_c))); + v_log2p = wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, v_log2total, v_lp); v_depth = wuffs_softfloat__arith__f64_to_u64_trunc(&self->private_data.f_sf, wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_log2p, 4602678819172646912u)); v_bits = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_bits, wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u32(&self->private_data.f_sf, v_c), v_log2p)); if (v_depth > 15u) { @@ -85569,11 +85569,13 @@ wuffs_brotli__bit_cost__population_cost( while (v_i < 18u) { v_p = ((uint64_t)(v_depth_histo[v_i])); v_sum += v_p; - v_retval = wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, v_retval, wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_p), wuffs_brotli__bit_cost__fast_log2(self, v_p))); + v_lp = wuffs_brotli__bit_cost__fast_log2_memo(self, v_p); + v_retval = wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, v_retval, wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_p), v_lp)); v_i += 1u; } if (v_sum != 0u) { - v_retval = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_retval, wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_sum), wuffs_brotli__bit_cost__fast_log2(self, v_sum))); + v_lp = wuffs_brotli__bit_cost__fast_log2_memo(self, v_sum); + v_retval = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_retval, wuffs_softfloat__arith__f64_mul(&self->private_data.f_sf, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_sum), v_lp)); } if (wuffs_softfloat__arith__f64_lt(&self->private_data.f_sf, v_retval, wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_sum)) != 0u) { v_retval = wuffs_softfloat__arith__f64_from_u64(&self->private_data.f_sf, v_sum); @@ -87178,12 +87180,15 @@ wuffs_brotli__encoder__find_match_length_with_limit( WUFFS_BASE__GENERATED_C_CODE static uint64_t wuffs_brotli__metablock_builder__bit_cost( - const wuffs_brotli__metablock_builder* self, + wuffs_brotli__metablock_builder* self, uint64_t a_count) { + uint64_t v_c = 0; + if (a_count == 0u) { return 13835058055282163712u; } - return wuffs_brotli__bit_cost__fast_log2(&self->private_data.f_bc, a_count); + v_c = wuffs_brotli__bit_cost__fast_log2_memo(&self->private_data.f_bc, a_count); + return v_c; } // -------- func brotli.metablock_builder.histogram_add_vector @@ -87570,7 +87575,7 @@ wuffs_brotli__metablock_builder__find_blocks( wuffs_private_impl__bulk_memset(&self->private_data.f_insert_cost[0], 35200u * (size_t)8u, 0u); v_i = 0u; while (v_i < a_num_histograms) { - self->private_data.f_insert_cost[(v_i & 255u)] = wuffs_brotli__bit_cost__fast_log2(&self->private_data.f_bc, (wuffs_brotli__metablock_builder__h_total(self, a_wb, ((uint64_t)(a_histograms + ((uint64_t)(v_i * v_hs)))), a_ds) & 4294967295u)); + self->private_data.f_insert_cost[(v_i & 255u)] = wuffs_brotli__bit_cost__fast_log2_memo(&self->private_data.f_bc, (wuffs_brotli__metablock_builder__h_total(self, a_wb, ((uint64_t)(a_histograms + ((uint64_t)(v_i * v_hs)))), a_ds) & 4294967295u)); v_i += 1u; } v_i = v_alphabet_size; @@ -87580,7 +87585,8 @@ wuffs_brotli__metablock_builder__find_blocks( while (v_j < a_num_histograms) { v_k = ((uint64_t)(((uint64_t)(v_i * a_num_histograms)) + v_j)); if (v_k < 35200u) { - self->private_data.f_insert_cost[v_k] = wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, self->private_data.f_insert_cost[(v_j & 255u)], wuffs_brotli__metablock_builder__bit_cost(self, ((uint64_t)(wuffs_brotli__metablock_builder__ld32(self, a_wb, ((uint64_t)(((uint64_t)(a_histograms + ((uint64_t)(v_j * v_hs)))) + ((uint64_t)(v_i * 4u))))))))); + v_c = wuffs_brotli__metablock_builder__bit_cost(self, ((uint64_t)(wuffs_brotli__metablock_builder__ld32(self, a_wb, ((uint64_t)(((uint64_t)(a_histograms + ((uint64_t)(v_j * v_hs)))) + ((uint64_t)(v_i * 4u)))))))); + self->private_data.f_insert_cost[v_k] = wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, self->private_data.f_insert_cost[(v_j & 255u)], v_c); } v_j += 1u; } @@ -87750,6 +87756,7 @@ wuffs_brotli__metablock_builder__cluster_blocks( uint64_t a_num_blocks, uint64_t a_block_ids, uint32_t a_ds) { + uint64_t v_c = 0; uint64_t v_hs = 0; uint64_t v_histogram_symbols = 0; uint64_t v_u32s = 0; @@ -87839,11 +87846,12 @@ wuffs_brotli__metablock_builder__cluster_blocks( v_pos += 1u; v_k += 1u; } + v_c = wuffs_brotli__metablock_builder__h_population_cost(self, a_wb, ((uint64_t)(v_histograms + ((uint64_t)(v_j * v_hs)))), a_ds); wuffs_brotli__metablock_builder__h_set_bit_cost(self, a_wb, ((uint64_t)(v_histograms + ((uint64_t)(v_j * v_hs)))), a_ds, - wuffs_brotli__metablock_builder__h_population_cost(self, a_wb, ((uint64_t)(v_histograms + ((uint64_t)(v_j * v_hs)))), a_ds)); + v_c); wuffs_brotli__metablock_builder__st32(self, a_wb, ((uint64_t)(v_new_clusters + ((uint64_t)(v_j * 4u)))), ((uint32_t)(v_j))); wuffs_brotli__metablock_builder__st32(self, a_wb, ((uint64_t)(v_symbols + ((uint64_t)(v_j * 4u)))), ((uint32_t)(v_j))); wuffs_brotli__metablock_builder__st32(self, a_wb, ((uint64_t)(v_sizes + ((uint64_t)(v_j * 4u)))), 1u); @@ -89521,6 +89529,8 @@ wuffs_brotli__metablock_builder__histogram_bit_cost_distance( uint64_t a_histogram, uint64_t a_candidate, uint64_t a_tmp) { + uint64_t v_c = 0; + if (wuffs_brotli__metablock_builder__h_total(self, a_wb, a_histogram, a_ds) == 0u) { return 0u; } @@ -89534,7 +89544,8 @@ wuffs_brotli__metablock_builder__histogram_bit_cost_distance( a_tmp, a_candidate, a_ds); - return wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, wuffs_brotli__metablock_builder__h_population_cost(self, a_wb, a_tmp, a_ds), wuffs_brotli__metablock_builder__h_bit_cost(self, a_wb, a_candidate, a_ds)); + v_c = wuffs_brotli__metablock_builder__h_population_cost(self, a_wb, a_tmp, a_ds); + return wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, v_c, wuffs_brotli__metablock_builder__h_bit_cost(self, a_wb, a_candidate, a_ds)); } // -------- func brotli.metablock_builder.histogram_remap @@ -89698,6 +89709,7 @@ wuffs_brotli__metablock_builder__cluster_histograms( uint64_t a_max_histograms, uint64_t a_out_histos, uint64_t a_histogram_symbols) { + uint64_t v_c = 0; uint64_t v_hs = 0; uint64_t v_mark = 0; uint64_t v_cluster_size = 0; @@ -89735,11 +89747,12 @@ wuffs_brotli__metablock_builder__cluster_histograms( ((uint64_t)(a_out_histos + ((uint64_t)(v_i * v_hs)))), ((uint64_t)(a_in_histos + ((uint64_t)(v_i * v_hs)))), a_ds); + v_c = wuffs_brotli__metablock_builder__h_population_cost(self, a_wb, ((uint64_t)(a_in_histos + ((uint64_t)(v_i * v_hs)))), a_ds); wuffs_brotli__metablock_builder__h_set_bit_cost(self, a_wb, ((uint64_t)(a_out_histos + ((uint64_t)(v_i * v_hs)))), a_ds, - wuffs_brotli__metablock_builder__h_population_cost(self, a_wb, ((uint64_t)(a_in_histos + ((uint64_t)(v_i * v_hs)))), a_ds)); + v_c); wuffs_brotli__metablock_builder__st32(self, a_wb, ((uint64_t)(a_histogram_symbols + ((uint64_t)(v_i * 4u)))), ((uint32_t)(v_i))); v_i += 1u; } @@ -96423,7 +96436,8 @@ wuffs_brotli__encoder__estimate_bit_costs_for_literals_utf8( } else if (v_utf8_pos == 2u) { v_iw = v_iwu2; } - v_lit_cost = wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, wuffs_brotli__bit_cost__fast_log2(&self->private_data.f_bc, v_iw), wuffs_brotli__bit_cost__fast_log2(&self->private_data.f_bc, v_histo)); + v_lit_cost = wuffs_brotli__bit_cost__fast_log2_memo(&self->private_data.f_bc, v_iw); + v_lit_cost = wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, v_lit_cost, wuffs_brotli__bit_cost__fast_log2(&self->private_data.f_bc, v_histo)); v_lit_cost = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_lit_cost, 4584030313835631162u); wuffs_brotli__encoder__lc_finish_cost(self, a_wb, @@ -96494,7 +96508,8 @@ wuffs_brotli__encoder__estimate_bit_costs_for_literals( if (v_histo == 0u) { v_histo = 1u; } - v_lit_cost = wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, wuffs_brotli__bit_cost__fast_log2(&self->private_data.f_bc, v_in_window), wuffs_brotli__bit_cost__fast_log2(&self->private_data.f_bc, v_histo)); + v_lit_cost = wuffs_brotli__bit_cost__fast_log2_memo(&self->private_data.f_bc, v_in_window); + v_lit_cost = wuffs_softfloat__arith__f64_sub(&self->private_data.f_sf, v_lit_cost, wuffs_brotli__bit_cost__fast_log2(&self->private_data.f_bc, v_histo)); v_lit_cost = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, v_lit_cost, 4584015902316823577u); wuffs_brotli__encoder__lc_finish_cost(self, a_wb, @@ -96824,12 +96839,15 @@ wuffs_brotli__metablock_builder__h_set_bit_cost( WUFFS_BASE__GENERATED_C_CODE static uint64_t wuffs_brotli__metablock_builder__h_population_cost( - const wuffs_brotli__metablock_builder* self, + wuffs_brotli__metablock_builder* self, wuffs_base__slice_u8 a_wb, uint64_t a_h, uint32_t a_ds) { + uint64_t v_c = 0; + if (a_h <= ((uint64_t)(a_wb.len))) { - return wuffs_brotli__bit_cost__population_cost(&self->private_data.f_bc, wuffs_base__slice_u8__subslice_i(a_wb, a_h), a_ds); + v_c = wuffs_brotli__bit_cost__population_cost(&self->private_data.f_bc, wuffs_base__slice_u8__subslice_i(a_wb, a_h), a_ds); + return v_c; } return 0u; } @@ -97085,7 +97103,8 @@ wuffs_brotli__metablock_builder__compute_distance_cost( } v_i += 1u; } - self->private_impl.f_dist_cost = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, wuffs_brotli__metablock_builder__h_population_cost(self, a_wb, a_tmp, 544u), v_extra_bits); + self->private_impl.f_dist_cost = wuffs_brotli__metablock_builder__h_population_cost(self, a_wb, a_tmp, 544u); + self->private_impl.f_dist_cost = wuffs_softfloat__arith__f64_add(&self->private_data.f_sf, self->private_impl.f_dist_cost, v_extra_bits); return true; } @@ -97205,7 +97224,6 @@ wuffs_brotli__metablock_builder__build_meta_block( uint32_t v_v = 0; wuffs_brotli__metablock_builder__reset_arena(self, ((uint64_t)(a_wb.len))); - wuffs_brotli__bit_cost__init_log2_cache(&self->private_data.f_bc); v_orig_npostfix = self->private_impl.f_dist_postfix_bits; v_orig_ndirect = self->private_impl.f_dist_num_direct_codes; v_orig_asm = self->private_impl.f_dist_alphabet_size_max; @@ -97754,12 +97772,15 @@ wuffs_brotli__encoder__gbs_histo_off( WUFFS_BASE__GENERATED_C_CODE static uint64_t wuffs_brotli__encoder__gbs_entropy( - const wuffs_brotli__encoder* self, + wuffs_brotli__encoder* self, wuffs_base__slice_u8 a_wb, uint64_t a_off, uint64_t a_size) { + uint64_t v_e = 0; + if (a_off <= ((uint64_t)(a_wb.len))) { - return wuffs_brotli__bit_cost__bits_entropy(&self->private_data.f_bc, wuffs_base__slice_u8__subslice_i(a_wb, a_off), a_size); + v_e = wuffs_brotli__bit_cost__bits_entropy(&self->private_data.f_bc, wuffs_base__slice_u8__subslice_i(a_wb, a_off), a_size); + return v_e; } return 0u; } @@ -99396,18 +99417,20 @@ wuffs_brotli__encoder__hq_extend_var_region( WUFFS_BASE__GENERATED_C_CODE WUFFS_BASE__MAYBE_STATIC uint64_t wuffs_brotli__metablock_tester__bits_entropy( - const wuffs_brotli__metablock_tester* self, + wuffs_brotli__metablock_tester* self, wuffs_base__slice_u8 a_population, uint64_t a_size) { if (!self) { return 0; } - if ((self->private_impl.magic != WUFFS_BASE__MAGIC) && - (self->private_impl.magic != WUFFS_BASE__DISABLED)) { + if (self->private_impl.magic != WUFFS_BASE__MAGIC) { return 0; } - return wuffs_brotli__bit_cost__bits_entropy(&self->private_data.f_bc, a_population, a_size); + uint64_t v_c = 0; + + v_c = wuffs_brotli__bit_cost__bits_entropy(&self->private_data.f_bc, a_population, a_size); + return v_c; } // -------- func brotli.metablock_tester.population_cost @@ -99415,23 +99438,27 @@ wuffs_brotli__metablock_tester__bits_entropy( WUFFS_BASE__GENERATED_C_CODE WUFFS_BASE__MAYBE_STATIC uint64_t wuffs_brotli__metablock_tester__population_cost( - const wuffs_brotli__metablock_tester* self, + wuffs_brotli__metablock_tester* self, wuffs_base__slice_u8 a_histogram, uint32_t a_data_size) { if (!self) { return 0; } - if ((self->private_impl.magic != WUFFS_BASE__MAGIC) && - (self->private_impl.magic != WUFFS_BASE__DISABLED)) { + if (self->private_impl.magic != WUFFS_BASE__MAGIC) { return 0; } + uint64_t v_c = 0; + if (a_data_size == 256u) { - return wuffs_brotli__bit_cost__population_cost(&self->private_data.f_bc, a_histogram, 256u); + v_c = wuffs_brotli__bit_cost__population_cost(&self->private_data.f_bc, a_histogram, 256u); + return v_c; } else if (a_data_size == 704u) { - return wuffs_brotli__bit_cost__population_cost(&self->private_data.f_bc, a_histogram, 704u); + v_c = wuffs_brotli__bit_cost__population_cost(&self->private_data.f_bc, a_histogram, 704u); + return v_c; } else if (a_data_size == 544u) { - return wuffs_brotli__bit_cost__population_cost(&self->private_data.f_bc, a_histogram, 544u); + v_c = wuffs_brotli__bit_cost__population_cost(&self->private_data.f_bc, a_histogram, 544u); + return v_c; } return 0u; } @@ -99481,7 +99508,6 @@ wuffs_brotli__metablock_tester__split_block( wuffs_brotli__metablock_builder__set_params(&self->private_data.f_mb, a_quality, false, false); wuffs_brotli__metablock_builder__reset_arena(&self->private_data.f_mb, ((uint64_t)(a_workbuf.len))); - wuffs_brotli__bit_cost__init_log2_cache(&self->private_data.f_mb.private_data.f_bc); wuffs_brotli__metablock_builder__alloc_block_splits(&self->private_data.f_mb, a_cmds, a_num_commands); if (self->private_data.f_mb.private_impl.f_oom) { return wuffs_base__make_status(wuffs_base__error__bad_workbuf_length); @@ -99535,7 +99561,6 @@ wuffs_brotli__metablock_tester__cluster_histograms( return 18446744073709551615u; } v_hs = wuffs_brotli__metablock_builder__hsize(&self->private_data.f_mb, v_ds); - wuffs_brotli__bit_cost__init_log2_cache(&self->private_data.f_mb.private_data.f_bc); wuffs_brotli__metablock_builder__reset_arena(&self->private_data.f_mb, ((uint64_t)(a_workbuf.len))); v_n = ((uint64_t)(a_in_size * v_hs)); v_in_off = wuffs_brotli__metablock_builder__alloc(&self->private_data.f_mb, v_n); diff --git a/std/brotli/README.md b/std/brotli/README.md index 28ff1e1b..daa973de 100644 --- a/std/brotli/README.md +++ b/std/brotli/README.md @@ -896,7 +896,12 @@ uses it at qualities 10 and 11 (and its smaller helpers at lower qualities). Its fixed-size state (about 0.8 MiB: FindBlocks' cost arrays and a 64Ki-entry FastLog2 cache in its `bit_cost`) lives in the struct; everything that the C -code allocates on the heap lives in a caller-supplied workbuf. +code allocates on the heap lives in a caller-supplied workbuf. The FastLog2 +cache (like the encoder's own `bit_cost` one) is filled lazily, one entry at +a time, when `fast_log2_memo` first needs that entry (a bitmap, zeroed when the +struct is initialized, records which entries are valid): typically only a few +thousand of the 65536 entries are ever used, and each one costs a (slow, exact) +software `log2`. Data formats (all little-endian, matching the C structs' memory layout on a 64-bit little-endian machine, so that results can be compared with `memcmp`): diff --git a/std/brotli/encode_bit_cost.wuffs b/std/brotli/encode_bit_cost.wuffs index f000465c..81b157d4 100644 --- a/std/brotli/encode_bit_cost.wuffs +++ b/std/brotli/encode_bit_cost.wuffs @@ -12,8 +12,8 @@ // This file mirrors bit_cost.c, bit_cost_inc.h and fast_log.h. The functions // are methods of the bit_cost struct (whose only state is the stateless -// softfloat arith), so that other structs (the encoder, the -// metablock_builder) can embed one with a single field. +// softfloat arith and a lazily filled FastLog2 cache), so that other structs +// (the encoder, the metablock_builder) can embed one with a single field. // // Doubles are IEEE 754 binary64 bit patterns (base.u64). A uint32_t array // (e.g. a histogram's data_) is a slice of little-endian bytes, because @@ -48,43 +48,23 @@ pri const F64_HUGE_VAL : base.u64 = 0x7FF0_0000_0000_0000 // F64_1E99 is the C code's 1e99. pri const F64_1E99 : base.u64 = 0x547D_42AE_A287_9F2E -// LOG2_CACHE_SIZE is the size of the bit_cost.log2_cache table. -pri const LOG2_CACHE_SIZE : base.u64 = 65536 - pri struct bit_cost?( - // log2_cache_ready is whether log2_cache has been filled (by - // init_log2_cache). - log2_cache_ready : base.bool, + // Bit (v & 63) of log2_valid[v >> 6] is whether log2_cache[v] holds + // FastLog2(v). This bitmap is zeroed when the struct is initialized + // (but log2_cache need not be), so the cache starts empty. + log2_valid : array[1024] base.u64, sf : softfloat.arith, ) + ( - // log2_cache[v] is FastLog2(v), for v < LOG2_CACHE_SIZE. It is only a - // speed optimization (glibc's log2 is expensive to emulate): the - // values are computed by the same functions, so the results are - // identical with or without it. + // log2_cache[v] is FastLog2(v), for 256 <= v < 65536, filled lazily + // (see fast_log2_memo). It is only a speed optimization (glibc's log2 + // is expensive to emulate): the values are computed by the same + // functions, so the results are identical with or without it. Few of + // the 65536 entries are typically needed, so filling it lazily is + // much cheaper than filling it all up front. log2_cache : array[65536] base.u64, ) -// init_log2_cache fills the log2_cache table (if it is not already filled). -// Calling it is optional, but it makes fast_log2 faster. -pri func bit_cost.init_log2_cache!() { - var v : base.u64 - - if this.log2_cache_ready { - return nothing - } - v = 0 - while v < 256 { - this.log2_cache[v] = LOG2_TABLE[v] - v += 1 - } - while v < 65536 { - this.log2_cache[v] = this.sf.log2_u64(v: v) - v += 1 - } - this.log2_cache_ready = true -} - // ld32 returns the u32le at s[i ..], or 0 if that is out of bounds (which // the callers never do). pri func bit_cost.ld32(s: roslice base.u8, i: base.u64) base.u32 { @@ -138,23 +118,41 @@ pri func bit_cost.st64!(s: slice base.u8, i: base.u64, v: base.u64) { } // Mirrors FastLog2 in fast_log.h. The result is the bit pattern of an IEEE -// 754 binary64 value. +// 754 binary64 value. It uses (but does not fill) log2_cache: see +// fast_log2_memo. pri func bit_cost.fast_log2(v: base.u64) base.u64 { if args.v < 256 { return LOG2_TABLE[args.v] - } else if this.log2_cache_ready and (args.v < 65536) { - return this.log2_cache[args.v] + } else if args.v < 65536 { + if ((this.log2_valid[args.v >> 6] >> (args.v & 63)) & 1) <> 0 { + return this.log2_cache[args.v] + } } return this.sf.log2_u64(v: args.v) } +// fast_log2_memo is like fast_log2 (it also mirrors FastLog2) but it fills +// the log2_cache entry for v (if v is in the cache's range). +pri func bit_cost.fast_log2_memo!(v: base.u64) base.u64 { + if args.v < 256 { + return LOG2_TABLE[args.v] + } else if args.v >= 65536 { + return this.sf.log2_u64(v: args.v) + } else if ((this.log2_valid[args.v >> 6] >> (args.v & 63)) & 1) == 0 { + this.log2_cache[args.v] = this.sf.log2_u64(v: args.v) + this.log2_valid[args.v >> 6] |= (1 as base.u64) << (args.v & 63) + } + return this.log2_cache[args.v] +} + // Mirrors BrotliBitsEntropy in bit_cost.c. population holds (at least) size // u32le values. The C code's "goto odd_number_of_elements_left" only changes // the loop's unrolling, not the order of the operations. -pri func bit_cost.bits_entropy(population: roslice base.u8, size: base.u64) base.u64 { +pri func bit_cost.bits_entropy!(population: roslice base.u8, size: base.u64) base.u64 { var sum : base.u64 var retval : base.u64 var p : base.u64 + var lp : base.u64 var q : roslice base.u8 var n : base.u64 @@ -168,12 +166,14 @@ pri func bit_cost.bits_entropy(population: roslice base.u8, size: base.u64) base q = q[4 ..] } sum ~mod+= p + lp = this.fast_log2_memo!(v: p) retval = this.sf.f64_sub(a: retval, b: this.sf.f64_mul( - a: this.sf.f64_from_u64(a: p), b: this.fast_log2(v: p))) + a: this.sf.f64_from_u64(a: p), b: lp)) } if sum <> 0 { + lp = this.fast_log2_memo!(v: sum) retval = this.sf.f64_add(a: retval, b: this.sf.f64_mul( - a: this.sf.f64_from_u64(a: sum), b: this.fast_log2(v: sum))) + a: this.sf.f64_from_u64(a: sum), b: lp)) } if this.sf.f64_lt(a: retval, b: this.sf.f64_from_u64(a: sum)) <> 0 { retval = this.sf.f64_from_u64(a: sum) @@ -184,7 +184,7 @@ pri func bit_cost.bits_entropy(population: roslice base.u8, size: base.u64) base // Mirrors BrotliPopulationCostLiteral, BrotliPopulationCostCommand and // BrotliPopulationCostDistance (FN(BrotliPopulationCost) in bit_cost_inc.h). // histogram is a histogram of data_size symbols (see the layout above). -pri func bit_cost.population_cost(histogram: roslice base.u8, data_size: base.u32[..= 704]) base.u64 { +pri func bit_cost.population_cost!(histogram: roslice base.u8, data_size: base.u32[..= 704]) base.u64 { var total : base.u64 var count : base.u32 var s : array[4] base.u32 @@ -203,6 +203,7 @@ pri func bit_cost.population_cost(histogram: roslice base.u8, data_size: base.u3 var depth_histo : array[18] base.u32 var log2total : base.u64 var log2p : base.u64 + var lp : base.u64 var depth : base.u64 var reps : base.u32 var sum : base.u64 @@ -284,12 +285,13 @@ pri func bit_cost.population_cost(histogram: roslice base.u8, data_size: base.u3 // Compute the entropy of the histogram and simultaneously build a // simplified histogram of the code length codes. max_depth = 1 - log2total = this.fast_log2(v: total) + log2total = this.fast_log2_memo!(v: total) i = 0 while i < args.data_size { c = this.ld32(s: args.histogram, i: (i as base.u64) * 4) if c > 0 { - log2p = this.sf.f64_sub(a: log2total, b: this.fast_log2(v: c as base.u64)) + lp = this.fast_log2_memo!(v: c as base.u64) + log2p = this.sf.f64_sub(a: log2total, b: lp) depth = this.sf.f64_to_u64_trunc(a: this.sf.f64_add(a: log2p, b: 0x3FE0_0000_0000_0000)) bits = this.sf.f64_add(a: bits, b: this.sf.f64_mul( a: this.sf.f64_from_u32(a: c), b: log2p)) @@ -340,13 +342,15 @@ pri func bit_cost.population_cost(histogram: roslice base.u8, data_size: base.u3 while i < 18 { p = depth_histo[i] as base.u64 sum ~mod+= p + lp = this.fast_log2_memo!(v: p) retval = this.sf.f64_sub(a: retval, b: this.sf.f64_mul( - a: this.sf.f64_from_u64(a: p), b: this.fast_log2(v: p))) + a: this.sf.f64_from_u64(a: p), b: lp)) i ~mod+= 1 } if sum <> 0 { + lp = this.fast_log2_memo!(v: sum) retval = this.sf.f64_add(a: retval, b: this.sf.f64_mul( - a: this.sf.f64_from_u64(a: sum), b: this.fast_log2(v: sum))) + a: this.sf.f64_from_u64(a: sum), b: lp)) } if this.sf.f64_lt(a: retval, b: this.sf.f64_from_u64(a: sum)) <> 0 { retval = this.sf.f64_from_u64(a: sum) diff --git a/std/brotli/encode_block_splitter.wuffs b/std/brotli/encode_block_splitter.wuffs index e95b44e2..b13173e7 100644 --- a/std/brotli/encode_block_splitter.wuffs +++ b/std/brotli/encode_block_splitter.wuffs @@ -31,12 +31,15 @@ pri const HISTOGRAMS_PER_BATCH : base.u64 = 64 pri const CLUSTERS_PER_BATCH : base.u64 = 16 // Mirrors BitCost in block_splitter.c. -pri func metablock_builder.bit_cost(count: base.u64) base.u64 { +pri func metablock_builder.bit_cost!(count: base.u64) base.u64 { + var c : base.u64 + if args.count == 0 { // -2.0 return 0xC000_0000_0000_0000 } - return this.bc.fast_log2(v: args.count) + c = this.bc.fast_log2_memo!(v: args.count) + return c } // Mirrors FN(HistogramAddVector) in histogram_inc.h, adding the n symbols at @@ -314,7 +317,7 @@ pri func metablock_builder.find_blocks!(wb: slice base.u8, data: base.u64, lengt i = 0 while i < args.num_histograms { // FastLog2((uint32_t)histograms[i].total_count_). - this.insert_cost[i & 255] = this.bc.fast_log2( + this.insert_cost[i & 255] = this.bc.fast_log2_memo!( v: this.h_total(wb: args.wb, h: args.histograms ~mod+ (i ~mod* hs), ds: args.ds) & 0xFFFF_FFFF) i ~mod+= 1 } @@ -326,9 +329,8 @@ pri func metablock_builder.find_blocks!(wb: slice base.u8, data: base.u64, lengt while j < args.num_histograms { k = (i ~mod* args.num_histograms) ~mod+ j if k < 35200 { - this.insert_cost[k] = this.sf.f64_sub( - a: this.insert_cost[j & 255], - b: this.bit_cost(count: this.ld32(wb: args.wb, i: (args.histograms ~mod+ (j ~mod* hs)) ~mod+ (i ~mod* 4)) as base.u64)) + c = this.bit_cost!(count: this.ld32(wb: args.wb, i: (args.histograms ~mod+ (j ~mod* hs)) ~mod+ (i ~mod* 4)) as base.u64) + this.insert_cost[k] = this.sf.f64_sub(a: this.insert_cost[j & 255], b: c) } j ~mod+= 1 } @@ -471,6 +473,7 @@ pri func metablock_builder.build_block_histograms!(wb: slice base.u8, data: base // this.split_etc[cat]. pri func metablock_builder.cluster_blocks!(wb: slice base.u8, cat: base.u32[..= 2], data: base.u64, length: base.u64, width: base.u32, num_blocks: base.u64, block_ids: base.u64, ds: base.u32[..= 704]) { + var c : base.u64 var hs : base.u64 var histogram_symbols : base.u64 var u32s : base.u64 @@ -562,8 +565,8 @@ pri func metablock_builder.cluster_blocks!(wb: slice base.u8, cat: base.u32[..= pos ~mod+= 1 k ~mod+= 1 } - this.h_set_bit_cost!(wb: args.wb, h: histograms ~mod+ (j ~mod* hs), ds: args.ds, - bit_cost: this.h_population_cost(wb: args.wb, h: histograms ~mod+ (j ~mod* hs), ds: args.ds)) + c = this.h_population_cost!(wb: args.wb, h: histograms ~mod+ (j ~mod* hs), ds: args.ds) + this.h_set_bit_cost!(wb: args.wb, h: histograms ~mod+ (j ~mod* hs), ds: args.ds, bit_cost: c) this.st32!(wb: args.wb, i: new_clusters ~mod+ (j ~mod* 4), v: (j & 0xFFFF_FFFF) as base.u32) this.st32!(wb: args.wb, i: symbols ~mod+ (j ~mod* 4), v: (j & 0xFFFF_FFFF) as base.u32) this.st32!(wb: args.wb, i: sizes ~mod+ (j ~mod* 4), v: 1) diff --git a/std/brotli/encode_cluster.wuffs b/std/brotli/encode_cluster.wuffs index b8a24408..805bd04b 100644 --- a/std/brotli/encode_cluster.wuffs +++ b/std/brotli/encode_cluster.wuffs @@ -108,7 +108,7 @@ pri func metablock_builder.compare_and_push_to_queue!(wb: slice base.u8, ds: bas } this.h_copy!(wb: args.wb, h: args.tmp, v: h1, ds: args.ds) this.h_add_histogram!(wb: args.wb, h: args.tmp, v: h2, ds: args.ds) - c = this.h_population_cost(wb: args.wb, h: args.tmp, ds: args.ds) + c = this.h_population_cost!(wb: args.wb, h: args.tmp, ds: args.ds) if this.sf.f64_lt(a: c, b: this.sf.f64_sub(a: threshold, b: cost_diff)) <> 0 { cost_combo = c is_good_pair = true @@ -271,14 +271,15 @@ pri func metablock_builder.histogram_combine!(wb: slice base.u8, ds: base.u32[.. // Mirrors FN(BrotliHistogramBitCostDistance) in cluster_inc.h. pri func metablock_builder.histogram_bit_cost_distance!(wb: slice base.u8, ds: base.u32[..= 704], histogram: base.u64, candidate: base.u64, tmp: base.u64) base.u64 { + var c : base.u64 + if this.h_total(wb: args.wb, h: args.histogram, ds: args.ds) == 0 { return 0 } this.h_copy!(wb: args.wb, h: args.tmp, v: args.histogram, ds: args.ds) this.h_add_histogram!(wb: args.wb, h: args.tmp, v: args.candidate, ds: args.ds) - return this.sf.f64_sub( - a: this.h_population_cost(wb: args.wb, h: args.tmp, ds: args.ds), - b: this.h_bit_cost(wb: args.wb, h: args.candidate, ds: args.ds)) + c = this.h_population_cost!(wb: args.wb, h: args.tmp, ds: args.ds) + return this.sf.f64_sub(a: c, b: this.h_bit_cost(wb: args.wb, h: args.candidate, ds: args.ds)) } // Mirrors FN(BrotliHistogramRemap) in cluster_inc.h. @@ -395,6 +396,7 @@ pri func metablock_builder.histogram_reindex!(wb: slice base.u8, ds: base.u32[.. // histogram_symbols. It returns the C code's *out_size. pri func metablock_builder.cluster_histograms!(wb: slice base.u8, ds: base.u32[..= 704], in_histos: base.u64, in_size: base.u64, max_histograms: base.u64, out_histos: base.u64, histogram_symbols: base.u64) base.u64 { + var c : base.u64 var hs : base.u64 var mark : base.u64 var cluster_size : base.u64 @@ -426,8 +428,8 @@ pri func metablock_builder.cluster_histograms!(wb: slice base.u8, ds: base.u32[. i = 0 while i < args.in_size { this.h_copy!(wb: args.wb, h: args.out_histos ~mod+ (i ~mod* hs), v: args.in_histos ~mod+ (i ~mod* hs), ds: args.ds) - this.h_set_bit_cost!(wb: args.wb, h: args.out_histos ~mod+ (i ~mod* hs), ds: args.ds, - bit_cost: this.h_population_cost(wb: args.wb, h: args.in_histos ~mod+ (i ~mod* hs), ds: args.ds)) + c = this.h_population_cost!(wb: args.wb, h: args.in_histos ~mod+ (i ~mod* hs), ds: args.ds) + this.h_set_bit_cost!(wb: args.wb, h: args.out_histos ~mod+ (i ~mod* hs), ds: args.ds, bit_cost: c) this.st32!(wb: args.wb, i: args.histogram_symbols ~mod+ (i ~mod* 4), v: (i & 0xFFFF_FFFF) as base.u32) i ~mod+= 1 } diff --git a/std/brotli/encode_literal_cost.wuffs b/std/brotli/encode_literal_cost.wuffs index 7fabc9c8..3b0846cf 100644 --- a/std/brotli/encode_literal_cost.wuffs +++ b/std/brotli/encode_literal_cost.wuffs @@ -277,7 +277,8 @@ pri func encoder.estimate_bit_costs_for_literals_utf8!(wb: slice base.u8, pos: b } else if utf8_pos == 2 { iw = iwu2 } - lit_cost = this.sf.f64_sub(a: this.bc.fast_log2(v: iw), b: this.bc.fast_log2(v: histo)) + lit_cost = this.bc.fast_log2_memo!(v: iw) + lit_cost = this.sf.f64_sub(a: lit_cost, b: this.bc.fast_log2(v: histo)) lit_cost = this.sf.f64_add(a: lit_cost, b: LC_F64_0_02905) this.lc_finish_cost!(wb: args.wb, cost_off: args.cost_off, i: i, lit_cost: lit_cost, utf8: true) i ~mod+= 1 @@ -337,7 +338,8 @@ pri func encoder.estimate_bit_costs_for_literals!(wb: slice base.u8, pos: base.u if histo == 0 { histo = 1 } - lit_cost = this.sf.f64_sub(a: this.bc.fast_log2(v: in_window), b: this.bc.fast_log2(v: histo)) + lit_cost = this.bc.fast_log2_memo!(v: in_window) + lit_cost = this.sf.f64_sub(a: lit_cost, b: this.bc.fast_log2(v: histo)) lit_cost = this.sf.f64_add(a: lit_cost, b: LC_F64_0_029) this.lc_finish_cost!(wb: args.wb, cost_off: args.cost_off, i: i, lit_cost: lit_cost, utf8: false) i ~mod+= 1 diff --git a/std/brotli/encode_metablock.wuffs b/std/brotli/encode_metablock.wuffs index fbbd15b4..9da3c388 100644 --- a/std/brotli/encode_metablock.wuffs +++ b/std/brotli/encode_metablock.wuffs @@ -264,9 +264,12 @@ pri func metablock_builder.h_set_bit_cost!(wb: slice base.u8, h: base.u64, ds: b this.st64!(wb: args.wb, i: args.h ~mod+ (((args.ds as base.u64) * 4) + 8), v: args.bit_cost) } -pri func metablock_builder.h_population_cost(wb: roslice base.u8, h: base.u64, ds: base.u32[..= 704]) base.u64 { +pri func metablock_builder.h_population_cost!(wb: roslice base.u8, h: base.u64, ds: base.u32[..= 704]) base.u64 { + var c : base.u64 + if args.h <= args.wb.length() { - return this.bc.population_cost(histogram: args.wb[args.h ..], data_size: args.ds) + c = this.bc.population_cost!(histogram: args.wb[args.h ..], data_size: args.ds) + return c } return 0 } @@ -482,9 +485,8 @@ pri func metablock_builder.compute_distance_cost!(wb: slice base.u8, cmds: rosli } i ~mod+= 1 } - this.dist_cost = this.sf.f64_add( - a: this.h_population_cost(wb: args.wb, h: args.tmp, ds: HISTOGRAM_DISTANCE_DATA_SIZE), - b: extra_bits) + this.dist_cost = this.h_population_cost!(wb: args.wb, h: args.tmp, ds: HISTOGRAM_DISTANCE_DATA_SIZE) + this.dist_cost = this.sf.f64_add(a: this.dist_cost, b: extra_bits) return true } @@ -592,7 +594,6 @@ pri func metablock_builder.build_meta_block!(wb: slice base.u8, ringbuffer: rosl var v : base.u32 this.reset_arena!(wb_len: args.wb.length()) - this.bc.init_log2_cache!() orig_npostfix = this.dist_postfix_bits orig_ndirect = this.dist_num_direct_codes diff --git a/std/brotli/encode_metablock_greedy.wuffs b/std/brotli/encode_metablock_greedy.wuffs index eaec8b21..103d6bbf 100644 --- a/std/brotli/encode_metablock_greedy.wuffs +++ b/std/brotli/encode_metablock_greedy.wuffs @@ -47,9 +47,12 @@ pri func encoder.gbs_histo_off(c: base.u32[..= 3], ix: base.u64) base.u64 { // gbs_entropy returns BrotliBitsEntropy of the size u32le values at // args.wb[off ..]. -pri func encoder.gbs_entropy(wb: roslice base.u8, off: base.u64, size: base.u64) base.u64 { +pri func encoder.gbs_entropy!(wb: roslice base.u8, off: base.u64, size: base.u64) base.u64 { + var e : base.u64 + if args.off <= args.wb.length() { - return this.bc.bits_entropy(population: args.wb[args.off ..], size: args.size) + e = this.bc.bits_entropy!(population: args.wb[args.off ..], size: args.size) + return e } return 0 } @@ -162,7 +165,7 @@ pri func encoder.gbs_finish_block!(wb: slice base.u8, c: base.u32[..= 3], is_fin this.set_byte!(s: args.wb, i: types_off, v: 0) i = 0 while i < nc { - e = this.gbs_entropy(wb: args.wb, off: this.gbs_histo_off(c: args.c, ix: i), size: alphabet_size) + e = this.gbs_entropy!(wb: args.wb, off: this.gbs_histo_off(c: args.c, ix: i), size: alphabet_size) this.gbs_last_entropy[((le_base as base.u64) ~mod+ i) & 127] = e this.gbs_last_entropy[(((le_base as base.u64) ~mod+ nc) ~mod+ i) & 127] = e i ~mod+= 1 @@ -185,7 +188,7 @@ pri func encoder.gbs_finish_block!(wb: slice base.u8, c: base.u32[..= 3], is_fin last1 = this.gbs_last_histogram_ix[(args.c * 2) + 1] i = 0 while i < nc { - e = this.gbs_entropy(wb: args.wb, off: this.gbs_histo_off(c: args.c, ix: curr ~mod+ i), size: alphabet_size) + e = this.gbs_entropy!(wb: args.wb, off: this.gbs_histo_off(c: args.c, ix: curr ~mod+ i), size: alphabet_size) this.gbs_entropy_tmp[i & 15] = e j = 0 while j < 2 { @@ -197,7 +200,7 @@ pri func encoder.gbs_finish_block!(wb: slice base.u8, c: base.u32[..= 3], is_fin } else { this.gbs_add_histogram!(wb: args.wb, dst: tmp, src: this.gbs_histo_off(c: args.c, ix: last1 ~mod+ i), n: data_size) } - this.gbs_comb_entropy[jx & 31] = this.gbs_entropy(wb: args.wb, off: tmp, size: alphabet_size) + this.gbs_comb_entropy[jx & 31] = this.gbs_entropy!(wb: args.wb, off: tmp, size: alphabet_size) d = this.sf.f64_sub( a: this.sf.f64_sub(a: this.gbs_comb_entropy[jx & 31], b: e), b: this.gbs_last_entropy[((le_base as base.u64) ~mod+ jx) & 127]) diff --git a/std/brotli/encode_tester.wuffs b/std/brotli/encode_tester.wuffs index 71aa8c45..855921c4 100644 --- a/std/brotli/encode_tester.wuffs +++ b/std/brotli/encode_tester.wuffs @@ -29,8 +29,11 @@ pub struct metablock_tester?( ) // bits_entropy mirrors BrotliBitsEntropy. population holds size u32le values. -pub func metablock_tester.bits_entropy(population: roslice base.u8, size: base.u64) base.u64 { - return this.bc.bits_entropy(population: args.population, size: args.size) +pub func metablock_tester.bits_entropy!(population: roslice base.u8, size: base.u64) base.u64 { + var c : base.u64 + + c = this.bc.bits_entropy!(population: args.population, size: args.size) + return c } // population_cost mirrors BrotliPopulationCostLiteral (data_size 256), @@ -38,13 +41,18 @@ pub func metablock_tester.bits_entropy(population: roslice base.u8, size: base.u // histogram has the C struct's layout: data_size u32le counts, then the // total_count_ u64le (and then the bit_cost_, which is ignored). It returns 0 // for any other data_size. -pub func metablock_tester.population_cost(histogram: roslice base.u8, data_size: base.u32) base.u64 { +pub func metablock_tester.population_cost!(histogram: roslice base.u8, data_size: base.u32) base.u64 { + var c : base.u64 + if args.data_size == 256 { - return this.bc.population_cost(histogram: args.histogram, data_size: 256) + c = this.bc.population_cost!(histogram: args.histogram, data_size: 256) + return c } else if args.data_size == 704 { - return this.bc.population_cost(histogram: args.histogram, data_size: 704) + c = this.bc.population_cost!(histogram: args.histogram, data_size: 704) + return c } else if args.data_size == 544 { - return this.bc.population_cost(histogram: args.histogram, data_size: 544) + c = this.bc.population_cost!(histogram: args.histogram, data_size: 544) + return c } return 0 } @@ -62,7 +70,6 @@ pub func metablock_tester.split_block!(cmds: roslice base.u8, num_commands: base data: roslice base.u8, pos: base.u64, mask: base.u64, quality: base.u32, workbuf: slice base.u8) base.status { this.mb.set_params!(quality: args.quality, large_window: false, disable_literal_context_modeling: false) this.mb.reset_arena!(wb_len: args.workbuf.length()) - this.mb.bc.init_log2_cache!() this.mb.alloc_block_splits!(cmds: args.cmds, num_commands: args.num_commands) if this.mb.oom { return base."#bad workbuf length" @@ -102,7 +109,6 @@ pub func metablock_tester.cluster_histograms!(data_size: base.u32, in_histos: ro return 0xFFFF_FFFF_FFFF_FFFF } hs = this.mb.hsize(ds: ds) - this.mb.bc.init_log2_cache!() this.mb.reset_arena!(wb_len: args.workbuf.length()) n = args.in_size ~mod* hs in_off = this.mb.alloc!(n: n) From ab5930ffbbd6be21694df2ee7b5f6734213e5e71 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 10:18:57 +0000 Subject: [PATCH 3/6] std/softfloat: return early for zero operands, merge the f64 add/sub fast paths and add a faster f32_from_u32 f64_add and f64_sub now share one fast path for normal operands (f64_add_normal) and both return early when one operand is zero. The f32 add and sub do the same, and f32_from_u32 converts values below 2^24 directly. All results are unchanged: the sweep, the old-vs-new differential test and the ops harness report zero mismatches. --- release/c/wuffs-unsupported-snapshot.c | 199 +++++++++--------------- std/softfloat/README.md | 13 +- std/softfloat/softfloat.wuffs | 203 +++++++++---------------- std/softfloat/softfloat_f32.wuffs | 49 ++++-- 4 files changed, 195 insertions(+), 269 deletions(-) diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 6cfc619f..6e516516 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -41206,14 +41206,7 @@ wuffs_softfloat__arith__f64_propagate_nan( WUFFS_BASE__GENERATED_C_CODE static uint64_t -wuffs_softfloat__arith__f64_add_normal_mags( - const wuffs_softfloat__arith* self, - uint64_t a_a, - uint64_t a_b); - -WUFFS_BASE__GENERATED_C_CODE -static uint64_t -wuffs_softfloat__arith__f64_sub_normal_mags( +wuffs_softfloat__arith__f64_add_normal( const wuffs_softfloat__arith* self, uint64_t a_a, uint64_t a_b); @@ -41897,13 +41890,14 @@ wuffs_softfloat__arith__f64_add( uint64_t v_z = 0; if ((((uint64_t)(((a_a >> 52u) & 2047u) - 1u)) < 2046u) && (((uint64_t)(((a_b >> 52u) & 2047u) - 1u)) < 2046u)) { - if (((a_a ^ a_b) >> 63u) == 0u) { - return wuffs_softfloat__arith__f64_add_normal_mags(self, a_a, a_b); - } - v_z = wuffs_softfloat__arith__f64_sub_normal_mags(self, a_a, a_b); + v_z = wuffs_softfloat__arith__f64_add_normal(self, a_a, a_b); if (v_z != 18446744073709551615u) { return v_z; } + } else if (((a_b & 9223372036854775807u) == 0u) && (((uint64_t)((a_a & 9223372036854775807u) - 1u)) < 9218868437227405311u)) { + return a_a; + } else if (((a_a & 9223372036854775807u) == 0u) && (((uint64_t)((a_b & 9223372036854775807u) - 1u)) < 9218868437227405311u)) { + return a_b; } if (((a_a ^ a_b) >> 63u) == 0u) { return wuffs_softfloat__arith__f64_add_mags(self, a_a, a_b, (a_a & 9223372036854775808u)); @@ -41930,13 +41924,14 @@ wuffs_softfloat__arith__f64_sub( uint64_t v_z = 0; if ((((uint64_t)(((a_a >> 52u) & 2047u) - 1u)) < 2046u) && (((uint64_t)(((a_b >> 52u) & 2047u) - 1u)) < 2046u)) { - if (((a_a ^ a_b) >> 63u) != 0u) { - return wuffs_softfloat__arith__f64_add_normal_mags(self, a_a, a_b); - } - v_z = wuffs_softfloat__arith__f64_sub_normal_mags(self, a_a, a_b); + v_z = wuffs_softfloat__arith__f64_add_normal(self, a_a, (a_b ^ 9223372036854775808u)); if (v_z != 18446744073709551615u) { return v_z; } + } else if (((a_b & 9223372036854775807u) == 0u) && (((uint64_t)((a_a & 9223372036854775807u) - 1u)) < 9218868437227405311u)) { + return a_a; + } else if (((a_a & 9223372036854775807u) == 0u) && (((uint64_t)((a_b & 9223372036854775807u) - 1u)) < 9218868437227405311u)) { + return (a_b ^ 9223372036854775808u); } if (((a_a ^ a_b) >> 63u) == 0u) { return wuffs_softfloat__arith__f64_sub_mags(self, a_a, a_b, (a_a & 9223372036854775808u)); @@ -41944,133 +41939,71 @@ wuffs_softfloat__arith__f64_sub( return wuffs_softfloat__arith__f64_add_mags(self, a_a, a_b, (a_a & 9223372036854775808u)); } -// -------- func softfloat.arith.f64_add_normal_mags +// -------- func softfloat.arith.f64_add_normal WUFFS_BASE__GENERATED_C_CODE static uint64_t -wuffs_softfloat__arith__f64_add_normal_mags( +wuffs_softfloat__arith__f64_add_normal( const wuffs_softfloat__arith* self, uint64_t a_a, uint64_t a_b) { - uint32_t v_ea = 0; - uint32_t v_eb = 0; - uint64_t v_ma = 0; - uint64_t v_mb = 0; + uint64_t v_x = 0; + uint64_t v_y = 0; + uint32_t v_ex = 0; uint32_t v_d = 0; + uint64_t v_mx = 0; + uint64_t v_my = 0; uint64_t v_z = 0; - uint64_t v_rb = 0; - - v_ea = ((uint32_t)(((a_a >> 52u) & 2047u))); - v_eb = ((uint32_t)(((a_b >> 52u) & 2047u))); - v_ma = (((a_a & 4503599627370495u) | 4503599627370496u) << 9u); - v_mb = (((a_b & 4503599627370495u) | 4503599627370496u) << 9u); - if (v_ea < v_eb) { - v_d = (((uint32_t)(v_eb - v_ea)) & 2047u); - v_ea = v_eb; - v_z = v_ma; - v_ma = v_mb; - v_mb = v_z; - } else { - v_d = (((uint32_t)(v_ea - v_eb)) & 2047u); - } - if (v_d >= 63u) { - v_mb = 1u; - } else if (v_d > 0u) { - if (((uint64_t)(v_mb << (64u - v_d))) != 0u) { - v_mb = ((v_mb >> v_d) | 1u); - } else { - v_mb = (v_mb >> v_d); - } - } - v_z = (v_ma + v_mb); - if (v_z >= 4611686018427387904u) { - v_ea += 1u; - v_z = ((v_z >> 1u) | (v_z & 1u)); - } - v_rb = (v_z & 511u); - v_z = (((uint64_t)(v_z + 256u)) >> 9u); - if (v_rb == 256u) { - v_z &= 18446744073709551614u; - } - v_z = ((uint64_t)(((uint64_t)(((uint64_t)(((uint32_t)(v_ea - 1u)))) << 52u)) + v_z)); - if (v_z >= 9218868437227405312u) { - v_z = 9218868437227405312u; - } - return ((a_a & 9223372036854775808u) | v_z); -} - -// -------- func softfloat.arith.f64_sub_normal_mags - -WUFFS_BASE__GENERATED_C_CODE -static uint64_t -wuffs_softfloat__arith__f64_sub_normal_mags( - const wuffs_softfloat__arith* self, - uint64_t a_a, - uint64_t a_b) { - uint64_t v_sign = 0; - uint32_t v_ea = 0; - uint32_t v_eb = 0; - uint64_t v_fa = 0; - uint64_t v_fb = 0; - uint64_t v_ma = 0; - uint64_t v_mb = 0; - uint32_t v_d = 0; uint32_t v_s = 0; - uint64_t v_z = 0; uint64_t v_rb = 0; - v_sign = (a_a & 9223372036854775808u); - v_ea = ((uint32_t)(((a_a >> 52u) & 2047u))); - v_eb = ((uint32_t)(((a_b >> 52u) & 2047u))); - v_fa = (a_a & 4503599627370495u); - v_fb = (a_b & 4503599627370495u); - if (v_ea == v_eb) { - if (v_fa == v_fb) { + v_x = a_a; + v_y = a_b; + if ((v_y & 9223372036854775807u) > (v_x & 9223372036854775807u)) { + v_x = a_b; + v_y = a_a; + } + v_ex = ((uint32_t)(((v_x >> 52u) & 2047u))); + v_d = ((uint32_t)(v_ex - ((uint32_t)(((v_y >> 52u) & 2047u))))); + if (v_d > 63u) { + v_d = 63u; + } + v_mx = (((v_x & 4503599627370495u) | 4503599627370496u) << 10u); + v_my = (((v_y & 4503599627370495u) | 4503599627370496u) << 10u); + if ((v_my & ((uint64_t)((((uint64_t)(1u)) << (v_d & 63u)) - 1u))) != 0u) { + v_my = ((v_my >> (v_d & 63u)) | 1u); + } else { + v_my = (v_my >> (v_d & 63u)); + } + if (((v_x ^ v_y) >> 63u) == 0u) { + v_z = (v_mx + v_my); + if (v_z >= 9223372036854775808u) { + v_ex += 1u; + v_z = ((v_z >> 1u) | (v_z & 1u)); + } + } else { + v_z = ((uint64_t)(v_mx - v_my)); + if (v_z == 0u) { return 0u; - } else if (v_fa < v_fb) { - v_sign ^= 9223372036854775808u; - v_z = ((uint64_t)(v_fb - v_fa)); - } else { - v_z = ((uint64_t)(v_fa - v_fb)); + } else if (v_z < 4611686018427387904u) { + v_s = (((uint32_t)(wuffs_softfloat__arith__clz64(self, v_z) - 1u)) & 63u); + if (v_s >= v_ex) { + return 18446744073709551615u; + } + v_ex = (((uint32_t)(v_ex - v_s)) & 2047u); + v_z = ((uint64_t)(v_z << v_s)); } - v_s = (((uint32_t)(wuffs_softfloat__arith__clz64(self, v_z) - 11u)) & 63u); - if (v_s >= v_ea) { - return 18446744073709551615u; - } - return (v_sign | ((uint64_t)(((uint64_t)(((uint64_t)(((uint32_t)(((uint32_t)(v_ea - v_s)) - 1u)))) << 52u)) + ((uint64_t)(v_z << v_s))))); } - v_ma = ((v_fa | 4503599627370496u) << 10u); - v_mb = ((v_fb | 4503599627370496u) << 10u); - if (v_ea < v_eb) { - v_sign ^= 9223372036854775808u; - v_d = (((uint32_t)(v_eb - v_ea)) & 2047u); - v_ea = v_eb; - v_z = v_ma; - v_ma = v_mb; - v_mb = v_z; - } else { - v_d = (((uint32_t)(v_ea - v_eb)) & 2047u); - } - if (v_d >= 63u) { - v_mb = 1u; - } else if (((uint64_t)(v_mb << ((64u - v_d) & 63u))) != 0u) { - v_mb = ((v_mb >> v_d) | 1u); - } else { - v_mb = (v_mb >> v_d); - } - v_z = ((uint64_t)(v_ma - v_mb)); - v_s = (((uint32_t)(wuffs_softfloat__arith__clz64(self, v_z) - 1u)) & 63u); - if (v_s >= v_ea) { - return 18446744073709551615u; - } - v_ea = (((uint32_t)(v_ea - v_s)) & 2047u); - v_z = ((uint64_t)(v_z << v_s)); v_rb = (v_z & 1023u); v_z = (((uint64_t)(v_z + 512u)) >> 10u); if (v_rb == 512u) { v_z &= 18446744073709551614u; } - return (v_sign | ((uint64_t)(((uint64_t)(((uint64_t)(((uint32_t)(v_ea - 1u)))) << 52u)) + v_z))); + v_z = ((uint64_t)(((uint64_t)(((uint64_t)(((uint32_t)(v_ex - 1u)))) << 52u)) + v_z)); + if (v_z >= 9218868437227405312u) { + v_z = 9218868437227405312u; + } + return ((v_x & 9223372036854775808u) | v_z); } // -------- func softfloat.arith.f64_add_mags @@ -42832,6 +42765,10 @@ wuffs_softfloat__arith__f32_add( if (v_z != 4294967295u) { return v_z; } + } else if (((a_b & 2147483647u) == 0u) && (((uint32_t)((a_a & 2147483647u) - 1u)) < 2139095039u)) { + return a_a; + } else if (((a_a & 2147483647u) == 0u) && (((uint32_t)((a_b & 2147483647u) - 1u)) < 2139095039u)) { + return a_b; } return wuffs_softfloat__arith__slow_f32_add(self, a_a, a_b); } @@ -42862,6 +42799,10 @@ wuffs_softfloat__arith__f32_sub( if (v_z != 4294967295u) { return v_z; } + } else if (((a_b & 2147483647u) == 0u) && (((uint32_t)((a_a & 2147483647u) - 1u)) < 2139095039u)) { + return a_a; + } else if (((a_a & 2147483647u) == 0u) && (((uint32_t)((a_b & 2147483647u) - 1u)) < 2139095039u)) { + return (a_b ^ 2147483648u); } return wuffs_softfloat__arith__slow_f32_sub(self, a_a, a_b); } @@ -43334,6 +43275,16 @@ wuffs_softfloat__arith__f32_from_u32( return 0; } + uint32_t v_c = 0; + + if (a_a == 0u) { + return 0u; + } else if (a_a < 16777216u) { + v_c = wuffs_softfloat__arith__clz32(self, a_a); + if (v_c >= 8u) { + return ((uint32_t)(((uint32_t)(((uint32_t)(157u - v_c)) << 23u)) + ((uint32_t)(a_a << (v_c - 8u))))); + } + } return wuffs_softfloat__arith__f64_to_f32(self, wuffs_softfloat__arith__f64_from_u32(self, a_a)); } diff --git a/std/softfloat/README.md b/std/softfloat/README.md index 23c560bd..84ae6e9b 100644 --- a/std/softfloat/README.md +++ b/std/softfloat/README.md @@ -97,15 +97,18 @@ with the final remainder as the sticky bit. Leading zero counts use a 256-entry table. `f64_add` and `f64_sub` have a fast path for the common case where both -operands are normal (`f64_add_normal_mags` and `f64_sub_normal_mags`): it -skips the special-case handling, aligns the significands with a sticky bit, -adds or subtracts them and rounds inline. Only a subtraction whose result is -subnormal falls back to the general code (`f64_add_mags`, `f64_sub_mags`). +operands are normal (`f64_add_normal`): it orders the operands by magnitude, +aligns the significands with a sticky bit, adds or subtracts them and rounds +inline. Only a result that is subnormal falls back to the general code +(`f64_add_mags`, `f64_sub_mags`). Adding (or subtracting) a zero and a finite +non-zero value also returns early. Likewise, `f32_add`, `f32_sub` and `f32_mul` compute the result directly in binary32 (with 32-bit significands for add and sub and an exact 48-bit product for mul) when both operands are normal and the result is normal (or, -for add and mul, overflows to infinity). Otherwise (zeroes, subnormal +for add and mul, overflows to infinity), and add and sub return early for a +zero plus a finite non-zero value. `f32_from_u32` converts integers below +`1 << 24` (which are exact) directly. Otherwise (zeroes, subnormal operands or results, infinities and NaNs), they, and `f32_div` always, convert their operands exactly to `f64`, perform the `f64` operation and then round to `f32`. This double rounding is innocuous: binary64's 53 bits of diff --git a/std/softfloat/softfloat.wuffs b/std/softfloat/softfloat.wuffs index c1392b67..0df236f9 100644 --- a/std/softfloat/softfloat.wuffs +++ b/std/softfloat/softfloat.wuffs @@ -367,13 +367,16 @@ pub func arith.f64_add(a: base.u64, b: base.u64) base.u64 { if ((((args.a >> 52) & 0x7FF) ~mod- 1) < 0x7FE) and ((((args.b >> 52) & 0x7FF) ~mod- 1) < 0x7FE) { // Both operands are normal: take the fast path. - if ((args.a ^ args.b) >> 63) == 0 { - return this.f64_add_normal_mags(a: args.a, b: args.b) - } - z = this.f64_sub_normal_mags(a: args.a, b: args.b) + z = this.f64_add_normal(a: args.a, b: args.b) if z <> 0xFFFF_FFFF_FFFF_FFFF { return z } + } else if ((args.b & F64_ABS) == 0) and (((args.a & F64_ABS) ~mod- 1) < (F64_INF - 1)) { + // b is a zero and a is finite and non-zero, so (a + b) is a. + return args.a + } else if ((args.a & F64_ABS) == 0) and (((args.b & F64_ABS) ~mod- 1) < (F64_INF - 1)) { + // a is a zero and b is finite and non-zero, so (a + b) is b. + return args.b } if ((args.a ^ args.b) >> 63) == 0 { return this.f64_add_mags(a: args.a, b: args.b, sign: args.a & F64_SIGN) @@ -386,14 +389,19 @@ pub func arith.f64_sub(a: base.u64, b: base.u64) base.u64 { var z : base.u64 if ((((args.a >> 52) & 0x7FF) ~mod- 1) < 0x7FE) and ((((args.b >> 52) & 0x7FF) ~mod- 1) < 0x7FE) { - // Both operands are normal: take the fast path. - if ((args.a ^ args.b) >> 63) <> 0 { - return this.f64_add_normal_mags(a: args.a, b: args.b) - } - z = this.f64_sub_normal_mags(a: args.a, b: args.b) + // Both operands are normal: take the fast path. For non-NaN + // operands, (a - b) is exactly (a + -b), including the sign of a + // zero result. + z = this.f64_add_normal(a: args.a, b: args.b ^ F64_SIGN) if z <> 0xFFFF_FFFF_FFFF_FFFF { return z } + } else if ((args.b & F64_ABS) == 0) and (((args.a & F64_ABS) ~mod- 1) < (F64_INF - 1)) { + // b is a zero and a is finite and non-zero, so (a - b) is a. + return args.a + } else if ((args.a & F64_ABS) == 0) and (((args.b & F64_ABS) ~mod- 1) < (F64_INF - 1)) { + // a is a zero and b is finite and non-zero, so (a - b) is -b. + return args.b ^ F64_SIGN } if ((args.a ^ args.b) >> 63) == 0 { return this.f64_sub_mags(a: args.a, b: args.b, sign: args.a & F64_SIGN) @@ -401,141 +409,80 @@ pub func arith.f64_sub(a: base.u64, b: base.u64) base.u64 { return this.f64_add_mags(a: args.a, b: args.b, sign: args.a & F64_SIGN) } -// f64_add_normal_mags returns (sign(a) * (|a| + |b|)), for normal a and b. -// It is a faster version of f64_add_mags. -pri func arith.f64_add_normal_mags(a: base.u64, b: base.u64) base.u64 { - var ea : base.u32[..= 0x800] - var eb : base.u32[..= 0x7FF] - var ma : base.u64[..= 0x3FFF_FFFF_FFFF_FE00] - var mb : base.u64[..= 0x3FFF_FFFF_FFFF_FE00] - var d : base.u32[..= 0x7FF] +// f64_add_normal returns (a + b), for normal a and b, unless the result is +// subnormal, in which case it returns 0xFFFF_FFFF_FFFF_FFFF (a NaN, which it +// never otherwise returns) and the caller must take its slow path. It is a +// faster (but less general) version of f64_add_mags and f64_sub_mags. +pri func arith.f64_add_normal(a: base.u64, b: base.u64) base.u64 { + var x : base.u64 + var y : base.u64 + var ex : base.u32[..= 0x800] + var d : base.u32 + var mx : base.u64[..= 0x7FFF_FFFF_FFFF_FFFF] + var my : base.u64[..= 0x7FFF_FFFF_FFFF_FFFF] var z : base.u64 + var s : base.u32[..= 63] var rb : base.u64 - ea = ((args.a >> 52) & 0x7FF) as base.u32 - eb = ((args.b >> 52) & 0x7FF) as base.u32 - // The significands, with their implicit bit at bit 61 and 9 extra bits - // of precision at the bottom. - ma = ((args.a & F64_FRAC) | F64_HIDDEN) << 9 - mb = ((args.b & F64_FRAC) | F64_HIDDEN) << 9 - if ea < eb { - d = (eb ~mod- ea) & 0x7FF - ea = eb - z = ma - ma = mb - mb = z + // Order the operands so that |x| >= |y|. The result has x's sign. + x = args.a + y = args.b + if (y & F64_ABS) > (x & F64_ABS) { + x = args.b + y = args.a + } + ex = ((x >> 52) & 0x7FF) as base.u32 + d = ex ~mod- (((y >> 52) & 0x7FF) as base.u32) + if d > 63 { + d = 63 + } + // The significands, with their implicit bit at bit 62 and 10 extra bits + // of precision at the bottom. Align my with mx, keeping a sticky bit. + mx = ((x & F64_FRAC) | F64_HIDDEN) << 10 + my = ((y & F64_FRAC) | F64_HIDDEN) << 10 + if (my & (((1 as base.u64) << (d & 63)) ~mod- 1)) <> 0 { + my = (my >> (d & 63)) | 1 } else { - d = (ea ~mod- eb) & 0x7FF + my = my >> (d & 63) } - if d >= 63 { - mb = 1 - } else if d > 0 { - if (mb ~mod<< (64 - d)) <> 0 { - mb = (mb >> d) | 1 - } else { - mb = mb >> d + + if ((x ^ y) >> 63) == 0 { + z = mx + my + if z >= 0x8000_0000_0000_0000 { + ex += 1 + z = (z >> 1) | (z & 1) } - } - z = ma + mb - if z >= 0x4000_0000_0000_0000 { - ea += 1 - z = (z >> 1) | (z & 1) - } - rb = z & 0x1FF - z = (z ~mod+ 0x100) >> 9 - if rb == 0x100 { - z &= 0xFFFF_FFFF_FFFF_FFFE - } - // ea is in [1 ..= 0x7FF] and z is in [1 << 52 ..= 1 << 53]. Adding z - // (whose bit 52 is the implicit bit) carries any rounding overflow into - // the exponent field. - z = (((ea ~mod- 1) as base.u64) ~mod<< 52) ~mod+ z - if z >= F64_INF { - z = F64_INF - } - return (args.a & F64_SIGN) | z -} - -// f64_sub_normal_mags returns (sign(a) * (|a| - |b|)), for normal a and b, -// unless the result is subnormal, in which case it returns -// 0xFFFF_FFFF_FFFF_FFFF (a NaN, which it never otherwise returns) and the -// caller must take its slow path. It is a faster version of f64_sub_mags. -pri func arith.f64_sub_normal_mags(a: base.u64, b: base.u64) base.u64 { - var sign : base.u64 - var ea : base.u32[..= 0x7FF] - var eb : base.u32[..= 0x7FF] - var fa : base.u64[..= 0x000F_FFFF_FFFF_FFFF] - var fb : base.u64[..= 0x000F_FFFF_FFFF_FFFF] - var ma : base.u64 - var mb : base.u64 - var d : base.u32[..= 0x7FF] - var s : base.u32[..= 63] - var z : base.u64 - var rb : base.u64 - - sign = args.a & F64_SIGN - ea = ((args.a >> 52) & 0x7FF) as base.u32 - eb = ((args.b >> 52) & 0x7FF) as base.u32 - fa = args.a & F64_FRAC - fb = args.b & F64_FRAC - - if ea == eb { - // The difference is exact. - if fa == fb { + } else { + // mx >= my, since |x| >= |y|. + z = mx ~mod- my + if z == 0 { // Exact cancellation gives +0 when rounding to nearest. return 0 - } else if fa < fb { - sign ^= F64_SIGN - z = fb ~mod- fa - } else { - z = fa ~mod- fb + } else if z < 0x4000_0000_0000_0000 { + // If (d >= 2) then z >= (1 << 61) and the normalization shift + // is 1. Otherwise, no bits were shifted out of my. + s = (this.clz64(a: z) ~mod- 1) & 63 + if s >= ex { + return 0xFFFF_FFFF_FFFF_FFFF + } + ex = (ex ~mod- s) & 0x7FF + z = z ~mod<< s } - // z is in [1 ..= F64_FRAC]. Shift its top bit to bit 52. - s = (this.clz64(a: z) ~mod- 11) & 63 - if s >= ea { - return 0xFFFF_FFFF_FFFF_FFFF - } - return sign | (((((ea ~mod- s) ~mod- 1) as base.u64) ~mod<< 52) ~mod+ (z ~mod<< s)) } - // The significands, with their implicit bit at bit 62 and 10 extra bits - // of precision at the bottom. - ma = (fa | F64_HIDDEN) << 10 - mb = (fb | F64_HIDDEN) << 10 - if ea < eb { - sign ^= F64_SIGN - d = (eb ~mod- ea) & 0x7FF - ea = eb - z = ma - ma = mb - mb = z - } else { - d = (ea ~mod- eb) & 0x7FF - } - if d >= 63 { - mb = 1 - } else if (mb ~mod<< ((64 - d) & 63)) <> 0 { - mb = (mb >> d) | 1 - } else { - mb = mb >> d - } - // ma > mb, so z is non-zero. If (d >= 2) then z >= (1 << 61) and the - // normalization shift is at most 1. Otherwise, no bits were shifted out. - z = ma ~mod- mb - s = (this.clz64(a: z) ~mod- 1) & 63 - if s >= ea { - return 0xFFFF_FFFF_FFFF_FFFF - } - ea = (ea ~mod- s) & 0x7FF - z = z ~mod<< s + // Round to nearest, ties to even. z is now in [1 << 52 ..= 1 << 53] and + // ex in [1 ..= 0x7FF]. Adding z (whose bit 52 is the implicit bit) to the + // shifted exponent carries any rounding overflow into the exponent field. rb = z & 0x3FF z = (z ~mod+ 0x200) >> 10 if rb == 0x200 { z &= 0xFFFF_FFFF_FFFF_FFFE } - // ea is in [1 ..= 0x7FE] and z is in [1 << 52 ..= 1 << 53], so this does - // not overflow. - return sign | ((((ea ~mod- 1) as base.u64) ~mod<< 52) ~mod+ z) + z = (((ex ~mod- 1) as base.u64) ~mod<< 52) ~mod+ z + if z >= F64_INF { + z = F64_INF + } + return (x & F64_SIGN) | z } // f64_add_mags returns (sign * (|a| + |b|)). diff --git a/std/softfloat/softfloat_f32.wuffs b/std/softfloat/softfloat_f32.wuffs index ce616e11..32ca0e39 100644 --- a/std/softfloat/softfloat_f32.wuffs +++ b/std/softfloat/softfloat_f32.wuffs @@ -10,18 +10,19 @@ // -------- f32 (binary32) operations. // -// When both operands are normal (and, for sub_mags and mul, when the result -// is normal too), the f32 add, sub and mul functions compute the result -// directly in binary32 (with 32-bit or 64-bit integer significands). This is -// the common case. Otherwise (zeroes, subnormals, infinities, NaNs and -// subnormal results), they and f32_div convert their (exactly representable) -// operands to f64, perform the f64 operation and round the result to f32 -// (the slow_etc functions). Double rounding is innocuous here: binary64 has -// more than (2 * 24) + 2 bits of precision, which suffices for +, -, * and / -// (Figueroa, "When is double rounding innocuous?", 1995). This is also -// exhaustively cross-checked against hardware by the tests. The fast paths -// never return a NaN, so NaN results (and their payloads) always come from -// the slow paths. +// When both operands are normal (and, unless it overflows to infinity, the +// result is normal too), the f32 add, sub and mul functions compute the +// result directly in binary32 (with 32-bit or 64-bit integer significands). +// This is the common case. Adding or subtracting a zero and a finite +// non-zero operand is also handled directly. Otherwise (zeroes, subnormals, +// infinities, NaNs and subnormal results), they and f32_div convert their +// (exactly representable) operands to f64, perform the f64 operation and +// round the result to f32 (the slow_etc functions). Double rounding is +// innocuous here: binary64 has more than (2 * 24) + 2 bits of precision, +// which suffices for +, -, * and / (Figueroa, "When is double rounding +// innocuous?", 1995). This is also exhaustively cross-checked against +// hardware by the tests. The fast paths never return a NaN, so NaN results +// (and their payloads) always come from the slow paths. pri const F32_SIGN : base.u32 = 0x8000_0000 pri const F32_ABS : base.u32 = 0x7FFF_FFFF @@ -90,6 +91,12 @@ pub func arith.f32_add(a: base.u32, b: base.u32) base.u32 { if z <> 0xFFFF_FFFF { return z } + } else if ((args.b & F32_ABS) == 0) and (((args.a & F32_ABS) ~mod- 1) < (F32_INF - 1)) { + // b is a zero and a is finite and non-zero, so (a + b) is a. + return args.a + } else if ((args.a & F32_ABS) == 0) and (((args.b & F32_ABS) ~mod- 1) < (F32_INF - 1)) { + // a is a zero and b is finite and non-zero, so (a + b) is b. + return args.b } return this.slow_f32_add(a: args.a, b: args.b) } @@ -106,6 +113,12 @@ pub func arith.f32_sub(a: base.u32, b: base.u32) base.u32 { if z <> 0xFFFF_FFFF { return z } + } else if ((args.b & F32_ABS) == 0) and (((args.a & F32_ABS) ~mod- 1) < (F32_INF - 1)) { + // b is a zero and a is finite and non-zero, so (a - b) is a. + return args.a + } else if ((args.a & F32_ABS) == 0) and (((args.b & F32_ABS) ~mod- 1) < (F32_INF - 1)) { + // a is a zero and b is finite and non-zero, so (a - b) is -b. + return args.b ^ F32_SIGN } return this.slow_f32_sub(a: args.a, b: args.b) } @@ -415,6 +428,18 @@ pub func arith.f32_max(a: base.u32, b: base.u32) base.u32 { // f32_from_u32 returns the f32 nearest to the integer a (C's (float)a). pub func arith.f32_from_u32(a: base.u32) base.u32 { + var c : base.u32[..= 31] + + if args.a == 0 { + return 0 + } else if args.a < 0x100_0000 { + // a is exactly representable. Shift its top bit to bit 23 (the + // implicit bit, which is then added to the exponent field). + c = this.clz32(a: args.a) + if c >= 8 { + return ((157 ~mod- c) ~mod<< 23) ~mod+ (args.a ~mod<< (c - 8)) + } + } // The u32 to f64 conversion is exact, so there is only one rounding. return this.f64_to_f32(a: this.f64_from_u32(a: args.a)) } From 44a9ca086ca92c4be9cca764a995e3d0235b372b Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 10:30:04 +0000 Subject: [PATCH 4/6] std/softfloat: compare non-negative, non-NaN operands directly The f64 and f32 comparison, min and max functions now compare the bit patterns directly when neither operand is negative or a NaN, and call the unchanged general code (now f64_cmp_slow and f32_cmp_slow) otherwise. Results are unchanged: the ops harness and the old-vs-new differential test report zero mismatches. --- release/c/wuffs-unsupported-snapshot.c | 180 ++++++++++++++++++++++--- std/softfloat/README.md | 5 + std/softfloat/softfloat.wuffs | 79 +++++++++-- std/softfloat/softfloat_f32.wuffs | 79 +++++++++-- 4 files changed, 311 insertions(+), 32 deletions(-) diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 6e516516..3fd992ac 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -41204,6 +41204,13 @@ wuffs_softfloat__arith__f64_propagate_nan( uint64_t a_a, uint64_t a_b); +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_softfloat__arith__f64_cmp_slow( + const wuffs_softfloat__arith* self, + uint64_t a_a, + uint64_t a_b); + WUFFS_BASE__GENERATED_C_CODE static uint64_t wuffs_softfloat__arith__f64_add_normal( @@ -41270,6 +41277,13 @@ wuffs_softfloat__arith__slow_f32_mul( uint32_t a_a, uint32_t a_b); +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_softfloat__arith__f32_cmp_slow( + const wuffs_softfloat__arith* self, + uint32_t a_a, + uint32_t a_b); + // ---------------- VTables // ---------------- Initializer Implementations @@ -41668,6 +41682,25 @@ wuffs_softfloat__arith__f64_cmp( return 0; } + if ((a_a | a_b) <= 9218868437227405312u) { + if (a_a < a_b) { + return 0u; + } else if (a_a == a_b) { + return 1u; + } + return 2u; + } + return wuffs_softfloat__arith__f64_cmp_slow(self, a_a, a_b); +} + +// -------- func softfloat.arith.f64_cmp_slow + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_softfloat__arith__f64_cmp_slow( + const wuffs_softfloat__arith* self, + uint64_t a_a, + uint64_t a_b) { uint64_t v_ka = 0; uint64_t v_kb = 0; @@ -41708,7 +41741,13 @@ wuffs_softfloat__arith__f64_eq( return 0; } - if (wuffs_softfloat__arith__f64_cmp(self, a_a, a_b) == 1u) { + if ((a_a | a_b) <= 9218868437227405312u) { + if (a_a == a_b) { + return 1u; + } + return 0u; + } + if (wuffs_softfloat__arith__f64_cmp_slow(self, a_a, a_b) == 1u) { return 1u; } return 0u; @@ -41730,7 +41769,13 @@ wuffs_softfloat__arith__f64_ne( return 0; } - if (wuffs_softfloat__arith__f64_cmp(self, a_a, a_b) == 1u) { + if ((a_a | a_b) <= 9218868437227405312u) { + if (a_a != a_b) { + return 1u; + } + return 0u; + } + if (wuffs_softfloat__arith__f64_cmp_slow(self, a_a, a_b) == 1u) { return 0u; } return 1u; @@ -41752,7 +41797,13 @@ wuffs_softfloat__arith__f64_lt( return 0; } - if (wuffs_softfloat__arith__f64_cmp(self, a_a, a_b) == 0u) { + if ((a_a | a_b) <= 9218868437227405312u) { + if (a_a < a_b) { + return 1u; + } + return 0u; + } + if (wuffs_softfloat__arith__f64_cmp_slow(self, a_a, a_b) == 0u) { return 1u; } return 0u; @@ -41774,7 +41825,13 @@ wuffs_softfloat__arith__f64_le( return 0; } - if (wuffs_softfloat__arith__f64_cmp(self, a_a, a_b) <= 1u) { + if ((a_a | a_b) <= 9218868437227405312u) { + if (a_a <= a_b) { + return 1u; + } + return 0u; + } + if (wuffs_softfloat__arith__f64_cmp_slow(self, a_a, a_b) <= 1u) { return 1u; } return 0u; @@ -41796,7 +41853,13 @@ wuffs_softfloat__arith__f64_gt( return 0; } - if (wuffs_softfloat__arith__f64_cmp(self, a_a, a_b) == 2u) { + if ((a_a | a_b) <= 9218868437227405312u) { + if (a_a > a_b) { + return 1u; + } + return 0u; + } + if (wuffs_softfloat__arith__f64_cmp_slow(self, a_a, a_b) == 2u) { return 1u; } return 0u; @@ -41820,7 +41883,13 @@ wuffs_softfloat__arith__f64_ge( uint32_t v_c = 0; - v_c = wuffs_softfloat__arith__f64_cmp(self, a_a, a_b); + if ((a_a | a_b) <= 9218868437227405312u) { + if (a_a >= a_b) { + return 1u; + } + return 0u; + } + v_c = wuffs_softfloat__arith__f64_cmp_slow(self, a_a, a_b); if ((v_c == 1u) || (v_c == 2u)) { return 1u; } @@ -41843,7 +41912,13 @@ wuffs_softfloat__arith__f64_min( return 0; } - if (wuffs_softfloat__arith__f64_cmp(self, a_a, a_b) == 0u) { + if ((a_a | a_b) <= 9218868437227405312u) { + if (a_a < a_b) { + return a_a; + } + return a_b; + } + if (wuffs_softfloat__arith__f64_cmp_slow(self, a_a, a_b) == 0u) { return a_a; } return a_b; @@ -41865,7 +41940,13 @@ wuffs_softfloat__arith__f64_max( return 0; } - if (wuffs_softfloat__arith__f64_cmp(self, a_a, a_b) == 2u) { + if ((a_a | a_b) <= 9218868437227405312u) { + if (a_a > a_b) { + return a_a; + } + return a_b; + } + if (wuffs_softfloat__arith__f64_cmp_slow(self, a_a, a_b) == 2u) { return a_a; } return a_b; @@ -43057,6 +43138,25 @@ wuffs_softfloat__arith__f32_cmp( return 0; } + if ((a_a | a_b) <= 2139095040u) { + if (a_a < a_b) { + return 0u; + } else if (a_a == a_b) { + return 1u; + } + return 2u; + } + return wuffs_softfloat__arith__f32_cmp_slow(self, a_a, a_b); +} + +// -------- func softfloat.arith.f32_cmp_slow + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_softfloat__arith__f32_cmp_slow( + const wuffs_softfloat__arith* self, + uint32_t a_a, + uint32_t a_b) { uint32_t v_ka = 0; uint32_t v_kb = 0; @@ -43097,7 +43197,13 @@ wuffs_softfloat__arith__f32_eq( return 0; } - if (wuffs_softfloat__arith__f32_cmp(self, a_a, a_b) == 1u) { + if ((a_a | a_b) <= 2139095040u) { + if (a_a == a_b) { + return 1u; + } + return 0u; + } + if (wuffs_softfloat__arith__f32_cmp_slow(self, a_a, a_b) == 1u) { return 1u; } return 0u; @@ -43119,7 +43225,13 @@ wuffs_softfloat__arith__f32_ne( return 0; } - if (wuffs_softfloat__arith__f32_cmp(self, a_a, a_b) == 1u) { + if ((a_a | a_b) <= 2139095040u) { + if (a_a != a_b) { + return 1u; + } + return 0u; + } + if (wuffs_softfloat__arith__f32_cmp_slow(self, a_a, a_b) == 1u) { return 0u; } return 1u; @@ -43141,7 +43253,13 @@ wuffs_softfloat__arith__f32_lt( return 0; } - if (wuffs_softfloat__arith__f32_cmp(self, a_a, a_b) == 0u) { + if ((a_a | a_b) <= 2139095040u) { + if (a_a < a_b) { + return 1u; + } + return 0u; + } + if (wuffs_softfloat__arith__f32_cmp_slow(self, a_a, a_b) == 0u) { return 1u; } return 0u; @@ -43163,7 +43281,13 @@ wuffs_softfloat__arith__f32_le( return 0; } - if (wuffs_softfloat__arith__f32_cmp(self, a_a, a_b) <= 1u) { + if ((a_a | a_b) <= 2139095040u) { + if (a_a <= a_b) { + return 1u; + } + return 0u; + } + if (wuffs_softfloat__arith__f32_cmp_slow(self, a_a, a_b) <= 1u) { return 1u; } return 0u; @@ -43185,7 +43309,13 @@ wuffs_softfloat__arith__f32_gt( return 0; } - if (wuffs_softfloat__arith__f32_cmp(self, a_a, a_b) == 2u) { + if ((a_a | a_b) <= 2139095040u) { + if (a_a > a_b) { + return 1u; + } + return 0u; + } + if (wuffs_softfloat__arith__f32_cmp_slow(self, a_a, a_b) == 2u) { return 1u; } return 0u; @@ -43209,7 +43339,13 @@ wuffs_softfloat__arith__f32_ge( uint32_t v_c = 0; - v_c = wuffs_softfloat__arith__f32_cmp(self, a_a, a_b); + if ((a_a | a_b) <= 2139095040u) { + if (a_a >= a_b) { + return 1u; + } + return 0u; + } + v_c = wuffs_softfloat__arith__f32_cmp_slow(self, a_a, a_b); if ((v_c == 1u) || (v_c == 2u)) { return 1u; } @@ -43232,7 +43368,13 @@ wuffs_softfloat__arith__f32_min( return 0; } - if (wuffs_softfloat__arith__f32_cmp(self, a_a, a_b) == 0u) { + if ((a_a | a_b) <= 2139095040u) { + if (a_a < a_b) { + return a_a; + } + return a_b; + } + if (wuffs_softfloat__arith__f32_cmp_slow(self, a_a, a_b) == 0u) { return a_a; } return a_b; @@ -43254,7 +43396,13 @@ wuffs_softfloat__arith__f32_max( return 0; } - if (wuffs_softfloat__arith__f32_cmp(self, a_a, a_b) == 2u) { + if ((a_a | a_b) <= 2139095040u) { + if (a_a > a_b) { + return a_a; + } + return a_b; + } + if (wuffs_softfloat__arith__f32_cmp_slow(self, a_a, a_b) == 2u) { return a_a; } return a_b; diff --git a/std/softfloat/README.md b/std/softfloat/README.md index 84ae6e9b..e9b1dc43 100644 --- a/std/softfloat/README.md +++ b/std/softfloat/README.md @@ -118,6 +118,11 @@ Newsletter, 1995). The tests confirm this against hardware, including for subnormal `f32` results. The fast paths never produce a NaN, so NaN results (and their payloads) are exactly those of the "via `f64`" computation. +The comparison functions (`f64_cmp`, `f64_lt`, `f64_min`, etc. and their `f32` +counterparts) compare the bit patterns directly when neither operand is +negative or a NaN (when `a | b` is at most the bit pattern of `+inf`), and +otherwise call `f64_cmp_slow` or `f32_cmp_slow`. + ## log2 diff --git a/std/softfloat/softfloat.wuffs b/std/softfloat/softfloat.wuffs index 0df236f9..65bd77e5 100644 --- a/std/softfloat/softfloat.wuffs +++ b/std/softfloat/softfloat.wuffs @@ -265,6 +265,21 @@ pub func arith.f64_abs(a: base.u64) base.u64 { // f64_cmp compares a and b, returning 0 if (a < b), 1 if (a == b), 2 if (a // > b) and 3 if they are unordered (either is a NaN). -0 and +0 compare equal. pub func arith.f64_cmp(a: base.u64, b: base.u64) base.u32 { + if (args.a | args.b) <= F64_INF { + // Fast path: neither is negative (the sign bits are clear) or a NaN + // (both are at most +inf), so the bit patterns sort like the values. + if args.a < args.b { + return 0 + } else if args.a == args.b { + return 1 + } + return 2 + } + return this.f64_cmp_slow(a: args.a, b: args.b) +} + +// f64_cmp_slow is f64_cmp without the fast path. +pri func arith.f64_cmp_slow(a: base.u64, b: base.u64) base.u32 { var ka : base.u64 var kb : base.u64 @@ -297,35 +312,65 @@ pub func arith.f64_cmp(a: base.u64, b: base.u64) base.u32 { // true), and -0 equals +0. pub func arith.f64_eq(a: base.u64, b: base.u64) base.u32 { - if this.f64_cmp(a: args.a, b: args.b) == 1 { + if (args.a | args.b) <= F64_INF { + if args.a == args.b { + return 1 + } + return 0 + } + if this.f64_cmp_slow(a: args.a, b: args.b) == 1 { return 1 } return 0 } pub func arith.f64_ne(a: base.u64, b: base.u64) base.u32 { - if this.f64_cmp(a: args.a, b: args.b) == 1 { + if (args.a | args.b) <= F64_INF { + if args.a <> args.b { + return 1 + } + return 0 + } + if this.f64_cmp_slow(a: args.a, b: args.b) == 1 { return 0 } return 1 } pub func arith.f64_lt(a: base.u64, b: base.u64) base.u32 { - if this.f64_cmp(a: args.a, b: args.b) == 0 { + if (args.a | args.b) <= F64_INF { + if args.a < args.b { + return 1 + } + return 0 + } + if this.f64_cmp_slow(a: args.a, b: args.b) == 0 { return 1 } return 0 } pub func arith.f64_le(a: base.u64, b: base.u64) base.u32 { - if this.f64_cmp(a: args.a, b: args.b) <= 1 { + if (args.a | args.b) <= F64_INF { + if args.a <= args.b { + return 1 + } + return 0 + } + if this.f64_cmp_slow(a: args.a, b: args.b) <= 1 { return 1 } return 0 } pub func arith.f64_gt(a: base.u64, b: base.u64) base.u32 { - if this.f64_cmp(a: args.a, b: args.b) == 2 { + if (args.a | args.b) <= F64_INF { + if args.a > args.b { + return 1 + } + return 0 + } + if this.f64_cmp_slow(a: args.a, b: args.b) == 2 { return 1 } return 0 @@ -334,7 +379,13 @@ pub func arith.f64_gt(a: base.u64, b: base.u64) base.u32 { pub func arith.f64_ge(a: base.u64, b: base.u64) base.u32 { var c : base.u32 - c = this.f64_cmp(a: args.a, b: args.b) + if (args.a | args.b) <= F64_INF { + if args.a >= args.b { + return 1 + } + return 0 + } + c = this.f64_cmp_slow(a: args.a, b: args.b) if (c == 1) or (c == 2) { return 1 } @@ -345,7 +396,13 @@ pub func arith.f64_ge(a: base.u64, b: base.u64) base.u32 { // Note that this is not IEEE 754 minNum: f64_min(NaN, x) and f64_min(x, NaN) // both return the second argument, and f64_min(-0, +0) returns +0. pub func arith.f64_min(a: base.u64, b: base.u64) base.u64 { - if this.f64_cmp(a: args.a, b: args.b) == 0 { + if (args.a | args.b) <= F64_INF { + if args.a < args.b { + return args.a + } + return args.b + } + if this.f64_cmp_slow(a: args.a, b: args.b) == 0 { return args.a } return args.b @@ -353,7 +410,13 @@ pub func arith.f64_min(a: base.u64, b: base.u64) base.u64 { // f64_max returns ((a > b) ? a : b), like brotli's BROTLI_MAX(double, a, b). pub func arith.f64_max(a: base.u64, b: base.u64) base.u64 { - if this.f64_cmp(a: args.a, b: args.b) == 2 { + if (args.a | args.b) <= F64_INF { + if args.a > args.b { + return args.a + } + return args.b + } + if this.f64_cmp_slow(a: args.a, b: args.b) == 2 { return args.a } return args.b diff --git a/std/softfloat/softfloat_f32.wuffs b/std/softfloat/softfloat_f32.wuffs index 32ca0e39..c49d64e9 100644 --- a/std/softfloat/softfloat_f32.wuffs +++ b/std/softfloat/softfloat_f32.wuffs @@ -334,6 +334,21 @@ pub func arith.f32_div(a: base.u32, b: base.u32) base.u32 { // f32_cmp compares a and b, returning 0 if (a < b), 1 if (a == b), 2 if (a // > b) and 3 if they are unordered (either is a NaN). -0 and +0 compare equal. pub func arith.f32_cmp(a: base.u32, b: base.u32) base.u32 { + if (args.a | args.b) <= F32_INF { + // Fast path: neither is negative (the sign bits are clear) or a NaN + // (both are at most +inf), so the bit patterns sort like the values. + if args.a < args.b { + return 0 + } else if args.a == args.b { + return 1 + } + return 2 + } + return this.f32_cmp_slow(a: args.a, b: args.b) +} + +// f32_cmp_slow is f32_cmp without the fast path. +pri func arith.f32_cmp_slow(a: base.u32, b: base.u32) base.u32 { var ka : base.u32 var kb : base.u32 @@ -364,35 +379,65 @@ pub func arith.f32_cmp(a: base.u32, b: base.u32) base.u32 { // true), and -0 equals +0. pub func arith.f32_eq(a: base.u32, b: base.u32) base.u32 { - if this.f32_cmp(a: args.a, b: args.b) == 1 { + if (args.a | args.b) <= F32_INF { + if args.a == args.b { + return 1 + } + return 0 + } + if this.f32_cmp_slow(a: args.a, b: args.b) == 1 { return 1 } return 0 } pub func arith.f32_ne(a: base.u32, b: base.u32) base.u32 { - if this.f32_cmp(a: args.a, b: args.b) == 1 { + if (args.a | args.b) <= F32_INF { + if args.a <> args.b { + return 1 + } + return 0 + } + if this.f32_cmp_slow(a: args.a, b: args.b) == 1 { return 0 } return 1 } pub func arith.f32_lt(a: base.u32, b: base.u32) base.u32 { - if this.f32_cmp(a: args.a, b: args.b) == 0 { + if (args.a | args.b) <= F32_INF { + if args.a < args.b { + return 1 + } + return 0 + } + if this.f32_cmp_slow(a: args.a, b: args.b) == 0 { return 1 } return 0 } pub func arith.f32_le(a: base.u32, b: base.u32) base.u32 { - if this.f32_cmp(a: args.a, b: args.b) <= 1 { + if (args.a | args.b) <= F32_INF { + if args.a <= args.b { + return 1 + } + return 0 + } + if this.f32_cmp_slow(a: args.a, b: args.b) <= 1 { return 1 } return 0 } pub func arith.f32_gt(a: base.u32, b: base.u32) base.u32 { - if this.f32_cmp(a: args.a, b: args.b) == 2 { + if (args.a | args.b) <= F32_INF { + if args.a > args.b { + return 1 + } + return 0 + } + if this.f32_cmp_slow(a: args.a, b: args.b) == 2 { return 1 } return 0 @@ -401,7 +446,13 @@ pub func arith.f32_gt(a: base.u32, b: base.u32) base.u32 { pub func arith.f32_ge(a: base.u32, b: base.u32) base.u32 { var c : base.u32 - c = this.f32_cmp(a: args.a, b: args.b) + if (args.a | args.b) <= F32_INF { + if args.a >= args.b { + return 1 + } + return 0 + } + c = this.f32_cmp_slow(a: args.a, b: args.b) if (c == 1) or (c == 2) { return 1 } @@ -412,7 +463,13 @@ pub func arith.f32_ge(a: base.u32, b: base.u32) base.u32 { // Note that this is not IEEE 754 minNum: f32_min(NaN, x) and f32_min(x, NaN) // both return the second argument, and f32_min(-0, +0) returns +0. pub func arith.f32_min(a: base.u32, b: base.u32) base.u32 { - if this.f32_cmp(a: args.a, b: args.b) == 0 { + if (args.a | args.b) <= F32_INF { + if args.a < args.b { + return args.a + } + return args.b + } + if this.f32_cmp_slow(a: args.a, b: args.b) == 0 { return args.a } return args.b @@ -420,7 +477,13 @@ pub func arith.f32_min(a: base.u32, b: base.u32) base.u32 { // f32_max returns ((a > b) ? a : b), like brotli's BROTLI_MAX(float, a, b). pub func arith.f32_max(a: base.u32, b: base.u32) base.u32 { - if this.f32_cmp(a: args.a, b: args.b) == 2 { + if (args.a | args.b) <= F32_INF { + if args.a > args.b { + return args.a + } + return args.b + } + if this.f32_cmp_slow(a: args.a, b: args.b) == 2 { return args.a } return args.b From 4dff581dd0cfa3659025c14b762d995d177e4fdf Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 10:43:02 +0000 Subject: [PATCH 5/6] script: check the softfloat comparisons in manual-test-softfloat-sweep.c Add f32_compare and f64_compare, which run the cmp, eq, ne, lt, le, gt, ge, min and max functions on every pair of 594 structured values against the hardware's comparison operators. --- script/manual-test-softfloat-sweep.c | 122 ++++++++++++++++++++++++++- std/softfloat/README.md | 10 ++- 2 files changed, 127 insertions(+), 5 deletions(-) diff --git a/script/manual-test-softfloat-sweep.c b/script/manual-test-softfloat-sweep.c index f7b5cfc5..b47b3301 100644 --- a/script/manual-test-softfloat-sweep.c +++ b/script/manual-test-softfloat-sweep.c @@ -43,9 +43,17 @@ // -o /tmp/sfsweep // // (as one command line), then run /tmp/sfsweep (or /tmp/sfsweep OPNAME to run -// only one op: f32_add, f32_sub, f32_mul, f32_div, f64_to_f32, f32_to_f64, -// f32_from_u32, f32_from_i32 or f32_from_u64). It exits with a non-zero -// status if there were any mismatches. +// only one op: f32_add, f32_sub, f32_mul, f32_div, f64_add, f64_sub, f64_mul, +// f64_div, f64_to_f32, f32_to_f64, f32_from_u32, f32_from_i32, f32_from_u64, +// f32_compare or f64_compare). It exits with a non-zero status if there were +// any mismatches. +// +// The f32_compare and f64_compare checks run the cmp, eq, ne, lt, le, gt, ge, +// min and max functions on every pair of a set of values (both signs, zero, +// subnormal, normal and maximal exponents, boundary mantissas and their +// neighbors, infinities and NaNs), against the hardware's comparison +// operators. For min and max, the result must be the bits of +// ((a < b) ? a : b) and ((a > b) ? a : b). #define _POSIX_C_SOURCE 200809L @@ -464,6 +472,99 @@ static const struct { {"f32_from_u64", ((uint64_t)64) << 26}, }; +static uint64_t // +check_compare32() { + static const uint32_t exps[] = {0, 1, 2, 126, 127, 128, 253, 254, 255}; + static const uint32_t mans[] = {0x000000, 0x000001, 0x000002, 0x000003, + 0x3FFFFF, 0x400000, 0x400001, 0x7FFFFE, + 0x7FFFFF, 0x123456, 0x654321}; + static uint32_t v[2 * 9 * 11 * 3]; + uint32_t n = 0; + for (uint32_t s = 0; s < 2; s++) { + for (uint32_t e = 0; e < 9; e++) { + for (uint32_t m = 0; m < 11; m++) { + uint32_t x = (s << 31) | (exps[e] << 23) | mans[m]; + v[n++] = x; + v[n++] = x + 1; + v[n++] = (x & 0x7FFFFFFF) ? (x - 1) : x; + } + } + } + uint64_t mismatches = 0; + for (uint32_t i = 0; i < n; i++) { + for (uint32_t j = 0; j < n; j++) { + uint32_t a = v[i]; + uint32_t b = v[j]; + float x = f_of(a); + float y = f_of(b); + uint32_t c = ((x != x) || (y != y)) ? 3 : (x < y) ? 0 : (x == y) ? 1 : 2; + bool ok = (wuffs_softfloat__arith__f32_cmp(&g_sf, a, b) == c) && + (wuffs_softfloat__arith__f32_eq(&g_sf, a, b) == (x == y)) && + (wuffs_softfloat__arith__f32_ne(&g_sf, a, b) == (x != y)) && + (wuffs_softfloat__arith__f32_lt(&g_sf, a, b) == (x < y)) && + (wuffs_softfloat__arith__f32_le(&g_sf, a, b) == (x <= y)) && + (wuffs_softfloat__arith__f32_gt(&g_sf, a, b) == (x > y)) && + (wuffs_softfloat__arith__f32_ge(&g_sf, a, b) == (x >= y)) && + (wuffs_softfloat__arith__f32_min(&g_sf, a, b) == + ((x < y) ? a : b)) && + (wuffs_softfloat__arith__f32_max(&g_sf, a, b) == + ((x > y) ? a : b)); + if (!ok && (mismatches++ < 5)) { + printf("MISMATCH f32_compare(0x%08" PRIX32 ", 0x%08" PRIX32 ")\n", a, + b); + } + } + } + return mismatches; +} + +static uint64_t // +check_compare64() { + static const uint64_t exps[] = {0, 1, 2, 1022, 1023, 1024, 2045, 2046, 2047}; + static const uint64_t mans[] = { + 0x0000000000000, 0x0000000000001, 0x0000000000002, 0x0000000000003, + 0x7FFFFFFFFFFFF, 0x8000000000000, 0x8000000000001, 0xFFFFFFFFFFFFE, + 0xFFFFFFFFFFFFF, 0x0123456789ABC, 0xFEDCBA9876543}; + static uint64_t v[2 * 9 * 11 * 3]; + uint32_t n = 0; + for (uint64_t s = 0; s < 2; s++) { + for (uint32_t e = 0; e < 9; e++) { + for (uint32_t m = 0; m < 11; m++) { + uint64_t x = (s << 63) | (exps[e] << 52) | mans[m]; + v[n++] = x; + v[n++] = x + 1; + v[n++] = (x & 0x7FFFFFFFFFFFFFFFull) ? (x - 1) : x; + } + } + } + uint64_t mismatches = 0; + for (uint32_t i = 0; i < n; i++) { + for (uint32_t j = 0; j < n; j++) { + uint64_t a = v[i]; + uint64_t b = v[j]; + double x = d_of(a); + double y = d_of(b); + uint32_t c = ((x != x) || (y != y)) ? 3 : (x < y) ? 0 : (x == y) ? 1 : 2; + bool ok = (wuffs_softfloat__arith__f64_cmp(&g_sf, a, b) == c) && + (wuffs_softfloat__arith__f64_eq(&g_sf, a, b) == (x == y)) && + (wuffs_softfloat__arith__f64_ne(&g_sf, a, b) == (x != y)) && + (wuffs_softfloat__arith__f64_lt(&g_sf, a, b) == (x < y)) && + (wuffs_softfloat__arith__f64_le(&g_sf, a, b) == (x <= y)) && + (wuffs_softfloat__arith__f64_gt(&g_sf, a, b) == (x > y)) && + (wuffs_softfloat__arith__f64_ge(&g_sf, a, b) == (x >= y)) && + (wuffs_softfloat__arith__f64_min(&g_sf, a, b) == + ((x < y) ? a : b)) && + (wuffs_softfloat__arith__f64_max(&g_sf, a, b) == + ((x > y) ? a : b)); + if (!ok && (mismatches++ < 5)) { + printf("MISMATCH f64_compare(0x%016" PRIX64 ", 0x%016" PRIX64 ")\n", a, + b); + } + } + } + return mismatches; +} + static double // now() { struct timespec t; @@ -564,6 +665,21 @@ main(int argc, char** argv) { fflush(stdout); } + for (int w = 0; w < 2; w++) { + const char* name = w ? "f64_compare" : "f32_compare"; + if (only && strcmp(only, name)) { + continue; + } + double t0 = now(); + uint64_t mismatches = w ? check_compare64() : check_compare32(); + total_mismatches += mismatches; + printf("%-14s %12" PRIu64 " operand pairs, %8" PRIu64 + " mismatches, %6.1f seconds\n", + name, (uint64_t)(2 * 9 * 11 * 3) * (2 * 9 * 11 * 3), mismatches, + now() - t0); + fflush(stdout); + } + printf("%s: %" PRIu64 " mismatches in total (%.0f seconds)\n", total_mismatches ? "FAIL" : "PASS", total_mismatches, now() - t_start); diff --git a/std/softfloat/README.md b/std/softfloat/README.md index e9b1dc43..e2294330 100644 --- a/std/softfloat/README.md +++ b/std/softfloat/README.md @@ -219,8 +219,12 @@ operations (every pair of exponents in `[0 ..= 2047]`, 9 x 9 mantissa patterns, NaN results checked bit-for-bit against the NaN rule above), and checks `f64_to_f32` (326 exponents times 2²⁰ fraction patterns times both signs), `f32_to_f64` (every `f32`), `f32_from_u32` and `f32_from_i32` (every -32-bit integer) and `f32_from_u64` (2³² structured values). Result (x86-64, -gcc 13 `-O2`, 4 threads): +32-bit integer) and `f32_from_u64` (2³² structured values). It also checks +the `f32` and `f64` cmp, eq, ne, lt, le, gt, ge, min and max functions on +every pair of 594 values (both signs, zero, subnormal, normal and maximal +exponents, boundary mantissas and their neighbors, infinities and NaNs) +against the hardware's comparison operators. Result (x86-64, gcc 13 `-O2`, 4 +threads): f32_add 419430400 operand pairs, 0 mismatches f32_sub 419430400 operand pairs, 0 mismatches @@ -235,4 +239,6 @@ gcc 13 `-O2`, 4 threads): f32_from_u32 4294967296 operands, 0 mismatches f32_from_i32 4294967296 operands, 0 mismatches f32_from_u64 4294967296 operands, 0 mismatches + f32_compare 352836 operand pairs, 0 mismatches + f64_compare 352836 operand pairs, 0 mismatches PASS: 0 mismatches in total From 85fd6770d5f91e6855726dcea1b6b26951a576a1 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 12:10:41 +0000 Subject: [PATCH 6/6] std/brotli: add decoder fast paths Add non-coroutine read_symbol_fast and read_bits_fast helpers that are tried before the coroutine versions, and a process_commands_fast loop that keeps a 64-bit read-ahead bit buffer and hands back the whole unconsumed bytes when it exits. process_commands now tracks its position in a command (cmd_state) so that both loops can resume each other. LZ77 copies that neither wrap nor reach the ring buffer's end are done in chunks of at most the distance, which matches a byte-by-byte copy. Output is unchanged. On alice29 (decoder reused) decoding is about 1.8x to 2.1x faster for q1, q5, q9 and q11 inputs. --- release/c/wuffs-unsupported-snapshot.c | 785 ++++++++++++++++++++++--- std/brotli/README.md | 33 +- std/brotli/decode_brotli.wuffs | 94 +++ std/brotli/decode_metablock.wuffs | 496 ++++++++++++++-- 4 files changed, 1286 insertions(+), 122 deletions(-) diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 3fd992ac..c60c3ee6 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -9610,6 +9610,7 @@ struct wuffs_brotli__decoder__struct { bool f_implicit_zero_distance; uint32_t f_insert_length; uint32_t f_copy_length; + uint32_t f_cmd_state; bool f_quirk_large_window; bool f_large_window; bool f_started; @@ -9763,7 +9764,6 @@ struct wuffs_brotli__decoder__struct { } s_decode_block_switch; struct { uint32_t v_cmd_code; - uint32_t v_insert_length; uint32_t v_tree; uint32_t v_distance; uint32_t v_i; @@ -9774,6 +9774,7 @@ struct wuffs_brotli__decoder__struct { } s_process_commands; struct { uint32_t v_code; + uint32_t v_nbits; } s_read_distance; } private_data; @@ -77416,6 +77417,20 @@ wuffs_brotli__decoder__read_symbol( wuffs_base__io_buffer* a_src, uint32_t a_hoff); +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__decoder__read_bits_fast( + wuffs_brotli__decoder* self, + wuffs_base__io_buffer* a_src, + uint32_t a_n); + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__decoder__read_symbol_fast( + wuffs_brotli__decoder* self, + wuffs_base__io_buffer* a_src, + uint32_t a_hoff); + WUFFS_BASE__GENERATED_C_CODE static bool wuffs_brotli__decoder__jump_to_byte_boundary( @@ -77695,6 +77710,13 @@ wuffs_brotli__decoder__process_commands( wuffs_base__slice_u8 a_workbuf, wuffs_base__slice_u8 a_dictbuf); +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__status +wuffs_brotli__decoder__process_commands_fast( + wuffs_brotli__decoder* self, + wuffs_base__io_buffer* a_src, + wuffs_base__slice_u8 a_workbuf); + WUFFS_BASE__GENERATED_C_CODE static wuffs_base__empty_struct wuffs_brotli__decoder__update_dist_htree_index( @@ -81730,6 +81752,132 @@ wuffs_brotli__decoder__read_symbol( return status; } +// -------- func brotli.decoder.read_bits_fast + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__decoder__read_bits_fast( + wuffs_brotli__decoder* self, + wuffs_base__io_buffer* a_src, + uint32_t a_n) { + uint32_t v_n = 0; + uint32_t v_bits = 0; + uint32_t v_n_bits = 0; + + const uint8_t* iop_a_src = NULL; + const uint8_t* io0_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + const uint8_t* io1_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + const uint8_t* io2_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + if (a_src && a_src->data.ptr) { + io0_a_src = a_src->data.ptr; + io1_a_src = io0_a_src + a_src->meta.ri; + iop_a_src = io1_a_src; + io2_a_src = io0_a_src + a_src->meta.wi; + } + + v_n = a_n; + v_bits = self->private_impl.f_bits; + v_n_bits = self->private_impl.f_n_bits; + while (v_n_bits < v_n) { + if (((uint64_t)(io2_a_src - iop_a_src)) <= 0u) { + self->private_impl.f_bits = v_bits; + self->private_impl.f_n_bits = v_n_bits; + if (a_src && a_src->data.ptr) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + return 1u; + } + v_bits |= (((uint32_t)(wuffs_base__peek_u8be__no_bounds_check(iop_a_src))) << v_n_bits); + iop_a_src += 1u; + v_n_bits += 8u; + } + self->private_impl.f_bits_value = (v_bits & ((((uint32_t)(1u)) << v_n) - 1u)); + self->private_impl.f_bits = (v_bits >> v_n); + self->private_impl.f_n_bits = (v_n_bits - v_n); + if (a_src && a_src->data.ptr) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + return 0u; +} + +// -------- func brotli.decoder.read_symbol_fast + +WUFFS_BASE__GENERATED_C_CODE +static uint32_t +wuffs_brotli__decoder__read_symbol_fast( + wuffs_brotli__decoder* self, + wuffs_base__io_buffer* a_src, + uint32_t a_hoff) { + uint32_t v_hoff = 0; + uint32_t v_bits = 0; + uint32_t v_n_bits = 0; + uint32_t v_i = 0; + uint32_t v_e = 0; + uint32_t v_e_bits = 0; + + const uint8_t* iop_a_src = NULL; + const uint8_t* io0_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + const uint8_t* io1_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + const uint8_t* io2_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + if (a_src && a_src->data.ptr) { + io0_a_src = a_src->data.ptr; + io1_a_src = io0_a_src + a_src->meta.ri; + iop_a_src = io1_a_src; + io2_a_src = io0_a_src + a_src->meta.wi; + } + + v_hoff = a_hoff; + v_bits = self->private_impl.f_bits; + v_n_bits = self->private_impl.f_n_bits; + while (true) { + v_i = ((uint32_t)(v_hoff + (v_bits & 255u))); + if (v_i >= 678034u) { + break; + } + v_e = ((uint32_t)(self->private_data.f_huffs[v_i])); + v_e_bits = (v_e & 15u); + if (v_e_bits <= 8u) { + if (v_n_bits >= v_e_bits) { + self->private_impl.f_symbol = (v_e >> 4u); + self->private_impl.f_bits = (v_bits >> v_e_bits); + self->private_impl.f_n_bits = (v_n_bits - v_e_bits); + if (a_src && a_src->data.ptr) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + return 0u; + } + } else if (v_n_bits > 8u) { + v_i = ((uint32_t)(((uint32_t)(v_hoff + (v_e >> 4u))) + ((v_bits >> 8u) & ((((uint32_t)(1u)) << (v_e_bits - 8u)) - 1u)))); + if (v_i >= 678034u) { + break; + } + v_e = ((uint32_t)(self->private_data.f_huffs[v_i])); + v_e_bits = ((v_e & 15u) + 8u); + if (v_n_bits >= v_e_bits) { + self->private_impl.f_symbol = (v_e >> 4u); + self->private_impl.f_bits = (v_bits >> v_e_bits); + self->private_impl.f_n_bits = (v_n_bits - v_e_bits); + if (a_src && a_src->data.ptr) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + return 0u; + } + } + if ((v_n_bits >= 24u) || (((uint64_t)(io2_a_src - iop_a_src)) <= 0u)) { + break; + } + v_bits |= (((uint32_t)(wuffs_base__peek_u8be__no_bounds_check(iop_a_src))) << v_n_bits); + iop_a_src += 1u; + v_n_bits += 8u; + } + self->private_impl.f_bits = v_bits; + self->private_impl.f_n_bits = v_n_bits; + if (a_src && a_src->data.ptr) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + return 1u; +} + // -------- func brotli.decoder.jump_to_byte_boundary WUFFS_BASE__GENERATED_C_CODE @@ -84518,7 +84666,6 @@ wuffs_brotli__decoder__process_commands( wuffs_base__status status = wuffs_base__make_status(NULL); uint32_t v_cmd_code = 0; - uint32_t v_insert_length = 0; uint32_t v_p1 = 0; uint32_t v_p2 = 0; uint32_t v_mode_offset = 0; @@ -84534,11 +84681,31 @@ wuffs_brotli__decoder__process_commands( uint32_t v_address = 0; uint32_t v_chunk = 0; uint64_t v_j = 0; + uint32_t v_fr = 0; + uint64_t v_dst_i = 0; + uint64_t v_src_i = 0; + uint64_t v_k64 = 0; + uint64_t v_n64 = 0; + uint64_t v_dlo = 0; + uint64_t v_dhi = 0; + uint64_t v_slo = 0; + uint64_t v_shi = 0; + wuffs_base__status v_fs = wuffs_base__make_status(NULL); + + const uint8_t* iop_a_src = NULL; + const uint8_t* io0_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + const uint8_t* io1_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + const uint8_t* io2_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + if (a_src && a_src->data.ptr) { + io0_a_src = a_src->data.ptr; + io1_a_src = io0_a_src + a_src->meta.ri; + iop_a_src = io1_a_src; + io2_a_src = io0_a_src + a_src->meta.wi; + } uint32_t coro_susp_point = self->private_impl.p_process_commands; if (coro_susp_point) { v_cmd_code = self->private_data.s_process_commands.v_cmd_code; - v_insert_length = self->private_data.s_process_commands.v_insert_length; v_tree = self->private_data.s_process_commands.v_tree; v_distance = self->private_data.s_process_commands.v_distance; v_i = self->private_data.s_process_commands.v_i; @@ -84551,47 +84718,134 @@ wuffs_brotli__decoder__process_commands( WUFFS_BASE__COROUTINE_SUSPENSION_POINT_0; while (true) { - if (self->private_impl.f_block_length[1u] == 0u) { - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(1); - status = wuffs_brotli__decoder__decode_block_switch(self, a_src, 1u); - if (status.repr) { - goto suspend; + if ((self->private_impl.f_cmd_state == 0u) && (((uint64_t)(io2_a_src - iop_a_src)) >= 16u)) { + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + v_fs = wuffs_brotli__decoder__process_commands_fast(self, a_src, a_workbuf); + if (a_src) { + iop_a_src = a_src->data.ptr + a_src->meta.ri; + } + if ( ! wuffs_base__status__is_ok(&v_fs)) { + status = v_fs; + if (wuffs_base__status__is_error(&status)) { + goto exit; + } else if (wuffs_base__status__is_suspension(&status)) { + status = wuffs_base__make_status(wuffs_base__error__cannot_return_a_suspension); + goto exit; + } + goto ok; + } else if (self->private_impl.f_cmd_state == 4u) { + self->private_impl.f_cmd_state = 0u; + status = wuffs_base__make_status(NULL); + goto ok; } - self->private_impl.f_htree_command = (self->private_impl.f_block_type_rb[3u] & 255u); - continue; } - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(2); - status = wuffs_brotli__decoder__read_symbol(self, a_src, (161792u + (self->private_impl.f_htree_command * 1080u))); - if (status.repr) { - goto suspend; + if (self->private_impl.f_cmd_state == 0u) { + if (self->private_impl.f_block_length[1u] == 0u) { + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(1); + status = wuffs_brotli__decoder__decode_block_switch(self, a_src, 1u); + if (a_src) { + iop_a_src = a_src->data.ptr + a_src->meta.ri; + } + if (status.repr) { + goto suspend; + } + self->private_impl.f_htree_command = (self->private_impl.f_block_type_rb[3u] & 255u); + continue; + } + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + v_fr = wuffs_brotli__decoder__read_symbol_fast(self, a_src, (161792u + (self->private_impl.f_htree_command * 1080u))); + if (a_src) { + iop_a_src = a_src->data.ptr + a_src->meta.ri; + } + if (v_fr != 0u) { + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(2); + status = wuffs_brotli__decoder__read_symbol(self, a_src, (161792u + (self->private_impl.f_htree_command * 1080u))); + if (a_src) { + iop_a_src = a_src->data.ptr + a_src->meta.ri; + } + if (status.repr) { + goto suspend; + } + } + if (self->private_impl.f_symbol >= 704u) { + status = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); + goto exit; + } + v_cmd_code = wuffs_base__u32__min(self->private_impl.f_symbol, 703u); + self->private_impl.f_implicit_zero_distance = (WUFFS_BROTLI__CMD_IMPLICIT_ZERO_DISTANCE[v_cmd_code] != 0u); + self->private_impl.f_distance_context = ((uint32_t)(WUFFS_BROTLI__CMD_DISTANCE_CONTEXT[v_cmd_code])); + wuffs_brotli__decoder__update_dist_htree_index(self); + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + v_fr = wuffs_brotli__decoder__read_bits_fast(self, a_src, ((uint32_t)(WUFFS_BROTLI__CMD_INSERT_LEN_EXTRA_BITS[v_cmd_code]))); + if (a_src) { + iop_a_src = a_src->data.ptr + a_src->meta.ri; + } + if (v_fr != 0u) { + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(3); + status = wuffs_brotli__decoder__read_bits(self, a_src, ((uint32_t)(WUFFS_BROTLI__CMD_INSERT_LEN_EXTRA_BITS[v_cmd_code]))); + if (a_src) { + iop_a_src = a_src->data.ptr + a_src->meta.ri; + } + if (status.repr) { + goto suspend; + } + } + self->private_impl.f_insert_length = ((uint32_t)(WUFFS_BROTLI__CMD_INSERT_LEN_OFFSET[v_cmd_code] + self->private_impl.f_bits_value)); + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + v_fr = wuffs_brotli__decoder__read_bits_fast(self, a_src, ((uint32_t)(WUFFS_BROTLI__CMD_COPY_LEN_EXTRA_BITS[v_cmd_code]))); + if (a_src) { + iop_a_src = a_src->data.ptr + a_src->meta.ri; + } + if (v_fr != 0u) { + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(4); + status = wuffs_brotli__decoder__read_bits(self, a_src, ((uint32_t)(WUFFS_BROTLI__CMD_COPY_LEN_EXTRA_BITS[v_cmd_code]))); + if (a_src) { + iop_a_src = a_src->data.ptr + a_src->meta.ri; + } + if (status.repr) { + goto suspend; + } + } + self->private_impl.f_copy_length = ((uint32_t)(WUFFS_BROTLI__CMD_COPY_LEN_OFFSET[v_cmd_code] + self->private_impl.f_bits_value)); + self->private_impl.f_block_length[1u] -= 1u; + if (self->private_impl.f_insert_length > 0u) { + self->private_impl.f_mb_remaining -= self->private_impl.f_insert_length; + self->private_impl.f_cmd_state = 1u; + } else { + self->private_impl.f_cmd_state = 2u; + } } - if (self->private_impl.f_symbol >= 704u) { - status = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); - goto exit; - } - v_cmd_code = wuffs_base__u32__min(self->private_impl.f_symbol, 703u); - self->private_impl.f_implicit_zero_distance = (WUFFS_BROTLI__CMD_IMPLICIT_ZERO_DISTANCE[v_cmd_code] != 0u); - self->private_impl.f_distance_context = ((uint32_t)(WUFFS_BROTLI__CMD_DISTANCE_CONTEXT[v_cmd_code])); - wuffs_brotli__decoder__update_dist_htree_index(self); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(3); - status = wuffs_brotli__decoder__read_bits(self, a_src, ((uint32_t)(WUFFS_BROTLI__CMD_INSERT_LEN_EXTRA_BITS[v_cmd_code]))); - if (status.repr) { - goto suspend; - } - v_insert_length = ((uint32_t)(WUFFS_BROTLI__CMD_INSERT_LEN_OFFSET[v_cmd_code] + self->private_impl.f_bits_value)); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(4); - status = wuffs_brotli__decoder__read_bits(self, a_src, ((uint32_t)(WUFFS_BROTLI__CMD_COPY_LEN_EXTRA_BITS[v_cmd_code]))); - if (status.repr) { - goto suspend; - } - self->private_impl.f_copy_length = ((uint32_t)(WUFFS_BROTLI__CMD_COPY_LEN_OFFSET[v_cmd_code] + self->private_impl.f_bits_value)); - self->private_impl.f_block_length[1u] -= 1u; - if (v_insert_length > 0u) { - self->private_impl.f_mb_remaining -= v_insert_length; - while (v_insert_length > 0u) { + if (self->private_impl.f_cmd_state == 1u) { + while (self->private_impl.f_insert_length > 0u) { if (self->private_impl.f_block_length[0u] == 0u) { + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } WUFFS_BASE__COROUTINE_SUSPENSION_POINT(5); status = wuffs_brotli__decoder__decode_block_switch(self, a_src, 0u); + if (a_src) { + iop_a_src = a_src->data.ptr + a_src->meta.ri; + } if (status.repr) { goto suspend; } @@ -84603,14 +84857,29 @@ wuffs_brotli__decoder__process_commands( v_mode_offset = (self->private_impl.f_context_mode << 9u); v_ctx = ((uint32_t)(((uint8_t)(WUFFS_BROTLI__CONTEXT_LOOKUP[(v_mode_offset | v_p1)] | WUFFS_BROTLI__CONTEXT_LOOKUP[(v_mode_offset | 256u | v_p2)])))); v_tree = ((uint32_t)(self->private_data.f_context_maps[(self->private_impl.f_context_map_slice + v_ctx)])); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(6); - status = wuffs_brotli__decoder__read_symbol(self, a_src, (v_tree * 632u)); - if (status.repr) { - goto suspend; + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + v_fr = wuffs_brotli__decoder__read_symbol_fast(self, a_src, (v_tree * 632u)); + if (a_src) { + iop_a_src = a_src->data.ptr + a_src->meta.ri; + } + if (v_fr != 0u) { + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(6); + status = wuffs_brotli__decoder__read_symbol(self, a_src, (v_tree * 632u)); + if (a_src) { + iop_a_src = a_src->data.ptr + a_src->meta.ri; + } + if (status.repr) { + goto suspend; + } } wuffs_brotli__decoder__put_byte(self, a_workbuf, ((uint8_t)(self->private_impl.f_symbol))); self->private_impl.f_block_length[0u] -= 1u; - v_insert_length -= 1u; + self->private_impl.f_insert_length -= 1u; if (self->private_impl.f_pos == self->private_impl.f_rb_size) { WUFFS_BASE__COROUTINE_SUSPENSION_POINT(7); status = wuffs_brotli__decoder__ring_event(self); @@ -84619,30 +84888,47 @@ wuffs_brotli__decoder__process_commands( } } } + self->private_impl.f_cmd_state = 2u; if ((self->private_impl.f_mb_remaining == 0u) || (self->private_impl.f_mb_remaining >= 2147483648u)) { + self->private_impl.f_cmd_state = 0u; status = wuffs_base__make_status(NULL); goto ok; } } - if (self->private_impl.f_implicit_zero_distance) { - self->private_impl.f_distance_context = 1u; - self->private_impl.f_dist_rb_idx -= 1u; - self->private_impl.f_distance_code = self->private_impl.f_dist_rb[(self->private_impl.f_dist_rb_idx & 3u)]; - } else { - if (self->private_impl.f_block_length[2u] == 0u) { - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(8); - status = wuffs_brotli__decoder__decode_block_switch(self, a_src, 2u); + if (self->private_impl.f_cmd_state == 2u) { + if (self->private_impl.f_implicit_zero_distance) { + self->private_impl.f_distance_context = 1u; + self->private_impl.f_dist_rb_idx -= 1u; + self->private_impl.f_distance_code = self->private_impl.f_dist_rb[(self->private_impl.f_dist_rb_idx & 3u)]; + } else { + if (self->private_impl.f_block_length[2u] == 0u) { + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(8); + status = wuffs_brotli__decoder__decode_block_switch(self, a_src, 2u); + if (a_src) { + iop_a_src = a_src->data.ptr + a_src->meta.ri; + } + if (status.repr) { + goto suspend; + } + self->private_impl.f_dist_context_map_slice = ((self->private_impl.f_block_type_rb[5u] & 255u) << 2u); + wuffs_brotli__decoder__update_dist_htree_index(self); + } + if (a_src) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(9); + status = wuffs_brotli__decoder__read_distance(self, a_src); + if (a_src) { + iop_a_src = a_src->data.ptr + a_src->meta.ri; + } if (status.repr) { goto suspend; } - self->private_impl.f_dist_context_map_slice = ((self->private_impl.f_block_type_rb[5u] & 255u) << 2u); - wuffs_brotli__decoder__update_dist_htree_index(self); - } - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(9); - status = wuffs_brotli__decoder__read_distance(self, a_src); - if (status.repr) { - goto suspend; } + self->private_impl.f_cmd_state = 3u; } if (self->private_impl.f_max_distance != self->private_impl.f_max_backward_distance) { self->private_impl.f_max_distance = wuffs_base__u32__min(self->private_impl.f_pos, self->private_impl.f_max_backward_distance); @@ -84732,6 +85018,33 @@ wuffs_brotli__decoder__process_commands( self->private_impl.f_dist_rb[(self->private_impl.f_dist_rb_idx & 3u)] = v_distance; self->private_impl.f_dist_rb_idx += 1u; self->private_impl.f_mb_remaining -= v_i; + v_dst_i = ((uint64_t)(self->private_impl.f_pos)); + v_src_i = ((uint64_t)((((uint32_t)(self->private_impl.f_pos - v_distance)) & ((uint32_t)(self->private_impl.f_window_size - 1u))))); + if (((v_dst_i + ((uint64_t)(v_i))) < ((uint64_t)(self->private_impl.f_rb_size))) && + ((v_src_i + ((uint64_t)(v_i))) <= ((uint64_t)(self->private_impl.f_window_size))) && + (((uint64_t)(self->private_impl.f_window_size)) <= ((uint64_t)(a_workbuf.len))) && + (v_distance > 0u)) { + v_k64 = 0u; + while (v_k64 < ((uint64_t)(v_i))) { + v_n64 = ((uint64_t)(((uint64_t)(v_i)) - v_k64)); + v_n64 = wuffs_base__u64__min(v_n64, ((uint64_t)(v_distance))); + v_dlo = ((uint64_t)(v_dst_i + v_k64)); + v_dhi = ((uint64_t)(v_dlo + v_n64)); + v_slo = ((uint64_t)(v_src_i + v_k64)); + v_shi = ((uint64_t)(v_slo + v_n64)); + if ((v_n64 == 0u) || + (v_dlo > v_dhi) || + (v_dhi > ((uint64_t)(a_workbuf.len))) || + (v_slo > v_shi) || + (v_shi > ((uint64_t)(a_workbuf.len)))) { + break; + } + wuffs_private_impl__slice_u8__copy_from_slice(wuffs_base__slice_u8__subslice_ij(a_workbuf, v_dlo, v_dhi), wuffs_base__slice_u8__subslice_ij(a_workbuf, v_slo, v_shi)); + v_k64 += v_n64; + } + self->private_impl.f_pos += ((uint32_t)(v_k64)); + v_i -= ((uint32_t)(v_k64)); + } while (v_i > 0u) { v_b = wuffs_brotli__decoder__get_byte(self, a_workbuf, v_distance); wuffs_brotli__decoder__put_byte(self, a_workbuf, v_b); @@ -84745,6 +85058,7 @@ wuffs_brotli__decoder__process_commands( } } } + self->private_impl.f_cmd_state = 0u; if ((self->private_impl.f_mb_remaining == 0u) || (self->private_impl.f_mb_remaining >= 2147483648u)) { status = wuffs_base__make_status(NULL); goto ok; @@ -84760,7 +85074,6 @@ wuffs_brotli__decoder__process_commands( suspend: self->private_impl.p_process_commands = wuffs_base__status__is_suspension(&status) ? coro_susp_point : 0; self->private_data.s_process_commands.v_cmd_code = v_cmd_code; - self->private_data.s_process_commands.v_insert_length = v_insert_length; self->private_data.s_process_commands.v_tree = v_tree; self->private_data.s_process_commands.v_distance = v_distance; self->private_data.s_process_commands.v_i = v_i; @@ -84771,6 +85084,330 @@ wuffs_brotli__decoder__process_commands( goto exit; exit: + if (a_src && a_src->data.ptr) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + + return status; +} + +// -------- func brotli.decoder.process_commands_fast + +WUFFS_BASE__GENERATED_C_CODE +static wuffs_base__status +wuffs_brotli__decoder__process_commands_fast( + wuffs_brotli__decoder* self, + wuffs_base__io_buffer* a_src, + wuffs_base__slice_u8 a_workbuf) { + wuffs_base__status status = wuffs_base__make_status(NULL); + + wuffs_base__status v_ret = wuffs_base__make_status(NULL); + uint64_t v_bits = 0; + uint32_t v_n_bits = 0; + uint32_t v_i = 0; + uint32_t v_e = 0; + uint32_t v_e_bits = 0; + uint32_t v_cmd_code = 0; + uint32_t v_nb = 0; + uint32_t v_p1 = 0; + uint32_t v_p2 = 0; + uint32_t v_mode_offset = 0; + uint32_t v_ctx = 0; + uint32_t v_tree = 0; + uint32_t v_code = 0; + uint32_t v_nbits = 0; + uint32_t v_lo = 0; + uint32_t v_max_distance = 0; + uint32_t v_distance = 0; + uint32_t v_n = 0; + uint64_t v_dst_i = 0; + uint64_t v_src_i = 0; + uint64_t v_k64 = 0; + uint64_t v_n64 = 0; + uint64_t v_dlo = 0; + uint64_t v_dhi = 0; + uint64_t v_slo = 0; + uint64_t v_shi = 0; + + const uint8_t* iop_a_src = NULL; + const uint8_t* io0_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + const uint8_t* io1_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + const uint8_t* io2_a_src WUFFS_BASE__POTENTIALLY_UNUSED = NULL; + if (a_src && a_src->data.ptr) { + io0_a_src = a_src->data.ptr; + io1_a_src = io0_a_src + a_src->meta.ri; + iop_a_src = io1_a_src; + io2_a_src = io0_a_src + a_src->meta.wi; + } + + if (self->private_impl.f_n_bits >= 8u) { + status = wuffs_base__make_status(NULL); + goto ok; + } + v_n_bits = (self->private_impl.f_n_bits & 7u); + v_bits = (((uint64_t)(self->private_impl.f_bits)) & ((((uint64_t)(1u)) << v_n_bits) - 1u)); + while (true) { + if (self->private_impl.f_cmd_state == 0u) { + if ((self->private_impl.f_block_length[1u] == 0u) || (((uint64_t)(io2_a_src - iop_a_src)) < 16u)) { + break; + } + v_bits |= ((uint64_t)(wuffs_base__peek_u64le__no_bounds_check(iop_a_src) << v_n_bits)); + iop_a_src += ((63u - v_n_bits) >> 3u); + v_n_bits |= 56u; + v_i = ((uint32_t)((161792u + (self->private_impl.f_htree_command * 1080u)) + ((uint32_t)((v_bits & 255u))))); + if (v_i >= 678034u) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); + break; + } + v_e = ((uint32_t)(self->private_data.f_huffs[v_i])); + v_e_bits = (v_e & 15u); + if (v_e_bits > 8u) { + v_i = ((uint32_t)(((uint32_t)((161792u + (self->private_impl.f_htree_command * 1080u)) + (v_e >> 4u))) + (((uint32_t)(((v_bits >> 8u) & 32767u))) & ((((uint32_t)(1u)) << (v_e_bits - 8u)) - 1u)))); + if (v_i >= 678034u) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); + break; + } + v_e = ((uint32_t)(self->private_data.f_huffs[v_i])); + v_e_bits = ((v_e & 15u) + 8u); + } + if (v_n_bits < v_e_bits) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_bit_reader); + break; + } + v_bits >>= v_e_bits; + v_n_bits -= v_e_bits; + if ((v_e >> 4u) >= 704u) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); + break; + } + self->private_impl.f_symbol = (v_e >> 4u); + v_cmd_code = wuffs_base__u32__min(self->private_impl.f_symbol, 703u); + self->private_impl.f_implicit_zero_distance = (WUFFS_BROTLI__CMD_IMPLICIT_ZERO_DISTANCE[v_cmd_code] != 0u); + self->private_impl.f_distance_context = ((uint32_t)(WUFFS_BROTLI__CMD_DISTANCE_CONTEXT[v_cmd_code])); + wuffs_brotli__decoder__update_dist_htree_index(self); + v_nb = ((uint32_t)(WUFFS_BROTLI__CMD_INSERT_LEN_EXTRA_BITS[v_cmd_code])); + if (v_n_bits < v_nb) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_bit_reader); + break; + } + self->private_impl.f_bits_value = ((uint32_t)((v_bits & ((((uint64_t)(1u)) << v_nb) - 1u)))); + v_bits >>= v_nb; + v_n_bits -= v_nb; + self->private_impl.f_insert_length = ((uint32_t)(WUFFS_BROTLI__CMD_INSERT_LEN_OFFSET[v_cmd_code] + self->private_impl.f_bits_value)); + if (((uint64_t)(io2_a_src - iop_a_src)) < 8u) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_bit_reader); + break; + } + v_bits |= ((uint64_t)(wuffs_base__peek_u64le__no_bounds_check(iop_a_src) << v_n_bits)); + iop_a_src += ((63u - v_n_bits) >> 3u); + v_n_bits |= 56u; + v_nb = ((uint32_t)(WUFFS_BROTLI__CMD_COPY_LEN_EXTRA_BITS[v_cmd_code])); + if (v_n_bits < v_nb) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_bit_reader); + break; + } + self->private_impl.f_bits_value = ((uint32_t)((v_bits & ((((uint64_t)(1u)) << v_nb) - 1u)))); + v_bits >>= v_nb; + v_n_bits -= v_nb; + self->private_impl.f_copy_length = ((uint32_t)(WUFFS_BROTLI__CMD_COPY_LEN_OFFSET[v_cmd_code] + self->private_impl.f_bits_value)); + self->private_impl.f_block_length[1u] -= 1u; + if (self->private_impl.f_insert_length > 0u) { + self->private_impl.f_mb_remaining -= self->private_impl.f_insert_length; + self->private_impl.f_cmd_state = 1u; + } else { + self->private_impl.f_cmd_state = 2u; + } + } + if (self->private_impl.f_cmd_state == 1u) { + if (((uint64_t)(self->private_impl.f_window_size)) > ((uint64_t)(a_workbuf.len))) { + break; + } + v_p1 = ((uint32_t)(wuffs_brotli__decoder__get_byte(self, a_workbuf, 1u))); + v_p2 = ((uint32_t)(wuffs_brotli__decoder__get_byte(self, a_workbuf, 2u))); + while (self->private_impl.f_insert_length > 0u) { + if ((self->private_impl.f_block_length[0u] == 0u) || (((uint32_t)(self->private_impl.f_pos + 1u)) >= self->private_impl.f_rb_size)) { + goto label__outer__break; + } + if (v_n_bits < 24u) { + if (((uint64_t)(io2_a_src - iop_a_src)) < 8u) { + goto label__outer__break; + } + v_bits |= ((uint64_t)(wuffs_base__peek_u64le__no_bounds_check(iop_a_src) << v_n_bits)); + iop_a_src += ((63u - v_n_bits) >> 3u); + v_n_bits |= 56u; + } + v_mode_offset = (self->private_impl.f_context_mode << 9u); + v_ctx = ((uint32_t)(((uint8_t)(WUFFS_BROTLI__CONTEXT_LOOKUP[(v_mode_offset | v_p1)] | WUFFS_BROTLI__CONTEXT_LOOKUP[(v_mode_offset | 256u | v_p2)])))); + v_tree = ((uint32_t)(self->private_data.f_context_maps[(self->private_impl.f_context_map_slice + v_ctx)])); + v_i = ((uint32_t)((v_tree * 632u) + ((uint32_t)((v_bits & 255u))))); + if (v_i >= 678034u) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); + goto label__outer__break; + } + v_e = ((uint32_t)(self->private_data.f_huffs[v_i])); + v_e_bits = (v_e & 15u); + if (v_e_bits > 8u) { + v_i = ((uint32_t)(((uint32_t)((v_tree * 632u) + (v_e >> 4u))) + (((uint32_t)(((v_bits >> 8u) & 32767u))) & ((((uint32_t)(1u)) << (v_e_bits - 8u)) - 1u)))); + if (v_i >= 678034u) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); + goto label__outer__break; + } + v_e = ((uint32_t)(self->private_data.f_huffs[v_i])); + v_e_bits = ((v_e & 15u) + 8u); + } + if (v_n_bits < v_e_bits) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_bit_reader); + goto label__outer__break; + } + v_bits >>= v_e_bits; + v_n_bits -= v_e_bits; + self->private_impl.f_symbol = (v_e >> 4u); + wuffs_brotli__decoder__put_byte(self, a_workbuf, ((uint8_t)(self->private_impl.f_symbol))); + v_p2 = v_p1; + v_p1 = (self->private_impl.f_symbol & 255u); + self->private_impl.f_block_length[0u] -= 1u; + self->private_impl.f_insert_length -= 1u; + } + self->private_impl.f_cmd_state = 2u; + if ((self->private_impl.f_mb_remaining == 0u) || (self->private_impl.f_mb_remaining >= 2147483648u)) { + self->private_impl.f_cmd_state = 4u; + break; + } + } + if (self->private_impl.f_cmd_state == 2u) { + if (self->private_impl.f_implicit_zero_distance) { + self->private_impl.f_distance_context = 1u; + self->private_impl.f_dist_rb_idx -= 1u; + self->private_impl.f_distance_code = self->private_impl.f_dist_rb[(self->private_impl.f_dist_rb_idx & 3u)]; + } else { + if ((self->private_impl.f_block_length[2u] == 0u) || (((uint64_t)(io2_a_src - iop_a_src)) < 8u)) { + break; + } + v_bits |= ((uint64_t)(wuffs_base__peek_u64le__no_bounds_check(iop_a_src) << v_n_bits)); + iop_a_src += ((63u - v_n_bits) >> 3u); + v_n_bits |= 56u; + v_i = ((uint32_t)((438272u + (self->private_impl.f_dist_htree_index * 920u)) + ((uint32_t)((v_bits & 255u))))); + if (v_i >= 678034u) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); + break; + } + v_e = ((uint32_t)(self->private_data.f_huffs[v_i])); + v_e_bits = (v_e & 15u); + if (v_e_bits > 8u) { + v_i = ((uint32_t)(((uint32_t)((438272u + (self->private_impl.f_dist_htree_index * 920u)) + (v_e >> 4u))) + (((uint32_t)(((v_bits >> 8u) & 32767u))) & ((((uint32_t)(1u)) << (v_e_bits - 8u)) - 1u)))); + if (v_i >= 678034u) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); + break; + } + v_e = ((uint32_t)(self->private_data.f_huffs[v_i])); + v_e_bits = ((v_e & 15u) + 8u); + } + if (v_n_bits < v_e_bits) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_bit_reader); + break; + } + v_bits >>= v_e_bits; + v_n_bits -= v_e_bits; + self->private_impl.f_symbol = (v_e >> 4u); + v_code = self->private_impl.f_symbol; + self->private_impl.f_block_length[2u] -= 1u; + self->private_impl.f_distance_context = 0u; + if (v_code < 16u) { + wuffs_brotli__decoder__take_distance_from_ring_buffer(self, v_code); + } else { + if ((v_code >= self->private_impl.f_distance_alphabet_size) || (v_code >= 1128u)) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); + break; + } + v_nbits = ((uint32_t)(wuffs_base__u8__min(self->private_data.f_dist_extra_bits[v_code], 32u))); + if (v_n_bits < v_nbits) { + v_ret = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_bit_reader); + break; + } + v_lo = ((uint32_t)((v_bits & ((((uint64_t)(1u)) << v_nbits) - 1u)))); + self->private_impl.f_bits_value = v_lo; + v_bits >>= v_nbits; + v_n_bits -= v_nbits; + self->private_impl.f_distance_code = ((uint32_t)(((uint64_t)(((uint64_t)(self->private_data.f_dist_offset[v_code])) + ((uint64_t)(((uint64_t)(self->private_impl.f_bits_value)) << self->private_impl.f_distance_postfix_bits)))))); + } + } + self->private_impl.f_cmd_state = 3u; + } + v_max_distance = self->private_impl.f_max_distance; + if (v_max_distance != self->private_impl.f_max_backward_distance) { + v_max_distance = wuffs_base__u32__min(self->private_impl.f_pos, self->private_impl.f_max_backward_distance); + } + v_distance = self->private_impl.f_distance_code; + v_n = self->private_impl.f_copy_length; + if ((v_distance > v_max_distance) || (v_distance == 0u)) { + break; + } + v_dst_i = ((uint64_t)(self->private_impl.f_pos)); + v_src_i = ((uint64_t)((((uint32_t)(self->private_impl.f_pos - v_distance)) & ((uint32_t)(self->private_impl.f_window_size - 1u))))); + if (((v_dst_i + ((uint64_t)(v_n))) >= ((uint64_t)(self->private_impl.f_rb_size))) || ((v_src_i + ((uint64_t)(v_n))) > ((uint64_t)(self->private_impl.f_window_size))) || (((uint64_t)(self->private_impl.f_window_size)) > ((uint64_t)(a_workbuf.len)))) { + break; + } + self->private_impl.f_max_distance = v_max_distance; + self->private_impl.f_dist_rb[(self->private_impl.f_dist_rb_idx & 3u)] = v_distance; + self->private_impl.f_dist_rb_idx += 1u; + self->private_impl.f_mb_remaining -= v_n; + v_k64 = 0u; + while (v_k64 < ((uint64_t)(v_n))) { + v_n64 = ((uint64_t)(((uint64_t)(v_n)) - v_k64)); + v_n64 = wuffs_base__u64__min(v_n64, ((uint64_t)(v_distance))); + v_dlo = ((uint64_t)(v_dst_i + v_k64)); + v_dhi = ((uint64_t)(v_dlo + v_n64)); + v_slo = ((uint64_t)(v_src_i + v_k64)); + v_shi = ((uint64_t)(v_slo + v_n64)); + if ((v_n64 == 0u) || + (v_dlo > v_dhi) || + (v_dhi > ((uint64_t)(a_workbuf.len))) || + (v_slo > v_shi) || + (v_shi > ((uint64_t)(a_workbuf.len)))) { + break; + } + wuffs_private_impl__slice_u8__copy_from_slice(wuffs_base__slice_u8__subslice_ij(a_workbuf, v_dlo, v_dhi), wuffs_base__slice_u8__subslice_ij(a_workbuf, v_slo, v_shi)); + v_k64 += v_n64; + } + self->private_impl.f_pos += ((uint32_t)(v_k64)); + v_n -= ((uint32_t)(v_k64)); + while (v_n > 0u) { + wuffs_brotli__decoder__put_byte(self, a_workbuf, wuffs_brotli__decoder__get_byte(self, a_workbuf, v_distance)); + v_n -= 1u; + } + self->private_impl.f_cmd_state = 0u; + if ((self->private_impl.f_mb_remaining == 0u) || (self->private_impl.f_mb_remaining >= 2147483648u)) { + self->private_impl.f_cmd_state = 4u; + break; + } + } + label__outer__break:; + while (v_n_bits >= 8u) { + if ( ! (iop_a_src > io1_a_src)) { + status = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_bit_reader); + goto exit; + } + iop_a_src--; + v_n_bits -= 8u; + } + self->private_impl.f_bits = ((uint32_t)((v_bits & ((((uint64_t)(1u)) << v_n_bits) - 1u)))); + self->private_impl.f_n_bits = v_n_bits; + status = v_ret; + if (wuffs_base__status__is_error(&status)) { + goto exit; + } else if (wuffs_base__status__is_suspension(&status)) { + status = wuffs_base__make_status(wuffs_base__error__cannot_return_a_suspension); + goto exit; + } + goto ok; + + ok: + goto exit; + exit: + if (a_src && a_src->data.ptr) { + a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); + } + return status; } @@ -84794,18 +85431,24 @@ wuffs_brotli__decoder__read_distance( wuffs_base__status status = wuffs_base__make_status(NULL); uint32_t v_code = 0; + uint32_t v_fr = 0; + uint32_t v_nbits = 0; uint32_t coro_susp_point = self->private_impl.p_read_distance; if (coro_susp_point) { v_code = self->private_data.s_read_distance.v_code; + v_nbits = self->private_data.s_read_distance.v_nbits; } switch (coro_susp_point) { WUFFS_BASE__COROUTINE_SUSPENSION_POINT_0; - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(1); - status = wuffs_brotli__decoder__read_symbol(self, a_src, (438272u + (self->private_impl.f_dist_htree_index * 920u))); - if (status.repr) { - goto suspend; + v_fr = wuffs_brotli__decoder__read_symbol_fast(self, a_src, (438272u + (self->private_impl.f_dist_htree_index * 920u))); + if (v_fr != 0u) { + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(1); + status = wuffs_brotli__decoder__read_symbol(self, a_src, (438272u + (self->private_impl.f_dist_htree_index * 920u))); + if (status.repr) { + goto suspend; + } } v_code = self->private_impl.f_symbol; self->private_impl.f_block_length[2u] -= 1u; @@ -84819,10 +85462,17 @@ wuffs_brotli__decoder__read_distance( status = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); goto exit; } - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(2); - status = wuffs_brotli__decoder__read_bits32(self, a_src, ((uint32_t)(wuffs_base__u8__min(self->private_data.f_dist_extra_bits[v_code], 32u)))); - if (status.repr) { - goto suspend; + v_nbits = ((uint32_t)(wuffs_base__u8__min(self->private_data.f_dist_extra_bits[v_code], 32u))); + v_fr = 1u; + if (v_nbits <= 24u) { + v_fr = wuffs_brotli__decoder__read_bits_fast(self, a_src, v_nbits); + } + if (v_fr != 0u) { + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(2); + status = wuffs_brotli__decoder__read_bits32(self, a_src, v_nbits); + if (status.repr) { + goto suspend; + } } if (v_code >= 1128u) { status = wuffs_base__make_status(wuffs_brotli__error__internal_error_inconsistent_huffman_table); @@ -84839,6 +85489,7 @@ wuffs_brotli__decoder__read_distance( suspend: self->private_impl.p_read_distance = wuffs_base__status__is_suspension(&status) ? coro_susp_point : 0; self->private_data.s_read_distance.v_code = v_code; + self->private_data.s_read_distance.v_nbits = v_nbits; goto exit; exit: diff --git a/std/brotli/README.md b/std/brotli/README.md index daa973de..d16e596b 100644 --- a/std/brotli/README.md +++ b/std/brotli/README.md @@ -55,8 +55,9 @@ Without the large-window quirk and without attached dictionaries (see the window size plus some slack. The slack mirrors the C code's `kRingBufferWriteAheadSlack`, which its fast paths use to write (and discard) a few bytes past the ring buffer's end. This -implementation does not yet have such fast paths, but it still reserves the -slack, so that adding them later will not change the `workbuf_len` API. A +implementation's fast paths (see below) do not write past the ring buffer's +end, but it still reserves the slack, so that adding such fast paths later +will not change the `workbuf_len` API. A workbuf shorter than the stream's window size plus 542 bytes is rejected with `base."#bad workbuf length"`. @@ -138,12 +139,25 @@ invariants for memory safety but Wuffs requires explicit bounds checks. - The bit reader reads input bytes only when it needs them (like the C code's "safe" `BrotliSafeReadBits` and `SafeReadSymbol` functions), so that - `transform_io` can suspend at any point. There is no "fast path" (like the C - code's `ProcessCommands`, as opposed to `SafeProcessCommands`). Both paths - produce the same output and errors. -- LZ77 copies and dictionary words are written to the ring buffer one byte at - a time, with ring buffer events (the C code's `WriteRingBuffer` and - `WrapRingBuffer` calls) at the same positions as the C code. + `transform_io` can suspend at any point. Between reads, fewer than 8 bits + are buffered. The fast path, `process_commands_fast` (like the C code's + `ProcessCommands`, as opposed to `SafeProcessCommands`), is not a coroutine. + It reads ahead (up to 8 bytes at a time) into a 64-bit bit buffer while the + input has enough bytes, and when it returns to `process_commands` (at a + block switch, a ring buffer event, a dictionary reference or when the input + runs low), it gives back the whole bytes left in that buffer, which restores + the "fewer than 8 bits" state exactly. `process_commands` tracks where it is + within a command (`cmd_state`), so that either can pick up where the other + left off. Within `process_commands`, `read_symbol_fast` and `read_bits_fast` + (which are not coroutines either) read the same bytes at the same times as + the coroutines `read_symbol` and `read_bits`, which finish the job when the + input runs out. All paths produce the same output and errors. +- Dictionary words, and LZ77 copies that wrap around the physical ring buffer + or reach the C code's (virtual) ring buffer size, are written to the ring + buffer one byte at a time, with ring buffer events (the C code's + `WriteRingBuffer` and `WrapRingBuffer` calls) at the same positions as the + C code. Other LZ77 copies are written in chunks of at most the copy distance + (so that each chunk's source bytes are final), which is equivalent. - On error, the Wuffs decoder may have already written more output to `dst` than libbrotlidec would have (libbrotlidec does not flush on error), but what both have written agrees. @@ -153,7 +167,8 @@ invariants for memory safety but Wuffs requires explicit bounds checks. - Metadata callbacks (metadata blocks are currently skipped). See the "Metadata Blocks" section below. -- Fast paths. +- More fast paths, for example for dictionary words, for copies that wrap + around the ring buffer and for block switches. - Compound dictionary matches at encoder qualities 10 and 11 (see the "Encoder" section below). diff --git a/std/brotli/decode_brotli.wuffs b/std/brotli/decode_brotli.wuffs index eccf6bd2..495e7c55 100644 --- a/std/brotli/decode_brotli.wuffs +++ b/std/brotli/decode_brotli.wuffs @@ -230,6 +230,13 @@ pub struct decoder? implements base.io_transformer( insert_length : base.u32, copy_length : base.u32, + // cmd_state is where process_commands (and process_commands_fast) + // are within a command: 0 at its start, 1 when reading its literals + // (insert_length is how many are left), 2 when reading its distance + // and 3 when applying its copy. process_commands_fast sets it to 4 + // when the metablock is done. + cmd_state : base.u32[..= 4], + // quirk_large_window is the QUIRK_LARGE_WINDOW quirk (the C code's // BROTLI_DECODER_PARAM_LARGE_WINDOW parameter). large_window is // whether the stream is a large-window stream (the C code's @@ -679,6 +686,93 @@ pri func decoder.read_symbol?(src: base.io_reader, hoff: base.u32) { this.n_bits = n_bits } +// read_bits_fast is like read_bits but is not a coroutine. It reads input +// bytes exactly when read_bits would, for as long as args.src has them. It +// returns 0 when it has set this.bits_value, or 1 when read_bits must finish +// the job (this.bits and this.n_bits hold the bits read so far). +pri func decoder.read_bits_fast!(src: base.io_reader, n: base.u32[..= 24]) base.u32 { + var n : base.u32[..= 24] + var bits : base.u32 + var n_bits : base.u32 + + n = args.n + bits = this.bits + n_bits = this.n_bits + while n_bits < n, + post n_bits >= n, + { + if args.src.length() <= 0 { + this.bits = bits + this.n_bits = n_bits + return 1 + } + assert n_bits < 24 via "a < b: a < c; c <= b"(c: n) + bits |= args.src.peek_u8_as_u32() << n_bits + args.src.skip_u32_fast!(actual: 1, worst_case: 1) + n_bits += 8 + } + this.bits_value = bits & (((1 as base.u32) << n) - 1) + this.bits = bits >> n + this.n_bits = n_bits - n + return 0 +} + +// read_symbol_fast is like read_symbol but is not a coroutine. It reads +// input bytes exactly when read_symbol would, for as long as args.src has +// them. It returns 0 when it has set this.symbol, or 1 when read_symbol must +// finish the job (this.bits and this.n_bits hold the bits read so far). In +// particular, read_symbol (not read_symbol_fast) returns any error. +pri func decoder.read_symbol_fast!(src: base.io_reader, hoff: base.u32) base.u32 { + var hoff : base.u32 + var bits : base.u32 + var n_bits : base.u32 + var i : base.u32 + var e : base.u32 + var e_bits : base.u32[..= 23] + + hoff = args.hoff + bits = this.bits + n_bits = this.n_bits + while true { + i = hoff ~mod+ (bits & 0xFF) + if i >= HUFFS_LEN { + break + } + e = this.huffs[i] as base.u32 + e_bits = e & 15 + if e_bits <= 8 { + if n_bits >= e_bits { + this.symbol = e >> 4 + this.bits = bits >> e_bits + this.n_bits = n_bits - e_bits + return 0 + } + } else if n_bits > 8 { + i = (hoff ~mod+ (e >> 4)) ~mod+ ((bits >> 8) & (((1 as base.u32) << (e_bits - 8)) - 1)) + if i >= HUFFS_LEN { + break + } + e = this.huffs[i] as base.u32 + e_bits = (e & 15) + 8 + if n_bits >= e_bits { + this.symbol = e >> 4 + this.bits = bits >> e_bits + this.n_bits = n_bits - e_bits + return 0 + } + } + if (n_bits >= 24) or (args.src.length() <= 0) { + break + } + bits |= args.src.peek_u8_as_u32() << n_bits + args.src.skip_u32_fast!(actual: 1, worst_case: 1) + n_bits += 8 + } + this.bits = bits + this.n_bits = n_bits + return 1 +} + // jump_to_byte_boundary drops the bits up to the next byte boundary and // returns whether they were all zero. Mirrors BrotliJumpToByteBoundary in // bit_reader.h. diff --git a/std/brotli/decode_metablock.wuffs b/std/brotli/decode_metablock.wuffs index f3d29d85..ae301fec 100644 --- a/std/brotli/decode_metablock.wuffs +++ b/std/brotli/decode_metablock.wuffs @@ -418,7 +418,6 @@ pri func decoder.calculate_distance_lut!() { // Mirrors ProcessCommandsInternal in decode.c. pri func decoder.process_commands?(src: base.io_reader, workbuf: slice base.u8, dictbuf: roslice base.u8) { var cmd_code : base.u32[..= 703] - var insert_length : base.u32 var p1 : base.u32[..= 0xFF] var p2 : base.u32[..= 0xFF] var mode_offset : base.u32[..= 0x600] @@ -434,37 +433,72 @@ pri func decoder.process_commands?(src: base.io_reader, workbuf: slice base.u8, var address : base.u32 var chunk : base.u32[..= 14] var j : base.u64 + var fr : base.u32 + var dst_i : base.u64 + var src_i : base.u64 + var k64 : base.u64 + var n64 : base.u64 + var dlo : base.u64 + var dhi : base.u64 + var slo : base.u64 + var shi : base.u64 + var fs : base.status while true { - // CommandBegin. - if this.block_length[1] == 0 { - // Mirrors DecodeCommandBlockSwitch. - this.decode_block_switch?(src: args.src, t: 1) - this.htree_command = this.block_type_rb[3] & 0xFF - continue + if (this.cmd_state == 0) and (args.src.length() >= 16) { + fs = this.process_commands_fast!(src: args.src, workbuf: args.workbuf) + if not fs.is_ok() { + return fs + } else if this.cmd_state == 4 { + this.cmd_state = 0 + return ok + } } - // Mirrors ReadCommandInternal. - this.read_symbol?(src: args.src, - hoff: CMD_HUFFS_BASE + (this.htree_command * CMD_HUFFS_STRIDE)) - if this.symbol >= 704 { - return "#internal error: inconsistent Huffman table" + if this.cmd_state == 0 { + // CommandBegin. + if this.block_length[1] == 0 { + // Mirrors DecodeCommandBlockSwitch. + this.decode_block_switch?(src: args.src, t: 1) + this.htree_command = this.block_type_rb[3] & 0xFF + continue + } + + // Mirrors ReadCommandInternal. + fr = this.read_symbol_fast!(src: args.src, hoff: CMD_HUFFS_BASE + (this.htree_command * CMD_HUFFS_STRIDE)) + if fr <> 0 { + this.read_symbol?(src: args.src, + hoff: CMD_HUFFS_BASE + (this.htree_command * CMD_HUFFS_STRIDE)) + } + if this.symbol >= 704 { + return "#internal error: inconsistent Huffman table" + } + cmd_code = this.symbol.min(no_more_than: 703) + this.implicit_zero_distance = CMD_IMPLICIT_ZERO_DISTANCE[cmd_code] <> 0 + this.distance_context = CMD_DISTANCE_CONTEXT[cmd_code] as base.u32 + this.update_dist_htree_index!() + fr = this.read_bits_fast!(src: args.src, n: CMD_INSERT_LEN_EXTRA_BITS[cmd_code] as base.u32) + if fr <> 0 { + this.read_bits?(src: args.src, n: CMD_INSERT_LEN_EXTRA_BITS[cmd_code] as base.u32) + } + this.insert_length = CMD_INSERT_LEN_OFFSET[cmd_code] ~mod+ this.bits_value + fr = this.read_bits_fast!(src: args.src, n: CMD_COPY_LEN_EXTRA_BITS[cmd_code] as base.u32) + if fr <> 0 { + this.read_bits?(src: args.src, n: CMD_COPY_LEN_EXTRA_BITS[cmd_code] as base.u32) + } + this.copy_length = CMD_COPY_LEN_OFFSET[cmd_code] ~mod+ this.bits_value + this.block_length[1] ~mod-= 1 + if this.insert_length > 0 { + this.mb_remaining ~mod-= this.insert_length + this.cmd_state = 1 + } else { + this.cmd_state = 2 + } } - cmd_code = this.symbol.min(no_more_than: 703) - this.implicit_zero_distance = CMD_IMPLICIT_ZERO_DISTANCE[cmd_code] <> 0 - this.distance_context = CMD_DISTANCE_CONTEXT[cmd_code] as base.u32 - this.update_dist_htree_index!() - this.read_bits?(src: args.src, n: CMD_INSERT_LEN_EXTRA_BITS[cmd_code] as base.u32) - insert_length = CMD_INSERT_LEN_OFFSET[cmd_code] ~mod+ this.bits_value - this.read_bits?(src: args.src, n: CMD_COPY_LEN_EXTRA_BITS[cmd_code] as base.u32) - this.copy_length = CMD_COPY_LEN_OFFSET[cmd_code] ~mod+ this.bits_value - this.block_length[1] ~mod-= 1 - - if insert_length > 0 { - this.mb_remaining ~mod-= insert_length + if this.cmd_state == 1 { // CommandInner. Read the literals in the command. - while insert_length > 0 { + while this.insert_length > 0 { if this.block_length[0] == 0 { // Mirrors DecodeLiteralBlockSwitch. this.decode_block_switch?(src: args.src, t: 0) @@ -478,39 +512,48 @@ pri func decoder.process_commands?(src: base.io_reader, workbuf: slice base.u8, ctx = (CONTEXT_LOOKUP[mode_offset | p1] | CONTEXT_LOOKUP[mode_offset | 0x100 | p2]) as base.u32 tree = this.context_maps[this.context_map_slice + ctx] as base.u32 - this.read_symbol?(src: args.src, hoff: LIT_HUFFS_BASE + (tree * LIT_HUFFS_STRIDE)) + fr = this.read_symbol_fast!(src: args.src, hoff: LIT_HUFFS_BASE + (tree * LIT_HUFFS_STRIDE)) + if fr <> 0 { + this.read_symbol?(src: args.src, hoff: LIT_HUFFS_BASE + (tree * LIT_HUFFS_STRIDE)) + } this.put_byte!(workbuf: args.workbuf, b: (this.symbol & 0xFF) as base.u8) this.block_length[0] ~mod-= 1 - insert_length -= 1 + this.insert_length ~mod-= 1 if this.pos == this.rb_size { this.ring_event?() } } + this.cmd_state = 2 if (this.mb_remaining == 0) or (this.mb_remaining >= 0x8000_0000) { // BROTLI_STATE_METABLOCK_DONE. + this.cmd_state = 0 return ok } } - // CommandPostDecodeLiterals. - if this.implicit_zero_distance { - // Implicit distance case. - this.distance_context = 1 - this.dist_rb_idx ~mod-= 1 - this.distance_code = this.dist_rb[this.dist_rb_idx & 3] - } else { - // Read distance code in the command, unless it was implicitly - // zero. - if this.block_length[2] == 0 { - // Mirrors DecodeDistanceBlockSwitch. - this.decode_block_switch?(src: args.src, t: 2) - this.dist_context_map_slice = (this.block_type_rb[5] & 0xFF) << 2 - this.update_dist_htree_index!() + if this.cmd_state == 2 { + // CommandPostDecodeLiterals. + if this.implicit_zero_distance { + // Implicit distance case. + this.distance_context = 1 + this.dist_rb_idx ~mod-= 1 + this.distance_code = this.dist_rb[this.dist_rb_idx & 3] + } else { + // Read distance code in the command, unless it was implicitly + // zero. + if this.block_length[2] == 0 { + // Mirrors DecodeDistanceBlockSwitch. + this.decode_block_switch?(src: args.src, t: 2) + this.dist_context_map_slice = (this.block_type_rb[5] & 0xFF) << 2 + this.update_dist_htree_index!() + } + this.read_distance?(src: args.src) } - this.read_distance?(src: args.src) + this.cmd_state = 3 } + // this.cmd_state == 3. if this.max_distance <> this.max_backward_distance { this.max_distance = this.pos.min(no_more_than: this.max_backward_distance) } @@ -604,6 +647,37 @@ pri func decoder.process_commands?(src: base.io_reader, workbuf: slice base.u8, this.dist_rb[this.dist_rb_idx & 3] = distance this.dist_rb_idx ~mod+= 1 this.mb_remaining ~mod-= i + // Fast path: when the copy neither wraps around the physical ring + // buffer (at either end) nor makes this.pos reach this.rb_size, + // copy the bytes in chunks of at most distance bytes. Each chunk's + // source bytes are final (and do not overlap the chunk's + // destination), so this matches the byte by byte loop below. + dst_i = this.pos as base.u64 + src_i = ((this.pos ~mod- distance) & (this.window_size ~mod- 1)) as base.u64 + if ((dst_i + (i as base.u64)) < (this.rb_size as base.u64)) and + ((src_i + (i as base.u64)) <= (this.window_size as base.u64)) and + ((this.window_size as base.u64) <= args.workbuf.length()) and + (distance > 0) { + k64 = 0 + while k64 < (i as base.u64) { + n64 = (i as base.u64) ~mod- k64 + n64 = n64.min(no_more_than: distance as base.u64) + dlo = dst_i ~mod+ k64 + dhi = dlo ~mod+ n64 + slo = src_i ~mod+ k64 + shi = slo ~mod+ n64 + if (n64 == 0) or (dlo > dhi) or (dhi > args.workbuf.length()) or + (slo > shi) or (shi > args.workbuf.length()) { + break + } + args.workbuf[dlo .. dhi].copy_from_slice!(s: args.workbuf[slo .. shi]) + k64 ~mod+= n64 + } + // k64 == i, unless the (unreachable) break above was taken, + // in which case the loop below copies the remaining bytes. + this.pos ~mod+= (k64 & 0xFFFF_FFFF) as base.u32 + i ~mod-= (k64 & 0xFFFF_FFFF) as base.u32 + } while i > 0 { b = this.get_byte(workbuf: args.workbuf, distance: distance) this.put_byte!(workbuf: args.workbuf, b: b) @@ -614,6 +688,7 @@ pri func decoder.process_commands?(src: base.io_reader, workbuf: slice base.u8, } } + this.cmd_state = 0 if (this.mb_remaining == 0) or (this.mb_remaining >= 0x8000_0000) { // BROTLI_STATE_METABLOCK_DONE. return ok @@ -621,6 +696,323 @@ pri func decoder.process_commands?(src: base.io_reader, workbuf: slice base.u8, } } +// process_commands_fast is a fast path for process_commands. It is not a +// coroutine. It handles commands, or parts of commands (see +// decoder.cmd_state), for as long as args.src holds enough bytes and the work +// needs no block switch, ring buffer event or dictionary reference. Then it +// returns, and process_commands picks up where it left off. +// +// It reads ahead, up to 8 bytes at a time, into a 64-bit bit buffer. When it +// returns, it gives back (undoes) the whole bytes left in that buffer. The +// coroutines read_symbol and read_bits read an input byte only when they +// need more bits, so that between reads, fewer than 8 bits are buffered. +// Giving back the whole bytes restores that state exactly, so this function +// reads the same bits and leaves the decoder in the same state as +// process_commands would, and it writes the same bytes and returns the same +// errors. It sets this.cmd_state to 4 when the metablock is done. +pri func decoder.process_commands_fast!(src: base.io_reader, workbuf: slice base.u8) base.status { + var ret : base.status + var bits : base.u64 + var n_bits : base.u32[..= 63] + var i : base.u32 + var e : base.u32 + var e_bits : base.u32[..= 23] + var cmd_code : base.u32[..= 703] + var nb : base.u32[..= 24] + var p1 : base.u32[..= 0xFF] + var p2 : base.u32[..= 0xFF] + var mode_offset : base.u32[..= 0x600] + var ctx : base.u32[..= 63] + var tree : base.u32[..= 0xFF] + var code : base.u32 + var nbits : base.u32[..= 32] + var lo : base.u32 + var max_distance : base.u32 + var distance : base.u32 + var n : base.u32 + var dst_i : base.u64 + var src_i : base.u64 + var k64 : base.u64 + var n64 : base.u64 + var dlo : base.u64 + var dhi : base.u64 + var slo : base.u64 + var shi : base.u64 + + if this.n_bits >= 8 { + return ok + } + n_bits = this.n_bits & 7 + bits = (this.bits as base.u64) & (((1 as base.u64) << n_bits) - 1) + + while.outer true { + if this.cmd_state == 0 { + // Two refills, of at most 8 bytes each. + if (this.block_length[1] == 0) or (args.src.length() < 16) { + break.outer + } + bits |= args.src.peek_u64le() ~mod<< n_bits + args.src.skip_u32_fast!(actual: (63 - n_bits) >> 3, worst_case: 8) + n_bits |= 56 + // Mirrors ReadCommandInternal. + i = (CMD_HUFFS_BASE + (this.htree_command * CMD_HUFFS_STRIDE)) ~mod+ ((bits & 0xFF) as base.u32) + if i >= HUFFS_LEN { + ret = "#internal error: inconsistent Huffman table" + break.outer + } + e = this.huffs[i] as base.u32 + e_bits = e & 15 + if e_bits > 8 { + i = ((CMD_HUFFS_BASE + (this.htree_command * CMD_HUFFS_STRIDE)) ~mod+ (e >> 4)) ~mod+ ((((bits >> 8) & 0x7FFF) as base.u32) & (((1 as base.u32) << (e_bits - 8)) - 1)) + if i >= HUFFS_LEN { + ret = "#internal error: inconsistent Huffman table" + break.outer + } + e = this.huffs[i] as base.u32 + e_bits = (e & 15) + 8 + } + if n_bits < e_bits { + ret = "#internal error: inconsistent bit reader" + break.outer + } + bits >>= e_bits + n_bits -= e_bits + if (e >> 4) >= 704 { + ret = "#internal error: inconsistent Huffman table" + break.outer + } + this.symbol = e >> 4 + cmd_code = this.symbol.min(no_more_than: 703) + this.implicit_zero_distance = CMD_IMPLICIT_ZERO_DISTANCE[cmd_code] <> 0 + this.distance_context = CMD_DISTANCE_CONTEXT[cmd_code] as base.u32 + this.update_dist_htree_index!() + nb = CMD_INSERT_LEN_EXTRA_BITS[cmd_code] as base.u32 + if n_bits < nb { + ret = "#internal error: inconsistent bit reader" + break.outer + } + this.bits_value = ((bits & (((1 as base.u64) << nb) - 1)) & 0xFFFF_FFFF) as base.u32 + bits >>= nb + n_bits -= nb + this.insert_length = CMD_INSERT_LEN_OFFSET[cmd_code] ~mod+ this.bits_value + if args.src.length() < 8 { + ret = "#internal error: inconsistent bit reader" + break.outer + } + bits |= args.src.peek_u64le() ~mod<< n_bits + args.src.skip_u32_fast!(actual: (63 - n_bits) >> 3, worst_case: 8) + n_bits |= 56 + nb = CMD_COPY_LEN_EXTRA_BITS[cmd_code] as base.u32 + if n_bits < nb { + ret = "#internal error: inconsistent bit reader" + break.outer + } + this.bits_value = ((bits & (((1 as base.u64) << nb) - 1)) & 0xFFFF_FFFF) as base.u32 + bits >>= nb + n_bits -= nb + this.copy_length = CMD_COPY_LEN_OFFSET[cmd_code] ~mod+ this.bits_value + this.block_length[1] ~mod-= 1 + if this.insert_length > 0 { + this.mb_remaining ~mod-= this.insert_length + this.cmd_state = 1 + } else { + this.cmd_state = 2 + } + } + + if this.cmd_state == 1 { + // CommandInner. Read the literals in the command. p1 and p2 are + // the last two bytes written to the ring buffer (put_byte always + // writes when the physical ring buffer fits in args.workbuf). + if (this.window_size as base.u64) > args.workbuf.length() { + break.outer + } + p1 = this.get_byte(workbuf: args.workbuf, distance: 1) as base.u32 + p2 = this.get_byte(workbuf: args.workbuf, distance: 2) as base.u32 + while this.insert_length > 0 { + if (this.block_length[0] == 0) or ((this.pos ~mod+ 1) >= this.rb_size) { + break.outer + } + if n_bits < 24 { + if args.src.length() < 8 { + break.outer + } + bits |= args.src.peek_u64le() ~mod<< n_bits + args.src.skip_u32_fast!(actual: (63 - n_bits) >> 3, worst_case: 8) + n_bits |= 56 + } + mode_offset = this.context_mode << 9 + ctx = (CONTEXT_LOOKUP[mode_offset | p1] | + CONTEXT_LOOKUP[mode_offset | 0x100 | p2]) as base.u32 + tree = this.context_maps[this.context_map_slice + ctx] as base.u32 + i = (LIT_HUFFS_BASE + (tree * LIT_HUFFS_STRIDE)) ~mod+ ((bits & 0xFF) as base.u32) + if i >= HUFFS_LEN { + ret = "#internal error: inconsistent Huffman table" + break.outer + } + e = this.huffs[i] as base.u32 + e_bits = e & 15 + if e_bits > 8 { + i = ((LIT_HUFFS_BASE + (tree * LIT_HUFFS_STRIDE)) ~mod+ (e >> 4)) ~mod+ ((((bits >> 8) & 0x7FFF) as base.u32) & (((1 as base.u32) << (e_bits - 8)) - 1)) + if i >= HUFFS_LEN { + ret = "#internal error: inconsistent Huffman table" + break.outer + } + e = this.huffs[i] as base.u32 + e_bits = (e & 15) + 8 + } + if n_bits < e_bits { + ret = "#internal error: inconsistent bit reader" + break.outer + } + bits >>= e_bits + n_bits -= e_bits + this.symbol = e >> 4 + this.put_byte!(workbuf: args.workbuf, b: (this.symbol & 0xFF) as base.u8) + p2 = p1 + p1 = this.symbol & 0xFF + this.block_length[0] ~mod-= 1 + this.insert_length ~mod-= 1 + } + this.cmd_state = 2 + if (this.mb_remaining == 0) or (this.mb_remaining >= 0x8000_0000) { + this.cmd_state = 4 + break.outer + } + } + + if this.cmd_state == 2 { + // CommandPostDecodeLiterals. + if this.implicit_zero_distance { + this.distance_context = 1 + this.dist_rb_idx ~mod-= 1 + this.distance_code = this.dist_rb[this.dist_rb_idx & 3] + } else { + // One refill: 56 bits suffice for a symbol and 32 extra bits. + if (this.block_length[2] == 0) or (args.src.length() < 8) { + break.outer + } + bits |= args.src.peek_u64le() ~mod<< n_bits + args.src.skip_u32_fast!(actual: (63 - n_bits) >> 3, worst_case: 8) + n_bits |= 56 + // Mirrors read_distance. + i = (DIST_HUFFS_BASE + (this.dist_htree_index * DIST_HUFFS_STRIDE)) ~mod+ ((bits & 0xFF) as base.u32) + if i >= HUFFS_LEN { + ret = "#internal error: inconsistent Huffman table" + break.outer + } + e = this.huffs[i] as base.u32 + e_bits = e & 15 + if e_bits > 8 { + i = ((DIST_HUFFS_BASE + (this.dist_htree_index * DIST_HUFFS_STRIDE)) ~mod+ (e >> 4)) ~mod+ ((((bits >> 8) & 0x7FFF) as base.u32) & (((1 as base.u32) << (e_bits - 8)) - 1)) + if i >= HUFFS_LEN { + ret = "#internal error: inconsistent Huffman table" + break.outer + } + e = this.huffs[i] as base.u32 + e_bits = (e & 15) + 8 + } + if n_bits < e_bits { + ret = "#internal error: inconsistent bit reader" + break.outer + } + bits >>= e_bits + n_bits -= e_bits + this.symbol = e >> 4 + code = this.symbol + this.block_length[2] ~mod-= 1 + this.distance_context = 0 + if code < 16 { + this.take_distance_from_ring_buffer!(code: code) + } else { + if (code >= this.distance_alphabet_size) or (code >= 1128) { + ret = "#internal error: inconsistent Huffman table" + break.outer + } + nbits = this.dist_extra_bits[code].min(no_more_than: 32) as base.u32 + if n_bits < nbits { + ret = "#internal error: inconsistent bit reader" + break.outer + } + // Like read_bits32, this is (lo | (hi << 16)) when nbits + // is more than 24. + lo = ((bits & (((1 as base.u64) << nbits) - 1)) & 0xFFFF_FFFF) as base.u32 + this.bits_value = lo + bits >>= nbits + n_bits -= nbits + this.distance_code = (((this.dist_offset[code] as base.u64) ~mod+ + ((this.bits_value as base.u64) ~mod<< this.distance_postfix_bits)) & 0xFFFF_FFFF) as base.u32 + } + } + this.cmd_state = 3 + } + + // this.cmd_state == 3. Only handle LZ77 copies that need no ring + // buffer event and do not wrap around the physical ring buffer. + max_distance = this.max_distance + if max_distance <> this.max_backward_distance { + max_distance = this.pos.min(no_more_than: this.max_backward_distance) + } + distance = this.distance_code + n = this.copy_length + if (distance > max_distance) or (distance == 0) { + break.outer + } + dst_i = this.pos as base.u64 + src_i = ((this.pos ~mod- distance) & (this.window_size ~mod- 1)) as base.u64 + if ((dst_i + (n as base.u64)) >= (this.rb_size as base.u64)) or + ((src_i + (n as base.u64)) > (this.window_size as base.u64)) or + ((this.window_size as base.u64) > args.workbuf.length()) { + break.outer + } + this.max_distance = max_distance + this.dist_rb[this.dist_rb_idx & 3] = distance + this.dist_rb_idx ~mod+= 1 + this.mb_remaining ~mod-= n + // Copy in chunks of at most distance bytes, like process_commands. + k64 = 0 + while k64 < (n as base.u64) { + n64 = (n as base.u64) ~mod- k64 + n64 = n64.min(no_more_than: distance as base.u64) + dlo = dst_i ~mod+ k64 + dhi = dlo ~mod+ n64 + slo = src_i ~mod+ k64 + shi = slo ~mod+ n64 + if (n64 == 0) or (dlo > dhi) or (dhi > args.workbuf.length()) or + (slo > shi) or (shi > args.workbuf.length()) { + break + } + args.workbuf[dlo .. dhi].copy_from_slice!(s: args.workbuf[slo .. shi]) + k64 ~mod+= n64 + } + this.pos ~mod+= (k64 & 0xFFFF_FFFF) as base.u32 + n ~mod-= (k64 & 0xFFFF_FFFF) as base.u32 + // n == 0, unless the (unreachable) break above was taken. This.pos + // stays below this.rb_size, so there is no ring buffer event. + while n > 0 { + this.put_byte!(workbuf: args.workbuf, b: this.get_byte(workbuf: args.workbuf, distance: distance)) + n -= 1 + } + this.cmd_state = 0 + if (this.mb_remaining == 0) or (this.mb_remaining >= 0x8000_0000) { + this.cmd_state = 4 + break.outer + } + }.outer + + // Give back the whole bytes left in the bit buffer. + while n_bits >= 8 { + if not args.src.can_undo_byte() { + return "#internal error: inconsistent bit reader" + } + args.src.undo_byte!() + n_bits -= 8 + } + this.bits = ((bits & (((1 as base.u64) << n_bits) - 1)) & 0xFFFF_FFFF) as base.u32 + this.n_bits = n_bits + return ret +} + // update_dist_htree_index sets this.dist_htree_index from the distance // context map. Mirrors part of ReadCommandInternal and // DecodeDistanceBlockSwitch in decode.c. @@ -631,10 +1023,15 @@ pri func decoder.update_dist_htree_index!() { // Mirrors ReadDistanceInternal in decode.c. pri func decoder.read_distance?(src: base.io_reader) { - var code : base.u32 + var code : base.u32 + var fr : base.u32 + var nbits : base.u32[..= 32] - this.read_symbol?(src: args.src, - hoff: DIST_HUFFS_BASE + (this.dist_htree_index * DIST_HUFFS_STRIDE)) + fr = this.read_symbol_fast!(src: args.src, hoff: DIST_HUFFS_BASE + (this.dist_htree_index * DIST_HUFFS_STRIDE)) + if fr <> 0 { + this.read_symbol?(src: args.src, + hoff: DIST_HUFFS_BASE + (this.dist_htree_index * DIST_HUFFS_STRIDE)) + } code = this.symbol this.block_length[2] ~mod-= 1 // Convert the distance code to the actual distance by possibly looking up @@ -649,7 +1046,14 @@ pri func decoder.read_distance?(src: base.io_reader) { } // The C code uses BrotliSafeReadBits32, as large-window brotli can have // more than 24 extra bits. - this.read_bits32?(src: args.src, n: this.dist_extra_bits[code].min(no_more_than: 32) as base.u32) + nbits = this.dist_extra_bits[code].min(no_more_than: 32) as base.u32 + fr = 1 + if nbits <= 24 { + fr = this.read_bits_fast!(src: args.src, n: nbits) + } + if fr <> 0 { + this.read_bits32?(src: args.src, n: nbits) + } if code >= 1128 { return "#internal error: inconsistent Huffman table" }