std/softfloat: add fast paths for f32 and f64 add, sub and mul

f32_add, f32_sub and f32_mul now compute the result directly in binary32
when both operands are normal and the result is normal (or overflows),
instead of always going through f64. f64_add and f64_sub get a similar
fast path for normal operands. Every other case takes the previous code
path, so results (including NaN payloads) are unchanged.

Add script/manual-test-softfloat-sweep.c, which checks the f32 and f64
add, sub, mul and div operations over every pair of operand exponents
with many mantissa patterns, and the f32 conversions, against the
hardware and (bit-for-bit, NaNs included) the via-f64 definitions.
This commit is contained in:
Claude committed 2026-10-27 09:26:09 +00:00
1 parent d72d46ea97
commit 1d1c338981
5 files changed
+1474 -18

No files matched your search

+449 -2
View File
@@ -41133,6 +41133,12 @@ wuffs_softfloat__arith__clz64(
const wuffs_softfloat__arith* self,
uint64_t a_a);
WUFFS_BASE__GENERATED_C_CODE
static uint32_t
wuffs_softfloat__arith__clz32(
const wuffs_softfloat__arith* self,
uint32_t a_a);
WUFFS_BASE__GENERATED_C_CODE
static uint64_t
wuffs_softfloat__arith__shift_right_jam64(
@@ -41198,6 +41204,20 @@ wuffs_softfloat__arith__f64_propagate_nan(
uint64_t a_a,
uint64_t a_b);
WUFFS_BASE__GENERATED_C_CODE
static uint64_t
wuffs_softfloat__arith__f64_add_normal_mags(
const wuffs_softfloat__arith* self,
uint64_t a_a,
uint64_t a_b);
WUFFS_BASE__GENERATED_C_CODE
static uint64_t
wuffs_softfloat__arith__f64_sub_normal_mags(
const wuffs_softfloat__arith* self,
uint64_t a_a,
uint64_t a_b);
WUFFS_BASE__GENERATED_C_CODE
static uint64_t
wuffs_softfloat__arith__f64_add_mags(
@@ -41222,6 +41242,41 @@ wuffs_softfloat__arith__round_pack_f32(
uint32_t a_exp,
uint32_t a_sig);
WUFFS_BASE__GENERATED_C_CODE
static uint32_t
wuffs_softfloat__arith__f32_add_mags(
const wuffs_softfloat__arith* self,
uint32_t a_a,
uint32_t a_b);
WUFFS_BASE__GENERATED_C_CODE
static uint32_t
wuffs_softfloat__arith__f32_sub_mags(
const wuffs_softfloat__arith* self,
uint32_t a_a,
uint32_t a_b);
WUFFS_BASE__GENERATED_C_CODE
static uint32_t
wuffs_softfloat__arith__slow_f32_add(
const wuffs_softfloat__arith* self,
uint32_t a_a,
uint32_t a_b);
WUFFS_BASE__GENERATED_C_CODE
static uint32_t
wuffs_softfloat__arith__slow_f32_sub(
const wuffs_softfloat__arith* self,
uint32_t a_a,
uint32_t a_b);
WUFFS_BASE__GENERATED_C_CODE
static uint32_t
wuffs_softfloat__arith__slow_f32_mul(
const wuffs_softfloat__arith* self,
uint32_t a_a,
uint32_t a_b);
// ---------------- VTables
// ---------------- Initializer Implementations
@@ -41322,6 +41377,29 @@ wuffs_softfloat__arith__clz64(
(((uint32_t)(WUFFS_SOFTFLOAT__CLZ8[(v_x >> 56u)])) & 7u)) & 63u);
}
// -------- func softfloat.arith.clz32
WUFFS_BASE__GENERATED_C_CODE
static uint32_t
wuffs_softfloat__arith__clz32(
const wuffs_softfloat__arith* self,
uint32_t a_a) {
uint32_t v_x = 0;
uint32_t v_n16 = 0;
uint32_t v_n8 = 0;
v_x = a_a;
if ((v_x >> 16u) == 0u) {
v_x = ((uint32_t)(v_x << 16u));
v_n16 = 16u;
}
if ((v_x >> 24u) == 0u) {
v_x = ((uint32_t)(v_x << 8u));
v_n8 = 8u;
}
return ((v_n16 + v_n8 + (((uint32_t)(WUFFS_SOFTFLOAT__CLZ8[(v_x >> 24u)])) & 7u)) & 31u);
}
// -------- func softfloat.arith.shift_right_jam64
WUFFS_BASE__GENERATED_C_CODE
@@ -41816,6 +41894,17 @@ wuffs_softfloat__arith__f64_add(
return 0;
}
uint64_t v_z = 0;
if ((((uint64_t)(((a_a >> 52u) & 2047u) - 1u)) < 2046u) && (((uint64_t)(((a_b >> 52u) & 2047u) - 1u)) < 2046u)) {
if (((a_a ^ a_b) >> 63u) == 0u) {
return wuffs_softfloat__arith__f64_add_normal_mags(self, a_a, a_b);
}
v_z = wuffs_softfloat__arith__f64_sub_normal_mags(self, a_a, a_b);
if (v_z != 18446744073709551615u) {
return v_z;
}
}
if (((a_a ^ a_b) >> 63u) == 0u) {
return wuffs_softfloat__arith__f64_add_mags(self, a_a, a_b, (a_a & 9223372036854775808u));
}
@@ -41838,12 +41927,152 @@ wuffs_softfloat__arith__f64_sub(
return 0;
}
uint64_t v_z = 0;
if ((((uint64_t)(((a_a >> 52u) & 2047u) - 1u)) < 2046u) && (((uint64_t)(((a_b >> 52u) & 2047u) - 1u)) < 2046u)) {
if (((a_a ^ a_b) >> 63u) != 0u) {
return wuffs_softfloat__arith__f64_add_normal_mags(self, a_a, a_b);
}
v_z = wuffs_softfloat__arith__f64_sub_normal_mags(self, a_a, a_b);
if (v_z != 18446744073709551615u) {
return v_z;
}
}
if (((a_a ^ a_b) >> 63u) == 0u) {
return wuffs_softfloat__arith__f64_sub_mags(self, a_a, a_b, (a_a & 9223372036854775808u));
}
return wuffs_softfloat__arith__f64_add_mags(self, a_a, a_b, (a_a & 9223372036854775808u));
}
// -------- func softfloat.arith.f64_add_normal_mags
WUFFS_BASE__GENERATED_C_CODE
static uint64_t
wuffs_softfloat__arith__f64_add_normal_mags(
const wuffs_softfloat__arith* self,
uint64_t a_a,
uint64_t a_b) {
uint32_t v_ea = 0;
uint32_t v_eb = 0;
uint64_t v_ma = 0;
uint64_t v_mb = 0;
uint32_t v_d = 0;
uint64_t v_z = 0;
uint64_t v_rb = 0;
v_ea = ((uint32_t)(((a_a >> 52u) & 2047u)));
v_eb = ((uint32_t)(((a_b >> 52u) & 2047u)));
v_ma = (((a_a & 4503599627370495u) | 4503599627370496u) << 9u);
v_mb = (((a_b & 4503599627370495u) | 4503599627370496u) << 9u);
if (v_ea < v_eb) {
v_d = (((uint32_t)(v_eb - v_ea)) & 2047u);
v_ea = v_eb;
v_z = v_ma;
v_ma = v_mb;
v_mb = v_z;
} else {
v_d = (((uint32_t)(v_ea - v_eb)) & 2047u);
}
if (v_d >= 63u) {
v_mb = 1u;
} else if (v_d > 0u) {
if (((uint64_t)(v_mb << (64u - v_d))) != 0u) {
v_mb = ((v_mb >> v_d) | 1u);
} else {
v_mb = (v_mb >> v_d);
}
}
v_z = (v_ma + v_mb);
if (v_z >= 4611686018427387904u) {
v_ea += 1u;
v_z = ((v_z >> 1u) | (v_z & 1u));
}
v_rb = (v_z & 511u);
v_z = (((uint64_t)(v_z + 256u)) >> 9u);
if (v_rb == 256u) {
v_z &= 18446744073709551614u;
}
v_z = ((uint64_t)(((uint64_t)(((uint64_t)(((uint32_t)(v_ea - 1u)))) << 52u)) + v_z));
if (v_z >= 9218868437227405312u) {
v_z = 9218868437227405312u;
}
return ((a_a & 9223372036854775808u) | v_z);
}
// -------- func softfloat.arith.f64_sub_normal_mags
WUFFS_BASE__GENERATED_C_CODE
static uint64_t
wuffs_softfloat__arith__f64_sub_normal_mags(
const wuffs_softfloat__arith* self,
uint64_t a_a,
uint64_t a_b) {
uint64_t v_sign = 0;
uint32_t v_ea = 0;
uint32_t v_eb = 0;
uint64_t v_fa = 0;
uint64_t v_fb = 0;
uint64_t v_ma = 0;
uint64_t v_mb = 0;
uint32_t v_d = 0;
uint32_t v_s = 0;
uint64_t v_z = 0;
uint64_t v_rb = 0;
v_sign = (a_a & 9223372036854775808u);
v_ea = ((uint32_t)(((a_a >> 52u) & 2047u)));
v_eb = ((uint32_t)(((a_b >> 52u) & 2047u)));
v_fa = (a_a & 4503599627370495u);
v_fb = (a_b & 4503599627370495u);
if (v_ea == v_eb) {
if (v_fa == v_fb) {
return 0u;
} else if (v_fa < v_fb) {
v_sign ^= 9223372036854775808u;
v_z = ((uint64_t)(v_fb - v_fa));
} else {
v_z = ((uint64_t)(v_fa - v_fb));
}
v_s = (((uint32_t)(wuffs_softfloat__arith__clz64(self, v_z) - 11u)) & 63u);
if (v_s >= v_ea) {
return 18446744073709551615u;
}
return (v_sign | ((uint64_t)(((uint64_t)(((uint64_t)(((uint32_t)(((uint32_t)(v_ea - v_s)) - 1u)))) << 52u)) + ((uint64_t)(v_z << v_s)))));
}
v_ma = ((v_fa | 4503599627370496u) << 10u);
v_mb = ((v_fb | 4503599627370496u) << 10u);
if (v_ea < v_eb) {
v_sign ^= 9223372036854775808u;
v_d = (((uint32_t)(v_eb - v_ea)) & 2047u);
v_ea = v_eb;
v_z = v_ma;
v_ma = v_mb;
v_mb = v_z;
} else {
v_d = (((uint32_t)(v_ea - v_eb)) & 2047u);
}
if (v_d >= 63u) {
v_mb = 1u;
} else if (((uint64_t)(v_mb << ((64u - v_d) & 63u))) != 0u) {
v_mb = ((v_mb >> v_d) | 1u);
} else {
v_mb = (v_mb >> v_d);
}
v_z = ((uint64_t)(v_ma - v_mb));
v_s = (((uint32_t)(wuffs_softfloat__arith__clz64(self, v_z) - 1u)) & 63u);
if (v_s >= v_ea) {
return 18446744073709551615u;
}
v_ea = (((uint32_t)(v_ea - v_s)) & 2047u);
v_z = ((uint64_t)(v_z << v_s));
v_rb = (v_z & 1023u);
v_z = (((uint64_t)(v_z + 512u)) >> 10u);
if (v_rb == 512u) {
v_z &= 18446744073709551614u;
}
return (v_sign | ((uint64_t)(((uint64_t)(((uint64_t)(((uint32_t)(v_ea - 1u)))) << 52u)) + v_z)));
}
// -------- func softfloat.arith.f64_add_mags
WUFFS_BASE__GENERATED_C_CODE
@@ -42593,7 +42822,18 @@ wuffs_softfloat__arith__f32_add(
return 0;
}
return wuffs_softfloat__arith__f64_to_f32(self, wuffs_softfloat__arith__f64_add(self, wuffs_softfloat__arith__f32_to_f64(self, a_a), wuffs_softfloat__arith__f32_to_f64(self, a_b)));
uint32_t v_z = 0;
if ((((uint32_t)(((a_a >> 23u) & 255u) - 1u)) < 254u) && (((uint32_t)(((a_b >> 23u) & 255u) - 1u)) < 254u)) {
if (((a_a ^ a_b) >> 31u) == 0u) {
return wuffs_softfloat__arith__f32_add_mags(self, a_a, a_b);
}
v_z = wuffs_softfloat__arith__f32_sub_mags(self, a_a, a_b);
if (v_z != 4294967295u) {
return v_z;
}
}
return wuffs_softfloat__arith__slow_f32_add(self, a_a, a_b);
}
// -------- func softfloat.arith.f32_sub
@@ -42612,7 +42852,18 @@ wuffs_softfloat__arith__f32_sub(
return 0;
}
return wuffs_softfloat__arith__f64_to_f32(self, wuffs_softfloat__arith__f64_sub(self, wuffs_softfloat__arith__f32_to_f64(self, a_a), wuffs_softfloat__arith__f32_to_f64(self, a_b)));
uint32_t v_z = 0;
if ((((uint32_t)(((a_a >> 23u) & 255u) - 1u)) < 254u) && (((uint32_t)(((a_b >> 23u) & 255u) - 1u)) < 254u)) {
if (((a_a ^ a_b) >> 31u) != 0u) {
return wuffs_softfloat__arith__f32_add_mags(self, a_a, a_b);
}
v_z = wuffs_softfloat__arith__f32_sub_mags(self, a_a, a_b);
if (v_z != 4294967295u) {
return v_z;
}
}
return wuffs_softfloat__arith__slow_f32_sub(self, a_a, a_b);
}
// -------- func softfloat.arith.f32_mul
@@ -42631,6 +42882,202 @@ wuffs_softfloat__arith__f32_mul(
return 0;
}
uint32_t v_ea = 0;
uint32_t v_eb = 0;
uint32_t v_e = 0;
uint64_t v_p = 0;
uint32_t v_m = 0;
uint32_t v_r = 0;
uint32_t v_z = 0;
v_ea = ((a_a >> 23u) & 255u);
v_eb = ((a_b >> 23u) & 255u);
if ((((uint32_t)(v_ea - 1u)) < 254u) && (((uint32_t)(v_eb - 1u)) < 254u)) {
v_p = (((uint64_t)(((a_a & 8388607u) | 8388608u))) * ((uint64_t)(((a_b & 8388607u) | 8388608u))));
v_e = (v_ea + v_eb);
if (v_p >= 140737488355328u) {
v_e += 1u;
v_m = ((uint32_t)(((v_p >> 24u) & 16777215u)));
v_r = ((uint32_t)(((uint32_t)((v_p & 16777215u))) - 8388608u));
} else {
v_m = ((uint32_t)(((v_p >> 23u) & 16777215u)));
v_r = ((uint32_t)(((uint32_t)((v_p & 8388607u))) - 4194304u));
}
if ((v_r != 0u) && (v_r < 2147483648u)) {
v_m += 1u;
} else if (v_r == 0u) {
v_m += (v_m & 1u);
}
if ((v_e >= 128u) && (v_e <= 381u)) {
v_z = ((uint32_t)(((v_e - 128u) << 23u) + v_m));
if (v_z >= 2139095040u) {
v_z = 2139095040u;
}
return (((a_a ^ a_b) & 2147483648u) | v_z);
}
}
return wuffs_softfloat__arith__slow_f32_mul(self, a_a, a_b);
}
// -------- func softfloat.arith.f32_add_mags
WUFFS_BASE__GENERATED_C_CODE
static uint32_t
wuffs_softfloat__arith__f32_add_mags(
const wuffs_softfloat__arith* self,
uint32_t a_a,
uint32_t a_b) {
uint32_t v_ea = 0;
uint32_t v_eb = 0;
uint32_t v_ma = 0;
uint32_t v_mb = 0;
uint32_t v_d = 0;
uint32_t v_z = 0;
uint32_t v_rb = 0;
v_ea = ((a_a >> 23u) & 255u);
v_eb = ((a_b >> 23u) & 255u);
v_ma = (((a_a & 8388607u) | 8388608u) << 6u);
v_mb = (((a_b & 8388607u) | 8388608u) << 6u);
if (v_ea < v_eb) {
v_d = (((uint32_t)(v_eb - v_ea)) & 255u);
v_ea = v_eb;
v_z = v_ma;
v_ma = v_mb;
v_mb = v_z;
} else {
v_d = (((uint32_t)(v_ea - v_eb)) & 255u);
}
if (v_d >= 31u) {
v_mb = 1u;
} else if (v_d > 0u) {
if (((uint32_t)(v_mb << (32u - v_d))) != 0u) {
v_mb = ((v_mb >> v_d) | 1u);
} else {
v_mb = (v_mb >> v_d);
}
}
v_z = ((v_ma & 1073741823u) + (v_mb & 1073741823u));
if (v_z >= 1073741824u) {
v_ea += 1u;
v_z = ((v_z >> 1u) | (v_z & 1u));
}
v_rb = (v_z & 63u);
v_z = (((uint32_t)(v_z + 32u)) >> 6u);
if (v_rb == 32u) {
v_z &= 4294967294u;
}
v_z = ((uint32_t)(((uint32_t)(((uint32_t)(v_ea - 1u)) << 23u)) + v_z));
if (v_z >= 2139095040u) {
v_z = 2139095040u;
}
return ((a_a & 2147483648u) | v_z);
}
// -------- func softfloat.arith.f32_sub_mags
WUFFS_BASE__GENERATED_C_CODE
static uint32_t
wuffs_softfloat__arith__f32_sub_mags(
const wuffs_softfloat__arith* self,
uint32_t a_a,
uint32_t a_b) {
uint32_t v_sign = 0;
uint32_t v_ea = 0;
uint32_t v_eb = 0;
uint32_t v_fa = 0;
uint32_t v_fb = 0;
uint32_t v_ma = 0;
uint32_t v_mb = 0;
uint32_t v_d = 0;
uint32_t v_s = 0;
uint32_t v_z = 0;
uint32_t v_rb = 0;
v_sign = (a_a & 2147483648u);
v_ea = ((a_a >> 23u) & 255u);
v_eb = ((a_b >> 23u) & 255u);
v_fa = (a_a & 8388607u);
v_fb = (a_b & 8388607u);
if (v_ea == v_eb) {
if (v_fa == v_fb) {
return 0u;
} else if (v_fa < v_fb) {
v_sign ^= 2147483648u;
v_z = ((uint32_t)(v_fb - v_fa));
} else {
v_z = ((uint32_t)(v_fa - v_fb));
}
v_s = (((uint32_t)(wuffs_softfloat__arith__clz32(self, v_z) - 8u)) & 31u);
if (v_s >= v_ea) {
return 4294967295u;
}
return (v_sign | ((uint32_t)(((uint32_t)(((uint32_t)(((uint32_t)(v_ea - v_s)) - 1u)) << 23u)) + ((uint32_t)(v_z << v_s)))));
}
v_ma = ((v_fa | 8388608u) << 7u);
v_mb = ((v_fb | 8388608u) << 7u);
if (v_ea < v_eb) {
v_sign ^= 2147483648u;
v_d = (((uint32_t)(v_eb - v_ea)) & 255u);
v_ea = v_eb;
v_z = v_ma;
v_ma = v_mb;
v_mb = v_z;
} else {
v_d = (((uint32_t)(v_ea - v_eb)) & 255u);
}
if (v_d >= 31u) {
v_mb = 1u;
} else if (((uint32_t)(v_mb << ((32u - v_d) & 31u))) != 0u) {
v_mb = ((v_mb >> v_d) | 1u);
} else {
v_mb = (v_mb >> v_d);
}
v_z = ((uint32_t)(v_ma - v_mb));
v_s = (((uint32_t)(wuffs_softfloat__arith__clz32(self, v_z) - 1u)) & 31u);
if (v_s >= v_ea) {
return 4294967295u;
}
v_ea = (((uint32_t)(v_ea - v_s)) & 255u);
v_z = ((uint32_t)(v_z << v_s));
v_rb = (v_z & 127u);
v_z = (((uint32_t)(v_z + 64u)) >> 7u);
if (v_rb == 64u) {
v_z &= 4294967294u;
}
return (v_sign | ((uint32_t)(((uint32_t)(((uint32_t)(v_ea - 1u)) << 23u)) + v_z)));
}
// -------- func softfloat.arith.slow_f32_add
WUFFS_BASE__GENERATED_C_CODE
static uint32_t
wuffs_softfloat__arith__slow_f32_add(
const wuffs_softfloat__arith* self,
uint32_t a_a,
uint32_t a_b) {
return wuffs_softfloat__arith__f64_to_f32(self, wuffs_softfloat__arith__f64_add(self, wuffs_softfloat__arith__f32_to_f64(self, a_a), wuffs_softfloat__arith__f32_to_f64(self, a_b)));
}
// -------- func softfloat.arith.slow_f32_sub
WUFFS_BASE__GENERATED_C_CODE
static uint32_t
wuffs_softfloat__arith__slow_f32_sub(
const wuffs_softfloat__arith* self,
uint32_t a_a,
uint32_t a_b) {
return wuffs_softfloat__arith__f64_to_f32(self, wuffs_softfloat__arith__f64_sub(self, wuffs_softfloat__arith__f32_to_f64(self, a_a), wuffs_softfloat__arith__f32_to_f64(self, a_b)));
}
// -------- func softfloat.arith.slow_f32_mul
WUFFS_BASE__GENERATED_C_CODE
static uint32_t
wuffs_softfloat__arith__slow_f32_mul(
const wuffs_softfloat__arith* self,
uint32_t a_a,
uint32_t a_b) {
return wuffs_softfloat__arith__f64_to_f32(self, wuffs_softfloat__arith__f64_mul(self, wuffs_softfloat__arith__f32_to_f64(self, a_a), wuffs_softfloat__arith__f32_to_f64(self, a_b)));
}
+571
View File
@@ -0,0 +1,571 @@
// Copyright 2026 The Wuffs Authors.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// SPDX-License-Identifier: Apache-2.0 OR MIT
// ----------------
// manual-test-softfloat-sweep.c systematically checks Wuffs' std/softfloat
// binary32 operations (and the f64 to / from f32 conversions) over every
// combination of operand exponents, signs and mantissa patterns, plus
// exhaustive sweeps of the integer conversions. Each f32 result is compared
// against:
//
// - the C compiler's native (hardware) float arithmetic: the bits must be
// identical, except that a NaN only has to match a NaN, and
// - the "via f64" definition, f64_to_f32(f64_op(f32_to_f64(a),
// f32_to_f64(b))), computed with the package's own (public) f64 functions:
// here the bits must be identical even for NaNs, so that the NaN payloads
// and signs produced by the f32 fast paths' slow path fallbacks are
// exactly those of the via-f64 implementation.
//
// The operands for the binary operations are every (exponent_a, exponent_b)
// pair in [0 ..= 255] x [0 ..= 255], times both signs for each operand, times
// NUM_MANTISSAS x NUM_MANTISSAS mantissa patterns (zero, one, all ones, only
// top / bottom bits, half-way patterns and pseudo-random ones, so that exact
// results, rounding ties, carries, cancellation, subnormal results, overflow
// to infinity and NaN payloads are all covered).
//
// The f64 add, sub, mul and div operations are checked the same way (every
// exponent pair in [0 ..= 2047] x [0 ..= 2047], both signs, NUM_MANTISSAS64
// x NUM_MANTISSAS64 mantissa patterns) against the hardware, with NaN
// results checked bit-for-bit against the package's documented NaN rule:
// the first NaN operand, quietened, else the default NaN 0xFFF8_0000_0000_0000.
//
// Build it WITHOUT -ffast-math and WITHOUT -mfma / -march=native:
//
// gcc -O2 -std=c99 script/manual-test-softfloat-sweep.c -lm -lpthread
// -o /tmp/sfsweep
//
// (as one command line), then run /tmp/sfsweep (or /tmp/sfsweep OPNAME to run
// only one op: f32_add, f32_sub, f32_mul, f32_div, f64_to_f32, f32_to_f64,
// f32_from_u32, f32_from_i32 or f32_from_u64). It exits with a non-zero
// status if there were any mismatches.
#define _POSIX_C_SOURCE 200809L
#include <inttypes.h>
#include <pthread.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#define WUFFS_IMPLEMENTATION
#define WUFFS_CONFIG__STATIC_FUNCTIONS
#define WUFFS_CONFIG__MODULES
#define WUFFS_CONFIG__MODULE__BASE
#define WUFFS_CONFIG__MODULE__SOFTFLOAT
#include "../release/c/wuffs-unsupported-snapshot.c"
#if defined(__FAST_MATH__) || defined(__FMA__)
#error "Build without -ffast-math and without -mfma / -march=native"
#endif
#define NUM_THREADS 4
static wuffs_softfloat__arith g_sf;
static float //
f_of(uint32_t u) {
float f;
memcpy(&f, &u, 4);
return f;
}
static uint32_t //
u_of_f(float f) {
uint32_t u;
memcpy(&u, &f, 4);
return u;
}
static double //
d_of(uint64_t u) {
double d;
memcpy(&d, &u, 8);
return d;
}
static uint64_t //
u_of_d(double d) {
uint64_t u;
memcpy(&u, &d, 8);
return u;
}
static bool //
is_nan32(uint32_t u) {
return (u & 0x7FFFFFFFu) > 0x7F800000u;
}
static bool //
is_nan64(uint64_t u) {
return (u & 0x7FFFFFFFFFFFFFFFull) > 0x7FF0000000000000ull;
}
// ---------------- Mantissa patterns.
#define NUM_MANTISSAS 40
static uint32_t g_mantissas[NUM_MANTISSAS];
static void //
init_mantissas() {
static const uint32_t fixed[] = {
0x000000, 0x000001, 0x000002, 0x000003, 0x7FFFFF, 0x7FFFFE, 0x7FFFFD,
0x400000, 0x400001, 0x3FFFFF, 0x200000, 0x600000, 0x000100, 0x0000FF,
0x100001, 0x080000, 0x7C0000, 0x000040, 0x00003F, 0x555555, 0x2AAAAA,
0x7FFF80, 0x000080, 0x400080, 0x3FFF80,
};
size_t n = sizeof(fixed) / sizeof(fixed[0]);
memcpy(g_mantissas, fixed, sizeof(fixed));
uint64_t x = 0x9E3779B97F4A7C15ull;
for (size_t i = n; i < NUM_MANTISSAS; i++) {
x ^= x >> 12;
x ^= x << 25;
x ^= x >> 27;
uint32_t r = (uint32_t)((x * 0x2545F4914F6CDD1Dull) >> 41) & 0x7FFFFF;
if (i & 1) {
r &= 0x7FF000 | (r & 0xF); // Sparse.
}
g_mantissas[i] = r;
}
}
#define NUM_MANTISSAS64 9
static uint64_t g_mantissas64[NUM_MANTISSAS64];
static void //
init_mantissas64() {
static const uint64_t fixed[] = {
0x0000000000000ull, 0x0000000000001ull, 0xFFFFFFFFFFFFFull,
0x8000000000000ull, 0x8000000000001ull, 0x0000000000200ull,
};
size_t n = sizeof(fixed) / sizeof(fixed[0]);
memcpy(g_mantissas64, fixed, sizeof(fixed));
uint64_t x = 0x0123456789ABCDEFull;
for (size_t i = n; i < NUM_MANTISSAS64; i++) {
x ^= x >> 12;
x ^= x << 25;
x ^= x >> 27;
uint64_t r = (x * 0x2545F4914F6CDD1Dull) >> 12;
if (i & 1) {
r &= 0xFFF00000000FFull; // Sparse.
}
g_mantissas64[i] = r;
}
}
// ---------------- Binary ops.
typedef uint32_t (*wuffs_op)(const wuffs_softfloat__arith*, uint32_t, uint32_t);
typedef uint64_t (*wuffs_op64)(const wuffs_softfloat__arith*,
uint64_t,
uint64_t);
static float //
hw_add(float a, float b) {
return a + b;
}
static float //
hw_sub(float a, float b) {
return a - b;
}
static float //
hw_mul(float a, float b) {
return a * b;
}
static float //
hw_div(float a, float b) {
return a / b;
}
static struct {
const char* name;
wuffs_op op;
wuffs_op64 op64;
float (*hw)(float, float);
} g_binops[] = {
{"f32_add", wuffs_softfloat__arith__f32_add, wuffs_softfloat__arith__f64_add,
hw_add},
{"f32_sub", wuffs_softfloat__arith__f32_sub, wuffs_softfloat__arith__f64_sub,
hw_sub},
{"f32_mul", wuffs_softfloat__arith__f32_mul, wuffs_softfloat__arith__f64_mul,
hw_mul},
{"f32_div", wuffs_softfloat__arith__f32_div, wuffs_softfloat__arith__f64_div,
hw_div},
};
static double //
hw_add64(double a, double b) {
return a + b;
}
static double //
hw_sub64(double a, double b) {
return a - b;
}
static double //
hw_mul64(double a, double b) {
return a * b;
}
static double //
hw_div64(double a, double b) {
return a / b;
}
static struct {
const char* name;
wuffs_op64 op64;
double (*hw)(double, double);
} g_binops64[] = {
{"f64_add", wuffs_softfloat__arith__f64_add, hw_add64},
{"f64_sub", wuffs_softfloat__arith__f64_sub, hw_sub64},
{"f64_mul", wuffs_softfloat__arith__f64_mul, hw_mul64},
{"f64_div", wuffs_softfloat__arith__f64_div, hw_div64},
};
typedef struct {
int which;
uint32_t ea_lo;
uint32_t ea_hi;
uint64_t count;
uint64_t mismatches;
} job;
static pthread_mutex_t g_print_mutex = PTHREAD_MUTEX_INITIALIZER;
static void //
report(job* j, const char* name, uint32_t a, uint32_t b, uint32_t have,
uint32_t want_hw, uint32_t want_via) {
j->mismatches++;
if (j->mismatches <= 5) {
pthread_mutex_lock(&g_print_mutex);
printf("MISMATCH %s(0x%08" PRIX32 ", 0x%08" PRIX32 "): have 0x%08" PRIX32
", hardware 0x%08" PRIX32 ", via f64 0x%08" PRIX32 "\n",
name, a, b, have, want_hw, want_via);
pthread_mutex_unlock(&g_print_mutex);
}
}
static void* //
run_binop(void* arg) {
job* j = (job*)arg;
wuffs_op op = g_binops[j->which].op;
wuffs_op64 op64 = g_binops[j->which].op64;
float (*hw)(float, float) = g_binops[j->which].hw;
const char* name = g_binops[j->which].name;
for (uint32_t ea = j->ea_lo; ea < j->ea_hi; ea++) {
for (uint32_t eb = 0; eb < 256; eb++) {
for (uint32_t ma = 0; ma < NUM_MANTISSAS; ma++) {
for (uint32_t mb = 0; mb < NUM_MANTISSAS; mb++) {
for (uint32_t signs = 0; signs < 4; signs++) {
uint32_t a = ((signs & 1) << 31) | (ea << 23) | g_mantissas[ma];
uint32_t b = ((signs >> 1) << 31) | (eb << 23) | g_mantissas[mb];
uint32_t have = op(&g_sf, a, b);
uint32_t want_hw = u_of_f(hw(f_of(a), f_of(b)));
uint32_t want_via = wuffs_softfloat__arith__f64_to_f32(
&g_sf, op64(&g_sf, wuffs_softfloat__arith__f32_to_f64(&g_sf, a),
wuffs_softfloat__arith__f32_to_f64(&g_sf, b)));
j->count++;
if (have != want_via) {
report(j, name, a, b, have, want_hw, want_via);
} else if (is_nan32(have) ? !is_nan32(want_hw)
: (have != want_hw)) {
report(j, name, a, b, have, want_hw, want_via);
}
}
}
}
}
}
return NULL;
}
static void* //
run_binop64(void* arg) {
job* j = (job*)arg;
wuffs_op64 op64 = g_binops64[j->which].op64;
double (*hw)(double, double) = g_binops64[j->which].hw;
const char* name = g_binops64[j->which].name;
for (uint64_t ea = j->ea_lo; ea < j->ea_hi; ea++) {
for (uint64_t eb = 0; eb < 2048; eb++) {
for (uint32_t ma = 0; ma < NUM_MANTISSAS64; ma++) {
for (uint32_t mb = 0; mb < NUM_MANTISSAS64; mb++) {
for (uint64_t signs = 0; signs < 4; signs++) {
uint64_t a = ((signs & 1) << 63) | (ea << 52) | g_mantissas64[ma];
uint64_t b = ((signs >> 1) << 63) | (eb << 52) | g_mantissas64[mb];
uint64_t have = op64(&g_sf, a, b);
uint64_t want = u_of_d(hw(d_of(a), d_of(b)));
j->count++;
if (is_nan64(want)) {
want = is_nan64(a) ? (a | 0x0008000000000000ull)
: is_nan64(b) ? (b | 0x0008000000000000ull)
: 0xFFF8000000000000ull;
}
if (have != want) {
j->mismatches++;
if (j->mismatches <= 5) {
pthread_mutex_lock(&g_print_mutex);
printf("MISMATCH %s(0x%016" PRIX64 ", 0x%016" PRIX64
"): have 0x%016" PRIX64 ", want 0x%016" PRIX64 "\n",
name, a, b, have, want);
pthread_mutex_unlock(&g_print_mutex);
}
}
}
}
}
}
}
return NULL;
}
// ---------------- Conversions.
typedef struct {
int which;
uint64_t lo;
uint64_t hi;
uint64_t count;
uint64_t mismatches;
} cjob;
static void //
creport(cjob* j, const char* name, uint64_t a, uint64_t have, uint64_t want) {
j->mismatches++;
if (j->mismatches <= 5) {
pthread_mutex_lock(&g_print_mutex);
printf("MISMATCH %s(0x%016" PRIX64 "): have 0x%016" PRIX64
", want 0x%016" PRIX64 "\n",
name, a, have, want);
pthread_mutex_unlock(&g_print_mutex);
}
}
// f32_from_u64 operands: (1 << top) plus, for top <= 26, every pattern of
// the bits below it or, for larger top, every pattern of the 13 bits just
// below it (where the rounding happens) and of the 13 bottom bits (sticky).
static uint64_t //
u64_operand(uint64_t i) {
uint32_t top = (uint32_t)(i >> 26) & 63;
uint64_t low = i & 0x3FFFFFF;
uint64_t below = low;
if (top > 26) {
below = ((low >> 13) << (top - 13)) | (low & 0x1FFF);
}
below &= (((uint64_t)1) << top) - 1;
return (((uint64_t)1) << top) | below;
}
// f64_to_f32_exponent maps [0 ..= 325] to the f64 biased exponents 0, 1, 2,
// 0x340 ..= 0x47F (the f32 normal and subnormal range, and a little more on
// either side) and 0x7FD, 0x7FE and 0x7FF.
static uint64_t //
f64_to_f32_exponent(uint32_t i) {
if (i < 3) {
return i;
} else if (i < 323) {
return 0x340 + (i - 3);
}
return 0x7FD + (i - 323);
}
static void* //
run_conv(void* arg) {
cjob* j = (cjob*)arg;
for (uint64_t i = j->lo; i < j->hi; i++) {
j->count++;
switch (j->which) {
case 0: { // f64_to_f32: exponents near the f32 range, many fractions.
uint64_t e = f64_to_f32_exponent((uint32_t)(i >> 20));
uint64_t f = i & 0xFFFFF;
// 10 bits at the top of the fraction, 10 around the rounding point
// (bit 29) and 6 (repeated) at the bottom.
uint64_t frac = ((f >> 10) << 42) | ((f & 0x3FF) << 24) |
((f & 0x3F) * 0x41);
for (int s = 0; s < 2; s++) {
uint64_t a = ((uint64_t)s << 63) | (e << 52) | frac;
uint32_t have = wuffs_softfloat__arith__f64_to_f32(&g_sf, a);
uint32_t want = u_of_f((float)d_of(a));
if (is_nan32(have) ? !is_nan32(want) : (have != want)) {
creport(j, "f64_to_f32", a, have, want);
} else if (is_nan32(have) &&
(have != (((uint32_t)(a >> 32) & 0x80000000u) |
0x7FC00000u |
(uint32_t)((a & 0xFFFFFFFFFFFFFull) >> 29)))) {
creport(j, "f64_to_f32 (NaN payload)", a, have, want);
}
}
break;
}
case 1: { // f32_to_f64: every f32.
uint32_t a = (uint32_t)i;
uint64_t have = wuffs_softfloat__arith__f32_to_f64(&g_sf, a);
uint64_t want = u_of_d((double)f_of(a));
if (is_nan64(have) ? !is_nan64(want) : (have != want)) {
creport(j, "f32_to_f64", a, have, want);
} else if (is_nan64(have) &&
(have != (((uint64_t)(a >> 31) << 63) |
0x7FF8000000000000ull |
((uint64_t)(a & 0x7FFFFF) << 29)))) {
creport(j, "f32_to_f64 (NaN payload)", a, have, want);
}
break;
}
case 2: { // f32_from_u32: every u32.
uint32_t a = (uint32_t)i;
uint32_t have = wuffs_softfloat__arith__f32_from_u32(&g_sf, a);
uint32_t want = u_of_f((float)a);
if (have != want) {
creport(j, "f32_from_u32", a, have, want);
}
break;
}
case 3: { // f32_from_i32: every i32.
int32_t a = (int32_t)(uint32_t)i;
uint32_t have = wuffs_softfloat__arith__f32_from_i32(&g_sf, a);
uint32_t want = u_of_f((float)a);
if (have != want) {
creport(j, "f32_from_i32", (uint32_t)a, have, want);
}
break;
}
case 4: { // f32_from_u64.
uint64_t a = u64_operand(i);
uint32_t have = wuffs_softfloat__arith__f32_from_u64(&g_sf, a);
uint32_t want = u_of_f((float)a);
if (have != want) {
creport(j, "f32_from_u64", a, have, want);
}
break;
}
}
}
return NULL;
}
static const struct {
const char* name;
uint64_t n;
} g_convs[] = {
{"f64_to_f32", ((uint64_t)326) << 20},
{"f32_to_f64", ((uint64_t)1) << 32},
{"f32_from_u32", ((uint64_t)1) << 32},
{"f32_from_i32", ((uint64_t)1) << 32},
{"f32_from_u64", ((uint64_t)64) << 26},
};
static double //
now() {
struct timespec t;
clock_gettime(CLOCK_MONOTONIC, &t);
return (double)t.tv_sec + (1e-9 * (double)t.tv_nsec);
}
int //
main(int argc, char** argv) {
const char* only = (argc > 1) ? argv[1] : NULL;
wuffs_base__status status = wuffs_softfloat__arith__initialize(
&g_sf, sizeof g_sf, WUFFS_VERSION, 0);
if (status.repr) {
printf("initialize: %s\n", status.repr);
return 1;
}
init_mantissas();
init_mantissas64();
uint64_t total_mismatches = 0;
double t_start = now();
for (int w = 0; w < (int)(sizeof(g_binops) / sizeof(g_binops[0])); w++) {
if (only && strcmp(only, g_binops[w].name)) {
continue;
}
double t0 = now();
job jobs[NUM_THREADS];
pthread_t threads[NUM_THREADS];
for (int t = 0; t < NUM_THREADS; t++) {
jobs[t] = (job){w, (uint32_t)(t * 256 / NUM_THREADS),
(uint32_t)((t + 1) * 256 / NUM_THREADS), 0, 0};
pthread_create(&threads[t], NULL, run_binop, &jobs[t]);
}
uint64_t count = 0, mismatches = 0;
for (int t = 0; t < NUM_THREADS; t++) {
pthread_join(threads[t], NULL);
count += jobs[t].count;
mismatches += jobs[t].mismatches;
}
total_mismatches += mismatches;
printf("%-14s %12" PRIu64 " operand pairs, %8" PRIu64
" mismatches, %6.1f seconds\n",
g_binops[w].name, count, mismatches, now() - t0);
fflush(stdout);
}
for (int w = 0; w < (int)(sizeof(g_binops64) / sizeof(g_binops64[0]));
w++) {
if (only && strcmp(only, g_binops64[w].name)) {
continue;
}
double t0 = now();
job jobs[NUM_THREADS];
pthread_t threads[NUM_THREADS];
for (int t = 0; t < NUM_THREADS; t++) {
jobs[t] = (job){w, (uint32_t)(t * 2048 / NUM_THREADS),
(uint32_t)((t + 1) * 2048 / NUM_THREADS), 0, 0};
pthread_create(&threads[t], NULL, run_binop64, &jobs[t]);
}
uint64_t count = 0, mismatches = 0;
for (int t = 0; t < NUM_THREADS; t++) {
pthread_join(threads[t], NULL);
count += jobs[t].count;
mismatches += jobs[t].mismatches;
}
total_mismatches += mismatches;
printf("%-14s %12" PRIu64 " operand pairs, %8" PRIu64
" mismatches, %6.1f seconds\n",
g_binops64[w].name, count, mismatches, now() - t0);
fflush(stdout);
}
for (int w = 0; w < (int)(sizeof(g_convs) / sizeof(g_convs[0])); w++) {
if (only && strcmp(only, g_convs[w].name)) {
continue;
}
double t0 = now();
cjob jobs[NUM_THREADS];
pthread_t threads[NUM_THREADS];
uint64_t n = g_convs[w].n;
for (int t = 0; t < NUM_THREADS; t++) {
jobs[t] = (cjob){w, (n / NUM_THREADS) * (uint64_t)t,
(t == NUM_THREADS - 1) ? n
: (n / NUM_THREADS) * (t + 1),
0, 0};
pthread_create(&threads[t], NULL, run_conv, &jobs[t]);
}
uint64_t count = 0, mismatches = 0;
for (int t = 0; t < NUM_THREADS; t++) {
pthread_join(threads[t], NULL);
count += jobs[t].count;
mismatches += jobs[t].mismatches;
}
total_mismatches += mismatches;
printf("%-14s %12" PRIu64 " operands, %8" PRIu64
" mismatches, %6.1f seconds\n",
g_convs[w].name, count, mismatches, now() - t0);
fflush(stdout);
}
printf("%s: %" PRIu64 " mismatches in total (%.0f seconds)\n",
total_mismatches ? "FAIL" : "PASS", total_mismatches,
now() - t_start);
return total_mismatches ? 1 : 0;
}
+47 -6
View File
@@ -96,12 +96,24 @@ integer long division, 11 quotient bits per (64-bit integer) division step,
with the final remainder as the sticky bit. Leading zero counts use a 256-entry
table.
The `f32` add, sub, mul and div operations convert their operands exactly to
`f64`, perform the `f64` operation and then round to `f32`. This double
rounding is innocuous: binary64's 53 bits of precision are more than `(2 * 24)
+ 2`, which suffices for `+`, `-`, `*` and `/` (S. A. Figueroa, "When is double
rounding innocuous?", ACM SIGNUM Newsletter, 1995). The tests confirm this
against hardware, including for subnormal `f32` results.
`f64_add` and `f64_sub` have a fast path for the common case where both
operands are normal (`f64_add_normal_mags` and `f64_sub_normal_mags`): it
skips the special-case handling, aligns the significands with a sticky bit,
adds or subtracts them and rounds inline. Only a subtraction whose result is
subnormal falls back to the general code (`f64_add_mags`, `f64_sub_mags`).
Likewise, `f32_add`, `f32_sub` and `f32_mul` compute the result directly in
binary32 (with 32-bit significands for add and sub and an exact 48-bit
product for mul) when both operands are normal and the result is normal (or,
for add and mul, overflows to infinity). Otherwise (zeroes, subnormal
operands or results, infinities and NaNs), they, and `f32_div` always,
convert their operands exactly to `f64`, perform the `f64` operation and then
round to `f32`. This double rounding is innocuous: binary64's 53 bits of
precision are more than `(2 * 24) + 2`, which suffices for `+`, `-`, `*` and
`/` (S. A. Figueroa, "When is double rounding innocuous?", ACM SIGNUM
Newsletter, 1995). The tests confirm this against hardware, including for
subnormal `f32` results. The fast paths never produce a NaN, so NaN results
(and their payloads) are exactly those of the "via `f64`" computation.
## log2
@@ -187,3 +199,32 @@ threads):
(6 minutes 20 seconds wall clock in total)
No exception table is needed.
`script/manual-test-softfloat-sweep.c` systematically checks the `f32` add,
sub, mul and div operations over every pair of operand exponents (`[0 ..=
255]` squared), both signs and 40 x 40 mantissa patterns (exact results, ties,
carries, cancellation, subnormal results, overflow, NaN payloads): each result
must equal the hardware's (any NaN matching any NaN) and, bit-for-bit
including NaNs, the "via `f64`" composition `f64_to_f32(f64_op(f32_to_f64(a),
f32_to_f64(b)))`. It does the same for the `f64` add, sub, mul and div
operations (every pair of exponents in `[0 ..= 2047]`, 9 x 9 mantissa
patterns, NaN results checked bit-for-bit against the NaN rule above), and
checks `f64_to_f32` (326 exponents times 2²⁰ fraction patterns times both
signs), `f32_to_f64` (every `f32`), `f32_from_u32` and `f32_from_i32` (every
32-bit integer) and `f32_from_u64` (2³² structured values). Result (x86-64,
gcc 13 `-O2`, 4 threads):
f32_add 419430400 operand pairs, 0 mismatches
f32_sub 419430400 operand pairs, 0 mismatches
f32_mul 419430400 operand pairs, 0 mismatches
f32_div 419430400 operand pairs, 0 mismatches
f64_add 1358954496 operand pairs, 0 mismatches
f64_sub 1358954496 operand pairs, 0 mismatches
f64_mul 1358954496 operand pairs, 0 mismatches
f64_div 1358954496 operand pairs, 0 mismatches
f64_to_f32 341835776 operands, 0 mismatches
f32_to_f64 4294967296 operands, 0 mismatches
f32_from_u32 4294967296 operands, 0 mismatches
f32_from_i32 4294967296 operands, 0 mismatches
f32_from_u64 4294967296 operands, 0 mismatches
PASS: 0 mismatches in total
+179
View File
@@ -61,6 +61,24 @@ pri func arith.clz64(a: base.u64) base.u32[..= 63] {
return (n32 + n16 + n8 + ((CLZ8[x >> 56] as base.u32) & 7)) & 63
}
// clz32 returns the number of leading zero bits of a, which must be non-zero.
pri func arith.clz32(a: base.u32) base.u32[..= 31] {
var x : base.u32
var n16 : base.u32[..= 16]
var n8 : base.u32[..= 8]
x = args.a
if (x >> 16) == 0 {
x = x ~mod<< 16
n16 = 16
}
if (x >> 24) == 0 {
x = x ~mod<< 8
n8 = 8
}
return (n16 + n8 + ((CLZ8[x >> 24] as base.u32) & 7)) & 31
}
// shift_right_jam64 returns (a >> dist), except that if any non-zero bits
// were shifted out then the least significant bit of the result is set. This
// "sticky bit" is needed to round correctly.
@@ -345,6 +363,18 @@ pub func arith.f64_max(a: base.u64, b: base.u64) base.u64 {
// f64_add returns (a + b).
pub func arith.f64_add(a: base.u64, b: base.u64) base.u64 {
var z : base.u64
if ((((args.a >> 52) & 0x7FF) ~mod- 1) < 0x7FE) and ((((args.b >> 52) & 0x7FF) ~mod- 1) < 0x7FE) {
// Both operands are normal: take the fast path.
if ((args.a ^ args.b) >> 63) == 0 {
return this.f64_add_normal_mags(a: args.a, b: args.b)
}
z = this.f64_sub_normal_mags(a: args.a, b: args.b)
if z <> 0xFFFF_FFFF_FFFF_FFFF {
return z
}
}
if ((args.a ^ args.b) >> 63) == 0 {
return this.f64_add_mags(a: args.a, b: args.b, sign: args.a & F64_SIGN)
}
@@ -353,12 +383,161 @@ pub func arith.f64_add(a: base.u64, b: base.u64) base.u64 {
// f64_sub returns (a - b).
pub func arith.f64_sub(a: base.u64, b: base.u64) base.u64 {
var z : base.u64
if ((((args.a >> 52) & 0x7FF) ~mod- 1) < 0x7FE) and ((((args.b >> 52) & 0x7FF) ~mod- 1) < 0x7FE) {
// Both operands are normal: take the fast path.
if ((args.a ^ args.b) >> 63) <> 0 {
return this.f64_add_normal_mags(a: args.a, b: args.b)
}
z = this.f64_sub_normal_mags(a: args.a, b: args.b)
if z <> 0xFFFF_FFFF_FFFF_FFFF {
return z
}
}
if ((args.a ^ args.b) >> 63) == 0 {
return this.f64_sub_mags(a: args.a, b: args.b, sign: args.a & F64_SIGN)
}
return this.f64_add_mags(a: args.a, b: args.b, sign: args.a & F64_SIGN)
}
// f64_add_normal_mags returns (sign(a) * (|a| + |b|)), for normal a and b.
// It is a faster version of f64_add_mags.
pri func arith.f64_add_normal_mags(a: base.u64, b: base.u64) base.u64 {
var ea : base.u32[..= 0x800]
var eb : base.u32[..= 0x7FF]
var ma : base.u64[..= 0x3FFF_FFFF_FFFF_FE00]
var mb : base.u64[..= 0x3FFF_FFFF_FFFF_FE00]
var d : base.u32[..= 0x7FF]
var z : base.u64
var rb : base.u64
ea = ((args.a >> 52) & 0x7FF) as base.u32
eb = ((args.b >> 52) & 0x7FF) as base.u32
// The significands, with their implicit bit at bit 61 and 9 extra bits
// of precision at the bottom.
ma = ((args.a & F64_FRAC) | F64_HIDDEN) << 9
mb = ((args.b & F64_FRAC) | F64_HIDDEN) << 9
if ea < eb {
d = (eb ~mod- ea) & 0x7FF
ea = eb
z = ma
ma = mb
mb = z
} else {
d = (ea ~mod- eb) & 0x7FF
}
if d >= 63 {
mb = 1
} else if d > 0 {
if (mb ~mod<< (64 - d)) <> 0 {
mb = (mb >> d) | 1
} else {
mb = mb >> d
}
}
z = ma + mb
if z >= 0x4000_0000_0000_0000 {
ea += 1
z = (z >> 1) | (z & 1)
}
rb = z & 0x1FF
z = (z ~mod+ 0x100) >> 9
if rb == 0x100 {
z &= 0xFFFF_FFFF_FFFF_FFFE
}
// ea is in [1 ..= 0x7FF] and z is in [1 << 52 ..= 1 << 53]. Adding z
// (whose bit 52 is the implicit bit) carries any rounding overflow into
// the exponent field.
z = (((ea ~mod- 1) as base.u64) ~mod<< 52) ~mod+ z
if z >= F64_INF {
z = F64_INF
}
return (args.a & F64_SIGN) | z
}
// f64_sub_normal_mags returns (sign(a) * (|a| - |b|)), for normal a and b,
// unless the result is subnormal, in which case it returns
// 0xFFFF_FFFF_FFFF_FFFF (a NaN, which it never otherwise returns) and the
// caller must take its slow path. It is a faster version of f64_sub_mags.
pri func arith.f64_sub_normal_mags(a: base.u64, b: base.u64) base.u64 {
var sign : base.u64
var ea : base.u32[..= 0x7FF]
var eb : base.u32[..= 0x7FF]
var fa : base.u64[..= 0x000F_FFFF_FFFF_FFFF]
var fb : base.u64[..= 0x000F_FFFF_FFFF_FFFF]
var ma : base.u64
var mb : base.u64
var d : base.u32[..= 0x7FF]
var s : base.u32[..= 63]
var z : base.u64
var rb : base.u64
sign = args.a & F64_SIGN
ea = ((args.a >> 52) & 0x7FF) as base.u32
eb = ((args.b >> 52) & 0x7FF) as base.u32
fa = args.a & F64_FRAC
fb = args.b & F64_FRAC
if ea == eb {
// The difference is exact.
if fa == fb {
// Exact cancellation gives +0 when rounding to nearest.
return 0
} else if fa < fb {
sign ^= F64_SIGN
z = fb ~mod- fa
} else {
z = fa ~mod- fb
}
// z is in [1 ..= F64_FRAC]. Shift its top bit to bit 52.
s = (this.clz64(a: z) ~mod- 11) & 63
if s >= ea {
return 0xFFFF_FFFF_FFFF_FFFF
}
return sign | (((((ea ~mod- s) ~mod- 1) as base.u64) ~mod<< 52) ~mod+ (z ~mod<< s))
}
// The significands, with their implicit bit at bit 62 and 10 extra bits
// of precision at the bottom.
ma = (fa | F64_HIDDEN) << 10
mb = (fb | F64_HIDDEN) << 10
if ea < eb {
sign ^= F64_SIGN
d = (eb ~mod- ea) & 0x7FF
ea = eb
z = ma
ma = mb
mb = z
} else {
d = (ea ~mod- eb) & 0x7FF
}
if d >= 63 {
mb = 1
} else if (mb ~mod<< ((64 - d) & 63)) <> 0 {
mb = (mb >> d) | 1
} else {
mb = mb >> d
}
// ma > mb, so z is non-zero. If (d >= 2) then z >= (1 << 61) and the
// normalization shift is at most 1. Otherwise, no bits were shifted out.
z = ma ~mod- mb
s = (this.clz64(a: z) ~mod- 1) & 63
if s >= ea {
return 0xFFFF_FFFF_FFFF_FFFF
}
ea = (ea ~mod- s) & 0x7FF
z = z ~mod<< s
rb = z & 0x3FF
z = (z ~mod+ 0x200) >> 10
if rb == 0x200 {
z &= 0xFFFF_FFFF_FFFF_FFFE
}
// ea is in [1 ..= 0x7FE] and z is in [1 << 52 ..= 1 << 53], so this does
// not overflow.
return sign | ((((ea ~mod- 1) as base.u64) ~mod<< 52) ~mod+ z)
}
// f64_add_mags returns (sign * (|a| + |b|)).
pri func arith.f64_add_mags(a: base.u64, b: base.u64, sign: base.u64) base.u64 {
var exp_a : base.u32
+228 -10
View File
@@ -10,12 +10,18 @@
// -------- f32 (binary32) operations.
//
// The f32 add, sub, mul and div functions convert their (exactly
// representable) operands to f64, perform the f64 operation and round the
// result to f32. Double rounding is innocuous here: binary64 has more than
// (2 * 24) + 2 bits of precision, which suffices for +, -, * and / (Figueroa,
// "When is double rounding innocuous?", 1995). This is also exhaustively
// cross-checked against hardware by the tests.
// When both operands are normal (and, for sub_mags and mul, when the result
// is normal too), the f32 add, sub and mul functions compute the result
// directly in binary32 (with 32-bit or 64-bit integer significands). This is
// the common case. Otherwise (zeroes, subnormals, infinities, NaNs and
// subnormal results), they and f32_div convert their (exactly representable)
// operands to f64, perform the f64 operation and round the result to f32
// (the slow_etc functions). Double rounding is innocuous here: binary64 has
// more than (2 * 24) + 2 bits of precision, which suffices for +, -, * and /
// (Figueroa, "When is double rounding innocuous?", 1995). This is also
// exhaustively cross-checked against hardware by the tests. The fast paths
// never return a NaN, so NaN results (and their payloads) always come from
// the slow paths.
pri const F32_SIGN : base.u32 = 0x8000_0000
pri const F32_ABS : base.u32 = 0x7FFF_FFFF
@@ -74,20 +80,232 @@ pub func arith.f32_abs(a: base.u32) base.u32 {
// f32_add returns (a + b).
pub func arith.f32_add(a: base.u32, b: base.u32) base.u32 {
var z : base.u32
if ((((args.a >> 23) & 0xFF) ~mod- 1) < 0xFE) and ((((args.b >> 23) & 0xFF) ~mod- 1) < 0xFE) {
if ((args.a ^ args.b) >> 31) == 0 {
return this.f32_add_mags(a: args.a, b: args.b)
}
z = this.f32_sub_mags(a: args.a, b: args.b)
if z <> 0xFFFF_FFFF {
return z
}
}
return this.slow_f32_add(a: args.a, b: args.b)
}
// f32_sub returns (a - b).
pub func arith.f32_sub(a: base.u32, b: base.u32) base.u32 {
var z : base.u32
if ((((args.a >> 23) & 0xFF) ~mod- 1) < 0xFE) and ((((args.b >> 23) & 0xFF) ~mod- 1) < 0xFE) {
if ((args.a ^ args.b) >> 31) <> 0 {
return this.f32_add_mags(a: args.a, b: args.b)
}
z = this.f32_sub_mags(a: args.a, b: args.b)
if z <> 0xFFFF_FFFF {
return z
}
}
return this.slow_f32_sub(a: args.a, b: args.b)
}
// f32_mul returns (a * b).
pub func arith.f32_mul(a: base.u32, b: base.u32) base.u32 {
var ea : base.u32[..= 0xFF]
var eb : base.u32[..= 0xFF]
var e : base.u32[..= 0x1FF]
var p : base.u64
var m : base.u32
var r : base.u32
var z : base.u32
ea = (args.a >> 23) & 0xFF
eb = (args.b >> 23) & 0xFF
if ((ea ~mod- 1) < 0xFE) and ((eb ~mod- 1) < 0xFE) {
// Both operands are normal. The exact product of the two 24-bit
// significands, p, is in [1 << 46, 1 << 48). The biased exponent of
// the result, e, is (ea + eb - 127) (plus 1 if p >= (1 << 47)).
p = (((args.a & 0x7F_FFFF) | 0x80_0000) as base.u64) *
(((args.b & 0x7F_FFFF) | 0x80_0000) as base.u64)
e = ea + eb
if p >= 0x8000_0000_0000 {
e += 1
m = ((p >> 24) & 0xFF_FFFF) as base.u32
r = ((p & 0xFF_FFFF) as base.u32) ~mod- 0x80_0000
} else {
m = ((p >> 23) & 0xFF_FFFF) as base.u32
r = ((p & 0x7F_FFFF) as base.u32) ~mod- 0x40_0000
}
// r is the remainder minus half an ULP, as a two's complement
// integer. Round to nearest, ties to even.
if (r <> 0) and (r < 0x8000_0000) {
m ~mod+= 1
} else if r == 0 {
m ~mod+= m & 1
}
if (e >= 128) and (e <= 381) {
// Adding m (whose bit 23 is the implicit bit) carries any
// rounding overflow into the exponent field.
z = ((e - 128) << 23) ~mod+ m
if z >= F32_INF {
z = F32_INF
}
return ((args.a ^ args.b) & F32_SIGN) | z
}
}
return this.slow_f32_mul(a: args.a, b: args.b)
}
// f32_add_mags returns (sign(a) * (|a| + |b|)), for normal a and b.
pri func arith.f32_add_mags(a: base.u32, b: base.u32) base.u32 {
var ea : base.u32[..= 0x100]
var eb : base.u32[..= 0xFF]
var ma : base.u32[..= 0x3FFF_FFC0]
var mb : base.u32[..= 0x3FFF_FFC0]
var d : base.u32[..= 0xFF]
var z : base.u32
var rb : base.u32
ea = (args.a >> 23) & 0xFF
eb = (args.b >> 23) & 0xFF
// The significands, with their implicit bit at bit 29 and 6 extra bits
// of precision at the bottom.
ma = ((args.a & 0x7F_FFFF) | 0x80_0000) << 6
mb = ((args.b & 0x7F_FFFF) | 0x80_0000) << 6
if ea < eb {
d = (eb ~mod- ea) & 0xFF
ea = eb
z = ma
ma = mb
mb = z
} else {
d = (ea ~mod- eb) & 0xFF
}
if d >= 31 {
mb = 1
} else if d > 0 {
if (mb ~mod<< (32 - d)) <> 0 {
mb = (mb >> d) | 1
} else {
mb = mb >> d
}
}
z = (ma & 0x3FFF_FFFF) + (mb & 0x3FFF_FFFF)
if z >= 0x4000_0000 {
ea += 1
z = (z >> 1) | (z & 1)
}
rb = z & 0x3F
z = (z ~mod+ 0x20) >> 6
if rb == 0x20 {
z &= 0xFFFF_FFFE
}
// ea is in [1 ..= 255] and z is in [1 << 23 ..= 1 << 24].
z = ((ea ~mod- 1) ~mod<< 23) ~mod+ z
if z >= F32_INF {
z = F32_INF
}
return (args.a & F32_SIGN) | z
}
// f32_sub_mags returns (sign(a) * (|a| - |b|)), for normal a and b, unless
// the result is subnormal, in which case it returns 0xFFFF_FFFF (a NaN, which
// it never otherwise returns) and the caller must take its slow path.
pri func arith.f32_sub_mags(a: base.u32, b: base.u32) base.u32 {
var sign : base.u32
var ea : base.u32[..= 0xFF]
var eb : base.u32[..= 0xFF]
var fa : base.u32[..= 0x7F_FFFF]
var fb : base.u32[..= 0x7F_FFFF]
var ma : base.u32
var mb : base.u32
var d : base.u32[..= 0xFF]
var s : base.u32[..= 31]
var z : base.u32
var rb : base.u32
sign = args.a & F32_SIGN
ea = (args.a >> 23) & 0xFF
eb = (args.b >> 23) & 0xFF
fa = args.a & 0x7F_FFFF
fb = args.b & 0x7F_FFFF
if ea == eb {
// The difference is exact.
if fa == fb {
// Exact cancellation gives +0 when rounding to nearest.
return 0
} else if fa < fb {
sign ^= F32_SIGN
z = fb ~mod- fa
} else {
z = fa ~mod- fb
}
// z is in [1 ..= 0x7F_FFFF]. Shift its top bit to bit 23.
s = (this.clz32(a: z) ~mod- 8) & 31
if s >= ea {
return 0xFFFF_FFFF
}
return sign | ((((ea ~mod- s) ~mod- 1) ~mod<< 23) ~mod+ (z ~mod<< s))
}
// The significands, with their implicit bit at bit 30 and 7 extra bits
// of precision at the bottom.
ma = (fa | 0x80_0000) << 7
mb = (fb | 0x80_0000) << 7
if ea < eb {
sign ^= F32_SIGN
d = (eb ~mod- ea) & 0xFF
ea = eb
z = ma
ma = mb
mb = z
} else {
d = (ea ~mod- eb) & 0xFF
}
if d >= 31 {
mb = 1
} else if (mb ~mod<< ((32 - d) & 31)) <> 0 {
mb = (mb >> d) | 1
} else {
mb = mb >> d
}
// ma > mb, so z is non-zero. If (d >= 2) then z >= (1 << 29) and the
// normalization shift is at most 1. Otherwise, no bits were shifted out.
z = ma ~mod- mb
s = (this.clz32(a: z) ~mod- 1) & 31
if s >= ea {
return 0xFFFF_FFFF
}
ea = (ea ~mod- s) & 0xFF
z = z ~mod<< s
rb = z & 0x7F
z = (z ~mod+ 0x40) >> 7
if rb == 0x40 {
z &= 0xFFFF_FFFE
}
// ea is in [1 ..= 254] and z is in [1 << 23 ..= 1 << 24], so this does
// not overflow.
return sign | (((ea ~mod- 1) ~mod<< 23) ~mod+ z)
}
// slow_f32_add returns (a + b), computed via f64.
pri func arith.slow_f32_add(a: base.u32, b: base.u32) base.u32 {
return this.f64_to_f32(a: this.f64_add(
a: this.f32_to_f64(a: args.a),
b: this.f32_to_f64(a: args.b)))
}
// f32_sub returns (a - b).
pub func arith.f32_sub(a: base.u32, b: base.u32) base.u32 {
// slow_f32_sub returns (a - b), computed via f64.
pri func arith.slow_f32_sub(a: base.u32, b: base.u32) base.u32 {
return this.f64_to_f32(a: this.f64_sub(
a: this.f32_to_f64(a: args.a),
b: this.f32_to_f64(a: args.b)))
}
// f32_mul returns (a * b).
pub func arith.f32_mul(a: base.u32, b: base.u32) base.u32 {
// slow_f32_mul returns (a * b), computed via f64.
pri func arith.slow_f32_mul(a: base.u32, b: base.u32) base.u32 {
return this.f64_to_f32(a: this.f64_mul(
a: this.f32_to_f64(a: args.a),
b: this.f32_to_f64(a: args.b)))