mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-20 16:47:28 +01:00
Merge claude/project-thread-0izcgq (speed work)
This commit is contained in:
15 files changed
+3327
-329
No files matched your search
+30
-10
@@ -55,8 +55,9 @@ Without the large-window quirk and without attached dictionaries (see the
|
||||
window size plus some slack. The slack
|
||||
mirrors the C code's `kRingBufferWriteAheadSlack`, which its fast paths use to
|
||||
write (and discard) a few bytes past the ring buffer's end. This
|
||||
implementation does not yet have such fast paths, but it still reserves the
|
||||
slack, so that adding them later will not change the `workbuf_len` API. A
|
||||
implementation's fast paths (see below) do not write past the ring buffer's
|
||||
end, but it still reserves the slack, so that adding such fast paths later
|
||||
will not change the `workbuf_len` API. A
|
||||
workbuf shorter than the stream's window size plus 542 bytes is rejected with
|
||||
`base."#bad workbuf length"`.
|
||||
|
||||
@@ -158,12 +159,25 @@ invariants for memory safety but Wuffs requires explicit bounds checks.
|
||||
|
||||
- The bit reader reads input bytes only when it needs them (like the C code's
|
||||
"safe" `BrotliSafeReadBits` and `SafeReadSymbol` functions), so that
|
||||
`transform_io` can suspend at any point. There is no "fast path" (like the C
|
||||
code's `ProcessCommands`, as opposed to `SafeProcessCommands`). Both paths
|
||||
produce the same output and errors.
|
||||
- LZ77 copies and dictionary words are written to the ring buffer one byte at
|
||||
a time, with ring buffer events (the C code's `WriteRingBuffer` and
|
||||
`WrapRingBuffer` calls) at the same positions as the C code.
|
||||
`transform_io` can suspend at any point. Between reads, fewer than 8 bits
|
||||
are buffered. The fast path, `process_commands_fast` (like the C code's
|
||||
`ProcessCommands`, as opposed to `SafeProcessCommands`), is not a coroutine.
|
||||
It reads ahead (up to 8 bytes at a time) into a 64-bit bit buffer while the
|
||||
input has enough bytes, and when it returns to `process_commands` (at a
|
||||
block switch, a ring buffer event, a dictionary reference or when the input
|
||||
runs low), it gives back the whole bytes left in that buffer, which restores
|
||||
the "fewer than 8 bits" state exactly. `process_commands` tracks where it is
|
||||
within a command (`cmd_state`), so that either can pick up where the other
|
||||
left off. Within `process_commands`, `read_symbol_fast` and `read_bits_fast`
|
||||
(which are not coroutines either) read the same bytes at the same times as
|
||||
the coroutines `read_symbol` and `read_bits`, which finish the job when the
|
||||
input runs out. All paths produce the same output and errors.
|
||||
- Dictionary words, and LZ77 copies that wrap around the physical ring buffer
|
||||
or reach the C code's (virtual) ring buffer size, are written to the ring
|
||||
buffer one byte at a time, with ring buffer events (the C code's
|
||||
`WriteRingBuffer` and `WrapRingBuffer` calls) at the same positions as the
|
||||
C code. Other LZ77 copies are written in chunks of at most the copy distance
|
||||
(so that each chunk's source bytes are final), which is equivalent.
|
||||
- On error, the Wuffs decoder may have already written more output to `dst`
|
||||
than libbrotlidec would have (libbrotlidec does not flush on error), but
|
||||
what both have written agrees.
|
||||
@@ -173,7 +187,8 @@ invariants for memory safety but Wuffs requires explicit bounds checks.
|
||||
|
||||
- Metadata callbacks (metadata blocks are currently skipped). See the
|
||||
"Metadata Blocks" section below.
|
||||
- Fast paths.
|
||||
- More fast paths, for example for dictionary words, for copies that wrap
|
||||
around the ring buffer and for block switches.
|
||||
|
||||
|
||||
## Encoder
|
||||
@@ -969,7 +984,12 @@ uses it at qualities 10 and 11 (and its smaller helpers at lower
|
||||
qualities). Its
|
||||
fixed-size state (about 0.8 MiB: FindBlocks' cost arrays and a 64Ki-entry
|
||||
FastLog2 cache in its `bit_cost`) lives in the struct; everything that the C
|
||||
code allocates on the heap lives in a caller-supplied workbuf.
|
||||
code allocates on the heap lives in a caller-supplied workbuf. The FastLog2
|
||||
cache (like the encoder's own `bit_cost` one) is filled lazily, one entry at
|
||||
a time, when `fast_log2_memo` first needs that entry (a bitmap, zeroed when the
|
||||
struct is initialized, records which entries are valid): typically only a few
|
||||
thousand of the 65536 entries are ever used, and each one costs a (slow, exact)
|
||||
software `log2`.
|
||||
|
||||
Data formats (all little-endian, matching the C structs' memory layout on a
|
||||
64-bit little-endian machine, so that results can be compared with `memcmp`):
|
||||
|
||||
@@ -230,6 +230,13 @@ pub struct decoder? implements base.io_transformer(
|
||||
insert_length : base.u32,
|
||||
copy_length : base.u32,
|
||||
|
||||
// cmd_state is where process_commands (and process_commands_fast)
|
||||
// are within a command: 0 at its start, 1 when reading its literals
|
||||
// (insert_length is how many are left), 2 when reading its distance
|
||||
// and 3 when applying its copy. process_commands_fast sets it to 4
|
||||
// when the metablock is done.
|
||||
cmd_state : base.u32[..= 4],
|
||||
|
||||
// quirk_large_window is the QUIRK_LARGE_WINDOW quirk (the C code's
|
||||
// BROTLI_DECODER_PARAM_LARGE_WINDOW parameter). large_window is
|
||||
// whether the stream is a large-window stream (the C code's
|
||||
@@ -679,6 +686,93 @@ pri func decoder.read_symbol?(src: base.io_reader, hoff: base.u32) {
|
||||
this.n_bits = n_bits
|
||||
}
|
||||
|
||||
// read_bits_fast is like read_bits but is not a coroutine. It reads input
|
||||
// bytes exactly when read_bits would, for as long as args.src has them. It
|
||||
// returns 0 when it has set this.bits_value, or 1 when read_bits must finish
|
||||
// the job (this.bits and this.n_bits hold the bits read so far).
|
||||
pri func decoder.read_bits_fast!(src: base.io_reader, n: base.u32[..= 24]) base.u32 {
|
||||
var n : base.u32[..= 24]
|
||||
var bits : base.u32
|
||||
var n_bits : base.u32
|
||||
|
||||
n = args.n
|
||||
bits = this.bits
|
||||
n_bits = this.n_bits
|
||||
while n_bits < n,
|
||||
post n_bits >= n,
|
||||
{
|
||||
if args.src.length() <= 0 {
|
||||
this.bits = bits
|
||||
this.n_bits = n_bits
|
||||
return 1
|
||||
}
|
||||
assert n_bits < 24 via "a < b: a < c; c <= b"(c: n)
|
||||
bits |= args.src.peek_u8_as_u32() << n_bits
|
||||
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
|
||||
n_bits += 8
|
||||
}
|
||||
this.bits_value = bits & (((1 as base.u32) << n) - 1)
|
||||
this.bits = bits >> n
|
||||
this.n_bits = n_bits - n
|
||||
return 0
|
||||
}
|
||||
|
||||
// read_symbol_fast is like read_symbol but is not a coroutine. It reads
|
||||
// input bytes exactly when read_symbol would, for as long as args.src has
|
||||
// them. It returns 0 when it has set this.symbol, or 1 when read_symbol must
|
||||
// finish the job (this.bits and this.n_bits hold the bits read so far). In
|
||||
// particular, read_symbol (not read_symbol_fast) returns any error.
|
||||
pri func decoder.read_symbol_fast!(src: base.io_reader, hoff: base.u32) base.u32 {
|
||||
var hoff : base.u32
|
||||
var bits : base.u32
|
||||
var n_bits : base.u32
|
||||
var i : base.u32
|
||||
var e : base.u32
|
||||
var e_bits : base.u32[..= 23]
|
||||
|
||||
hoff = args.hoff
|
||||
bits = this.bits
|
||||
n_bits = this.n_bits
|
||||
while true {
|
||||
i = hoff ~mod+ (bits & 0xFF)
|
||||
if i >= HUFFS_LEN {
|
||||
break
|
||||
}
|
||||
e = this.huffs[i] as base.u32
|
||||
e_bits = e & 15
|
||||
if e_bits <= 8 {
|
||||
if n_bits >= e_bits {
|
||||
this.symbol = e >> 4
|
||||
this.bits = bits >> e_bits
|
||||
this.n_bits = n_bits - e_bits
|
||||
return 0
|
||||
}
|
||||
} else if n_bits > 8 {
|
||||
i = (hoff ~mod+ (e >> 4)) ~mod+ ((bits >> 8) & (((1 as base.u32) << (e_bits - 8)) - 1))
|
||||
if i >= HUFFS_LEN {
|
||||
break
|
||||
}
|
||||
e = this.huffs[i] as base.u32
|
||||
e_bits = (e & 15) + 8
|
||||
if n_bits >= e_bits {
|
||||
this.symbol = e >> 4
|
||||
this.bits = bits >> e_bits
|
||||
this.n_bits = n_bits - e_bits
|
||||
return 0
|
||||
}
|
||||
}
|
||||
if (n_bits >= 24) or (args.src.length() <= 0) {
|
||||
break
|
||||
}
|
||||
bits |= args.src.peek_u8_as_u32() << n_bits
|
||||
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
|
||||
n_bits += 8
|
||||
}
|
||||
this.bits = bits
|
||||
this.n_bits = n_bits
|
||||
return 1
|
||||
}
|
||||
|
||||
// jump_to_byte_boundary drops the bits up to the next byte boundary and
|
||||
// returns whether they were all zero. Mirrors BrotliJumpToByteBoundary in
|
||||
// bit_reader.h.
|
||||
|
||||
@@ -418,7 +418,6 @@ pri func decoder.calculate_distance_lut!() {
|
||||
// Mirrors ProcessCommandsInternal in decode.c.
|
||||
pri func decoder.process_commands?(src: base.io_reader, workbuf: slice base.u8, dictbuf: roslice base.u8) {
|
||||
var cmd_code : base.u32[..= 703]
|
||||
var insert_length : base.u32
|
||||
var p1 : base.u32[..= 0xFF]
|
||||
var p2 : base.u32[..= 0xFF]
|
||||
var mode_offset : base.u32[..= 0x600]
|
||||
@@ -434,37 +433,72 @@ pri func decoder.process_commands?(src: base.io_reader, workbuf: slice base.u8,
|
||||
var address : base.u32
|
||||
var chunk : base.u32[..= 14]
|
||||
var j : base.u64
|
||||
var fr : base.u32
|
||||
var dst_i : base.u64
|
||||
var src_i : base.u64
|
||||
var k64 : base.u64
|
||||
var n64 : base.u64
|
||||
var dlo : base.u64
|
||||
var dhi : base.u64
|
||||
var slo : base.u64
|
||||
var shi : base.u64
|
||||
var fs : base.status
|
||||
|
||||
while true {
|
||||
// CommandBegin.
|
||||
if this.block_length[1] == 0 {
|
||||
// Mirrors DecodeCommandBlockSwitch.
|
||||
this.decode_block_switch?(src: args.src, t: 1)
|
||||
this.htree_command = this.block_type_rb[3] & 0xFF
|
||||
continue
|
||||
if (this.cmd_state == 0) and (args.src.length() >= 16) {
|
||||
fs = this.process_commands_fast!(src: args.src, workbuf: args.workbuf)
|
||||
if not fs.is_ok() {
|
||||
return fs
|
||||
} else if this.cmd_state == 4 {
|
||||
this.cmd_state = 0
|
||||
return ok
|
||||
}
|
||||
}
|
||||
|
||||
// Mirrors ReadCommandInternal.
|
||||
this.read_symbol?(src: args.src,
|
||||
hoff: CMD_HUFFS_BASE + (this.htree_command * CMD_HUFFS_STRIDE))
|
||||
if this.symbol >= 704 {
|
||||
return "#internal error: inconsistent Huffman table"
|
||||
if this.cmd_state == 0 {
|
||||
// CommandBegin.
|
||||
if this.block_length[1] == 0 {
|
||||
// Mirrors DecodeCommandBlockSwitch.
|
||||
this.decode_block_switch?(src: args.src, t: 1)
|
||||
this.htree_command = this.block_type_rb[3] & 0xFF
|
||||
continue
|
||||
}
|
||||
|
||||
// Mirrors ReadCommandInternal.
|
||||
fr = this.read_symbol_fast!(src: args.src, hoff: CMD_HUFFS_BASE + (this.htree_command * CMD_HUFFS_STRIDE))
|
||||
if fr <> 0 {
|
||||
this.read_symbol?(src: args.src,
|
||||
hoff: CMD_HUFFS_BASE + (this.htree_command * CMD_HUFFS_STRIDE))
|
||||
}
|
||||
if this.symbol >= 704 {
|
||||
return "#internal error: inconsistent Huffman table"
|
||||
}
|
||||
cmd_code = this.symbol.min(no_more_than: 703)
|
||||
this.implicit_zero_distance = CMD_IMPLICIT_ZERO_DISTANCE[cmd_code] <> 0
|
||||
this.distance_context = CMD_DISTANCE_CONTEXT[cmd_code] as base.u32
|
||||
this.update_dist_htree_index!()
|
||||
fr = this.read_bits_fast!(src: args.src, n: CMD_INSERT_LEN_EXTRA_BITS[cmd_code] as base.u32)
|
||||
if fr <> 0 {
|
||||
this.read_bits?(src: args.src, n: CMD_INSERT_LEN_EXTRA_BITS[cmd_code] as base.u32)
|
||||
}
|
||||
this.insert_length = CMD_INSERT_LEN_OFFSET[cmd_code] ~mod+ this.bits_value
|
||||
fr = this.read_bits_fast!(src: args.src, n: CMD_COPY_LEN_EXTRA_BITS[cmd_code] as base.u32)
|
||||
if fr <> 0 {
|
||||
this.read_bits?(src: args.src, n: CMD_COPY_LEN_EXTRA_BITS[cmd_code] as base.u32)
|
||||
}
|
||||
this.copy_length = CMD_COPY_LEN_OFFSET[cmd_code] ~mod+ this.bits_value
|
||||
this.block_length[1] ~mod-= 1
|
||||
if this.insert_length > 0 {
|
||||
this.mb_remaining ~mod-= this.insert_length
|
||||
this.cmd_state = 1
|
||||
} else {
|
||||
this.cmd_state = 2
|
||||
}
|
||||
}
|
||||
cmd_code = this.symbol.min(no_more_than: 703)
|
||||
this.implicit_zero_distance = CMD_IMPLICIT_ZERO_DISTANCE[cmd_code] <> 0
|
||||
this.distance_context = CMD_DISTANCE_CONTEXT[cmd_code] as base.u32
|
||||
this.update_dist_htree_index!()
|
||||
this.read_bits?(src: args.src, n: CMD_INSERT_LEN_EXTRA_BITS[cmd_code] as base.u32)
|
||||
insert_length = CMD_INSERT_LEN_OFFSET[cmd_code] ~mod+ this.bits_value
|
||||
this.read_bits?(src: args.src, n: CMD_COPY_LEN_EXTRA_BITS[cmd_code] as base.u32)
|
||||
this.copy_length = CMD_COPY_LEN_OFFSET[cmd_code] ~mod+ this.bits_value
|
||||
this.block_length[1] ~mod-= 1
|
||||
|
||||
if insert_length > 0 {
|
||||
this.mb_remaining ~mod-= insert_length
|
||||
|
||||
if this.cmd_state == 1 {
|
||||
// CommandInner. Read the literals in the command.
|
||||
while insert_length > 0 {
|
||||
while this.insert_length > 0 {
|
||||
if this.block_length[0] == 0 {
|
||||
// Mirrors DecodeLiteralBlockSwitch.
|
||||
this.decode_block_switch?(src: args.src, t: 0)
|
||||
@@ -478,39 +512,48 @@ pri func decoder.process_commands?(src: base.io_reader, workbuf: slice base.u8,
|
||||
ctx = (CONTEXT_LOOKUP[mode_offset | p1] |
|
||||
CONTEXT_LOOKUP[mode_offset | 0x100 | p2]) as base.u32
|
||||
tree = this.context_maps[this.context_map_slice + ctx] as base.u32
|
||||
this.read_symbol?(src: args.src, hoff: LIT_HUFFS_BASE + (tree * LIT_HUFFS_STRIDE))
|
||||
fr = this.read_symbol_fast!(src: args.src, hoff: LIT_HUFFS_BASE + (tree * LIT_HUFFS_STRIDE))
|
||||
if fr <> 0 {
|
||||
this.read_symbol?(src: args.src, hoff: LIT_HUFFS_BASE + (tree * LIT_HUFFS_STRIDE))
|
||||
}
|
||||
this.put_byte!(workbuf: args.workbuf, b: (this.symbol & 0xFF) as base.u8)
|
||||
this.block_length[0] ~mod-= 1
|
||||
insert_length -= 1
|
||||
this.insert_length ~mod-= 1
|
||||
if this.pos == this.rb_size {
|
||||
this.ring_event?()
|
||||
}
|
||||
}
|
||||
|
||||
this.cmd_state = 2
|
||||
if (this.mb_remaining == 0) or (this.mb_remaining >= 0x8000_0000) {
|
||||
// BROTLI_STATE_METABLOCK_DONE.
|
||||
this.cmd_state = 0
|
||||
return ok
|
||||
}
|
||||
}
|
||||
|
||||
// CommandPostDecodeLiterals.
|
||||
if this.implicit_zero_distance {
|
||||
// Implicit distance case.
|
||||
this.distance_context = 1
|
||||
this.dist_rb_idx ~mod-= 1
|
||||
this.distance_code = this.dist_rb[this.dist_rb_idx & 3]
|
||||
} else {
|
||||
// Read distance code in the command, unless it was implicitly
|
||||
// zero.
|
||||
if this.block_length[2] == 0 {
|
||||
// Mirrors DecodeDistanceBlockSwitch.
|
||||
this.decode_block_switch?(src: args.src, t: 2)
|
||||
this.dist_context_map_slice = (this.block_type_rb[5] & 0xFF) << 2
|
||||
this.update_dist_htree_index!()
|
||||
if this.cmd_state == 2 {
|
||||
// CommandPostDecodeLiterals.
|
||||
if this.implicit_zero_distance {
|
||||
// Implicit distance case.
|
||||
this.distance_context = 1
|
||||
this.dist_rb_idx ~mod-= 1
|
||||
this.distance_code = this.dist_rb[this.dist_rb_idx & 3]
|
||||
} else {
|
||||
// Read distance code in the command, unless it was implicitly
|
||||
// zero.
|
||||
if this.block_length[2] == 0 {
|
||||
// Mirrors DecodeDistanceBlockSwitch.
|
||||
this.decode_block_switch?(src: args.src, t: 2)
|
||||
this.dist_context_map_slice = (this.block_type_rb[5] & 0xFF) << 2
|
||||
this.update_dist_htree_index!()
|
||||
}
|
||||
this.read_distance?(src: args.src)
|
||||
}
|
||||
this.read_distance?(src: args.src)
|
||||
this.cmd_state = 3
|
||||
}
|
||||
|
||||
// this.cmd_state == 3.
|
||||
if this.max_distance <> this.max_backward_distance {
|
||||
this.max_distance = this.pos.min(no_more_than: this.max_backward_distance)
|
||||
}
|
||||
@@ -604,6 +647,37 @@ pri func decoder.process_commands?(src: base.io_reader, workbuf: slice base.u8,
|
||||
this.dist_rb[this.dist_rb_idx & 3] = distance
|
||||
this.dist_rb_idx ~mod+= 1
|
||||
this.mb_remaining ~mod-= i
|
||||
// Fast path: when the copy neither wraps around the physical ring
|
||||
// buffer (at either end) nor makes this.pos reach this.rb_size,
|
||||
// copy the bytes in chunks of at most distance bytes. Each chunk's
|
||||
// source bytes are final (and do not overlap the chunk's
|
||||
// destination), so this matches the byte by byte loop below.
|
||||
dst_i = this.pos as base.u64
|
||||
src_i = ((this.pos ~mod- distance) & (this.window_size ~mod- 1)) as base.u64
|
||||
if ((dst_i + (i as base.u64)) < (this.rb_size as base.u64)) and
|
||||
((src_i + (i as base.u64)) <= (this.window_size as base.u64)) and
|
||||
((this.window_size as base.u64) <= args.workbuf.length()) and
|
||||
(distance > 0) {
|
||||
k64 = 0
|
||||
while k64 < (i as base.u64) {
|
||||
n64 = (i as base.u64) ~mod- k64
|
||||
n64 = n64.min(no_more_than: distance as base.u64)
|
||||
dlo = dst_i ~mod+ k64
|
||||
dhi = dlo ~mod+ n64
|
||||
slo = src_i ~mod+ k64
|
||||
shi = slo ~mod+ n64
|
||||
if (n64 == 0) or (dlo > dhi) or (dhi > args.workbuf.length()) or
|
||||
(slo > shi) or (shi > args.workbuf.length()) {
|
||||
break
|
||||
}
|
||||
args.workbuf[dlo .. dhi].copy_from_slice!(s: args.workbuf[slo .. shi])
|
||||
k64 ~mod+= n64
|
||||
}
|
||||
// k64 == i, unless the (unreachable) break above was taken,
|
||||
// in which case the loop below copies the remaining bytes.
|
||||
this.pos ~mod+= (k64 & 0xFFFF_FFFF) as base.u32
|
||||
i ~mod-= (k64 & 0xFFFF_FFFF) as base.u32
|
||||
}
|
||||
while i > 0 {
|
||||
b = this.get_byte(workbuf: args.workbuf, distance: distance)
|
||||
this.put_byte!(workbuf: args.workbuf, b: b)
|
||||
@@ -614,6 +688,7 @@ pri func decoder.process_commands?(src: base.io_reader, workbuf: slice base.u8,
|
||||
}
|
||||
}
|
||||
|
||||
this.cmd_state = 0
|
||||
if (this.mb_remaining == 0) or (this.mb_remaining >= 0x8000_0000) {
|
||||
// BROTLI_STATE_METABLOCK_DONE.
|
||||
return ok
|
||||
@@ -621,6 +696,323 @@ pri func decoder.process_commands?(src: base.io_reader, workbuf: slice base.u8,
|
||||
}
|
||||
}
|
||||
|
||||
// process_commands_fast is a fast path for process_commands. It is not a
|
||||
// coroutine. It handles commands, or parts of commands (see
|
||||
// decoder.cmd_state), for as long as args.src holds enough bytes and the work
|
||||
// needs no block switch, ring buffer event or dictionary reference. Then it
|
||||
// returns, and process_commands picks up where it left off.
|
||||
//
|
||||
// It reads ahead, up to 8 bytes at a time, into a 64-bit bit buffer. When it
|
||||
// returns, it gives back (undoes) the whole bytes left in that buffer. The
|
||||
// coroutines read_symbol and read_bits read an input byte only when they
|
||||
// need more bits, so that between reads, fewer than 8 bits are buffered.
|
||||
// Giving back the whole bytes restores that state exactly, so this function
|
||||
// reads the same bits and leaves the decoder in the same state as
|
||||
// process_commands would, and it writes the same bytes and returns the same
|
||||
// errors. It sets this.cmd_state to 4 when the metablock is done.
|
||||
pri func decoder.process_commands_fast!(src: base.io_reader, workbuf: slice base.u8) base.status {
|
||||
var ret : base.status
|
||||
var bits : base.u64
|
||||
var n_bits : base.u32[..= 63]
|
||||
var i : base.u32
|
||||
var e : base.u32
|
||||
var e_bits : base.u32[..= 23]
|
||||
var cmd_code : base.u32[..= 703]
|
||||
var nb : base.u32[..= 24]
|
||||
var p1 : base.u32[..= 0xFF]
|
||||
var p2 : base.u32[..= 0xFF]
|
||||
var mode_offset : base.u32[..= 0x600]
|
||||
var ctx : base.u32[..= 63]
|
||||
var tree : base.u32[..= 0xFF]
|
||||
var code : base.u32
|
||||
var nbits : base.u32[..= 32]
|
||||
var lo : base.u32
|
||||
var max_distance : base.u32
|
||||
var distance : base.u32
|
||||
var n : base.u32
|
||||
var dst_i : base.u64
|
||||
var src_i : base.u64
|
||||
var k64 : base.u64
|
||||
var n64 : base.u64
|
||||
var dlo : base.u64
|
||||
var dhi : base.u64
|
||||
var slo : base.u64
|
||||
var shi : base.u64
|
||||
|
||||
if this.n_bits >= 8 {
|
||||
return ok
|
||||
}
|
||||
n_bits = this.n_bits & 7
|
||||
bits = (this.bits as base.u64) & (((1 as base.u64) << n_bits) - 1)
|
||||
|
||||
while.outer true {
|
||||
if this.cmd_state == 0 {
|
||||
// Two refills, of at most 8 bytes each.
|
||||
if (this.block_length[1] == 0) or (args.src.length() < 16) {
|
||||
break.outer
|
||||
}
|
||||
bits |= args.src.peek_u64le() ~mod<< n_bits
|
||||
args.src.skip_u32_fast!(actual: (63 - n_bits) >> 3, worst_case: 8)
|
||||
n_bits |= 56
|
||||
// Mirrors ReadCommandInternal.
|
||||
i = (CMD_HUFFS_BASE + (this.htree_command * CMD_HUFFS_STRIDE)) ~mod+ ((bits & 0xFF) as base.u32)
|
||||
if i >= HUFFS_LEN {
|
||||
ret = "#internal error: inconsistent Huffman table"
|
||||
break.outer
|
||||
}
|
||||
e = this.huffs[i] as base.u32
|
||||
e_bits = e & 15
|
||||
if e_bits > 8 {
|
||||
i = ((CMD_HUFFS_BASE + (this.htree_command * CMD_HUFFS_STRIDE)) ~mod+ (e >> 4)) ~mod+ ((((bits >> 8) & 0x7FFF) as base.u32) & (((1 as base.u32) << (e_bits - 8)) - 1))
|
||||
if i >= HUFFS_LEN {
|
||||
ret = "#internal error: inconsistent Huffman table"
|
||||
break.outer
|
||||
}
|
||||
e = this.huffs[i] as base.u32
|
||||
e_bits = (e & 15) + 8
|
||||
}
|
||||
if n_bits < e_bits {
|
||||
ret = "#internal error: inconsistent bit reader"
|
||||
break.outer
|
||||
}
|
||||
bits >>= e_bits
|
||||
n_bits -= e_bits
|
||||
if (e >> 4) >= 704 {
|
||||
ret = "#internal error: inconsistent Huffman table"
|
||||
break.outer
|
||||
}
|
||||
this.symbol = e >> 4
|
||||
cmd_code = this.symbol.min(no_more_than: 703)
|
||||
this.implicit_zero_distance = CMD_IMPLICIT_ZERO_DISTANCE[cmd_code] <> 0
|
||||
this.distance_context = CMD_DISTANCE_CONTEXT[cmd_code] as base.u32
|
||||
this.update_dist_htree_index!()
|
||||
nb = CMD_INSERT_LEN_EXTRA_BITS[cmd_code] as base.u32
|
||||
if n_bits < nb {
|
||||
ret = "#internal error: inconsistent bit reader"
|
||||
break.outer
|
||||
}
|
||||
this.bits_value = ((bits & (((1 as base.u64) << nb) - 1)) & 0xFFFF_FFFF) as base.u32
|
||||
bits >>= nb
|
||||
n_bits -= nb
|
||||
this.insert_length = CMD_INSERT_LEN_OFFSET[cmd_code] ~mod+ this.bits_value
|
||||
if args.src.length() < 8 {
|
||||
ret = "#internal error: inconsistent bit reader"
|
||||
break.outer
|
||||
}
|
||||
bits |= args.src.peek_u64le() ~mod<< n_bits
|
||||
args.src.skip_u32_fast!(actual: (63 - n_bits) >> 3, worst_case: 8)
|
||||
n_bits |= 56
|
||||
nb = CMD_COPY_LEN_EXTRA_BITS[cmd_code] as base.u32
|
||||
if n_bits < nb {
|
||||
ret = "#internal error: inconsistent bit reader"
|
||||
break.outer
|
||||
}
|
||||
this.bits_value = ((bits & (((1 as base.u64) << nb) - 1)) & 0xFFFF_FFFF) as base.u32
|
||||
bits >>= nb
|
||||
n_bits -= nb
|
||||
this.copy_length = CMD_COPY_LEN_OFFSET[cmd_code] ~mod+ this.bits_value
|
||||
this.block_length[1] ~mod-= 1
|
||||
if this.insert_length > 0 {
|
||||
this.mb_remaining ~mod-= this.insert_length
|
||||
this.cmd_state = 1
|
||||
} else {
|
||||
this.cmd_state = 2
|
||||
}
|
||||
}
|
||||
|
||||
if this.cmd_state == 1 {
|
||||
// CommandInner. Read the literals in the command. p1 and p2 are
|
||||
// the last two bytes written to the ring buffer (put_byte always
|
||||
// writes when the physical ring buffer fits in args.workbuf).
|
||||
if (this.window_size as base.u64) > args.workbuf.length() {
|
||||
break.outer
|
||||
}
|
||||
p1 = this.get_byte(workbuf: args.workbuf, distance: 1) as base.u32
|
||||
p2 = this.get_byte(workbuf: args.workbuf, distance: 2) as base.u32
|
||||
while this.insert_length > 0 {
|
||||
if (this.block_length[0] == 0) or ((this.pos ~mod+ 1) >= this.rb_size) {
|
||||
break.outer
|
||||
}
|
||||
if n_bits < 24 {
|
||||
if args.src.length() < 8 {
|
||||
break.outer
|
||||
}
|
||||
bits |= args.src.peek_u64le() ~mod<< n_bits
|
||||
args.src.skip_u32_fast!(actual: (63 - n_bits) >> 3, worst_case: 8)
|
||||
n_bits |= 56
|
||||
}
|
||||
mode_offset = this.context_mode << 9
|
||||
ctx = (CONTEXT_LOOKUP[mode_offset | p1] |
|
||||
CONTEXT_LOOKUP[mode_offset | 0x100 | p2]) as base.u32
|
||||
tree = this.context_maps[this.context_map_slice + ctx] as base.u32
|
||||
i = (LIT_HUFFS_BASE + (tree * LIT_HUFFS_STRIDE)) ~mod+ ((bits & 0xFF) as base.u32)
|
||||
if i >= HUFFS_LEN {
|
||||
ret = "#internal error: inconsistent Huffman table"
|
||||
break.outer
|
||||
}
|
||||
e = this.huffs[i] as base.u32
|
||||
e_bits = e & 15
|
||||
if e_bits > 8 {
|
||||
i = ((LIT_HUFFS_BASE + (tree * LIT_HUFFS_STRIDE)) ~mod+ (e >> 4)) ~mod+ ((((bits >> 8) & 0x7FFF) as base.u32) & (((1 as base.u32) << (e_bits - 8)) - 1))
|
||||
if i >= HUFFS_LEN {
|
||||
ret = "#internal error: inconsistent Huffman table"
|
||||
break.outer
|
||||
}
|
||||
e = this.huffs[i] as base.u32
|
||||
e_bits = (e & 15) + 8
|
||||
}
|
||||
if n_bits < e_bits {
|
||||
ret = "#internal error: inconsistent bit reader"
|
||||
break.outer
|
||||
}
|
||||
bits >>= e_bits
|
||||
n_bits -= e_bits
|
||||
this.symbol = e >> 4
|
||||
this.put_byte!(workbuf: args.workbuf, b: (this.symbol & 0xFF) as base.u8)
|
||||
p2 = p1
|
||||
p1 = this.symbol & 0xFF
|
||||
this.block_length[0] ~mod-= 1
|
||||
this.insert_length ~mod-= 1
|
||||
}
|
||||
this.cmd_state = 2
|
||||
if (this.mb_remaining == 0) or (this.mb_remaining >= 0x8000_0000) {
|
||||
this.cmd_state = 4
|
||||
break.outer
|
||||
}
|
||||
}
|
||||
|
||||
if this.cmd_state == 2 {
|
||||
// CommandPostDecodeLiterals.
|
||||
if this.implicit_zero_distance {
|
||||
this.distance_context = 1
|
||||
this.dist_rb_idx ~mod-= 1
|
||||
this.distance_code = this.dist_rb[this.dist_rb_idx & 3]
|
||||
} else {
|
||||
// One refill: 56 bits suffice for a symbol and 32 extra bits.
|
||||
if (this.block_length[2] == 0) or (args.src.length() < 8) {
|
||||
break.outer
|
||||
}
|
||||
bits |= args.src.peek_u64le() ~mod<< n_bits
|
||||
args.src.skip_u32_fast!(actual: (63 - n_bits) >> 3, worst_case: 8)
|
||||
n_bits |= 56
|
||||
// Mirrors read_distance.
|
||||
i = (DIST_HUFFS_BASE + (this.dist_htree_index * DIST_HUFFS_STRIDE)) ~mod+ ((bits & 0xFF) as base.u32)
|
||||
if i >= HUFFS_LEN {
|
||||
ret = "#internal error: inconsistent Huffman table"
|
||||
break.outer
|
||||
}
|
||||
e = this.huffs[i] as base.u32
|
||||
e_bits = e & 15
|
||||
if e_bits > 8 {
|
||||
i = ((DIST_HUFFS_BASE + (this.dist_htree_index * DIST_HUFFS_STRIDE)) ~mod+ (e >> 4)) ~mod+ ((((bits >> 8) & 0x7FFF) as base.u32) & (((1 as base.u32) << (e_bits - 8)) - 1))
|
||||
if i >= HUFFS_LEN {
|
||||
ret = "#internal error: inconsistent Huffman table"
|
||||
break.outer
|
||||
}
|
||||
e = this.huffs[i] as base.u32
|
||||
e_bits = (e & 15) + 8
|
||||
}
|
||||
if n_bits < e_bits {
|
||||
ret = "#internal error: inconsistent bit reader"
|
||||
break.outer
|
||||
}
|
||||
bits >>= e_bits
|
||||
n_bits -= e_bits
|
||||
this.symbol = e >> 4
|
||||
code = this.symbol
|
||||
this.block_length[2] ~mod-= 1
|
||||
this.distance_context = 0
|
||||
if code < 16 {
|
||||
this.take_distance_from_ring_buffer!(code: code)
|
||||
} else {
|
||||
if (code >= this.distance_alphabet_size) or (code >= 1128) {
|
||||
ret = "#internal error: inconsistent Huffman table"
|
||||
break.outer
|
||||
}
|
||||
nbits = this.dist_extra_bits[code].min(no_more_than: 32) as base.u32
|
||||
if n_bits < nbits {
|
||||
ret = "#internal error: inconsistent bit reader"
|
||||
break.outer
|
||||
}
|
||||
// Like read_bits32, this is (lo | (hi << 16)) when nbits
|
||||
// is more than 24.
|
||||
lo = ((bits & (((1 as base.u64) << nbits) - 1)) & 0xFFFF_FFFF) as base.u32
|
||||
this.bits_value = lo
|
||||
bits >>= nbits
|
||||
n_bits -= nbits
|
||||
this.distance_code = (((this.dist_offset[code] as base.u64) ~mod+
|
||||
((this.bits_value as base.u64) ~mod<< this.distance_postfix_bits)) & 0xFFFF_FFFF) as base.u32
|
||||
}
|
||||
}
|
||||
this.cmd_state = 3
|
||||
}
|
||||
|
||||
// this.cmd_state == 3. Only handle LZ77 copies that need no ring
|
||||
// buffer event and do not wrap around the physical ring buffer.
|
||||
max_distance = this.max_distance
|
||||
if max_distance <> this.max_backward_distance {
|
||||
max_distance = this.pos.min(no_more_than: this.max_backward_distance)
|
||||
}
|
||||
distance = this.distance_code
|
||||
n = this.copy_length
|
||||
if (distance > max_distance) or (distance == 0) {
|
||||
break.outer
|
||||
}
|
||||
dst_i = this.pos as base.u64
|
||||
src_i = ((this.pos ~mod- distance) & (this.window_size ~mod- 1)) as base.u64
|
||||
if ((dst_i + (n as base.u64)) >= (this.rb_size as base.u64)) or
|
||||
((src_i + (n as base.u64)) > (this.window_size as base.u64)) or
|
||||
((this.window_size as base.u64) > args.workbuf.length()) {
|
||||
break.outer
|
||||
}
|
||||
this.max_distance = max_distance
|
||||
this.dist_rb[this.dist_rb_idx & 3] = distance
|
||||
this.dist_rb_idx ~mod+= 1
|
||||
this.mb_remaining ~mod-= n
|
||||
// Copy in chunks of at most distance bytes, like process_commands.
|
||||
k64 = 0
|
||||
while k64 < (n as base.u64) {
|
||||
n64 = (n as base.u64) ~mod- k64
|
||||
n64 = n64.min(no_more_than: distance as base.u64)
|
||||
dlo = dst_i ~mod+ k64
|
||||
dhi = dlo ~mod+ n64
|
||||
slo = src_i ~mod+ k64
|
||||
shi = slo ~mod+ n64
|
||||
if (n64 == 0) or (dlo > dhi) or (dhi > args.workbuf.length()) or
|
||||
(slo > shi) or (shi > args.workbuf.length()) {
|
||||
break
|
||||
}
|
||||
args.workbuf[dlo .. dhi].copy_from_slice!(s: args.workbuf[slo .. shi])
|
||||
k64 ~mod+= n64
|
||||
}
|
||||
this.pos ~mod+= (k64 & 0xFFFF_FFFF) as base.u32
|
||||
n ~mod-= (k64 & 0xFFFF_FFFF) as base.u32
|
||||
// n == 0, unless the (unreachable) break above was taken. This.pos
|
||||
// stays below this.rb_size, so there is no ring buffer event.
|
||||
while n > 0 {
|
||||
this.put_byte!(workbuf: args.workbuf, b: this.get_byte(workbuf: args.workbuf, distance: distance))
|
||||
n -= 1
|
||||
}
|
||||
this.cmd_state = 0
|
||||
if (this.mb_remaining == 0) or (this.mb_remaining >= 0x8000_0000) {
|
||||
this.cmd_state = 4
|
||||
break.outer
|
||||
}
|
||||
}.outer
|
||||
|
||||
// Give back the whole bytes left in the bit buffer.
|
||||
while n_bits >= 8 {
|
||||
if not args.src.can_undo_byte() {
|
||||
return "#internal error: inconsistent bit reader"
|
||||
}
|
||||
args.src.undo_byte!()
|
||||
n_bits -= 8
|
||||
}
|
||||
this.bits = ((bits & (((1 as base.u64) << n_bits) - 1)) & 0xFFFF_FFFF) as base.u32
|
||||
this.n_bits = n_bits
|
||||
return ret
|
||||
}
|
||||
|
||||
// update_dist_htree_index sets this.dist_htree_index from the distance
|
||||
// context map. Mirrors part of ReadCommandInternal and
|
||||
// DecodeDistanceBlockSwitch in decode.c.
|
||||
@@ -631,10 +1023,15 @@ pri func decoder.update_dist_htree_index!() {
|
||||
|
||||
// Mirrors ReadDistanceInternal in decode.c.
|
||||
pri func decoder.read_distance?(src: base.io_reader) {
|
||||
var code : base.u32
|
||||
var code : base.u32
|
||||
var fr : base.u32
|
||||
var nbits : base.u32[..= 32]
|
||||
|
||||
this.read_symbol?(src: args.src,
|
||||
hoff: DIST_HUFFS_BASE + (this.dist_htree_index * DIST_HUFFS_STRIDE))
|
||||
fr = this.read_symbol_fast!(src: args.src, hoff: DIST_HUFFS_BASE + (this.dist_htree_index * DIST_HUFFS_STRIDE))
|
||||
if fr <> 0 {
|
||||
this.read_symbol?(src: args.src,
|
||||
hoff: DIST_HUFFS_BASE + (this.dist_htree_index * DIST_HUFFS_STRIDE))
|
||||
}
|
||||
code = this.symbol
|
||||
this.block_length[2] ~mod-= 1
|
||||
// Convert the distance code to the actual distance by possibly looking up
|
||||
@@ -649,7 +1046,14 @@ pri func decoder.read_distance?(src: base.io_reader) {
|
||||
}
|
||||
// The C code uses BrotliSafeReadBits32, as large-window brotli can have
|
||||
// more than 24 extra bits.
|
||||
this.read_bits32?(src: args.src, n: this.dist_extra_bits[code].min(no_more_than: 32) as base.u32)
|
||||
nbits = this.dist_extra_bits[code].min(no_more_than: 32) as base.u32
|
||||
fr = 1
|
||||
if nbits <= 24 {
|
||||
fr = this.read_bits_fast!(src: args.src, n: nbits)
|
||||
}
|
||||
if fr <> 0 {
|
||||
this.read_bits32?(src: args.src, n: nbits)
|
||||
}
|
||||
if code >= 1128 {
|
||||
return "#internal error: inconsistent Huffman table"
|
||||
}
|
||||
|
||||
@@ -12,8 +12,8 @@
|
||||
|
||||
// This file mirrors bit_cost.c, bit_cost_inc.h and fast_log.h. The functions
|
||||
// are methods of the bit_cost struct (whose only state is the stateless
|
||||
// softfloat arith), so that other structs (the encoder, the
|
||||
// metablock_builder) can embed one with a single field.
|
||||
// softfloat arith and a lazily filled FastLog2 cache), so that other structs
|
||||
// (the encoder, the metablock_builder) can embed one with a single field.
|
||||
//
|
||||
// Doubles are IEEE 754 binary64 bit patterns (base.u64). A uint32_t array
|
||||
// (e.g. a histogram's data_) is a slice of little-endian bytes, because
|
||||
@@ -48,43 +48,23 @@ pri const F64_HUGE_VAL : base.u64 = 0x7FF0_0000_0000_0000
|
||||
// F64_1E99 is the C code's 1e99.
|
||||
pri const F64_1E99 : base.u64 = 0x547D_42AE_A287_9F2E
|
||||
|
||||
// LOG2_CACHE_SIZE is the size of the bit_cost.log2_cache table.
|
||||
pri const LOG2_CACHE_SIZE : base.u64 = 65536
|
||||
|
||||
pri struct bit_cost?(
|
||||
// log2_cache_ready is whether log2_cache has been filled (by
|
||||
// init_log2_cache).
|
||||
log2_cache_ready : base.bool,
|
||||
// Bit (v & 63) of log2_valid[v >> 6] is whether log2_cache[v] holds
|
||||
// FastLog2(v). This bitmap is zeroed when the struct is initialized
|
||||
// (but log2_cache need not be), so the cache starts empty.
|
||||
log2_valid : array[1024] base.u64,
|
||||
|
||||
sf : softfloat.arith,
|
||||
) + (
|
||||
// log2_cache[v] is FastLog2(v), for v < LOG2_CACHE_SIZE. It is only a
|
||||
// speed optimization (glibc's log2 is expensive to emulate): the
|
||||
// values are computed by the same functions, so the results are
|
||||
// identical with or without it.
|
||||
// log2_cache[v] is FastLog2(v), for 256 <= v < 65536, filled lazily
|
||||
// (see fast_log2_memo). It is only a speed optimization (glibc's log2
|
||||
// is expensive to emulate): the values are computed by the same
|
||||
// functions, so the results are identical with or without it. Few of
|
||||
// the 65536 entries are typically needed, so filling it lazily is
|
||||
// much cheaper than filling it all up front.
|
||||
log2_cache : array[65536] base.u64,
|
||||
)
|
||||
|
||||
// init_log2_cache fills the log2_cache table (if it is not already filled).
|
||||
// Calling it is optional, but it makes fast_log2 faster.
|
||||
pri func bit_cost.init_log2_cache!() {
|
||||
var v : base.u64
|
||||
|
||||
if this.log2_cache_ready {
|
||||
return nothing
|
||||
}
|
||||
v = 0
|
||||
while v < 256 {
|
||||
this.log2_cache[v] = LOG2_TABLE[v]
|
||||
v += 1
|
||||
}
|
||||
while v < 65536 {
|
||||
this.log2_cache[v] = this.sf.log2_u64(v: v)
|
||||
v += 1
|
||||
}
|
||||
this.log2_cache_ready = true
|
||||
}
|
||||
|
||||
// ld32 returns the u32le at s[i ..], or 0 if that is out of bounds (which
|
||||
// the callers never do).
|
||||
pri func bit_cost.ld32(s: roslice base.u8, i: base.u64) base.u32 {
|
||||
@@ -138,23 +118,41 @@ pri func bit_cost.st64!(s: slice base.u8, i: base.u64, v: base.u64) {
|
||||
}
|
||||
|
||||
// Mirrors FastLog2 in fast_log.h. The result is the bit pattern of an IEEE
|
||||
// 754 binary64 value.
|
||||
// 754 binary64 value. It uses (but does not fill) log2_cache: see
|
||||
// fast_log2_memo.
|
||||
pri func bit_cost.fast_log2(v: base.u64) base.u64 {
|
||||
if args.v < 256 {
|
||||
return LOG2_TABLE[args.v]
|
||||
} else if this.log2_cache_ready and (args.v < 65536) {
|
||||
return this.log2_cache[args.v]
|
||||
} else if args.v < 65536 {
|
||||
if ((this.log2_valid[args.v >> 6] >> (args.v & 63)) & 1) <> 0 {
|
||||
return this.log2_cache[args.v]
|
||||
}
|
||||
}
|
||||
return this.sf.log2_u64(v: args.v)
|
||||
}
|
||||
|
||||
// fast_log2_memo is like fast_log2 (it also mirrors FastLog2) but it fills
|
||||
// the log2_cache entry for v (if v is in the cache's range).
|
||||
pri func bit_cost.fast_log2_memo!(v: base.u64) base.u64 {
|
||||
if args.v < 256 {
|
||||
return LOG2_TABLE[args.v]
|
||||
} else if args.v >= 65536 {
|
||||
return this.sf.log2_u64(v: args.v)
|
||||
} else if ((this.log2_valid[args.v >> 6] >> (args.v & 63)) & 1) == 0 {
|
||||
this.log2_cache[args.v] = this.sf.log2_u64(v: args.v)
|
||||
this.log2_valid[args.v >> 6] |= (1 as base.u64) << (args.v & 63)
|
||||
}
|
||||
return this.log2_cache[args.v]
|
||||
}
|
||||
|
||||
// Mirrors BrotliBitsEntropy in bit_cost.c. population holds (at least) size
|
||||
// u32le values. The C code's "goto odd_number_of_elements_left" only changes
|
||||
// the loop's unrolling, not the order of the operations.
|
||||
pri func bit_cost.bits_entropy(population: roslice base.u8, size: base.u64) base.u64 {
|
||||
pri func bit_cost.bits_entropy!(population: roslice base.u8, size: base.u64) base.u64 {
|
||||
var sum : base.u64
|
||||
var retval : base.u64
|
||||
var p : base.u64
|
||||
var lp : base.u64
|
||||
var q : roslice base.u8
|
||||
var n : base.u64
|
||||
|
||||
@@ -168,12 +166,14 @@ pri func bit_cost.bits_entropy(population: roslice base.u8, size: base.u64) base
|
||||
q = q[4 ..]
|
||||
}
|
||||
sum ~mod+= p
|
||||
lp = this.fast_log2_memo!(v: p)
|
||||
retval = this.sf.f64_sub(a: retval, b: this.sf.f64_mul(
|
||||
a: this.sf.f64_from_u64(a: p), b: this.fast_log2(v: p)))
|
||||
a: this.sf.f64_from_u64(a: p), b: lp))
|
||||
}
|
||||
if sum <> 0 {
|
||||
lp = this.fast_log2_memo!(v: sum)
|
||||
retval = this.sf.f64_add(a: retval, b: this.sf.f64_mul(
|
||||
a: this.sf.f64_from_u64(a: sum), b: this.fast_log2(v: sum)))
|
||||
a: this.sf.f64_from_u64(a: sum), b: lp))
|
||||
}
|
||||
if this.sf.f64_lt(a: retval, b: this.sf.f64_from_u64(a: sum)) <> 0 {
|
||||
retval = this.sf.f64_from_u64(a: sum)
|
||||
@@ -184,7 +184,7 @@ pri func bit_cost.bits_entropy(population: roslice base.u8, size: base.u64) base
|
||||
// Mirrors BrotliPopulationCostLiteral, BrotliPopulationCostCommand and
|
||||
// BrotliPopulationCostDistance (FN(BrotliPopulationCost) in bit_cost_inc.h).
|
||||
// histogram is a histogram of data_size symbols (see the layout above).
|
||||
pri func bit_cost.population_cost(histogram: roslice base.u8, data_size: base.u32[..= 704]) base.u64 {
|
||||
pri func bit_cost.population_cost!(histogram: roslice base.u8, data_size: base.u32[..= 704]) base.u64 {
|
||||
var total : base.u64
|
||||
var count : base.u32
|
||||
var s : array[4] base.u32
|
||||
@@ -203,6 +203,7 @@ pri func bit_cost.population_cost(histogram: roslice base.u8, data_size: base.u3
|
||||
var depth_histo : array[18] base.u32
|
||||
var log2total : base.u64
|
||||
var log2p : base.u64
|
||||
var lp : base.u64
|
||||
var depth : base.u64
|
||||
var reps : base.u32
|
||||
var sum : base.u64
|
||||
@@ -284,12 +285,13 @@ pri func bit_cost.population_cost(histogram: roslice base.u8, data_size: base.u3
|
||||
// Compute the entropy of the histogram and simultaneously build a
|
||||
// simplified histogram of the code length codes.
|
||||
max_depth = 1
|
||||
log2total = this.fast_log2(v: total)
|
||||
log2total = this.fast_log2_memo!(v: total)
|
||||
i = 0
|
||||
while i < args.data_size {
|
||||
c = this.ld32(s: args.histogram, i: (i as base.u64) * 4)
|
||||
if c > 0 {
|
||||
log2p = this.sf.f64_sub(a: log2total, b: this.fast_log2(v: c as base.u64))
|
||||
lp = this.fast_log2_memo!(v: c as base.u64)
|
||||
log2p = this.sf.f64_sub(a: log2total, b: lp)
|
||||
depth = this.sf.f64_to_u64_trunc(a: this.sf.f64_add(a: log2p, b: 0x3FE0_0000_0000_0000))
|
||||
bits = this.sf.f64_add(a: bits, b: this.sf.f64_mul(
|
||||
a: this.sf.f64_from_u32(a: c), b: log2p))
|
||||
@@ -340,13 +342,15 @@ pri func bit_cost.population_cost(histogram: roslice base.u8, data_size: base.u3
|
||||
while i < 18 {
|
||||
p = depth_histo[i] as base.u64
|
||||
sum ~mod+= p
|
||||
lp = this.fast_log2_memo!(v: p)
|
||||
retval = this.sf.f64_sub(a: retval, b: this.sf.f64_mul(
|
||||
a: this.sf.f64_from_u64(a: p), b: this.fast_log2(v: p)))
|
||||
a: this.sf.f64_from_u64(a: p), b: lp))
|
||||
i ~mod+= 1
|
||||
}
|
||||
if sum <> 0 {
|
||||
lp = this.fast_log2_memo!(v: sum)
|
||||
retval = this.sf.f64_add(a: retval, b: this.sf.f64_mul(
|
||||
a: this.sf.f64_from_u64(a: sum), b: this.fast_log2(v: sum)))
|
||||
a: this.sf.f64_from_u64(a: sum), b: lp))
|
||||
}
|
||||
if this.sf.f64_lt(a: retval, b: this.sf.f64_from_u64(a: sum)) <> 0 {
|
||||
retval = this.sf.f64_from_u64(a: sum)
|
||||
|
||||
@@ -31,12 +31,15 @@ pri const HISTOGRAMS_PER_BATCH : base.u64 = 64
|
||||
pri const CLUSTERS_PER_BATCH : base.u64 = 16
|
||||
|
||||
// Mirrors BitCost in block_splitter.c.
|
||||
pri func metablock_builder.bit_cost(count: base.u64) base.u64 {
|
||||
pri func metablock_builder.bit_cost!(count: base.u64) base.u64 {
|
||||
var c : base.u64
|
||||
|
||||
if args.count == 0 {
|
||||
// -2.0
|
||||
return 0xC000_0000_0000_0000
|
||||
}
|
||||
return this.bc.fast_log2(v: args.count)
|
||||
c = this.bc.fast_log2_memo!(v: args.count)
|
||||
return c
|
||||
}
|
||||
|
||||
// Mirrors FN(HistogramAddVector) in histogram_inc.h, adding the n symbols at
|
||||
@@ -314,7 +317,7 @@ pri func metablock_builder.find_blocks!(wb: slice base.u8, data: base.u64, lengt
|
||||
i = 0
|
||||
while i < args.num_histograms {
|
||||
// FastLog2((uint32_t)histograms[i].total_count_).
|
||||
this.insert_cost[i & 255] = this.bc.fast_log2(
|
||||
this.insert_cost[i & 255] = this.bc.fast_log2_memo!(
|
||||
v: this.h_total(wb: args.wb, h: args.histograms ~mod+ (i ~mod* hs), ds: args.ds) & 0xFFFF_FFFF)
|
||||
i ~mod+= 1
|
||||
}
|
||||
@@ -326,9 +329,8 @@ pri func metablock_builder.find_blocks!(wb: slice base.u8, data: base.u64, lengt
|
||||
while j < args.num_histograms {
|
||||
k = (i ~mod* args.num_histograms) ~mod+ j
|
||||
if k < 35200 {
|
||||
this.insert_cost[k] = this.sf.f64_sub(
|
||||
a: this.insert_cost[j & 255],
|
||||
b: this.bit_cost(count: this.ld32(wb: args.wb, i: (args.histograms ~mod+ (j ~mod* hs)) ~mod+ (i ~mod* 4)) as base.u64))
|
||||
c = this.bit_cost!(count: this.ld32(wb: args.wb, i: (args.histograms ~mod+ (j ~mod* hs)) ~mod+ (i ~mod* 4)) as base.u64)
|
||||
this.insert_cost[k] = this.sf.f64_sub(a: this.insert_cost[j & 255], b: c)
|
||||
}
|
||||
j ~mod+= 1
|
||||
}
|
||||
@@ -471,6 +473,7 @@ pri func metablock_builder.build_block_histograms!(wb: slice base.u8, data: base
|
||||
// this.split_etc[cat].
|
||||
pri func metablock_builder.cluster_blocks!(wb: slice base.u8, cat: base.u32[..= 2], data: base.u64, length: base.u64, width: base.u32,
|
||||
num_blocks: base.u64, block_ids: base.u64, ds: base.u32[..= 704]) {
|
||||
var c : base.u64
|
||||
var hs : base.u64
|
||||
var histogram_symbols : base.u64
|
||||
var u32s : base.u64
|
||||
@@ -562,8 +565,8 @@ pri func metablock_builder.cluster_blocks!(wb: slice base.u8, cat: base.u32[..=
|
||||
pos ~mod+= 1
|
||||
k ~mod+= 1
|
||||
}
|
||||
this.h_set_bit_cost!(wb: args.wb, h: histograms ~mod+ (j ~mod* hs), ds: args.ds,
|
||||
bit_cost: this.h_population_cost(wb: args.wb, h: histograms ~mod+ (j ~mod* hs), ds: args.ds))
|
||||
c = this.h_population_cost!(wb: args.wb, h: histograms ~mod+ (j ~mod* hs), ds: args.ds)
|
||||
this.h_set_bit_cost!(wb: args.wb, h: histograms ~mod+ (j ~mod* hs), ds: args.ds, bit_cost: c)
|
||||
this.st32!(wb: args.wb, i: new_clusters ~mod+ (j ~mod* 4), v: (j & 0xFFFF_FFFF) as base.u32)
|
||||
this.st32!(wb: args.wb, i: symbols ~mod+ (j ~mod* 4), v: (j & 0xFFFF_FFFF) as base.u32)
|
||||
this.st32!(wb: args.wb, i: sizes ~mod+ (j ~mod* 4), v: 1)
|
||||
|
||||
@@ -108,7 +108,7 @@ pri func metablock_builder.compare_and_push_to_queue!(wb: slice base.u8, ds: bas
|
||||
}
|
||||
this.h_copy!(wb: args.wb, h: args.tmp, v: h1, ds: args.ds)
|
||||
this.h_add_histogram!(wb: args.wb, h: args.tmp, v: h2, ds: args.ds)
|
||||
c = this.h_population_cost(wb: args.wb, h: args.tmp, ds: args.ds)
|
||||
c = this.h_population_cost!(wb: args.wb, h: args.tmp, ds: args.ds)
|
||||
if this.sf.f64_lt(a: c, b: this.sf.f64_sub(a: threshold, b: cost_diff)) <> 0 {
|
||||
cost_combo = c
|
||||
is_good_pair = true
|
||||
@@ -271,14 +271,15 @@ pri func metablock_builder.histogram_combine!(wb: slice base.u8, ds: base.u32[..
|
||||
// Mirrors FN(BrotliHistogramBitCostDistance) in cluster_inc.h.
|
||||
pri func metablock_builder.histogram_bit_cost_distance!(wb: slice base.u8, ds: base.u32[..= 704],
|
||||
histogram: base.u64, candidate: base.u64, tmp: base.u64) base.u64 {
|
||||
var c : base.u64
|
||||
|
||||
if this.h_total(wb: args.wb, h: args.histogram, ds: args.ds) == 0 {
|
||||
return 0
|
||||
}
|
||||
this.h_copy!(wb: args.wb, h: args.tmp, v: args.histogram, ds: args.ds)
|
||||
this.h_add_histogram!(wb: args.wb, h: args.tmp, v: args.candidate, ds: args.ds)
|
||||
return this.sf.f64_sub(
|
||||
a: this.h_population_cost(wb: args.wb, h: args.tmp, ds: args.ds),
|
||||
b: this.h_bit_cost(wb: args.wb, h: args.candidate, ds: args.ds))
|
||||
c = this.h_population_cost!(wb: args.wb, h: args.tmp, ds: args.ds)
|
||||
return this.sf.f64_sub(a: c, b: this.h_bit_cost(wb: args.wb, h: args.candidate, ds: args.ds))
|
||||
}
|
||||
|
||||
// Mirrors FN(BrotliHistogramRemap) in cluster_inc.h.
|
||||
@@ -395,6 +396,7 @@ pri func metablock_builder.histogram_reindex!(wb: slice base.u8, ds: base.u32[..
|
||||
// histogram_symbols. It returns the C code's *out_size.
|
||||
pri func metablock_builder.cluster_histograms!(wb: slice base.u8, ds: base.u32[..= 704], in_histos: base.u64, in_size: base.u64,
|
||||
max_histograms: base.u64, out_histos: base.u64, histogram_symbols: base.u64) base.u64 {
|
||||
var c : base.u64
|
||||
var hs : base.u64
|
||||
var mark : base.u64
|
||||
var cluster_size : base.u64
|
||||
@@ -426,8 +428,8 @@ pri func metablock_builder.cluster_histograms!(wb: slice base.u8, ds: base.u32[.
|
||||
i = 0
|
||||
while i < args.in_size {
|
||||
this.h_copy!(wb: args.wb, h: args.out_histos ~mod+ (i ~mod* hs), v: args.in_histos ~mod+ (i ~mod* hs), ds: args.ds)
|
||||
this.h_set_bit_cost!(wb: args.wb, h: args.out_histos ~mod+ (i ~mod* hs), ds: args.ds,
|
||||
bit_cost: this.h_population_cost(wb: args.wb, h: args.in_histos ~mod+ (i ~mod* hs), ds: args.ds))
|
||||
c = this.h_population_cost!(wb: args.wb, h: args.in_histos ~mod+ (i ~mod* hs), ds: args.ds)
|
||||
this.h_set_bit_cost!(wb: args.wb, h: args.out_histos ~mod+ (i ~mod* hs), ds: args.ds, bit_cost: c)
|
||||
this.st32!(wb: args.wb, i: args.histogram_symbols ~mod+ (i ~mod* 4), v: (i & 0xFFFF_FFFF) as base.u32)
|
||||
i ~mod+= 1
|
||||
}
|
||||
|
||||
@@ -277,7 +277,8 @@ pri func encoder.estimate_bit_costs_for_literals_utf8!(wb: slice base.u8, pos: b
|
||||
} else if utf8_pos == 2 {
|
||||
iw = iwu2
|
||||
}
|
||||
lit_cost = this.sf.f64_sub(a: this.bc.fast_log2(v: iw), b: this.bc.fast_log2(v: histo))
|
||||
lit_cost = this.bc.fast_log2_memo!(v: iw)
|
||||
lit_cost = this.sf.f64_sub(a: lit_cost, b: this.bc.fast_log2(v: histo))
|
||||
lit_cost = this.sf.f64_add(a: lit_cost, b: LC_F64_0_02905)
|
||||
this.lc_finish_cost!(wb: args.wb, cost_off: args.cost_off, i: i, lit_cost: lit_cost, utf8: true)
|
||||
i ~mod+= 1
|
||||
@@ -337,7 +338,8 @@ pri func encoder.estimate_bit_costs_for_literals!(wb: slice base.u8, pos: base.u
|
||||
if histo == 0 {
|
||||
histo = 1
|
||||
}
|
||||
lit_cost = this.sf.f64_sub(a: this.bc.fast_log2(v: in_window), b: this.bc.fast_log2(v: histo))
|
||||
lit_cost = this.bc.fast_log2_memo!(v: in_window)
|
||||
lit_cost = this.sf.f64_sub(a: lit_cost, b: this.bc.fast_log2(v: histo))
|
||||
lit_cost = this.sf.f64_add(a: lit_cost, b: LC_F64_0_029)
|
||||
this.lc_finish_cost!(wb: args.wb, cost_off: args.cost_off, i: i, lit_cost: lit_cost, utf8: false)
|
||||
i ~mod+= 1
|
||||
|
||||
@@ -264,9 +264,12 @@ pri func metablock_builder.h_set_bit_cost!(wb: slice base.u8, h: base.u64, ds: b
|
||||
this.st64!(wb: args.wb, i: args.h ~mod+ (((args.ds as base.u64) * 4) + 8), v: args.bit_cost)
|
||||
}
|
||||
|
||||
pri func metablock_builder.h_population_cost(wb: roslice base.u8, h: base.u64, ds: base.u32[..= 704]) base.u64 {
|
||||
pri func metablock_builder.h_population_cost!(wb: roslice base.u8, h: base.u64, ds: base.u32[..= 704]) base.u64 {
|
||||
var c : base.u64
|
||||
|
||||
if args.h <= args.wb.length() {
|
||||
return this.bc.population_cost(histogram: args.wb[args.h ..], data_size: args.ds)
|
||||
c = this.bc.population_cost!(histogram: args.wb[args.h ..], data_size: args.ds)
|
||||
return c
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -482,9 +485,8 @@ pri func metablock_builder.compute_distance_cost!(wb: slice base.u8, cmds: rosli
|
||||
}
|
||||
i ~mod+= 1
|
||||
}
|
||||
this.dist_cost = this.sf.f64_add(
|
||||
a: this.h_population_cost(wb: args.wb, h: args.tmp, ds: HISTOGRAM_DISTANCE_DATA_SIZE),
|
||||
b: extra_bits)
|
||||
this.dist_cost = this.h_population_cost!(wb: args.wb, h: args.tmp, ds: HISTOGRAM_DISTANCE_DATA_SIZE)
|
||||
this.dist_cost = this.sf.f64_add(a: this.dist_cost, b: extra_bits)
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -592,7 +594,6 @@ pri func metablock_builder.build_meta_block!(wb: slice base.u8, ringbuffer: rosl
|
||||
var v : base.u32
|
||||
|
||||
this.reset_arena!(wb_len: args.wb.length())
|
||||
this.bc.init_log2_cache!()
|
||||
|
||||
orig_npostfix = this.dist_postfix_bits
|
||||
orig_ndirect = this.dist_num_direct_codes
|
||||
|
||||
@@ -47,9 +47,12 @@ pri func encoder.gbs_histo_off(c: base.u32[..= 3], ix: base.u64) base.u64 {
|
||||
|
||||
// gbs_entropy returns BrotliBitsEntropy of the size u32le values at
|
||||
// args.wb[off ..].
|
||||
pri func encoder.gbs_entropy(wb: roslice base.u8, off: base.u64, size: base.u64) base.u64 {
|
||||
pri func encoder.gbs_entropy!(wb: roslice base.u8, off: base.u64, size: base.u64) base.u64 {
|
||||
var e : base.u64
|
||||
|
||||
if args.off <= args.wb.length() {
|
||||
return this.bc.bits_entropy(population: args.wb[args.off ..], size: args.size)
|
||||
e = this.bc.bits_entropy!(population: args.wb[args.off ..], size: args.size)
|
||||
return e
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -162,7 +165,7 @@ pri func encoder.gbs_finish_block!(wb: slice base.u8, c: base.u32[..= 3], is_fin
|
||||
this.set_byte!(s: args.wb, i: types_off, v: 0)
|
||||
i = 0
|
||||
while i < nc {
|
||||
e = this.gbs_entropy(wb: args.wb, off: this.gbs_histo_off(c: args.c, ix: i), size: alphabet_size)
|
||||
e = this.gbs_entropy!(wb: args.wb, off: this.gbs_histo_off(c: args.c, ix: i), size: alphabet_size)
|
||||
this.gbs_last_entropy[((le_base as base.u64) ~mod+ i) & 127] = e
|
||||
this.gbs_last_entropy[(((le_base as base.u64) ~mod+ nc) ~mod+ i) & 127] = e
|
||||
i ~mod+= 1
|
||||
@@ -185,7 +188,7 @@ pri func encoder.gbs_finish_block!(wb: slice base.u8, c: base.u32[..= 3], is_fin
|
||||
last1 = this.gbs_last_histogram_ix[(args.c * 2) + 1]
|
||||
i = 0
|
||||
while i < nc {
|
||||
e = this.gbs_entropy(wb: args.wb, off: this.gbs_histo_off(c: args.c, ix: curr ~mod+ i), size: alphabet_size)
|
||||
e = this.gbs_entropy!(wb: args.wb, off: this.gbs_histo_off(c: args.c, ix: curr ~mod+ i), size: alphabet_size)
|
||||
this.gbs_entropy_tmp[i & 15] = e
|
||||
j = 0
|
||||
while j < 2 {
|
||||
@@ -197,7 +200,7 @@ pri func encoder.gbs_finish_block!(wb: slice base.u8, c: base.u32[..= 3], is_fin
|
||||
} else {
|
||||
this.gbs_add_histogram!(wb: args.wb, dst: tmp, src: this.gbs_histo_off(c: args.c, ix: last1 ~mod+ i), n: data_size)
|
||||
}
|
||||
this.gbs_comb_entropy[jx & 31] = this.gbs_entropy(wb: args.wb, off: tmp, size: alphabet_size)
|
||||
this.gbs_comb_entropy[jx & 31] = this.gbs_entropy!(wb: args.wb, off: tmp, size: alphabet_size)
|
||||
d = this.sf.f64_sub(
|
||||
a: this.sf.f64_sub(a: this.gbs_comb_entropy[jx & 31], b: e),
|
||||
b: this.gbs_last_entropy[((le_base as base.u64) ~mod+ jx) & 127])
|
||||
|
||||
@@ -29,8 +29,11 @@ pub struct metablock_tester?(
|
||||
)
|
||||
|
||||
// bits_entropy mirrors BrotliBitsEntropy. population holds size u32le values.
|
||||
pub func metablock_tester.bits_entropy(population: roslice base.u8, size: base.u64) base.u64 {
|
||||
return this.bc.bits_entropy(population: args.population, size: args.size)
|
||||
pub func metablock_tester.bits_entropy!(population: roslice base.u8, size: base.u64) base.u64 {
|
||||
var c : base.u64
|
||||
|
||||
c = this.bc.bits_entropy!(population: args.population, size: args.size)
|
||||
return c
|
||||
}
|
||||
|
||||
// population_cost mirrors BrotliPopulationCostLiteral (data_size 256),
|
||||
@@ -38,13 +41,18 @@ pub func metablock_tester.bits_entropy(population: roslice base.u8, size: base.u
|
||||
// histogram has the C struct's layout: data_size u32le counts, then the
|
||||
// total_count_ u64le (and then the bit_cost_, which is ignored). It returns 0
|
||||
// for any other data_size.
|
||||
pub func metablock_tester.population_cost(histogram: roslice base.u8, data_size: base.u32) base.u64 {
|
||||
pub func metablock_tester.population_cost!(histogram: roslice base.u8, data_size: base.u32) base.u64 {
|
||||
var c : base.u64
|
||||
|
||||
if args.data_size == 256 {
|
||||
return this.bc.population_cost(histogram: args.histogram, data_size: 256)
|
||||
c = this.bc.population_cost!(histogram: args.histogram, data_size: 256)
|
||||
return c
|
||||
} else if args.data_size == 704 {
|
||||
return this.bc.population_cost(histogram: args.histogram, data_size: 704)
|
||||
c = this.bc.population_cost!(histogram: args.histogram, data_size: 704)
|
||||
return c
|
||||
} else if args.data_size == 544 {
|
||||
return this.bc.population_cost(histogram: args.histogram, data_size: 544)
|
||||
c = this.bc.population_cost!(histogram: args.histogram, data_size: 544)
|
||||
return c
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -62,7 +70,6 @@ pub func metablock_tester.split_block!(cmds: roslice base.u8, num_commands: base
|
||||
data: roslice base.u8, pos: base.u64, mask: base.u64, quality: base.u32, workbuf: slice base.u8) base.status {
|
||||
this.mb.set_params!(quality: args.quality, large_window: false, disable_literal_context_modeling: false)
|
||||
this.mb.reset_arena!(wb_len: args.workbuf.length())
|
||||
this.mb.bc.init_log2_cache!()
|
||||
this.mb.alloc_block_splits!(cmds: args.cmds, num_commands: args.num_commands)
|
||||
if this.mb.oom {
|
||||
return base."#bad workbuf length"
|
||||
@@ -102,7 +109,6 @@ pub func metablock_tester.cluster_histograms!(data_size: base.u32, in_histos: ro
|
||||
return 0xFFFF_FFFF_FFFF_FFFF
|
||||
}
|
||||
hs = this.mb.hsize(ds: ds)
|
||||
this.mb.bc.init_log2_cache!()
|
||||
this.mb.reset_arena!(wb_len: args.workbuf.length())
|
||||
n = args.in_size ~mod* hs
|
||||
in_off = this.mb.alloc!(n: n)
|
||||
|
||||
+61
-6
@@ -96,12 +96,32 @@ integer long division, 11 quotient bits per (64-bit integer) division step,
|
||||
with the final remainder as the sticky bit. Leading zero counts use a 256-entry
|
||||
table.
|
||||
|
||||
The `f32` add, sub, mul and div operations convert their operands exactly to
|
||||
`f64`, perform the `f64` operation and then round to `f32`. This double
|
||||
rounding is innocuous: binary64's 53 bits of precision are more than `(2 * 24)
|
||||
+ 2`, which suffices for `+`, `-`, `*` and `/` (S. A. Figueroa, "When is double
|
||||
rounding innocuous?", ACM SIGNUM Newsletter, 1995). The tests confirm this
|
||||
against hardware, including for subnormal `f32` results.
|
||||
`f64_add` and `f64_sub` have a fast path for the common case where both
|
||||
operands are normal (`f64_add_normal`): it orders the operands by magnitude,
|
||||
aligns the significands with a sticky bit, adds or subtracts them and rounds
|
||||
inline. Only a result that is subnormal falls back to the general code
|
||||
(`f64_add_mags`, `f64_sub_mags`). Adding (or subtracting) a zero and a finite
|
||||
non-zero value also returns early.
|
||||
|
||||
Likewise, `f32_add`, `f32_sub` and `f32_mul` compute the result directly in
|
||||
binary32 (with 32-bit significands for add and sub and an exact 48-bit
|
||||
product for mul) when both operands are normal and the result is normal (or,
|
||||
for add and mul, overflows to infinity), and add and sub return early for a
|
||||
zero plus a finite non-zero value. `f32_from_u32` converts integers below
|
||||
`1 << 24` (which are exact) directly. Otherwise (zeroes, subnormal
|
||||
operands or results, infinities and NaNs), they, and `f32_div` always,
|
||||
convert their operands exactly to `f64`, perform the `f64` operation and then
|
||||
round to `f32`. This double rounding is innocuous: binary64's 53 bits of
|
||||
precision are more than `(2 * 24) + 2`, which suffices for `+`, `-`, `*` and
|
||||
`/` (S. A. Figueroa, "When is double rounding innocuous?", ACM SIGNUM
|
||||
Newsletter, 1995). The tests confirm this against hardware, including for
|
||||
subnormal `f32` results. The fast paths never produce a NaN, so NaN results
|
||||
(and their payloads) are exactly those of the "via `f64`" computation.
|
||||
|
||||
The comparison functions (`f64_cmp`, `f64_lt`, `f64_min`, etc. and their `f32`
|
||||
counterparts) compare the bit patterns directly when neither operand is
|
||||
negative or a NaN (when `a | b` is at most the bit pattern of `+inf`), and
|
||||
otherwise call `f64_cmp_slow` or `f32_cmp_slow`.
|
||||
|
||||
|
||||
## log2
|
||||
@@ -187,3 +207,38 @@ threads):
|
||||
(6 minutes 20 seconds wall clock in total)
|
||||
|
||||
No exception table is needed.
|
||||
|
||||
`script/manual-test-softfloat-sweep.c` systematically checks the `f32` add,
|
||||
sub, mul and div operations over every pair of operand exponents (`[0 ..=
|
||||
255]` squared), both signs and 40 x 40 mantissa patterns (exact results, ties,
|
||||
carries, cancellation, subnormal results, overflow, NaN payloads): each result
|
||||
must equal the hardware's (any NaN matching any NaN) and, bit-for-bit
|
||||
including NaNs, the "via `f64`" composition `f64_to_f32(f64_op(f32_to_f64(a),
|
||||
f32_to_f64(b)))`. It does the same for the `f64` add, sub, mul and div
|
||||
operations (every pair of exponents in `[0 ..= 2047]`, 9 x 9 mantissa
|
||||
patterns, NaN results checked bit-for-bit against the NaN rule above), and
|
||||
checks `f64_to_f32` (326 exponents times 2²⁰ fraction patterns times both
|
||||
signs), `f32_to_f64` (every `f32`), `f32_from_u32` and `f32_from_i32` (every
|
||||
32-bit integer) and `f32_from_u64` (2³² structured values). It also checks
|
||||
the `f32` and `f64` cmp, eq, ne, lt, le, gt, ge, min and max functions on
|
||||
every pair of 594 values (both signs, zero, subnormal, normal and maximal
|
||||
exponents, boundary mantissas and their neighbors, infinities and NaNs)
|
||||
against the hardware's comparison operators. Result (x86-64, gcc 13 `-O2`, 4
|
||||
threads):
|
||||
|
||||
f32_add 419430400 operand pairs, 0 mismatches
|
||||
f32_sub 419430400 operand pairs, 0 mismatches
|
||||
f32_mul 419430400 operand pairs, 0 mismatches
|
||||
f32_div 419430400 operand pairs, 0 mismatches
|
||||
f64_add 1358954496 operand pairs, 0 mismatches
|
||||
f64_sub 1358954496 operand pairs, 0 mismatches
|
||||
f64_mul 1358954496 operand pairs, 0 mismatches
|
||||
f64_div 1358954496 operand pairs, 0 mismatches
|
||||
f64_to_f32 341835776 operands, 0 mismatches
|
||||
f32_to_f64 4294967296 operands, 0 mismatches
|
||||
f32_from_u32 4294967296 operands, 0 mismatches
|
||||
f32_from_i32 4294967296 operands, 0 mismatches
|
||||
f32_from_u64 4294967296 operands, 0 mismatches
|
||||
f32_compare 352836 operand pairs, 0 mismatches
|
||||
f64_compare 352836 operand pairs, 0 mismatches
|
||||
PASS: 0 mismatches in total
|
||||
@@ -61,6 +61,24 @@ pri func arith.clz64(a: base.u64) base.u32[..= 63] {
|
||||
return (n32 + n16 + n8 + ((CLZ8[x >> 56] as base.u32) & 7)) & 63
|
||||
}
|
||||
|
||||
// clz32 returns the number of leading zero bits of a, which must be non-zero.
|
||||
pri func arith.clz32(a: base.u32) base.u32[..= 31] {
|
||||
var x : base.u32
|
||||
var n16 : base.u32[..= 16]
|
||||
var n8 : base.u32[..= 8]
|
||||
|
||||
x = args.a
|
||||
if (x >> 16) == 0 {
|
||||
x = x ~mod<< 16
|
||||
n16 = 16
|
||||
}
|
||||
if (x >> 24) == 0 {
|
||||
x = x ~mod<< 8
|
||||
n8 = 8
|
||||
}
|
||||
return (n16 + n8 + ((CLZ8[x >> 24] as base.u32) & 7)) & 31
|
||||
}
|
||||
|
||||
// shift_right_jam64 returns (a >> dist), except that if any non-zero bits
|
||||
// were shifted out then the least significant bit of the result is set. This
|
||||
// "sticky bit" is needed to round correctly.
|
||||
@@ -247,6 +265,21 @@ pub func arith.f64_abs(a: base.u64) base.u64 {
|
||||
// f64_cmp compares a and b, returning 0 if (a < b), 1 if (a == b), 2 if (a
|
||||
// > b) and 3 if they are unordered (either is a NaN). -0 and +0 compare equal.
|
||||
pub func arith.f64_cmp(a: base.u64, b: base.u64) base.u32 {
|
||||
if (args.a | args.b) <= F64_INF {
|
||||
// Fast path: neither is negative (the sign bits are clear) or a NaN
|
||||
// (both are at most +inf), so the bit patterns sort like the values.
|
||||
if args.a < args.b {
|
||||
return 0
|
||||
} else if args.a == args.b {
|
||||
return 1
|
||||
}
|
||||
return 2
|
||||
}
|
||||
return this.f64_cmp_slow(a: args.a, b: args.b)
|
||||
}
|
||||
|
||||
// f64_cmp_slow is f64_cmp without the fast path.
|
||||
pri func arith.f64_cmp_slow(a: base.u64, b: base.u64) base.u32 {
|
||||
var ka : base.u64
|
||||
var kb : base.u64
|
||||
|
||||
@@ -279,35 +312,65 @@ pub func arith.f64_cmp(a: base.u64, b: base.u64) base.u32 {
|
||||
// true), and -0 equals +0.
|
||||
|
||||
pub func arith.f64_eq(a: base.u64, b: base.u64) base.u32 {
|
||||
if this.f64_cmp(a: args.a, b: args.b) == 1 {
|
||||
if (args.a | args.b) <= F64_INF {
|
||||
if args.a == args.b {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
if this.f64_cmp_slow(a: args.a, b: args.b) == 1 {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
pub func arith.f64_ne(a: base.u64, b: base.u64) base.u32 {
|
||||
if this.f64_cmp(a: args.a, b: args.b) == 1 {
|
||||
if (args.a | args.b) <= F64_INF {
|
||||
if args.a <> args.b {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
if this.f64_cmp_slow(a: args.a, b: args.b) == 1 {
|
||||
return 0
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
pub func arith.f64_lt(a: base.u64, b: base.u64) base.u32 {
|
||||
if this.f64_cmp(a: args.a, b: args.b) == 0 {
|
||||
if (args.a | args.b) <= F64_INF {
|
||||
if args.a < args.b {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
if this.f64_cmp_slow(a: args.a, b: args.b) == 0 {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
pub func arith.f64_le(a: base.u64, b: base.u64) base.u32 {
|
||||
if this.f64_cmp(a: args.a, b: args.b) <= 1 {
|
||||
if (args.a | args.b) <= F64_INF {
|
||||
if args.a <= args.b {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
if this.f64_cmp_slow(a: args.a, b: args.b) <= 1 {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
pub func arith.f64_gt(a: base.u64, b: base.u64) base.u32 {
|
||||
if this.f64_cmp(a: args.a, b: args.b) == 2 {
|
||||
if (args.a | args.b) <= F64_INF {
|
||||
if args.a > args.b {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
if this.f64_cmp_slow(a: args.a, b: args.b) == 2 {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
@@ -316,7 +379,13 @@ pub func arith.f64_gt(a: base.u64, b: base.u64) base.u32 {
|
||||
pub func arith.f64_ge(a: base.u64, b: base.u64) base.u32 {
|
||||
var c : base.u32
|
||||
|
||||
c = this.f64_cmp(a: args.a, b: args.b)
|
||||
if (args.a | args.b) <= F64_INF {
|
||||
if args.a >= args.b {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
c = this.f64_cmp_slow(a: args.a, b: args.b)
|
||||
if (c == 1) or (c == 2) {
|
||||
return 1
|
||||
}
|
||||
@@ -327,7 +396,13 @@ pub func arith.f64_ge(a: base.u64, b: base.u64) base.u32 {
|
||||
// Note that this is not IEEE 754 minNum: f64_min(NaN, x) and f64_min(x, NaN)
|
||||
// both return the second argument, and f64_min(-0, +0) returns +0.
|
||||
pub func arith.f64_min(a: base.u64, b: base.u64) base.u64 {
|
||||
if this.f64_cmp(a: args.a, b: args.b) == 0 {
|
||||
if (args.a | args.b) <= F64_INF {
|
||||
if args.a < args.b {
|
||||
return args.a
|
||||
}
|
||||
return args.b
|
||||
}
|
||||
if this.f64_cmp_slow(a: args.a, b: args.b) == 0 {
|
||||
return args.a
|
||||
}
|
||||
return args.b
|
||||
@@ -335,7 +410,13 @@ pub func arith.f64_min(a: base.u64, b: base.u64) base.u64 {
|
||||
|
||||
// f64_max returns ((a > b) ? a : b), like brotli's BROTLI_MAX(double, a, b).
|
||||
pub func arith.f64_max(a: base.u64, b: base.u64) base.u64 {
|
||||
if this.f64_cmp(a: args.a, b: args.b) == 2 {
|
||||
if (args.a | args.b) <= F64_INF {
|
||||
if args.a > args.b {
|
||||
return args.a
|
||||
}
|
||||
return args.b
|
||||
}
|
||||
if this.f64_cmp_slow(a: args.a, b: args.b) == 2 {
|
||||
return args.a
|
||||
}
|
||||
return args.b
|
||||
@@ -345,6 +426,21 @@ pub func arith.f64_max(a: base.u64, b: base.u64) base.u64 {
|
||||
|
||||
// f64_add returns (a + b).
|
||||
pub func arith.f64_add(a: base.u64, b: base.u64) base.u64 {
|
||||
var z : base.u64
|
||||
|
||||
if ((((args.a >> 52) & 0x7FF) ~mod- 1) < 0x7FE) and ((((args.b >> 52) & 0x7FF) ~mod- 1) < 0x7FE) {
|
||||
// Both operands are normal: take the fast path.
|
||||
z = this.f64_add_normal(a: args.a, b: args.b)
|
||||
if z <> 0xFFFF_FFFF_FFFF_FFFF {
|
||||
return z
|
||||
}
|
||||
} else if ((args.b & F64_ABS) == 0) and (((args.a & F64_ABS) ~mod- 1) < (F64_INF - 1)) {
|
||||
// b is a zero and a is finite and non-zero, so (a + b) is a.
|
||||
return args.a
|
||||
} else if ((args.a & F64_ABS) == 0) and (((args.b & F64_ABS) ~mod- 1) < (F64_INF - 1)) {
|
||||
// a is a zero and b is finite and non-zero, so (a + b) is b.
|
||||
return args.b
|
||||
}
|
||||
if ((args.a ^ args.b) >> 63) == 0 {
|
||||
return this.f64_add_mags(a: args.a, b: args.b, sign: args.a & F64_SIGN)
|
||||
}
|
||||
@@ -353,12 +449,105 @@ pub func arith.f64_add(a: base.u64, b: base.u64) base.u64 {
|
||||
|
||||
// f64_sub returns (a - b).
|
||||
pub func arith.f64_sub(a: base.u64, b: base.u64) base.u64 {
|
||||
var z : base.u64
|
||||
|
||||
if ((((args.a >> 52) & 0x7FF) ~mod- 1) < 0x7FE) and ((((args.b >> 52) & 0x7FF) ~mod- 1) < 0x7FE) {
|
||||
// Both operands are normal: take the fast path. For non-NaN
|
||||
// operands, (a - b) is exactly (a + -b), including the sign of a
|
||||
// zero result.
|
||||
z = this.f64_add_normal(a: args.a, b: args.b ^ F64_SIGN)
|
||||
if z <> 0xFFFF_FFFF_FFFF_FFFF {
|
||||
return z
|
||||
}
|
||||
} else if ((args.b & F64_ABS) == 0) and (((args.a & F64_ABS) ~mod- 1) < (F64_INF - 1)) {
|
||||
// b is a zero and a is finite and non-zero, so (a - b) is a.
|
||||
return args.a
|
||||
} else if ((args.a & F64_ABS) == 0) and (((args.b & F64_ABS) ~mod- 1) < (F64_INF - 1)) {
|
||||
// a is a zero and b is finite and non-zero, so (a - b) is -b.
|
||||
return args.b ^ F64_SIGN
|
||||
}
|
||||
if ((args.a ^ args.b) >> 63) == 0 {
|
||||
return this.f64_sub_mags(a: args.a, b: args.b, sign: args.a & F64_SIGN)
|
||||
}
|
||||
return this.f64_add_mags(a: args.a, b: args.b, sign: args.a & F64_SIGN)
|
||||
}
|
||||
|
||||
// f64_add_normal returns (a + b), for normal a and b, unless the result is
|
||||
// subnormal, in which case it returns 0xFFFF_FFFF_FFFF_FFFF (a NaN, which it
|
||||
// never otherwise returns) and the caller must take its slow path. It is a
|
||||
// faster (but less general) version of f64_add_mags and f64_sub_mags.
|
||||
pri func arith.f64_add_normal(a: base.u64, b: base.u64) base.u64 {
|
||||
var x : base.u64
|
||||
var y : base.u64
|
||||
var ex : base.u32[..= 0x800]
|
||||
var d : base.u32
|
||||
var mx : base.u64[..= 0x7FFF_FFFF_FFFF_FFFF]
|
||||
var my : base.u64[..= 0x7FFF_FFFF_FFFF_FFFF]
|
||||
var z : base.u64
|
||||
var s : base.u32[..= 63]
|
||||
var rb : base.u64
|
||||
|
||||
// Order the operands so that |x| >= |y|. The result has x's sign.
|
||||
x = args.a
|
||||
y = args.b
|
||||
if (y & F64_ABS) > (x & F64_ABS) {
|
||||
x = args.b
|
||||
y = args.a
|
||||
}
|
||||
ex = ((x >> 52) & 0x7FF) as base.u32
|
||||
d = ex ~mod- (((y >> 52) & 0x7FF) as base.u32)
|
||||
if d > 63 {
|
||||
d = 63
|
||||
}
|
||||
// The significands, with their implicit bit at bit 62 and 10 extra bits
|
||||
// of precision at the bottom. Align my with mx, keeping a sticky bit.
|
||||
mx = ((x & F64_FRAC) | F64_HIDDEN) << 10
|
||||
my = ((y & F64_FRAC) | F64_HIDDEN) << 10
|
||||
if (my & (((1 as base.u64) << (d & 63)) ~mod- 1)) <> 0 {
|
||||
my = (my >> (d & 63)) | 1
|
||||
} else {
|
||||
my = my >> (d & 63)
|
||||
}
|
||||
|
||||
if ((x ^ y) >> 63) == 0 {
|
||||
z = mx + my
|
||||
if z >= 0x8000_0000_0000_0000 {
|
||||
ex += 1
|
||||
z = (z >> 1) | (z & 1)
|
||||
}
|
||||
} else {
|
||||
// mx >= my, since |x| >= |y|.
|
||||
z = mx ~mod- my
|
||||
if z == 0 {
|
||||
// Exact cancellation gives +0 when rounding to nearest.
|
||||
return 0
|
||||
} else if z < 0x4000_0000_0000_0000 {
|
||||
// If (d >= 2) then z >= (1 << 61) and the normalization shift
|
||||
// is 1. Otherwise, no bits were shifted out of my.
|
||||
s = (this.clz64(a: z) ~mod- 1) & 63
|
||||
if s >= ex {
|
||||
return 0xFFFF_FFFF_FFFF_FFFF
|
||||
}
|
||||
ex = (ex ~mod- s) & 0x7FF
|
||||
z = z ~mod<< s
|
||||
}
|
||||
}
|
||||
|
||||
// Round to nearest, ties to even. z is now in [1 << 52 ..= 1 << 53] and
|
||||
// ex in [1 ..= 0x7FF]. Adding z (whose bit 52 is the implicit bit) to the
|
||||
// shifted exponent carries any rounding overflow into the exponent field.
|
||||
rb = z & 0x3FF
|
||||
z = (z ~mod+ 0x200) >> 10
|
||||
if rb == 0x200 {
|
||||
z &= 0xFFFF_FFFF_FFFF_FFFE
|
||||
}
|
||||
z = (((ex ~mod- 1) as base.u64) ~mod<< 52) ~mod+ z
|
||||
if z >= F64_INF {
|
||||
z = F64_INF
|
||||
}
|
||||
return (x & F64_SIGN) | z
|
||||
}
|
||||
|
||||
// f64_add_mags returns (sign * (|a| + |b|)).
|
||||
pri func arith.f64_add_mags(a: base.u64, b: base.u64, sign: base.u64) base.u64 {
|
||||
var exp_a : base.u32
|
||||
|
||||
@@ -10,12 +10,19 @@
|
||||
|
||||
// -------- f32 (binary32) operations.
|
||||
//
|
||||
// The f32 add, sub, mul and div functions convert their (exactly
|
||||
// representable) operands to f64, perform the f64 operation and round the
|
||||
// result to f32. Double rounding is innocuous here: binary64 has more than
|
||||
// (2 * 24) + 2 bits of precision, which suffices for +, -, * and / (Figueroa,
|
||||
// "When is double rounding innocuous?", 1995). This is also exhaustively
|
||||
// cross-checked against hardware by the tests.
|
||||
// When both operands are normal (and, unless it overflows to infinity, the
|
||||
// result is normal too), the f32 add, sub and mul functions compute the
|
||||
// result directly in binary32 (with 32-bit or 64-bit integer significands).
|
||||
// This is the common case. Adding or subtracting a zero and a finite
|
||||
// non-zero operand is also handled directly. Otherwise (zeroes, subnormals,
|
||||
// infinities, NaNs and subnormal results), they and f32_div convert their
|
||||
// (exactly representable) operands to f64, perform the f64 operation and
|
||||
// round the result to f32 (the slow_etc functions). Double rounding is
|
||||
// innocuous here: binary64 has more than (2 * 24) + 2 bits of precision,
|
||||
// which suffices for +, -, * and / (Figueroa, "When is double rounding
|
||||
// innocuous?", 1995). This is also exhaustively cross-checked against
|
||||
// hardware by the tests. The fast paths never return a NaN, so NaN results
|
||||
// (and their payloads) always come from the slow paths.
|
||||
|
||||
pri const F32_SIGN : base.u32 = 0x8000_0000
|
||||
pri const F32_ABS : base.u32 = 0x7FFF_FFFF
|
||||
@@ -74,20 +81,244 @@ pub func arith.f32_abs(a: base.u32) base.u32 {
|
||||
|
||||
// f32_add returns (a + b).
|
||||
pub func arith.f32_add(a: base.u32, b: base.u32) base.u32 {
|
||||
var z : base.u32
|
||||
|
||||
if ((((args.a >> 23) & 0xFF) ~mod- 1) < 0xFE) and ((((args.b >> 23) & 0xFF) ~mod- 1) < 0xFE) {
|
||||
if ((args.a ^ args.b) >> 31) == 0 {
|
||||
return this.f32_add_mags(a: args.a, b: args.b)
|
||||
}
|
||||
z = this.f32_sub_mags(a: args.a, b: args.b)
|
||||
if z <> 0xFFFF_FFFF {
|
||||
return z
|
||||
}
|
||||
} else if ((args.b & F32_ABS) == 0) and (((args.a & F32_ABS) ~mod- 1) < (F32_INF - 1)) {
|
||||
// b is a zero and a is finite and non-zero, so (a + b) is a.
|
||||
return args.a
|
||||
} else if ((args.a & F32_ABS) == 0) and (((args.b & F32_ABS) ~mod- 1) < (F32_INF - 1)) {
|
||||
// a is a zero and b is finite and non-zero, so (a + b) is b.
|
||||
return args.b
|
||||
}
|
||||
return this.slow_f32_add(a: args.a, b: args.b)
|
||||
}
|
||||
|
||||
// f32_sub returns (a - b).
|
||||
pub func arith.f32_sub(a: base.u32, b: base.u32) base.u32 {
|
||||
var z : base.u32
|
||||
|
||||
if ((((args.a >> 23) & 0xFF) ~mod- 1) < 0xFE) and ((((args.b >> 23) & 0xFF) ~mod- 1) < 0xFE) {
|
||||
if ((args.a ^ args.b) >> 31) <> 0 {
|
||||
return this.f32_add_mags(a: args.a, b: args.b)
|
||||
}
|
||||
z = this.f32_sub_mags(a: args.a, b: args.b)
|
||||
if z <> 0xFFFF_FFFF {
|
||||
return z
|
||||
}
|
||||
} else if ((args.b & F32_ABS) == 0) and (((args.a & F32_ABS) ~mod- 1) < (F32_INF - 1)) {
|
||||
// b is a zero and a is finite and non-zero, so (a - b) is a.
|
||||
return args.a
|
||||
} else if ((args.a & F32_ABS) == 0) and (((args.b & F32_ABS) ~mod- 1) < (F32_INF - 1)) {
|
||||
// a is a zero and b is finite and non-zero, so (a - b) is -b.
|
||||
return args.b ^ F32_SIGN
|
||||
}
|
||||
return this.slow_f32_sub(a: args.a, b: args.b)
|
||||
}
|
||||
|
||||
// f32_mul returns (a * b).
|
||||
pub func arith.f32_mul(a: base.u32, b: base.u32) base.u32 {
|
||||
var ea : base.u32[..= 0xFF]
|
||||
var eb : base.u32[..= 0xFF]
|
||||
var e : base.u32[..= 0x1FF]
|
||||
var p : base.u64
|
||||
var m : base.u32
|
||||
var r : base.u32
|
||||
var z : base.u32
|
||||
|
||||
ea = (args.a >> 23) & 0xFF
|
||||
eb = (args.b >> 23) & 0xFF
|
||||
if ((ea ~mod- 1) < 0xFE) and ((eb ~mod- 1) < 0xFE) {
|
||||
// Both operands are normal. The exact product of the two 24-bit
|
||||
// significands, p, is in [1 << 46, 1 << 48). The biased exponent of
|
||||
// the result, e, is (ea + eb - 127) (plus 1 if p >= (1 << 47)).
|
||||
p = (((args.a & 0x7F_FFFF) | 0x80_0000) as base.u64) *
|
||||
(((args.b & 0x7F_FFFF) | 0x80_0000) as base.u64)
|
||||
e = ea + eb
|
||||
if p >= 0x8000_0000_0000 {
|
||||
e += 1
|
||||
m = ((p >> 24) & 0xFF_FFFF) as base.u32
|
||||
r = ((p & 0xFF_FFFF) as base.u32) ~mod- 0x80_0000
|
||||
} else {
|
||||
m = ((p >> 23) & 0xFF_FFFF) as base.u32
|
||||
r = ((p & 0x7F_FFFF) as base.u32) ~mod- 0x40_0000
|
||||
}
|
||||
// r is the remainder minus half an ULP, as a two's complement
|
||||
// integer. Round to nearest, ties to even.
|
||||
if (r <> 0) and (r < 0x8000_0000) {
|
||||
m ~mod+= 1
|
||||
} else if r == 0 {
|
||||
m ~mod+= m & 1
|
||||
}
|
||||
if (e >= 128) and (e <= 381) {
|
||||
// Adding m (whose bit 23 is the implicit bit) carries any
|
||||
// rounding overflow into the exponent field.
|
||||
z = ((e - 128) << 23) ~mod+ m
|
||||
if z >= F32_INF {
|
||||
z = F32_INF
|
||||
}
|
||||
return ((args.a ^ args.b) & F32_SIGN) | z
|
||||
}
|
||||
}
|
||||
return this.slow_f32_mul(a: args.a, b: args.b)
|
||||
}
|
||||
|
||||
// f32_add_mags returns (sign(a) * (|a| + |b|)), for normal a and b.
|
||||
pri func arith.f32_add_mags(a: base.u32, b: base.u32) base.u32 {
|
||||
var ea : base.u32[..= 0x100]
|
||||
var eb : base.u32[..= 0xFF]
|
||||
var ma : base.u32[..= 0x3FFF_FFC0]
|
||||
var mb : base.u32[..= 0x3FFF_FFC0]
|
||||
var d : base.u32[..= 0xFF]
|
||||
var z : base.u32
|
||||
var rb : base.u32
|
||||
|
||||
ea = (args.a >> 23) & 0xFF
|
||||
eb = (args.b >> 23) & 0xFF
|
||||
// The significands, with their implicit bit at bit 29 and 6 extra bits
|
||||
// of precision at the bottom.
|
||||
ma = ((args.a & 0x7F_FFFF) | 0x80_0000) << 6
|
||||
mb = ((args.b & 0x7F_FFFF) | 0x80_0000) << 6
|
||||
if ea < eb {
|
||||
d = (eb ~mod- ea) & 0xFF
|
||||
ea = eb
|
||||
z = ma
|
||||
ma = mb
|
||||
mb = z
|
||||
} else {
|
||||
d = (ea ~mod- eb) & 0xFF
|
||||
}
|
||||
if d >= 31 {
|
||||
mb = 1
|
||||
} else if d > 0 {
|
||||
if (mb ~mod<< (32 - d)) <> 0 {
|
||||
mb = (mb >> d) | 1
|
||||
} else {
|
||||
mb = mb >> d
|
||||
}
|
||||
}
|
||||
z = (ma & 0x3FFF_FFFF) + (mb & 0x3FFF_FFFF)
|
||||
if z >= 0x4000_0000 {
|
||||
ea += 1
|
||||
z = (z >> 1) | (z & 1)
|
||||
}
|
||||
rb = z & 0x3F
|
||||
z = (z ~mod+ 0x20) >> 6
|
||||
if rb == 0x20 {
|
||||
z &= 0xFFFF_FFFE
|
||||
}
|
||||
// ea is in [1 ..= 255] and z is in [1 << 23 ..= 1 << 24].
|
||||
z = ((ea ~mod- 1) ~mod<< 23) ~mod+ z
|
||||
if z >= F32_INF {
|
||||
z = F32_INF
|
||||
}
|
||||
return (args.a & F32_SIGN) | z
|
||||
}
|
||||
|
||||
// f32_sub_mags returns (sign(a) * (|a| - |b|)), for normal a and b, unless
|
||||
// the result is subnormal, in which case it returns 0xFFFF_FFFF (a NaN, which
|
||||
// it never otherwise returns) and the caller must take its slow path.
|
||||
pri func arith.f32_sub_mags(a: base.u32, b: base.u32) base.u32 {
|
||||
var sign : base.u32
|
||||
var ea : base.u32[..= 0xFF]
|
||||
var eb : base.u32[..= 0xFF]
|
||||
var fa : base.u32[..= 0x7F_FFFF]
|
||||
var fb : base.u32[..= 0x7F_FFFF]
|
||||
var ma : base.u32
|
||||
var mb : base.u32
|
||||
var d : base.u32[..= 0xFF]
|
||||
var s : base.u32[..= 31]
|
||||
var z : base.u32
|
||||
var rb : base.u32
|
||||
|
||||
sign = args.a & F32_SIGN
|
||||
ea = (args.a >> 23) & 0xFF
|
||||
eb = (args.b >> 23) & 0xFF
|
||||
fa = args.a & 0x7F_FFFF
|
||||
fb = args.b & 0x7F_FFFF
|
||||
|
||||
if ea == eb {
|
||||
// The difference is exact.
|
||||
if fa == fb {
|
||||
// Exact cancellation gives +0 when rounding to nearest.
|
||||
return 0
|
||||
} else if fa < fb {
|
||||
sign ^= F32_SIGN
|
||||
z = fb ~mod- fa
|
||||
} else {
|
||||
z = fa ~mod- fb
|
||||
}
|
||||
// z is in [1 ..= 0x7F_FFFF]. Shift its top bit to bit 23.
|
||||
s = (this.clz32(a: z) ~mod- 8) & 31
|
||||
if s >= ea {
|
||||
return 0xFFFF_FFFF
|
||||
}
|
||||
return sign | ((((ea ~mod- s) ~mod- 1) ~mod<< 23) ~mod+ (z ~mod<< s))
|
||||
}
|
||||
|
||||
// The significands, with their implicit bit at bit 30 and 7 extra bits
|
||||
// of precision at the bottom.
|
||||
ma = (fa | 0x80_0000) << 7
|
||||
mb = (fb | 0x80_0000) << 7
|
||||
if ea < eb {
|
||||
sign ^= F32_SIGN
|
||||
d = (eb ~mod- ea) & 0xFF
|
||||
ea = eb
|
||||
z = ma
|
||||
ma = mb
|
||||
mb = z
|
||||
} else {
|
||||
d = (ea ~mod- eb) & 0xFF
|
||||
}
|
||||
if d >= 31 {
|
||||
mb = 1
|
||||
} else if (mb ~mod<< ((32 - d) & 31)) <> 0 {
|
||||
mb = (mb >> d) | 1
|
||||
} else {
|
||||
mb = mb >> d
|
||||
}
|
||||
// ma > mb, so z is non-zero. If (d >= 2) then z >= (1 << 29) and the
|
||||
// normalization shift is at most 1. Otherwise, no bits were shifted out.
|
||||
z = ma ~mod- mb
|
||||
s = (this.clz32(a: z) ~mod- 1) & 31
|
||||
if s >= ea {
|
||||
return 0xFFFF_FFFF
|
||||
}
|
||||
ea = (ea ~mod- s) & 0xFF
|
||||
z = z ~mod<< s
|
||||
rb = z & 0x7F
|
||||
z = (z ~mod+ 0x40) >> 7
|
||||
if rb == 0x40 {
|
||||
z &= 0xFFFF_FFFE
|
||||
}
|
||||
// ea is in [1 ..= 254] and z is in [1 << 23 ..= 1 << 24], so this does
|
||||
// not overflow.
|
||||
return sign | (((ea ~mod- 1) ~mod<< 23) ~mod+ z)
|
||||
}
|
||||
|
||||
// slow_f32_add returns (a + b), computed via f64.
|
||||
pri func arith.slow_f32_add(a: base.u32, b: base.u32) base.u32 {
|
||||
return this.f64_to_f32(a: this.f64_add(
|
||||
a: this.f32_to_f64(a: args.a),
|
||||
b: this.f32_to_f64(a: args.b)))
|
||||
}
|
||||
|
||||
// f32_sub returns (a - b).
|
||||
pub func arith.f32_sub(a: base.u32, b: base.u32) base.u32 {
|
||||
// slow_f32_sub returns (a - b), computed via f64.
|
||||
pri func arith.slow_f32_sub(a: base.u32, b: base.u32) base.u32 {
|
||||
return this.f64_to_f32(a: this.f64_sub(
|
||||
a: this.f32_to_f64(a: args.a),
|
||||
b: this.f32_to_f64(a: args.b)))
|
||||
}
|
||||
|
||||
// f32_mul returns (a * b).
|
||||
pub func arith.f32_mul(a: base.u32, b: base.u32) base.u32 {
|
||||
// slow_f32_mul returns (a * b), computed via f64.
|
||||
pri func arith.slow_f32_mul(a: base.u32, b: base.u32) base.u32 {
|
||||
return this.f64_to_f32(a: this.f64_mul(
|
||||
a: this.f32_to_f64(a: args.a),
|
||||
b: this.f32_to_f64(a: args.b)))
|
||||
@@ -103,6 +334,21 @@ pub func arith.f32_div(a: base.u32, b: base.u32) base.u32 {
|
||||
// f32_cmp compares a and b, returning 0 if (a < b), 1 if (a == b), 2 if (a
|
||||
// > b) and 3 if they are unordered (either is a NaN). -0 and +0 compare equal.
|
||||
pub func arith.f32_cmp(a: base.u32, b: base.u32) base.u32 {
|
||||
if (args.a | args.b) <= F32_INF {
|
||||
// Fast path: neither is negative (the sign bits are clear) or a NaN
|
||||
// (both are at most +inf), so the bit patterns sort like the values.
|
||||
if args.a < args.b {
|
||||
return 0
|
||||
} else if args.a == args.b {
|
||||
return 1
|
||||
}
|
||||
return 2
|
||||
}
|
||||
return this.f32_cmp_slow(a: args.a, b: args.b)
|
||||
}
|
||||
|
||||
// f32_cmp_slow is f32_cmp without the fast path.
|
||||
pri func arith.f32_cmp_slow(a: base.u32, b: base.u32) base.u32 {
|
||||
var ka : base.u32
|
||||
var kb : base.u32
|
||||
|
||||
@@ -133,35 +379,65 @@ pub func arith.f32_cmp(a: base.u32, b: base.u32) base.u32 {
|
||||
// true), and -0 equals +0.
|
||||
|
||||
pub func arith.f32_eq(a: base.u32, b: base.u32) base.u32 {
|
||||
if this.f32_cmp(a: args.a, b: args.b) == 1 {
|
||||
if (args.a | args.b) <= F32_INF {
|
||||
if args.a == args.b {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
if this.f32_cmp_slow(a: args.a, b: args.b) == 1 {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
pub func arith.f32_ne(a: base.u32, b: base.u32) base.u32 {
|
||||
if this.f32_cmp(a: args.a, b: args.b) == 1 {
|
||||
if (args.a | args.b) <= F32_INF {
|
||||
if args.a <> args.b {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
if this.f32_cmp_slow(a: args.a, b: args.b) == 1 {
|
||||
return 0
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
pub func arith.f32_lt(a: base.u32, b: base.u32) base.u32 {
|
||||
if this.f32_cmp(a: args.a, b: args.b) == 0 {
|
||||
if (args.a | args.b) <= F32_INF {
|
||||
if args.a < args.b {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
if this.f32_cmp_slow(a: args.a, b: args.b) == 0 {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
pub func arith.f32_le(a: base.u32, b: base.u32) base.u32 {
|
||||
if this.f32_cmp(a: args.a, b: args.b) <= 1 {
|
||||
if (args.a | args.b) <= F32_INF {
|
||||
if args.a <= args.b {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
if this.f32_cmp_slow(a: args.a, b: args.b) <= 1 {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
pub func arith.f32_gt(a: base.u32, b: base.u32) base.u32 {
|
||||
if this.f32_cmp(a: args.a, b: args.b) == 2 {
|
||||
if (args.a | args.b) <= F32_INF {
|
||||
if args.a > args.b {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
if this.f32_cmp_slow(a: args.a, b: args.b) == 2 {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
@@ -170,7 +446,13 @@ pub func arith.f32_gt(a: base.u32, b: base.u32) base.u32 {
|
||||
pub func arith.f32_ge(a: base.u32, b: base.u32) base.u32 {
|
||||
var c : base.u32
|
||||
|
||||
c = this.f32_cmp(a: args.a, b: args.b)
|
||||
if (args.a | args.b) <= F32_INF {
|
||||
if args.a >= args.b {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
c = this.f32_cmp_slow(a: args.a, b: args.b)
|
||||
if (c == 1) or (c == 2) {
|
||||
return 1
|
||||
}
|
||||
@@ -181,7 +463,13 @@ pub func arith.f32_ge(a: base.u32, b: base.u32) base.u32 {
|
||||
// Note that this is not IEEE 754 minNum: f32_min(NaN, x) and f32_min(x, NaN)
|
||||
// both return the second argument, and f32_min(-0, +0) returns +0.
|
||||
pub func arith.f32_min(a: base.u32, b: base.u32) base.u32 {
|
||||
if this.f32_cmp(a: args.a, b: args.b) == 0 {
|
||||
if (args.a | args.b) <= F32_INF {
|
||||
if args.a < args.b {
|
||||
return args.a
|
||||
}
|
||||
return args.b
|
||||
}
|
||||
if this.f32_cmp_slow(a: args.a, b: args.b) == 0 {
|
||||
return args.a
|
||||
}
|
||||
return args.b
|
||||
@@ -189,7 +477,13 @@ pub func arith.f32_min(a: base.u32, b: base.u32) base.u32 {
|
||||
|
||||
// f32_max returns ((a > b) ? a : b), like brotli's BROTLI_MAX(float, a, b).
|
||||
pub func arith.f32_max(a: base.u32, b: base.u32) base.u32 {
|
||||
if this.f32_cmp(a: args.a, b: args.b) == 2 {
|
||||
if (args.a | args.b) <= F32_INF {
|
||||
if args.a > args.b {
|
||||
return args.a
|
||||
}
|
||||
return args.b
|
||||
}
|
||||
if this.f32_cmp_slow(a: args.a, b: args.b) == 2 {
|
||||
return args.a
|
||||
}
|
||||
return args.b
|
||||
@@ -197,6 +491,18 @@ pub func arith.f32_max(a: base.u32, b: base.u32) base.u32 {
|
||||
|
||||
// f32_from_u32 returns the f32 nearest to the integer a (C's (float)a).
|
||||
pub func arith.f32_from_u32(a: base.u32) base.u32 {
|
||||
var c : base.u32[..= 31]
|
||||
|
||||
if args.a == 0 {
|
||||
return 0
|
||||
} else if args.a < 0x100_0000 {
|
||||
// a is exactly representable. Shift its top bit to bit 23 (the
|
||||
// implicit bit, which is then added to the exponent field).
|
||||
c = this.clz32(a: args.a)
|
||||
if c >= 8 {
|
||||
return ((157 ~mod- c) ~mod<< 23) ~mod+ (args.a ~mod<< (c - 8))
|
||||
}
|
||||
}
|
||||
// The u32 to f64 conversion is exact, so there is only one rounding.
|
||||
return this.f64_to_f32(a: this.f64_from_u32(a: args.a))
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user