Merge claude/project-thread-0izcgq (speed work)

This commit is contained in:
Claude committed 2026-10-27 15:13:58 +00:00
commit 73de2fe182
15 files changed
+3327 -329

No files matched your search

+30 -10
View File
@@ -55,8 +55,9 @@ Without the large-window quirk and without attached dictionaries (see the
window size plus some slack. The slack
mirrors the C code's `kRingBufferWriteAheadSlack`, which its fast paths use to
write (and discard) a few bytes past the ring buffer's end. This
implementation does not yet have such fast paths, but it still reserves the
slack, so that adding them later will not change the `workbuf_len` API. A
implementation's fast paths (see below) do not write past the ring buffer's
end, but it still reserves the slack, so that adding such fast paths later
will not change the `workbuf_len` API. A
workbuf shorter than the stream's window size plus 542 bytes is rejected with
`base."#bad workbuf length"`.
@@ -158,12 +159,25 @@ invariants for memory safety but Wuffs requires explicit bounds checks.
- The bit reader reads input bytes only when it needs them (like the C code's
"safe" `BrotliSafeReadBits` and `SafeReadSymbol` functions), so that
`transform_io` can suspend at any point. There is no "fast path" (like the C
code's `ProcessCommands`, as opposed to `SafeProcessCommands`). Both paths
produce the same output and errors.
- LZ77 copies and dictionary words are written to the ring buffer one byte at
a time, with ring buffer events (the C code's `WriteRingBuffer` and
`WrapRingBuffer` calls) at the same positions as the C code.
`transform_io` can suspend at any point. Between reads, fewer than 8 bits
are buffered. The fast path, `process_commands_fast` (like the C code's
`ProcessCommands`, as opposed to `SafeProcessCommands`), is not a coroutine.
It reads ahead (up to 8 bytes at a time) into a 64-bit bit buffer while the
input has enough bytes, and when it returns to `process_commands` (at a
block switch, a ring buffer event, a dictionary reference or when the input
runs low), it gives back the whole bytes left in that buffer, which restores
the "fewer than 8 bits" state exactly. `process_commands` tracks where it is
within a command (`cmd_state`), so that either can pick up where the other
left off. Within `process_commands`, `read_symbol_fast` and `read_bits_fast`
(which are not coroutines either) read the same bytes at the same times as
the coroutines `read_symbol` and `read_bits`, which finish the job when the
input runs out. All paths produce the same output and errors.
- Dictionary words, and LZ77 copies that wrap around the physical ring buffer
or reach the C code's (virtual) ring buffer size, are written to the ring
buffer one byte at a time, with ring buffer events (the C code's
`WriteRingBuffer` and `WrapRingBuffer` calls) at the same positions as the
C code. Other LZ77 copies are written in chunks of at most the copy distance
(so that each chunk's source bytes are final), which is equivalent.
- On error, the Wuffs decoder may have already written more output to `dst`
than libbrotlidec would have (libbrotlidec does not flush on error), but
what both have written agrees.
@@ -173,7 +187,8 @@ invariants for memory safety but Wuffs requires explicit bounds checks.
- Metadata callbacks (metadata blocks are currently skipped). See the
"Metadata Blocks" section below.
- Fast paths.
- More fast paths, for example for dictionary words, for copies that wrap
around the ring buffer and for block switches.
## Encoder
@@ -969,7 +984,12 @@ uses it at qualities 10 and 11 (and its smaller helpers at lower
qualities). Its
fixed-size state (about 0.8 MiB: FindBlocks' cost arrays and a 64Ki-entry
FastLog2 cache in its `bit_cost`) lives in the struct; everything that the C
code allocates on the heap lives in a caller-supplied workbuf.
code allocates on the heap lives in a caller-supplied workbuf. The FastLog2
cache (like the encoder's own `bit_cost` one) is filled lazily, one entry at
a time, when `fast_log2_memo` first needs that entry (a bitmap, zeroed when the
struct is initialized, records which entries are valid): typically only a few
thousand of the 65536 entries are ever used, and each one costs a (slow, exact)
software `log2`.
Data formats (all little-endian, matching the C structs' memory layout on a
64-bit little-endian machine, so that results can be compared with `memcmp`):
+94
View File
@@ -230,6 +230,13 @@ pub struct decoder? implements base.io_transformer(
insert_length : base.u32,
copy_length : base.u32,
// cmd_state is where process_commands (and process_commands_fast)
// are within a command: 0 at its start, 1 when reading its literals
// (insert_length is how many are left), 2 when reading its distance
// and 3 when applying its copy. process_commands_fast sets it to 4
// when the metablock is done.
cmd_state : base.u32[..= 4],
// quirk_large_window is the QUIRK_LARGE_WINDOW quirk (the C code's
// BROTLI_DECODER_PARAM_LARGE_WINDOW parameter). large_window is
// whether the stream is a large-window stream (the C code's
@@ -679,6 +686,93 @@ pri func decoder.read_symbol?(src: base.io_reader, hoff: base.u32) {
this.n_bits = n_bits
}
// read_bits_fast is like read_bits but is not a coroutine. It reads input
// bytes exactly when read_bits would, for as long as args.src has them. It
// returns 0 when it has set this.bits_value, or 1 when read_bits must finish
// the job (this.bits and this.n_bits hold the bits read so far).
pri func decoder.read_bits_fast!(src: base.io_reader, n: base.u32[..= 24]) base.u32 {
var n : base.u32[..= 24]
var bits : base.u32
var n_bits : base.u32
n = args.n
bits = this.bits
n_bits = this.n_bits
while n_bits < n,
post n_bits >= n,
{
if args.src.length() <= 0 {
this.bits = bits
this.n_bits = n_bits
return 1
}
assert n_bits < 24 via "a < b: a < c; c <= b"(c: n)
bits |= args.src.peek_u8_as_u32() << n_bits
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
n_bits += 8
}
this.bits_value = bits & (((1 as base.u32) << n) - 1)
this.bits = bits >> n
this.n_bits = n_bits - n
return 0
}
// read_symbol_fast is like read_symbol but is not a coroutine. It reads
// input bytes exactly when read_symbol would, for as long as args.src has
// them. It returns 0 when it has set this.symbol, or 1 when read_symbol must
// finish the job (this.bits and this.n_bits hold the bits read so far). In
// particular, read_symbol (not read_symbol_fast) returns any error.
pri func decoder.read_symbol_fast!(src: base.io_reader, hoff: base.u32) base.u32 {
var hoff : base.u32
var bits : base.u32
var n_bits : base.u32
var i : base.u32
var e : base.u32
var e_bits : base.u32[..= 23]
hoff = args.hoff
bits = this.bits
n_bits = this.n_bits
while true {
i = hoff ~mod+ (bits & 0xFF)
if i >= HUFFS_LEN {
break
}
e = this.huffs[i] as base.u32
e_bits = e & 15
if e_bits <= 8 {
if n_bits >= e_bits {
this.symbol = e >> 4
this.bits = bits >> e_bits
this.n_bits = n_bits - e_bits
return 0
}
} else if n_bits > 8 {
i = (hoff ~mod+ (e >> 4)) ~mod+ ((bits >> 8) & (((1 as base.u32) << (e_bits - 8)) - 1))
if i >= HUFFS_LEN {
break
}
e = this.huffs[i] as base.u32
e_bits = (e & 15) + 8
if n_bits >= e_bits {
this.symbol = e >> 4
this.bits = bits >> e_bits
this.n_bits = n_bits - e_bits
return 0
}
}
if (n_bits >= 24) or (args.src.length() <= 0) {
break
}
bits |= args.src.peek_u8_as_u32() << n_bits
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
n_bits += 8
}
this.bits = bits
this.n_bits = n_bits
return 1
}
// jump_to_byte_boundary drops the bits up to the next byte boundary and
// returns whether they were all zero. Mirrors BrotliJumpToByteBoundary in
// bit_reader.h.
+450 -46
View File
@@ -418,7 +418,6 @@ pri func decoder.calculate_distance_lut!() {
// Mirrors ProcessCommandsInternal in decode.c.
pri func decoder.process_commands?(src: base.io_reader, workbuf: slice base.u8, dictbuf: roslice base.u8) {
var cmd_code : base.u32[..= 703]
var insert_length : base.u32
var p1 : base.u32[..= 0xFF]
var p2 : base.u32[..= 0xFF]
var mode_offset : base.u32[..= 0x600]
@@ -434,37 +433,72 @@ pri func decoder.process_commands?(src: base.io_reader, workbuf: slice base.u8,
var address : base.u32
var chunk : base.u32[..= 14]
var j : base.u64
var fr : base.u32
var dst_i : base.u64
var src_i : base.u64
var k64 : base.u64
var n64 : base.u64
var dlo : base.u64
var dhi : base.u64
var slo : base.u64
var shi : base.u64
var fs : base.status
while true {
// CommandBegin.
if this.block_length[1] == 0 {
// Mirrors DecodeCommandBlockSwitch.
this.decode_block_switch?(src: args.src, t: 1)
this.htree_command = this.block_type_rb[3] & 0xFF
continue
if (this.cmd_state == 0) and (args.src.length() >= 16) {
fs = this.process_commands_fast!(src: args.src, workbuf: args.workbuf)
if not fs.is_ok() {
return fs
} else if this.cmd_state == 4 {
this.cmd_state = 0
return ok
}
}
// Mirrors ReadCommandInternal.
this.read_symbol?(src: args.src,
hoff: CMD_HUFFS_BASE + (this.htree_command * CMD_HUFFS_STRIDE))
if this.symbol >= 704 {
return "#internal error: inconsistent Huffman table"
if this.cmd_state == 0 {
// CommandBegin.
if this.block_length[1] == 0 {
// Mirrors DecodeCommandBlockSwitch.
this.decode_block_switch?(src: args.src, t: 1)
this.htree_command = this.block_type_rb[3] & 0xFF
continue
}
// Mirrors ReadCommandInternal.
fr = this.read_symbol_fast!(src: args.src, hoff: CMD_HUFFS_BASE + (this.htree_command * CMD_HUFFS_STRIDE))
if fr <> 0 {
this.read_symbol?(src: args.src,
hoff: CMD_HUFFS_BASE + (this.htree_command * CMD_HUFFS_STRIDE))
}
if this.symbol >= 704 {
return "#internal error: inconsistent Huffman table"
}
cmd_code = this.symbol.min(no_more_than: 703)
this.implicit_zero_distance = CMD_IMPLICIT_ZERO_DISTANCE[cmd_code] <> 0
this.distance_context = CMD_DISTANCE_CONTEXT[cmd_code] as base.u32
this.update_dist_htree_index!()
fr = this.read_bits_fast!(src: args.src, n: CMD_INSERT_LEN_EXTRA_BITS[cmd_code] as base.u32)
if fr <> 0 {
this.read_bits?(src: args.src, n: CMD_INSERT_LEN_EXTRA_BITS[cmd_code] as base.u32)
}
this.insert_length = CMD_INSERT_LEN_OFFSET[cmd_code] ~mod+ this.bits_value
fr = this.read_bits_fast!(src: args.src, n: CMD_COPY_LEN_EXTRA_BITS[cmd_code] as base.u32)
if fr <> 0 {
this.read_bits?(src: args.src, n: CMD_COPY_LEN_EXTRA_BITS[cmd_code] as base.u32)
}
this.copy_length = CMD_COPY_LEN_OFFSET[cmd_code] ~mod+ this.bits_value
this.block_length[1] ~mod-= 1
if this.insert_length > 0 {
this.mb_remaining ~mod-= this.insert_length
this.cmd_state = 1
} else {
this.cmd_state = 2
}
}
cmd_code = this.symbol.min(no_more_than: 703)
this.implicit_zero_distance = CMD_IMPLICIT_ZERO_DISTANCE[cmd_code] <> 0
this.distance_context = CMD_DISTANCE_CONTEXT[cmd_code] as base.u32
this.update_dist_htree_index!()
this.read_bits?(src: args.src, n: CMD_INSERT_LEN_EXTRA_BITS[cmd_code] as base.u32)
insert_length = CMD_INSERT_LEN_OFFSET[cmd_code] ~mod+ this.bits_value
this.read_bits?(src: args.src, n: CMD_COPY_LEN_EXTRA_BITS[cmd_code] as base.u32)
this.copy_length = CMD_COPY_LEN_OFFSET[cmd_code] ~mod+ this.bits_value
this.block_length[1] ~mod-= 1
if insert_length > 0 {
this.mb_remaining ~mod-= insert_length
if this.cmd_state == 1 {
// CommandInner. Read the literals in the command.
while insert_length > 0 {
while this.insert_length > 0 {
if this.block_length[0] == 0 {
// Mirrors DecodeLiteralBlockSwitch.
this.decode_block_switch?(src: args.src, t: 0)
@@ -478,39 +512,48 @@ pri func decoder.process_commands?(src: base.io_reader, workbuf: slice base.u8,
ctx = (CONTEXT_LOOKUP[mode_offset | p1] |
CONTEXT_LOOKUP[mode_offset | 0x100 | p2]) as base.u32
tree = this.context_maps[this.context_map_slice + ctx] as base.u32
this.read_symbol?(src: args.src, hoff: LIT_HUFFS_BASE + (tree * LIT_HUFFS_STRIDE))
fr = this.read_symbol_fast!(src: args.src, hoff: LIT_HUFFS_BASE + (tree * LIT_HUFFS_STRIDE))
if fr <> 0 {
this.read_symbol?(src: args.src, hoff: LIT_HUFFS_BASE + (tree * LIT_HUFFS_STRIDE))
}
this.put_byte!(workbuf: args.workbuf, b: (this.symbol & 0xFF) as base.u8)
this.block_length[0] ~mod-= 1
insert_length -= 1
this.insert_length ~mod-= 1
if this.pos == this.rb_size {
this.ring_event?()
}
}
this.cmd_state = 2
if (this.mb_remaining == 0) or (this.mb_remaining >= 0x8000_0000) {
// BROTLI_STATE_METABLOCK_DONE.
this.cmd_state = 0
return ok
}
}
// CommandPostDecodeLiterals.
if this.implicit_zero_distance {
// Implicit distance case.
this.distance_context = 1
this.dist_rb_idx ~mod-= 1
this.distance_code = this.dist_rb[this.dist_rb_idx & 3]
} else {
// Read distance code in the command, unless it was implicitly
// zero.
if this.block_length[2] == 0 {
// Mirrors DecodeDistanceBlockSwitch.
this.decode_block_switch?(src: args.src, t: 2)
this.dist_context_map_slice = (this.block_type_rb[5] & 0xFF) << 2
this.update_dist_htree_index!()
if this.cmd_state == 2 {
// CommandPostDecodeLiterals.
if this.implicit_zero_distance {
// Implicit distance case.
this.distance_context = 1
this.dist_rb_idx ~mod-= 1
this.distance_code = this.dist_rb[this.dist_rb_idx & 3]
} else {
// Read distance code in the command, unless it was implicitly
// zero.
if this.block_length[2] == 0 {
// Mirrors DecodeDistanceBlockSwitch.
this.decode_block_switch?(src: args.src, t: 2)
this.dist_context_map_slice = (this.block_type_rb[5] & 0xFF) << 2
this.update_dist_htree_index!()
}
this.read_distance?(src: args.src)
}
this.read_distance?(src: args.src)
this.cmd_state = 3
}
// this.cmd_state == 3.
if this.max_distance <> this.max_backward_distance {
this.max_distance = this.pos.min(no_more_than: this.max_backward_distance)
}
@@ -604,6 +647,37 @@ pri func decoder.process_commands?(src: base.io_reader, workbuf: slice base.u8,
this.dist_rb[this.dist_rb_idx & 3] = distance
this.dist_rb_idx ~mod+= 1
this.mb_remaining ~mod-= i
// Fast path: when the copy neither wraps around the physical ring
// buffer (at either end) nor makes this.pos reach this.rb_size,
// copy the bytes in chunks of at most distance bytes. Each chunk's
// source bytes are final (and do not overlap the chunk's
// destination), so this matches the byte by byte loop below.
dst_i = this.pos as base.u64
src_i = ((this.pos ~mod- distance) & (this.window_size ~mod- 1)) as base.u64
if ((dst_i + (i as base.u64)) < (this.rb_size as base.u64)) and
((src_i + (i as base.u64)) <= (this.window_size as base.u64)) and
((this.window_size as base.u64) <= args.workbuf.length()) and
(distance > 0) {
k64 = 0
while k64 < (i as base.u64) {
n64 = (i as base.u64) ~mod- k64
n64 = n64.min(no_more_than: distance as base.u64)
dlo = dst_i ~mod+ k64
dhi = dlo ~mod+ n64
slo = src_i ~mod+ k64
shi = slo ~mod+ n64
if (n64 == 0) or (dlo > dhi) or (dhi > args.workbuf.length()) or
(slo > shi) or (shi > args.workbuf.length()) {
break
}
args.workbuf[dlo .. dhi].copy_from_slice!(s: args.workbuf[slo .. shi])
k64 ~mod+= n64
}
// k64 == i, unless the (unreachable) break above was taken,
// in which case the loop below copies the remaining bytes.
this.pos ~mod+= (k64 & 0xFFFF_FFFF) as base.u32
i ~mod-= (k64 & 0xFFFF_FFFF) as base.u32
}
while i > 0 {
b = this.get_byte(workbuf: args.workbuf, distance: distance)
this.put_byte!(workbuf: args.workbuf, b: b)
@@ -614,6 +688,7 @@ pri func decoder.process_commands?(src: base.io_reader, workbuf: slice base.u8,
}
}
this.cmd_state = 0
if (this.mb_remaining == 0) or (this.mb_remaining >= 0x8000_0000) {
// BROTLI_STATE_METABLOCK_DONE.
return ok
@@ -621,6 +696,323 @@ pri func decoder.process_commands?(src: base.io_reader, workbuf: slice base.u8,
}
}
// process_commands_fast is a fast path for process_commands. It is not a
// coroutine. It handles commands, or parts of commands (see
// decoder.cmd_state), for as long as args.src holds enough bytes and the work
// needs no block switch, ring buffer event or dictionary reference. Then it
// returns, and process_commands picks up where it left off.
//
// It reads ahead, up to 8 bytes at a time, into a 64-bit bit buffer. When it
// returns, it gives back (undoes) the whole bytes left in that buffer. The
// coroutines read_symbol and read_bits read an input byte only when they
// need more bits, so that between reads, fewer than 8 bits are buffered.
// Giving back the whole bytes restores that state exactly, so this function
// reads the same bits and leaves the decoder in the same state as
// process_commands would, and it writes the same bytes and returns the same
// errors. It sets this.cmd_state to 4 when the metablock is done.
pri func decoder.process_commands_fast!(src: base.io_reader, workbuf: slice base.u8) base.status {
var ret : base.status
var bits : base.u64
var n_bits : base.u32[..= 63]
var i : base.u32
var e : base.u32
var e_bits : base.u32[..= 23]
var cmd_code : base.u32[..= 703]
var nb : base.u32[..= 24]
var p1 : base.u32[..= 0xFF]
var p2 : base.u32[..= 0xFF]
var mode_offset : base.u32[..= 0x600]
var ctx : base.u32[..= 63]
var tree : base.u32[..= 0xFF]
var code : base.u32
var nbits : base.u32[..= 32]
var lo : base.u32
var max_distance : base.u32
var distance : base.u32
var n : base.u32
var dst_i : base.u64
var src_i : base.u64
var k64 : base.u64
var n64 : base.u64
var dlo : base.u64
var dhi : base.u64
var slo : base.u64
var shi : base.u64
if this.n_bits >= 8 {
return ok
}
n_bits = this.n_bits & 7
bits = (this.bits as base.u64) & (((1 as base.u64) << n_bits) - 1)
while.outer true {
if this.cmd_state == 0 {
// Two refills, of at most 8 bytes each.
if (this.block_length[1] == 0) or (args.src.length() < 16) {
break.outer
}
bits |= args.src.peek_u64le() ~mod<< n_bits
args.src.skip_u32_fast!(actual: (63 - n_bits) >> 3, worst_case: 8)
n_bits |= 56
// Mirrors ReadCommandInternal.
i = (CMD_HUFFS_BASE + (this.htree_command * CMD_HUFFS_STRIDE)) ~mod+ ((bits & 0xFF) as base.u32)
if i >= HUFFS_LEN {
ret = "#internal error: inconsistent Huffman table"
break.outer
}
e = this.huffs[i] as base.u32
e_bits = e & 15
if e_bits > 8 {
i = ((CMD_HUFFS_BASE + (this.htree_command * CMD_HUFFS_STRIDE)) ~mod+ (e >> 4)) ~mod+ ((((bits >> 8) & 0x7FFF) as base.u32) & (((1 as base.u32) << (e_bits - 8)) - 1))
if i >= HUFFS_LEN {
ret = "#internal error: inconsistent Huffman table"
break.outer
}
e = this.huffs[i] as base.u32
e_bits = (e & 15) + 8
}
if n_bits < e_bits {
ret = "#internal error: inconsistent bit reader"
break.outer
}
bits >>= e_bits
n_bits -= e_bits
if (e >> 4) >= 704 {
ret = "#internal error: inconsistent Huffman table"
break.outer
}
this.symbol = e >> 4
cmd_code = this.symbol.min(no_more_than: 703)
this.implicit_zero_distance = CMD_IMPLICIT_ZERO_DISTANCE[cmd_code] <> 0
this.distance_context = CMD_DISTANCE_CONTEXT[cmd_code] as base.u32
this.update_dist_htree_index!()
nb = CMD_INSERT_LEN_EXTRA_BITS[cmd_code] as base.u32
if n_bits < nb {
ret = "#internal error: inconsistent bit reader"
break.outer
}
this.bits_value = ((bits & (((1 as base.u64) << nb) - 1)) & 0xFFFF_FFFF) as base.u32
bits >>= nb
n_bits -= nb
this.insert_length = CMD_INSERT_LEN_OFFSET[cmd_code] ~mod+ this.bits_value
if args.src.length() < 8 {
ret = "#internal error: inconsistent bit reader"
break.outer
}
bits |= args.src.peek_u64le() ~mod<< n_bits
args.src.skip_u32_fast!(actual: (63 - n_bits) >> 3, worst_case: 8)
n_bits |= 56
nb = CMD_COPY_LEN_EXTRA_BITS[cmd_code] as base.u32
if n_bits < nb {
ret = "#internal error: inconsistent bit reader"
break.outer
}
this.bits_value = ((bits & (((1 as base.u64) << nb) - 1)) & 0xFFFF_FFFF) as base.u32
bits >>= nb
n_bits -= nb
this.copy_length = CMD_COPY_LEN_OFFSET[cmd_code] ~mod+ this.bits_value
this.block_length[1] ~mod-= 1
if this.insert_length > 0 {
this.mb_remaining ~mod-= this.insert_length
this.cmd_state = 1
} else {
this.cmd_state = 2
}
}
if this.cmd_state == 1 {
// CommandInner. Read the literals in the command. p1 and p2 are
// the last two bytes written to the ring buffer (put_byte always
// writes when the physical ring buffer fits in args.workbuf).
if (this.window_size as base.u64) > args.workbuf.length() {
break.outer
}
p1 = this.get_byte(workbuf: args.workbuf, distance: 1) as base.u32
p2 = this.get_byte(workbuf: args.workbuf, distance: 2) as base.u32
while this.insert_length > 0 {
if (this.block_length[0] == 0) or ((this.pos ~mod+ 1) >= this.rb_size) {
break.outer
}
if n_bits < 24 {
if args.src.length() < 8 {
break.outer
}
bits |= args.src.peek_u64le() ~mod<< n_bits
args.src.skip_u32_fast!(actual: (63 - n_bits) >> 3, worst_case: 8)
n_bits |= 56
}
mode_offset = this.context_mode << 9
ctx = (CONTEXT_LOOKUP[mode_offset | p1] |
CONTEXT_LOOKUP[mode_offset | 0x100 | p2]) as base.u32
tree = this.context_maps[this.context_map_slice + ctx] as base.u32
i = (LIT_HUFFS_BASE + (tree * LIT_HUFFS_STRIDE)) ~mod+ ((bits & 0xFF) as base.u32)
if i >= HUFFS_LEN {
ret = "#internal error: inconsistent Huffman table"
break.outer
}
e = this.huffs[i] as base.u32
e_bits = e & 15
if e_bits > 8 {
i = ((LIT_HUFFS_BASE + (tree * LIT_HUFFS_STRIDE)) ~mod+ (e >> 4)) ~mod+ ((((bits >> 8) & 0x7FFF) as base.u32) & (((1 as base.u32) << (e_bits - 8)) - 1))
if i >= HUFFS_LEN {
ret = "#internal error: inconsistent Huffman table"
break.outer
}
e = this.huffs[i] as base.u32
e_bits = (e & 15) + 8
}
if n_bits < e_bits {
ret = "#internal error: inconsistent bit reader"
break.outer
}
bits >>= e_bits
n_bits -= e_bits
this.symbol = e >> 4
this.put_byte!(workbuf: args.workbuf, b: (this.symbol & 0xFF) as base.u8)
p2 = p1
p1 = this.symbol & 0xFF
this.block_length[0] ~mod-= 1
this.insert_length ~mod-= 1
}
this.cmd_state = 2
if (this.mb_remaining == 0) or (this.mb_remaining >= 0x8000_0000) {
this.cmd_state = 4
break.outer
}
}
if this.cmd_state == 2 {
// CommandPostDecodeLiterals.
if this.implicit_zero_distance {
this.distance_context = 1
this.dist_rb_idx ~mod-= 1
this.distance_code = this.dist_rb[this.dist_rb_idx & 3]
} else {
// One refill: 56 bits suffice for a symbol and 32 extra bits.
if (this.block_length[2] == 0) or (args.src.length() < 8) {
break.outer
}
bits |= args.src.peek_u64le() ~mod<< n_bits
args.src.skip_u32_fast!(actual: (63 - n_bits) >> 3, worst_case: 8)
n_bits |= 56
// Mirrors read_distance.
i = (DIST_HUFFS_BASE + (this.dist_htree_index * DIST_HUFFS_STRIDE)) ~mod+ ((bits & 0xFF) as base.u32)
if i >= HUFFS_LEN {
ret = "#internal error: inconsistent Huffman table"
break.outer
}
e = this.huffs[i] as base.u32
e_bits = e & 15
if e_bits > 8 {
i = ((DIST_HUFFS_BASE + (this.dist_htree_index * DIST_HUFFS_STRIDE)) ~mod+ (e >> 4)) ~mod+ ((((bits >> 8) & 0x7FFF) as base.u32) & (((1 as base.u32) << (e_bits - 8)) - 1))
if i >= HUFFS_LEN {
ret = "#internal error: inconsistent Huffman table"
break.outer
}
e = this.huffs[i] as base.u32
e_bits = (e & 15) + 8
}
if n_bits < e_bits {
ret = "#internal error: inconsistent bit reader"
break.outer
}
bits >>= e_bits
n_bits -= e_bits
this.symbol = e >> 4
code = this.symbol
this.block_length[2] ~mod-= 1
this.distance_context = 0
if code < 16 {
this.take_distance_from_ring_buffer!(code: code)
} else {
if (code >= this.distance_alphabet_size) or (code >= 1128) {
ret = "#internal error: inconsistent Huffman table"
break.outer
}
nbits = this.dist_extra_bits[code].min(no_more_than: 32) as base.u32
if n_bits < nbits {
ret = "#internal error: inconsistent bit reader"
break.outer
}
// Like read_bits32, this is (lo | (hi << 16)) when nbits
// is more than 24.
lo = ((bits & (((1 as base.u64) << nbits) - 1)) & 0xFFFF_FFFF) as base.u32
this.bits_value = lo
bits >>= nbits
n_bits -= nbits
this.distance_code = (((this.dist_offset[code] as base.u64) ~mod+
((this.bits_value as base.u64) ~mod<< this.distance_postfix_bits)) & 0xFFFF_FFFF) as base.u32
}
}
this.cmd_state = 3
}
// this.cmd_state == 3. Only handle LZ77 copies that need no ring
// buffer event and do not wrap around the physical ring buffer.
max_distance = this.max_distance
if max_distance <> this.max_backward_distance {
max_distance = this.pos.min(no_more_than: this.max_backward_distance)
}
distance = this.distance_code
n = this.copy_length
if (distance > max_distance) or (distance == 0) {
break.outer
}
dst_i = this.pos as base.u64
src_i = ((this.pos ~mod- distance) & (this.window_size ~mod- 1)) as base.u64
if ((dst_i + (n as base.u64)) >= (this.rb_size as base.u64)) or
((src_i + (n as base.u64)) > (this.window_size as base.u64)) or
((this.window_size as base.u64) > args.workbuf.length()) {
break.outer
}
this.max_distance = max_distance
this.dist_rb[this.dist_rb_idx & 3] = distance
this.dist_rb_idx ~mod+= 1
this.mb_remaining ~mod-= n
// Copy in chunks of at most distance bytes, like process_commands.
k64 = 0
while k64 < (n as base.u64) {
n64 = (n as base.u64) ~mod- k64
n64 = n64.min(no_more_than: distance as base.u64)
dlo = dst_i ~mod+ k64
dhi = dlo ~mod+ n64
slo = src_i ~mod+ k64
shi = slo ~mod+ n64
if (n64 == 0) or (dlo > dhi) or (dhi > args.workbuf.length()) or
(slo > shi) or (shi > args.workbuf.length()) {
break
}
args.workbuf[dlo .. dhi].copy_from_slice!(s: args.workbuf[slo .. shi])
k64 ~mod+= n64
}
this.pos ~mod+= (k64 & 0xFFFF_FFFF) as base.u32
n ~mod-= (k64 & 0xFFFF_FFFF) as base.u32
// n == 0, unless the (unreachable) break above was taken. This.pos
// stays below this.rb_size, so there is no ring buffer event.
while n > 0 {
this.put_byte!(workbuf: args.workbuf, b: this.get_byte(workbuf: args.workbuf, distance: distance))
n -= 1
}
this.cmd_state = 0
if (this.mb_remaining == 0) or (this.mb_remaining >= 0x8000_0000) {
this.cmd_state = 4
break.outer
}
}.outer
// Give back the whole bytes left in the bit buffer.
while n_bits >= 8 {
if not args.src.can_undo_byte() {
return "#internal error: inconsistent bit reader"
}
args.src.undo_byte!()
n_bits -= 8
}
this.bits = ((bits & (((1 as base.u64) << n_bits) - 1)) & 0xFFFF_FFFF) as base.u32
this.n_bits = n_bits
return ret
}
// update_dist_htree_index sets this.dist_htree_index from the distance
// context map. Mirrors part of ReadCommandInternal and
// DecodeDistanceBlockSwitch in decode.c.
@@ -631,10 +1023,15 @@ pri func decoder.update_dist_htree_index!() {
// Mirrors ReadDistanceInternal in decode.c.
pri func decoder.read_distance?(src: base.io_reader) {
var code : base.u32
var code : base.u32
var fr : base.u32
var nbits : base.u32[..= 32]
this.read_symbol?(src: args.src,
hoff: DIST_HUFFS_BASE + (this.dist_htree_index * DIST_HUFFS_STRIDE))
fr = this.read_symbol_fast!(src: args.src, hoff: DIST_HUFFS_BASE + (this.dist_htree_index * DIST_HUFFS_STRIDE))
if fr <> 0 {
this.read_symbol?(src: args.src,
hoff: DIST_HUFFS_BASE + (this.dist_htree_index * DIST_HUFFS_STRIDE))
}
code = this.symbol
this.block_length[2] ~mod-= 1
// Convert the distance code to the actual distance by possibly looking up
@@ -649,7 +1046,14 @@ pri func decoder.read_distance?(src: base.io_reader) {
}
// The C code uses BrotliSafeReadBits32, as large-window brotli can have
// more than 24 extra bits.
this.read_bits32?(src: args.src, n: this.dist_extra_bits[code].min(no_more_than: 32) as base.u32)
nbits = this.dist_extra_bits[code].min(no_more_than: 32) as base.u32
fr = 1
if nbits <= 24 {
fr = this.read_bits_fast!(src: args.src, n: nbits)
}
if fr <> 0 {
this.read_bits32?(src: args.src, n: nbits)
}
if code >= 1128 {
return "#internal error: inconsistent Huffman table"
}
+47 -43
View File
@@ -12,8 +12,8 @@
// This file mirrors bit_cost.c, bit_cost_inc.h and fast_log.h. The functions
// are methods of the bit_cost struct (whose only state is the stateless
// softfloat arith), so that other structs (the encoder, the
// metablock_builder) can embed one with a single field.
// softfloat arith and a lazily filled FastLog2 cache), so that other structs
// (the encoder, the metablock_builder) can embed one with a single field.
//
// Doubles are IEEE 754 binary64 bit patterns (base.u64). A uint32_t array
// (e.g. a histogram's data_) is a slice of little-endian bytes, because
@@ -48,43 +48,23 @@ pri const F64_HUGE_VAL : base.u64 = 0x7FF0_0000_0000_0000
// F64_1E99 is the C code's 1e99.
pri const F64_1E99 : base.u64 = 0x547D_42AE_A287_9F2E
// LOG2_CACHE_SIZE is the size of the bit_cost.log2_cache table.
pri const LOG2_CACHE_SIZE : base.u64 = 65536
pri struct bit_cost?(
// log2_cache_ready is whether log2_cache has been filled (by
// init_log2_cache).
log2_cache_ready : base.bool,
// Bit (v & 63) of log2_valid[v >> 6] is whether log2_cache[v] holds
// FastLog2(v). This bitmap is zeroed when the struct is initialized
// (but log2_cache need not be), so the cache starts empty.
log2_valid : array[1024] base.u64,
sf : softfloat.arith,
) + (
// log2_cache[v] is FastLog2(v), for v < LOG2_CACHE_SIZE. It is only a
// speed optimization (glibc's log2 is expensive to emulate): the
// values are computed by the same functions, so the results are
// identical with or without it.
// log2_cache[v] is FastLog2(v), for 256 <= v < 65536, filled lazily
// (see fast_log2_memo). It is only a speed optimization (glibc's log2
// is expensive to emulate): the values are computed by the same
// functions, so the results are identical with or without it. Few of
// the 65536 entries are typically needed, so filling it lazily is
// much cheaper than filling it all up front.
log2_cache : array[65536] base.u64,
)
// init_log2_cache fills the log2_cache table (if it is not already filled).
// Calling it is optional, but it makes fast_log2 faster.
pri func bit_cost.init_log2_cache!() {
var v : base.u64
if this.log2_cache_ready {
return nothing
}
v = 0
while v < 256 {
this.log2_cache[v] = LOG2_TABLE[v]
v += 1
}
while v < 65536 {
this.log2_cache[v] = this.sf.log2_u64(v: v)
v += 1
}
this.log2_cache_ready = true
}
// ld32 returns the u32le at s[i ..], or 0 if that is out of bounds (which
// the callers never do).
pri func bit_cost.ld32(s: roslice base.u8, i: base.u64) base.u32 {
@@ -138,23 +118,41 @@ pri func bit_cost.st64!(s: slice base.u8, i: base.u64, v: base.u64) {
}
// Mirrors FastLog2 in fast_log.h. The result is the bit pattern of an IEEE
// 754 binary64 value.
// 754 binary64 value. It uses (but does not fill) log2_cache: see
// fast_log2_memo.
pri func bit_cost.fast_log2(v: base.u64) base.u64 {
if args.v < 256 {
return LOG2_TABLE[args.v]
} else if this.log2_cache_ready and (args.v < 65536) {
return this.log2_cache[args.v]
} else if args.v < 65536 {
if ((this.log2_valid[args.v >> 6] >> (args.v & 63)) & 1) <> 0 {
return this.log2_cache[args.v]
}
}
return this.sf.log2_u64(v: args.v)
}
// fast_log2_memo is like fast_log2 (it also mirrors FastLog2) but it fills
// the log2_cache entry for v (if v is in the cache's range).
pri func bit_cost.fast_log2_memo!(v: base.u64) base.u64 {
if args.v < 256 {
return LOG2_TABLE[args.v]
} else if args.v >= 65536 {
return this.sf.log2_u64(v: args.v)
} else if ((this.log2_valid[args.v >> 6] >> (args.v & 63)) & 1) == 0 {
this.log2_cache[args.v] = this.sf.log2_u64(v: args.v)
this.log2_valid[args.v >> 6] |= (1 as base.u64) << (args.v & 63)
}
return this.log2_cache[args.v]
}
// Mirrors BrotliBitsEntropy in bit_cost.c. population holds (at least) size
// u32le values. The C code's "goto odd_number_of_elements_left" only changes
// the loop's unrolling, not the order of the operations.
pri func bit_cost.bits_entropy(population: roslice base.u8, size: base.u64) base.u64 {
pri func bit_cost.bits_entropy!(population: roslice base.u8, size: base.u64) base.u64 {
var sum : base.u64
var retval : base.u64
var p : base.u64
var lp : base.u64
var q : roslice base.u8
var n : base.u64
@@ -168,12 +166,14 @@ pri func bit_cost.bits_entropy(population: roslice base.u8, size: base.u64) base
q = q[4 ..]
}
sum ~mod+= p
lp = this.fast_log2_memo!(v: p)
retval = this.sf.f64_sub(a: retval, b: this.sf.f64_mul(
a: this.sf.f64_from_u64(a: p), b: this.fast_log2(v: p)))
a: this.sf.f64_from_u64(a: p), b: lp))
}
if sum <> 0 {
lp = this.fast_log2_memo!(v: sum)
retval = this.sf.f64_add(a: retval, b: this.sf.f64_mul(
a: this.sf.f64_from_u64(a: sum), b: this.fast_log2(v: sum)))
a: this.sf.f64_from_u64(a: sum), b: lp))
}
if this.sf.f64_lt(a: retval, b: this.sf.f64_from_u64(a: sum)) <> 0 {
retval = this.sf.f64_from_u64(a: sum)
@@ -184,7 +184,7 @@ pri func bit_cost.bits_entropy(population: roslice base.u8, size: base.u64) base
// Mirrors BrotliPopulationCostLiteral, BrotliPopulationCostCommand and
// BrotliPopulationCostDistance (FN(BrotliPopulationCost) in bit_cost_inc.h).
// histogram is a histogram of data_size symbols (see the layout above).
pri func bit_cost.population_cost(histogram: roslice base.u8, data_size: base.u32[..= 704]) base.u64 {
pri func bit_cost.population_cost!(histogram: roslice base.u8, data_size: base.u32[..= 704]) base.u64 {
var total : base.u64
var count : base.u32
var s : array[4] base.u32
@@ -203,6 +203,7 @@ pri func bit_cost.population_cost(histogram: roslice base.u8, data_size: base.u3
var depth_histo : array[18] base.u32
var log2total : base.u64
var log2p : base.u64
var lp : base.u64
var depth : base.u64
var reps : base.u32
var sum : base.u64
@@ -284,12 +285,13 @@ pri func bit_cost.population_cost(histogram: roslice base.u8, data_size: base.u3
// Compute the entropy of the histogram and simultaneously build a
// simplified histogram of the code length codes.
max_depth = 1
log2total = this.fast_log2(v: total)
log2total = this.fast_log2_memo!(v: total)
i = 0
while i < args.data_size {
c = this.ld32(s: args.histogram, i: (i as base.u64) * 4)
if c > 0 {
log2p = this.sf.f64_sub(a: log2total, b: this.fast_log2(v: c as base.u64))
lp = this.fast_log2_memo!(v: c as base.u64)
log2p = this.sf.f64_sub(a: log2total, b: lp)
depth = this.sf.f64_to_u64_trunc(a: this.sf.f64_add(a: log2p, b: 0x3FE0_0000_0000_0000))
bits = this.sf.f64_add(a: bits, b: this.sf.f64_mul(
a: this.sf.f64_from_u32(a: c), b: log2p))
@@ -340,13 +342,15 @@ pri func bit_cost.population_cost(histogram: roslice base.u8, data_size: base.u3
while i < 18 {
p = depth_histo[i] as base.u64
sum ~mod+= p
lp = this.fast_log2_memo!(v: p)
retval = this.sf.f64_sub(a: retval, b: this.sf.f64_mul(
a: this.sf.f64_from_u64(a: p), b: this.fast_log2(v: p)))
a: this.sf.f64_from_u64(a: p), b: lp))
i ~mod+= 1
}
if sum <> 0 {
lp = this.fast_log2_memo!(v: sum)
retval = this.sf.f64_add(a: retval, b: this.sf.f64_mul(
a: this.sf.f64_from_u64(a: sum), b: this.fast_log2(v: sum)))
a: this.sf.f64_from_u64(a: sum), b: lp))
}
if this.sf.f64_lt(a: retval, b: this.sf.f64_from_u64(a: sum)) <> 0 {
retval = this.sf.f64_from_u64(a: sum)
+11 -8
View File
@@ -31,12 +31,15 @@ pri const HISTOGRAMS_PER_BATCH : base.u64 = 64
pri const CLUSTERS_PER_BATCH : base.u64 = 16
// Mirrors BitCost in block_splitter.c.
pri func metablock_builder.bit_cost(count: base.u64) base.u64 {
pri func metablock_builder.bit_cost!(count: base.u64) base.u64 {
var c : base.u64
if args.count == 0 {
// -2.0
return 0xC000_0000_0000_0000
}
return this.bc.fast_log2(v: args.count)
c = this.bc.fast_log2_memo!(v: args.count)
return c
}
// Mirrors FN(HistogramAddVector) in histogram_inc.h, adding the n symbols at
@@ -314,7 +317,7 @@ pri func metablock_builder.find_blocks!(wb: slice base.u8, data: base.u64, lengt
i = 0
while i < args.num_histograms {
// FastLog2((uint32_t)histograms[i].total_count_).
this.insert_cost[i & 255] = this.bc.fast_log2(
this.insert_cost[i & 255] = this.bc.fast_log2_memo!(
v: this.h_total(wb: args.wb, h: args.histograms ~mod+ (i ~mod* hs), ds: args.ds) & 0xFFFF_FFFF)
i ~mod+= 1
}
@@ -326,9 +329,8 @@ pri func metablock_builder.find_blocks!(wb: slice base.u8, data: base.u64, lengt
while j < args.num_histograms {
k = (i ~mod* args.num_histograms) ~mod+ j
if k < 35200 {
this.insert_cost[k] = this.sf.f64_sub(
a: this.insert_cost[j & 255],
b: this.bit_cost(count: this.ld32(wb: args.wb, i: (args.histograms ~mod+ (j ~mod* hs)) ~mod+ (i ~mod* 4)) as base.u64))
c = this.bit_cost!(count: this.ld32(wb: args.wb, i: (args.histograms ~mod+ (j ~mod* hs)) ~mod+ (i ~mod* 4)) as base.u64)
this.insert_cost[k] = this.sf.f64_sub(a: this.insert_cost[j & 255], b: c)
}
j ~mod+= 1
}
@@ -471,6 +473,7 @@ pri func metablock_builder.build_block_histograms!(wb: slice base.u8, data: base
// this.split_etc[cat].
pri func metablock_builder.cluster_blocks!(wb: slice base.u8, cat: base.u32[..= 2], data: base.u64, length: base.u64, width: base.u32,
num_blocks: base.u64, block_ids: base.u64, ds: base.u32[..= 704]) {
var c : base.u64
var hs : base.u64
var histogram_symbols : base.u64
var u32s : base.u64
@@ -562,8 +565,8 @@ pri func metablock_builder.cluster_blocks!(wb: slice base.u8, cat: base.u32[..=
pos ~mod+= 1
k ~mod+= 1
}
this.h_set_bit_cost!(wb: args.wb, h: histograms ~mod+ (j ~mod* hs), ds: args.ds,
bit_cost: this.h_population_cost(wb: args.wb, h: histograms ~mod+ (j ~mod* hs), ds: args.ds))
c = this.h_population_cost!(wb: args.wb, h: histograms ~mod+ (j ~mod* hs), ds: args.ds)
this.h_set_bit_cost!(wb: args.wb, h: histograms ~mod+ (j ~mod* hs), ds: args.ds, bit_cost: c)
this.st32!(wb: args.wb, i: new_clusters ~mod+ (j ~mod* 4), v: (j & 0xFFFF_FFFF) as base.u32)
this.st32!(wb: args.wb, i: symbols ~mod+ (j ~mod* 4), v: (j & 0xFFFF_FFFF) as base.u32)
this.st32!(wb: args.wb, i: sizes ~mod+ (j ~mod* 4), v: 1)
+8 -6
View File
@@ -108,7 +108,7 @@ pri func metablock_builder.compare_and_push_to_queue!(wb: slice base.u8, ds: bas
}
this.h_copy!(wb: args.wb, h: args.tmp, v: h1, ds: args.ds)
this.h_add_histogram!(wb: args.wb, h: args.tmp, v: h2, ds: args.ds)
c = this.h_population_cost(wb: args.wb, h: args.tmp, ds: args.ds)
c = this.h_population_cost!(wb: args.wb, h: args.tmp, ds: args.ds)
if this.sf.f64_lt(a: c, b: this.sf.f64_sub(a: threshold, b: cost_diff)) <> 0 {
cost_combo = c
is_good_pair = true
@@ -271,14 +271,15 @@ pri func metablock_builder.histogram_combine!(wb: slice base.u8, ds: base.u32[..
// Mirrors FN(BrotliHistogramBitCostDistance) in cluster_inc.h.
pri func metablock_builder.histogram_bit_cost_distance!(wb: slice base.u8, ds: base.u32[..= 704],
histogram: base.u64, candidate: base.u64, tmp: base.u64) base.u64 {
var c : base.u64
if this.h_total(wb: args.wb, h: args.histogram, ds: args.ds) == 0 {
return 0
}
this.h_copy!(wb: args.wb, h: args.tmp, v: args.histogram, ds: args.ds)
this.h_add_histogram!(wb: args.wb, h: args.tmp, v: args.candidate, ds: args.ds)
return this.sf.f64_sub(
a: this.h_population_cost(wb: args.wb, h: args.tmp, ds: args.ds),
b: this.h_bit_cost(wb: args.wb, h: args.candidate, ds: args.ds))
c = this.h_population_cost!(wb: args.wb, h: args.tmp, ds: args.ds)
return this.sf.f64_sub(a: c, b: this.h_bit_cost(wb: args.wb, h: args.candidate, ds: args.ds))
}
// Mirrors FN(BrotliHistogramRemap) in cluster_inc.h.
@@ -395,6 +396,7 @@ pri func metablock_builder.histogram_reindex!(wb: slice base.u8, ds: base.u32[..
// histogram_symbols. It returns the C code's *out_size.
pri func metablock_builder.cluster_histograms!(wb: slice base.u8, ds: base.u32[..= 704], in_histos: base.u64, in_size: base.u64,
max_histograms: base.u64, out_histos: base.u64, histogram_symbols: base.u64) base.u64 {
var c : base.u64
var hs : base.u64
var mark : base.u64
var cluster_size : base.u64
@@ -426,8 +428,8 @@ pri func metablock_builder.cluster_histograms!(wb: slice base.u8, ds: base.u32[.
i = 0
while i < args.in_size {
this.h_copy!(wb: args.wb, h: args.out_histos ~mod+ (i ~mod* hs), v: args.in_histos ~mod+ (i ~mod* hs), ds: args.ds)
this.h_set_bit_cost!(wb: args.wb, h: args.out_histos ~mod+ (i ~mod* hs), ds: args.ds,
bit_cost: this.h_population_cost(wb: args.wb, h: args.in_histos ~mod+ (i ~mod* hs), ds: args.ds))
c = this.h_population_cost!(wb: args.wb, h: args.in_histos ~mod+ (i ~mod* hs), ds: args.ds)
this.h_set_bit_cost!(wb: args.wb, h: args.out_histos ~mod+ (i ~mod* hs), ds: args.ds, bit_cost: c)
this.st32!(wb: args.wb, i: args.histogram_symbols ~mod+ (i ~mod* 4), v: (i & 0xFFFF_FFFF) as base.u32)
i ~mod+= 1
}
+4 -2
View File
@@ -277,7 +277,8 @@ pri func encoder.estimate_bit_costs_for_literals_utf8!(wb: slice base.u8, pos: b
} else if utf8_pos == 2 {
iw = iwu2
}
lit_cost = this.sf.f64_sub(a: this.bc.fast_log2(v: iw), b: this.bc.fast_log2(v: histo))
lit_cost = this.bc.fast_log2_memo!(v: iw)
lit_cost = this.sf.f64_sub(a: lit_cost, b: this.bc.fast_log2(v: histo))
lit_cost = this.sf.f64_add(a: lit_cost, b: LC_F64_0_02905)
this.lc_finish_cost!(wb: args.wb, cost_off: args.cost_off, i: i, lit_cost: lit_cost, utf8: true)
i ~mod+= 1
@@ -337,7 +338,8 @@ pri func encoder.estimate_bit_costs_for_literals!(wb: slice base.u8, pos: base.u
if histo == 0 {
histo = 1
}
lit_cost = this.sf.f64_sub(a: this.bc.fast_log2(v: in_window), b: this.bc.fast_log2(v: histo))
lit_cost = this.bc.fast_log2_memo!(v: in_window)
lit_cost = this.sf.f64_sub(a: lit_cost, b: this.bc.fast_log2(v: histo))
lit_cost = this.sf.f64_add(a: lit_cost, b: LC_F64_0_029)
this.lc_finish_cost!(wb: args.wb, cost_off: args.cost_off, i: i, lit_cost: lit_cost, utf8: false)
i ~mod+= 1
+7 -6
View File
@@ -264,9 +264,12 @@ pri func metablock_builder.h_set_bit_cost!(wb: slice base.u8, h: base.u64, ds: b
this.st64!(wb: args.wb, i: args.h ~mod+ (((args.ds as base.u64) * 4) + 8), v: args.bit_cost)
}
pri func metablock_builder.h_population_cost(wb: roslice base.u8, h: base.u64, ds: base.u32[..= 704]) base.u64 {
pri func metablock_builder.h_population_cost!(wb: roslice base.u8, h: base.u64, ds: base.u32[..= 704]) base.u64 {
var c : base.u64
if args.h <= args.wb.length() {
return this.bc.population_cost(histogram: args.wb[args.h ..], data_size: args.ds)
c = this.bc.population_cost!(histogram: args.wb[args.h ..], data_size: args.ds)
return c
}
return 0
}
@@ -482,9 +485,8 @@ pri func metablock_builder.compute_distance_cost!(wb: slice base.u8, cmds: rosli
}
i ~mod+= 1
}
this.dist_cost = this.sf.f64_add(
a: this.h_population_cost(wb: args.wb, h: args.tmp, ds: HISTOGRAM_DISTANCE_DATA_SIZE),
b: extra_bits)
this.dist_cost = this.h_population_cost!(wb: args.wb, h: args.tmp, ds: HISTOGRAM_DISTANCE_DATA_SIZE)
this.dist_cost = this.sf.f64_add(a: this.dist_cost, b: extra_bits)
return true
}
@@ -592,7 +594,6 @@ pri func metablock_builder.build_meta_block!(wb: slice base.u8, ringbuffer: rosl
var v : base.u32
this.reset_arena!(wb_len: args.wb.length())
this.bc.init_log2_cache!()
orig_npostfix = this.dist_postfix_bits
orig_ndirect = this.dist_num_direct_codes
+8 -5
View File
@@ -47,9 +47,12 @@ pri func encoder.gbs_histo_off(c: base.u32[..= 3], ix: base.u64) base.u64 {
// gbs_entropy returns BrotliBitsEntropy of the size u32le values at
// args.wb[off ..].
pri func encoder.gbs_entropy(wb: roslice base.u8, off: base.u64, size: base.u64) base.u64 {
pri func encoder.gbs_entropy!(wb: roslice base.u8, off: base.u64, size: base.u64) base.u64 {
var e : base.u64
if args.off <= args.wb.length() {
return this.bc.bits_entropy(population: args.wb[args.off ..], size: args.size)
e = this.bc.bits_entropy!(population: args.wb[args.off ..], size: args.size)
return e
}
return 0
}
@@ -162,7 +165,7 @@ pri func encoder.gbs_finish_block!(wb: slice base.u8, c: base.u32[..= 3], is_fin
this.set_byte!(s: args.wb, i: types_off, v: 0)
i = 0
while i < nc {
e = this.gbs_entropy(wb: args.wb, off: this.gbs_histo_off(c: args.c, ix: i), size: alphabet_size)
e = this.gbs_entropy!(wb: args.wb, off: this.gbs_histo_off(c: args.c, ix: i), size: alphabet_size)
this.gbs_last_entropy[((le_base as base.u64) ~mod+ i) & 127] = e
this.gbs_last_entropy[(((le_base as base.u64) ~mod+ nc) ~mod+ i) & 127] = e
i ~mod+= 1
@@ -185,7 +188,7 @@ pri func encoder.gbs_finish_block!(wb: slice base.u8, c: base.u32[..= 3], is_fin
last1 = this.gbs_last_histogram_ix[(args.c * 2) + 1]
i = 0
while i < nc {
e = this.gbs_entropy(wb: args.wb, off: this.gbs_histo_off(c: args.c, ix: curr ~mod+ i), size: alphabet_size)
e = this.gbs_entropy!(wb: args.wb, off: this.gbs_histo_off(c: args.c, ix: curr ~mod+ i), size: alphabet_size)
this.gbs_entropy_tmp[i & 15] = e
j = 0
while j < 2 {
@@ -197,7 +200,7 @@ pri func encoder.gbs_finish_block!(wb: slice base.u8, c: base.u32[..= 3], is_fin
} else {
this.gbs_add_histogram!(wb: args.wb, dst: tmp, src: this.gbs_histo_off(c: args.c, ix: last1 ~mod+ i), n: data_size)
}
this.gbs_comb_entropy[jx & 31] = this.gbs_entropy(wb: args.wb, off: tmp, size: alphabet_size)
this.gbs_comb_entropy[jx & 31] = this.gbs_entropy!(wb: args.wb, off: tmp, size: alphabet_size)
d = this.sf.f64_sub(
a: this.sf.f64_sub(a: this.gbs_comb_entropy[jx & 31], b: e),
b: this.gbs_last_entropy[((le_base as base.u64) ~mod+ jx) & 127])
+14 -8
View File
@@ -29,8 +29,11 @@ pub struct metablock_tester?(
)
// bits_entropy mirrors BrotliBitsEntropy. population holds size u32le values.
pub func metablock_tester.bits_entropy(population: roslice base.u8, size: base.u64) base.u64 {
return this.bc.bits_entropy(population: args.population, size: args.size)
pub func metablock_tester.bits_entropy!(population: roslice base.u8, size: base.u64) base.u64 {
var c : base.u64
c = this.bc.bits_entropy!(population: args.population, size: args.size)
return c
}
// population_cost mirrors BrotliPopulationCostLiteral (data_size 256),
@@ -38,13 +41,18 @@ pub func metablock_tester.bits_entropy(population: roslice base.u8, size: base.u
// histogram has the C struct's layout: data_size u32le counts, then the
// total_count_ u64le (and then the bit_cost_, which is ignored). It returns 0
// for any other data_size.
pub func metablock_tester.population_cost(histogram: roslice base.u8, data_size: base.u32) base.u64 {
pub func metablock_tester.population_cost!(histogram: roslice base.u8, data_size: base.u32) base.u64 {
var c : base.u64
if args.data_size == 256 {
return this.bc.population_cost(histogram: args.histogram, data_size: 256)
c = this.bc.population_cost!(histogram: args.histogram, data_size: 256)
return c
} else if args.data_size == 704 {
return this.bc.population_cost(histogram: args.histogram, data_size: 704)
c = this.bc.population_cost!(histogram: args.histogram, data_size: 704)
return c
} else if args.data_size == 544 {
return this.bc.population_cost(histogram: args.histogram, data_size: 544)
c = this.bc.population_cost!(histogram: args.histogram, data_size: 544)
return c
}
return 0
}
@@ -62,7 +70,6 @@ pub func metablock_tester.split_block!(cmds: roslice base.u8, num_commands: base
data: roslice base.u8, pos: base.u64, mask: base.u64, quality: base.u32, workbuf: slice base.u8) base.status {
this.mb.set_params!(quality: args.quality, large_window: false, disable_literal_context_modeling: false)
this.mb.reset_arena!(wb_len: args.workbuf.length())
this.mb.bc.init_log2_cache!()
this.mb.alloc_block_splits!(cmds: args.cmds, num_commands: args.num_commands)
if this.mb.oom {
return base."#bad workbuf length"
@@ -102,7 +109,6 @@ pub func metablock_tester.cluster_histograms!(data_size: base.u32, in_histos: ro
return 0xFFFF_FFFF_FFFF_FFFF
}
hs = this.mb.hsize(ds: ds)
this.mb.bc.init_log2_cache!()
this.mb.reset_arena!(wb_len: args.workbuf.length())
n = args.in_size ~mod* hs
in_off = this.mb.alloc!(n: n)
+61 -6
View File
@@ -96,12 +96,32 @@ integer long division, 11 quotient bits per (64-bit integer) division step,
with the final remainder as the sticky bit. Leading zero counts use a 256-entry
table.
The `f32` add, sub, mul and div operations convert their operands exactly to
`f64`, perform the `f64` operation and then round to `f32`. This double
rounding is innocuous: binary64's 53 bits of precision are more than `(2 * 24)
+ 2`, which suffices for `+`, `-`, `*` and `/` (S. A. Figueroa, "When is double
rounding innocuous?", ACM SIGNUM Newsletter, 1995). The tests confirm this
against hardware, including for subnormal `f32` results.
`f64_add` and `f64_sub` have a fast path for the common case where both
operands are normal (`f64_add_normal`): it orders the operands by magnitude,
aligns the significands with a sticky bit, adds or subtracts them and rounds
inline. Only a result that is subnormal falls back to the general code
(`f64_add_mags`, `f64_sub_mags`). Adding (or subtracting) a zero and a finite
non-zero value also returns early.
Likewise, `f32_add`, `f32_sub` and `f32_mul` compute the result directly in
binary32 (with 32-bit significands for add and sub and an exact 48-bit
product for mul) when both operands are normal and the result is normal (or,
for add and mul, overflows to infinity), and add and sub return early for a
zero plus a finite non-zero value. `f32_from_u32` converts integers below
`1 << 24` (which are exact) directly. Otherwise (zeroes, subnormal
operands or results, infinities and NaNs), they, and `f32_div` always,
convert their operands exactly to `f64`, perform the `f64` operation and then
round to `f32`. This double rounding is innocuous: binary64's 53 bits of
precision are more than `(2 * 24) + 2`, which suffices for `+`, `-`, `*` and
`/` (S. A. Figueroa, "When is double rounding innocuous?", ACM SIGNUM
Newsletter, 1995). The tests confirm this against hardware, including for
subnormal `f32` results. The fast paths never produce a NaN, so NaN results
(and their payloads) are exactly those of the "via `f64`" computation.
The comparison functions (`f64_cmp`, `f64_lt`, `f64_min`, etc. and their `f32`
counterparts) compare the bit patterns directly when neither operand is
negative or a NaN (when `a | b` is at most the bit pattern of `+inf`), and
otherwise call `f64_cmp_slow` or `f32_cmp_slow`.
## log2
@@ -187,3 +207,38 @@ threads):
(6 minutes 20 seconds wall clock in total)
No exception table is needed.
`script/manual-test-softfloat-sweep.c` systematically checks the `f32` add,
sub, mul and div operations over every pair of operand exponents (`[0 ..=
255]` squared), both signs and 40 x 40 mantissa patterns (exact results, ties,
carries, cancellation, subnormal results, overflow, NaN payloads): each result
must equal the hardware's (any NaN matching any NaN) and, bit-for-bit
including NaNs, the "via `f64`" composition `f64_to_f32(f64_op(f32_to_f64(a),
f32_to_f64(b)))`. It does the same for the `f64` add, sub, mul and div
operations (every pair of exponents in `[0 ..= 2047]`, 9 x 9 mantissa
patterns, NaN results checked bit-for-bit against the NaN rule above), and
checks `f64_to_f32` (326 exponents times 2²⁰ fraction patterns times both
signs), `f32_to_f64` (every `f32`), `f32_from_u32` and `f32_from_i32` (every
32-bit integer) and `f32_from_u64` (2³² structured values). It also checks
the `f32` and `f64` cmp, eq, ne, lt, le, gt, ge, min and max functions on
every pair of 594 values (both signs, zero, subnormal, normal and maximal
exponents, boundary mantissas and their neighbors, infinities and NaNs)
against the hardware's comparison operators. Result (x86-64, gcc 13 `-O2`, 4
threads):
f32_add 419430400 operand pairs, 0 mismatches
f32_sub 419430400 operand pairs, 0 mismatches
f32_mul 419430400 operand pairs, 0 mismatches
f32_div 419430400 operand pairs, 0 mismatches
f64_add 1358954496 operand pairs, 0 mismatches
f64_sub 1358954496 operand pairs, 0 mismatches
f64_mul 1358954496 operand pairs, 0 mismatches
f64_div 1358954496 operand pairs, 0 mismatches
f64_to_f32 341835776 operands, 0 mismatches
f32_to_f64 4294967296 operands, 0 mismatches
f32_from_u32 4294967296 operands, 0 mismatches
f32_from_i32 4294967296 operands, 0 mismatches
f32_from_u64 4294967296 operands, 0 mismatches
f32_compare 352836 operand pairs, 0 mismatches
f64_compare 352836 operand pairs, 0 mismatches
PASS: 0 mismatches in total
+197 -8
View File
@@ -61,6 +61,24 @@ pri func arith.clz64(a: base.u64) base.u32[..= 63] {
return (n32 + n16 + n8 + ((CLZ8[x >> 56] as base.u32) & 7)) & 63
}
// clz32 returns the number of leading zero bits of a, which must be non-zero.
pri func arith.clz32(a: base.u32) base.u32[..= 31] {
var x : base.u32
var n16 : base.u32[..= 16]
var n8 : base.u32[..= 8]
x = args.a
if (x >> 16) == 0 {
x = x ~mod<< 16
n16 = 16
}
if (x >> 24) == 0 {
x = x ~mod<< 8
n8 = 8
}
return (n16 + n8 + ((CLZ8[x >> 24] as base.u32) & 7)) & 31
}
// shift_right_jam64 returns (a >> dist), except that if any non-zero bits
// were shifted out then the least significant bit of the result is set. This
// "sticky bit" is needed to round correctly.
@@ -247,6 +265,21 @@ pub func arith.f64_abs(a: base.u64) base.u64 {
// f64_cmp compares a and b, returning 0 if (a < b), 1 if (a == b), 2 if (a
// > b) and 3 if they are unordered (either is a NaN). -0 and +0 compare equal.
pub func arith.f64_cmp(a: base.u64, b: base.u64) base.u32 {
if (args.a | args.b) <= F64_INF {
// Fast path: neither is negative (the sign bits are clear) or a NaN
// (both are at most +inf), so the bit patterns sort like the values.
if args.a < args.b {
return 0
} else if args.a == args.b {
return 1
}
return 2
}
return this.f64_cmp_slow(a: args.a, b: args.b)
}
// f64_cmp_slow is f64_cmp without the fast path.
pri func arith.f64_cmp_slow(a: base.u64, b: base.u64) base.u32 {
var ka : base.u64
var kb : base.u64
@@ -279,35 +312,65 @@ pub func arith.f64_cmp(a: base.u64, b: base.u64) base.u32 {
// true), and -0 equals +0.
pub func arith.f64_eq(a: base.u64, b: base.u64) base.u32 {
if this.f64_cmp(a: args.a, b: args.b) == 1 {
if (args.a | args.b) <= F64_INF {
if args.a == args.b {
return 1
}
return 0
}
if this.f64_cmp_slow(a: args.a, b: args.b) == 1 {
return 1
}
return 0
}
pub func arith.f64_ne(a: base.u64, b: base.u64) base.u32 {
if this.f64_cmp(a: args.a, b: args.b) == 1 {
if (args.a | args.b) <= F64_INF {
if args.a <> args.b {
return 1
}
return 0
}
if this.f64_cmp_slow(a: args.a, b: args.b) == 1 {
return 0
}
return 1
}
pub func arith.f64_lt(a: base.u64, b: base.u64) base.u32 {
if this.f64_cmp(a: args.a, b: args.b) == 0 {
if (args.a | args.b) <= F64_INF {
if args.a < args.b {
return 1
}
return 0
}
if this.f64_cmp_slow(a: args.a, b: args.b) == 0 {
return 1
}
return 0
}
pub func arith.f64_le(a: base.u64, b: base.u64) base.u32 {
if this.f64_cmp(a: args.a, b: args.b) <= 1 {
if (args.a | args.b) <= F64_INF {
if args.a <= args.b {
return 1
}
return 0
}
if this.f64_cmp_slow(a: args.a, b: args.b) <= 1 {
return 1
}
return 0
}
pub func arith.f64_gt(a: base.u64, b: base.u64) base.u32 {
if this.f64_cmp(a: args.a, b: args.b) == 2 {
if (args.a | args.b) <= F64_INF {
if args.a > args.b {
return 1
}
return 0
}
if this.f64_cmp_slow(a: args.a, b: args.b) == 2 {
return 1
}
return 0
@@ -316,7 +379,13 @@ pub func arith.f64_gt(a: base.u64, b: base.u64) base.u32 {
pub func arith.f64_ge(a: base.u64, b: base.u64) base.u32 {
var c : base.u32
c = this.f64_cmp(a: args.a, b: args.b)
if (args.a | args.b) <= F64_INF {
if args.a >= args.b {
return 1
}
return 0
}
c = this.f64_cmp_slow(a: args.a, b: args.b)
if (c == 1) or (c == 2) {
return 1
}
@@ -327,7 +396,13 @@ pub func arith.f64_ge(a: base.u64, b: base.u64) base.u32 {
// Note that this is not IEEE 754 minNum: f64_min(NaN, x) and f64_min(x, NaN)
// both return the second argument, and f64_min(-0, +0) returns +0.
pub func arith.f64_min(a: base.u64, b: base.u64) base.u64 {
if this.f64_cmp(a: args.a, b: args.b) == 0 {
if (args.a | args.b) <= F64_INF {
if args.a < args.b {
return args.a
}
return args.b
}
if this.f64_cmp_slow(a: args.a, b: args.b) == 0 {
return args.a
}
return args.b
@@ -335,7 +410,13 @@ pub func arith.f64_min(a: base.u64, b: base.u64) base.u64 {
// f64_max returns ((a > b) ? a : b), like brotli's BROTLI_MAX(double, a, b).
pub func arith.f64_max(a: base.u64, b: base.u64) base.u64 {
if this.f64_cmp(a: args.a, b: args.b) == 2 {
if (args.a | args.b) <= F64_INF {
if args.a > args.b {
return args.a
}
return args.b
}
if this.f64_cmp_slow(a: args.a, b: args.b) == 2 {
return args.a
}
return args.b
@@ -345,6 +426,21 @@ pub func arith.f64_max(a: base.u64, b: base.u64) base.u64 {
// f64_add returns (a + b).
pub func arith.f64_add(a: base.u64, b: base.u64) base.u64 {
var z : base.u64
if ((((args.a >> 52) & 0x7FF) ~mod- 1) < 0x7FE) and ((((args.b >> 52) & 0x7FF) ~mod- 1) < 0x7FE) {
// Both operands are normal: take the fast path.
z = this.f64_add_normal(a: args.a, b: args.b)
if z <> 0xFFFF_FFFF_FFFF_FFFF {
return z
}
} else if ((args.b & F64_ABS) == 0) and (((args.a & F64_ABS) ~mod- 1) < (F64_INF - 1)) {
// b is a zero and a is finite and non-zero, so (a + b) is a.
return args.a
} else if ((args.a & F64_ABS) == 0) and (((args.b & F64_ABS) ~mod- 1) < (F64_INF - 1)) {
// a is a zero and b is finite and non-zero, so (a + b) is b.
return args.b
}
if ((args.a ^ args.b) >> 63) == 0 {
return this.f64_add_mags(a: args.a, b: args.b, sign: args.a & F64_SIGN)
}
@@ -353,12 +449,105 @@ pub func arith.f64_add(a: base.u64, b: base.u64) base.u64 {
// f64_sub returns (a - b).
pub func arith.f64_sub(a: base.u64, b: base.u64) base.u64 {
var z : base.u64
if ((((args.a >> 52) & 0x7FF) ~mod- 1) < 0x7FE) and ((((args.b >> 52) & 0x7FF) ~mod- 1) < 0x7FE) {
// Both operands are normal: take the fast path. For non-NaN
// operands, (a - b) is exactly (a + -b), including the sign of a
// zero result.
z = this.f64_add_normal(a: args.a, b: args.b ^ F64_SIGN)
if z <> 0xFFFF_FFFF_FFFF_FFFF {
return z
}
} else if ((args.b & F64_ABS) == 0) and (((args.a & F64_ABS) ~mod- 1) < (F64_INF - 1)) {
// b is a zero and a is finite and non-zero, so (a - b) is a.
return args.a
} else if ((args.a & F64_ABS) == 0) and (((args.b & F64_ABS) ~mod- 1) < (F64_INF - 1)) {
// a is a zero and b is finite and non-zero, so (a - b) is -b.
return args.b ^ F64_SIGN
}
if ((args.a ^ args.b) >> 63) == 0 {
return this.f64_sub_mags(a: args.a, b: args.b, sign: args.a & F64_SIGN)
}
return this.f64_add_mags(a: args.a, b: args.b, sign: args.a & F64_SIGN)
}
// f64_add_normal returns (a + b), for normal a and b, unless the result is
// subnormal, in which case it returns 0xFFFF_FFFF_FFFF_FFFF (a NaN, which it
// never otherwise returns) and the caller must take its slow path. It is a
// faster (but less general) version of f64_add_mags and f64_sub_mags.
pri func arith.f64_add_normal(a: base.u64, b: base.u64) base.u64 {
var x : base.u64
var y : base.u64
var ex : base.u32[..= 0x800]
var d : base.u32
var mx : base.u64[..= 0x7FFF_FFFF_FFFF_FFFF]
var my : base.u64[..= 0x7FFF_FFFF_FFFF_FFFF]
var z : base.u64
var s : base.u32[..= 63]
var rb : base.u64
// Order the operands so that |x| >= |y|. The result has x's sign.
x = args.a
y = args.b
if (y & F64_ABS) > (x & F64_ABS) {
x = args.b
y = args.a
}
ex = ((x >> 52) & 0x7FF) as base.u32
d = ex ~mod- (((y >> 52) & 0x7FF) as base.u32)
if d > 63 {
d = 63
}
// The significands, with their implicit bit at bit 62 and 10 extra bits
// of precision at the bottom. Align my with mx, keeping a sticky bit.
mx = ((x & F64_FRAC) | F64_HIDDEN) << 10
my = ((y & F64_FRAC) | F64_HIDDEN) << 10
if (my & (((1 as base.u64) << (d & 63)) ~mod- 1)) <> 0 {
my = (my >> (d & 63)) | 1
} else {
my = my >> (d & 63)
}
if ((x ^ y) >> 63) == 0 {
z = mx + my
if z >= 0x8000_0000_0000_0000 {
ex += 1
z = (z >> 1) | (z & 1)
}
} else {
// mx >= my, since |x| >= |y|.
z = mx ~mod- my
if z == 0 {
// Exact cancellation gives +0 when rounding to nearest.
return 0
} else if z < 0x4000_0000_0000_0000 {
// If (d >= 2) then z >= (1 << 61) and the normalization shift
// is 1. Otherwise, no bits were shifted out of my.
s = (this.clz64(a: z) ~mod- 1) & 63
if s >= ex {
return 0xFFFF_FFFF_FFFF_FFFF
}
ex = (ex ~mod- s) & 0x7FF
z = z ~mod<< s
}
}
// Round to nearest, ties to even. z is now in [1 << 52 ..= 1 << 53] and
// ex in [1 ..= 0x7FF]. Adding z (whose bit 52 is the implicit bit) to the
// shifted exponent carries any rounding overflow into the exponent field.
rb = z & 0x3FF
z = (z ~mod+ 0x200) >> 10
if rb == 0x200 {
z &= 0xFFFF_FFFF_FFFF_FFFE
}
z = (((ex ~mod- 1) as base.u64) ~mod<< 52) ~mod+ z
if z >= F64_INF {
z = F64_INF
}
return (x & F64_SIGN) | z
}
// f64_add_mags returns (sign * (|a| + |b|)).
pri func arith.f64_add_mags(a: base.u64, b: base.u64, sign: base.u64) base.u64 {
var exp_a : base.u32
+324 -18
View File
@@ -10,12 +10,19 @@
// -------- f32 (binary32) operations.
//
// The f32 add, sub, mul and div functions convert their (exactly
// representable) operands to f64, perform the f64 operation and round the
// result to f32. Double rounding is innocuous here: binary64 has more than
// (2 * 24) + 2 bits of precision, which suffices for +, -, * and / (Figueroa,
// "When is double rounding innocuous?", 1995). This is also exhaustively
// cross-checked against hardware by the tests.
// When both operands are normal (and, unless it overflows to infinity, the
// result is normal too), the f32 add, sub and mul functions compute the
// result directly in binary32 (with 32-bit or 64-bit integer significands).
// This is the common case. Adding or subtracting a zero and a finite
// non-zero operand is also handled directly. Otherwise (zeroes, subnormals,
// infinities, NaNs and subnormal results), they and f32_div convert their
// (exactly representable) operands to f64, perform the f64 operation and
// round the result to f32 (the slow_etc functions). Double rounding is
// innocuous here: binary64 has more than (2 * 24) + 2 bits of precision,
// which suffices for +, -, * and / (Figueroa, "When is double rounding
// innocuous?", 1995). This is also exhaustively cross-checked against
// hardware by the tests. The fast paths never return a NaN, so NaN results
// (and their payloads) always come from the slow paths.
pri const F32_SIGN : base.u32 = 0x8000_0000
pri const F32_ABS : base.u32 = 0x7FFF_FFFF
@@ -74,20 +81,244 @@ pub func arith.f32_abs(a: base.u32) base.u32 {
// f32_add returns (a + b).
pub func arith.f32_add(a: base.u32, b: base.u32) base.u32 {
var z : base.u32
if ((((args.a >> 23) & 0xFF) ~mod- 1) < 0xFE) and ((((args.b >> 23) & 0xFF) ~mod- 1) < 0xFE) {
if ((args.a ^ args.b) >> 31) == 0 {
return this.f32_add_mags(a: args.a, b: args.b)
}
z = this.f32_sub_mags(a: args.a, b: args.b)
if z <> 0xFFFF_FFFF {
return z
}
} else if ((args.b & F32_ABS) == 0) and (((args.a & F32_ABS) ~mod- 1) < (F32_INF - 1)) {
// b is a zero and a is finite and non-zero, so (a + b) is a.
return args.a
} else if ((args.a & F32_ABS) == 0) and (((args.b & F32_ABS) ~mod- 1) < (F32_INF - 1)) {
// a is a zero and b is finite and non-zero, so (a + b) is b.
return args.b
}
return this.slow_f32_add(a: args.a, b: args.b)
}
// f32_sub returns (a - b).
pub func arith.f32_sub(a: base.u32, b: base.u32) base.u32 {
var z : base.u32
if ((((args.a >> 23) & 0xFF) ~mod- 1) < 0xFE) and ((((args.b >> 23) & 0xFF) ~mod- 1) < 0xFE) {
if ((args.a ^ args.b) >> 31) <> 0 {
return this.f32_add_mags(a: args.a, b: args.b)
}
z = this.f32_sub_mags(a: args.a, b: args.b)
if z <> 0xFFFF_FFFF {
return z
}
} else if ((args.b & F32_ABS) == 0) and (((args.a & F32_ABS) ~mod- 1) < (F32_INF - 1)) {
// b is a zero and a is finite and non-zero, so (a - b) is a.
return args.a
} else if ((args.a & F32_ABS) == 0) and (((args.b & F32_ABS) ~mod- 1) < (F32_INF - 1)) {
// a is a zero and b is finite and non-zero, so (a - b) is -b.
return args.b ^ F32_SIGN
}
return this.slow_f32_sub(a: args.a, b: args.b)
}
// f32_mul returns (a * b).
pub func arith.f32_mul(a: base.u32, b: base.u32) base.u32 {
var ea : base.u32[..= 0xFF]
var eb : base.u32[..= 0xFF]
var e : base.u32[..= 0x1FF]
var p : base.u64
var m : base.u32
var r : base.u32
var z : base.u32
ea = (args.a >> 23) & 0xFF
eb = (args.b >> 23) & 0xFF
if ((ea ~mod- 1) < 0xFE) and ((eb ~mod- 1) < 0xFE) {
// Both operands are normal. The exact product of the two 24-bit
// significands, p, is in [1 << 46, 1 << 48). The biased exponent of
// the result, e, is (ea + eb - 127) (plus 1 if p >= (1 << 47)).
p = (((args.a & 0x7F_FFFF) | 0x80_0000) as base.u64) *
(((args.b & 0x7F_FFFF) | 0x80_0000) as base.u64)
e = ea + eb
if p >= 0x8000_0000_0000 {
e += 1
m = ((p >> 24) & 0xFF_FFFF) as base.u32
r = ((p & 0xFF_FFFF) as base.u32) ~mod- 0x80_0000
} else {
m = ((p >> 23) & 0xFF_FFFF) as base.u32
r = ((p & 0x7F_FFFF) as base.u32) ~mod- 0x40_0000
}
// r is the remainder minus half an ULP, as a two's complement
// integer. Round to nearest, ties to even.
if (r <> 0) and (r < 0x8000_0000) {
m ~mod+= 1
} else if r == 0 {
m ~mod+= m & 1
}
if (e >= 128) and (e <= 381) {
// Adding m (whose bit 23 is the implicit bit) carries any
// rounding overflow into the exponent field.
z = ((e - 128) << 23) ~mod+ m
if z >= F32_INF {
z = F32_INF
}
return ((args.a ^ args.b) & F32_SIGN) | z
}
}
return this.slow_f32_mul(a: args.a, b: args.b)
}
// f32_add_mags returns (sign(a) * (|a| + |b|)), for normal a and b.
pri func arith.f32_add_mags(a: base.u32, b: base.u32) base.u32 {
var ea : base.u32[..= 0x100]
var eb : base.u32[..= 0xFF]
var ma : base.u32[..= 0x3FFF_FFC0]
var mb : base.u32[..= 0x3FFF_FFC0]
var d : base.u32[..= 0xFF]
var z : base.u32
var rb : base.u32
ea = (args.a >> 23) & 0xFF
eb = (args.b >> 23) & 0xFF
// The significands, with their implicit bit at bit 29 and 6 extra bits
// of precision at the bottom.
ma = ((args.a & 0x7F_FFFF) | 0x80_0000) << 6
mb = ((args.b & 0x7F_FFFF) | 0x80_0000) << 6
if ea < eb {
d = (eb ~mod- ea) & 0xFF
ea = eb
z = ma
ma = mb
mb = z
} else {
d = (ea ~mod- eb) & 0xFF
}
if d >= 31 {
mb = 1
} else if d > 0 {
if (mb ~mod<< (32 - d)) <> 0 {
mb = (mb >> d) | 1
} else {
mb = mb >> d
}
}
z = (ma & 0x3FFF_FFFF) + (mb & 0x3FFF_FFFF)
if z >= 0x4000_0000 {
ea += 1
z = (z >> 1) | (z & 1)
}
rb = z & 0x3F
z = (z ~mod+ 0x20) >> 6
if rb == 0x20 {
z &= 0xFFFF_FFFE
}
// ea is in [1 ..= 255] and z is in [1 << 23 ..= 1 << 24].
z = ((ea ~mod- 1) ~mod<< 23) ~mod+ z
if z >= F32_INF {
z = F32_INF
}
return (args.a & F32_SIGN) | z
}
// f32_sub_mags returns (sign(a) * (|a| - |b|)), for normal a and b, unless
// the result is subnormal, in which case it returns 0xFFFF_FFFF (a NaN, which
// it never otherwise returns) and the caller must take its slow path.
pri func arith.f32_sub_mags(a: base.u32, b: base.u32) base.u32 {
var sign : base.u32
var ea : base.u32[..= 0xFF]
var eb : base.u32[..= 0xFF]
var fa : base.u32[..= 0x7F_FFFF]
var fb : base.u32[..= 0x7F_FFFF]
var ma : base.u32
var mb : base.u32
var d : base.u32[..= 0xFF]
var s : base.u32[..= 31]
var z : base.u32
var rb : base.u32
sign = args.a & F32_SIGN
ea = (args.a >> 23) & 0xFF
eb = (args.b >> 23) & 0xFF
fa = args.a & 0x7F_FFFF
fb = args.b & 0x7F_FFFF
if ea == eb {
// The difference is exact.
if fa == fb {
// Exact cancellation gives +0 when rounding to nearest.
return 0
} else if fa < fb {
sign ^= F32_SIGN
z = fb ~mod- fa
} else {
z = fa ~mod- fb
}
// z is in [1 ..= 0x7F_FFFF]. Shift its top bit to bit 23.
s = (this.clz32(a: z) ~mod- 8) & 31
if s >= ea {
return 0xFFFF_FFFF
}
return sign | ((((ea ~mod- s) ~mod- 1) ~mod<< 23) ~mod+ (z ~mod<< s))
}
// The significands, with their implicit bit at bit 30 and 7 extra bits
// of precision at the bottom.
ma = (fa | 0x80_0000) << 7
mb = (fb | 0x80_0000) << 7
if ea < eb {
sign ^= F32_SIGN
d = (eb ~mod- ea) & 0xFF
ea = eb
z = ma
ma = mb
mb = z
} else {
d = (ea ~mod- eb) & 0xFF
}
if d >= 31 {
mb = 1
} else if (mb ~mod<< ((32 - d) & 31)) <> 0 {
mb = (mb >> d) | 1
} else {
mb = mb >> d
}
// ma > mb, so z is non-zero. If (d >= 2) then z >= (1 << 29) and the
// normalization shift is at most 1. Otherwise, no bits were shifted out.
z = ma ~mod- mb
s = (this.clz32(a: z) ~mod- 1) & 31
if s >= ea {
return 0xFFFF_FFFF
}
ea = (ea ~mod- s) & 0xFF
z = z ~mod<< s
rb = z & 0x7F
z = (z ~mod+ 0x40) >> 7
if rb == 0x40 {
z &= 0xFFFF_FFFE
}
// ea is in [1 ..= 254] and z is in [1 << 23 ..= 1 << 24], so this does
// not overflow.
return sign | (((ea ~mod- 1) ~mod<< 23) ~mod+ z)
}
// slow_f32_add returns (a + b), computed via f64.
pri func arith.slow_f32_add(a: base.u32, b: base.u32) base.u32 {
return this.f64_to_f32(a: this.f64_add(
a: this.f32_to_f64(a: args.a),
b: this.f32_to_f64(a: args.b)))
}
// f32_sub returns (a - b).
pub func arith.f32_sub(a: base.u32, b: base.u32) base.u32 {
// slow_f32_sub returns (a - b), computed via f64.
pri func arith.slow_f32_sub(a: base.u32, b: base.u32) base.u32 {
return this.f64_to_f32(a: this.f64_sub(
a: this.f32_to_f64(a: args.a),
b: this.f32_to_f64(a: args.b)))
}
// f32_mul returns (a * b).
pub func arith.f32_mul(a: base.u32, b: base.u32) base.u32 {
// slow_f32_mul returns (a * b), computed via f64.
pri func arith.slow_f32_mul(a: base.u32, b: base.u32) base.u32 {
return this.f64_to_f32(a: this.f64_mul(
a: this.f32_to_f64(a: args.a),
b: this.f32_to_f64(a: args.b)))
@@ -103,6 +334,21 @@ pub func arith.f32_div(a: base.u32, b: base.u32) base.u32 {
// f32_cmp compares a and b, returning 0 if (a < b), 1 if (a == b), 2 if (a
// > b) and 3 if they are unordered (either is a NaN). -0 and +0 compare equal.
pub func arith.f32_cmp(a: base.u32, b: base.u32) base.u32 {
if (args.a | args.b) <= F32_INF {
// Fast path: neither is negative (the sign bits are clear) or a NaN
// (both are at most +inf), so the bit patterns sort like the values.
if args.a < args.b {
return 0
} else if args.a == args.b {
return 1
}
return 2
}
return this.f32_cmp_slow(a: args.a, b: args.b)
}
// f32_cmp_slow is f32_cmp without the fast path.
pri func arith.f32_cmp_slow(a: base.u32, b: base.u32) base.u32 {
var ka : base.u32
var kb : base.u32
@@ -133,35 +379,65 @@ pub func arith.f32_cmp(a: base.u32, b: base.u32) base.u32 {
// true), and -0 equals +0.
pub func arith.f32_eq(a: base.u32, b: base.u32) base.u32 {
if this.f32_cmp(a: args.a, b: args.b) == 1 {
if (args.a | args.b) <= F32_INF {
if args.a == args.b {
return 1
}
return 0
}
if this.f32_cmp_slow(a: args.a, b: args.b) == 1 {
return 1
}
return 0
}
pub func arith.f32_ne(a: base.u32, b: base.u32) base.u32 {
if this.f32_cmp(a: args.a, b: args.b) == 1 {
if (args.a | args.b) <= F32_INF {
if args.a <> args.b {
return 1
}
return 0
}
if this.f32_cmp_slow(a: args.a, b: args.b) == 1 {
return 0
}
return 1
}
pub func arith.f32_lt(a: base.u32, b: base.u32) base.u32 {
if this.f32_cmp(a: args.a, b: args.b) == 0 {
if (args.a | args.b) <= F32_INF {
if args.a < args.b {
return 1
}
return 0
}
if this.f32_cmp_slow(a: args.a, b: args.b) == 0 {
return 1
}
return 0
}
pub func arith.f32_le(a: base.u32, b: base.u32) base.u32 {
if this.f32_cmp(a: args.a, b: args.b) <= 1 {
if (args.a | args.b) <= F32_INF {
if args.a <= args.b {
return 1
}
return 0
}
if this.f32_cmp_slow(a: args.a, b: args.b) <= 1 {
return 1
}
return 0
}
pub func arith.f32_gt(a: base.u32, b: base.u32) base.u32 {
if this.f32_cmp(a: args.a, b: args.b) == 2 {
if (args.a | args.b) <= F32_INF {
if args.a > args.b {
return 1
}
return 0
}
if this.f32_cmp_slow(a: args.a, b: args.b) == 2 {
return 1
}
return 0
@@ -170,7 +446,13 @@ pub func arith.f32_gt(a: base.u32, b: base.u32) base.u32 {
pub func arith.f32_ge(a: base.u32, b: base.u32) base.u32 {
var c : base.u32
c = this.f32_cmp(a: args.a, b: args.b)
if (args.a | args.b) <= F32_INF {
if args.a >= args.b {
return 1
}
return 0
}
c = this.f32_cmp_slow(a: args.a, b: args.b)
if (c == 1) or (c == 2) {
return 1
}
@@ -181,7 +463,13 @@ pub func arith.f32_ge(a: base.u32, b: base.u32) base.u32 {
// Note that this is not IEEE 754 minNum: f32_min(NaN, x) and f32_min(x, NaN)
// both return the second argument, and f32_min(-0, +0) returns +0.
pub func arith.f32_min(a: base.u32, b: base.u32) base.u32 {
if this.f32_cmp(a: args.a, b: args.b) == 0 {
if (args.a | args.b) <= F32_INF {
if args.a < args.b {
return args.a
}
return args.b
}
if this.f32_cmp_slow(a: args.a, b: args.b) == 0 {
return args.a
}
return args.b
@@ -189,7 +477,13 @@ pub func arith.f32_min(a: base.u32, b: base.u32) base.u32 {
// f32_max returns ((a > b) ? a : b), like brotli's BROTLI_MAX(float, a, b).
pub func arith.f32_max(a: base.u32, b: base.u32) base.u32 {
if this.f32_cmp(a: args.a, b: args.b) == 2 {
if (args.a | args.b) <= F32_INF {
if args.a > args.b {
return args.a
}
return args.b
}
if this.f32_cmp_slow(a: args.a, b: args.b) == 2 {
return args.a
}
return args.b
@@ -197,6 +491,18 @@ pub func arith.f32_max(a: base.u32, b: base.u32) base.u32 {
// f32_from_u32 returns the f32 nearest to the integer a (C's (float)a).
pub func arith.f32_from_u32(a: base.u32) base.u32 {
var c : base.u32[..= 31]
if args.a == 0 {
return 0
} else if args.a < 0x100_0000 {
// a is exactly representable. Shift its top bit to bit 23 (the
// implicit bit, which is then added to the exponent field).
c = this.clz32(a: args.a)
if c >= 8 {
return ((157 ~mod- c) ~mod<< 23) ~mod+ (args.a ~mod<< (c - 8))
}
}
// The u32 to f64 conversion is exact, so there is only one rounding.
return this.f64_to_f32(a: this.f64_from_u32(a: args.a))
}