mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-19 16:47:28 +01:00
std/brotli: speed up qualities 0 to 9 and the decoder
Encoder (output stays byte-exact with libbrotlienc v1.2.1): - H58/H68 (q5-6): match the tags 8 at a time (SWAR, like GetMatchingTagMask's non-SSE2 code) and visit the matches with a de Bruijn ctz; inline the bucket entry check into the H5/H6 and H58/H68 loops, caching the 4 bytes compared at best_len; a dedicated H58/H68 store and hash. - find_last_distances caches the current byte at best_len. - ctz64 and a branch-free log2_floor_non_zero (de Bruijn tables), and find_match_length_with_limit uses ctz. - load32le, load64le, peek_u16, poke_u16 and poke_u32 use one bounds check each. Decoder (same output and statuses): - The literal fast path keeps pos and the counts in local variables. - Copies of at most 16 bytes, with a distance of at least 16, copy 16 bytes (like the C code; the window gap keeps the extra bytes unread). script/manual-test-brotli-float.c also checks log2_floor_non_zero (every u32) and ctz64. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UZ978BikvkegL2JUkqoB1e
This commit is contained in:
8 files changed
+716
-377
No files matched your search
+16
-8
@@ -883,7 +883,9 @@ by both this package's decoder and libbrotlidec.
|
||||
`script/manual-test-brotli-float.c` checks the encoder's package-private
|
||||
float fast paths (`encode_float.wuffs`, which spare the hot loops the
|
||||
cross-package `std/softfloat` calls) against the C compiler's hardware float
|
||||
and double arithmetic, for operands biased towards their edge cases.
|
||||
and double arithmetic, for operands biased towards their edge cases, and
|
||||
checks the integer bit helpers (`log2_floor_non_zero` for every 32-bit value,
|
||||
and `ctz64`).
|
||||
`script/manual-test-brotli-encoder-parity.c` runs a larger grid over every
|
||||
file in `test/data/brotli`, some larger files in `test/data` (up to about
|
||||
400 KiB) and some synthetic inputs (up to 1 MiB). Its `-q=LO-HI` and
|
||||
@@ -1131,13 +1133,19 @@ metablock) for as long as it needs the results.
|
||||
- `workbuf_len` is a worst case (or, at qualities 10 and 11, a practical
|
||||
bound). A future option could take the input length (like the C code's
|
||||
size hint) to give a tighter bound.
|
||||
- Speed. Compressing the 3 MiB `test/data/harvesters.bmp` in one call (on
|
||||
x86-64, lgwin 22), the Wuffs encoder takes about 1.2 times libbrotlienc's
|
||||
time at quality 5, 1.7 times at quality 9, 4 times at quality 10 and 3.5
|
||||
times at quality 11 (on text, 2 to 3 times at qualities 10 and 11), mostly
|
||||
because of the soft float arithmetic in the Zopfli cost model, the block
|
||||
splitter and histogram clustering. Decoding takes about 1.2 times
|
||||
libbrotlidec's time.
|
||||
- Speed. On a mix of text, JSON, map data, an image and an ISO image (on
|
||||
x86-64, lgwin 22, one call each, reusing the workbuf), the Wuffs encoder
|
||||
takes about 1.2 to 1.8 times libbrotlienc's time at qualities 0 to 8 (about
|
||||
1.8 times at the qualities 4 to 6 that web servers use for on-the-fly
|
||||
compression) and about the same time at quality 9. At qualities 10 and 11
|
||||
it takes about 2 times libbrotlienc's time on text and 3.5 to 4 times on
|
||||
images (such as the 3 MiB `test/data/harvesters.bmp`), mostly because of
|
||||
the soft float arithmetic in the Zopfli cost model, the block splitter and
|
||||
histogram clustering. Decoding takes about 1.2 to 1.6 times libbrotlidec's
|
||||
time. The workbuf is sized for the window (about 48 to 62 MiB at lgwin 22
|
||||
and qualities 5 to 9), so callers that compress many small inputs should
|
||||
reuse it: allocating and first touching a fresh one for each call costs
|
||||
more than the compression of a small input.
|
||||
- Serialized (shared) dictionaries, which only an experimental
|
||||
(`BROTLI_EXPERIMENTAL`) libbrotlienc build supports.
|
||||
- Changing parameters mid-stream.
|
||||
|
||||
@@ -738,6 +738,13 @@ pri func decoder.process_commands_fast!(src: base.io_reader, workbuf: slice base
|
||||
var dhi : base.u64
|
||||
var slo : base.u64
|
||||
var shi : base.u64
|
||||
var cm_slice : base.u32[..= 16320]
|
||||
var wmask : base.u32
|
||||
var pos : base.u32
|
||||
var lit_n : base.u32
|
||||
var lit_i : base.u32
|
||||
var wi : base.u64
|
||||
var copied16 : base.bool
|
||||
|
||||
if this.n_bits >= 8 {
|
||||
return ok
|
||||
@@ -829,26 +836,38 @@ pri func decoder.process_commands_fast!(src: base.io_reader, workbuf: slice base
|
||||
}
|
||||
p1 = this.get_byte(workbuf: args.workbuf, distance: 1) as base.u32
|
||||
p2 = this.get_byte(workbuf: args.workbuf, distance: 2) as base.u32
|
||||
while this.insert_length > 0 {
|
||||
if (this.block_length[0] == 0) or ((this.pos ~mod+ 1) >= this.rb_size) {
|
||||
break.outer
|
||||
}
|
||||
mode_offset = this.context_mode << 9
|
||||
cm_slice = this.context_map_slice
|
||||
wmask = this.window_size ~mod- 1
|
||||
pos = this.pos
|
||||
// lit_n is how many literals the loop below decodes before this
|
||||
// command's literals, this block's literals or the ring buffer
|
||||
// (before the next ring_event) run out. The loop keeps pos, the
|
||||
// counts and the symbol in local variables, and writes them back
|
||||
// to this.etc fields before every break.
|
||||
lit_n = this.insert_length.min(no_more_than: this.block_length[0])
|
||||
if (pos ~mod+ 1) >= this.rb_size {
|
||||
lit_n = 0
|
||||
} else {
|
||||
lit_n = lit_n.min(no_more_than: (this.rb_size ~mod- pos) ~mod- 1)
|
||||
}
|
||||
lit_i = 0
|
||||
while lit_i < lit_n {
|
||||
if n_bits < 24 {
|
||||
if args.src.length() < 8 {
|
||||
break.outer
|
||||
break
|
||||
}
|
||||
bits |= args.src.peek_u64le() ~mod<< n_bits
|
||||
args.src.skip_u32_fast!(actual: (63 - n_bits) >> 3, worst_case: 8)
|
||||
n_bits |= 56
|
||||
}
|
||||
mode_offset = this.context_mode << 9
|
||||
ctx = (CONTEXT_LOOKUP[mode_offset | p1] |
|
||||
CONTEXT_LOOKUP[mode_offset | 0x100 | p2]) as base.u32
|
||||
tree = this.context_maps[this.context_map_slice + ctx] as base.u32
|
||||
tree = this.context_maps[cm_slice + ctx] as base.u32
|
||||
i = (LIT_HUFFS_BASE + (tree * LIT_HUFFS_STRIDE)) ~mod+ ((bits & 0xFF) as base.u32)
|
||||
if i >= HUFFS_LEN {
|
||||
ret = "#internal error: inconsistent Huffman table"
|
||||
break.outer
|
||||
break
|
||||
}
|
||||
e = this.huffs[i] as base.u32
|
||||
e_bits = e & 15
|
||||
@@ -856,23 +875,34 @@ pri func decoder.process_commands_fast!(src: base.io_reader, workbuf: slice base
|
||||
i = ((LIT_HUFFS_BASE + (tree * LIT_HUFFS_STRIDE)) ~mod+ (e >> 4)) ~mod+ ((((bits >> 8) & 0x7FFF) as base.u32) & (((1 as base.u32) << (e_bits - 8)) - 1))
|
||||
if i >= HUFFS_LEN {
|
||||
ret = "#internal error: inconsistent Huffman table"
|
||||
break.outer
|
||||
break
|
||||
}
|
||||
e = this.huffs[i] as base.u32
|
||||
e_bits = (e & 15) + 8
|
||||
}
|
||||
if n_bits < e_bits {
|
||||
ret = "#internal error: inconsistent bit reader"
|
||||
break.outer
|
||||
break
|
||||
}
|
||||
bits >>= e_bits
|
||||
n_bits -= e_bits
|
||||
this.symbol = e >> 4
|
||||
this.put_byte!(workbuf: args.workbuf, b: (this.symbol & 0xFF) as base.u8)
|
||||
p2 = p1
|
||||
p1 = this.symbol & 0xFF
|
||||
this.block_length[0] ~mod-= 1
|
||||
this.insert_length ~mod-= 1
|
||||
p1 = (e >> 4) & 0xFF
|
||||
wi = (pos & wmask) as base.u64
|
||||
if wi < args.workbuf.length() {
|
||||
args.workbuf[wi] = p1 as base.u8
|
||||
}
|
||||
pos ~mod+= 1
|
||||
lit_i ~mod+= 1
|
||||
}
|
||||
if lit_i > 0 {
|
||||
this.symbol = p1
|
||||
this.pos = pos
|
||||
this.block_length[0] ~mod-= lit_i
|
||||
this.insert_length ~mod-= lit_i
|
||||
}
|
||||
if (ret.is_error()) or (this.insert_length > 0) {
|
||||
break.outer
|
||||
}
|
||||
this.cmd_state = 2
|
||||
if (this.mb_remaining == 0) or (this.mb_remaining >= 0x8000_0000) {
|
||||
@@ -971,6 +1001,18 @@ pri func decoder.process_commands_fast!(src: base.io_reader, workbuf: slice base
|
||||
this.mb_remaining ~mod-= n
|
||||
// Copy in chunks of at most distance bytes, like process_commands.
|
||||
k64 = 0
|
||||
if (n <= 16) and (distance >= 16) and
|
||||
((dst_i ~mod+ 16) <= (this.rb_size as base.u64)) and
|
||||
((src_i ~mod+ 16) <= (this.window_size as base.u64)) {
|
||||
// Like the C code, copy 16 bytes, overwriting up to 16 - n bytes
|
||||
// after the copy's end. Those bytes are more than the maximum
|
||||
// backward distance (BROTLI_WINDOW_GAP is 16) away from any later
|
||||
// position, so they are never read before being rewritten.
|
||||
copied16 = this.copy16!(workbuf: args.workbuf, d: dst_i, s: src_i)
|
||||
if copied16 {
|
||||
k64 = n as base.u64
|
||||
}
|
||||
}
|
||||
while k64 < (n as base.u64) {
|
||||
n64 = (n as base.u64) ~mod- k64
|
||||
n64 = n64.min(no_more_than: distance as base.u64)
|
||||
@@ -1098,3 +1140,34 @@ pri func decoder.take_distance_from_ring_buffer!(code: base.u32[..= 15]) {
|
||||
this.distance_code = v - 3
|
||||
}
|
||||
}
|
||||
|
||||
// copy16 copies the 16 bytes at workbuf[s ..] to workbuf[d ..], if both are
|
||||
// in bounds (and returns whether it did). The two ranges must not overlap.
|
||||
pri func decoder.copy16!(workbuf: slice base.u8, d: base.u64, s: base.u64) base.bool {
|
||||
var lo : base.u64
|
||||
var hi : base.u64
|
||||
var q : slice base.u8
|
||||
var r : slice base.u8
|
||||
|
||||
if (args.s > args.workbuf.length()) or (args.d > args.workbuf.length()) {
|
||||
return false
|
||||
}
|
||||
q = args.workbuf[args.s ..]
|
||||
if q.length() >= 16 {
|
||||
lo = q.peek_u64le()
|
||||
q = q[8 ..]
|
||||
if q.length() >= 8 {
|
||||
hi = q.peek_u64le()
|
||||
q = args.workbuf[args.d ..]
|
||||
if q.length() >= 16 {
|
||||
r = q[8 ..]
|
||||
q.poke_u64le!(a: lo)
|
||||
if r.length() >= 8 {
|
||||
r.poke_u64le!(a: hi)
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -125,33 +125,27 @@ pri func encoder.store_meta_block_header!(len: base.u64, is_uncompressed: base.b
|
||||
}
|
||||
}
|
||||
|
||||
// LOG2_DEBRUIJN32 maps (v * 0x07C4_ACDD) >> 27, for v = (1 << (k + 1)) - 1,
|
||||
// to k.
|
||||
pri const LOG2_DEBRUIJN32 : roarray[32] base.u32[..= 31] = [
|
||||
0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30,
|
||||
8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31,
|
||||
]
|
||||
|
||||
// Mirrors Log2FloorNonZero in fast_log.h, which (with BROTLI_BSR32) truncates
|
||||
// its argument to 32 bits. It returns 0 for an argument of 0.
|
||||
pri func encoder.log2_floor_non_zero(n: base.u64) base.u32[..= 31] {
|
||||
var x : base.u32
|
||||
var r : base.u32
|
||||
|
||||
// Smear the highest set bit of the low 32 bits of n rightwards, then
|
||||
// look up its position.
|
||||
x = (args.n & 0xFFFF_FFFF) as base.u32
|
||||
if x >= 0x1_0000 {
|
||||
x >>= 16
|
||||
r = 16
|
||||
}
|
||||
if x >= 0x100 {
|
||||
x >>= 8
|
||||
r ~mod+= 8
|
||||
}
|
||||
if x >= 0x10 {
|
||||
x >>= 4
|
||||
r ~mod+= 4
|
||||
}
|
||||
if x >= 4 {
|
||||
x >>= 2
|
||||
r ~mod+= 2
|
||||
}
|
||||
if x >= 2 {
|
||||
r ~mod+= 1
|
||||
}
|
||||
return r & 31
|
||||
x |= x >> 1
|
||||
x |= x >> 2
|
||||
x |= x >> 4
|
||||
x |= x >> 8
|
||||
x |= x >> 16
|
||||
return LOG2_DEBRUIJN32[(x ~mod* 0x07C4_ACDD) >> 27]
|
||||
}
|
||||
|
||||
// Mirrors FastLog2 in fast_log.h. The result is the bit pattern of an IEEE
|
||||
@@ -193,12 +187,10 @@ pri func encoder.bits_entropy_of_histogram() base.u64 {
|
||||
// code's BROTLI_UNALIGNED_LOAD64LE. The C code never reads out of bounds, so
|
||||
// the zero result for an out of bounds read should never be used.
|
||||
pri func encoder.load64le(s: roslice base.u8, i: base.u64) base.u64 {
|
||||
var q : roslice base.u8
|
||||
|
||||
if args.i < args.s.length() {
|
||||
q = args.s[args.i ..]
|
||||
if q.length() >= 8 {
|
||||
return q.peek_u64le()
|
||||
if args.i < 0xFFFF_FFFF_FFFF_0000 {
|
||||
if (args.i + 8) <= args.s.length() {
|
||||
assert args.i <= (args.i + 8) via "a <= (a + b): 0 <= b"(b: 8)
|
||||
return args.s[args.i .. args.i + 8].peek_u64le()
|
||||
}
|
||||
}
|
||||
return 0
|
||||
@@ -206,12 +198,10 @@ pri func encoder.load64le(s: roslice base.u8, i: base.u64) base.u64 {
|
||||
|
||||
// load32le is like load64le but for 4 bytes.
|
||||
pri func encoder.load32le(s: roslice base.u8, i: base.u64) base.u32 {
|
||||
var q : roslice base.u8
|
||||
|
||||
if args.i < args.s.length() {
|
||||
q = args.s[args.i ..]
|
||||
if q.length() >= 4 {
|
||||
return q.peek_u32le()
|
||||
if args.i < 0xFFFF_FFFF_FFFF_0000 {
|
||||
if (args.i + 4) <= args.s.length() {
|
||||
assert args.i <= (args.i + 4) via "a <= (a + b): 0 <= b"(b: 4)
|
||||
return args.s[args.i .. args.i + 4].peek_u32le()
|
||||
}
|
||||
}
|
||||
return 0
|
||||
@@ -235,11 +225,7 @@ pri func encoder.find_match_length_with_limit(s: roslice base.u8, i1: base.u64,
|
||||
while (n >= 8) and (a.length() >= 8) and (b.length() >= 8) {
|
||||
x = a.peek_u64le() ^ b.peek_u64le()
|
||||
if x <> 0 {
|
||||
while (x & 0xFF) == 0 {
|
||||
matched ~mod+= 1
|
||||
x >>= 8
|
||||
}
|
||||
return matched
|
||||
return matched ~mod+ (this.ctz64(x: x) >> 3)
|
||||
}
|
||||
matched ~mod+= 8
|
||||
n -= 8
|
||||
|
||||
+159
-95
@@ -400,7 +400,6 @@ pri func encoder.hasher_store!(wb: slice base.u8, ix: base.u64) {
|
||||
var bsize : base.u64
|
||||
var off : base.u64
|
||||
var num : base.u16
|
||||
var hash : base.u32
|
||||
var bank : base.u64
|
||||
var idx : base.u64
|
||||
var delta : base.u64
|
||||
@@ -408,6 +407,10 @@ pri func encoder.hasher_store!(wb: slice base.u8, ix: base.u64) {
|
||||
var fc_bits : base.u32[..= 16]
|
||||
|
||||
htype = this.hasher_type
|
||||
if (htype == 58) or (htype == 68) {
|
||||
this.hasher_store_h58!(wb: args.wb, ix: args.ix)
|
||||
return nothing
|
||||
}
|
||||
if this.rb_buf_off > args.wb.length() {
|
||||
return nothing
|
||||
}
|
||||
@@ -427,16 +430,6 @@ pri func encoder.hasher_store!(wb: slice base.u8, ix: base.u64) {
|
||||
this.poke_u16!(s: args.wb, i: this.h_off0 ~mod+ (2 ~mod* (key as base.u64)), v: num ~mod+ 1)
|
||||
this.poke_u32!(s: args.wb, i: this.h_off1 ~mod+ (4 ~mod* off), v: (args.ix & 0xFFFF_FFFF) as base.u32)
|
||||
|
||||
} else if (htype == 58) or (htype == 68) {
|
||||
hash = this.hash_longest(data: data, i: args.ix & (this.rb_mask as base.u64))
|
||||
key = hash >> 8
|
||||
num = this.peek_u16(s: args.wb, i: this.h_off0 ~mod+ (2 ~mod* (key as base.u64)))
|
||||
off = ((num as base.u64) & (((1 as base.u64) << this.h_block_bits) - 1)) ~mod+
|
||||
((key as base.u64) << this.h_block_bits)
|
||||
this.poke_u16!(s: args.wb, i: this.h_off0 ~mod+ (2 ~mod* (key as base.u64)), v: num ~mod- 1)
|
||||
this.poke_u32!(s: args.wb, i: this.h_off2 ~mod+ (4 ~mod* off), v: (args.ix & 0xFFFF_FFFF) as base.u32)
|
||||
this.set_byte!(s: args.wb, i: this.h_off1 ~mod+ off, v: (hash & 0xFF) as base.u8)
|
||||
|
||||
} else if (htype == 40) or (htype == 41) or (htype == 42) {
|
||||
key = this.hash_forgetful(data: data, i: args.ix & (this.rb_mask as base.u64))
|
||||
fc_bits = this.fc_bank_bits()
|
||||
@@ -457,6 +450,36 @@ pri func encoder.hasher_store!(wb: slice base.u8, ix: base.u64) {
|
||||
}
|
||||
}
|
||||
|
||||
// hasher_store_h58 is FN(Store) for H58 and H68.
|
||||
pri func encoder.hasher_store_h58!(wb: slice base.u8, ix: base.u64) {
|
||||
var hash : base.u32
|
||||
var key : base.u64
|
||||
var koff : base.u64
|
||||
var num : base.u16
|
||||
var off : base.u64
|
||||
|
||||
hash = this.hash_h58(wb: args.wb, ix: args.ix)
|
||||
key = (hash >> 8) as base.u64
|
||||
koff = this.h_off0 ~mod+ (2 ~mod* key)
|
||||
num = this.peek_u16(s: args.wb, i: koff)
|
||||
off = ((num as base.u64) & (((1 as base.u64) << this.h_block_bits) - 1)) ~mod+ (key << this.h_block_bits)
|
||||
this.poke_u16!(s: args.wb, i: koff, v: num ~mod- 1)
|
||||
this.poke_u32!(s: args.wb, i: this.h_off2 ~mod+ (4 ~mod* off), v: (args.ix & 0xFFFF_FFFF) as base.u32)
|
||||
this.set_byte!(s: args.wb, i: this.h_off1 ~mod+ off, v: (hash & 0xFF) as base.u8)
|
||||
}
|
||||
|
||||
// hash_h58 is hash_longest for H58 and H68, of the ring buffer bytes at
|
||||
// position ix (masked).
|
||||
pri func encoder.hash_h58(wb: roslice base.u8, ix: base.u64) base.u32 {
|
||||
var i : base.u64
|
||||
|
||||
i = this.rb_buf_off ~mod+ (args.ix & (this.rb_mask as base.u64))
|
||||
if this.hasher_type == 58 {
|
||||
return (this.load32le(s: args.wb, i: i) ~mod* HASH_MUL_32) >> ((24 - this.h_bucket_bits) & 31)
|
||||
}
|
||||
return ((this.load64le(s: args.wb, i: i) ~mod* ((HASH_MUL_64 << 24) & 0xFFFF_FFFF_FFFF_FFFF)) >> 41) as base.u32
|
||||
}
|
||||
|
||||
// Mirrors FN(StoreRange) of every hasher.
|
||||
pri func encoder.hasher_store_range!(wb: slice base.u8, ix_start: base.u64, ix_end: base.u64) {
|
||||
var i : base.u64
|
||||
@@ -803,6 +826,8 @@ pri func encoder.find_last_distances!(data: roslice base.u8, cur_ix: base.u64, m
|
||||
var prev_ix : base.u64
|
||||
var len : base.u64
|
||||
var score : base.u64
|
||||
var cur_byte : base.u8
|
||||
var n : base.u32
|
||||
|
||||
mask = this.rb_mask as base.u64
|
||||
cur_ix_masked = args.cur_ix & mask
|
||||
@@ -811,15 +836,14 @@ pri func encoder.find_last_distances!(data: roslice base.u8, cur_ix: base.u64, m
|
||||
this.sr_len = 0
|
||||
this.sr_len_code_delta = 0
|
||||
|
||||
// cur_byte is data[cur_ix_masked + best_len], updated with best_len.
|
||||
cur_byte = this.byte_at(s: args.data, i: cur_ix_masked ~mod+ best_len)
|
||||
n = this.h_num_last_distances
|
||||
i = 0
|
||||
while i < this.h_num_last_distances {
|
||||
while i < n {
|
||||
backward = this.dist_cache_u64(i: i)
|
||||
prev_ix = args.cur_ix ~mod- backward
|
||||
if prev_ix >= args.cur_ix {
|
||||
i ~mod+= 1
|
||||
continue
|
||||
}
|
||||
if backward > args.max_backward {
|
||||
if (prev_ix >= args.cur_ix) or (backward > args.max_backward) {
|
||||
i ~mod+= 1
|
||||
continue
|
||||
}
|
||||
@@ -828,7 +852,7 @@ pri func encoder.find_last_distances!(data: roslice base.u8, cur_ix: base.u64, m
|
||||
break
|
||||
}
|
||||
if ((prev_ix ~mod+ best_len) > mask) or
|
||||
(this.byte_at(s: args.data, i: cur_ix_masked ~mod+ best_len) <> this.byte_at(s: args.data, i: prev_ix ~mod+ best_len)) {
|
||||
(cur_byte <> this.byte_at(s: args.data, i: prev_ix ~mod+ best_len)) {
|
||||
i ~mod+= 1
|
||||
continue
|
||||
}
|
||||
@@ -845,6 +869,7 @@ pri func encoder.find_last_distances!(data: roslice base.u8, cur_ix: base.u64, m
|
||||
this.sr_len = best_len
|
||||
this.sr_distance = backward
|
||||
this.sr_score = best_score
|
||||
cur_byte = this.byte_at(s: args.data, i: cur_ix_masked ~mod+ best_len)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -853,55 +878,6 @@ pri func encoder.find_last_distances!(data: roslice base.u8, cur_ix: base.u64, m
|
||||
return best_len
|
||||
}
|
||||
|
||||
// try_bucket_entry is the body of the bucket loop shared by the H5, H6, H58
|
||||
// and H68 FN(FindLongestMatch) functions, for one prev_ix candidate. It
|
||||
// returns 0 to continue, 1 to break, or 2 + the new best_len.
|
||||
pri func encoder.try_bucket_entry!(data: roslice base.u8, cur_ix: base.u64, prev_ix_in: base.u64, best_len: base.u64, max_length: base.u64, max_backward: base.u64) base.u64 {
|
||||
var mask : base.u64
|
||||
var cur_ix_masked : base.u64
|
||||
var prev_ix : base.u64
|
||||
var backward : base.u64
|
||||
var len : base.u64
|
||||
var score : base.u64
|
||||
|
||||
mask = this.rb_mask as base.u64
|
||||
cur_ix_masked = args.cur_ix & mask
|
||||
backward = args.cur_ix ~mod- args.prev_ix_in
|
||||
if backward > args.max_backward {
|
||||
return 1
|
||||
}
|
||||
prev_ix = args.prev_ix_in & mask
|
||||
if (cur_ix_masked ~mod+ args.best_len) > mask {
|
||||
return 1
|
||||
}
|
||||
if ((prev_ix ~mod+ args.best_len) > mask) or
|
||||
(this.load32le(s: args.data, i: (cur_ix_masked ~mod+ args.best_len) ~mod- 3) <>
|
||||
this.load32le(s: args.data, i: (prev_ix ~mod+ args.best_len) ~mod- 3)) {
|
||||
return 0
|
||||
}
|
||||
if (this.hasher_type == 6) or (this.hasher_type == 68) {
|
||||
if this.load32le(s: args.data, i: cur_ix_masked) <> this.load32le(s: args.data, i: prev_ix) {
|
||||
return 0
|
||||
}
|
||||
len = this.find_match_length_with_limit(s: args.data, i1: prev_ix ~mod+ 4, i2: cur_ix_masked ~mod+ 4,
|
||||
limit: args.max_length ~mod- 4) ~mod+ 4
|
||||
} else {
|
||||
len = this.find_match_length_with_limit(s: args.data, i1: prev_ix, i2: cur_ix_masked, limit: args.max_length)
|
||||
if len < 4 {
|
||||
return 0
|
||||
}
|
||||
}
|
||||
score = this.backward_reference_score(copy_length: len, backward_reference_offset: backward)
|
||||
if this.sr_score < score {
|
||||
// The C code's best_score is always equal to out->score here.
|
||||
this.sr_len = len
|
||||
this.sr_distance = backward
|
||||
this.sr_score = score
|
||||
return 2 ~mod+ len
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// Mirrors FN(FindLongestMatch) in hash_longest_match_inc.h (H5) and
|
||||
// hash_longest_match64_inc.h (H6).
|
||||
pri func encoder.find_longest_match_h5!(wb: slice base.u8, cur_ix: base.u64, max_length: base.u64, max_backward: base.u64, dictionary_distance: base.u64, max_distance: base.u64) {
|
||||
@@ -916,7 +892,13 @@ pri func encoder.find_longest_match_h5!(wb: slice base.u8, cur_ix: base.u64, max
|
||||
var bucket : base.u64
|
||||
var down : base.u64
|
||||
var i : base.u64
|
||||
var r : base.u64
|
||||
var mask : base.u64
|
||||
var is64 : base.bool
|
||||
var cur4 : base.u32
|
||||
var prev_ix : base.u64
|
||||
var backward : base.u64
|
||||
var len : base.u64
|
||||
var score : base.u64
|
||||
|
||||
if this.rb_buf_off > args.wb.length() {
|
||||
return nothing
|
||||
@@ -938,20 +920,46 @@ pri func encoder.find_longest_match_h5!(wb: slice base.u8, cur_ix: base.u64, max
|
||||
if (num as base.u64) > block_size {
|
||||
down = (num as base.u64) ~mod- block_size
|
||||
}
|
||||
// Try the bucket's entries, newest first. Like the C code, this
|
||||
// compares 4 bytes ending at best_len + 1 before the full match.
|
||||
mask = this.rb_mask as base.u64
|
||||
is64 = this.hasher_type == 6
|
||||
cur4 = this.load32le(s: data, i: (cur_ix_masked ~mod+ best_len) ~mod- 3)
|
||||
i = num as base.u64
|
||||
while i > down {
|
||||
i ~mod-= 1
|
||||
r = this.try_bucket_entry!(
|
||||
data: data,
|
||||
cur_ix: args.cur_ix,
|
||||
prev_ix_in: this.load32le(s: args.wb, i: bucket ~mod+ (4 ~mod* (i & block_mask))) as base.u64,
|
||||
best_len: best_len,
|
||||
max_length: args.max_length,
|
||||
max_backward: args.max_backward)
|
||||
if r == 1 {
|
||||
prev_ix = this.load32le(s: args.wb, i: bucket ~mod+ (4 ~mod* (i & block_mask))) as base.u64
|
||||
backward = args.cur_ix ~mod- prev_ix
|
||||
if backward > args.max_backward {
|
||||
break
|
||||
} else if r >= 2 {
|
||||
best_len = r ~mod- 2
|
||||
}
|
||||
prev_ix &= mask
|
||||
if (cur_ix_masked ~mod+ best_len) > mask {
|
||||
break
|
||||
}
|
||||
if ((prev_ix ~mod+ best_len) > mask) or
|
||||
(cur4 <> this.load32le(s: data, i: (prev_ix ~mod+ best_len) ~mod- 3)) {
|
||||
continue
|
||||
}
|
||||
if is64 {
|
||||
if this.load32le(s: data, i: cur_ix_masked) <> this.load32le(s: data, i: prev_ix) {
|
||||
continue
|
||||
}
|
||||
len = this.find_match_length_with_limit(s: data, i1: prev_ix ~mod+ 4, i2: cur_ix_masked ~mod+ 4,
|
||||
limit: args.max_length ~mod- 4) ~mod+ 4
|
||||
} else {
|
||||
len = this.find_match_length_with_limit(s: data, i1: prev_ix, i2: cur_ix_masked, limit: args.max_length)
|
||||
if len < 4 {
|
||||
continue
|
||||
}
|
||||
}
|
||||
score = this.backward_reference_score(copy_length: len, backward_reference_offset: backward)
|
||||
if this.sr_score < score {
|
||||
this.sr_len = len
|
||||
this.sr_distance = backward
|
||||
this.sr_score = score
|
||||
best_len = len
|
||||
cur4 = this.load32le(s: data, i: (cur_ix_masked ~mod+ best_len) ~mod- 3)
|
||||
}
|
||||
}
|
||||
this.poke_u32!(s: args.wb, i: bucket ~mod+ (4 ~mod* ((num as base.u64) & block_mask)), v: (args.cur_ix & 0xFFFF_FFFF) as base.u32)
|
||||
@@ -984,7 +992,17 @@ pri func encoder.find_longest_match_h58!(wb: slice base.u8, cur_ix: base.u64, ma
|
||||
var limit : base.u64
|
||||
var k : base.u64
|
||||
var rb_index : base.u64
|
||||
var r : base.u64
|
||||
var splat : base.u64
|
||||
var matches : base.u64
|
||||
var chunk : base.u64
|
||||
var j : base.u64
|
||||
var mask : base.u64
|
||||
var is64 : base.bool
|
||||
var cur4 : base.u32
|
||||
var prev_ix : base.u64
|
||||
var backward : base.u64
|
||||
var len : base.u64
|
||||
var score : base.u64
|
||||
|
||||
if this.rb_buf_off > args.wb.length() {
|
||||
return nothing
|
||||
@@ -992,7 +1010,7 @@ pri func encoder.find_longest_match_h58!(wb: slice base.u8, cur_ix: base.u64, ma
|
||||
data = args.wb[this.rb_buf_off ..]
|
||||
cur_ix_masked = args.cur_ix & (this.rb_mask as base.u64)
|
||||
min_score = this.sr_score
|
||||
hash = this.hash_longest(data: data, i: cur_ix_masked)
|
||||
hash = this.hash_h58(wb: args.wb, ix: args.cur_ix)
|
||||
key = (hash >> 8) as base.u64
|
||||
block_size = (1 as base.u64) << this.h_block_bits
|
||||
block_mask = block_size ~mod- 1
|
||||
@@ -1014,24 +1032,70 @@ pri func encoder.find_longest_match_h58!(wb: slice base.u8, cur_ix: base.u64, ma
|
||||
if block_size > (n as base.u64) {
|
||||
limit = n as base.u64
|
||||
}
|
||||
k = 0
|
||||
while k < limit {
|
||||
// Like GetMatchingTagMask's non-SSE2 code: bit j of matches is whether
|
||||
// the tag of entry j matches, 8 entries at a time.
|
||||
splat = (tag as base.u64) ~mod* 0x0101_0101_0101_0101
|
||||
matches = 0
|
||||
j = 0
|
||||
while j < block_size {
|
||||
chunk = this.load64le(s: args.wb, i: tag_bucket ~mod+ j) ^ splat
|
||||
// The 0x80 bit of each byte is set when that byte is non-zero.
|
||||
chunk = (((chunk | 0x8080_8080_8080_8080) ~mod- 0x0101_0101_0101_0101) | chunk) & 0x8080_8080_8080_8080
|
||||
chunk = (chunk ^ 0x8080_8080_8080_8080) >> 7
|
||||
matches |= ((chunk ~mod* 0x0102_0408_1020_4080) >> 56) ~mod<< (j & 63)
|
||||
j ~mod+= 8
|
||||
}
|
||||
// Rotate so that bit k is entry (head + k) & block_mask.
|
||||
if head > 0 {
|
||||
matches = ((matches >> (head & 63)) | (matches ~mod<< ((block_size ~mod- head) & 63))) &
|
||||
((0xFFFF_FFFF_FFFF_FFFF as base.u64) >> ((64 ~mod- block_size) & 63))
|
||||
}
|
||||
if limit < 64 {
|
||||
matches &= ((1 as base.u64) << (limit & 63)) ~mod- 1
|
||||
}
|
||||
// Try the entries whose tags match, in the C code's order (from head).
|
||||
// Like the C code, this compares 4 bytes ending at best_len + 1 before
|
||||
// the full match.
|
||||
mask = this.rb_mask as base.u64
|
||||
is64 = this.hasher_type == 68
|
||||
cur4 = this.load32le(s: data, i: (cur_ix_masked ~mod+ best_len) ~mod- 3)
|
||||
while matches <> 0 {
|
||||
k = this.ctz64(x: matches)
|
||||
matches &= matches ~mod- 1
|
||||
rb_index = (head ~mod+ k) & block_mask
|
||||
if this.byte_at(s: args.wb, i: tag_bucket ~mod+ rb_index) == tag {
|
||||
r = this.try_bucket_entry!(
|
||||
data: data,
|
||||
cur_ix: args.cur_ix,
|
||||
prev_ix_in: this.load32le(s: args.wb, i: bucket ~mod+ (4 ~mod* rb_index)) as base.u64,
|
||||
best_len: best_len,
|
||||
max_length: args.max_length,
|
||||
max_backward: args.max_backward)
|
||||
if r == 1 {
|
||||
break
|
||||
} else if r >= 2 {
|
||||
best_len = r ~mod- 2
|
||||
prev_ix = this.load32le(s: args.wb, i: bucket ~mod+ (4 ~mod* rb_index)) as base.u64
|
||||
backward = args.cur_ix ~mod- prev_ix
|
||||
if backward > args.max_backward {
|
||||
break
|
||||
}
|
||||
prev_ix &= mask
|
||||
if (cur_ix_masked ~mod+ best_len) > mask {
|
||||
break
|
||||
}
|
||||
if ((prev_ix ~mod+ best_len) > mask) or
|
||||
(cur4 <> this.load32le(s: data, i: (prev_ix ~mod+ best_len) ~mod- 3)) {
|
||||
continue
|
||||
}
|
||||
if is64 {
|
||||
if this.load32le(s: data, i: cur_ix_masked) <> this.load32le(s: data, i: prev_ix) {
|
||||
continue
|
||||
}
|
||||
len = this.find_match_length_with_limit(s: data, i1: prev_ix ~mod+ 4, i2: cur_ix_masked ~mod+ 4,
|
||||
limit: args.max_length ~mod- 4) ~mod+ 4
|
||||
} else {
|
||||
len = this.find_match_length_with_limit(s: data, i1: prev_ix, i2: cur_ix_masked, limit: args.max_length)
|
||||
if len < 4 {
|
||||
continue
|
||||
}
|
||||
}
|
||||
k ~mod+= 1
|
||||
score = this.backward_reference_score(copy_length: len, backward_reference_offset: backward)
|
||||
if this.sr_score < score {
|
||||
this.sr_len = len
|
||||
this.sr_distance = backward
|
||||
this.sr_score = score
|
||||
best_len = len
|
||||
cur4 = this.load32le(s: data, i: (cur_ix_masked ~mod+ best_len) ~mod- 3)
|
||||
}
|
||||
}
|
||||
this.poke_u32!(s: args.wb, i: bucket ~mod+ (4 ~mod* ((num as base.u64) & block_mask)), v: (args.cur_ix & 0xFFFF_FFFF) as base.u32)
|
||||
this.set_byte!(s: args.wb, i: tag_bucket ~mod+ ((num as base.u64) & block_mask), v: tag)
|
||||
|
||||
@@ -283,16 +283,16 @@ pri func metablock_builder.h_population_cost!(wb: roslice base.u8, h: base.u64,
|
||||
// Mirrors Log2FloorNonZero in fast_log.h (which truncates n to 32 bits).
|
||||
pri func metablock_builder.log2_floor_non_zero(n: base.u64) base.u32[..= 31] {
|
||||
var x : base.u32
|
||||
var r : base.u32[..= 31]
|
||||
|
||||
// Smear the highest set bit of the low 32 bits of n rightwards, then
|
||||
// look up its position.
|
||||
x = (args.n & 0xFFFF_FFFF) as base.u32
|
||||
while x > 1 {
|
||||
x >>= 1
|
||||
if r < 31 {
|
||||
r += 1
|
||||
}
|
||||
}
|
||||
return r
|
||||
x |= x >> 1
|
||||
x |= x >> 2
|
||||
x |= x >> 4
|
||||
x |= x >> 8
|
||||
x |= x >> 16
|
||||
return LOG2_DEBRUIJN32[(x ~mod* 0x07C4_ACDD) >> 27]
|
||||
}
|
||||
|
||||
// Mirrors PrefixEncodeCopyDistance in prefix.h. It returns the code (the
|
||||
|
||||
@@ -18,6 +18,21 @@
|
||||
// out of bounds branches here (which do nothing, or return 0) should never be
|
||||
// taken.
|
||||
|
||||
// CTZ64_DEBRUIJN maps ((x & -x) * 0x03F7_9D71_B4CB_0A89) >> 58 to the
|
||||
// number of trailing zero bits of the non-zero x.
|
||||
pri const CTZ64_DEBRUIJN : roarray[64] base.u8[..= 63] = [
|
||||
0, 1, 48, 2, 57, 49, 28, 3, 61, 58, 50, 42, 38, 29, 17, 4,
|
||||
62, 55, 59, 36, 53, 51, 43, 22, 45, 39, 33, 30, 24, 18, 12, 5,
|
||||
63, 47, 56, 27, 60, 41, 37, 16, 54, 35, 52, 21, 44, 32, 23, 11,
|
||||
46, 26, 40, 15, 34, 20, 31, 10, 25, 14, 19, 9, 13, 8, 7, 6,
|
||||
]
|
||||
|
||||
// ctz64 returns the number of trailing zero bits of x, which must be
|
||||
// non-zero (for x == 0 it returns 0).
|
||||
pri func encoder.ctz64(x: base.u64) base.u64[..= 63] {
|
||||
return CTZ64_DEBRUIJN[((args.x & (0 ~mod- args.x)) ~mod* 0x03F7_9D71_B4CB_0A89) >> 58] as base.u64
|
||||
}
|
||||
|
||||
// byte_at returns s[i], or 0 if i is out of bounds.
|
||||
pri func encoder.byte_at(s: roslice base.u8, i: base.u64) base.u8 {
|
||||
if args.i < args.s.length() {
|
||||
@@ -28,12 +43,10 @@ pri func encoder.byte_at(s: roslice base.u8, i: base.u64) base.u8 {
|
||||
|
||||
// peek_u16 returns the 2 bytes (little-endian) of s starting at i.
|
||||
pri func encoder.peek_u16(s: roslice base.u8, i: base.u64) base.u16 {
|
||||
var q : roslice base.u8
|
||||
|
||||
if args.i < args.s.length() {
|
||||
q = args.s[args.i ..]
|
||||
if q.length() >= 2 {
|
||||
return q.peek_u16le()
|
||||
if args.i < 0xFFFF_FFFF_FFFF_0000 {
|
||||
if (args.i + 2) <= args.s.length() {
|
||||
assert args.i <= (args.i + 2) via "a <= (a + b): 0 <= b"(b: 2)
|
||||
return args.s[args.i .. args.i + 2].peek_u16le()
|
||||
}
|
||||
}
|
||||
return 0
|
||||
@@ -41,24 +54,20 @@ pri func encoder.peek_u16(s: roslice base.u8, i: base.u64) base.u16 {
|
||||
|
||||
// poke_u16 sets the 2 bytes (little-endian) of s starting at i.
|
||||
pri func encoder.poke_u16!(s: slice base.u8, i: base.u64, v: base.u16) {
|
||||
var q : slice base.u8
|
||||
|
||||
if args.i < args.s.length() {
|
||||
q = args.s[args.i ..]
|
||||
if q.length() >= 2 {
|
||||
q.poke_u16le!(a: args.v)
|
||||
if args.i < 0xFFFF_FFFF_FFFF_0000 {
|
||||
if (args.i + 2) <= args.s.length() {
|
||||
assert args.i <= (args.i + 2) via "a <= (a + b): 0 <= b"(b: 2)
|
||||
args.s[args.i .. args.i + 2].poke_u16le!(a: args.v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// poke_u32 sets the 4 bytes (little-endian) of s starting at i.
|
||||
pri func encoder.poke_u32!(s: slice base.u8, i: base.u64, v: base.u32) {
|
||||
var q : slice base.u8
|
||||
|
||||
if args.i < args.s.length() {
|
||||
q = args.s[args.i ..]
|
||||
if q.length() >= 4 {
|
||||
q.poke_u32le!(a: args.v)
|
||||
if args.i < 0xFFFF_FFFF_FFFF_0000 {
|
||||
if (args.i + 4) <= args.s.length() {
|
||||
assert args.i <= (args.i + 4) via "a <= (a + b): 0 <= b"(b: 4)
|
||||
args.s[args.i .. args.i + 4].poke_u32le!(a: args.v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user