Files
wuffs/std/brotli/encode_bit_cost.wuffs
T
Claude 49356e06e4 std/brotli: fill the FastLog2 caches lazily
Each bit_cost (one in the encoder, one in its metablock_builder) has a
64Ki-entry FastLog2 cache. The metablock_builder filled all of it before
building each quality 10 or 11 metablock (about 20 ms per encoder, since
each entry is an exact software log2), and the encoder's own cache was
never filled, so the quality 10 and 11 literal cost estimates computed
one software log2 per input byte.

Instead, a bitmap (zeroed when the struct is initialized) records which
cache entries are valid, and the new fast_log2_memo fills an entry the
first time it is needed. Typically only a few thousand entries are used.
bits_entropy, population_cost and their callers become impure (!) to
call it. The values are computed by the same functions, so the results
are unchanged.

On alice29 at quality 10, the encoder executes about 20% fewer
instructions, and a 1000-byte input takes 6 ms instead of 21 ms.
2026-10-27 09:57:01 +00:00

360 lines
13 KiB
Plaintext

// Copyright 2026 The Wuffs Authors.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// SPDX-License-Identifier: Apache-2.0 OR MIT
// --------
// This file mirrors bit_cost.c, bit_cost_inc.h and fast_log.h. The functions
// are methods of the bit_cost struct (whose only state is the stateless
// softfloat arith and a lazily filled FastLog2 cache), so that other structs
// (the encoder, the metablock_builder) can embed one with a single field.
//
// Doubles are IEEE 754 binary64 bit patterns (base.u64). A uint32_t array
// (e.g. a histogram's data_) is a slice of little-endian bytes, because
// Wuffs' C code generator does not support "slice base.u32" arguments.
//
// A histogram (the C code's HistogramLiteral, HistogramCommand and
// HistogramDistance structs) is (4 * data_size) + 16 bytes, exactly the C
// struct's memory layout on a 64-bit little-endian machine:
//
// [0 .. 4 * data_size) data_, as u32le values
// [4 * data_size .. (4 * data_size) + 8) total_count_, as a u64le
// [(4 * data_size) + 8 .. (4 * data_size) + 16) bit_cost_, as the u64le
// bits of a binary64
//
// data_size is 256 (HISTOGRAM_LITERAL_DATA_SIZE), 704
// (HISTOGRAM_COMMAND_DATA_SIZE) or 544 (HISTOGRAM_DISTANCE_DATA_SIZE). The C
// code's templated functions (the FN(X) macros, instantiated three times) are
// one Wuffs function each, with a data_size argument.
pri const HISTOGRAM_LITERAL_DATA_SIZE : base.u32 = 256
pri const HISTOGRAM_COMMAND_DATA_SIZE : base.u32 = 704
pri const HISTOGRAM_DISTANCE_DATA_SIZE : base.u32 = 544
// The byte sizes of the three histogram types.
pri const HISTOGRAM_LITERAL_SIZE : base.u64 = 1040
pri const HISTOGRAM_COMMAND_SIZE : base.u64 = 2832
pri const HISTOGRAM_DISTANCE_SIZE : base.u64 = 2192
// F64_HUGE_VAL is +infinity (the C code's HUGE_VAL).
pri const F64_HUGE_VAL : base.u64 = 0x7FF0_0000_0000_0000
// F64_1E99 is the C code's 1e99.
pri const F64_1E99 : base.u64 = 0x547D_42AE_A287_9F2E
pri struct bit_cost?(
// Bit (v & 63) of log2_valid[v >> 6] is whether log2_cache[v] holds
// FastLog2(v). This bitmap is zeroed when the struct is initialized
// (but log2_cache need not be), so the cache starts empty.
log2_valid : array[1024] base.u64,
sf : softfloat.arith,
) + (
// log2_cache[v] is FastLog2(v), for 256 <= v < 65536, filled lazily
// (see fast_log2_memo). It is only a speed optimization (glibc's log2
// is expensive to emulate): the values are computed by the same
// functions, so the results are identical with or without it. Few of
// the 65536 entries are typically needed, so filling it lazily is
// much cheaper than filling it all up front.
log2_cache : array[65536] base.u64,
)
// ld32 returns the u32le at s[i ..], or 0 if that is out of bounds (which
// the callers never do).
pri func bit_cost.ld32(s: roslice base.u8, i: base.u64) base.u32 {
var q : roslice base.u8
if args.i < args.s.length() {
q = args.s[args.i ..]
if q.length() >= 4 {
return q.peek_u32le()
}
}
return 0
}
// ld64 is like ld32 but for a u64le.
pri func bit_cost.ld64(s: roslice base.u8, i: base.u64) base.u64 {
var q : roslice base.u8
if args.i < args.s.length() {
q = args.s[args.i ..]
if q.length() >= 8 {
return q.peek_u64le()
}
}
return 0
}
// st32 stores v as a u32le at s[i ..], or does nothing if that is out of
// bounds (which the callers never do).
pri func bit_cost.st32!(s: slice base.u8, i: base.u64, v: base.u32) {
var q : slice base.u8
if args.i < args.s.length() {
q = args.s[args.i ..]
if q.length() >= 4 {
q.poke_u32le!(a: args.v)
}
}
}
// st64 is like st32 but for a u64le.
pri func bit_cost.st64!(s: slice base.u8, i: base.u64, v: base.u64) {
var q : slice base.u8
if args.i < args.s.length() {
q = args.s[args.i ..]
if q.length() >= 8 {
q.poke_u64le!(a: args.v)
}
}
}
// Mirrors FastLog2 in fast_log.h. The result is the bit pattern of an IEEE
// 754 binary64 value. It uses (but does not fill) log2_cache: see
// fast_log2_memo.
pri func bit_cost.fast_log2(v: base.u64) base.u64 {
if args.v < 256 {
return LOG2_TABLE[args.v]
} else if args.v < 65536 {
if ((this.log2_valid[args.v >> 6] >> (args.v & 63)) & 1) <> 0 {
return this.log2_cache[args.v]
}
}
return this.sf.log2_u64(v: args.v)
}
// fast_log2_memo is like fast_log2 (it also mirrors FastLog2) but it fills
// the log2_cache entry for v (if v is in the cache's range).
pri func bit_cost.fast_log2_memo!(v: base.u64) base.u64 {
if args.v < 256 {
return LOG2_TABLE[args.v]
} else if args.v >= 65536 {
return this.sf.log2_u64(v: args.v)
} else if ((this.log2_valid[args.v >> 6] >> (args.v & 63)) & 1) == 0 {
this.log2_cache[args.v] = this.sf.log2_u64(v: args.v)
this.log2_valid[args.v >> 6] |= (1 as base.u64) << (args.v & 63)
}
return this.log2_cache[args.v]
}
// Mirrors BrotliBitsEntropy in bit_cost.c. population holds (at least) size
// u32le values. The C code's "goto odd_number_of_elements_left" only changes
// the loop's unrolling, not the order of the operations.
pri func bit_cost.bits_entropy!(population: roslice base.u8, size: base.u64) base.u64 {
var sum : base.u64
var retval : base.u64
var p : base.u64
var lp : base.u64
var q : roslice base.u8
var n : base.u64
q = args.population
n = args.size
while n > 0 {
n -= 1
p = 0
if q.length() >= 4 {
p = q.peek_u32le() as base.u64
q = q[4 ..]
}
sum ~mod+= p
lp = this.fast_log2_memo!(v: p)
retval = this.sf.f64_sub(a: retval, b: this.sf.f64_mul(
a: this.sf.f64_from_u64(a: p), b: lp))
}
if sum <> 0 {
lp = this.fast_log2_memo!(v: sum)
retval = this.sf.f64_add(a: retval, b: this.sf.f64_mul(
a: this.sf.f64_from_u64(a: sum), b: lp))
}
if this.sf.f64_lt(a: retval, b: this.sf.f64_from_u64(a: sum)) <> 0 {
retval = this.sf.f64_from_u64(a: sum)
}
return retval
}
// Mirrors BrotliPopulationCostLiteral, BrotliPopulationCostCommand and
// BrotliPopulationCostDistance (FN(BrotliPopulationCost) in bit_cost_inc.h).
// histogram is a histogram of data_size symbols (see the layout above).
pri func bit_cost.population_cost!(histogram: roslice base.u8, data_size: base.u32[..= 704]) base.u64 {
var total : base.u64
var count : base.u32
var s : array[4] base.u32
var bits : base.u64
var i : base.u32
var j : base.u32
var k : base.u32
var c : base.u32
var histo : array[4] base.u32
var h0 : base.u32
var h1 : base.u32
var h2 : base.u32
var h23 : base.u32
var histomax : base.u32
var max_depth : base.u64[..= 15]
var depth_histo : array[18] base.u32
var log2total : base.u64
var log2p : base.u64
var lp : base.u64
var depth : base.u64
var reps : base.u32
var sum : base.u64
var retval : base.u64
var p : base.u64
total = this.ld64(s: args.histogram, i: (args.data_size as base.u64) * 4)
if total == 0 {
// kOneSymbolHistogramCost.
return 0x4028_0000_0000_0000
}
i = 0
while i < args.data_size {
if this.ld32(s: args.histogram, i: (i as base.u64) * 4) > 0 {
if count < 4 {
s[count] = i
}
count ~mod+= 1
if count > 4 {
break
}
}
i ~mod+= 1
}
if count == 1 {
// kOneSymbolHistogramCost.
return 0x4028_0000_0000_0000
}
if count == 2 {
// kTwoSymbolHistogramCost + (double)total_count_.
return this.sf.f64_add(a: 0x4034_0000_0000_0000, b: this.sf.f64_from_u64(a: total))
}
if count == 3 {
h0 = this.ld32(s: args.histogram, i: (s[0] as base.u64) * 4)
h1 = this.ld32(s: args.histogram, i: (s[1] as base.u64) * 4)
h2 = this.ld32(s: args.histogram, i: (s[2] as base.u64) * 4)
histomax = h0.max(no_less_than: h1.max(no_less_than: h2))
// kThreeSymbolHistogramCost + 2 * (histo0 + histo1 + histo2) -
// histomax, where the integer expression is uint32_t arithmetic.
return this.sf.f64_sub(
a: this.sf.f64_add(
a: 0x403C_0000_0000_0000,
b: this.sf.f64_from_u32(a: 2 ~mod* ((h0 ~mod+ h1) ~mod+ h2))),
b: this.sf.f64_from_u32(a: histomax))
}
if count == 4 {
i = 0
while i < 4 {
histo[i] = this.ld32(s: args.histogram, i: (s[i] as base.u64) * 4)
i ~mod+= 1
}
// Sort.
i = 0
while i < 4 {
j = i ~mod+ 1
while j < 4 {
if histo[j] > histo[i & 3] {
k = histo[j]
histo[j] = histo[i & 3]
histo[i & 3] = k
}
j ~mod+= 1
}
i ~mod+= 1
}
h23 = histo[2] ~mod+ histo[3]
histomax = h23.max(no_less_than: histo[0])
// kFourSymbolHistogramCost + 3 * h23 + 2 * (histo[0] + histo[1]) -
// histomax, evaluated left to right.
return this.sf.f64_sub(
a: this.sf.f64_add(
a: this.sf.f64_add(
a: 0x4042_8000_0000_0000,
b: this.sf.f64_from_u32(a: 3 ~mod* h23)),
b: this.sf.f64_from_u32(a: 2 ~mod* (histo[0] ~mod+ histo[1]))),
b: this.sf.f64_from_u32(a: histomax))
}
// Compute the entropy of the histogram and simultaneously build a
// simplified histogram of the code length codes.
max_depth = 1
log2total = this.fast_log2_memo!(v: total)
i = 0
while i < args.data_size {
c = this.ld32(s: args.histogram, i: (i as base.u64) * 4)
if c > 0 {
lp = this.fast_log2_memo!(v: c as base.u64)
log2p = this.sf.f64_sub(a: log2total, b: lp)
depth = this.sf.f64_to_u64_trunc(a: this.sf.f64_add(a: log2p, b: 0x3FE0_0000_0000_0000))
bits = this.sf.f64_add(a: bits, b: this.sf.f64_mul(
a: this.sf.f64_from_u32(a: c), b: log2p))
if depth > 15 {
depth = 15
}
if depth > (max_depth as base.u64) {
max_depth = depth & 15
}
depth_histo[depth & 15] ~mod+= 1
i ~mod+= 1
} else {
// Compute the run length of zeros.
reps = 1
k = i ~mod+ 1
while k < args.data_size {
if this.ld32(s: args.histogram, i: (k as base.u64) * 4) <> 0 {
break
}
reps ~mod+= 1
k ~mod+= 1
}
i ~sat+= reps
if i >= args.data_size {
// Don't add any cost for the last zero run.
break
}
if reps < 3 {
depth_histo[0] ~mod+= reps
} else {
reps -= 2
while reps > 0 {
depth_histo[17] ~mod+= 1
// Add the 3 extra bits for the 17 code length code.
bits = this.sf.f64_add(a: bits, b: 0x4008_0000_0000_0000)
reps >>= 3
}
}
}
}
// Add the estimated encoding cost of the code length code histogram.
bits = this.sf.f64_add(a: bits, b: this.sf.f64_from_u64(a: 18 + (2 * max_depth)))
// Add the entropy of the code length code histogram. This is
// BrotliBitsEntropy(depth_histo, BROTLI_CODE_LENGTH_CODES), inlined
// because depth_histo is a local array.
i = 0
while i < 18 {
p = depth_histo[i] as base.u64
sum ~mod+= p
lp = this.fast_log2_memo!(v: p)
retval = this.sf.f64_sub(a: retval, b: this.sf.f64_mul(
a: this.sf.f64_from_u64(a: p), b: lp))
i ~mod+= 1
}
if sum <> 0 {
lp = this.fast_log2_memo!(v: sum)
retval = this.sf.f64_add(a: retval, b: this.sf.f64_mul(
a: this.sf.f64_from_u64(a: sum), b: lp))
}
if this.sf.f64_lt(a: retval, b: this.sf.f64_from_u64(a: sum)) <> 0 {
retval = this.sf.f64_from_u64(a: sum)
}
return this.sf.f64_add(a: bits, b: retval)
}