Files
wuffs/std/brotli/encode_metablock_greedy.wuffs
T
Claude 044437a8ec std/brotli: speed up the encoder at qualities 10 and 11, byte-exactly
- encode_float.wuffs: package-private fast paths for the f32/f64 add, sub,
  mul, compare and int conversion calls in the encoder's hot loops. Calling
  std/softfloat is a cross-package public call (a receiver check, never
  inlined). Each fast path returns exactly the softfloat result and falls
  back to it for the uncommon cases. script/manual-test-brotli-float.c
  cross-checks them against hardware float and double arithmetic.
- Zopfli update_nodes: compute a length's cost only when its copy code
  changes, hoist the distance cache values and the continuation byte, and
  use a table for small int to f32 conversions.
- FindBlocks: memoize (block_switch_bitcost + insert_cost) for capped costs
  and skip the subtraction for costs at or above a per-byte threshold
  (S(c) = c - min_cost is monotone, so such costs would be capped anyway).
- BrotliPopulationCost: per-call memo of each count's term and depth, and
  skip the zero terms of the entropy sums.
- std/softfloat: a 64-bit f32_add_mags.

Output is unchanged (mimic tests on gcc and clang, parity sweeps).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UZ978BikvkegL2JUkqoB1e
2026-10-27 23:21:31 +00:00

446 lines
20 KiB
Plaintext

// Copyright 2026 The Wuffs Authors.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// SPDX-License-Identifier: Apache-2.0 OR MIT
// --------
// This file mirrors BrotliBuildMetaBlockGreedy (metablock.c and
// metablock_inc.h), the metablock builder for qualities 4 to 9, and
// BrotliOptimizeHistograms (metablock.c) for its output. The latter uses
// metablock_builder.optimize_huffman_counts_for_rle (see
// encode_metablock.wuffs) for BrotliOptimizeHuffmanCountsForRle.
//
// The C code's three FN(BlockSplitter)s and its ContextBlockSplitter are one
// greedy block splitter here (the gbs_etc functions and fields), indexed by
// the block category c: 0 for literals, 1 for commands and 2 for distances.
// A FN(BlockSplitter) is a ContextBlockSplitter with one context: the two
// make the same decisions, with the same floating point operations (adding
// to an initial 0.0 does not change a non-zero sum, and a zero sum compares
// the same either way).
//
// The metablock split (the C code's MetaBlockSplit) is in the workbuf: for
// each category, the block types (u8), the block lengths (u32le) and the
// histograms (data_size u32le counts each, without the C code's total_count_
// and bit_cost_ fields, which BrotliStoreMetaBlock does not use). The
// literal context map (when there is more than one literal context) is
// stored as u8 values: they are less than 256.
// Mirrors BROTLI_MAX_NUMBER_OF_BLOCK_TYPES in constants.h.
pri const GBS_MAX_NUMBER_OF_BLOCK_TYPES : base.u32 = 256
// The doubles 400.0, 500.0, 100.0 and 20.0.
pri const GBS_F64_400 : base.u64 = 0x4079_0000_0000_0000
pri const GBS_F64_500 : base.u64 = 0x407F_4000_0000_0000
pri const GBS_F64_100 : base.u64 = 0x4059_0000_0000_0000
pri const GBS_F64_20 : base.u64 = 0x4034_0000_0000_0000
// gbs_histo_off returns the workbuf offset of category c's histogram ix.
pri func encoder.gbs_histo_off(c: base.u32[..= 3], ix: base.u64) base.u64 {
return this.gbs_histo_off_base[args.c] ~mod+ (args.ix ~mod* ((this.gbs_data_size[args.c] as base.u64) * 4))
}
// gbs_entropy returns BrotliBitsEntropy of the size u32le values at
// args.wb[off ..].
pri func encoder.gbs_entropy!(wb: roslice base.u8, off: base.u64, size: base.u64) base.u64 {
var e : base.u64
if args.off <= args.wb.length() {
e = this.bc.bits_entropy!(population: args.wb[args.off ..], size: args.size)
return e
}
return 0
}
// gbs_clear sets the n u32le values at args.wb[off ..] to zero.
pri func encoder.gbs_clear!(wb: slice base.u8, off: base.u64, n: base.u64) {
this.fill_bytes!(s: args.wb, i: args.off, n: args.n ~mod* 4, v: 0)
}
// gbs_copy copies the n u32le values at args.wb[src ..] to args.wb[dst ..].
pri func encoder.gbs_copy!(wb: slice base.u8, dst: base.u64, src: base.u64, n: base.u64) {
var i : base.u64
i = 0
while i < args.n {
this.poke_u32!(s: args.wb, i: args.dst ~mod+ (i ~mod* 4),
v: this.load32le(s: args.wb, i: args.src ~mod+ (i ~mod* 4)))
i ~mod+= 1
}
}
// gbs_add_histogram adds the n u32le values at args.wb[src ..] to those at
// args.wb[dst ..].
pri func encoder.gbs_add_histogram!(wb: slice base.u8, dst: base.u64, src: base.u64, n: base.u64) {
var i : base.u64
i = 0
while i < args.n {
this.poke_u32!(s: args.wb, i: args.dst ~mod+ (i ~mod* 4),
v: this.load32le(s: args.wb, i: args.dst ~mod+ (i ~mod* 4)) ~mod+
this.load32le(s: args.wb, i: args.src ~mod+ (i ~mod* 4)))
i ~mod+= 1
}
}
// Mirrors FN(InitBlockSplitter) in metablock_inc.h and
// InitContextBlockSplitter in metablock.c. The workbuf offsets of the
// category's split and histograms were set by compute_layout.
pri func encoder.gbs_init!(wb: slice base.u8, c: base.u32[..= 3], alphabet_size: base.u32, data_size: base.u32,
num_contexts: base.u32, min_block_size: base.u32, split_threshold: base.u64, num_symbols: base.u64) {
var max_num_blocks : base.u64
var max_num_types : base.u64
var nc : base.u32
nc = args.num_contexts.max(no_less_than: 1)
this.gbs_alphabet_size[args.c] = args.alphabet_size
this.gbs_data_size[args.c] = args.data_size
this.gbs_num_contexts[args.c] = nc
this.gbs_max_block_types[args.c] = GBS_MAX_NUMBER_OF_BLOCK_TYPES / nc
this.gbs_min_block_size[args.c] = args.min_block_size.max(no_less_than: 1)
this.gbs_split_threshold[args.c] = args.split_threshold
this.gbs_num_blocks[args.c] = 0
this.gbs_num_types[args.c] = 0
this.gbs_target_block_size[args.c] = this.gbs_min_block_size[args.c] as base.u64
this.gbs_block_size[args.c] = 0
this.gbs_curr_histogram_ix[args.c] = 0
this.gbs_merge_last_count[args.c] = 0
max_num_blocks = (args.num_symbols / (this.gbs_min_block_size[args.c] as base.u64)) ~mod+ 1
max_num_types = max_num_blocks.min(no_more_than: (this.gbs_max_block_types[args.c] as base.u64) + 1)
this.gbs_split_num_blocks[args.c] = max_num_blocks
this.gbs_histograms_size[args.c] = max_num_types ~mod* (nc as base.u64)
// Clear only current histogram.
this.gbs_clear!(wb: args.wb, off: this.gbs_histo_off(c: args.c, ix: 0),
n: (nc as base.u64) ~mod* (args.data_size as base.u64))
this.gbs_last_histogram_ix[(args.c * 2) + 0] = 0
this.gbs_last_histogram_ix[(args.c * 2) + 1] = 0
}
// Mirrors FN(BlockSplitterFinishBlock) in metablock_inc.h and
// ContextBlockSplitterFinishBlock in metablock.c.
pri func encoder.gbs_finish_block!(wb: slice base.u8, c: base.u32[..= 3], is_final: base.bool) {
var nc : base.u64
var i : base.u64
var j : base.u64
var jx : base.u64
var alphabet_size : base.u64
var data_size : base.u64
var num_blocks : base.u64
var curr : base.u64
var tmp : base.u64
var e : base.u64
var d : base.u64
var diff0 : base.u64
var diff1 : base.u64
var threshold : base.u64
var types_off : base.u64
var lengths_off : base.u64
var le_base : base.u32[..= 96]
var last0 : base.u64
var last1 : base.u64
nc = this.gbs_num_contexts[args.c] as base.u64
alphabet_size = this.gbs_alphabet_size[args.c] as base.u64
data_size = this.gbs_data_size[args.c] as base.u64
num_blocks = this.gbs_num_blocks[args.c]
curr = this.gbs_curr_histogram_ix[args.c]
threshold = this.gbs_split_threshold[args.c]
types_off = this.gbs_types_off[args.c]
lengths_off = this.gbs_lengths_off[args.c]
le_base = args.c * 32
if this.gbs_block_size[args.c] < (this.gbs_min_block_size[args.c] as base.u64) {
this.gbs_block_size[args.c] = this.gbs_min_block_size[args.c] as base.u64
}
if num_blocks == 0 {
// Create first block.
this.poke_u32!(s: args.wb, i: lengths_off, v: (this.gbs_block_size[args.c] & 0xFFFF_FFFF) as base.u32)
this.set_byte!(s: args.wb, i: types_off, v: 0)
i = 0
while i < nc {
e = this.gbs_entropy!(wb: args.wb, off: this.gbs_histo_off(c: args.c, ix: i), size: alphabet_size)
this.gbs_last_entropy[((le_base as base.u64) ~mod+ i) & 127] = e
this.gbs_last_entropy[(((le_base as base.u64) ~mod+ nc) ~mod+ i) & 127] = e
i ~mod+= 1
}
this.gbs_num_blocks[args.c] = num_blocks ~mod+ 1
this.gbs_num_types[args.c] ~mod+= 1
curr ~mod+= nc
this.gbs_curr_histogram_ix[args.c] = curr
if curr < this.gbs_histograms_size[args.c] {
this.gbs_clear!(wb: args.wb, off: this.gbs_histo_off(c: args.c, ix: curr), n: nc ~mod* data_size)
}
this.gbs_block_size[args.c] = 0
} else if this.gbs_block_size[args.c] > 0 {
// Try merging the set of histograms for the current block type with
// the respective set of histograms for the last and second last
// block types. Decide over the split based on the total reduction of
// entropy across all contexts.
last0 = this.gbs_last_histogram_ix[(args.c * 2) + 0]
last1 = this.gbs_last_histogram_ix[(args.c * 2) + 1]
i = 0
while i < nc {
e = this.gbs_entropy!(wb: args.wb, off: this.gbs_histo_off(c: args.c, ix: curr ~mod+ i), size: alphabet_size)
this.gbs_entropy_tmp[i & 15] = e
j = 0
while j < 2 {
jx = (j ~mod* nc) ~mod+ i
tmp = this.mb_comb_off ~mod+ ((jx ~mod* data_size) ~mod* 4)
this.gbs_copy!(wb: args.wb, dst: tmp, src: this.gbs_histo_off(c: args.c, ix: curr ~mod+ i), n: data_size)
if j == 0 {
this.gbs_add_histogram!(wb: args.wb, dst: tmp, src: this.gbs_histo_off(c: args.c, ix: last0 ~mod+ i), n: data_size)
} else {
this.gbs_add_histogram!(wb: args.wb, dst: tmp, src: this.gbs_histo_off(c: args.c, ix: last1 ~mod+ i), n: data_size)
}
this.gbs_comb_entropy[jx & 31] = this.gbs_entropy!(wb: args.wb, off: tmp, size: alphabet_size)
d = this.bc.f64_sub(
a: this.bc.f64_sub(a: this.gbs_comb_entropy[jx & 31], b: e),
b: this.gbs_last_entropy[((le_base as base.u64) ~mod+ jx) & 127])
if j == 0 {
if nc == 1 {
diff0 = d
} else {
diff0 = this.bc.f64_add(a: diff0, b: d)
}
} else {
if nc == 1 {
diff1 = d
} else {
diff1 = this.bc.f64_add(a: diff1, b: d)
}
}
j ~mod+= 1
}
i ~mod+= 1
}
if (this.gbs_num_types[args.c] < this.gbs_max_block_types[args.c]) and
(this.bc.f64_gt(a: diff0, b: threshold) <> 0) and
(this.bc.f64_gt(a: diff1, b: threshold) <> 0) {
// Create new block.
this.poke_u32!(s: args.wb, i: lengths_off ~mod+ (num_blocks ~mod* 4),
v: (this.gbs_block_size[args.c] & 0xFFFF_FFFF) as base.u32)
this.set_byte!(s: args.wb, i: types_off ~mod+ num_blocks,
v: (this.gbs_num_types[args.c] & 0xFF) as base.u8)
this.gbs_last_histogram_ix[(args.c * 2) + 1] = last0
this.gbs_last_histogram_ix[(args.c * 2) + 0] = (this.gbs_num_types[args.c] as base.u64) ~mod* nc
i = 0
while i < nc {
this.gbs_last_entropy[(((le_base as base.u64) ~mod+ nc) ~mod+ i) & 127] =
this.gbs_last_entropy[((le_base as base.u64) ~mod+ i) & 127]
this.gbs_last_entropy[((le_base as base.u64) ~mod+ i) & 127] = this.gbs_entropy_tmp[i & 15]
i ~mod+= 1
}
this.gbs_num_blocks[args.c] = num_blocks ~mod+ 1
this.gbs_num_types[args.c] ~mod+= 1
curr ~mod+= nc
this.gbs_curr_histogram_ix[args.c] = curr
if curr < this.gbs_histograms_size[args.c] {
this.gbs_clear!(wb: args.wb, off: this.gbs_histo_off(c: args.c, ix: curr), n: nc ~mod* data_size)
}
this.gbs_block_size[args.c] = 0
this.gbs_merge_last_count[args.c] = 0
this.gbs_target_block_size[args.c] = this.gbs_min_block_size[args.c] as base.u64
} else if this.bc.f64_lt(a: diff1, b: this.bc.f64_sub(a: diff0, b: GBS_F64_20)) <> 0 {
// Combine this block with second last block.
this.poke_u32!(s: args.wb, i: lengths_off ~mod+ (num_blocks ~mod* 4),
v: (this.gbs_block_size[args.c] & 0xFFFF_FFFF) as base.u32)
this.set_byte!(s: args.wb, i: types_off ~mod+ num_blocks,
v: this.byte_at(s: args.wb, i: types_off ~mod+ (num_blocks ~mod- 2)))
this.gbs_last_histogram_ix[(args.c * 2) + 0] = last1
this.gbs_last_histogram_ix[(args.c * 2) + 1] = last0
i = 0
while i < nc {
this.gbs_copy!(wb: args.wb, dst: this.gbs_histo_off(c: args.c, ix: last1 ~mod+ i),
src: this.mb_comb_off ~mod+ (((nc ~mod+ i) ~mod* data_size) ~mod* 4), n: data_size)
this.gbs_last_entropy[(((le_base as base.u64) ~mod+ nc) ~mod+ i) & 127] =
this.gbs_last_entropy[((le_base as base.u64) ~mod+ i) & 127]
this.gbs_last_entropy[((le_base as base.u64) ~mod+ i) & 127] = this.gbs_comb_entropy[(nc ~mod+ i) & 31]
this.gbs_clear!(wb: args.wb, off: this.gbs_histo_off(c: args.c, ix: curr ~mod+ i), n: data_size)
i ~mod+= 1
}
this.gbs_num_blocks[args.c] = num_blocks ~mod+ 1
this.gbs_block_size[args.c] = 0
this.gbs_merge_last_count[args.c] = 0
this.gbs_target_block_size[args.c] = this.gbs_min_block_size[args.c] as base.u64
} else {
// Combine this block with last block.
tmp = lengths_off ~mod+ ((num_blocks ~mod- 1) ~mod* 4)
this.poke_u32!(s: args.wb, i: tmp,
v: this.load32le(s: args.wb, i: tmp) ~mod+ ((this.gbs_block_size[args.c] & 0xFFFF_FFFF) as base.u32))
i = 0
while i < nc {
this.gbs_copy!(wb: args.wb, dst: this.gbs_histo_off(c: args.c, ix: last0 ~mod+ i),
src: this.mb_comb_off ~mod+ ((i ~mod* data_size) ~mod* 4), n: data_size)
this.gbs_last_entropy[((le_base as base.u64) ~mod+ i) & 127] = this.gbs_comb_entropy[i & 31]
if this.gbs_num_types[args.c] == 1 {
this.gbs_last_entropy[(((le_base as base.u64) ~mod+ nc) ~mod+ i) & 127] =
this.gbs_last_entropy[((le_base as base.u64) ~mod+ i) & 127]
}
this.gbs_clear!(wb: args.wb, off: this.gbs_histo_off(c: args.c, ix: curr ~mod+ i), n: data_size)
i ~mod+= 1
}
this.gbs_block_size[args.c] = 0
this.gbs_merge_last_count[args.c] ~mod+= 1
if this.gbs_merge_last_count[args.c] > 1 {
this.gbs_target_block_size[args.c] ~mod+= this.gbs_min_block_size[args.c] as base.u64
}
}
}
if args.is_final {
this.gbs_histograms_size[args.c] = (this.gbs_num_types[args.c] as base.u64) ~mod* nc
this.gbs_split_num_blocks[args.c] = this.gbs_num_blocks[args.c]
}
}
// Mirrors FN(BlockSplitterAddSymbol) in metablock_inc.h and
// ContextBlockSplitterAddSymbol in metablock.c.
pri func encoder.gbs_add_symbol!(wb: slice base.u8, c: base.u32[..= 3], symbol: base.u64, context: base.u64) {
var off : base.u64
off = this.gbs_histo_off(c: args.c, ix: this.gbs_curr_histogram_ix[args.c] ~mod+ args.context) ~mod+ (args.symbol ~mod* 4)
this.poke_u32!(s: args.wb, i: off, v: this.load32le(s: args.wb, i: off) ~mod+ 1)
this.gbs_block_size[args.c] ~mod+= 1
if this.gbs_block_size[args.c] == this.gbs_target_block_size[args.c] {
this.gbs_finish_block!(wb: args.wb, c: args.c, is_final: false)
}
}
// static_context_map returns the C code's static_context_map[j] for the
// given number of literal contexts: kStaticContextMapSimpleUTF8 (2),
// kStaticContextMapContinuation (3) or kStaticContextMapComplexUTF8 (13).
pri func encoder.static_context_map(num_contexts: base.u32, j: base.u32) base.u32 {
if args.num_contexts == 2 {
return STATIC_CONTEXT_MAP_SIMPLE_UTF8[args.j & 63] as base.u32
} else if args.num_contexts == 3 {
return STATIC_CONTEXT_MAP_CONTINUATION[args.j & 63] as base.u32
} else if args.num_contexts == 13 {
return STATIC_CONTEXT_MAP_COMPLEX_UTF8[args.j & 63] as base.u32
}
return 0
}
// Mirrors BrotliBuildMetaBlockGreedy (and BrotliBuildMetaBlockGreedyInternal
// and MapStaticContexts) in metablock.c. The literal context mode is
// CONTEXT_UTF8, the only one below quality 10.
pri func encoder.build_meta_block_greedy!(wb: slice base.u8, pos: base.u64, prev_byte: base.u8, prev_byte2: base.u8,
num_contexts: base.u32, n_commands: base.u64) {
var pos : base.u64
var prev_byte : base.u8
var prev_byte2 : base.u8
var num_literals : base.u64
var i : base.u64
var j : base.u32
var k : base.u32
var literal : base.u8
var context : base.u32
var cmd_prefix : base.u16
var copy_len : base.u32
var num_types : base.u32
var offset : base.u32
pos = args.pos
prev_byte = args.prev_byte
prev_byte2 = args.prev_byte2
i = 0
while i < args.n_commands {
num_literals ~mod+= this.cmd_insert_len(wb: args.wb, i: i) as base.u64
i ~mod+= 1
}
this.gbs_init!(wb: args.wb, c: 0, alphabet_size: 256, data_size: 256, num_contexts: args.num_contexts,
min_block_size: 512, split_threshold: GBS_F64_400, num_symbols: num_literals)
this.gbs_init!(wb: args.wb, c: 1, alphabet_size: 704, data_size: 704, num_contexts: 1,
min_block_size: 1024, split_threshold: GBS_F64_500, num_symbols: args.n_commands)
this.gbs_init!(wb: args.wb, c: 2, alphabet_size: 64, data_size: 544, num_contexts: 1,
min_block_size: 512, split_threshold: GBS_F64_100, num_symbols: args.n_commands)
i = 0
while i < args.n_commands {
cmd_prefix = this.cmd_cmd_prefix(wb: args.wb, i: i)
this.gbs_add_symbol!(wb: args.wb, c: 1, symbol: cmd_prefix as base.u64, context: 0)
j = this.cmd_insert_len(wb: args.wb, i: i)
while j > 0 {
literal = this.rb_byte(wb: args.wb, pos: pos)
if args.num_contexts <= 1 {
this.gbs_add_symbol!(wb: args.wb, c: 0, symbol: literal as base.u64, context: 0)
} else {
context = (CONTEXT_LOOKUP[1024 + (prev_byte as base.u32)] |
CONTEXT_LOOKUP[1280 + (prev_byte2 as base.u32)]) as base.u32
this.gbs_add_symbol!(wb: args.wb, c: 0, symbol: literal as base.u64,
context: this.static_context_map(num_contexts: args.num_contexts, j: context) as base.u64)
}
prev_byte2 = prev_byte
prev_byte = literal
pos ~mod+= 1
j -= 1
}
copy_len = this.cmd_copy_len(wb: args.wb, i: i)
pos ~mod+= copy_len as base.u64
if copy_len <> 0 {
prev_byte2 = this.rb_byte(wb: args.wb, pos: pos ~mod- 2)
prev_byte = this.rb_byte(wb: args.wb, pos: pos ~mod- 1)
if cmd_prefix >= 128 {
this.gbs_add_symbol!(wb: args.wb, c: 2,
symbol: (this.cmd_dist_prefix(wb: args.wb, i: i) & 0x3FF) as base.u64, context: 0)
}
}
i ~mod+= 1
}
this.gbs_finish_block!(wb: args.wb, c: 0, is_final: true)
this.gbs_finish_block!(wb: args.wb, c: 1, is_final: true)
this.gbs_finish_block!(wb: args.wb, c: 2, is_final: true)
this.mb_cmap_size = 0
if args.num_contexts > 1 {
// MapStaticContexts.
num_types = this.gbs_num_types[0]
this.mb_cmap_size = (num_types as base.u64) << 6
k = 0
while k < num_types {
offset = k ~mod* args.num_contexts
j = 0
while j < 64 {
this.set_byte!(s: args.wb, i: this.mb_cmap_off ~mod+ (((k as base.u64) << 6) | (j as base.u64)),
v: ((offset ~mod+ this.static_context_map(num_contexts: args.num_contexts, j: j)) & 0xFF) as base.u8)
j ~mod+= 1
}
k ~mod+= 1
}
}
}
// Mirrors BrotliOptimizeHistograms in metablock.c.
pri func encoder.optimize_histograms!(wb: slice base.u8, num_distance_codes: base.u32) {
var i : base.u64
i = 0
while i < this.gbs_histograms_size[0] {
this.mb.optimize_huffman_counts_for_rle!(wb: args.wb, counts: this.gbs_histo_off(c: 0, ix: i), length: 256)
i ~mod+= 1
}
i = 0
while i < this.gbs_histograms_size[1] {
this.mb.optimize_huffman_counts_for_rle!(wb: args.wb, counts: this.gbs_histo_off(c: 1, ix: i), length: 704)
i ~mod+= 1
}
i = 0
while i < this.gbs_histograms_size[2] {
this.mb.optimize_huffman_counts_for_rle!(wb: args.wb, counts: this.gbs_histo_off(c: 2, ix: i),
length: args.num_distance_codes as base.u64)
i ~mod+= 1
}
}