std/brotli: add metablock_builder.workbuf_len_worst_case

Give the metablock builder an upper bound on the workbuf length that
build_meta_block needs, expose it through the metablock_tester and check
it against the high water mark in the parity harness. Add shard options
(-largeoff, -edge, -manytypes) and a stress phase that reaches 256 block
types to the harness.
This commit is contained in:
Claude committed 2026-10-27 01:03:11 +00:00
1 parent 202b406248
commit 9ad7ae6d69
4 files changed
+245 -6

No files matched your search

+76
View File
@@ -971,3 +971,79 @@ pri func metablock_builder.optimize_huffman_counts_for_rle!(wb: slice base.u8, c
i ~mod+= 1
}
}
// sbv_scratch_len is an upper bound on how much workbuf (beyond the split
// arrays) split_byte_vector uses, for length symbols with the given
// histogram size, symbols_per_histogram and max_histograms.
pri func metablock_builder.sbv_scratch_len(length: base.u64, hs: base.u64, symbols_per_histogram: base.u64, max_histograms: base.u64) base.u64 {
var nh : base.u64
var a : base.u64
var b : base.u64
var nb : base.u64
if args.length < 128 {
return 0
}
nh = (args.length / args.symbols_per_histogram.max(no_less_than: 1)) ~sat+ 1
nh = nh.min(no_more_than: args.max_histograms)
// The histograms and the switch signal bitmaps.
a = ((nh ~sat+ 1) ~mod* args.hs) ~sat+ (args.length ~mod* ((nh ~sat+ 7) >> 3))
// cluster_blocks, with num_blocks (and num_clusters) at most length.
nb = args.length
b = (nb ~mod* (args.hs ~sat+ 24)) ~sat+ 1024
b ~sat+= (nb.min(no_more_than: 64)) ~mod* args.hs
b ~sat+= (2049 * HISTOGRAM_PAIR_SIZE) ~sat+ (2 ~mod* args.hs)
nb = 64 ~mod* args.length
nb = nb.min(no_more_than: (args.length / 2) ~mod* args.length)
b ~sat+= (nb ~sat+ 1) ~mod* HISTOGRAM_PAIR_SIZE
return args.length ~sat+ a.max(no_less_than: b)
}
// cluster_scratch_len is an upper bound on how much workbuf
// cluster_histograms uses, for in_size histograms of hs bytes.
pri func metablock_builder.cluster_scratch_len(in_size: base.u64, hs: base.u64) base.u64 {
var n : base.u64
var p : base.u64
n = ((12 ~mod* args.in_size) ~sat+ (2049 * HISTOGRAM_PAIR_SIZE)) ~sat+ args.hs
p = 64 ~mod* args.in_size
p = p.min(no_more_than: (args.in_size / 2) ~mod* args.in_size)
n ~sat+= (p ~sat+ 1) ~mod* HISTOGRAM_PAIR_SIZE
return n ~sat+ (args.in_size ~mod* args.hs)
}
// workbuf_len_worst_case returns an upper bound on the workbuf length that
// build_meta_block needs for a metablock with num_literals literals (the sum
// of the commands' insert lengths) and num_commands commands. The bound
// assumes the worst case for every allocation that the C code sizes at run
// time (e.g. one FindBlocks block per symbol), so it is much larger than what
// typical inputs need. The wb_high field records the actual high water mark.
pri func metablock_builder.workbuf_len_worst_case(num_literals: base.u64, num_commands: base.u64) base.u64 {
var l : base.u64
var c : base.u64
var n : base.u64
var split : base.u64
// Metablocks are at most (1 << 24) bytes long, so these clamps (which
// keep the arithmetic below from overflowing) change nothing.
l = args.num_literals.min(no_more_than: 0xFFFF_FFFF)
c = args.num_commands.min(no_more_than: 0xFFFF_FFFF)
// The split arrays (types and lengths) for the three categories.
n = (5 ~mod* l.max(no_less_than: 1)) ~sat+ (10 ~mod* c.max(no_less_than: 1))
// split_block's scratch.
split = l ~sat+ this.sbv_scratch_len(length: l, hs: HISTOGRAM_LITERAL_SIZE, symbols_per_histogram: 544, max_histograms: 100)
split = split.max(no_less_than: (2 ~mod* c) ~sat+
this.sbv_scratch_len(length: c, hs: HISTOGRAM_COMMAND_SIZE, symbols_per_histogram: 530, max_histograms: 50))
split = split.max(no_less_than: (2 ~mod* c) ~sat+
this.sbv_scratch_len(length: c, hs: HISTOGRAM_DISTANCE_SIZE, symbols_per_histogram: 544, max_histograms: 50))
n ~sat+= split.max(no_less_than: HISTOGRAM_DISTANCE_SIZE)
// The MetaBlockSplit's histograms and context maps (with at most 256
// block types per category), plus the input histograms for clustering.
n ~sat+= (256 * HISTOGRAM_COMMAND_SIZE) + (16384 * 4) + (16384 * HISTOGRAM_LITERAL_SIZE) +
(1024 * 4) + (1024 * HISTOGRAM_DISTANCE_SIZE)
n ~sat+= (16384 * HISTOGRAM_LITERAL_SIZE) + (1024 * HISTOGRAM_DISTANCE_SIZE)
// cluster_histograms' scratch.
n ~sat+= this.cluster_scratch_len(in_size: 16384, hs: HISTOGRAM_LITERAL_SIZE).max(no_less_than:
this.cluster_scratch_len(in_size: 1024, hs: HISTOGRAM_DISTANCE_SIZE))
return n
}
+15 -2
View File
@@ -18,7 +18,9 @@
// their semantics may change at any time.
pub struct metablock_tester?(
dummy : base.u32,
// worst_case is the workbuf_len_worst_case for the last
// build_meta_block call.
worst_case : base.u64,
) + (
// Wuffs' C code generator expects same-package struct fields to be in
// the private_data section (after the "+").
@@ -129,8 +131,16 @@ pub func metablock_tester.build_meta_block!(workbuf: slice base.u8, ringbuffer:
prev_byte: base.u8, prev_byte2: base.u8, cmds: slice base.u8, num_commands: base.u64,
literal_context_mode: base.u32, quality: base.u32, large_window: base.u32,
disable_literal_context_modeling: base.u32, npostfix: base.u32, ndirect: base.u32) base.status {
var status : base.status
var status : base.status
var i : base.u64
var literals : base.u64
i = 0
while i < args.num_commands {
literals ~mod+= this.mb.ld32(wb: args.cmds, i: i ~mod* 16) as base.u64
i ~mod+= 1
}
this.worst_case = this.mb.workbuf_len_worst_case(num_literals: literals, num_commands: args.num_commands)
this.mb.set_params!(quality: args.quality, large_window: args.large_window <> 0,
disable_literal_context_modeling: args.disable_literal_context_modeling <> 0)
this.mb.init_distance_params!(npostfix: args.npostfix, ndirect: args.ndirect, large_window: args.large_window <> 0)
@@ -160,6 +170,7 @@ pub func metablock_tester.optimize_histograms!(workbuf: slice base.u8, num_dista
// num_direct_distance_codes, alphabet_size_max, alphabet_size_limit and
// max_distance.
// - 29: the highest workbuf offset used (the workbuf length needed).
// - 30: the workbuf_len_worst_case for the last build_meta_block call.
pub func metablock_tester.get(key: base.u32) base.u64 {
var c : base.u32[..= 2]
@@ -195,6 +206,8 @@ pub func metablock_tester.get(key: base.u32) base.u64 {
return this.mb.dist_max_distance
} else if args.key == 29 {
return this.mb.wb_high
} else if args.key == 30 {
return this.worst_case
}
return 0
}