std/brotli: add metablock_builder.workbuf_len_worst_case

Give the metablock builder an upper bound on the workbuf length that
build_meta_block needs, expose it through the metablock_tester and check
it against the high water mark in the parity harness. Add shard options
(-largeoff, -edge, -manytypes) and a stress phase that reaches 256 block
types to the harness.
This commit is contained in:
Claude committed 2026-10-27 01:03:11 +00:00
1 parent 202b406248
commit 9ad7ae6d69
4 files changed
+245 -6

No files matched your search

+121 -1
View File
@@ -10155,7 +10155,7 @@ struct wuffs_brotli__metablock_tester__struct {
uint32_t active_coroutine;
wuffs_base__vtable null_vtable;
uint32_t f_dummy;
uint64_t f_worst_case;
} private_impl;
struct {
@@ -61508,6 +61508,29 @@ wuffs_brotli__metablock_builder__optimize_huffman_counts_for_rle(
uint64_t a_counts,
uint64_t a_length);
WUFFS_BASE__GENERATED_C_CODE
static uint64_t
wuffs_brotli__metablock_builder__sbv_scratch_len(
const wuffs_brotli__metablock_builder* self,
uint64_t a_length,
uint64_t a_hs,
uint64_t a_symbols_per_histogram,
uint64_t a_max_histograms);
WUFFS_BASE__GENERATED_C_CODE
static uint64_t
wuffs_brotli__metablock_builder__cluster_scratch_len(
const wuffs_brotli__metablock_builder* self,
uint64_t a_in_size,
uint64_t a_hs);
WUFFS_BASE__GENERATED_C_CODE
static uint64_t
wuffs_brotli__metablock_builder__workbuf_len_worst_case(
const wuffs_brotli__metablock_builder* self,
uint64_t a_num_literals,
uint64_t a_num_commands);
// ---------------- VTables
const wuffs_base__io_transformer__func_ptrs
@@ -72490,6 +72513,93 @@ wuffs_brotli__metablock_builder__optimize_huffman_counts_for_rle(
return wuffs_base__make_empty_struct();
}
// -------- func brotli.metablock_builder.sbv_scratch_len
WUFFS_BASE__GENERATED_C_CODE
static uint64_t
wuffs_brotli__metablock_builder__sbv_scratch_len(
const wuffs_brotli__metablock_builder* self,
uint64_t a_length,
uint64_t a_hs,
uint64_t a_symbols_per_histogram,
uint64_t a_max_histograms) {
uint64_t v_nh = 0;
uint64_t v_a = 0;
uint64_t v_b = 0;
uint64_t v_nb = 0;
if (a_length < 128u) {
return 0u;
}
v_nh = wuffs_base__u64__sat_add((a_length / wuffs_base__u64__max(a_symbols_per_histogram, 1u)), 1u);
v_nh = wuffs_base__u64__min(v_nh, a_max_histograms);
v_a = wuffs_base__u64__sat_add(((uint64_t)(wuffs_base__u64__sat_add(v_nh, 1u) * a_hs)), ((uint64_t)(a_length * (wuffs_base__u64__sat_add(v_nh, 7u) >> 3u))));
v_nb = a_length;
v_b = wuffs_base__u64__sat_add(((uint64_t)(v_nb * wuffs_base__u64__sat_add(a_hs, 24u))), 1024u);
wuffs_private_impl__u64__sat_add_indirect(&v_b, ((uint64_t)(wuffs_base__u64__min(v_nb, 64u) * a_hs)));
wuffs_private_impl__u64__sat_add_indirect(&v_b, wuffs_base__u64__sat_add(49176u, ((uint64_t)(2u * a_hs))));
v_nb = ((uint64_t)(64u * a_length));
v_nb = wuffs_base__u64__min(v_nb, ((uint64_t)((a_length / 2u) * a_length)));
wuffs_private_impl__u64__sat_add_indirect(&v_b, ((uint64_t)(wuffs_base__u64__sat_add(v_nb, 1u) * 24u)));
return wuffs_base__u64__sat_add(a_length, wuffs_base__u64__max(v_a, v_b));
}
// -------- func brotli.metablock_builder.cluster_scratch_len
WUFFS_BASE__GENERATED_C_CODE
static uint64_t
wuffs_brotli__metablock_builder__cluster_scratch_len(
const wuffs_brotli__metablock_builder* self,
uint64_t a_in_size,
uint64_t a_hs) {
uint64_t v_n = 0;
uint64_t v_p = 0;
v_n = wuffs_base__u64__sat_add(wuffs_base__u64__sat_add(((uint64_t)(12u * a_in_size)), 49176u), a_hs);
v_p = ((uint64_t)(64u * a_in_size));
v_p = wuffs_base__u64__min(v_p, ((uint64_t)((a_in_size / 2u) * a_in_size)));
wuffs_private_impl__u64__sat_add_indirect(&v_n, ((uint64_t)(wuffs_base__u64__sat_add(v_p, 1u) * 24u)));
return wuffs_base__u64__sat_add(v_n, ((uint64_t)(a_in_size * a_hs)));
}
// -------- func brotli.metablock_builder.workbuf_len_worst_case
WUFFS_BASE__GENERATED_C_CODE
static uint64_t
wuffs_brotli__metablock_builder__workbuf_len_worst_case(
const wuffs_brotli__metablock_builder* self,
uint64_t a_num_literals,
uint64_t a_num_commands) {
uint64_t v_l = 0;
uint64_t v_c = 0;
uint64_t v_n = 0;
uint64_t v_split = 0;
v_l = wuffs_base__u64__min(a_num_literals, 4294967295u);
v_c = wuffs_base__u64__min(a_num_commands, 4294967295u);
v_n = wuffs_base__u64__sat_add(((uint64_t)(5u * wuffs_base__u64__max(v_l, 1u))), ((uint64_t)(10u * wuffs_base__u64__max(v_c, 1u))));
v_split = wuffs_base__u64__sat_add(v_l, wuffs_brotli__metablock_builder__sbv_scratch_len(self,
v_l,
1040u,
544u,
100u));
v_split = wuffs_base__u64__max(v_split, wuffs_base__u64__sat_add(((uint64_t)(2u * v_c)), wuffs_brotli__metablock_builder__sbv_scratch_len(self,
v_c,
2832u,
530u,
50u)));
v_split = wuffs_base__u64__max(v_split, wuffs_base__u64__sat_add(((uint64_t)(2u * v_c)), wuffs_brotli__metablock_builder__sbv_scratch_len(self,
v_c,
2192u,
544u,
50u)));
wuffs_private_impl__u64__sat_add_indirect(&v_n, wuffs_base__u64__max(v_split, 2192u));
wuffs_private_impl__u64__sat_add_indirect(&v_n, 20078592u);
wuffs_private_impl__u64__sat_add_indirect(&v_n, 19283968u);
wuffs_private_impl__u64__sat_add_indirect(&v_n, wuffs_base__u64__max(wuffs_brotli__metablock_builder__cluster_scratch_len(self, 16384u, 1040u), wuffs_brotli__metablock_builder__cluster_scratch_len(self, 1024u, 2192u)));
return v_n;
}
// -------- func brotli.metablock_tester.bits_entropy
WUFFS_BASE__GENERATED_C_CODE
@@ -72691,7 +72801,15 @@ wuffs_brotli__metablock_tester__build_meta_block(
}
wuffs_base__status v_status = wuffs_base__make_status(NULL);
uint64_t v_i = 0;
uint64_t v_literals = 0;
v_i = 0u;
while (v_i < a_num_commands) {
v_literals += ((uint64_t)(wuffs_brotli__metablock_builder__ld32(&self->private_data.f_mb, a_cmds, ((uint64_t)(v_i * 16u)))));
v_i += 1u;
}
self->private_impl.f_worst_case = wuffs_brotli__metablock_builder__workbuf_len_worst_case(&self->private_data.f_mb, v_literals, a_num_commands);
wuffs_brotli__metablock_builder__set_params(&self->private_data.f_mb, a_quality, (a_large_window != 0u), (a_disable_literal_context_modeling != 0u));
wuffs_brotli__metablock_builder__init_distance_params(&self->private_data.f_mb, a_npostfix, a_ndirect, (a_large_window != 0u));
v_status = wuffs_brotli__metablock_builder__build_meta_block(&self->private_data.f_mb,
@@ -72775,6 +72893,8 @@ wuffs_brotli__metablock_tester__get(
return self->private_data.f_mb.private_impl.f_dist_max_distance;
} else if (a_key == 29u) {
return self->private_data.f_mb.private_impl.f_wb_high;
} else if (a_key == 30u) {
return self->private_impl.f_worst_case;
}
return 0u;
}
+33 -3
View File
@@ -891,9 +891,16 @@ test_build_meta_block(block_t* b, int quality, bool disable_lcm,
}
{
uint64_t used = wuffs_brotli__metablock_tester__get(g_tester, 29);
uint64_t worst = wuffs_brotli__metablock_tester__get(g_tester, 30);
if (g_max_workbuf_used < used) {
g_max_workbuf_used = used;
}
g_num_checks++;
if (used > worst) {
g_num_mismatches++;
printf("workbuf used (%" PRIu64 ") > worst case (%" PRIu64 ") (%s)\n",
used, worst, g_config_name);
}
}
CHECK_EQ("dist.distance_postfix_bits", params.dist.distance_postfix_bits,
@@ -1080,6 +1087,7 @@ test_cluster_histograms(int iters) {
// ---------------- Configurations
static block_t g_block;
static uint32_t g_force_shift_max = 0;
static void //
@@ -1105,7 +1113,9 @@ run_config(size_t total, int style, int cmd_mode, bool do_split,
if (rng_below(8) == 0) {
quality = 4 + (int)rng_below(6);
}
g_shift_max = rng_below(2) ? 8192 : 64 + rng_below(2000);
g_shift_max = g_force_shift_max ? g_force_shift_max
: rng_below(2) ? 8192
: 64 + rng_below(2000);
g_dist_style = (int)rng_below(3);
g_dist_shift = 1 + rng_below(3);
g_dist_residue = rng_below(1u << g_dist_shift);
@@ -1152,6 +1162,7 @@ int //
main(int argc, char** argv) {
bool quick = false;
int n_bitcost = 200000, n_clusters = 3000, n_configs = 20000, n_large = 8;
int large_off = 0, n_many_types = 0, n_edge = 64;
int i;
for (i = 1; i < argc; i++) {
if (!strcmp(argv[i], "-quick")) {
@@ -1168,6 +1179,12 @@ main(int argc, char** argv) {
n_configs = atoi(argv[i] + 9);
} else if (!strncmp(argv[i], "-large=", 7)) {
n_large = atoi(argv[i] + 7);
} else if (!strncmp(argv[i], "-largeoff=", 10)) {
large_off = atoi(argv[i] + 10);
} else if (!strncmp(argv[i], "-manytypes=", 11)) {
n_many_types = atoi(argv[i] + 11);
} else if (!strncmp(argv[i], "-edge=", 6)) {
n_edge = atoi(argv[i] + 6);
} else if (!strncmp(argv[i], "-seed=", 6)) {
g_rng_state = strtoull(argv[i] + 6, NULL, 0) | 1;
}
@@ -1199,14 +1216,14 @@ main(int argc, char** argv) {
test_cluster_histograms(n_clusters);
// Edge cases: no commands, one command, one literal, all identical bytes,
// in every literal context mode.
for (i = 0; i < 64; i++) {
for (i = 0; i < n_edge; i++) {
static const size_t totals[8] = {0, 1, 2, 5, 127, 128, 129, 3000};
run_config(totals[i & 7], (i & 8) ? STYLE_IDENTICAL : STYLE_RANDOM,
(i >> 4) % 3, true, true);
}
// Large configurations: at least 1 MiB of literals, and at least 100k
// commands.
for (i = 0; i < n_large; i++) {
for (i = large_off; i < large_off + n_large; i++) {
static const int styles[4] = {STYLE_FILE, STYLE_RANDOM, STYLE_SHIFTING,
STYLE_UTF8};
if (i & 1) {
@@ -1220,6 +1237,19 @@ main(int argc, char** argv) {
i, g_num_mismatches);
fflush(stdout);
}
// Inputs whose byte distribution changes every few hundred bytes, which
// push the number of block types towards the 256 maximum (and the number
// of FindBlocks histograms towards kMaxLiteralHistograms).
for (i = 0; i < n_many_types; i++) {
g_force_shift_max = 64 + rng_below(600);
run_config((size_t)1 << (17 + rng_below(4)), STYLE_SHIFTING,
(int)rng_below(2), true, true);
g_force_shift_max = 0;
printf("many-types configuration %d done, %" PRIu64
" mismatches so far\n",
i, g_num_mismatches);
fflush(stdout);
}
{
int n = n_configs;
for (i = 0; i < n; i++) {
+76
View File
@@ -971,3 +971,79 @@ pri func metablock_builder.optimize_huffman_counts_for_rle!(wb: slice base.u8, c
i ~mod+= 1
}
}
// sbv_scratch_len is an upper bound on how much workbuf (beyond the split
// arrays) split_byte_vector uses, for length symbols with the given
// histogram size, symbols_per_histogram and max_histograms.
pri func metablock_builder.sbv_scratch_len(length: base.u64, hs: base.u64, symbols_per_histogram: base.u64, max_histograms: base.u64) base.u64 {
var nh : base.u64
var a : base.u64
var b : base.u64
var nb : base.u64
if args.length < 128 {
return 0
}
nh = (args.length / args.symbols_per_histogram.max(no_less_than: 1)) ~sat+ 1
nh = nh.min(no_more_than: args.max_histograms)
// The histograms and the switch signal bitmaps.
a = ((nh ~sat+ 1) ~mod* args.hs) ~sat+ (args.length ~mod* ((nh ~sat+ 7) >> 3))
// cluster_blocks, with num_blocks (and num_clusters) at most length.
nb = args.length
b = (nb ~mod* (args.hs ~sat+ 24)) ~sat+ 1024
b ~sat+= (nb.min(no_more_than: 64)) ~mod* args.hs
b ~sat+= (2049 * HISTOGRAM_PAIR_SIZE) ~sat+ (2 ~mod* args.hs)
nb = 64 ~mod* args.length
nb = nb.min(no_more_than: (args.length / 2) ~mod* args.length)
b ~sat+= (nb ~sat+ 1) ~mod* HISTOGRAM_PAIR_SIZE
return args.length ~sat+ a.max(no_less_than: b)
}
// cluster_scratch_len is an upper bound on how much workbuf
// cluster_histograms uses, for in_size histograms of hs bytes.
pri func metablock_builder.cluster_scratch_len(in_size: base.u64, hs: base.u64) base.u64 {
var n : base.u64
var p : base.u64
n = ((12 ~mod* args.in_size) ~sat+ (2049 * HISTOGRAM_PAIR_SIZE)) ~sat+ args.hs
p = 64 ~mod* args.in_size
p = p.min(no_more_than: (args.in_size / 2) ~mod* args.in_size)
n ~sat+= (p ~sat+ 1) ~mod* HISTOGRAM_PAIR_SIZE
return n ~sat+ (args.in_size ~mod* args.hs)
}
// workbuf_len_worst_case returns an upper bound on the workbuf length that
// build_meta_block needs for a metablock with num_literals literals (the sum
// of the commands' insert lengths) and num_commands commands. The bound
// assumes the worst case for every allocation that the C code sizes at run
// time (e.g. one FindBlocks block per symbol), so it is much larger than what
// typical inputs need. The wb_high field records the actual high water mark.
pri func metablock_builder.workbuf_len_worst_case(num_literals: base.u64, num_commands: base.u64) base.u64 {
var l : base.u64
var c : base.u64
var n : base.u64
var split : base.u64
// Metablocks are at most (1 << 24) bytes long, so these clamps (which
// keep the arithmetic below from overflowing) change nothing.
l = args.num_literals.min(no_more_than: 0xFFFF_FFFF)
c = args.num_commands.min(no_more_than: 0xFFFF_FFFF)
// The split arrays (types and lengths) for the three categories.
n = (5 ~mod* l.max(no_less_than: 1)) ~sat+ (10 ~mod* c.max(no_less_than: 1))
// split_block's scratch.
split = l ~sat+ this.sbv_scratch_len(length: l, hs: HISTOGRAM_LITERAL_SIZE, symbols_per_histogram: 544, max_histograms: 100)
split = split.max(no_less_than: (2 ~mod* c) ~sat+
this.sbv_scratch_len(length: c, hs: HISTOGRAM_COMMAND_SIZE, symbols_per_histogram: 530, max_histograms: 50))
split = split.max(no_less_than: (2 ~mod* c) ~sat+
this.sbv_scratch_len(length: c, hs: HISTOGRAM_DISTANCE_SIZE, symbols_per_histogram: 544, max_histograms: 50))
n ~sat+= split.max(no_less_than: HISTOGRAM_DISTANCE_SIZE)
// The MetaBlockSplit's histograms and context maps (with at most 256
// block types per category), plus the input histograms for clustering.
n ~sat+= (256 * HISTOGRAM_COMMAND_SIZE) + (16384 * 4) + (16384 * HISTOGRAM_LITERAL_SIZE) +
(1024 * 4) + (1024 * HISTOGRAM_DISTANCE_SIZE)
n ~sat+= (16384 * HISTOGRAM_LITERAL_SIZE) + (1024 * HISTOGRAM_DISTANCE_SIZE)
// cluster_histograms' scratch.
n ~sat+= this.cluster_scratch_len(in_size: 16384, hs: HISTOGRAM_LITERAL_SIZE).max(no_less_than:
this.cluster_scratch_len(in_size: 1024, hs: HISTOGRAM_DISTANCE_SIZE))
return n
}
+15 -2
View File
@@ -18,7 +18,9 @@
// their semantics may change at any time.
pub struct metablock_tester?(
dummy : base.u32,
// worst_case is the workbuf_len_worst_case for the last
// build_meta_block call.
worst_case : base.u64,
) + (
// Wuffs' C code generator expects same-package struct fields to be in
// the private_data section (after the "+").
@@ -129,8 +131,16 @@ pub func metablock_tester.build_meta_block!(workbuf: slice base.u8, ringbuffer:
prev_byte: base.u8, prev_byte2: base.u8, cmds: slice base.u8, num_commands: base.u64,
literal_context_mode: base.u32, quality: base.u32, large_window: base.u32,
disable_literal_context_modeling: base.u32, npostfix: base.u32, ndirect: base.u32) base.status {
var status : base.status
var status : base.status
var i : base.u64
var literals : base.u64
i = 0
while i < args.num_commands {
literals ~mod+= this.mb.ld32(wb: args.cmds, i: i ~mod* 16) as base.u64
i ~mod+= 1
}
this.worst_case = this.mb.workbuf_len_worst_case(num_literals: literals, num_commands: args.num_commands)
this.mb.set_params!(quality: args.quality, large_window: args.large_window <> 0,
disable_literal_context_modeling: args.disable_literal_context_modeling <> 0)
this.mb.init_distance_params!(npostfix: args.npostfix, ndirect: args.ndirect, large_window: args.large_window <> 0)
@@ -160,6 +170,7 @@ pub func metablock_tester.optimize_histograms!(workbuf: slice base.u8, num_dista
// num_direct_distance_codes, alphabet_size_max, alphabet_size_limit and
// max_distance.
// - 29: the highest workbuf offset used (the workbuf length needed).
// - 30: the workbuf_len_worst_case for the last build_meta_block call.
pub func metablock_tester.get(key: base.u32) base.u64 {
var c : base.u32[..= 2]
@@ -195,6 +206,8 @@ pub func metablock_tester.get(key: base.u32) base.u64 {
return this.mb.dist_max_distance
} else if args.key == 29 {
return this.mb.wb_high
} else if args.key == 30 {
return this.worst_case
}
return 0
}