From 5393d92234cf68edf8c34c9095e02200125d5157 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 01:12:51 +0000 Subject: [PATCH] std/brotli: document the metablock builder and add golden tests Bound the number of block types by the number of symbols in workbuf_len_worst_case, document the metablock_builder interface and its workbuf layout in README.md, and add deterministic bit cost and build_meta_block tests (with golden values from libbrotlienc v1.2.1) to test/c/std/brotli.c. --- release/c/wuffs-unsupported-snapshot.c | 14 +- std/brotli/README.md | 125 +++++++- std/brotli/encode_metablock.wuffs | 22 +- test/c/std/brotli.c | 381 +++++++++++++++++++++++++ 4 files changed, 531 insertions(+), 11 deletions(-) diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 9edd595e..3775c4f8 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -72572,6 +72572,8 @@ wuffs_brotli__metablock_builder__workbuf_len_worst_case( uint64_t a_num_commands) { uint64_t v_l = 0; uint64_t v_c = 0; + uint64_t v_tl = 0; + uint64_t v_tc = 0; uint64_t v_n = 0; uint64_t v_split = 0; @@ -72594,9 +72596,15 @@ wuffs_brotli__metablock_builder__workbuf_len_worst_case( 544u, 50u))); wuffs_private_impl__u64__sat_add_indirect(&v_n, wuffs_base__u64__max(v_split, 2192u)); - wuffs_private_impl__u64__sat_add_indirect(&v_n, 20078592u); - wuffs_private_impl__u64__sat_add_indirect(&v_n, 19283968u); - wuffs_private_impl__u64__sat_add_indirect(&v_n, wuffs_base__u64__max(wuffs_brotli__metablock_builder__cluster_scratch_len(self, 16384u, 1040u), wuffs_brotli__metablock_builder__cluster_scratch_len(self, 1024u, 2192u))); + v_tl = wuffs_base__u64__min(wuffs_base__u64__max(v_l, 1u), 256u); + v_tc = wuffs_base__u64__min(wuffs_base__u64__max(v_c, 1u), 256u); + wuffs_private_impl__u64__sat_add_indirect(&v_n, ((v_tc * 2832u) + + (v_tl * 64u * 4u) + + (v_tl * 64u * 1040u) + + (v_tc * 4u * 4u) + + (v_tc * 4u * 2192u))); + wuffs_private_impl__u64__sat_add_indirect(&v_n, ((v_tl * 64u * 1040u) + (v_tc * 4u * 2192u))); + wuffs_private_impl__u64__sat_add_indirect(&v_n, wuffs_base__u64__max(wuffs_brotli__metablock_builder__cluster_scratch_len(self, (v_tl * 64u), 1040u), wuffs_brotli__metablock_builder__cluster_scratch_len(self, (v_tc * 4u), 2192u))); return v_n; } diff --git a/std/brotli/README.md b/std/brotli/README.md index db513a40..25e0a3a7 100644 --- a/std/brotli/README.md +++ b/std/brotli/README.md @@ -195,6 +195,15 @@ The source files are: - `encode_histogram.wuffs`: `histogram.h`, `histogram_inc.h` and `histogram.c` (histogram helpers and `BrotliBuildHistogramsWithContext`), also as `bit_cost` methods. +- `encode_block_splitter.wuffs`: `block_splitter.c` and + `block_splitter_inc.h` (`BrotliSplitBlock`), as methods of the + `metablock_builder` struct. +- `encode_cluster.wuffs`: `cluster.c` and `cluster_inc.h` + (`BrotliClusterHistograms` and its helpers), also as `metablock_builder` + methods. +- `encode_metablock.wuffs`: the `metablock_builder` struct and `metablock.c` + (`BrotliInitDistanceParams`, `BrotliBuildMetaBlock`, + `BrotliOptimizeHistograms`). See "Encoder Metablock Builder" below. - `encode_tester.wuffs`: TEST ONLY. The `metablock_tester` struct exposes the internal functions above to C test programs. It is not part of the encoder API. @@ -330,10 +339,124 @@ some synthetic inputs). only in the C code's one-shot `BrotliEncoderCompress`). +### Encoder Metablock Builder + +The `metablock_builder` struct (a `pri struct`, embeddable in the encoder +with one field, e.g. `mb : metablock_builder` in the private data section) +implements the quality 10 and 11 metablock construction (`BrotliBuildMetaBlock`) +and `BrotliOptimizeHistograms`. It is not yet called by the encoder. Its +fixed-size state (about 0.8 MiB: FindBlocks' cost arrays and a 64Ki-entry +FastLog2 cache in its `bit_cost`) lives in the struct; everything that the C +code allocates on the heap lives in a caller-supplied workbuf. + +Data formats (all little-endian, matching the C structs' memory layout on a +64-bit little-endian machine, so that results can be compared with `memcmp`): + +- A command is 16 bytes: `insert_len_` (u32), `copy_len_` (u32), `dist_extra_` + (u32), `cmd_prefix_` (u16), `dist_prefix_` (u16), as built by `command.h`'s + `InitCommand` / `InitInsertCommand`. +- A histogram is `(4 * data_size) + 16` bytes: `data_size` u32 counts, the + u64 `total_count_` and the binary64 bits of `bit_cost_`. `data_size` is 256 + (literals, 1040 bytes), 704 (commands, 2832 bytes) or 544 (distances, 2192 + bytes). +- A BlockSplit's types are u8s and its lengths u32s. Context maps are u32s. +- Doubles are binary64 bit patterns (u64), computed with `std/softfloat` in + the C code's operation order. + +Usage, mirroring the C code's `WriteMetaBlockInternal` for quality >= 10: + +``` +// Once per metablock. The params are the C code's block_params, a copy of +// the encoder's params: init_distance_params must be called each time, +// because build_meta_block overwrites the dist_etc fields with the chosen +// distance params. +this.mb.set_params!(quality: q, large_window: lw, + disable_literal_context_modeling: dlcm) +this.mb.init_distance_params!(npostfix: np, ndirect: nd, large_window: lw) +status = this.mb.build_meta_block!(wb: workbuf, ringbuffer: rb, pos: pos, + mask: mask, prev_byte: pb, prev_byte2: pb2, cmds: cmds, + num_commands: n, literal_context_mode: mode) +// Optional (quality >= 4): ndc is min(this.mb.dist_alphabet_size_limit, 544). +this.mb.optimize_histograms!(wb: workbuf, num_distance_codes: ndc) +``` + +The exact signatures are: + +- `set_params!(quality: u32, large_window: bool, + disable_literal_context_modeling: bool)`. +- `init_distance_params!(npostfix: u32, ndirect: u32, large_window: bool)` + (`BrotliInitDistanceParams`; sets `dist_postfix_bits`, + `dist_num_direct_codes`, `dist_alphabet_size_max`, + `dist_alphabet_size_limit` and `dist_max_distance`). +- `build_meta_block!(wb: slice u8, ringbuffer: roslice u8, pos: u64, mask: + u64, prev_byte: u8, prev_byte2: u8, cmds: slice u8, num_commands: u64, + literal_context_mode: u32) status`. `ringbuffer` must be at least `mask + 1` + bytes. `literal_context_mode` is a `ContextType` (0 = LSB6, 1 = MSB6, 2 = + UTF8, 3 = SIGNED). Like the C code, it rewrites the commands' + `dist_prefix_` and `dist_extra_` fields for the chosen distance params and + leaves those params in the `dist_etc` fields. It returns + `"#bad workbuf length"` if `wb` is too short (the results are then invalid + but `cmds` may already be rewritten, so retrying needs the original + commands). +- `optimize_histograms!(wb: slice u8, num_distance_codes: u32)` works on the + MetaBlockSplit described by the fields below, whether `build_meta_block` + or the caller (e.g. a greedy builder) filled them in. +- `workbuf_len_worst_case(num_literals: u64, num_commands: u64) u64` is an + upper bound on the workbuf length that `build_meta_block` needs, where + `num_literals` is the sum of the commands' insert lengths. It assumes the + worst case for every run-time-sized allocation (one `FindBlocks` block per + symbol, 256 block types with 64 literal contexts each, full histogram pair + queues), so it is far larger than what real inputs need: about 76 MiB for + a 1 KiB metablock, 2.6 GiB for 1 MiB of literals and 128 Ki commands, and + 41 GiB for 16 MiB of literals and 4 Mi commands. These are the C code's + own worst cases (`ClusterBlocks` and `BrotliClusterHistograms` size their + histogram pair queues by the number of blocks and clusters), not artifacts + of the port. `wb_high` records what a call actually used: over the parity + harness's inputs, at most about 42 MB (for 1.3 MiB of literals split into + 256 literal block types). + +The MetaBlockSplit result is described by these fields, indexed by category +(0 = literal, 1 = command, 2 = distance), with offsets into `wb`: + +| Field | C code | +| ---------------------- | ----------------------------------------------- | +| `split_num_types[c]` | `{literal,command,distance}_split.num_types` | +| `split_num_blocks[c]` | `..._split.num_blocks` | +| `split_types_off[c]` | `..._split.types` (u8 array) | +| `split_lengths_off[c]` | `..._split.lengths` (u32 array) | +| `context_map_off[c]` | `{literal,distance}_context_map` (u32 array; none for commands) | +| `context_map_size[c]` | `{literal,distance}_context_map_size` | +| `histograms_off[c]` | `{literal,command,distance}_histograms` | +| `histograms_size[c]` | `..._histograms_size` | + +Workbuf layout during `build_meta_block` (a bump allocator, `wb_top`, carves +regions out of `wb` in this order; "scratch" regions are released when the +C code frees them, so later regions reuse their bytes): + +1. Scratch: one distance histogram for the distance parameter search. +2. The three BlockSplits' types and lengths arrays, sized for one block per + symbol (`max(1, n)` entries, where n is the number of literals, commands + or distance codes). +3. Scratch: `BrotliSplitBlock`'s literal / command code / distance code + arrays and `SplitByteVector`'s histograms, block ids, bitmaps and + `ClusterBlocks` state. +4. The command histograms, literal context map, literal histograms (room for + `64 * num_literal_types`), distance context map and distance histograms + (room for `4 * num_distance_types`). +5. Scratch: the per-context literal and distance histograms that are then + clustered, and `BrotliClusterHistograms`' state (cluster sizes, histogram + pairs, etc). + +After `build_meta_block` returns, the results (regions 2 and 4) are below +`wb_top`, and `wb[wb_top ..]` is free for the caller (e.g. for storing the +metablock) for as long as it needs the results. + + ### Encoder TODO - Qualities 2 to 11 (the C code's `EncodeData`, hashers, block splitting, - context modeling, Zopfli, etc). + context modeling, Zopfli, etc). The quality 10 and 11 metablock builder + exists (see above) but is not yet wired into the encoder. - Large-window encoding (only used at qualities 3 and above). - Custom and shared dictionaries (`BrotliEncoderAttachPreparedDictionary`). - Changing parameters mid-stream. diff --git a/std/brotli/encode_metablock.wuffs b/std/brotli/encode_metablock.wuffs index 0bfde74b..6cc0743f 100644 --- a/std/brotli/encode_metablock.wuffs +++ b/std/brotli/encode_metablock.wuffs @@ -1021,6 +1021,8 @@ pri func metablock_builder.cluster_scratch_len(in_size: base.u64, hs: base.u64) pri func metablock_builder.workbuf_len_worst_case(num_literals: base.u64, num_commands: base.u64) base.u64 { var l : base.u64 var c : base.u64 + var tl : base.u64 + var tc : base.u64 var n : base.u64 var split : base.u64 @@ -1037,13 +1039,19 @@ pri func metablock_builder.workbuf_len_worst_case(num_literals: base.u64, num_co split = split.max(no_less_than: (2 ~mod* c) ~sat+ this.sbv_scratch_len(length: c, hs: HISTOGRAM_DISTANCE_SIZE, symbols_per_histogram: 544, max_histograms: 50)) n ~sat+= split.max(no_less_than: HISTOGRAM_DISTANCE_SIZE) - // The MetaBlockSplit's histograms and context maps (with at most 256 - // block types per category), plus the input histograms for clustering. - n ~sat+= (256 * HISTOGRAM_COMMAND_SIZE) + (16384 * 4) + (16384 * HISTOGRAM_LITERAL_SIZE) + - (1024 * 4) + (1024 * HISTOGRAM_DISTANCE_SIZE) - n ~sat+= (16384 * HISTOGRAM_LITERAL_SIZE) + (1024 * HISTOGRAM_DISTANCE_SIZE) + // The number of block types is at most 256 (MAX_NUMBER_OF_HISTOGRAMS) + // and at most the number of symbols (but at least 1). tl bounds the + // literal block types and tc bounds both the command and the distance + // block types (there are at most as many distances as commands). + tl = l.max(no_less_than: 1).min(no_more_than: 256) + tc = c.max(no_less_than: 1).min(no_more_than: 256) + // The MetaBlockSplit's histograms and context maps, plus the input + // histograms for clustering. + n ~sat+= (tc * HISTOGRAM_COMMAND_SIZE) + (tl * 64 * 4) + (tl * 64 * HISTOGRAM_LITERAL_SIZE) + + (tc * 4 * 4) + (tc * 4 * HISTOGRAM_DISTANCE_SIZE) + n ~sat+= (tl * 64 * HISTOGRAM_LITERAL_SIZE) + (tc * 4 * HISTOGRAM_DISTANCE_SIZE) // cluster_histograms' scratch. - n ~sat+= this.cluster_scratch_len(in_size: 16384, hs: HISTOGRAM_LITERAL_SIZE).max(no_less_than: - this.cluster_scratch_len(in_size: 1024, hs: HISTOGRAM_DISTANCE_SIZE)) + n ~sat+= this.cluster_scratch_len(in_size: tl * 64, hs: HISTOGRAM_LITERAL_SIZE).max(no_less_than: + this.cluster_scratch_len(in_size: tc * 4, hs: HISTOGRAM_DISTANCE_SIZE)) return n } diff --git a/test/c/std/brotli.c b/test/c/std/brotli.c index 076d537a..ad92022c 100644 --- a/test/c/std/brotli.c +++ b/test/c/std/brotli.c @@ -1665,6 +1665,383 @@ bench_mimic_brotli_encode_alice29_q1() { #endif // WUFFS_MIMIC +// ---------------- Metablock Builder Tests + +// These tests check the encoder's metablock builder (the ports of +// libbrotlienc's bit_cost.c, histogram.c, block_splitter.c, cluster.c and +// metablock.c, exposed through the test-only metablock_tester struct) +// against golden values computed with libbrotlienc v1.2.1. The much larger +// script/manual-test-brotli-metablock-parity.c compares them with the C +// functions directly. + +static uint32_t // +mbt_log2_floor(uint64_t x) { + uint32_t n = 0; + while (x >>= 1) { + n++; + } + return n; +} + +static uint64_t // +mbt_fnv(uint64_t h, const uint8_t* p, size_t n) { + size_t i; + for (i = 0; i < n; i++) { + h = (h ^ p[i]) * 0x100000001B3ull; + } + return h; +} + +static uint64_t // +mbt_fnv_u64(uint64_t h, uint64_t v) { + uint8_t b[8]; + int i; + for (i = 0; i < 8; i++) { + b[i] = (uint8_t)(v >> (8 * i)); + } + return mbt_fnv(h, b, 8); +} + +static void // +mbt_put_u32le(uint8_t* p, uint32_t v) { + p[0] = (uint8_t)(v >> 0); + p[1] = (uint8_t)(v >> 8); + p[2] = (uint8_t)(v >> 16); + p[3] = (uint8_t)(v >> 24); +} + +// mbt_make_cmds writes the 16-byte commands (as command.h's InitCommand and +// InitInsertCommand would, for the given npostfix and ndirect) of a +// pseudo-random parse of total bytes to dst and returns how many there are. +// dst must have room for total + 1 commands. +static size_t // +mbt_make_cmds(uint8_t* dst, size_t total, uint32_t npostfix, uint32_t ndirect, + uint32_t seed) { + size_t n = 0; + size_t pos = 0; + uint32_t r = seed; + while (pos < total) { + size_t ins, copy, dcode, rem; + uint32_t inscode, copycode, dist_extra = 0; + uint16_t dist_prefix, cmd_prefix; + int use_last; + int insert_only = 0; + r = (r * 1103515245u) + 12345u; + ins = (r >> 16) % ((r & 0x300) ? 12 : 300); + r = (r * 1103515245u) + 12345u; + copy = 2 + ((r >> 16) % ((r & 0x700) ? 14 : 200)); + rem = total - pos; + if (ins > rem) { + ins = rem; + } + if ((rem - ins) < copy) { + ins = rem; + copy = 0; + } + if (copy == 0) { + // InitInsertCommand. + insert_only = 1; + dist_prefix = 16; + copy = 4; + use_last = 0; + } else { + r = (r * 1103515245u) + 12345u; + if ((r & 0x30000) == 0) { + dcode = (r >> 18) & 15; + } else { + size_t max = pos + ins; + dcode = 16 + (((r >> 8) % (max ? max : 1)) % 3000); + } + // PrefixEncodeCopyDistance. + if (dcode < 16 + ndirect) { + dist_prefix = (uint16_t)dcode; + } else { + size_t d = ((size_t)1 << (npostfix + 2)) + (dcode - 16 - ndirect); + size_t bucket = mbt_log2_floor(d) - 1; + size_t postfix = d & ((1u << npostfix) - 1); + size_t prefix = (d >> bucket) & 1; + size_t offset = (2 + prefix) << bucket; + size_t nbits = bucket - npostfix; + dist_prefix = (uint16_t)((nbits << 10) | + (16 + ndirect + + ((2 * (nbits - 1) + prefix) << npostfix) + + postfix)); + dist_extra = (uint32_t)((d - offset) >> npostfix); + } + use_last = (dist_prefix & 0x3FF) == 0; + } + // GetInsertLengthCode. + if (ins < 6) { + inscode = (uint32_t)ins; + } else if (ins < 130) { + uint32_t nb = mbt_log2_floor(ins - 2) - 1; + inscode = (nb << 1) + (uint32_t)((ins - 2) >> nb) + 2; + } else if (ins < 2114) { + inscode = mbt_log2_floor(ins - 66) + 10; + } else if (ins < 6210) { + inscode = 21; + } else if (ins < 22594) { + inscode = 22; + } else { + inscode = 23; + } + // GetCopyLengthCode. + if (copy < 10) { + copycode = (uint32_t)(copy - 2); + } else if (copy < 134) { + uint32_t nb = mbt_log2_floor(copy - 6) - 1; + copycode = (nb << 1) + (uint32_t)((copy - 6) >> nb) + 4; + } else if (copy < 2118) { + copycode = mbt_log2_floor(copy - 70) + 12; + } else { + copycode = 23; + } + // CombineLengthCodes. + { + uint32_t bits64 = (copycode & 7) | ((inscode & 7) << 3); + if (use_last && (inscode < 8) && (copycode < 16)) { + cmd_prefix = (uint16_t)((copycode < 8) ? bits64 : (bits64 | 64)); + } else { + uint32_t off = 2 * ((copycode >> 3) + 3 * (inscode >> 3)); + off = (off << 5) + 0x40 + ((0x520D40u >> off) & 0xC0); + cmd_prefix = (uint16_t)(off | bits64); + } + } + mbt_put_u32le(dst + 16 * n + 0, (uint32_t)ins); + mbt_put_u32le(dst + 16 * n + 4, insert_only ? (4u << 25) : (uint32_t)copy); + mbt_put_u32le(dst + 16 * n + 8, dist_extra); + dst[16 * n + 12] = (uint8_t)cmd_prefix; + dst[16 * n + 13] = (uint8_t)(cmd_prefix >> 8); + dst[16 * n + 14] = (uint8_t)dist_prefix; + dst[16 * n + 15] = (uint8_t)(dist_prefix >> 8); + n++; + pos += ins + (insert_only ? 0 : copy); + } + return n; +} + +typedef struct { + const char* filename; // NULL means the synthetic input. + uint32_t quality; + uint32_t mode; + uint32_t nolcm; + uint32_t npostfix; + uint32_t ndirect; + uint32_t large_window; + uint32_t seed; + uint64_t want_hash; +} mbt_case; + +static const mbt_case g_mbt_cases[] = { + {"test/data/romeo.txt", 11, 2, 0, 0, 0, 0, 0x12345678, + 0x91BCDB702326DA68}, + {"test/data/midsummer.txt", 10, 3, 1, 1, 4, 0, 0x9ABCDEF0, + 0xC1BC25672AFFE52C}, + {NULL, 11, 0, 0, 2, 8, 1, 0x0BADF00D, 0x106C079EF30D8335}, + {NULL, 10, 1, 0, 3, 120, 0, 0x00C0FFEE, 0x1CE5AB9FFAB4CEDB}, + {"test/data/pi.txt", 11, 2, 0, 1, 2, 0, 0x31415926, 0x4434059D684D4F0E}, + {"", 11, 2, 0, 0, 0, 0, 0x00000001, 0xAE70E50382B8388F}, +}; + +const char* // +test_wuffs_brotli_metablock_build_meta_block() { + CHECK_FOCUS(__func__); + wuffs_brotli__metablock_tester* t = wuffs_brotli__metablock_tester__alloc(); + if (!t) { + RETURN_FAIL("alloc failed"); + } + const char* ret = NULL; + uint8_t* rb = NULL; + uint8_t* cmds = NULL; + uint8_t* wb = NULL; + size_t wb_len = 1 << 20; + size_t i; + for (i = 0; i < WUFFS_TESTLIB_ARRAY_SIZE(g_mbt_cases); i++) { + const mbt_case* tc = &g_mbt_cases[i]; + size_t total = 0; + size_t rb_size = 1024; + size_t n, k; + int c; + uint64_t h = 0xCBF29CE484222325ull; + wuffs_base__status status; + wuffs_base__io_buffer src = ((wuffs_base__io_buffer){ + .data = g_have_slice_u8, + }); + if (!tc->filename) { + // A byte distribution that shifts every 700 bytes, 48 KiB. + uint32_t r = 1; + total = 49152; + for (k = 0; k < total; k++) { + r = (r * 1103515245u) + 12345u; + src.data.ptr[k] = (uint8_t)(((k / 700) * 37) + + ((r >> 16) % (1 + (k / 700) % 23))); + } + } else if (*tc->filename) { + ret = read_file(&src, tc->filename); + if (ret) { + goto done; + } + total = src.meta.wi; + } + while (rb_size < total) { + rb_size *= 2; + } + free(rb); + free(cmds); + rb = calloc(rb_size, 1); + cmds = malloc(16 * (total + 1)); + if (!rb || !cmds) { + ret = "out of memory"; + goto done; + } + if (total) { + memcpy(rb, src.data.ptr, total); + } + n = mbt_make_cmds(cmds, total, tc->npostfix, tc->ndirect, tc->seed); + while (true) { + if (!wb) { + wb = malloc(wb_len); + if (!wb) { + ret = "out of memory"; + goto done; + } + } + status = wuffs_brotli__metablock_tester__build_meta_block( + t, wuffs_base__make_slice_u8(wb, wb_len), + wuffs_base__make_slice_u8(rb, rb_size), 0, rb_size - 1, 0, 0, + wuffs_base__make_slice_u8(cmds, 16 * n), n, tc->mode, tc->quality, + tc->large_window, tc->nolcm, tc->npostfix, tc->ndirect); + if (status.repr != wuffs_base__error__bad_workbuf_length) { + break; + } + // The commands were rewritten, so regenerate them. + n = mbt_make_cmds(cmds, total, tc->npostfix, tc->ndirect, tc->seed); + free(wb); + wb = NULL; + wb_len *= 2; + } + if (status.repr) { + ret = status.repr; + goto done; + } + { + uint64_t ndc = wuffs_brotli__metablock_tester__get(t, 27); + wuffs_brotli__metablock_tester__optimize_histograms( + t, wuffs_base__make_slice_u8(wb, wb_len), + (uint32_t)((ndc < 544) ? ndc : 544)); + } + for (c = 0; c < 3; c++) { + uint64_t num_blocks = wuffs_brotli__metablock_tester__get(t, 4 * c + 1); + uint64_t types = wuffs_brotli__metablock_tester__get(t, 4 * c + 2); + uint64_t lengths = wuffs_brotli__metablock_tester__get(t, 4 * c + 3); + h = mbt_fnv_u64(h, wuffs_brotli__metablock_tester__get(t, 4 * c + 0)); + h = mbt_fnv_u64(h, num_blocks); + h = mbt_fnv(h, wb + types, num_blocks); + h = mbt_fnv(h, wb + lengths, 4 * num_blocks); + } + for (c = 0; c < 3; c += 2) { + uint64_t size = wuffs_brotli__metablock_tester__get(t, 15 + c); + h = mbt_fnv_u64(h, size); + h = mbt_fnv(h, wb + wuffs_brotli__metablock_tester__get(t, 12 + c), + 4 * size); + } + for (c = 0; c < 3; c++) { + static const size_t hsizes[3] = {1040, 2832, 2192}; + uint64_t size = wuffs_brotli__metablock_tester__get(t, 21 + c); + h = mbt_fnv_u64(h, size); + h = mbt_fnv(h, wb + wuffs_brotli__metablock_tester__get(t, 18 + c), + hsizes[c] * size); + } + for (c = 24; c < 29; c++) { + h = mbt_fnv_u64(h, wuffs_brotli__metablock_tester__get(t, c)); + } + h = mbt_fnv(h, cmds, 16 * n); + if (h != tc->want_hash) { + RETURN_FAIL("case %zu: hash: have 0x%016" PRIX64 ", want 0x%016" PRIX64, + i, h, tc->want_hash); + } + if (wuffs_brotli__metablock_tester__get(t, 29) > + wuffs_brotli__metablock_tester__get(t, 30)) { + RETURN_FAIL("case %zu: workbuf used exceeds the worst case", i); + } + } +done: + free(rb); + free(cmds); + free(wb); + free(t); + return ret; +} + +const char* // +test_wuffs_brotli_metablock_bit_cost() { + CHECK_FOCUS(__func__); + wuffs_brotli__metablock_tester* t = wuffs_brotli__metablock_tester__alloc(); + if (!t) { + RETURN_FAIL("alloc failed"); + } + // The C code's BrotliBitsEntropy and BrotliPopulationCost{Literal,Command, + // Distance} results, as binary64 bit patterns. + static const uint64_t want_pop[6][3] = { + {0x4028000000000000, 0x4028000000000000, 0x4028000000000000}, + {0x4028000000000000, 0x4028000000000000, 0x4028000000000000}, + {0x4055800000000000, 0x4055800000000000, 0x4055800000000000}, + {0x4055800000000000, 0x406AC00000000000, 0x406AC00000000000}, + {0x4055800000000000, 0x4076300000000000, 0x4076300000000000}, + {0x40F842EE8C1F42B3, 0x4113DB811CA8812F, 0x410D76A1EA720A2C}, + }; + static const uint32_t data_sizes[3] = {256, 704, 544}; + // Each histogram is data_size u32le counts, a u64le total_count_ and a + // binary64 bit_cost_. + uint8_t pop[4 * 704]; + uint8_t histo[(4 * 704) + 16]; + uint32_t counts[704]; + size_t i; + int k, c; + for (i = 0; i < 704; i++) { + counts[i] = (uint32_t)((i * i * 7 + 3) % 101); + mbt_put_u32le(pop + 4 * i, counts[i]); + } + { + uint64_t have = wuffs_brotli__metablock_tester__bits_entropy( + t, wuffs_base__make_slice_u8(pop, sizeof(pop)), 256); + if (have != 0x40F79A0C92B03A26ull) { + free(t); + RETURN_FAIL("bits_entropy: have 0x%016" PRIX64, have); + } + } + for (k = 0; k < 6; k++) { + static const int nz[6] = {0, 1, 2, 3, 4, 300}; + for (c = 0; c < 3; c++) { + uint64_t total = 0; + uint64_t have; + memset(histo, 0, sizeof(histo)); + for (i = 0; i < data_sizes[c]; i++) { + int on = (nz[k] >= 300) ? (counts[i] != 0) + : (((i % 151) == 7) && ((int)(i / 151) < nz[k])); + if (on) { + mbt_put_u32le(histo + 4 * i, counts[i] + 1); + total += counts[i] + 1; + } + } + for (i = 0; i < 8; i++) { + histo[(4 * data_sizes[c]) + i] = (uint8_t)(total >> (8 * i)); + } + have = wuffs_brotli__metablock_tester__population_cost( + t, wuffs_base__make_slice_u8(histo, (4 * data_sizes[c]) + 16), + data_sizes[c]); + if (have != want_pop[k][c]) { + free(t); + RETURN_FAIL("population_cost(k=%d, c=%d): have 0x%016" PRIX64 + ", want 0x%016" PRIX64, + k, c, have, want_pop[k][c]); + } + } + } + free(t); + return NULL; +} + // ---------------- Large Window and Dictionary Tests // See test/data/brotli/README.md for where the large-window (lw), raw @@ -2563,6 +2940,10 @@ proc g_tests[] = { test_wuffs_brotli_decode_large_window_far, test_wuffs_brotli_decode_large_window_quirk, + // Metablock builder tests. + test_wuffs_brotli_metablock_bit_cost, + test_wuffs_brotli_metablock_build_meta_block, + #ifdef WUFFS_MIMIC test_mimic_brotli_decode_alice29,