Merge branch 'm4b-metablock' into m4a-encoder-core

# Conflicts:
#	std/brotli/README.md
This commit is contained in:
Claude committed 2026-10-27 01:26:59 +00:00
commit b10d57f20f
4 files changed
+534 -17

No files matched your search

+11 -3
View File
@@ -86517,6 +86517,8 @@ wuffs_brotli__metablock_builder__workbuf_len_worst_case(
uint64_t a_num_commands) {
uint64_t v_l = 0;
uint64_t v_c = 0;
uint64_t v_tl = 0;
uint64_t v_tc = 0;
uint64_t v_n = 0;
uint64_t v_split = 0;
@@ -86539,9 +86541,15 @@ wuffs_brotli__metablock_builder__workbuf_len_worst_case(
544u,
50u)));
wuffs_private_impl__u64__sat_add_indirect(&v_n, wuffs_base__u64__max(v_split, 2192u));
wuffs_private_impl__u64__sat_add_indirect(&v_n, 20078592u);
wuffs_private_impl__u64__sat_add_indirect(&v_n, 19283968u);
wuffs_private_impl__u64__sat_add_indirect(&v_n, wuffs_base__u64__max(wuffs_brotli__metablock_builder__cluster_scratch_len(self, 16384u, 1040u), wuffs_brotli__metablock_builder__cluster_scratch_len(self, 1024u, 2192u)));
v_tl = wuffs_base__u64__min(wuffs_base__u64__max(v_l, 1u), 256u);
v_tc = wuffs_base__u64__min(wuffs_base__u64__max(v_c, 1u), 256u);
wuffs_private_impl__u64__sat_add_indirect(&v_n, ((v_tc * 2832u) +
(v_tl * 64u * 4u) +
(v_tl * 64u * 1040u) +
(v_tc * 4u * 4u) +
(v_tc * 4u * 2192u)));
wuffs_private_impl__u64__sat_add_indirect(&v_n, ((v_tl * 64u * 1040u) + (v_tc * 4u * 2192u)));
wuffs_private_impl__u64__sat_add_indirect(&v_n, wuffs_base__u64__max(wuffs_brotli__metablock_builder__cluster_scratch_len(self, (v_tl * 64u), 1040u), wuffs_brotli__metablock_builder__cluster_scratch_len(self, (v_tc * 4u), 2192u)));
return v_n;
}
+127 -7
View File
@@ -225,13 +225,19 @@ The source files are:
greedy block splitters, `metablock_inc.h`) and `BrotliOptimizeHistograms`
for its output.
- `encode_util.wuffs`: little-endian workbuf accessors.
- `encode_block_splitter.wuffs`, `encode_cluster.wuffs` and
`encode_metablock.wuffs`: the `metablock_builder` struct, with
`block_splitter.c`, `cluster.c` and `BrotliBuildMetaBlock` (which only
quality 10 and above use, and so the encoder does not call it yet). The
encoder uses some of its smaller functions (`BrotliInitDistanceParams`,
`PrefixEncodeCopyDistance`, `CommandRestoreDistanceCode` and
`BrotliOptimizeHuffmanCountsForRle`).
- `encode_block_splitter.wuffs`: `block_splitter.c` and
`block_splitter_inc.h` (`BrotliSplitBlock`), as methods of the
`metablock_builder` struct.
- `encode_cluster.wuffs`: `cluster.c` and `cluster_inc.h`
(`BrotliClusterHistograms` and its helpers), also as `metablock_builder`
methods.
- `encode_metablock.wuffs`: the `metablock_builder` struct and `metablock.c`
(`BrotliInitDistanceParams`, `BrotliBuildMetaBlock`,
`BrotliOptimizeHistograms`). See "Encoder Metablock Builder" below.
Only quality 10 and above use `BrotliBuildMetaBlock`, so the encoder does
not call it yet, but it uses some of the smaller functions
(`BrotliInitDistanceParams`, `PrefixEncodeCopyDistance`,
`CommandRestoreDistanceCode` and `BrotliOptimizeHuffmanCountsForRle`).
- `encode_tester.wuffs`: TEST ONLY. The `metablock_tester` struct exposes
the internal functions above to C test programs. It is not part of the
encoder API.
@@ -468,6 +474,120 @@ and for qualities 2 to 9 several hundred thousand, all byte-exact.
only in the C code's one-shot `BrotliEncoderCompress`).
### Encoder Metablock Builder
The `metablock_builder` struct (a `pri struct`, embeddable in the encoder
with one field, e.g. `mb : metablock_builder` in the private data section)
implements the quality 10 and 11 metablock construction (`BrotliBuildMetaBlock`)
and `BrotliOptimizeHistograms`. The encoder has one (its `mb` field) but
only calls its smaller helpers so far (see the source file list above). Its
fixed-size state (about 0.8 MiB: FindBlocks' cost arrays and a 64Ki-entry
FastLog2 cache in its `bit_cost`) lives in the struct; everything that the C
code allocates on the heap lives in a caller-supplied workbuf.
Data formats (all little-endian, matching the C structs' memory layout on a
64-bit little-endian machine, so that results can be compared with `memcmp`):
- A command is 16 bytes: `insert_len_` (u32), `copy_len_` (u32), `dist_extra_`
(u32), `cmd_prefix_` (u16), `dist_prefix_` (u16), as built by `command.h`'s
`InitCommand` / `InitInsertCommand`.
- A histogram is `(4 * data_size) + 16` bytes: `data_size` u32 counts, the
u64 `total_count_` and the binary64 bits of `bit_cost_`. `data_size` is 256
(literals, 1040 bytes), 704 (commands, 2832 bytes) or 544 (distances, 2192
bytes).
- A BlockSplit's types are u8s and its lengths u32s. Context maps are u32s.
- Doubles are binary64 bit patterns (u64), computed with `std/softfloat` in
the C code's operation order.
Usage, mirroring the C code's `WriteMetaBlockInternal` for quality >= 10:
```
// Once per metablock. The params are the C code's block_params, a copy of
// the encoder's params: init_distance_params must be called each time,
// because build_meta_block overwrites the dist_etc fields with the chosen
// distance params.
this.mb.set_params!(quality: q, large_window: lw,
disable_literal_context_modeling: dlcm)
this.mb.init_distance_params!(npostfix: np, ndirect: nd, large_window: lw)
status = this.mb.build_meta_block!(wb: workbuf, ringbuffer: rb, pos: pos,
mask: mask, prev_byte: pb, prev_byte2: pb2, cmds: cmds,
num_commands: n, literal_context_mode: mode)
// Optional (quality >= 4): ndc is min(this.mb.dist_alphabet_size_limit, 544).
this.mb.optimize_histograms!(wb: workbuf, num_distance_codes: ndc)
```
The exact signatures are:
- `set_params!(quality: u32, large_window: bool,
disable_literal_context_modeling: bool)`.
- `init_distance_params!(npostfix: u32, ndirect: u32, large_window: bool)`
(`BrotliInitDistanceParams`; sets `dist_postfix_bits`,
`dist_num_direct_codes`, `dist_alphabet_size_max`,
`dist_alphabet_size_limit` and `dist_max_distance`).
- `build_meta_block!(wb: slice u8, ringbuffer: roslice u8, pos: u64, mask:
u64, prev_byte: u8, prev_byte2: u8, cmds: slice u8, num_commands: u64,
literal_context_mode: u32) status`. `ringbuffer` must be at least `mask + 1`
bytes. `literal_context_mode` is a `ContextType` (0 = LSB6, 1 = MSB6, 2 =
UTF8, 3 = SIGNED). Like the C code, it rewrites the commands'
`dist_prefix_` and `dist_extra_` fields for the chosen distance params and
leaves those params in the `dist_etc` fields. It returns
`"#bad workbuf length"` if `wb` is too short (the results are then invalid
but `cmds` may already be rewritten, so retrying needs the original
commands).
- `optimize_histograms!(wb: slice u8, num_distance_codes: u32)` works on the
MetaBlockSplit described by the fields below, whether `build_meta_block`
or the caller (e.g. a greedy builder) filled them in.
- `workbuf_len_worst_case(num_literals: u64, num_commands: u64) u64` is an
upper bound on the workbuf length that `build_meta_block` needs, where
`num_literals` is the sum of the commands' insert lengths. It assumes the
worst case for every run-time-sized allocation (one `FindBlocks` block per
symbol, 256 block types with 64 literal contexts each, full histogram pair
queues), so it is far larger than what real inputs need: about 76 MiB for
a 1 KiB metablock, 2.6 GiB for 1 MiB of literals and 128 Ki commands, and
41 GiB for 16 MiB of literals and 4 Mi commands. These are the C code's
own worst cases (`ClusterBlocks` and `BrotliClusterHistograms` size their
histogram pair queues by the number of blocks and clusters), not artifacts
of the port. `wb_high` records what a call actually used: over the parity
harness's inputs, at most about 42 MB (for 1.3 MiB of literals split into
256 literal block types).
The MetaBlockSplit result is described by these fields, indexed by category
(0 = literal, 1 = command, 2 = distance), with offsets into `wb`:
| Field | C code |
| ---------------------- | ----------------------------------------------- |
| `split_num_types[c]` | `{literal,command,distance}_split.num_types` |
| `split_num_blocks[c]` | `..._split.num_blocks` |
| `split_types_off[c]` | `..._split.types` (u8 array) |
| `split_lengths_off[c]` | `..._split.lengths` (u32 array) |
| `context_map_off[c]` | `{literal,distance}_context_map` (u32 array; none for commands) |
| `context_map_size[c]` | `{literal,distance}_context_map_size` |
| `histograms_off[c]` | `{literal,command,distance}_histograms` |
| `histograms_size[c]` | `..._histograms_size` |
Workbuf layout during `build_meta_block` (a bump allocator, `wb_top`, carves
regions out of `wb` in this order; "scratch" regions are released when the
C code frees them, so later regions reuse their bytes):
1. Scratch: one distance histogram for the distance parameter search.
2. The three BlockSplits' types and lengths arrays, sized for one block per
symbol (`max(1, n)` entries, where n is the number of literals, commands
or distance codes).
3. Scratch: `BrotliSplitBlock`'s literal / command code / distance code
arrays and `SplitByteVector`'s histograms, block ids, bitmaps and
`ClusterBlocks` state.
4. The command histograms, literal context map, literal histograms (room for
`64 * num_literal_types`), distance context map and distance histograms
(room for `4 * num_distance_types`).
5. Scratch: the per-context literal and distance histograms that are then
clustered, and `BrotliClusterHistograms`' state (cluster sizes, histogram
pairs, etc).
After `build_meta_block` returns, the results (regions 2 and 4) are below
`wb_top`, and `wb[wb_top ..]` is free for the caller (e.g. for storing the
metablock) for as long as it needs the results.
### Encoder TODO
- Qualities 10 and 11 (Zopfli, the H10 binary tree hasher and
+15 -7
View File
@@ -1021,6 +1021,8 @@ pri func metablock_builder.cluster_scratch_len(in_size: base.u64, hs: base.u64)
pri func metablock_builder.workbuf_len_worst_case(num_literals: base.u64, num_commands: base.u64) base.u64 {
var l : base.u64
var c : base.u64
var tl : base.u64
var tc : base.u64
var n : base.u64
var split : base.u64
@@ -1037,13 +1039,19 @@ pri func metablock_builder.workbuf_len_worst_case(num_literals: base.u64, num_co
split = split.max(no_less_than: (2 ~mod* c) ~sat+
this.sbv_scratch_len(length: c, hs: HISTOGRAM_DISTANCE_SIZE, symbols_per_histogram: 544, max_histograms: 50))
n ~sat+= split.max(no_less_than: HISTOGRAM_DISTANCE_SIZE)
// The MetaBlockSplit's histograms and context maps (with at most 256
// block types per category), plus the input histograms for clustering.
n ~sat+= (256 * HISTOGRAM_COMMAND_SIZE) + (16384 * 4) + (16384 * HISTOGRAM_LITERAL_SIZE) +
(1024 * 4) + (1024 * HISTOGRAM_DISTANCE_SIZE)
n ~sat+= (16384 * HISTOGRAM_LITERAL_SIZE) + (1024 * HISTOGRAM_DISTANCE_SIZE)
// The number of block types is at most 256 (MAX_NUMBER_OF_HISTOGRAMS)
// and at most the number of symbols (but at least 1). tl bounds the
// literal block types and tc bounds both the command and the distance
// block types (there are at most as many distances as commands).
tl = l.max(no_less_than: 1).min(no_more_than: 256)
tc = c.max(no_less_than: 1).min(no_more_than: 256)
// The MetaBlockSplit's histograms and context maps, plus the input
// histograms for clustering.
n ~sat+= (tc * HISTOGRAM_COMMAND_SIZE) + (tl * 64 * 4) + (tl * 64 * HISTOGRAM_LITERAL_SIZE) +
(tc * 4 * 4) + (tc * 4 * HISTOGRAM_DISTANCE_SIZE)
n ~sat+= (tl * 64 * HISTOGRAM_LITERAL_SIZE) + (tc * 4 * HISTOGRAM_DISTANCE_SIZE)
// cluster_histograms' scratch.
n ~sat+= this.cluster_scratch_len(in_size: 16384, hs: HISTOGRAM_LITERAL_SIZE).max(no_less_than:
this.cluster_scratch_len(in_size: 1024, hs: HISTOGRAM_DISTANCE_SIZE))
n ~sat+= this.cluster_scratch_len(in_size: tl * 64, hs: HISTOGRAM_LITERAL_SIZE).max(no_less_than:
this.cluster_scratch_len(in_size: tc * 4, hs: HISTOGRAM_DISTANCE_SIZE))
return n
}
+381
View File
@@ -1907,6 +1907,383 @@ bench_mimic_brotli_encode_alice29_q9() {
#endif // WUFFS_MIMIC
// ---------------- Metablock Builder Tests
// These tests check the encoder's metablock builder (the ports of
// libbrotlienc's bit_cost.c, histogram.c, block_splitter.c, cluster.c and
// metablock.c, exposed through the test-only metablock_tester struct)
// against golden values computed with libbrotlienc v1.2.1. The much larger
// script/manual-test-brotli-metablock-parity.c compares them with the C
// functions directly.
static uint32_t //
mbt_log2_floor(uint64_t x) {
uint32_t n = 0;
while (x >>= 1) {
n++;
}
return n;
}
static uint64_t //
mbt_fnv(uint64_t h, const uint8_t* p, size_t n) {
size_t i;
for (i = 0; i < n; i++) {
h = (h ^ p[i]) * 0x100000001B3ull;
}
return h;
}
static uint64_t //
mbt_fnv_u64(uint64_t h, uint64_t v) {
uint8_t b[8];
int i;
for (i = 0; i < 8; i++) {
b[i] = (uint8_t)(v >> (8 * i));
}
return mbt_fnv(h, b, 8);
}
static void //
mbt_put_u32le(uint8_t* p, uint32_t v) {
p[0] = (uint8_t)(v >> 0);
p[1] = (uint8_t)(v >> 8);
p[2] = (uint8_t)(v >> 16);
p[3] = (uint8_t)(v >> 24);
}
// mbt_make_cmds writes the 16-byte commands (as command.h's InitCommand and
// InitInsertCommand would, for the given npostfix and ndirect) of a
// pseudo-random parse of total bytes to dst and returns how many there are.
// dst must have room for total + 1 commands.
static size_t //
mbt_make_cmds(uint8_t* dst, size_t total, uint32_t npostfix, uint32_t ndirect,
uint32_t seed) {
size_t n = 0;
size_t pos = 0;
uint32_t r = seed;
while (pos < total) {
size_t ins, copy, dcode, rem;
uint32_t inscode, copycode, dist_extra = 0;
uint16_t dist_prefix, cmd_prefix;
int use_last;
int insert_only = 0;
r = (r * 1103515245u) + 12345u;
ins = (r >> 16) % ((r & 0x300) ? 12 : 300);
r = (r * 1103515245u) + 12345u;
copy = 2 + ((r >> 16) % ((r & 0x700) ? 14 : 200));
rem = total - pos;
if (ins > rem) {
ins = rem;
}
if ((rem - ins) < copy) {
ins = rem;
copy = 0;
}
if (copy == 0) {
// InitInsertCommand.
insert_only = 1;
dist_prefix = 16;
copy = 4;
use_last = 0;
} else {
r = (r * 1103515245u) + 12345u;
if ((r & 0x30000) == 0) {
dcode = (r >> 18) & 15;
} else {
size_t max = pos + ins;
dcode = 16 + (((r >> 8) % (max ? max : 1)) % 3000);
}
// PrefixEncodeCopyDistance.
if (dcode < 16 + ndirect) {
dist_prefix = (uint16_t)dcode;
} else {
size_t d = ((size_t)1 << (npostfix + 2)) + (dcode - 16 - ndirect);
size_t bucket = mbt_log2_floor(d) - 1;
size_t postfix = d & ((1u << npostfix) - 1);
size_t prefix = (d >> bucket) & 1;
size_t offset = (2 + prefix) << bucket;
size_t nbits = bucket - npostfix;
dist_prefix = (uint16_t)((nbits << 10) |
(16 + ndirect +
((2 * (nbits - 1) + prefix) << npostfix) +
postfix));
dist_extra = (uint32_t)((d - offset) >> npostfix);
}
use_last = (dist_prefix & 0x3FF) == 0;
}
// GetInsertLengthCode.
if (ins < 6) {
inscode = (uint32_t)ins;
} else if (ins < 130) {
uint32_t nb = mbt_log2_floor(ins - 2) - 1;
inscode = (nb << 1) + (uint32_t)((ins - 2) >> nb) + 2;
} else if (ins < 2114) {
inscode = mbt_log2_floor(ins - 66) + 10;
} else if (ins < 6210) {
inscode = 21;
} else if (ins < 22594) {
inscode = 22;
} else {
inscode = 23;
}
// GetCopyLengthCode.
if (copy < 10) {
copycode = (uint32_t)(copy - 2);
} else if (copy < 134) {
uint32_t nb = mbt_log2_floor(copy - 6) - 1;
copycode = (nb << 1) + (uint32_t)((copy - 6) >> nb) + 4;
} else if (copy < 2118) {
copycode = mbt_log2_floor(copy - 70) + 12;
} else {
copycode = 23;
}
// CombineLengthCodes.
{
uint32_t bits64 = (copycode & 7) | ((inscode & 7) << 3);
if (use_last && (inscode < 8) && (copycode < 16)) {
cmd_prefix = (uint16_t)((copycode < 8) ? bits64 : (bits64 | 64));
} else {
uint32_t off = 2 * ((copycode >> 3) + 3 * (inscode >> 3));
off = (off << 5) + 0x40 + ((0x520D40u >> off) & 0xC0);
cmd_prefix = (uint16_t)(off | bits64);
}
}
mbt_put_u32le(dst + 16 * n + 0, (uint32_t)ins);
mbt_put_u32le(dst + 16 * n + 4, insert_only ? (4u << 25) : (uint32_t)copy);
mbt_put_u32le(dst + 16 * n + 8, dist_extra);
dst[16 * n + 12] = (uint8_t)cmd_prefix;
dst[16 * n + 13] = (uint8_t)(cmd_prefix >> 8);
dst[16 * n + 14] = (uint8_t)dist_prefix;
dst[16 * n + 15] = (uint8_t)(dist_prefix >> 8);
n++;
pos += ins + (insert_only ? 0 : copy);
}
return n;
}
typedef struct {
const char* filename; // NULL means the synthetic input.
uint32_t quality;
uint32_t mode;
uint32_t nolcm;
uint32_t npostfix;
uint32_t ndirect;
uint32_t large_window;
uint32_t seed;
uint64_t want_hash;
} mbt_case;
static const mbt_case g_mbt_cases[] = {
{"test/data/romeo.txt", 11, 2, 0, 0, 0, 0, 0x12345678,
0x91BCDB702326DA68},
{"test/data/midsummer.txt", 10, 3, 1, 1, 4, 0, 0x9ABCDEF0,
0xC1BC25672AFFE52C},
{NULL, 11, 0, 0, 2, 8, 1, 0x0BADF00D, 0x106C079EF30D8335},
{NULL, 10, 1, 0, 3, 120, 0, 0x00C0FFEE, 0x1CE5AB9FFAB4CEDB},
{"test/data/pi.txt", 11, 2, 0, 1, 2, 0, 0x31415926, 0x4434059D684D4F0E},
{"", 11, 2, 0, 0, 0, 0, 0x00000001, 0xAE70E50382B8388F},
};
const char* //
test_wuffs_brotli_metablock_build_meta_block() {
CHECK_FOCUS(__func__);
wuffs_brotli__metablock_tester* t = wuffs_brotli__metablock_tester__alloc();
if (!t) {
RETURN_FAIL("alloc failed");
}
const char* ret = NULL;
uint8_t* rb = NULL;
uint8_t* cmds = NULL;
uint8_t* wb = NULL;
size_t wb_len = 1 << 20;
size_t i;
for (i = 0; i < WUFFS_TESTLIB_ARRAY_SIZE(g_mbt_cases); i++) {
const mbt_case* tc = &g_mbt_cases[i];
size_t total = 0;
size_t rb_size = 1024;
size_t n, k;
int c;
uint64_t h = 0xCBF29CE484222325ull;
wuffs_base__status status;
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
.data = g_have_slice_u8,
});
if (!tc->filename) {
// A byte distribution that shifts every 700 bytes, 48 KiB.
uint32_t r = 1;
total = 49152;
for (k = 0; k < total; k++) {
r = (r * 1103515245u) + 12345u;
src.data.ptr[k] = (uint8_t)(((k / 700) * 37) +
((r >> 16) % (1 + (k / 700) % 23)));
}
} else if (*tc->filename) {
ret = read_file(&src, tc->filename);
if (ret) {
goto done;
}
total = src.meta.wi;
}
while (rb_size < total) {
rb_size *= 2;
}
free(rb);
free(cmds);
rb = calloc(rb_size, 1);
cmds = malloc(16 * (total + 1));
if (!rb || !cmds) {
ret = "out of memory";
goto done;
}
if (total) {
memcpy(rb, src.data.ptr, total);
}
n = mbt_make_cmds(cmds, total, tc->npostfix, tc->ndirect, tc->seed);
while (true) {
if (!wb) {
wb = malloc(wb_len);
if (!wb) {
ret = "out of memory";
goto done;
}
}
status = wuffs_brotli__metablock_tester__build_meta_block(
t, wuffs_base__make_slice_u8(wb, wb_len),
wuffs_base__make_slice_u8(rb, rb_size), 0, rb_size - 1, 0, 0,
wuffs_base__make_slice_u8(cmds, 16 * n), n, tc->mode, tc->quality,
tc->large_window, tc->nolcm, tc->npostfix, tc->ndirect);
if (status.repr != wuffs_base__error__bad_workbuf_length) {
break;
}
// The commands were rewritten, so regenerate them.
n = mbt_make_cmds(cmds, total, tc->npostfix, tc->ndirect, tc->seed);
free(wb);
wb = NULL;
wb_len *= 2;
}
if (status.repr) {
ret = status.repr;
goto done;
}
{
uint64_t ndc = wuffs_brotli__metablock_tester__get(t, 27);
wuffs_brotli__metablock_tester__optimize_histograms(
t, wuffs_base__make_slice_u8(wb, wb_len),
(uint32_t)((ndc < 544) ? ndc : 544));
}
for (c = 0; c < 3; c++) {
uint64_t num_blocks = wuffs_brotli__metablock_tester__get(t, 4 * c + 1);
uint64_t types = wuffs_brotli__metablock_tester__get(t, 4 * c + 2);
uint64_t lengths = wuffs_brotli__metablock_tester__get(t, 4 * c + 3);
h = mbt_fnv_u64(h, wuffs_brotli__metablock_tester__get(t, 4 * c + 0));
h = mbt_fnv_u64(h, num_blocks);
h = mbt_fnv(h, wb + types, num_blocks);
h = mbt_fnv(h, wb + lengths, 4 * num_blocks);
}
for (c = 0; c < 3; c += 2) {
uint64_t size = wuffs_brotli__metablock_tester__get(t, 15 + c);
h = mbt_fnv_u64(h, size);
h = mbt_fnv(h, wb + wuffs_brotli__metablock_tester__get(t, 12 + c),
4 * size);
}
for (c = 0; c < 3; c++) {
static const size_t hsizes[3] = {1040, 2832, 2192};
uint64_t size = wuffs_brotli__metablock_tester__get(t, 21 + c);
h = mbt_fnv_u64(h, size);
h = mbt_fnv(h, wb + wuffs_brotli__metablock_tester__get(t, 18 + c),
hsizes[c] * size);
}
for (c = 24; c < 29; c++) {
h = mbt_fnv_u64(h, wuffs_brotli__metablock_tester__get(t, c));
}
h = mbt_fnv(h, cmds, 16 * n);
if (h != tc->want_hash) {
RETURN_FAIL("case %zu: hash: have 0x%016" PRIX64 ", want 0x%016" PRIX64,
i, h, tc->want_hash);
}
if (wuffs_brotli__metablock_tester__get(t, 29) >
wuffs_brotli__metablock_tester__get(t, 30)) {
RETURN_FAIL("case %zu: workbuf used exceeds the worst case", i);
}
}
done:
free(rb);
free(cmds);
free(wb);
free(t);
return ret;
}
const char* //
test_wuffs_brotli_metablock_bit_cost() {
CHECK_FOCUS(__func__);
wuffs_brotli__metablock_tester* t = wuffs_brotli__metablock_tester__alloc();
if (!t) {
RETURN_FAIL("alloc failed");
}
// The C code's BrotliBitsEntropy and BrotliPopulationCost{Literal,Command,
// Distance} results, as binary64 bit patterns.
static const uint64_t want_pop[6][3] = {
{0x4028000000000000, 0x4028000000000000, 0x4028000000000000},
{0x4028000000000000, 0x4028000000000000, 0x4028000000000000},
{0x4055800000000000, 0x4055800000000000, 0x4055800000000000},
{0x4055800000000000, 0x406AC00000000000, 0x406AC00000000000},
{0x4055800000000000, 0x4076300000000000, 0x4076300000000000},
{0x40F842EE8C1F42B3, 0x4113DB811CA8812F, 0x410D76A1EA720A2C},
};
static const uint32_t data_sizes[3] = {256, 704, 544};
// Each histogram is data_size u32le counts, a u64le total_count_ and a
// binary64 bit_cost_.
uint8_t pop[4 * 704];
uint8_t histo[(4 * 704) + 16];
uint32_t counts[704];
size_t i;
int k, c;
for (i = 0; i < 704; i++) {
counts[i] = (uint32_t)((i * i * 7 + 3) % 101);
mbt_put_u32le(pop + 4 * i, counts[i]);
}
{
uint64_t have = wuffs_brotli__metablock_tester__bits_entropy(
t, wuffs_base__make_slice_u8(pop, sizeof(pop)), 256);
if (have != 0x40F79A0C92B03A26ull) {
free(t);
RETURN_FAIL("bits_entropy: have 0x%016" PRIX64, have);
}
}
for (k = 0; k < 6; k++) {
static const int nz[6] = {0, 1, 2, 3, 4, 300};
for (c = 0; c < 3; c++) {
uint64_t total = 0;
uint64_t have;
memset(histo, 0, sizeof(histo));
for (i = 0; i < data_sizes[c]; i++) {
int on = (nz[k] >= 300) ? (counts[i] != 0)
: (((i % 151) == 7) && ((int)(i / 151) < nz[k]));
if (on) {
mbt_put_u32le(histo + 4 * i, counts[i] + 1);
total += counts[i] + 1;
}
}
for (i = 0; i < 8; i++) {
histo[(4 * data_sizes[c]) + i] = (uint8_t)(total >> (8 * i));
}
have = wuffs_brotli__metablock_tester__population_cost(
t, wuffs_base__make_slice_u8(histo, (4 * data_sizes[c]) + 16),
data_sizes[c]);
if (have != want_pop[k][c]) {
free(t);
RETURN_FAIL("population_cost(k=%d, c=%d): have 0x%016" PRIX64
", want 0x%016" PRIX64,
k, c, have, want_pop[k][c]);
}
}
}
free(t);
return NULL;
}
// ---------------- Large Window and Dictionary Tests
// See test/data/brotli/README.md for where the large-window (lw), raw
@@ -2805,6 +3182,10 @@ proc g_tests[] = {
test_wuffs_brotli_decode_large_window_far,
test_wuffs_brotli_decode_large_window_quirk,
// Metablock builder tests.
test_wuffs_brotli_metablock_bit_cost,
test_wuffs_brotli_metablock_build_meta_block,
#ifdef WUFFS_MIMIC
test_mimic_brotli_decode_alice29,