mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-19 08:38:14 +01:00
Merge branch 'm4b-metablock' into m4a-encoder-core
# Conflicts: # std/brotli/README.md
This commit is contained in:
4 files changed
+534
-17
No files matched your search
@@ -86517,6 +86517,8 @@ wuffs_brotli__metablock_builder__workbuf_len_worst_case(
|
||||
uint64_t a_num_commands) {
|
||||
uint64_t v_l = 0;
|
||||
uint64_t v_c = 0;
|
||||
uint64_t v_tl = 0;
|
||||
uint64_t v_tc = 0;
|
||||
uint64_t v_n = 0;
|
||||
uint64_t v_split = 0;
|
||||
|
||||
@@ -86539,9 +86541,15 @@ wuffs_brotli__metablock_builder__workbuf_len_worst_case(
|
||||
544u,
|
||||
50u)));
|
||||
wuffs_private_impl__u64__sat_add_indirect(&v_n, wuffs_base__u64__max(v_split, 2192u));
|
||||
wuffs_private_impl__u64__sat_add_indirect(&v_n, 20078592u);
|
||||
wuffs_private_impl__u64__sat_add_indirect(&v_n, 19283968u);
|
||||
wuffs_private_impl__u64__sat_add_indirect(&v_n, wuffs_base__u64__max(wuffs_brotli__metablock_builder__cluster_scratch_len(self, 16384u, 1040u), wuffs_brotli__metablock_builder__cluster_scratch_len(self, 1024u, 2192u)));
|
||||
v_tl = wuffs_base__u64__min(wuffs_base__u64__max(v_l, 1u), 256u);
|
||||
v_tc = wuffs_base__u64__min(wuffs_base__u64__max(v_c, 1u), 256u);
|
||||
wuffs_private_impl__u64__sat_add_indirect(&v_n, ((v_tc * 2832u) +
|
||||
(v_tl * 64u * 4u) +
|
||||
(v_tl * 64u * 1040u) +
|
||||
(v_tc * 4u * 4u) +
|
||||
(v_tc * 4u * 2192u)));
|
||||
wuffs_private_impl__u64__sat_add_indirect(&v_n, ((v_tl * 64u * 1040u) + (v_tc * 4u * 2192u)));
|
||||
wuffs_private_impl__u64__sat_add_indirect(&v_n, wuffs_base__u64__max(wuffs_brotli__metablock_builder__cluster_scratch_len(self, (v_tl * 64u), 1040u), wuffs_brotli__metablock_builder__cluster_scratch_len(self, (v_tc * 4u), 2192u)));
|
||||
return v_n;
|
||||
}
|
||||
|
||||
|
||||
+127
-7
@@ -225,13 +225,19 @@ The source files are:
|
||||
greedy block splitters, `metablock_inc.h`) and `BrotliOptimizeHistograms`
|
||||
for its output.
|
||||
- `encode_util.wuffs`: little-endian workbuf accessors.
|
||||
- `encode_block_splitter.wuffs`, `encode_cluster.wuffs` and
|
||||
`encode_metablock.wuffs`: the `metablock_builder` struct, with
|
||||
`block_splitter.c`, `cluster.c` and `BrotliBuildMetaBlock` (which only
|
||||
quality 10 and above use, and so the encoder does not call it yet). The
|
||||
encoder uses some of its smaller functions (`BrotliInitDistanceParams`,
|
||||
`PrefixEncodeCopyDistance`, `CommandRestoreDistanceCode` and
|
||||
`BrotliOptimizeHuffmanCountsForRle`).
|
||||
- `encode_block_splitter.wuffs`: `block_splitter.c` and
|
||||
`block_splitter_inc.h` (`BrotliSplitBlock`), as methods of the
|
||||
`metablock_builder` struct.
|
||||
- `encode_cluster.wuffs`: `cluster.c` and `cluster_inc.h`
|
||||
(`BrotliClusterHistograms` and its helpers), also as `metablock_builder`
|
||||
methods.
|
||||
- `encode_metablock.wuffs`: the `metablock_builder` struct and `metablock.c`
|
||||
(`BrotliInitDistanceParams`, `BrotliBuildMetaBlock`,
|
||||
`BrotliOptimizeHistograms`). See "Encoder Metablock Builder" below.
|
||||
Only quality 10 and above use `BrotliBuildMetaBlock`, so the encoder does
|
||||
not call it yet, but it uses some of the smaller functions
|
||||
(`BrotliInitDistanceParams`, `PrefixEncodeCopyDistance`,
|
||||
`CommandRestoreDistanceCode` and `BrotliOptimizeHuffmanCountsForRle`).
|
||||
- `encode_tester.wuffs`: TEST ONLY. The `metablock_tester` struct exposes
|
||||
the internal functions above to C test programs. It is not part of the
|
||||
encoder API.
|
||||
@@ -468,6 +474,120 @@ and for qualities 2 to 9 several hundred thousand, all byte-exact.
|
||||
only in the C code's one-shot `BrotliEncoderCompress`).
|
||||
|
||||
|
||||
### Encoder Metablock Builder
|
||||
|
||||
The `metablock_builder` struct (a `pri struct`, embeddable in the encoder
|
||||
with one field, e.g. `mb : metablock_builder` in the private data section)
|
||||
implements the quality 10 and 11 metablock construction (`BrotliBuildMetaBlock`)
|
||||
and `BrotliOptimizeHistograms`. The encoder has one (its `mb` field) but
|
||||
only calls its smaller helpers so far (see the source file list above). Its
|
||||
fixed-size state (about 0.8 MiB: FindBlocks' cost arrays and a 64Ki-entry
|
||||
FastLog2 cache in its `bit_cost`) lives in the struct; everything that the C
|
||||
code allocates on the heap lives in a caller-supplied workbuf.
|
||||
|
||||
Data formats (all little-endian, matching the C structs' memory layout on a
|
||||
64-bit little-endian machine, so that results can be compared with `memcmp`):
|
||||
|
||||
- A command is 16 bytes: `insert_len_` (u32), `copy_len_` (u32), `dist_extra_`
|
||||
(u32), `cmd_prefix_` (u16), `dist_prefix_` (u16), as built by `command.h`'s
|
||||
`InitCommand` / `InitInsertCommand`.
|
||||
- A histogram is `(4 * data_size) + 16` bytes: `data_size` u32 counts, the
|
||||
u64 `total_count_` and the binary64 bits of `bit_cost_`. `data_size` is 256
|
||||
(literals, 1040 bytes), 704 (commands, 2832 bytes) or 544 (distances, 2192
|
||||
bytes).
|
||||
- A BlockSplit's types are u8s and its lengths u32s. Context maps are u32s.
|
||||
- Doubles are binary64 bit patterns (u64), computed with `std/softfloat` in
|
||||
the C code's operation order.
|
||||
|
||||
Usage, mirroring the C code's `WriteMetaBlockInternal` for quality >= 10:
|
||||
|
||||
```
|
||||
// Once per metablock. The params are the C code's block_params, a copy of
|
||||
// the encoder's params: init_distance_params must be called each time,
|
||||
// because build_meta_block overwrites the dist_etc fields with the chosen
|
||||
// distance params.
|
||||
this.mb.set_params!(quality: q, large_window: lw,
|
||||
disable_literal_context_modeling: dlcm)
|
||||
this.mb.init_distance_params!(npostfix: np, ndirect: nd, large_window: lw)
|
||||
status = this.mb.build_meta_block!(wb: workbuf, ringbuffer: rb, pos: pos,
|
||||
mask: mask, prev_byte: pb, prev_byte2: pb2, cmds: cmds,
|
||||
num_commands: n, literal_context_mode: mode)
|
||||
// Optional (quality >= 4): ndc is min(this.mb.dist_alphabet_size_limit, 544).
|
||||
this.mb.optimize_histograms!(wb: workbuf, num_distance_codes: ndc)
|
||||
```
|
||||
|
||||
The exact signatures are:
|
||||
|
||||
- `set_params!(quality: u32, large_window: bool,
|
||||
disable_literal_context_modeling: bool)`.
|
||||
- `init_distance_params!(npostfix: u32, ndirect: u32, large_window: bool)`
|
||||
(`BrotliInitDistanceParams`; sets `dist_postfix_bits`,
|
||||
`dist_num_direct_codes`, `dist_alphabet_size_max`,
|
||||
`dist_alphabet_size_limit` and `dist_max_distance`).
|
||||
- `build_meta_block!(wb: slice u8, ringbuffer: roslice u8, pos: u64, mask:
|
||||
u64, prev_byte: u8, prev_byte2: u8, cmds: slice u8, num_commands: u64,
|
||||
literal_context_mode: u32) status`. `ringbuffer` must be at least `mask + 1`
|
||||
bytes. `literal_context_mode` is a `ContextType` (0 = LSB6, 1 = MSB6, 2 =
|
||||
UTF8, 3 = SIGNED). Like the C code, it rewrites the commands'
|
||||
`dist_prefix_` and `dist_extra_` fields for the chosen distance params and
|
||||
leaves those params in the `dist_etc` fields. It returns
|
||||
`"#bad workbuf length"` if `wb` is too short (the results are then invalid
|
||||
but `cmds` may already be rewritten, so retrying needs the original
|
||||
commands).
|
||||
- `optimize_histograms!(wb: slice u8, num_distance_codes: u32)` works on the
|
||||
MetaBlockSplit described by the fields below, whether `build_meta_block`
|
||||
or the caller (e.g. a greedy builder) filled them in.
|
||||
- `workbuf_len_worst_case(num_literals: u64, num_commands: u64) u64` is an
|
||||
upper bound on the workbuf length that `build_meta_block` needs, where
|
||||
`num_literals` is the sum of the commands' insert lengths. It assumes the
|
||||
worst case for every run-time-sized allocation (one `FindBlocks` block per
|
||||
symbol, 256 block types with 64 literal contexts each, full histogram pair
|
||||
queues), so it is far larger than what real inputs need: about 76 MiB for
|
||||
a 1 KiB metablock, 2.6 GiB for 1 MiB of literals and 128 Ki commands, and
|
||||
41 GiB for 16 MiB of literals and 4 Mi commands. These are the C code's
|
||||
own worst cases (`ClusterBlocks` and `BrotliClusterHistograms` size their
|
||||
histogram pair queues by the number of blocks and clusters), not artifacts
|
||||
of the port. `wb_high` records what a call actually used: over the parity
|
||||
harness's inputs, at most about 42 MB (for 1.3 MiB of literals split into
|
||||
256 literal block types).
|
||||
|
||||
The MetaBlockSplit result is described by these fields, indexed by category
|
||||
(0 = literal, 1 = command, 2 = distance), with offsets into `wb`:
|
||||
|
||||
| Field | C code |
|
||||
| ---------------------- | ----------------------------------------------- |
|
||||
| `split_num_types[c]` | `{literal,command,distance}_split.num_types` |
|
||||
| `split_num_blocks[c]` | `..._split.num_blocks` |
|
||||
| `split_types_off[c]` | `..._split.types` (u8 array) |
|
||||
| `split_lengths_off[c]` | `..._split.lengths` (u32 array) |
|
||||
| `context_map_off[c]` | `{literal,distance}_context_map` (u32 array; none for commands) |
|
||||
| `context_map_size[c]` | `{literal,distance}_context_map_size` |
|
||||
| `histograms_off[c]` | `{literal,command,distance}_histograms` |
|
||||
| `histograms_size[c]` | `..._histograms_size` |
|
||||
|
||||
Workbuf layout during `build_meta_block` (a bump allocator, `wb_top`, carves
|
||||
regions out of `wb` in this order; "scratch" regions are released when the
|
||||
C code frees them, so later regions reuse their bytes):
|
||||
|
||||
1. Scratch: one distance histogram for the distance parameter search.
|
||||
2. The three BlockSplits' types and lengths arrays, sized for one block per
|
||||
symbol (`max(1, n)` entries, where n is the number of literals, commands
|
||||
or distance codes).
|
||||
3. Scratch: `BrotliSplitBlock`'s literal / command code / distance code
|
||||
arrays and `SplitByteVector`'s histograms, block ids, bitmaps and
|
||||
`ClusterBlocks` state.
|
||||
4. The command histograms, literal context map, literal histograms (room for
|
||||
`64 * num_literal_types`), distance context map and distance histograms
|
||||
(room for `4 * num_distance_types`).
|
||||
5. Scratch: the per-context literal and distance histograms that are then
|
||||
clustered, and `BrotliClusterHistograms`' state (cluster sizes, histogram
|
||||
pairs, etc).
|
||||
|
||||
After `build_meta_block` returns, the results (regions 2 and 4) are below
|
||||
`wb_top`, and `wb[wb_top ..]` is free for the caller (e.g. for storing the
|
||||
metablock) for as long as it needs the results.
|
||||
|
||||
|
||||
### Encoder TODO
|
||||
|
||||
- Qualities 10 and 11 (Zopfli, the H10 binary tree hasher and
|
||||
|
||||
@@ -1021,6 +1021,8 @@ pri func metablock_builder.cluster_scratch_len(in_size: base.u64, hs: base.u64)
|
||||
pri func metablock_builder.workbuf_len_worst_case(num_literals: base.u64, num_commands: base.u64) base.u64 {
|
||||
var l : base.u64
|
||||
var c : base.u64
|
||||
var tl : base.u64
|
||||
var tc : base.u64
|
||||
var n : base.u64
|
||||
var split : base.u64
|
||||
|
||||
@@ -1037,13 +1039,19 @@ pri func metablock_builder.workbuf_len_worst_case(num_literals: base.u64, num_co
|
||||
split = split.max(no_less_than: (2 ~mod* c) ~sat+
|
||||
this.sbv_scratch_len(length: c, hs: HISTOGRAM_DISTANCE_SIZE, symbols_per_histogram: 544, max_histograms: 50))
|
||||
n ~sat+= split.max(no_less_than: HISTOGRAM_DISTANCE_SIZE)
|
||||
// The MetaBlockSplit's histograms and context maps (with at most 256
|
||||
// block types per category), plus the input histograms for clustering.
|
||||
n ~sat+= (256 * HISTOGRAM_COMMAND_SIZE) + (16384 * 4) + (16384 * HISTOGRAM_LITERAL_SIZE) +
|
||||
(1024 * 4) + (1024 * HISTOGRAM_DISTANCE_SIZE)
|
||||
n ~sat+= (16384 * HISTOGRAM_LITERAL_SIZE) + (1024 * HISTOGRAM_DISTANCE_SIZE)
|
||||
// The number of block types is at most 256 (MAX_NUMBER_OF_HISTOGRAMS)
|
||||
// and at most the number of symbols (but at least 1). tl bounds the
|
||||
// literal block types and tc bounds both the command and the distance
|
||||
// block types (there are at most as many distances as commands).
|
||||
tl = l.max(no_less_than: 1).min(no_more_than: 256)
|
||||
tc = c.max(no_less_than: 1).min(no_more_than: 256)
|
||||
// The MetaBlockSplit's histograms and context maps, plus the input
|
||||
// histograms for clustering.
|
||||
n ~sat+= (tc * HISTOGRAM_COMMAND_SIZE) + (tl * 64 * 4) + (tl * 64 * HISTOGRAM_LITERAL_SIZE) +
|
||||
(tc * 4 * 4) + (tc * 4 * HISTOGRAM_DISTANCE_SIZE)
|
||||
n ~sat+= (tl * 64 * HISTOGRAM_LITERAL_SIZE) + (tc * 4 * HISTOGRAM_DISTANCE_SIZE)
|
||||
// cluster_histograms' scratch.
|
||||
n ~sat+= this.cluster_scratch_len(in_size: 16384, hs: HISTOGRAM_LITERAL_SIZE).max(no_less_than:
|
||||
this.cluster_scratch_len(in_size: 1024, hs: HISTOGRAM_DISTANCE_SIZE))
|
||||
n ~sat+= this.cluster_scratch_len(in_size: tl * 64, hs: HISTOGRAM_LITERAL_SIZE).max(no_less_than:
|
||||
this.cluster_scratch_len(in_size: tc * 4, hs: HISTOGRAM_DISTANCE_SIZE))
|
||||
return n
|
||||
}
|
||||
@@ -1907,6 +1907,383 @@ bench_mimic_brotli_encode_alice29_q9() {
|
||||
|
||||
#endif // WUFFS_MIMIC
|
||||
|
||||
// ---------------- Metablock Builder Tests
|
||||
|
||||
// These tests check the encoder's metablock builder (the ports of
|
||||
// libbrotlienc's bit_cost.c, histogram.c, block_splitter.c, cluster.c and
|
||||
// metablock.c, exposed through the test-only metablock_tester struct)
|
||||
// against golden values computed with libbrotlienc v1.2.1. The much larger
|
||||
// script/manual-test-brotli-metablock-parity.c compares them with the C
|
||||
// functions directly.
|
||||
|
||||
static uint32_t //
|
||||
mbt_log2_floor(uint64_t x) {
|
||||
uint32_t n = 0;
|
||||
while (x >>= 1) {
|
||||
n++;
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
static uint64_t //
|
||||
mbt_fnv(uint64_t h, const uint8_t* p, size_t n) {
|
||||
size_t i;
|
||||
for (i = 0; i < n; i++) {
|
||||
h = (h ^ p[i]) * 0x100000001B3ull;
|
||||
}
|
||||
return h;
|
||||
}
|
||||
|
||||
static uint64_t //
|
||||
mbt_fnv_u64(uint64_t h, uint64_t v) {
|
||||
uint8_t b[8];
|
||||
int i;
|
||||
for (i = 0; i < 8; i++) {
|
||||
b[i] = (uint8_t)(v >> (8 * i));
|
||||
}
|
||||
return mbt_fnv(h, b, 8);
|
||||
}
|
||||
|
||||
static void //
|
||||
mbt_put_u32le(uint8_t* p, uint32_t v) {
|
||||
p[0] = (uint8_t)(v >> 0);
|
||||
p[1] = (uint8_t)(v >> 8);
|
||||
p[2] = (uint8_t)(v >> 16);
|
||||
p[3] = (uint8_t)(v >> 24);
|
||||
}
|
||||
|
||||
// mbt_make_cmds writes the 16-byte commands (as command.h's InitCommand and
|
||||
// InitInsertCommand would, for the given npostfix and ndirect) of a
|
||||
// pseudo-random parse of total bytes to dst and returns how many there are.
|
||||
// dst must have room for total + 1 commands.
|
||||
static size_t //
|
||||
mbt_make_cmds(uint8_t* dst, size_t total, uint32_t npostfix, uint32_t ndirect,
|
||||
uint32_t seed) {
|
||||
size_t n = 0;
|
||||
size_t pos = 0;
|
||||
uint32_t r = seed;
|
||||
while (pos < total) {
|
||||
size_t ins, copy, dcode, rem;
|
||||
uint32_t inscode, copycode, dist_extra = 0;
|
||||
uint16_t dist_prefix, cmd_prefix;
|
||||
int use_last;
|
||||
int insert_only = 0;
|
||||
r = (r * 1103515245u) + 12345u;
|
||||
ins = (r >> 16) % ((r & 0x300) ? 12 : 300);
|
||||
r = (r * 1103515245u) + 12345u;
|
||||
copy = 2 + ((r >> 16) % ((r & 0x700) ? 14 : 200));
|
||||
rem = total - pos;
|
||||
if (ins > rem) {
|
||||
ins = rem;
|
||||
}
|
||||
if ((rem - ins) < copy) {
|
||||
ins = rem;
|
||||
copy = 0;
|
||||
}
|
||||
if (copy == 0) {
|
||||
// InitInsertCommand.
|
||||
insert_only = 1;
|
||||
dist_prefix = 16;
|
||||
copy = 4;
|
||||
use_last = 0;
|
||||
} else {
|
||||
r = (r * 1103515245u) + 12345u;
|
||||
if ((r & 0x30000) == 0) {
|
||||
dcode = (r >> 18) & 15;
|
||||
} else {
|
||||
size_t max = pos + ins;
|
||||
dcode = 16 + (((r >> 8) % (max ? max : 1)) % 3000);
|
||||
}
|
||||
// PrefixEncodeCopyDistance.
|
||||
if (dcode < 16 + ndirect) {
|
||||
dist_prefix = (uint16_t)dcode;
|
||||
} else {
|
||||
size_t d = ((size_t)1 << (npostfix + 2)) + (dcode - 16 - ndirect);
|
||||
size_t bucket = mbt_log2_floor(d) - 1;
|
||||
size_t postfix = d & ((1u << npostfix) - 1);
|
||||
size_t prefix = (d >> bucket) & 1;
|
||||
size_t offset = (2 + prefix) << bucket;
|
||||
size_t nbits = bucket - npostfix;
|
||||
dist_prefix = (uint16_t)((nbits << 10) |
|
||||
(16 + ndirect +
|
||||
((2 * (nbits - 1) + prefix) << npostfix) +
|
||||
postfix));
|
||||
dist_extra = (uint32_t)((d - offset) >> npostfix);
|
||||
}
|
||||
use_last = (dist_prefix & 0x3FF) == 0;
|
||||
}
|
||||
// GetInsertLengthCode.
|
||||
if (ins < 6) {
|
||||
inscode = (uint32_t)ins;
|
||||
} else if (ins < 130) {
|
||||
uint32_t nb = mbt_log2_floor(ins - 2) - 1;
|
||||
inscode = (nb << 1) + (uint32_t)((ins - 2) >> nb) + 2;
|
||||
} else if (ins < 2114) {
|
||||
inscode = mbt_log2_floor(ins - 66) + 10;
|
||||
} else if (ins < 6210) {
|
||||
inscode = 21;
|
||||
} else if (ins < 22594) {
|
||||
inscode = 22;
|
||||
} else {
|
||||
inscode = 23;
|
||||
}
|
||||
// GetCopyLengthCode.
|
||||
if (copy < 10) {
|
||||
copycode = (uint32_t)(copy - 2);
|
||||
} else if (copy < 134) {
|
||||
uint32_t nb = mbt_log2_floor(copy - 6) - 1;
|
||||
copycode = (nb << 1) + (uint32_t)((copy - 6) >> nb) + 4;
|
||||
} else if (copy < 2118) {
|
||||
copycode = mbt_log2_floor(copy - 70) + 12;
|
||||
} else {
|
||||
copycode = 23;
|
||||
}
|
||||
// CombineLengthCodes.
|
||||
{
|
||||
uint32_t bits64 = (copycode & 7) | ((inscode & 7) << 3);
|
||||
if (use_last && (inscode < 8) && (copycode < 16)) {
|
||||
cmd_prefix = (uint16_t)((copycode < 8) ? bits64 : (bits64 | 64));
|
||||
} else {
|
||||
uint32_t off = 2 * ((copycode >> 3) + 3 * (inscode >> 3));
|
||||
off = (off << 5) + 0x40 + ((0x520D40u >> off) & 0xC0);
|
||||
cmd_prefix = (uint16_t)(off | bits64);
|
||||
}
|
||||
}
|
||||
mbt_put_u32le(dst + 16 * n + 0, (uint32_t)ins);
|
||||
mbt_put_u32le(dst + 16 * n + 4, insert_only ? (4u << 25) : (uint32_t)copy);
|
||||
mbt_put_u32le(dst + 16 * n + 8, dist_extra);
|
||||
dst[16 * n + 12] = (uint8_t)cmd_prefix;
|
||||
dst[16 * n + 13] = (uint8_t)(cmd_prefix >> 8);
|
||||
dst[16 * n + 14] = (uint8_t)dist_prefix;
|
||||
dst[16 * n + 15] = (uint8_t)(dist_prefix >> 8);
|
||||
n++;
|
||||
pos += ins + (insert_only ? 0 : copy);
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
const char* filename; // NULL means the synthetic input.
|
||||
uint32_t quality;
|
||||
uint32_t mode;
|
||||
uint32_t nolcm;
|
||||
uint32_t npostfix;
|
||||
uint32_t ndirect;
|
||||
uint32_t large_window;
|
||||
uint32_t seed;
|
||||
uint64_t want_hash;
|
||||
} mbt_case;
|
||||
|
||||
static const mbt_case g_mbt_cases[] = {
|
||||
{"test/data/romeo.txt", 11, 2, 0, 0, 0, 0, 0x12345678,
|
||||
0x91BCDB702326DA68},
|
||||
{"test/data/midsummer.txt", 10, 3, 1, 1, 4, 0, 0x9ABCDEF0,
|
||||
0xC1BC25672AFFE52C},
|
||||
{NULL, 11, 0, 0, 2, 8, 1, 0x0BADF00D, 0x106C079EF30D8335},
|
||||
{NULL, 10, 1, 0, 3, 120, 0, 0x00C0FFEE, 0x1CE5AB9FFAB4CEDB},
|
||||
{"test/data/pi.txt", 11, 2, 0, 1, 2, 0, 0x31415926, 0x4434059D684D4F0E},
|
||||
{"", 11, 2, 0, 0, 0, 0, 0x00000001, 0xAE70E50382B8388F},
|
||||
};
|
||||
|
||||
const char* //
|
||||
test_wuffs_brotli_metablock_build_meta_block() {
|
||||
CHECK_FOCUS(__func__);
|
||||
wuffs_brotli__metablock_tester* t = wuffs_brotli__metablock_tester__alloc();
|
||||
if (!t) {
|
||||
RETURN_FAIL("alloc failed");
|
||||
}
|
||||
const char* ret = NULL;
|
||||
uint8_t* rb = NULL;
|
||||
uint8_t* cmds = NULL;
|
||||
uint8_t* wb = NULL;
|
||||
size_t wb_len = 1 << 20;
|
||||
size_t i;
|
||||
for (i = 0; i < WUFFS_TESTLIB_ARRAY_SIZE(g_mbt_cases); i++) {
|
||||
const mbt_case* tc = &g_mbt_cases[i];
|
||||
size_t total = 0;
|
||||
size_t rb_size = 1024;
|
||||
size_t n, k;
|
||||
int c;
|
||||
uint64_t h = 0xCBF29CE484222325ull;
|
||||
wuffs_base__status status;
|
||||
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
|
||||
.data = g_have_slice_u8,
|
||||
});
|
||||
if (!tc->filename) {
|
||||
// A byte distribution that shifts every 700 bytes, 48 KiB.
|
||||
uint32_t r = 1;
|
||||
total = 49152;
|
||||
for (k = 0; k < total; k++) {
|
||||
r = (r * 1103515245u) + 12345u;
|
||||
src.data.ptr[k] = (uint8_t)(((k / 700) * 37) +
|
||||
((r >> 16) % (1 + (k / 700) % 23)));
|
||||
}
|
||||
} else if (*tc->filename) {
|
||||
ret = read_file(&src, tc->filename);
|
||||
if (ret) {
|
||||
goto done;
|
||||
}
|
||||
total = src.meta.wi;
|
||||
}
|
||||
while (rb_size < total) {
|
||||
rb_size *= 2;
|
||||
}
|
||||
free(rb);
|
||||
free(cmds);
|
||||
rb = calloc(rb_size, 1);
|
||||
cmds = malloc(16 * (total + 1));
|
||||
if (!rb || !cmds) {
|
||||
ret = "out of memory";
|
||||
goto done;
|
||||
}
|
||||
if (total) {
|
||||
memcpy(rb, src.data.ptr, total);
|
||||
}
|
||||
n = mbt_make_cmds(cmds, total, tc->npostfix, tc->ndirect, tc->seed);
|
||||
while (true) {
|
||||
if (!wb) {
|
||||
wb = malloc(wb_len);
|
||||
if (!wb) {
|
||||
ret = "out of memory";
|
||||
goto done;
|
||||
}
|
||||
}
|
||||
status = wuffs_brotli__metablock_tester__build_meta_block(
|
||||
t, wuffs_base__make_slice_u8(wb, wb_len),
|
||||
wuffs_base__make_slice_u8(rb, rb_size), 0, rb_size - 1, 0, 0,
|
||||
wuffs_base__make_slice_u8(cmds, 16 * n), n, tc->mode, tc->quality,
|
||||
tc->large_window, tc->nolcm, tc->npostfix, tc->ndirect);
|
||||
if (status.repr != wuffs_base__error__bad_workbuf_length) {
|
||||
break;
|
||||
}
|
||||
// The commands were rewritten, so regenerate them.
|
||||
n = mbt_make_cmds(cmds, total, tc->npostfix, tc->ndirect, tc->seed);
|
||||
free(wb);
|
||||
wb = NULL;
|
||||
wb_len *= 2;
|
||||
}
|
||||
if (status.repr) {
|
||||
ret = status.repr;
|
||||
goto done;
|
||||
}
|
||||
{
|
||||
uint64_t ndc = wuffs_brotli__metablock_tester__get(t, 27);
|
||||
wuffs_brotli__metablock_tester__optimize_histograms(
|
||||
t, wuffs_base__make_slice_u8(wb, wb_len),
|
||||
(uint32_t)((ndc < 544) ? ndc : 544));
|
||||
}
|
||||
for (c = 0; c < 3; c++) {
|
||||
uint64_t num_blocks = wuffs_brotli__metablock_tester__get(t, 4 * c + 1);
|
||||
uint64_t types = wuffs_brotli__metablock_tester__get(t, 4 * c + 2);
|
||||
uint64_t lengths = wuffs_brotli__metablock_tester__get(t, 4 * c + 3);
|
||||
h = mbt_fnv_u64(h, wuffs_brotli__metablock_tester__get(t, 4 * c + 0));
|
||||
h = mbt_fnv_u64(h, num_blocks);
|
||||
h = mbt_fnv(h, wb + types, num_blocks);
|
||||
h = mbt_fnv(h, wb + lengths, 4 * num_blocks);
|
||||
}
|
||||
for (c = 0; c < 3; c += 2) {
|
||||
uint64_t size = wuffs_brotli__metablock_tester__get(t, 15 + c);
|
||||
h = mbt_fnv_u64(h, size);
|
||||
h = mbt_fnv(h, wb + wuffs_brotli__metablock_tester__get(t, 12 + c),
|
||||
4 * size);
|
||||
}
|
||||
for (c = 0; c < 3; c++) {
|
||||
static const size_t hsizes[3] = {1040, 2832, 2192};
|
||||
uint64_t size = wuffs_brotli__metablock_tester__get(t, 21 + c);
|
||||
h = mbt_fnv_u64(h, size);
|
||||
h = mbt_fnv(h, wb + wuffs_brotli__metablock_tester__get(t, 18 + c),
|
||||
hsizes[c] * size);
|
||||
}
|
||||
for (c = 24; c < 29; c++) {
|
||||
h = mbt_fnv_u64(h, wuffs_brotli__metablock_tester__get(t, c));
|
||||
}
|
||||
h = mbt_fnv(h, cmds, 16 * n);
|
||||
if (h != tc->want_hash) {
|
||||
RETURN_FAIL("case %zu: hash: have 0x%016" PRIX64 ", want 0x%016" PRIX64,
|
||||
i, h, tc->want_hash);
|
||||
}
|
||||
if (wuffs_brotli__metablock_tester__get(t, 29) >
|
||||
wuffs_brotli__metablock_tester__get(t, 30)) {
|
||||
RETURN_FAIL("case %zu: workbuf used exceeds the worst case", i);
|
||||
}
|
||||
}
|
||||
done:
|
||||
free(rb);
|
||||
free(cmds);
|
||||
free(wb);
|
||||
free(t);
|
||||
return ret;
|
||||
}
|
||||
|
||||
const char* //
|
||||
test_wuffs_brotli_metablock_bit_cost() {
|
||||
CHECK_FOCUS(__func__);
|
||||
wuffs_brotli__metablock_tester* t = wuffs_brotli__metablock_tester__alloc();
|
||||
if (!t) {
|
||||
RETURN_FAIL("alloc failed");
|
||||
}
|
||||
// The C code's BrotliBitsEntropy and BrotliPopulationCost{Literal,Command,
|
||||
// Distance} results, as binary64 bit patterns.
|
||||
static const uint64_t want_pop[6][3] = {
|
||||
{0x4028000000000000, 0x4028000000000000, 0x4028000000000000},
|
||||
{0x4028000000000000, 0x4028000000000000, 0x4028000000000000},
|
||||
{0x4055800000000000, 0x4055800000000000, 0x4055800000000000},
|
||||
{0x4055800000000000, 0x406AC00000000000, 0x406AC00000000000},
|
||||
{0x4055800000000000, 0x4076300000000000, 0x4076300000000000},
|
||||
{0x40F842EE8C1F42B3, 0x4113DB811CA8812F, 0x410D76A1EA720A2C},
|
||||
};
|
||||
static const uint32_t data_sizes[3] = {256, 704, 544};
|
||||
// Each histogram is data_size u32le counts, a u64le total_count_ and a
|
||||
// binary64 bit_cost_.
|
||||
uint8_t pop[4 * 704];
|
||||
uint8_t histo[(4 * 704) + 16];
|
||||
uint32_t counts[704];
|
||||
size_t i;
|
||||
int k, c;
|
||||
for (i = 0; i < 704; i++) {
|
||||
counts[i] = (uint32_t)((i * i * 7 + 3) % 101);
|
||||
mbt_put_u32le(pop + 4 * i, counts[i]);
|
||||
}
|
||||
{
|
||||
uint64_t have = wuffs_brotli__metablock_tester__bits_entropy(
|
||||
t, wuffs_base__make_slice_u8(pop, sizeof(pop)), 256);
|
||||
if (have != 0x40F79A0C92B03A26ull) {
|
||||
free(t);
|
||||
RETURN_FAIL("bits_entropy: have 0x%016" PRIX64, have);
|
||||
}
|
||||
}
|
||||
for (k = 0; k < 6; k++) {
|
||||
static const int nz[6] = {0, 1, 2, 3, 4, 300};
|
||||
for (c = 0; c < 3; c++) {
|
||||
uint64_t total = 0;
|
||||
uint64_t have;
|
||||
memset(histo, 0, sizeof(histo));
|
||||
for (i = 0; i < data_sizes[c]; i++) {
|
||||
int on = (nz[k] >= 300) ? (counts[i] != 0)
|
||||
: (((i % 151) == 7) && ((int)(i / 151) < nz[k]));
|
||||
if (on) {
|
||||
mbt_put_u32le(histo + 4 * i, counts[i] + 1);
|
||||
total += counts[i] + 1;
|
||||
}
|
||||
}
|
||||
for (i = 0; i < 8; i++) {
|
||||
histo[(4 * data_sizes[c]) + i] = (uint8_t)(total >> (8 * i));
|
||||
}
|
||||
have = wuffs_brotli__metablock_tester__population_cost(
|
||||
t, wuffs_base__make_slice_u8(histo, (4 * data_sizes[c]) + 16),
|
||||
data_sizes[c]);
|
||||
if (have != want_pop[k][c]) {
|
||||
free(t);
|
||||
RETURN_FAIL("population_cost(k=%d, c=%d): have 0x%016" PRIX64
|
||||
", want 0x%016" PRIX64,
|
||||
k, c, have, want_pop[k][c]);
|
||||
}
|
||||
}
|
||||
}
|
||||
free(t);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
// ---------------- Large Window and Dictionary Tests
|
||||
|
||||
// See test/data/brotli/README.md for where the large-window (lw), raw
|
||||
@@ -2805,6 +3182,10 @@ proc g_tests[] = {
|
||||
test_wuffs_brotli_decode_large_window_far,
|
||||
test_wuffs_brotli_decode_large_window_quirk,
|
||||
|
||||
// Metablock builder tests.
|
||||
test_wuffs_brotli_metablock_bit_cost,
|
||||
test_wuffs_brotli_metablock_build_meta_block,
|
||||
|
||||
#ifdef WUFFS_MIMIC
|
||||
|
||||
test_mimic_brotli_decode_alice29,
|
||||
|
||||
Reference in new issue
Block a user