test/c/std/brotli: add large-window encoder tests

Check large-window encoding at qualities 3 to 9 (lgwin 18 to 30, with
the H35, H55 and H65 composite hashers) against libbrotlienc, with round
trips through Wuffs' decoder (with the large-window quirk) and
libbrotlidec, over some small files and a 46 MiB input with repeats at
distances of about 20 and 40 MiB. Also check the large-window
workbuf_len and allow encoder workbufs of up to 3 GiB in the test
driver.
This commit is contained in:
Claude committed 2026-10-27 02:28:35 +00:00
1 parent 75b09b6700
commit dbb42802be
1 file changed
+284 -10
+284 -10
View File
@@ -894,7 +894,9 @@ wuffs_brotli_encode_plan(wuffs_base__io_buffer* dst,
wuffs_base__range_ii_u64 workbuf_len =
wuffs_brotli__encoder__workbuf_len(enc);
wuffs_base__slice_u8 workbuf = g_work_slice_u8;
if (workbuf_len.max_incl > (1ull << 31)) {
// Large-window brotli at lgwin 30 needs more than 2 GiB.
if ((workbuf_len.max_incl > (3ull << 30)) ||
(workbuf_len.max_incl > SIZE_MAX)) {
ret = "workbuf_len is too large";
goto cleanup;
} else if (workbuf_len.max_incl > g_work_slice_u8.len) {
@@ -1336,10 +1338,43 @@ test_wuffs_brotli_encode_parameters() {
src = make_io_buffer_from_string("", 0);
dst.meta.wi = 0;
// Qualities 10 and 11 (and 12, which sanitizes to 11) are unsupported, and
// so is large-window brotli at qualities 3 and above.
for (uint32_t q = 3; q <= 12; q++) {
bool large_window = q < 10;
// With the large-window quirk, lgwin can be up to 30 (instead of 24) at
// qualities 3 and above. Like the C code, quality 2 and below ignore the
// quirk. Above lgwin 24, the ring buffer is (2 << lgwin) bytes and, at
// quality 3, the H35 hasher has a 64 MiB rolling hash table on top of
// H3's 256 KiB.
for (uint32_t q = 2; q <= 3; q++) {
uint64_t wlens[3] = {0};
for (int k = 0; k < 3; k++) {
free(enc);
enc = wuffs_brotli__encoder__alloc();
if (!enc) {
RETURN_FAIL("alloc failed");
}
wuffs_brotli__encoder__set_quirk(
enc, WUFFS_BROTLI__QUIRK_ENCODER_QUALITY, q);
wuffs_brotli__encoder__set_quirk(enc, WUFFS_BROTLI__QUIRK_LGWIN,
(k == 0) ? 24 : 30);
if (k > 0) {
wuffs_brotli__encoder__set_quirk(
enc, WUFFS_BROTLI__QUIRK_LARGE_WINDOW, (k == 2) ? 1 : 0);
}
wlens[k] = wuffs_brotli__encoder__workbuf_len(enc).max_incl;
}
BROTLI_ENC_CHECK(wlens[0] == wlens[1],
"lgwin 30 without large window is lgwin 24");
if (q == 2) {
BROTLI_ENC_CHECK(wlens[0] == wlens[2],
"large window is ignored at quality 2");
} else {
BROTLI_ENC_CHECK(
wlens[2] == (wlens[0] + (2ull << 30) - (2ull << 24) + (64 << 20)),
"workbuf_len at q3w30 with large window");
}
}
// Qualities 10 and 11 (and 12, which sanitizes to 11) are unsupported.
for (uint32_t q = 10; q <= 12; q++) {
free(enc);
enc = wuffs_brotli__encoder__alloc();
if (!enc) {
@@ -1347,17 +1382,13 @@ test_wuffs_brotli_encode_parameters() {
}
wuffs_brotli__encoder__set_quirk(enc, WUFFS_BROTLI__QUIRK_ENCODER_QUALITY,
q);
if (large_window) {
wuffs_brotli__encoder__set_quirk(enc, WUFFS_BROTLI__QUIRK_LARGE_WINDOW,
1);
}
BROTLI_ENC_CHECK(wuffs_brotli__encoder__workbuf_len(enc).max_incl == 0,
"workbuf_len for an unsupported quality");
status = wuffs_brotli__encoder__transform_io(enc, &dst, &src,
g_work_slice_u8);
BROTLI_ENC_CHECK(
status.repr == wuffs_brotli__error__unsupported_encoder_quality,
large_window ? "large window at quality 3..9" : "quality 10..11");
"quality 10..11");
}
// Requests.
@@ -3133,6 +3164,245 @@ test_mimic_brotli_decode_large_window_far() {
#endif // WUFFS_MIMIC
// ---------------- Brotli Large Window Encoder Tests
// do_test_brotli_encode_large_window encodes data[:data_len] with the plan's
// config c (which should set the large-window quirk) and checks that
// decoding the result (with Wuffs' std/brotli decoder with the large-window
// quirk set, and, if mimic is true, with libbrotlidec) gives back the
// original data. If mimic is true, it also checks that libbrotlienc's output
// is the same. If max_compressed_len is non-zero, it checks that the
// compressed length is at most that. The decoded data goes in g_want.
const char* //
do_test_brotli_encode_large_window(const char* name,
const brotli_enc_config* c,
const uint8_t* data,
size_t data_len,
bool mimic,
size_t max_compressed_len) {
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
.data = g_src_slice_u8,
});
CHECK_STRING(make_brotli_enc_plan(&src, c, data, data_len));
wuffs_base__io_buffer have = ((wuffs_base__io_buffer){
.data = g_have_slice_u8,
});
wuffs_base__io_buffer src0 = src;
const char* status = wuffs_brotli_encode_plan(
&have, &src0, c->param_keys, c->param_values, c->num_params,
g_brotli_enc_step_ops, g_brotli_enc_step_lens, g_brotli_enc_num_steps,
c->dst_chunk);
if (status) {
RETURN_FAIL("%s: wuffs encode: %s", name, status);
} else if (max_compressed_len && (have.meta.wi > max_compressed_len)) {
RETURN_FAIL("%s: compressed length: have %zu, want at most %zu", name,
have.meta.wi, max_compressed_len);
}
have.meta.closed = true;
for (int decoder = 0; decoder < (mimic ? 3 : 1); decoder++) {
wuffs_base__io_buffer want = ((wuffs_base__io_buffer){
.data = g_want_slice_u8,
});
wuffs_base__io_buffer have1 = have;
if (decoder == 0) {
status = wuffs_brotli_decode_with_options(
&want, &have1, UINT64_MAX, UINT64_MAX, true, BROTLI_DICT_SET_NONE);
#ifdef WUFFS_MIMIC
} else if (decoder == 1) {
status = mimic_brotli_decode_large_window(
&want, &have1, UINT64_MAX, UINT64_MAX, true, BROTLI_DICT_SET_NONE);
} else {
wuffs_base__io_buffer src1 = src;
status = mimic_brotli_encode(
&want, &src1, c->param_keys, c->param_values, c->num_params,
g_brotli_enc_step_ops, g_brotli_enc_step_lens,
g_brotli_enc_num_steps, c->dst_chunk);
if (status) {
RETURN_FAIL("%s: mimic encode: %s", name, status);
}
status = check_io_buffers_equal("", &have, &want);
if (status) {
RETURN_FAIL("%s: %s", name, status);
}
continue;
#endif // WUFFS_MIMIC
}
const char* which = (decoder == 0) ? "wuffs" : "mimic";
if (status) {
RETURN_FAIL("%s: %s decode: %s", name, which, status);
} else if (have1.meta.ri != have1.meta.wi) {
RETURN_FAIL("%s: %s decode: trailing data", name, which);
} else if (want.meta.wi != data_len) {
RETURN_FAIL("%s: %s decoded length: have %zu, want %zu", name, which,
want.meta.wi, data_len);
} else if (memcmp(want.data.ptr, data, data_len)) {
RETURN_FAIL("%s: %s decoded bytes differ", name, which);
}
}
return NULL;
}
// do_test_brotli_encode_large_window_files runs a grid of large-window
// configurations (qualities 3 to 9, lgwin 18 to 30) over a few small files.
const char* //
do_test_brotli_encode_large_window_files(bool mimic) {
static const char* filenames[] = {
"test/data/romeo.txt",
"test/data/brotli/alice29.txt",
"test/data/brotli/random_org_10k.bin",
"test/data/brotli/quickfox_repeated",
};
static const uint32_t lgwins[] = {18, 25, 26, 28, 30};
wuffs_base__io_buffer data = ((wuffs_base__io_buffer){
.data = g_work_slice_u8,
});
for (size_t f = 0; f < WUFFS_TESTLIB_ARRAY_SIZE(filenames); f++) {
// g_work is free to use for the input: each config's workbuf (of more
// than 64 MiB) is malloc'ed.
data.meta.wi = 0;
data.meta.ri = 0;
data.meta.closed = false;
CHECK_STRING(read_file(&data, filenames[f]));
uint8_t* copy = malloc(data.meta.wi ? data.meta.wi : 1);
if (!copy) {
return "malloc failed";
}
memcpy(copy, data.data.ptr, data.meta.wi);
const char* ret = NULL;
for (uint32_t q = 3; (q <= 9) && !ret; q++) {
for (size_t w = 0; (w < 5) && !ret; w++) {
for (int k = 0; (k < 2) && !ret; k++) {
if ((k == 1) && (lgwins[w] != 26)) {
continue;
}
brotli_enc_config c = make_brotli_enc_config(q, lgwins[w]);
brotli_enc_config__add_param(&c, BROTLI_ENC_PARAM_LARGE_WINDOW, 1);
if (k == 1) {
// A big size hint selects H55 (at quality 4) and H65 (at
// qualities 5 to 9).
brotli_enc_config__add_param(&c, BROTLI_ENC_PARAM_SIZE_HINT,
1 << 20);
}
c.chunking = (w & 1) ? BROTLI_ENC_CHUNK_SINGLE
: BROTLI_ENC_CHUNK_RANDOM;
c.flush = (w & 1) ? BROTLI_ENC_FLUSH_NONE : BROTLI_ENC_FLUSH_MIDDLE;
c.seed = (q << 8) | w;
char s[256];
snprintf(s, sizeof s, "%s: q=%u lgwin=%u size_hint=%s",
filenames[f], q, lgwins[w], k ? "1<<20" : "0");
ret = do_test_brotli_encode_large_window(s, &c, copy, data.meta.wi,
mimic, 0);
}
}
}
free(copy);
if (ret) {
return ret;
}
}
return NULL;
}
// make_brotli_enc_far_data returns a malloc'ed buffer of pseudo-random bytes
// with two 2 MiB repeats at distances of (20 MiB + 64) and (40 MiB - 4096).
// The repeats' sources are not repeated elsewhere, so only an encoder with a
// large enough window (lgwin 25 or more for the first, 26 or more for the
// second) can compress it.
uint8_t* //
make_brotli_enc_far_data(size_t* len) {
const size_t mib = 1 << 20;
const size_t n = (46 * mib) + 12345;
uint8_t* p = malloc(n);
if (!p) {
return NULL;
}
brotli_enc_fill_random(p, n, 0xFA12);
memcpy(p + (21 * mib) + 64, p + (1 * mib), 2 * mib);
memcpy(p + (44 * mib) - 4096, p + (4 * mib), 2 * mib);
*len = n;
return p;
}
const char* //
do_test_brotli_encode_large_window_far(bool mimic) {
size_t len = 0;
uint8_t* data = make_brotli_enc_far_data(&len);
if (!data) {
return "malloc failed";
}
static const struct {
uint32_t quality;
uint32_t lgwin;
uint32_t size_hint;
uint32_t chunking;
size_t num_repeats;
} configs[] = {
// H35.
{3, 25, 0, BROTLI_ENC_CHUNK_SINGLE, 1},
{3, 30, 0, BROTLI_ENC_CHUNK_RANDOM, 2},
// H55.
{4, 26, 1 << 30, BROTLI_ENC_CHUNK_SINGLE, 2},
// H65.
{5, 30, 1 << 30, BROTLI_ENC_CHUNK_RANDOM, 2},
{7, 28, 0, BROTLI_ENC_CHUNK_SINGLE, 2},
{9, 26, 0, BROTLI_ENC_CHUNK_SINGLE, 2},
};
const char* ret = NULL;
for (size_t i = 0; (i < WUFFS_TESTLIB_ARRAY_SIZE(configs)) && !ret; i++) {
brotli_enc_config c =
make_brotli_enc_config(configs[i].quality, configs[i].lgwin);
brotli_enc_config__add_param(&c, BROTLI_ENC_PARAM_LARGE_WINDOW, 1);
if (configs[i].size_hint) {
brotli_enc_config__add_param(&c, BROTLI_ENC_PARAM_SIZE_HINT,
configs[i].size_hint);
}
c.chunking = configs[i].chunking;
c.flush = (configs[i].chunking == BROTLI_ENC_CHUNK_RANDOM)
? BROTLI_ENC_FLUSH_MIDDLE
: BROTLI_ENC_FLUSH_NONE;
c.seed = i;
char s[256];
snprintf(s, sizeof s, "far data: q=%u lgwin=%u", configs[i].quality,
configs[i].lgwin);
// Pseudo-random data is incompressible, except for the repeats (of
// 2 MiB each), which must be found (mostly) to meet this limit.
size_t max_compressed_len = len - (configs[i].num_repeats * (3 << 19));
ret = do_test_brotli_encode_large_window(s, &c, data, len, mimic,
max_compressed_len);
}
free(data);
return ret;
}
const char* //
test_wuffs_brotli_encode_large_window_files() {
CHECK_FOCUS(__func__);
return do_test_brotli_encode_large_window_files(false);
}
const char* //
test_wuffs_brotli_encode_large_window_far() {
CHECK_FOCUS(__func__);
return do_test_brotli_encode_large_window_far(false);
}
#ifdef WUFFS_MIMIC
const char* //
test_mimic_brotli_encode_large_window_files() {
CHECK_FOCUS(__func__);
return do_test_brotli_encode_large_window_files(true);
}
const char* //
test_mimic_brotli_encode_large_window_far() {
CHECK_FOCUS(__func__);
return do_test_brotli_encode_large_window_far(true);
}
#endif // WUFFS_MIMIC
// ---------------- Brotli Benches
const char* //
@@ -3204,6 +3474,8 @@ proc g_tests[] = {
// Encoder tests.
test_wuffs_brotli_encode_interface,
test_wuffs_brotli_encode_large_window_far,
test_wuffs_brotli_encode_large_window_files,
test_wuffs_brotli_encode_metadata_round_trip,
test_wuffs_brotli_encode_parameters,
test_wuffs_brotli_encode_round_trip_corpus,
@@ -3212,6 +3484,8 @@ proc g_tests[] = {
#ifdef WUFFS_MIMIC
test_mimic_brotli_encode_large_window_far,
test_mimic_brotli_encode_large_window_files,
test_mimic_brotli_encode_parity_corpus,
test_mimic_brotli_encode_parity_grid,
test_mimic_brotli_encode_parity_metadata,