mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-18 08:32:11 +01:00
test/c/std/brotli: add large-window encoder tests
Check large-window encoding at qualities 3 to 9 (lgwin 18 to 30, with the H35, H55 and H65 composite hashers) against libbrotlienc, with round trips through Wuffs' decoder (with the large-window quirk) and libbrotlidec, over some small files and a 46 MiB input with repeats at distances of about 20 and 40 MiB. Also check the large-window workbuf_len and allow encoder workbufs of up to 3 GiB in the test driver.
This commit is contained in:
1 file changed
+284
-10
+284
-10
@@ -894,7 +894,9 @@ wuffs_brotli_encode_plan(wuffs_base__io_buffer* dst,
|
||||
wuffs_base__range_ii_u64 workbuf_len =
|
||||
wuffs_brotli__encoder__workbuf_len(enc);
|
||||
wuffs_base__slice_u8 workbuf = g_work_slice_u8;
|
||||
if (workbuf_len.max_incl > (1ull << 31)) {
|
||||
// Large-window brotli at lgwin 30 needs more than 2 GiB.
|
||||
if ((workbuf_len.max_incl > (3ull << 30)) ||
|
||||
(workbuf_len.max_incl > SIZE_MAX)) {
|
||||
ret = "workbuf_len is too large";
|
||||
goto cleanup;
|
||||
} else if (workbuf_len.max_incl > g_work_slice_u8.len) {
|
||||
@@ -1336,10 +1338,43 @@ test_wuffs_brotli_encode_parameters() {
|
||||
src = make_io_buffer_from_string("", 0);
|
||||
dst.meta.wi = 0;
|
||||
|
||||
// Qualities 10 and 11 (and 12, which sanitizes to 11) are unsupported, and
|
||||
// so is large-window brotli at qualities 3 and above.
|
||||
for (uint32_t q = 3; q <= 12; q++) {
|
||||
bool large_window = q < 10;
|
||||
// With the large-window quirk, lgwin can be up to 30 (instead of 24) at
|
||||
// qualities 3 and above. Like the C code, quality 2 and below ignore the
|
||||
// quirk. Above lgwin 24, the ring buffer is (2 << lgwin) bytes and, at
|
||||
// quality 3, the H35 hasher has a 64 MiB rolling hash table on top of
|
||||
// H3's 256 KiB.
|
||||
for (uint32_t q = 2; q <= 3; q++) {
|
||||
uint64_t wlens[3] = {0};
|
||||
for (int k = 0; k < 3; k++) {
|
||||
free(enc);
|
||||
enc = wuffs_brotli__encoder__alloc();
|
||||
if (!enc) {
|
||||
RETURN_FAIL("alloc failed");
|
||||
}
|
||||
wuffs_brotli__encoder__set_quirk(
|
||||
enc, WUFFS_BROTLI__QUIRK_ENCODER_QUALITY, q);
|
||||
wuffs_brotli__encoder__set_quirk(enc, WUFFS_BROTLI__QUIRK_LGWIN,
|
||||
(k == 0) ? 24 : 30);
|
||||
if (k > 0) {
|
||||
wuffs_brotli__encoder__set_quirk(
|
||||
enc, WUFFS_BROTLI__QUIRK_LARGE_WINDOW, (k == 2) ? 1 : 0);
|
||||
}
|
||||
wlens[k] = wuffs_brotli__encoder__workbuf_len(enc).max_incl;
|
||||
}
|
||||
BROTLI_ENC_CHECK(wlens[0] == wlens[1],
|
||||
"lgwin 30 without large window is lgwin 24");
|
||||
if (q == 2) {
|
||||
BROTLI_ENC_CHECK(wlens[0] == wlens[2],
|
||||
"large window is ignored at quality 2");
|
||||
} else {
|
||||
BROTLI_ENC_CHECK(
|
||||
wlens[2] == (wlens[0] + (2ull << 30) - (2ull << 24) + (64 << 20)),
|
||||
"workbuf_len at q3w30 with large window");
|
||||
}
|
||||
}
|
||||
|
||||
// Qualities 10 and 11 (and 12, which sanitizes to 11) are unsupported.
|
||||
for (uint32_t q = 10; q <= 12; q++) {
|
||||
free(enc);
|
||||
enc = wuffs_brotli__encoder__alloc();
|
||||
if (!enc) {
|
||||
@@ -1347,17 +1382,13 @@ test_wuffs_brotli_encode_parameters() {
|
||||
}
|
||||
wuffs_brotli__encoder__set_quirk(enc, WUFFS_BROTLI__QUIRK_ENCODER_QUALITY,
|
||||
q);
|
||||
if (large_window) {
|
||||
wuffs_brotli__encoder__set_quirk(enc, WUFFS_BROTLI__QUIRK_LARGE_WINDOW,
|
||||
1);
|
||||
}
|
||||
BROTLI_ENC_CHECK(wuffs_brotli__encoder__workbuf_len(enc).max_incl == 0,
|
||||
"workbuf_len for an unsupported quality");
|
||||
status = wuffs_brotli__encoder__transform_io(enc, &dst, &src,
|
||||
g_work_slice_u8);
|
||||
BROTLI_ENC_CHECK(
|
||||
status.repr == wuffs_brotli__error__unsupported_encoder_quality,
|
||||
large_window ? "large window at quality 3..9" : "quality 10..11");
|
||||
"quality 10..11");
|
||||
}
|
||||
|
||||
// Requests.
|
||||
@@ -3133,6 +3164,245 @@ test_mimic_brotli_decode_large_window_far() {
|
||||
|
||||
#endif // WUFFS_MIMIC
|
||||
|
||||
// ---------------- Brotli Large Window Encoder Tests
|
||||
|
||||
// do_test_brotli_encode_large_window encodes data[:data_len] with the plan's
|
||||
// config c (which should set the large-window quirk) and checks that
|
||||
// decoding the result (with Wuffs' std/brotli decoder with the large-window
|
||||
// quirk set, and, if mimic is true, with libbrotlidec) gives back the
|
||||
// original data. If mimic is true, it also checks that libbrotlienc's output
|
||||
// is the same. If max_compressed_len is non-zero, it checks that the
|
||||
// compressed length is at most that. The decoded data goes in g_want.
|
||||
const char* //
|
||||
do_test_brotli_encode_large_window(const char* name,
|
||||
const brotli_enc_config* c,
|
||||
const uint8_t* data,
|
||||
size_t data_len,
|
||||
bool mimic,
|
||||
size_t max_compressed_len) {
|
||||
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
|
||||
.data = g_src_slice_u8,
|
||||
});
|
||||
CHECK_STRING(make_brotli_enc_plan(&src, c, data, data_len));
|
||||
wuffs_base__io_buffer have = ((wuffs_base__io_buffer){
|
||||
.data = g_have_slice_u8,
|
||||
});
|
||||
wuffs_base__io_buffer src0 = src;
|
||||
const char* status = wuffs_brotli_encode_plan(
|
||||
&have, &src0, c->param_keys, c->param_values, c->num_params,
|
||||
g_brotli_enc_step_ops, g_brotli_enc_step_lens, g_brotli_enc_num_steps,
|
||||
c->dst_chunk);
|
||||
if (status) {
|
||||
RETURN_FAIL("%s: wuffs encode: %s", name, status);
|
||||
} else if (max_compressed_len && (have.meta.wi > max_compressed_len)) {
|
||||
RETURN_FAIL("%s: compressed length: have %zu, want at most %zu", name,
|
||||
have.meta.wi, max_compressed_len);
|
||||
}
|
||||
have.meta.closed = true;
|
||||
|
||||
for (int decoder = 0; decoder < (mimic ? 3 : 1); decoder++) {
|
||||
wuffs_base__io_buffer want = ((wuffs_base__io_buffer){
|
||||
.data = g_want_slice_u8,
|
||||
});
|
||||
wuffs_base__io_buffer have1 = have;
|
||||
if (decoder == 0) {
|
||||
status = wuffs_brotli_decode_with_options(
|
||||
&want, &have1, UINT64_MAX, UINT64_MAX, true, BROTLI_DICT_SET_NONE);
|
||||
#ifdef WUFFS_MIMIC
|
||||
} else if (decoder == 1) {
|
||||
status = mimic_brotli_decode_large_window(
|
||||
&want, &have1, UINT64_MAX, UINT64_MAX, true, BROTLI_DICT_SET_NONE);
|
||||
} else {
|
||||
wuffs_base__io_buffer src1 = src;
|
||||
status = mimic_brotli_encode(
|
||||
&want, &src1, c->param_keys, c->param_values, c->num_params,
|
||||
g_brotli_enc_step_ops, g_brotli_enc_step_lens,
|
||||
g_brotli_enc_num_steps, c->dst_chunk);
|
||||
if (status) {
|
||||
RETURN_FAIL("%s: mimic encode: %s", name, status);
|
||||
}
|
||||
status = check_io_buffers_equal("", &have, &want);
|
||||
if (status) {
|
||||
RETURN_FAIL("%s: %s", name, status);
|
||||
}
|
||||
continue;
|
||||
#endif // WUFFS_MIMIC
|
||||
}
|
||||
const char* which = (decoder == 0) ? "wuffs" : "mimic";
|
||||
if (status) {
|
||||
RETURN_FAIL("%s: %s decode: %s", name, which, status);
|
||||
} else if (have1.meta.ri != have1.meta.wi) {
|
||||
RETURN_FAIL("%s: %s decode: trailing data", name, which);
|
||||
} else if (want.meta.wi != data_len) {
|
||||
RETURN_FAIL("%s: %s decoded length: have %zu, want %zu", name, which,
|
||||
want.meta.wi, data_len);
|
||||
} else if (memcmp(want.data.ptr, data, data_len)) {
|
||||
RETURN_FAIL("%s: %s decoded bytes differ", name, which);
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
// do_test_brotli_encode_large_window_files runs a grid of large-window
|
||||
// configurations (qualities 3 to 9, lgwin 18 to 30) over a few small files.
|
||||
const char* //
|
||||
do_test_brotli_encode_large_window_files(bool mimic) {
|
||||
static const char* filenames[] = {
|
||||
"test/data/romeo.txt",
|
||||
"test/data/brotli/alice29.txt",
|
||||
"test/data/brotli/random_org_10k.bin",
|
||||
"test/data/brotli/quickfox_repeated",
|
||||
};
|
||||
static const uint32_t lgwins[] = {18, 25, 26, 28, 30};
|
||||
wuffs_base__io_buffer data = ((wuffs_base__io_buffer){
|
||||
.data = g_work_slice_u8,
|
||||
});
|
||||
for (size_t f = 0; f < WUFFS_TESTLIB_ARRAY_SIZE(filenames); f++) {
|
||||
// g_work is free to use for the input: each config's workbuf (of more
|
||||
// than 64 MiB) is malloc'ed.
|
||||
data.meta.wi = 0;
|
||||
data.meta.ri = 0;
|
||||
data.meta.closed = false;
|
||||
CHECK_STRING(read_file(&data, filenames[f]));
|
||||
uint8_t* copy = malloc(data.meta.wi ? data.meta.wi : 1);
|
||||
if (!copy) {
|
||||
return "malloc failed";
|
||||
}
|
||||
memcpy(copy, data.data.ptr, data.meta.wi);
|
||||
const char* ret = NULL;
|
||||
for (uint32_t q = 3; (q <= 9) && !ret; q++) {
|
||||
for (size_t w = 0; (w < 5) && !ret; w++) {
|
||||
for (int k = 0; (k < 2) && !ret; k++) {
|
||||
if ((k == 1) && (lgwins[w] != 26)) {
|
||||
continue;
|
||||
}
|
||||
brotli_enc_config c = make_brotli_enc_config(q, lgwins[w]);
|
||||
brotli_enc_config__add_param(&c, BROTLI_ENC_PARAM_LARGE_WINDOW, 1);
|
||||
if (k == 1) {
|
||||
// A big size hint selects H55 (at quality 4) and H65 (at
|
||||
// qualities 5 to 9).
|
||||
brotli_enc_config__add_param(&c, BROTLI_ENC_PARAM_SIZE_HINT,
|
||||
1 << 20);
|
||||
}
|
||||
c.chunking = (w & 1) ? BROTLI_ENC_CHUNK_SINGLE
|
||||
: BROTLI_ENC_CHUNK_RANDOM;
|
||||
c.flush = (w & 1) ? BROTLI_ENC_FLUSH_NONE : BROTLI_ENC_FLUSH_MIDDLE;
|
||||
c.seed = (q << 8) | w;
|
||||
char s[256];
|
||||
snprintf(s, sizeof s, "%s: q=%u lgwin=%u size_hint=%s",
|
||||
filenames[f], q, lgwins[w], k ? "1<<20" : "0");
|
||||
ret = do_test_brotli_encode_large_window(s, &c, copy, data.meta.wi,
|
||||
mimic, 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
free(copy);
|
||||
if (ret) {
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
// make_brotli_enc_far_data returns a malloc'ed buffer of pseudo-random bytes
|
||||
// with two 2 MiB repeats at distances of (20 MiB + 64) and (40 MiB - 4096).
|
||||
// The repeats' sources are not repeated elsewhere, so only an encoder with a
|
||||
// large enough window (lgwin 25 or more for the first, 26 or more for the
|
||||
// second) can compress it.
|
||||
uint8_t* //
|
||||
make_brotli_enc_far_data(size_t* len) {
|
||||
const size_t mib = 1 << 20;
|
||||
const size_t n = (46 * mib) + 12345;
|
||||
uint8_t* p = malloc(n);
|
||||
if (!p) {
|
||||
return NULL;
|
||||
}
|
||||
brotli_enc_fill_random(p, n, 0xFA12);
|
||||
memcpy(p + (21 * mib) + 64, p + (1 * mib), 2 * mib);
|
||||
memcpy(p + (44 * mib) - 4096, p + (4 * mib), 2 * mib);
|
||||
*len = n;
|
||||
return p;
|
||||
}
|
||||
|
||||
const char* //
|
||||
do_test_brotli_encode_large_window_far(bool mimic) {
|
||||
size_t len = 0;
|
||||
uint8_t* data = make_brotli_enc_far_data(&len);
|
||||
if (!data) {
|
||||
return "malloc failed";
|
||||
}
|
||||
static const struct {
|
||||
uint32_t quality;
|
||||
uint32_t lgwin;
|
||||
uint32_t size_hint;
|
||||
uint32_t chunking;
|
||||
size_t num_repeats;
|
||||
} configs[] = {
|
||||
// H35.
|
||||
{3, 25, 0, BROTLI_ENC_CHUNK_SINGLE, 1},
|
||||
{3, 30, 0, BROTLI_ENC_CHUNK_RANDOM, 2},
|
||||
// H55.
|
||||
{4, 26, 1 << 30, BROTLI_ENC_CHUNK_SINGLE, 2},
|
||||
// H65.
|
||||
{5, 30, 1 << 30, BROTLI_ENC_CHUNK_RANDOM, 2},
|
||||
{7, 28, 0, BROTLI_ENC_CHUNK_SINGLE, 2},
|
||||
{9, 26, 0, BROTLI_ENC_CHUNK_SINGLE, 2},
|
||||
};
|
||||
const char* ret = NULL;
|
||||
for (size_t i = 0; (i < WUFFS_TESTLIB_ARRAY_SIZE(configs)) && !ret; i++) {
|
||||
brotli_enc_config c =
|
||||
make_brotli_enc_config(configs[i].quality, configs[i].lgwin);
|
||||
brotli_enc_config__add_param(&c, BROTLI_ENC_PARAM_LARGE_WINDOW, 1);
|
||||
if (configs[i].size_hint) {
|
||||
brotli_enc_config__add_param(&c, BROTLI_ENC_PARAM_SIZE_HINT,
|
||||
configs[i].size_hint);
|
||||
}
|
||||
c.chunking = configs[i].chunking;
|
||||
c.flush = (configs[i].chunking == BROTLI_ENC_CHUNK_RANDOM)
|
||||
? BROTLI_ENC_FLUSH_MIDDLE
|
||||
: BROTLI_ENC_FLUSH_NONE;
|
||||
c.seed = i;
|
||||
char s[256];
|
||||
snprintf(s, sizeof s, "far data: q=%u lgwin=%u", configs[i].quality,
|
||||
configs[i].lgwin);
|
||||
// Pseudo-random data is incompressible, except for the repeats (of
|
||||
// 2 MiB each), which must be found (mostly) to meet this limit.
|
||||
size_t max_compressed_len = len - (configs[i].num_repeats * (3 << 19));
|
||||
ret = do_test_brotli_encode_large_window(s, &c, data, len, mimic,
|
||||
max_compressed_len);
|
||||
}
|
||||
free(data);
|
||||
return ret;
|
||||
}
|
||||
|
||||
const char* //
|
||||
test_wuffs_brotli_encode_large_window_files() {
|
||||
CHECK_FOCUS(__func__);
|
||||
return do_test_brotli_encode_large_window_files(false);
|
||||
}
|
||||
|
||||
const char* //
|
||||
test_wuffs_brotli_encode_large_window_far() {
|
||||
CHECK_FOCUS(__func__);
|
||||
return do_test_brotli_encode_large_window_far(false);
|
||||
}
|
||||
|
||||
#ifdef WUFFS_MIMIC
|
||||
|
||||
const char* //
|
||||
test_mimic_brotli_encode_large_window_files() {
|
||||
CHECK_FOCUS(__func__);
|
||||
return do_test_brotli_encode_large_window_files(true);
|
||||
}
|
||||
|
||||
const char* //
|
||||
test_mimic_brotli_encode_large_window_far() {
|
||||
CHECK_FOCUS(__func__);
|
||||
return do_test_brotli_encode_large_window_far(true);
|
||||
}
|
||||
|
||||
#endif // WUFFS_MIMIC
|
||||
|
||||
// ---------------- Brotli Benches
|
||||
|
||||
const char* //
|
||||
@@ -3204,6 +3474,8 @@ proc g_tests[] = {
|
||||
// Encoder tests.
|
||||
|
||||
test_wuffs_brotli_encode_interface,
|
||||
test_wuffs_brotli_encode_large_window_far,
|
||||
test_wuffs_brotli_encode_large_window_files,
|
||||
test_wuffs_brotli_encode_metadata_round_trip,
|
||||
test_wuffs_brotli_encode_parameters,
|
||||
test_wuffs_brotli_encode_round_trip_corpus,
|
||||
@@ -3212,6 +3484,8 @@ proc g_tests[] = {
|
||||
|
||||
#ifdef WUFFS_MIMIC
|
||||
|
||||
test_mimic_brotli_encode_large_window_far,
|
||||
test_mimic_brotli_encode_large_window_files,
|
||||
test_mimic_brotli_encode_parity_corpus,
|
||||
test_mimic_brotli_encode_parity_grid,
|
||||
test_mimic_brotli_encode_parity_metadata,
|
||||
|
||||
Reference in new issue
Block a user