Files
Claude bc1b5fcd49 std/brotli: add QUIRK_REPORT_METADATA
The decoder can now report metadata blocks, like libbrotlidec's
BrotliDecoderSetMetadataCallbacks: transform_io returns
"@metadata reported", metadata_length gives the block's length and the
block's contents are written to dst. The differential fuzzer compares
the reported metadata with libbrotlidec's callbacks.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UZ978Fjdsyig4v3wYAZRg5
2026-10-27 17:06:08 +00:00

5102 lines
191 KiB
C

// Copyright 2026 The Wuffs Authors.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// SPDX-License-Identifier: Apache-2.0 OR MIT
// ----------------
/*
This test program is typically run indirectly, by the "wuffs test" or "wuffs
bench" commands. These commands take an optional "-mimic" flag to check that
Wuffs' output mimics (i.e. exactly matches) other libraries' output, such as
giflib for GIF, libpng for PNG, etc.
To manually run this test:
for CC in clang gcc; do
$CC -std=c99 -Wall -Werror brotli.c && ./a.out
rm -f a.out
done
Each edition should print "PASS", amongst other information, and exit(0).
Add the "wuffs mimic cflags" (everything after the colon below) to the C
compiler flags (after the .c file) to run the mimic tests.
To manually run the benchmarks, replace "-Wall -Werror" with "-O3" and replace
the first "./a.out" with "./a.out -bench". Combine these changes with the
"wuffs mimic cflags" to run the mimic benchmarks.
*/
// ¿ wuffs mimic cflags: -DWUFFS_MIMIC -lbrotlienc -lbrotlidec -lbrotlicommon -lm
// Wuffs ships as a "single file C library" or "header file library" as per
// https://github.com/nothings/stb/blob/master/docs/stb_howto.txt
//
// To use that single file as a "foo.c"-like implementation, instead of a
// "foo.h"-like header, #define WUFFS_IMPLEMENTATION before #include'ing or
// compiling it.
#define WUFFS_IMPLEMENTATION
// Defining the WUFFS_CONFIG__MODULE* macros are optional, but it lets users of
// release/c/etc.c choose which parts of Wuffs to build. That file contains the
// entire Wuffs standard library, implementing a variety of codecs and file
// formats. Without this macro definition, an optimizing compiler or linker may
// very well discard Wuffs code for unused codecs, but listing the Wuffs
// modules we use makes that process explicit. Preprocessing means that such
// code simply isn't compiled.
#define WUFFS_CONFIG__MODULES
#define WUFFS_CONFIG__MODULE__BASE
#define WUFFS_CONFIG__MODULE__BROTLI
#define WUFFS_CONFIG__MODULE__SOFTFLOAT
// If building this program in an environment that doesn't easily accommodate
// relative includes, you can use the script/inline-c-relative-includes.go
// program to generate a stand-alone C file.
#include "../../../release/c/wuffs-unsupported-snapshot.c"
#include "../testlib/testlib.c"
#ifdef WUFFS_MIMIC
#include "../mimiclib/brotli.c"
#endif
// ---------------- Golden Tests
golden_test g_brotli_alice29_gt = {
.want_filename = "test/data/brotli/alice29.txt",
.src_filename = "test/data/brotli/alice29.txt.compressed",
};
golden_test g_brotli_alice29_q11w10_gt = {
.want_filename = "test/data/brotli/alice29.txt",
.src_filename = "test/data/brotli/alice29.txt.q11w10.br",
};
golden_test g_brotli_romeo_gt = {
.want_filename = "test/data/romeo.txt",
.src_filename = "test/data/brotli/romeo.txt.q11w22.br",
};
// g_brotli_vectors lists every (compressed, decompressed) pair of test files.
// See test/data/brotli/README.md for where they came from.
struct {
const char* src_filename;
const char* want_filename;
} g_brotli_vectors[] = {
{"test/data/brotli/10x10y.compressed", "test/data/brotli/10x10y"},
{"test/data/brotli/256.bytes.q0w10.br", "test/data/256.bytes"},
{"test/data/brotli/256.bytes.q0w22.br", "test/data/256.bytes"},
{"test/data/brotli/256.bytes.q0w24.br", "test/data/256.bytes"},
{"test/data/brotli/256.bytes.q11w10.br", "test/data/256.bytes"},
{"test/data/brotli/256.bytes.q11w22.br", "test/data/256.bytes"},
{"test/data/brotli/256.bytes.q11w24.br", "test/data/256.bytes"},
{"test/data/brotli/256.bytes.q5w10.br", "test/data/256.bytes"},
{"test/data/brotli/256.bytes.q5w22.br", "test/data/256.bytes"},
{"test/data/brotli/256.bytes.q5w24.br", "test/data/256.bytes"},
{"test/data/brotli/64x.compressed", "test/data/brotli/64x"},
{"test/data/brotli/alice29.txt.compressed", "test/data/brotli/alice29.txt"},
{"test/data/brotli/alice29.txt.q11w10.br", "test/data/brotli/alice29.txt"},
{"test/data/brotli/alice29.txt.q1w10.br", "test/data/brotli/alice29.txt"},
{"test/data/brotli/alice29.txt.q5w10.br", "test/data/brotli/alice29.txt"},
{"test/data/brotli/alice29.txt.q9w16.br", "test/data/brotli/alice29.txt"},
{"test/data/brotli/asyoulik.txt.compressed",
"test/data/brotli/asyoulik.txt"},
{"test/data/brotli/backward65536.compressed",
"test/data/brotli/backward65536"},
{"test/data/brotli/compressed_file.compressed",
"test/data/brotli/compressed_file"},
{"test/data/brotli/compressed_repeated.compressed",
"test/data/brotli/compressed_repeated"},
{"test/data/brotli/cp1251-utf16le.compressed",
"test/data/brotli/cp1251-utf16le"},
{"test/data/brotli/cp852-utf8.compressed", "test/data/brotli/cp852-utf8"},
{"test/data/brotli/empty.compressed", "test/data/brotli/empty"},
{"test/data/brotli/empty.compressed.00", "test/data/brotli/empty"},
{"test/data/brotli/empty.compressed.01", "test/data/brotli/empty"},
{"test/data/brotli/empty.compressed.02", "test/data/brotli/empty"},
{"test/data/brotli/empty.compressed.03", "test/data/brotli/empty"},
{"test/data/brotli/empty.compressed.04", "test/data/brotli/empty"},
{"test/data/brotli/empty.compressed.05", "test/data/brotli/empty"},
{"test/data/brotli/empty.compressed.06", "test/data/brotli/empty"},
{"test/data/brotli/empty.compressed.07", "test/data/brotli/empty"},
{"test/data/brotli/empty.compressed.08", "test/data/brotli/empty"},
{"test/data/brotli/empty.compressed.09", "test/data/brotli/empty"},
{"test/data/brotli/empty.compressed.10", "test/data/brotli/empty"},
{"test/data/brotli/empty.compressed.11", "test/data/brotli/empty"},
{"test/data/brotli/empty.compressed.12", "test/data/brotli/empty"},
{"test/data/brotli/empty.compressed.13", "test/data/brotli/empty"},
{"test/data/brotli/empty.compressed.14", "test/data/brotli/empty"},
{"test/data/brotli/empty.compressed.15", "test/data/brotli/empty"},
{"test/data/brotli/empty.compressed.16", "test/data/brotli/empty"},
{"test/data/brotli/empty.compressed.17", "test/data/brotli/empty"},
{"test/data/brotli/empty.compressed.18", "test/data/brotli/empty"},
{"test/data/brotli/monkey.compressed", "test/data/brotli/monkey"},
{"test/data/brotli/quickfox.compressed", "test/data/brotli/quickfox"},
{"test/data/brotli/quickfox_repeated.compressed",
"test/data/brotli/quickfox_repeated"},
{"test/data/brotli/random_org_10k.bin.compressed",
"test/data/brotli/random_org_10k.bin"},
{"test/data/brotli/romeo.txt.q0w10.br", "test/data/romeo.txt"},
{"test/data/brotli/romeo.txt.q0w22.br", "test/data/romeo.txt"},
{"test/data/brotli/romeo.txt.q0w24.br", "test/data/romeo.txt"},
{"test/data/brotli/romeo.txt.q11w10.br", "test/data/romeo.txt"},
{"test/data/brotli/romeo.txt.q11w22.br", "test/data/romeo.txt"},
{"test/data/brotli/romeo.txt.q11w24.br", "test/data/romeo.txt"},
{"test/data/brotli/romeo.txt.q5w10.br", "test/data/romeo.txt"},
{"test/data/brotli/romeo.txt.q5w22.br", "test/data/romeo.txt"},
{"test/data/brotli/romeo.txt.q5w24.br", "test/data/romeo.txt"},
{"test/data/brotli/ukkonooa.compressed", "test/data/brotli/ukkonooa"},
{"test/data/brotli/x.compressed", "test/data/brotli/x"},
{"test/data/brotli/x.compressed.00", "test/data/brotli/x"},
{"test/data/brotli/x.compressed.01", "test/data/brotli/x"},
{"test/data/brotli/x.compressed.02", "test/data/brotli/x"},
{"test/data/brotli/x.compressed.03", "test/data/brotli/x"},
{"test/data/brotli/xyzzy.compressed", "test/data/brotli/xyzzy"},
{"test/data/brotli/zeros.compressed", "test/data/brotli/zeros"},
};
// g_brotli_malformed lists invalid (or truncated) inputs. Each want_status is
// the Wuffs status that the C code's BrotliDecoderErrorCode (in the comment)
// maps to. See std/brotli/README.md for that mapping. Each input is either a
// file (if src_filename is non-NULL) or inline data.
//
// The test/data/brotli/invalid.*.br files were found by mutating valid
// test/data/brotli files and keeping the smallest input that libbrotlidec
// v1.2.1 rejects with each error code.
//
// BROTLI_DECODER_ERROR_FORMAT_BLOCK_SWITCH is not covered: triggering it needs
// more than (1 << 24) commands in one metablock.
struct {
const char* src_filename;
const char* data;
size_t len;
const char* want_status;
} g_brotli_malformed[] = {
// BROTLI_DECODER_ERROR_FORMAT_EXUBERANT_NIBBLE.
{"test/data/brotli/invalid.exuberant-nibble.br", NULL, 0,
wuffs_brotli__error__bad_exuberant_nibble},
// BROTLI_DECODER_ERROR_FORMAT_RESERVED: a metadata block's reserved bit.
// The inline data's bits (LSB first) are: WBITS=0 (16), ISLAST=0,
// MNIBBLES=11 (metadata) and a reserved bit of 1.
{NULL, "\x1C", 1, wuffs_brotli__error__bad_reserved_bit},
{"test/data/brotli/invalid.reserved.br", NULL, 0,
wuffs_brotli__error__bad_reserved_bit},
// BROTLI_DECODER_ERROR_FORMAT_EXUBERANT_META_NIBBLE.
{"test/data/brotli/invalid.exuberant-meta-nibble.br", NULL, 0,
wuffs_brotli__error__bad_exuberant_metadata_nibble},
// BROTLI_DECODER_ERROR_FORMAT_SIMPLE_HUFFMAN_ALPHABET.
{"test/data/brotli/invalid.simple-huffman-alphabet.br", NULL, 0,
wuffs_brotli__error__bad_simple_huffman_alphabet},
// BROTLI_DECODER_ERROR_FORMAT_SIMPLE_HUFFMAN_SAME.
{"test/data/brotli/invalid.simple-huffman-same.br", NULL, 0,
wuffs_brotli__error__bad_simple_huffman_symbol},
// BROTLI_DECODER_ERROR_FORMAT_CL_SPACE.
{"test/data/brotli/invalid.cl-space.br", NULL, 0,
wuffs_brotli__error__bad_code_length_code_lengths},
// BROTLI_DECODER_ERROR_FORMAT_HUFFMAN_SPACE.
{"test/data/brotli/invalid.huffman-space.br", NULL, 0,
wuffs_brotli__error__bad_huffman_space},
// BROTLI_DECODER_ERROR_FORMAT_CONTEXT_MAP_REPEAT.
{"test/data/brotli/invalid.context-map-repeat.br", NULL, 0,
wuffs_brotli__error__bad_context_map_repeat},
// BROTLI_DECODER_ERROR_FORMAT_BLOCK_LENGTH_1.
{"test/data/brotli/invalid.block-length-1.br", NULL, 0,
wuffs_brotli__error__bad_block_length},
// BROTLI_DECODER_ERROR_FORMAT_BLOCK_LENGTH_2.
{"test/data/brotli/invalid.block-length-2.br", NULL, 0,
wuffs_brotli__error__bad_metablock_length},
// BROTLI_DECODER_ERROR_FORMAT_TRANSFORM.
{"test/data/brotli/invalid.transform.br", NULL, 0,
wuffs_brotli__error__bad_transform},
// BROTLI_DECODER_ERROR_FORMAT_DICTIONARY.
{"test/data/brotli/invalid.dictionary.br", NULL, 0,
wuffs_brotli__error__bad_dictionary},
// BROTLI_DECODER_ERROR_FORMAT_WINDOW_BITS: the large-window escape
// (WBITS = 0b0010001) is invalid unless large-window brotli is enabled.
{NULL, "\x11", 1, wuffs_brotli__error__unsupported_large_window},
// BROTLI_DECODER_ERROR_FORMAT_PADDING_1: an empty metadata block followed
// by a non-zero padding bit.
{NULL, "\x8C\x00", 2, wuffs_brotli__error__bad_padding},
// BROTLI_DECODER_ERROR_FORMAT_PADDING_2: an ISLAST and ISLASTEMPTY
// metablock followed by a non-zero padding bit.
{NULL, "\x0E", 1, wuffs_brotli__error__bad_final_padding},
// BROTLI_DECODER_ERROR_FORMAT_DISTANCE: a distance that is too far back.
{"test/data/brotli/invalid.distance.br", NULL, 0,
wuffs_brotli__error__bad_distance},
// BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT: an empty input.
{NULL, "", 0, wuffs_brotli__error__truncated_input},
// BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT: a truncated input (the first 20
// bytes of romeo.txt.q11w22.br).
{"test/data/brotli/invalid.truncated.br", NULL, 0,
wuffs_brotli__error__truncated_input},
};
const char* //
read_brotli_malformed(wuffs_base__io_buffer* src, size_t i) {
if (g_brotli_malformed[i].src_filename) {
return read_file(src, g_brotli_malformed[i].src_filename);
} else if (g_brotli_malformed[i].len > src->data.len) {
return "read_brotli_malformed: data is too long";
}
if (g_brotli_malformed[i].len > 0) {
memcpy(src->data.ptr, g_brotli_malformed[i].data,
g_brotli_malformed[i].len);
}
src->meta.wi = g_brotli_malformed[i].len;
src->meta.ri = 0;
src->meta.closed = true;
return NULL;
}
// ---------------- Brotli Tests
const char* //
test_wuffs_brotli_decode_interface() {
CHECK_FOCUS(__func__);
wuffs_brotli__decoder dec;
CHECK_STATUS("initialize",
wuffs_brotli__decoder__initialize(
&dec, sizeof dec, WUFFS_VERSION,
WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED));
return do_test__wuffs_base__io_transformer(
wuffs_brotli__decoder__upcast_as__wuffs_base__io_transformer(&dec),
"test/data/brotli/romeo.txt.q5w22.br", 0, SIZE_MAX, 942, 0x0A);
}
const char* //
wuffs_brotli_decode(wuffs_base__io_buffer* dst,
wuffs_base__io_buffer* src,
uint32_t wuffs_initialize_flags,
uint64_t wlimit,
uint64_t rlimit) {
wuffs_brotli__decoder dec;
CHECK_STATUS("initialize",
wuffs_brotli__decoder__initialize(
&dec, sizeof dec, WUFFS_VERSION, wuffs_initialize_flags));
while (true) {
wuffs_base__io_buffer limited_dst = make_limited_writer(*dst, wlimit);
wuffs_base__io_buffer limited_src = make_limited_reader(*src, rlimit);
wuffs_base__status status = wuffs_brotli__decoder__transform_io(
&dec, &limited_dst, &limited_src, g_work_slice_u8);
dst->meta.wi += limited_dst.meta.wi;
src->meta.ri += limited_src.meta.ri;
if (((wlimit < UINT64_MAX) &&
(status.repr == wuffs_base__suspension__short_write)) ||
((rlimit < UINT64_MAX) &&
(status.repr == wuffs_base__suspension__short_read))) {
continue;
}
return status.repr;
}
}
const char* //
do_test_brotli_decode_all(const char* (*codec_func)(wuffs_base__io_buffer*,
wuffs_base__io_buffer*,
uint32_t,
uint64_t,
uint64_t),
uint64_t wlimit,
uint64_t rlimit) {
size_t n = WUFFS_TESTLIB_ARRAY_SIZE(g_brotli_vectors);
for (size_t i = 0; i < n; i++) {
golden_test gt = ((golden_test){
.want_filename = g_brotli_vectors[i].want_filename,
.src_filename = g_brotli_vectors[i].src_filename,
});
const char* status = do_test_io_buffers(codec_func, &gt, wlimit, rlimit);
if (status) {
RETURN_FAIL("%s: %s", gt.src_filename, status);
}
}
return NULL;
}
const char* //
test_wuffs_brotli_decode_all() {
CHECK_FOCUS(__func__);
return do_test_brotli_decode_all(wuffs_brotli_decode, UINT64_MAX, UINT64_MAX);
}
const char* //
test_wuffs_brotli_decode_all_with_io_limits() {
CHECK_FOCUS(__func__);
return do_test_brotli_decode_all(wuffs_brotli_decode, 1, 1);
}
const char* //
test_wuffs_brotli_decode_alice29() {
CHECK_FOCUS(__func__);
return do_test_io_buffers(wuffs_brotli_decode, &g_brotli_alice29_gt,
UINT64_MAX, UINT64_MAX);
}
const char* //
test_wuffs_brotli_decode_alice29_q11w10() {
CHECK_FOCUS(__func__);
return do_test_io_buffers(wuffs_brotli_decode, &g_brotli_alice29_q11w10_gt,
UINT64_MAX, UINT64_MAX);
}
const char* //
test_wuffs_brotli_decode_romeo() {
CHECK_FOCUS(__func__);
return do_test_io_buffers(wuffs_brotli_decode, &g_brotli_romeo_gt, UINT64_MAX,
UINT64_MAX);
}
const char* //
test_wuffs_brotli_decode_malformed() {
CHECK_FOCUS(__func__);
size_t n = WUFFS_TESTLIB_ARRAY_SIZE(g_brotli_malformed);
for (size_t i = 0; i < n; i++) {
for (int j = 0; j < 2; j++) {
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
.data = g_src_slice_u8,
});
wuffs_base__io_buffer have = ((wuffs_base__io_buffer){
.data = g_have_slice_u8,
});
CHECK_STRING(read_brotli_malformed(&src, i));
uint64_t limit = (j == 0) ? UINT64_MAX : 1;
const char* have_status = wuffs_brotli_decode(
&have, &src, WUFFS_INITIALIZE__DEFAULT_OPTIONS, limit, limit);
// With an I/O limit, running out of (limited) input can make the C
// code's WriteRingBuffer reject an overrun metablock earlier, as
// BLOCK_LENGTH_1 instead of BLOCK_LENGTH_2.
const char* want_status = g_brotli_malformed[i].want_status;
if ((j == 1) &&
(want_status == wuffs_brotli__error__bad_metablock_length) &&
(have_status == wuffs_brotli__error__bad_block_length)) {
continue;
}
if (have_status != want_status) {
RETURN_FAIL("i=%zu, j=%d: have \"%s\", want \"%s\"", i, j, have_status,
want_status);
}
}
}
return NULL;
}
const char* //
test_wuffs_brotli_decode_truncated() {
CHECK_FOCUS(__func__);
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
.data = g_src_slice_u8,
});
wuffs_base__io_buffer have = ((wuffs_base__io_buffer){
.data = g_have_slice_u8,
});
CHECK_STRING(read_file(&src, "test/data/brotli/romeo.txt.q11w22.br"));
if (src.meta.wi < 2) {
RETURN_FAIL("source file was too short");
}
for (size_t n = 0; n < src.meta.wi; n++) {
wuffs_base__io_buffer s = src;
s.meta.wi = n;
s.meta.closed = true;
have.meta.wi = 0;
const char* have_status = wuffs_brotli_decode(
&have, &s, WUFFS_INITIALIZE__DEFAULT_OPTIONS, UINT64_MAX, UINT64_MAX);
if (have_status != wuffs_brotli__error__truncated_input) {
RETURN_FAIL("n=%zu: have \"%s\", want \"%s\"", n, have_status,
wuffs_brotli__error__truncated_input);
}
}
return NULL;
}
const char* //
test_wuffs_brotli_decode_trailing_data() {
CHECK_FOCUS(__func__);
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
.data = g_src_slice_u8,
});
wuffs_base__io_buffer have = ((wuffs_base__io_buffer){
.data = g_have_slice_u8,
});
wuffs_base__io_buffer want = ((wuffs_base__io_buffer){
.data = g_want_slice_u8,
});
CHECK_STRING(read_file(&src, "test/data/brotli/romeo.txt.q5w22.br"));
CHECK_STRING(read_file(&want, "test/data/romeo.txt"));
size_t n = src.meta.wi;
if ((n + 3) > src.data.len) {
RETURN_FAIL("source file was too long");
}
// Append some trailing data that the decoder should not read.
memcpy(src.data.ptr + n, "XYZ", 3);
src.meta.wi += 3;
for (int i = 0; i < 2; i++) {
uint64_t limit = (i == 0) ? UINT64_MAX : 1;
src.meta.ri = 0;
have.meta.wi = 0;
CHECK_STRING(wuffs_brotli_decode(
&have, &src, WUFFS_INITIALIZE__DEFAULT_OPTIONS, limit, limit));
CHECK_STRING(check_io_buffers_equal("", &have, &want));
if (src.meta.ri != n) {
RETURN_FAIL("i=%d: ri: have %zu, want %zu", i, src.meta.ri, n);
}
}
return NULL;
}
const char* //
test_wuffs_brotli_decode_workbuf_length() {
CHECK_FOCUS(__func__);
// romeo.txt.q5w22.br has a (1 << 22) byte window. The workbuf also needs
// 542 bytes of slack.
for (int i = 0; i < 3; i++) {
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
.data = g_src_slice_u8,
});
wuffs_base__io_buffer have = ((wuffs_base__io_buffer){
.data = g_have_slice_u8,
});
CHECK_STRING(read_file(&src, "test/data/brotli/romeo.txt.q5w22.br"));
wuffs_brotli__decoder dec;
CHECK_STATUS("initialize", wuffs_brotli__decoder__initialize(
&dec, sizeof dec, WUFFS_VERSION,
WUFFS_INITIALIZE__DEFAULT_OPTIONS));
wuffs_base__slice_u8 workbuf = g_work_slice_u8;
const char* want = NULL;
if (i == 0) {
workbuf.len = 1000;
want = wuffs_base__error__bad_workbuf_length;
} else if (i == 1) {
workbuf.len = (1 << 22) + 541;
want = wuffs_base__error__bad_workbuf_length;
} else {
workbuf.len = (1 << 22) + 542;
}
wuffs_base__status status =
wuffs_brotli__decoder__transform_io(&dec, &have, &src, workbuf);
if (status.repr != want) {
RETURN_FAIL("i=%d: have \"%s\", want \"%s\"", i, status.repr, want);
}
}
return NULL;
}
// ---------------- Mimic Tests
#ifdef WUFFS_MIMIC
const char* //
test_mimic_brotli_decode_all() {
CHECK_FOCUS(__func__);
return do_test_brotli_decode_all(mimic_brotli_decode, UINT64_MAX, UINT64_MAX);
}
const char* //
test_mimic_brotli_decode_alice29() {
CHECK_FOCUS(__func__);
return do_test_io_buffers(mimic_brotli_decode, &g_brotli_alice29_gt,
UINT64_MAX, UINT64_MAX);
}
const char* //
test_mimic_brotli_decode_romeo() {
CHECK_FOCUS(__func__);
return do_test_io_buffers(mimic_brotli_decode, &g_brotli_romeo_gt, UINT64_MAX,
UINT64_MAX);
}
// brotli_c_error_code returns the libbrotlidec BrotliDecoderErrorCode that
// corresponds to a Wuffs status. A truncated input maps to
// BROTLI_DECODER_NEEDS_MORE_INPUT (2).
int //
brotli_c_error_code(const char* status) {
static const struct {
const char* status;
int c_error_code;
} table[] = {
{wuffs_brotli__error__bad_exuberant_nibble, -1},
{wuffs_brotli__error__bad_reserved_bit, -2},
{wuffs_brotli__error__bad_exuberant_metadata_nibble, -3},
{wuffs_brotli__error__bad_simple_huffman_alphabet, -4},
{wuffs_brotli__error__bad_simple_huffman_symbol, -5},
{wuffs_brotli__error__bad_code_length_code_lengths, -6},
{wuffs_brotli__error__bad_huffman_space, -7},
{wuffs_brotli__error__bad_context_map_repeat, -8},
{wuffs_brotli__error__bad_block_length, -9},
{wuffs_brotli__error__bad_metablock_length, -10},
{wuffs_brotli__error__bad_transform, -11},
{wuffs_brotli__error__bad_dictionary, -12},
{wuffs_brotli__error__bad_window_bits, -13},
{wuffs_brotli__error__unsupported_large_window, -13},
{wuffs_brotli__error__bad_padding, -14},
{wuffs_brotli__error__bad_final_padding, -15},
{wuffs_brotli__error__bad_distance, -16},
{wuffs_brotli__error__bad_block_switch, -17},
{wuffs_brotli__error__bad_compound_dictionary, -18},
{wuffs_brotli__error__truncated_input, 2},
};
if (!status) {
return 1; // BROTLI_DECODER_SUCCESS.
}
for (size_t i = 0; i < WUFFS_TESTLIB_ARRAY_SIZE(table); i++) {
if (table[i].status == status) {
return table[i].c_error_code;
}
}
return 0;
}
// do_test_mimic_brotli_differential decodes src with both Wuffs and
// libbrotlidec and checks that they produce the same output and that they
// either both succeed or both fail.
const char* //
do_test_mimic_brotli_differential(const char* name,
wuffs_base__io_buffer* src,
bool want_error) {
wuffs_base__io_buffer have = ((wuffs_base__io_buffer){
.data = g_have_slice_u8,
});
wuffs_base__io_buffer want = ((wuffs_base__io_buffer){
.data = g_want_slice_u8,
});
wuffs_base__io_buffer src0 = *src;
const char* have_status = wuffs_brotli_decode(
&have, &src0, WUFFS_INITIALIZE__DEFAULT_OPTIONS, UINT64_MAX, UINT64_MAX);
wuffs_base__io_buffer src1 = *src;
int error_code = 0;
const char* want_status =
mimic_brotli_decode_error_code(&want, &src1, &error_code);
if ((have_status == NULL) != (want_status == NULL)) {
RETURN_FAIL("%s: have \"%s\", want \"%s\" (error code %d)", name,
have_status, want_status, error_code);
} else if ((have_status != NULL) &&
(brotli_c_error_code(have_status) != error_code)) {
RETURN_FAIL("%s: have \"%s\", want error code %d", name, have_status,
error_code);
} else if ((have_status != NULL) != want_error) {
RETURN_FAIL("%s: have \"%s\", want_error %d", name, have_status,
(int)want_error);
} else if (!want_error && (src0.meta.ri != src1.meta.ri)) {
RETURN_FAIL("%s: ri: have %zu, want %zu", name, src0.meta.ri, src1.meta.ri);
}
// On error, libbrotlidec only guarantees to have written the bytes that it
// flushed so far, so compare only the common prefix.
if (want_error) {
if (have.meta.wi > want.meta.wi) {
have.meta.wi = want.meta.wi;
} else {
want.meta.wi = have.meta.wi;
}
}
const char* status = check_io_buffers_equal("", &have, &want);
if (status) {
RETURN_FAIL("%s: %s", name, status);
}
return NULL;
}
const char* //
test_mimic_brotli_decode_differential() {
CHECK_FOCUS(__func__);
size_t n = WUFFS_TESTLIB_ARRAY_SIZE(g_brotli_vectors);
for (size_t i = 0; i < n; i++) {
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
.data = g_src_slice_u8,
});
CHECK_STRING(read_file(&src, g_brotli_vectors[i].src_filename));
CHECK_STRING(do_test_mimic_brotli_differential(
g_brotli_vectors[i].src_filename, &src, false));
}
return NULL;
}
const char* //
test_mimic_brotli_decode_malformed() {
CHECK_FOCUS(__func__);
size_t n = WUFFS_TESTLIB_ARRAY_SIZE(g_brotli_malformed);
for (size_t i = 0; i < n; i++) {
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
.data = g_src_slice_u8,
});
CHECK_STRING(read_brotli_malformed(&src, i));
char name[64];
snprintf(name, sizeof name, "g_brotli_malformed[%zu]", i);
CHECK_STRING(do_test_mimic_brotli_differential(name, &src, true));
}
return NULL;
}
// g_brotli_split_block_length is an invalid stream (found by differential
// fuzzing) whose last metablock is overrun. In one piece, libbrotlidec
// reports BLOCK_LENGTH_2 (and Wuffs "#bad metablock length"). Split after
// bytes 24, 25 and 30, libbrotlidec reports BLOCK_LENGTH_1, because of its
// internal input buffer, while Wuffs still reports "#bad metablock length".
// This is a documented difference (see "Status Mapping" in
// std/brotli/README.md).
static const uint8_t g_brotli_split_block_length[36] = {
0x09, 0x83, 0x04, 0x00, 0x00, 0x36, 0xE0, 0x28, 0x0A, 0x61, 0xD0, 0x09,
0x07, 0x93, 0xE3, 0xB6, 0x38, 0xE4, 0x02, 0x00, 0xB0, 0xDC, 0x1A, 0xA5,
0x6F, 0x3F, 0x00, 0x20, 0x00, 0xE2, 0xB0, 0x00, 0x6A, 0xDD, 0x07, 0x00,
};
const char* //
test_mimic_brotli_decode_split_block_length() {
CHECK_FOCUS(__func__);
static const size_t ends[2][4] = {{36, 0, 0, 0}, {24, 25, 30, 36}};
static const int want_c[2] = {-10, -9};
for (int split = 0; split < 2; split++) {
// libbrotlidec.
BrotliDecoderState* state = BrotliDecoderCreateInstance(NULL, NULL, NULL);
if (!state) {
RETURN_FAIL("BrotliDecoderCreateInstance failed");
}
size_t pos = 0;
size_t total_out = 0;
for (int k = 0; (k < 4) && ends[split][k]; k++) {
size_t avail_in = ends[split][k] - pos;
const uint8_t* next_in = g_brotli_split_block_length + pos;
size_t avail_out = g_have_slice_u8.len - total_out;
uint8_t* next_out = g_have_slice_u8.ptr + total_out;
BrotliDecoderResult r = BrotliDecoderDecompressStream(
state, &avail_in, &next_in, &avail_out, &next_out, NULL);
pos = ends[split][k] - avail_in;
total_out = g_have_slice_u8.len - avail_out;
if (r != BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT) {
break;
}
}
int code = (int)BrotliDecoderGetErrorCode(state);
BrotliDecoderDestroyInstance(state);
if (code != want_c[split]) {
RETURN_FAIL("split=%d: libbrotlidec error code: have %d, want %d", split,
code, want_c[split]);
}
// Wuffs.
wuffs_brotli__decoder dec;
CHECK_STATUS("initialize",
wuffs_brotli__decoder__initialize(
&dec, sizeof dec, WUFFS_VERSION,
WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED));
wuffs_base__io_buffer dst = ((wuffs_base__io_buffer){
.data = g_have_slice_u8,
});
wuffs_base__status status = wuffs_base__make_status(NULL);
pos = 0;
for (int k = 0; (k < 4) && ends[split][k]; k++) {
wuffs_base__io_buffer src = wuffs_base__ptr_u8__reader(
(uint8_t*)g_brotli_split_block_length, ends[split][k],
ends[split][k] == 36);
src.meta.ri = pos;
status = wuffs_brotli__decoder__transform_io(&dec, &dst, &src,
g_work_slice_u8);
pos = src.meta.ri;
if (status.repr != wuffs_base__suspension__short_read) {
break;
}
}
if (status.repr != wuffs_brotli__error__bad_metablock_length) {
RETURN_FAIL("split=%d: Wuffs status: have \"%s\", want \"%s\"", split,
status.repr, wuffs_brotli__error__bad_metablock_length);
}
}
return NULL;
}
#endif // WUFFS_MIMIC
// ---------------- Encoder Tests
// The encoder tests describe each encoding as a "plan": a sequence of steps.
// Step i applies the BrotliEncoderOperation g_brotli_enc_step_ops[i] (0 =
// PROCESS, 1 = FLUSH, 2 = FINISH, 3 = EMIT_METADATA) to the next
// g_brotli_enc_step_lens[i] bytes of the source. The wuffs_brotli_encode_plan
// and mimic_brotli_encode functions run a plan with Wuffs' std/brotli encoder
// and libbrotlienc, respectively, so that their outputs can be compared.
#define BROTLI_ENC_OP_PROCESS 0
#define BROTLI_ENC_OP_FLUSH 1
#define BROTLI_ENC_OP_FINISH 2
#define BROTLI_ENC_OP_EMIT_METADATA 3
// The BrotliEncoderParameter values.
#define BROTLI_ENC_PARAM_MODE 0
#define BROTLI_ENC_PARAM_QUALITY 1
#define BROTLI_ENC_PARAM_LGWIN 2
#define BROTLI_ENC_PARAM_LGBLOCK 3
#define BROTLI_ENC_PARAM_DISABLE_LITERAL_CONTEXT_MODELING 4
#define BROTLI_ENC_PARAM_SIZE_HINT 5
#define BROTLI_ENC_PARAM_LARGE_WINDOW 6
#define BROTLI_ENC_PARAM_NPOSTFIX 7
#define BROTLI_ENC_PARAM_NDIRECT 8
#define BROTLI_ENC_PARAM_STREAM_OFFSET 9
#define BROTLI_ENC_MAX_STEPS (1 << 19)
#define BROTLI_ENC_MAX_PARAMS 16
uint32_t g_brotli_enc_step_ops[BROTLI_ENC_MAX_STEPS];
size_t g_brotli_enc_step_lens[BROTLI_ENC_MAX_STEPS];
size_t g_brotli_enc_num_steps;
// The chunkings (how the input is split into steps).
#define BROTLI_ENC_CHUNK_SINGLE 0
#define BROTLI_ENC_CHUNK_1 1
#define BROTLI_ENC_CHUNK_7 2
#define BROTLI_ENC_CHUNK_4096 3
#define BROTLI_ENC_CHUNK_RANDOM 4
#define BROTLI_ENC_NUM_CHUNKINGS 5
// The flush modes.
#define BROTLI_ENC_FLUSH_NONE 0
#define BROTLI_ENC_FLUSH_EVERY_CHUNK 1
#define BROTLI_ENC_FLUSH_MIDDLE 2
#define BROTLI_ENC_NUM_FLUSHES 3
// The metadata positions.
#define BROTLI_ENC_METADATA_NONE 0
#define BROTLI_ENC_METADATA_START 1
#define BROTLI_ENC_METADATA_MIDDLE 2
#define BROTLI_ENC_METADATA_END 3
typedef struct {
uint32_t param_keys[BROTLI_ENC_MAX_PARAMS];
uint32_t param_values[BROTLI_ENC_MAX_PARAMS];
size_t num_params;
uint32_t chunking;
uint32_t flush;
uint32_t metadata_position;
size_t metadata_len;
size_t dst_chunk;
uint64_t seed;
} brotli_enc_config;
void //
brotli_enc_config__add_param(brotli_enc_config* c,
uint32_t key,
uint32_t value) {
if (c->num_params < BROTLI_ENC_MAX_PARAMS) {
c->param_keys[c->num_params] = key;
c->param_values[c->num_params] = value;
c->num_params++;
}
}
uint64_t //
brotli_enc_rand(uint64_t* state) {
// SplitMix64.
uint64_t z = (*state += 0x9E3779B97F4A7C15ull);
z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ull;
z = (z ^ (z >> 27)) * 0x94D049BB133111EBull;
return z ^ (z >> 31);
}
void //
brotli_enc_fill_random(uint8_t* p, size_t n, uint64_t seed) {
uint64_t state = seed;
for (size_t i = 0; i < n; i++) {
p[i] = (uint8_t)(brotli_enc_rand(&state) >> 56);
}
}
const char* //
brotli_enc_add_step(uint32_t op, size_t len) {
if (g_brotli_enc_num_steps >= BROTLI_ENC_MAX_STEPS) {
return "too many steps";
}
g_brotli_enc_step_ops[g_brotli_enc_num_steps] = op;
g_brotli_enc_step_lens[g_brotli_enc_num_steps] = len;
g_brotli_enc_num_steps++;
return NULL;
}
// make_brotli_enc_plan sets the g_brotli_enc_step_* plan for encoding the
// data[:data_len] input with the config c. It also writes the plan's source
// (the input, with any metadata bytes interleaved) to src.
const char* //
make_brotli_enc_plan(wuffs_base__io_buffer* src,
const brotli_enc_config* c,
const uint8_t* data,
size_t data_len) {
g_brotli_enc_num_steps = 0;
src->meta.ri = 0;
src->meta.wi = 0;
src->meta.pos = 0;
src->meta.closed = true;
if ((data_len + c->metadata_len) > src->data.len) {
return "make_brotli_enc_plan: src is too short";
}
uint64_t rng = c->seed;
size_t metadata_at = SIZE_MAX;
if (c->metadata_position == BROTLI_ENC_METADATA_START) {
metadata_at = 0;
} else if (c->metadata_position == BROTLI_ENC_METADATA_MIDDLE) {
metadata_at = data_len / 2;
} else if (c->metadata_position == BROTLI_ENC_METADATA_END) {
metadata_at = data_len;
}
bool flushed_middle = false;
size_t i = 0;
while (true) {
if (i == metadata_at) {
brotli_enc_fill_random(src->data.ptr + src->meta.wi, c->metadata_len,
c->seed ^ 0x5A5A);
src->meta.wi += c->metadata_len;
CHECK_STRING(
brotli_enc_add_step(BROTLI_ENC_OP_EMIT_METADATA, c->metadata_len));
metadata_at = SIZE_MAX;
}
size_t n = data_len - i;
switch (c->chunking) {
case BROTLI_ENC_CHUNK_1:
n = 1;
break;
case BROTLI_ENC_CHUNK_7:
n = 7;
break;
case BROTLI_ENC_CHUNK_4096:
n = 4096;
break;
case BROTLI_ENC_CHUNK_RANDOM: {
uint64_t r = brotli_enc_rand(&rng);
// Mostly small chunks, with occasional large ones.
n = (size_t)(1 + ((r & 0xFF) < 0xF0 ? ((r >> 8) % 300)
: ((r >> 8) % 70000)));
break;
}
}
if ((metadata_at != SIZE_MAX) && (metadata_at > i)) {
n = (n < (metadata_at - i)) ? n : (metadata_at - i);
}
if (n > (data_len - i)) {
n = data_len - i;
}
memcpy(src->data.ptr + src->meta.wi, data + i, n);
src->meta.wi += n;
i += n;
bool last = (i == data_len) && (metadata_at == SIZE_MAX);
if (last) {
CHECK_STRING(brotli_enc_add_step(BROTLI_ENC_OP_FINISH, n));
break;
} else if (c->flush == BROTLI_ENC_FLUSH_EVERY_CHUNK) {
CHECK_STRING(brotli_enc_add_step(BROTLI_ENC_OP_FLUSH, n));
} else {
CHECK_STRING(brotli_enc_add_step(BROTLI_ENC_OP_PROCESS, n));
if ((c->flush == BROTLI_ENC_FLUSH_MIDDLE) && !flushed_middle &&
((2 * i) >= data_len)) {
flushed_middle = true;
CHECK_STRING(brotli_enc_add_step(BROTLI_ENC_OP_FLUSH, 0));
}
}
}
return NULL;
}
// brotli_enc_quirk_key maps a BrotliEncoderParameter value to the
// corresponding std/brotli quirk key (or 0 if there is none).
uint32_t //
brotli_enc_quirk_key(uint32_t param) {
switch (param) {
case 0:
return WUFFS_BROTLI__QUIRK_MODE;
case 1:
return WUFFS_BROTLI__QUIRK_ENCODER_QUALITY;
case 2:
return WUFFS_BROTLI__QUIRK_LGWIN;
case 3:
return WUFFS_BROTLI__QUIRK_LGBLOCK;
case 4:
return WUFFS_BROTLI__QUIRK_DISABLE_LITERAL_CONTEXT_MODELING;
case 5:
return WUFFS_BROTLI__QUIRK_SIZE_HINT;
case 6:
return WUFFS_BROTLI__QUIRK_LARGE_WINDOW;
case 7:
return WUFFS_BROTLI__QUIRK_NPOSTFIX;
case 8:
return WUFFS_BROTLI__QUIRK_NDIRECT;
case 9:
return WUFFS_BROTLI__QUIRK_STREAM_OFFSET;
}
return 0;
}
const char* //
wuffs_brotli_encode_plan_with_dictionaries(wuffs_base__io_buffer* dst,
wuffs_base__io_buffer* src,
const uint32_t* param_keys,
const uint32_t* param_values,
size_t num_params,
size_t num_dicts,
const uint32_t* dict_types,
const wuffs_base__slice_u8* dicts,
const uint32_t* step_ops,
const size_t* step_lens,
size_t num_steps,
size_t dst_chunk);
// wuffs_brotli_encode_plan is the Wuffs equivalent of mimic_brotli_encode
// (in test/c/mimiclib/brotli.c).
const char* //
wuffs_brotli_encode_plan(wuffs_base__io_buffer* dst,
wuffs_base__io_buffer* src,
const uint32_t* param_keys,
const uint32_t* param_values,
size_t num_params,
const uint32_t* step_ops,
const size_t* step_lens,
size_t num_steps,
size_t dst_chunk) {
return wuffs_brotli_encode_plan_with_dictionaries(
dst, src, param_keys, param_values, num_params, 0, NULL, NULL, step_ops,
step_lens, num_steps, dst_chunk);
}
// wuffs_brotli_encode_plan_with_dictionaries is the Wuffs equivalent of
// mimic_brotli_encode_with_dictionaries: it adds (with add_dictionary) the
// num_dicts dictionaries after setting the quirks and before the first step.
// Each dict_types[i] is WUFFS_BROTLI__DICTIONARY_TYPE_RAW or
// WUFFS_BROTLI__DICTIONARY_TYPE_SERIALIZED.
const char* //
wuffs_brotli_encode_plan_with_dictionaries(wuffs_base__io_buffer* dst,
wuffs_base__io_buffer* src,
const uint32_t* param_keys,
const uint32_t* param_values,
size_t num_params,
size_t num_dicts,
const uint32_t* dict_types,
const wuffs_base__slice_u8* dicts,
const uint32_t* step_ops,
const size_t* step_lens,
size_t num_steps,
size_t dst_chunk) {
// The encoder struct is large (more than 1 MiB), mostly arrays that do not
// need zero-initializing, so don't use wuffs_brotli__encoder__alloc.
wuffs_brotli__encoder* enc = malloc(sizeof__wuffs_brotli__encoder());
if (!enc) {
return "malloc failed";
}
// At qualities 2 and above, the workbuf can be longer than
// g_work_slice_u8 (e.g. about 80 MiB at quality 9 and lgwin 22).
uint8_t* big_workbuf = NULL;
const char* ret = wuffs_brotli__encoder__initialize(
enc, sizeof__wuffs_brotli__encoder(), WUFFS_VERSION,
WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED)
.repr;
if (ret) {
goto cleanup;
}
for (size_t j = 0; j < num_params; j++) {
wuffs_base__status status = wuffs_brotli__encoder__set_quirk(
enc, brotli_enc_quirk_key(param_keys[j]),
param_values[j]);
if (status.repr) {
ret = status.repr;
goto cleanup;
}
}
wuffs_base__slice_u8 workbuf = g_work_slice_u8;
workbuf.len = 0;
size_t big_workbuf_len = 0;
for (size_t j = 0; j <= num_dicts; j++) {
// Each add_dictionary call needs add_dictionary_workbuf_len bytes and
// transform_io needs workbuf_len bytes, with the same dictionary bytes
// at the start.
uint64_t n = (j < num_dicts)
? wuffs_brotli__encoder__add_dictionary_workbuf_len(
enc, dicts[j].len)
: wuffs_brotli__encoder__workbuf_len(enc).min_incl;
// workbuf_len's minimum is used so that the qualities 10 and 11
// practical bound (see the README) is exercised. At other qualities,
// min_incl == max_incl. Large-window brotli at lgwin 30 needs more
// than 2 GiB.
if ((n > (3ull << 30)) || (n > SIZE_MAX)) {
ret = "workbuf_len is too large";
goto cleanup;
} else if ((n > g_work_slice_u8.len) && (n > big_workbuf_len)) {
uint8_t* p = realloc(big_workbuf, (size_t)n);
if (!p) {
ret = "malloc failed";
goto cleanup;
}
if (!big_workbuf && workbuf.len) {
// Keep the dictionaries added so far.
memcpy(p, g_work_slice_u8.ptr, workbuf.len);
}
big_workbuf = p;
big_workbuf_len = (size_t)n;
workbuf.ptr = big_workbuf;
}
workbuf.len = (size_t)n;
if (j < num_dicts) {
wuffs_base__status status = wuffs_brotli__encoder__add_dictionary(
enc, dict_types[j], dicts[j], workbuf);
if (status.repr) {
ret = status.repr;
goto cleanup;
}
}
}
for (size_t i = 0; i < num_steps; i++) {
size_t end = src->meta.ri + step_lens[i];
if (end > src->meta.wi) {
ret = "wuffs_brotli_encode_plan: step length exceeds the input";
goto cleanup;
}
uint32_t op = step_ops[i];
wuffs_base__status status = wuffs_base__make_status(NULL);
if (op == BROTLI_ENC_OP_FLUSH) {
status = wuffs_brotli__encoder__request_flush(enc);
} else if (op == BROTLI_ENC_OP_EMIT_METADATA) {
status = wuffs_brotli__encoder__request_emit_metadata(enc, step_lens[i]);
}
if (status.repr) {
ret = status.repr;
goto cleanup;
}
while (true) {
wuffs_base__io_buffer d = *dst;
if ((dst_chunk > 0) && ((d.data.len - d.meta.wi) > dst_chunk)) {
d.data.len = d.meta.wi + dst_chunk;
}
wuffs_base__io_buffer s = *src;
s.meta.wi = end;
s.meta.closed = (op == BROTLI_ENC_OP_FINISH);
status = wuffs_brotli__encoder__transform_io(enc, &d, &s, workbuf);
dst->meta.wi = d.meta.wi;
src->meta.ri = s.meta.ri;
if (status.repr == NULL) {
if (op != BROTLI_ENC_OP_FINISH) {
ret = "wuffs_brotli_encode_plan: unexpected ok status";
goto cleanup;
}
break;
} else if (status.repr == wuffs_base__suspension__short_write) {
if (dst->meta.wi == dst->data.len) {
ret = "wuffs_brotli_encode_plan: dst is full";
goto cleanup;
}
continue;
} else if ((status.repr == wuffs_base__suspension__short_read) &&
(op != BROTLI_ENC_OP_FINISH) && (src->meta.ri == end)) {
break;
}
ret = status.repr;
goto cleanup;
}
}
cleanup:
free(big_workbuf);
free(enc);
return ret;
}
// wuffs_brotli_encode encodes src to dst in one go, at the given quality and
// lgwin, with a dst limit of wlimit bytes per transform_io call.
const char* //
wuffs_brotli_encode(wuffs_base__io_buffer* dst,
wuffs_base__io_buffer* src,
uint32_t quality,
uint32_t lgwin,
size_t dst_chunk) {
uint32_t keys[2] = {BROTLI_ENC_PARAM_QUALITY, BROTLI_ENC_PARAM_LGWIN};
uint32_t values[2] = {quality, lgwin};
uint32_t op = BROTLI_ENC_OP_FINISH;
size_t len = wuffs_base__io_buffer__reader_length(src);
return wuffs_brotli_encode_plan(dst, src, keys, values, 2, &op, &len, 1,
dst_chunk);
}
// do_test_brotli_encode_round_trip encodes data[:data_len] (with the plan's
// config c) and then decodes it (with Wuffs' std/brotli decoder), checking
// that the result is the original data.
const char* //
do_test_brotli_encode_round_trip(const char* name,
const brotli_enc_config* c,
const uint8_t* data,
size_t data_len) {
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
.data = g_src_slice_u8,
});
CHECK_STRING(make_brotli_enc_plan(&src, c, data, data_len));
wuffs_base__io_buffer have = ((wuffs_base__io_buffer){
.data = g_have_slice_u8,
});
const char* status = wuffs_brotli_encode_plan(
&have, &src, c->param_keys, c->param_values, c->num_params,
g_brotli_enc_step_ops, g_brotli_enc_step_lens, g_brotli_enc_num_steps,
c->dst_chunk);
if (status) {
RETURN_FAIL("%s: encode: %s", name, status);
}
have.meta.closed = true;
wuffs_base__io_buffer decoded = ((wuffs_base__io_buffer){
.data = g_want_slice_u8,
});
status = wuffs_brotli_decode(&decoded, &have, WUFFS_INITIALIZE__DEFAULT_OPTIONS,
UINT64_MAX, UINT64_MAX);
if (status) {
RETURN_FAIL("%s: decode: %s", name, status);
} else if (have.meta.ri != have.meta.wi) {
RETURN_FAIL("%s: decode: trailing data", name);
} else if (decoded.meta.wi != data_len) {
RETURN_FAIL("%s: decoded length: have %zu, want %zu", name,
decoded.meta.wi, data_len);
} else if (memcmp(decoded.data.ptr, data, data_len)) {
RETURN_FAIL("%s: decoded bytes differ", name);
}
return NULL;
}
// brotli_enc_corpus_filenames lists the round trip (and parity) test inputs:
// every file in test/data/brotli plus some others.
const char* brotli_enc_corpus_filenames[] = {
"test/data/256.bytes",
"test/data/archive.iso",
"test/data/enwik5",
"test/data/nobel-prizes.json",
"test/data/pi.txt",
"test/data/romeo.txt",
"test/data/brotli/README.md",
};
// for_each_brotli_enc_corpus_file calls f on every corpus file, loading each
// one into a malloc'ed buffer.
const char* //
for_each_brotli_enc_corpus_file(
const char* (*f)(const char* name, const uint8_t* data, size_t len)) {
const char* names[256];
size_t num_names = 0;
for (size_t i = 0; i < WUFFS_TESTLIB_ARRAY_SIZE(brotli_enc_corpus_filenames);
i++) {
names[num_names++] = brotli_enc_corpus_filenames[i];
}
for (size_t i = 0; i < WUFFS_TESTLIB_ARRAY_SIZE(g_brotli_vectors); i++) {
names[num_names++] = g_brotli_vectors[i].src_filename;
names[num_names++] = g_brotli_vectors[i].want_filename;
}
for (size_t i = 0; i < WUFFS_TESTLIB_ARRAY_SIZE(g_brotli_malformed); i++) {
if (g_brotli_malformed[i].src_filename) {
names[num_names++] = g_brotli_malformed[i].src_filename;
}
}
uint8_t* buf = malloc(1 << 20);
if (!buf) {
return "malloc failed";
}
const char* ret = NULL;
for (size_t i = 0; i < num_names; i++) {
bool dup = false;
for (size_t j = 0; j < i; j++) {
if (!strcmp(names[i], names[j])) {
dup = true;
break;
}
}
if (dup) {
continue;
}
wuffs_base__io_buffer b = ((wuffs_base__io_buffer){
.data = wuffs_base__make_slice_u8(buf, 1 << 20),
});
ret = read_file(&b, names[i]);
if (!ret) {
ret = f(names[i], b.data.ptr, b.meta.wi);
}
if (ret) {
break;
}
}
free(buf);
return ret;
}
// brotli_enc_synthetic makes one of the synthetic test inputs: 0 is 1 MiB of
// zeroes, 1 is 256 KiB of pseudo-random bytes and 2 is a 100-byte pattern
// repeated to 64 KiB. It returns a malloc'ed buffer.
uint8_t* //
brotli_enc_synthetic(int which, size_t* len) {
size_t n = (which == 0) ? (1 << 20) : (which == 1) ? (256 << 10) : (64 << 10);
uint8_t* p = malloc(n);
if (!p) {
return NULL;
}
if (which == 0) {
memset(p, 0, n);
} else if (which == 1) {
brotli_enc_fill_random(p, n, 0x1234);
} else {
uint8_t pattern[100];
brotli_enc_fill_random(pattern, 100, 0x5678);
for (size_t i = 0; i < n; i++) {
p[i] = pattern[i % 100];
}
}
*len = n;
return p;
}
brotli_enc_config //
make_brotli_enc_config(uint32_t quality, uint32_t lgwin) {
brotli_enc_config c;
memset(&c, 0, sizeof c);
brotli_enc_config__add_param(&c, BROTLI_ENC_PARAM_QUALITY, quality);
brotli_enc_config__add_param(&c, BROTLI_ENC_PARAM_LGWIN, lgwin);
return c;
}
const char* //
do_test_brotli_encode_round_trip_file(const char* name,
const uint8_t* data,
size_t len) {
for (uint32_t q = 0; q <= 11; q++) {
for (int k = (q <= 1) ? 0 : 1; k < 3; k++) {
brotli_enc_config c = make_brotli_enc_config(q, (k == 0) ? 10 : 22);
if (k == 2) {
c.chunking = BROTLI_ENC_CHUNK_RANDOM;
c.flush = BROTLI_ENC_FLUSH_MIDDLE;
c.dst_chunk = 97;
c.seed = len;
}
char s[256];
snprintf(s, sizeof s, "%s (q=%u, k=%d)", name, q, k);
CHECK_STRING(do_test_brotli_encode_round_trip(s, &c, data, len));
}
}
return NULL;
}
const char* //
test_wuffs_brotli_encode_round_trip_corpus() {
CHECK_FOCUS(__func__);
return for_each_brotli_enc_corpus_file(do_test_brotli_encode_round_trip_file);
}
const char* //
test_wuffs_brotli_encode_round_trip_lengths() {
CHECK_FOCUS(__func__);
uint8_t data[40];
brotli_enc_fill_random(data, 20, 0x9999);
memcpy(data + 20, data, 20);
for (size_t n = 0; n <= 40; n++) {
char s[64];
snprintf(s, sizeof s, "length %zu", n);
CHECK_STRING(do_test_brotli_encode_round_trip_file(s, data, n));
}
return NULL;
}
const char* //
test_wuffs_brotli_encode_round_trip_synthetic() {
CHECK_FOCUS(__func__);
for (int which = 0; which < 3; which++) {
size_t len = 0;
uint8_t* data = brotli_enc_synthetic(which, &len);
if (!data) {
RETURN_FAIL("malloc failed");
}
char s[64];
snprintf(s, sizeof s, "synthetic #%d", which);
const char* status = do_test_brotli_encode_round_trip_file(s, data, len);
free(data);
if (status) {
return status;
}
}
return NULL;
}
// brotli_enc_long_zero_run fills data[:9200] with 6300 pseudo-random bytes
// followed by a copy of their first 2900 bytes, with one byte in every 21,
// 15, 13 or 11 changed. At quality 2 (BrotliStoreMetaBlockFast with more than
// 128 commands) that makes command codes 72 to 75 (last distance, one literal
// and a 10 to 20 byte copy) and 688 (6300 literals and an 80 byte copy), so
// the complex Huffman code for the commands has a run of 612 zero code
// lengths, more than the 511 that the encoder's repetition tables once had.
static void //
brotli_enc_long_zero_run(uint8_t* data) {
uint32_t x = 1;
for (int i = 0; i < 6300; i++) {
x ^= x << 13;
x ^= x >> 17;
x ^= x << 5;
data[i] = (uint8_t)x;
}
memcpy(data + 6300, data, 2900);
static const int steps[4] = {21, 15, 13, 11};
for (int i = 80, k = 0; i < 2900; i += steps[k++ & 3]) {
data[6300 + i] ^= 0x55;
}
}
const char* //
test_wuffs_brotli_encode_long_zero_run() {
CHECK_FOCUS(__func__);
uint8_t data[9200];
brotli_enc_long_zero_run(data);
brotli_enc_config c = make_brotli_enc_config(2, 16);
return do_test_brotli_encode_round_trip("long zero run", &c, data,
sizeof data);
}
const char* //
test_wuffs_brotli_encode_metadata_round_trip() {
CHECK_FOCUS(__func__);
wuffs_base__io_buffer data = ((wuffs_base__io_buffer){
.data = g_mimiclib_scratch_slice_u8,
});
CHECK_STRING(read_file(&data, "test/data/romeo.txt"));
static const size_t lens[] = {0, 1, 100, 70000};
for (uint32_t q = 0; q <= 11; q++) {
for (size_t i = 0; i < WUFFS_TESTLIB_ARRAY_SIZE(lens); i++) {
for (uint32_t pos = 1; pos <= 3; pos++) {
brotli_enc_config c = make_brotli_enc_config(q, 18);
c.chunking = BROTLI_ENC_CHUNK_7;
c.metadata_position = pos;
c.metadata_len = lens[i];
c.dst_chunk = (pos == 2) ? 3 : 0;
char s[64];
snprintf(s, sizeof s, "q=%u, len=%zu, pos=%u", q, lens[i], pos);
CHECK_STRING(do_test_brotli_encode_round_trip(s, &c, data.data.ptr,
data.meta.wi));
}
}
}
return NULL;
}
const char* //
test_wuffs_brotli_encode_parameters() {
CHECK_FOCUS(__func__);
wuffs_brotli__encoder* enc = wuffs_brotli__encoder__alloc();
if (!enc) {
RETURN_FAIL("alloc failed");
}
const char* ret = NULL;
wuffs_base__status status;
uint8_t buf[64];
wuffs_base__io_buffer dst =
wuffs_base__ptr_u8__writer(buf, sizeof buf);
wuffs_base__io_buffer src = make_io_buffer_from_string("", 0);
#define BROTLI_ENC_CHECK(cond, msg) \
if (!(cond)) { \
ret = msg; \
goto cleanup; \
}
// Unknown keys and out-of-range values.
status = wuffs_brotli__encoder__set_quirk(
enc, WUFFS_BROTLI__QUIRK_STREAM_OFFSET + 1, 0);
BROTLI_ENC_CHECK(status.repr == wuffs_base__error__unsupported_option,
"unknown key");
status = wuffs_brotli__encoder__set_quirk(enc, 12345, 0);
BROTLI_ENC_CHECK(status.repr == wuffs_base__error__unsupported_option,
"unknown key (2)");
status = wuffs_brotli__encoder__set_quirk(
enc, WUFFS_BROTLI__QUIRK_ENCODER_QUALITY, 0x100000000ull);
BROTLI_ENC_CHECK(status.repr == wuffs_base__error__bad_argument,
"value > 0xFFFFFFFF");
status = wuffs_brotli__encoder__set_quirk(
enc, WUFFS_BROTLI__QUIRK_DISABLE_LITERAL_CONTEXT_MODELING, 2);
BROTLI_ENC_CHECK(status.repr == wuffs_base__error__bad_argument,
"disable_lcm > 1");
status = wuffs_brotli__encoder__set_quirk(
enc, WUFFS_BROTLI__QUIRK_STREAM_OFFSET, (1 << 30) + 1);
BROTLI_ENC_CHECK(status.repr == wuffs_base__error__bad_argument,
"stream_offset > 1<<30");
status = wuffs_brotli__encoder__request_emit_metadata(enc, (1 << 24) + 1);
BROTLI_ENC_CHECK(status.repr == wuffs_base__error__bad_argument,
"metadata length > 1<<24");
// Raw values are stored as is, and sanitized later.
status = wuffs_brotli__encoder__set_quirk(
enc, WUFFS_BROTLI__QUIRK_LGWIN, 99);
BROTLI_ENC_CHECK(!status.repr, "set lgwin");
BROTLI_ENC_CHECK(wuffs_brotli__encoder__get_quirk(
enc, WUFFS_BROTLI__QUIRK_LGWIN) == 99,
"get lgwin");
// The default quality is 11 (and lgwin 99 is sanitized to 24).
BROTLI_ENC_CHECK(wuffs_brotli__encoder__workbuf_len(enc).min_incl > 0,
"workbuf_len at q11");
status = wuffs_brotli__encoder__set_quirk(
enc, WUFFS_BROTLI__QUIRK_ENCODER_QUALITY, 0xFFFFFFFF); // (int)-1.
BROTLI_ENC_CHECK(!status.repr, "set quality");
// Quality -1 is sanitized to 0 and lgwin 99 to 24.
BROTLI_ENC_CHECK(wuffs_brotli__encoder__workbuf_len(enc).max_incl ==
((2u << 24) + 503 + 16),
"workbuf_len at q0w24");
status = wuffs_brotli__encoder__set_quirk(
enc, WUFFS_BROTLI__QUIRK_LGWIN, 0xFFFFFFFF); // (int)-1.
BROTLI_ENC_CHECK(!status.repr, "set lgwin (2)");
BROTLI_ENC_CHECK(wuffs_brotli__encoder__workbuf_len(enc).max_incl ==
((2u << 10) + 503 + 16),
"workbuf_len at q0w10");
// max_compressed_size mirrors BrotliEncoderMaxCompressedSize.
BROTLI_ENC_CHECK(wuffs_brotli__encoder__max_compressed_size(enc, 0) == 2,
"max_compressed_size(0)");
BROTLI_ENC_CHECK(
wuffs_brotli__encoder__max_compressed_size(enc, 1 << 20) ==
((1 << 20) + 6 + (4 * 64)),
"max_compressed_size(1<<20)");
BROTLI_ENC_CHECK(
wuffs_brotli__encoder__max_compressed_size(enc, UINT64_MAX) == 0,
"max_compressed_size(overflow)");
// A too-short workbuf.
src.meta.closed = false;
status = wuffs_brotli__encoder__transform_io(
enc, &dst, &src, wuffs_base__make_slice_u8(g_work_slice_u8.ptr, 10));
BROTLI_ENC_CHECK(status.repr == wuffs_base__suspension__short_read,
"empty input");
// Parameters can't be changed after the first transform_io call.
status = wuffs_brotli__encoder__set_quirk(
enc, WUFFS_BROTLI__QUIRK_LGWIN, 16);
BROTLI_ENC_CHECK(status.repr == wuffs_base__error__bad_call_sequence,
"set_quirk after transform_io");
src.data.ptr = (uint8_t*)"abc";
src.data.len = 3;
src.meta.wi = 3;
status = wuffs_brotli__encoder__transform_io(
enc, &dst, &src, wuffs_base__make_slice_u8(g_work_slice_u8.ptr, 10));
BROTLI_ENC_CHECK(status.repr == wuffs_base__error__bad_workbuf_length,
"short workbuf");
// At qualities 2 to 9, workbuf_len depends on the parameters, and a
// too-short workbuf is rejected.
for (uint32_t q = 2; q <= 9; q++) {
free(enc);
enc = wuffs_brotli__encoder__alloc();
if (!enc) {
RETURN_FAIL("alloc failed");
}
wuffs_brotli__encoder__set_quirk(enc, WUFFS_BROTLI__QUIRK_ENCODER_QUALITY,
q);
wuffs_brotli__encoder__set_quirk(enc, WUFFS_BROTLI__QUIRK_LGWIN, 10);
uint64_t wlen = wuffs_brotli__encoder__workbuf_len(enc).max_incl;
BROTLI_ENC_CHECK((wlen > 0) && (wlen <= g_work_slice_u8.len),
"workbuf_len at quality 2..9");
wuffs_brotli__encoder__set_quirk(enc, WUFFS_BROTLI__QUIRK_LGWIN, 24);
BROTLI_ENC_CHECK(wuffs_brotli__encoder__workbuf_len(enc).max_incl > wlen,
"workbuf_len grows with lgwin");
wuffs_brotli__encoder__set_quirk(enc, WUFFS_BROTLI__QUIRK_LGWIN, 10);
src = make_io_buffer_from_string("abc", 3);
dst.meta.wi = 0;
status = wuffs_brotli__encoder__transform_io(
enc, &dst, &src,
wuffs_base__make_slice_u8(g_work_slice_u8.ptr, (size_t)(wlen / 2)));
BROTLI_ENC_CHECK(status.repr == wuffs_base__error__bad_workbuf_length,
"short workbuf at quality 2..9");
}
src = make_io_buffer_from_string("", 0);
dst.meta.wi = 0;
// With the large-window quirk, lgwin can be up to 30 (instead of 24) at
// qualities 3 and above. Like the C code, quality 2 and below ignore the
// quirk. Above lgwin 24, the ring buffer is (2 << lgwin) bytes and, at
// quality 3, the H35 hasher has a 64 MiB rolling hash table on top of
// H3's 256 KiB.
for (uint32_t q = 2; q <= 3; q++) {
uint64_t wlens[3] = {0};
for (int k = 0; k < 3; k++) {
free(enc);
enc = wuffs_brotli__encoder__alloc();
if (!enc) {
RETURN_FAIL("alloc failed");
}
wuffs_brotli__encoder__set_quirk(
enc, WUFFS_BROTLI__QUIRK_ENCODER_QUALITY, q);
wuffs_brotli__encoder__set_quirk(enc, WUFFS_BROTLI__QUIRK_LGWIN,
(k == 0) ? 24 : 30);
if (k > 0) {
wuffs_brotli__encoder__set_quirk(
enc, WUFFS_BROTLI__QUIRK_LARGE_WINDOW, (k == 2) ? 1 : 0);
}
wlens[k] = wuffs_brotli__encoder__workbuf_len(enc).max_incl;
}
BROTLI_ENC_CHECK(wlens[0] == wlens[1],
"lgwin 30 without large window is lgwin 24");
if (q == 2) {
BROTLI_ENC_CHECK(wlens[0] == wlens[2],
"large window is ignored at quality 2");
} else {
BROTLI_ENC_CHECK(
wlens[2] == (wlens[0] + (2ull << 30) - (2ull << 24) + (64 << 20)),
"workbuf_len at q3w30 with large window");
}
}
// At qualities 2 to 9, the ring buffer is the workbuf's last region (of
// (2 + (1 << rb_bits) + (1 << lgblock) + 7) bytes) and a shorter workbuf
// works if it holds (2 + input_length + 16) bytes of it (and the input is
// short enough for the ring buffer never to wrap).
{
free(enc);
enc = wuffs_brotli__encoder__alloc();
if (!enc) {
RETURN_FAIL("alloc failed");
}
wuffs_brotli__encoder__set_quirk(enc, WUFFS_BROTLI__QUIRK_ENCODER_QUALITY,
3);
wuffs_brotli__encoder__set_quirk(enc, WUFFS_BROTLI__QUIRK_LGWIN, 30);
wuffs_brotli__encoder__set_quirk(enc, WUFFS_BROTLI__QUIRK_LARGE_WINDOW, 1);
uint64_t wlen = wuffs_brotli__encoder__workbuf_len(enc).max_incl;
uint64_t rb_region = 2 + (2ull << 30) + (1 << 14) + 7;
BROTLI_ENC_CHECK(wlen > rb_region, "workbuf_len at q3w30");
size_t short_len = (size_t)(wlen - rb_region + 2 + 3 + 16);
uint8_t* short_workbuf = malloc(short_len);
if (!short_workbuf) {
RETURN_FAIL("malloc failed");
}
for (int k = 0; k < 2; k++) {
wuffs_brotli__encoder* e = wuffs_brotli__encoder__alloc();
if (!e) {
free(short_workbuf);
RETURN_FAIL("alloc failed");
}
wuffs_brotli__encoder__set_quirk(e, WUFFS_BROTLI__QUIRK_ENCODER_QUALITY,
3);
wuffs_brotli__encoder__set_quirk(e, WUFFS_BROTLI__QUIRK_LGWIN, 30);
wuffs_brotli__encoder__set_quirk(e, WUFFS_BROTLI__QUIRK_LARGE_WINDOW,
1);
wuffs_base__io_buffer s = make_io_buffer_from_string("abc", 3);
s.meta.closed = true;
dst.meta.wi = 0;
status = wuffs_brotli__encoder__transform_io(
e, &dst, &s,
wuffs_base__make_slice_u8(short_workbuf, short_len - (k ? 0 : 1)));
free(e);
if (k == 0) {
if (status.repr != wuffs_base__error__bad_workbuf_length) {
free(short_workbuf);
RETURN_FAIL("too short workbuf at q3w30: %s", status.repr);
}
} else if (status.repr) {
free(short_workbuf);
RETURN_FAIL("short workbuf at q3w30: %s", status.repr);
}
}
free(short_workbuf);
}
// At qualities 10 and 11, workbuf_len is a range (see the README's
// "Qualities 10 and 11 Workbuf Length" section).
for (uint32_t q = 10; q <= 12; q++) {
static const struct {
uint32_t lgwin;
uint64_t min_incl;
uint64_t max_incl;
} want[] = {
{16, 96686024ull, 302507168ull},
{22, 422539208ull, 5364573344ull},
{24, 813133768ull, 10148177056ull},
};
for (size_t i = 0; i < WUFFS_TESTLIB_ARRAY_SIZE(want); i++) {
free(enc);
enc = wuffs_brotli__encoder__alloc();
if (!enc) {
RETURN_FAIL("alloc failed");
}
wuffs_brotli__encoder__set_quirk(
enc, WUFFS_BROTLI__QUIRK_ENCODER_QUALITY, q);
wuffs_brotli__encoder__set_quirk(enc, WUFFS_BROTLI__QUIRK_LGWIN,
want[i].lgwin);
wuffs_base__range_ii_u64 r = wuffs_brotli__encoder__workbuf_len(enc);
if ((r.min_incl != want[i].min_incl) ||
(r.max_incl != want[i].max_incl)) {
RETURN_FAIL("workbuf_len at q=%u lgwin=%u: have %" PRIu64
"..%" PRIu64 ", want %" PRIu64 "..%" PRIu64,
q, want[i].lgwin, r.min_incl, r.max_incl,
want[i].min_incl, want[i].max_incl);
}
}
// A workbuf that does not even reach the ring buffer is rejected.
src = make_io_buffer_from_string("abc", 3);
dst.meta.wi = 0;
status = wuffs_brotli__encoder__transform_io(
enc, &dst, &src, wuffs_base__make_slice_u8(g_work_slice_u8.ptr, 1000));
BROTLI_ENC_CHECK(status.repr == wuffs_base__error__bad_workbuf_length,
"short workbuf at quality 10..11");
}
// Compound dictionaries at qualities 10 and 11. The want bytes are
// libbrotlienc's output (with the dictionary prepared and attached, lgwin
// 16 and BROTLI_OPERATION_FINISH), which is the same at both qualities.
for (uint32_t q = 10; q <= 11; q++) {
free(enc);
enc = wuffs_brotli__encoder__alloc();
if (!enc) {
RETURN_FAIL("alloc failed");
}
wuffs_brotli__encoder__set_quirk(enc, WUFFS_BROTLI__QUIRK_ENCODER_QUALITY,
q);
wuffs_brotli__encoder__set_quirk(enc, WUFFS_BROTLI__QUIRK_LGWIN, 16);
static const char dict_bytes[] = "a raw dictionary, for quality 11";
static const char input[] = "for quality 10 or 11, a raw dictionary";
static const uint8_t want[] = {
0xA2, 0x04, 0x00, 0xBF, 0x34, 0x08, 0x0B, 0x4C, 0x8C, 0x8C,
0x20, 0x18, 0x71, 0xEA, 0x81, 0x22, 0x4A, 0xEC, 0x4B,
};
wuffs_base__slice_u8 d = wuffs_base__make_slice_u8(
(uint8_t*)dict_bytes, sizeof dict_bytes - 1);
// The workbuf (whose prefix holds the prepared dictionary) is longer
// than g_work_slice_u8, so it is malloc'ed and then realloc'ed.
uint64_t n = wuffs_brotli__encoder__add_dictionary_workbuf_len(enc, d.len);
uint8_t* p = malloc((size_t)n);
BROTLI_ENC_CHECK(p, "malloc failed");
status = wuffs_brotli__encoder__add_dictionary(
enc, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, d,
wuffs_base__make_slice_u8(p, (size_t)n));
if (status.repr) {
free(p);
}
BROTLI_ENC_CHECK(!status.repr, "add_dictionary at quality 10..11");
n = wuffs_brotli__encoder__workbuf_len(enc).min_incl;
uint8_t* p2 = realloc(p, (size_t)n);
if (!p2) {
free(p);
}
BROTLI_ENC_CHECK(p2, "realloc failed");
src = make_io_buffer_from_string(input, sizeof input - 1);
src.meta.closed = true;
dst.meta.wi = 0;
status = wuffs_brotli__encoder__transform_io(
enc, &dst, &src, wuffs_base__make_slice_u8(p2, (size_t)n));
free(p2);
BROTLI_ENC_CHECK(!status.repr, "dictionary at quality 10..11");
BROTLI_ENC_CHECK((dst.meta.wi == sizeof want) &&
!memcmp(dst.data.ptr, want, sizeof want),
"dictionary at quality 10..11: output bytes");
}
src = make_io_buffer_from_string("", 0);
dst.meta.wi = 0;
src = make_io_buffer_from_string("", 0);
dst.meta.wi = 0;
// Requests.
free(enc);
enc = wuffs_brotli__encoder__alloc();
if (!enc) {
RETURN_FAIL("alloc failed");
}
status = wuffs_brotli__encoder__request_emit_metadata(enc, 1);
BROTLI_ENC_CHECK(!status.repr, "request_emit_metadata");
status = wuffs_brotli__encoder__request_flush(enc);
BROTLI_ENC_CHECK(status.repr == wuffs_base__error__bad_call_sequence,
"request_flush while metadata is pending");
status = wuffs_brotli__encoder__request_emit_metadata(enc, 1);
BROTLI_ENC_CHECK(status.repr == wuffs_base__error__bad_call_sequence,
"request_emit_metadata twice");
// A closed src with too few metadata bytes.
wuffs_brotli__encoder__set_quirk(enc, WUFFS_BROTLI__QUIRK_ENCODER_QUALITY, 1);
src = make_io_buffer_from_string("", 0);
status = wuffs_brotli__encoder__transform_io(enc, &dst, &src,
g_work_slice_u8);
BROTLI_ENC_CHECK(status.repr == wuffs_brotli__error__truncated_input,
"truncated metadata");
#undef BROTLI_ENC_CHECK
cleanup:
free(enc);
if (ret) {
RETURN_FAIL("%s: status \"%s\"", ret, status.repr);
}
return NULL;
}
const char* //
test_wuffs_brotli_encode_interface() {
CHECK_FOCUS(__func__);
wuffs_brotli__encoder* enc = wuffs_brotli__encoder__alloc();
if (!enc) {
RETURN_FAIL("alloc failed");
}
wuffs_brotli__encoder__set_quirk(enc, WUFFS_BROTLI__QUIRK_ENCODER_QUALITY, 1);
// Encoding romeo.txt at q1w22 gives 575 bytes, the last of which is 0xC3.
const char* status = do_test__wuffs_base__io_transformer(
wuffs_brotli__encoder__upcast_as__wuffs_base__io_transformer(enc),
"test/data/romeo.txt", 0, SIZE_MAX, 575, 0xC3);
free(enc);
return status;
}
// ---------------- Decoder Metadata Tests
// The metadata tests encode romeo.txt with metadata blocks (whose contents
// are also taken from romeo.txt) and then decode it with
// QUIRK_REPORT_METADATA. The decoded output ("out") and the metadata ("md",
// each block prefixed by its length as 4 little-endian bytes) are collected
// separately.
#define BROTLI_MD_NUM_STEPS 8
#define BROTLI_MD_MAX_BLOCKS 8
static const uint32_t g_brotli_md_step_ops[BROTLI_MD_NUM_STEPS] = {
BROTLI_ENC_OP_EMIT_METADATA, BROTLI_ENC_OP_PROCESS,
BROTLI_ENC_OP_EMIT_METADATA, BROTLI_ENC_OP_EMIT_METADATA,
BROTLI_ENC_OP_FLUSH, BROTLI_ENC_OP_EMIT_METADATA,
BROTLI_ENC_OP_PROCESS, BROTLI_ENC_OP_FINISH,
};
static const size_t g_brotli_md_step_lens[BROTLI_MD_NUM_STEPS] = {
3, 500, 10, 0, 0, 70000, 1000, 50000,
};
// brotli_md_want fills out and md with the expected results for the
// g_brotli_md_step_etc plan applied to src. It sets md_pos[i] to the length of
// out before the i'th metadata block.
static void //
brotli_md_want(wuffs_base__io_buffer* out,
wuffs_base__io_buffer* md,
size_t* md_pos,
const uint8_t* src) {
size_t n_md = 0;
for (size_t i = 0; i < BROTLI_MD_NUM_STEPS; i++) {
size_t n = g_brotli_md_step_lens[i];
if (g_brotli_md_step_ops[i] == BROTLI_ENC_OP_EMIT_METADATA) {
md_pos[n_md++] = out->meta.wi;
wuffs_base__poke_u32le__no_bounds_check(md->data.ptr + md->meta.wi,
(uint32_t)n);
md->meta.wi += 4;
memcpy(md->data.ptr + md->meta.wi, src, n);
md->meta.wi += n;
} else {
memcpy(out->data.ptr + out->meta.wi, src, n);
out->meta.wi += n;
}
src += n;
}
}
// wuffs_brotli_decode_metadata decodes src_ptr[.. src_len] with
// QUIRK_REPORT_METADATA, at most dst_chunk bytes of dst and src_chunk more
// bytes of src per transform_io call.
static const char* //
wuffs_brotli_decode_metadata(wuffs_base__io_buffer* out,
wuffs_base__io_buffer* md,
size_t* md_pos,
size_t* num_md,
const uint8_t* src_ptr,
size_t src_len,
size_t dst_chunk,
size_t src_chunk) {
wuffs_brotli__decoder dec;
CHECK_STATUS("initialize",
wuffs_brotli__decoder__initialize(
&dec, sizeof dec, WUFFS_VERSION,
WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED));
CHECK_STATUS("set_quirk",
wuffs_brotli__decoder__set_quirk(
&dec, WUFFS_BROTLI__QUIRK_REPORT_METADATA, 1));
if (wuffs_brotli__decoder__get_quirk(
&dec, WUFFS_BROTLI__QUIRK_REPORT_METADATA) != 1) {
RETURN_FAIL("get_quirk: have 0, want 1");
}
uint8_t tmp[4096];
if (dst_chunk > sizeof tmp) {
RETURN_FAIL("dst_chunk is too large");
}
size_t md_remaining = 0;
*num_md = 0;
wuffs_base__io_buffer src = wuffs_base__ptr_u8__reader(
(uint8_t*)src_ptr, src_chunk < src_len ? src_chunk : src_len,
src_chunk >= src_len);
while (true) {
wuffs_base__io_buffer dst = wuffs_base__ptr_u8__writer(tmp, dst_chunk);
wuffs_base__status status = wuffs_brotli__decoder__transform_io(
&dec, &dst, &src, g_work_slice_u8);
uint8_t* p = tmp;
size_t n = dst.meta.wi;
if (md_remaining > 0) {
size_t m = (n < md_remaining) ? n : md_remaining;
if (md->data.len - md->meta.wi < m) {
RETURN_FAIL("md is too short");
}
memcpy(md->data.ptr + md->meta.wi, p, m);
md->meta.wi += m;
md_remaining -= m;
p += m;
n -= m;
}
if (n > 0) {
if (md_remaining > 0) {
RETURN_FAIL("inconsistent md_remaining");
} else if (out->data.len - out->meta.wi < n) {
RETURN_FAIL("out is too short");
}
memcpy(out->data.ptr + out->meta.wi, p, n);
out->meta.wi += n;
}
if (status.repr == NULL) {
break;
} else if (status.repr == wuffs_base__note__metadata_reported) {
if (md_remaining > 0) {
RETURN_FAIL("metadata reported while copying metadata");
} else if (*num_md >= BROTLI_MD_MAX_BLOCKS) {
RETURN_FAIL("too many metadata blocks");
}
uint64_t len = wuffs_brotli__decoder__metadata_length(&dec);
md_pos[(*num_md)++] = out->meta.wi;
if (md->data.len - md->meta.wi < 4) {
RETURN_FAIL("md is too short");
}
wuffs_base__poke_u32le__no_bounds_check(md->data.ptr + md->meta.wi,
(uint32_t)len);
md->meta.wi += 4;
md_remaining = (size_t)len;
} else if (status.repr == wuffs_base__suspension__short_read) {
if (src.meta.closed) {
RETURN_FAIL("short read on closed src");
}
src.meta.wi = (src_len - src.meta.wi > src_chunk) ? src.meta.wi + src_chunk
: src_len;
src.meta.closed = src.meta.wi == src_len;
} else if (status.repr != wuffs_base__suspension__short_write) {
RETURN_FAIL("transform_io: \"%s\"", status.repr);
}
}
if (md_remaining > 0) {
RETURN_FAIL("incomplete metadata");
}
// Setting the quirk after decoding has started fails.
wuffs_base__status status = wuffs_brotli__decoder__set_quirk(
&dec, WUFFS_BROTLI__QUIRK_REPORT_METADATA, 0);
if (status.repr != wuffs_base__error__bad_call_sequence) {
RETURN_FAIL("set_quirk after start: have \"%s\", want \"%s\"", status.repr,
wuffs_base__error__bad_call_sequence);
}
return NULL;
}
static const char* //
do_test_brotli_decode_metadata(const char* (*encode)(wuffs_base__io_buffer*,
wuffs_base__io_buffer*,
const uint32_t*,
const uint32_t*,
size_t,
const uint32_t*,
const size_t*,
size_t,
size_t)) {
wuffs_base__io_buffer data = ((wuffs_base__io_buffer){
.data = g_mimiclib_scratch_slice_u8,
});
CHECK_STRING(read_file(&data, "test/data/romeo.txt"));
// The plan needs 121513 bytes, so repeat romeo.txt (which is shorter).
size_t total = 0;
for (size_t i = 0; i < BROTLI_MD_NUM_STEPS; i++) {
total += g_brotli_md_step_lens[i];
}
if (data.meta.wi == 0) {
RETURN_FAIL("empty romeo.txt");
}
while (data.meta.wi < total) {
size_t n = data.meta.wi;
if (n > total - data.meta.wi) {
n = total - data.meta.wi;
}
memmove(data.data.ptr + data.meta.wi, data.data.ptr, n);
data.meta.wi += n;
}
// The want and have halves of g_want_slice_u8 and g_have_slice_u8.
size_t half_w = g_want_slice_u8.len / 2;
size_t half_h = g_have_slice_u8.len / 2;
wuffs_base__io_buffer want_out = ((wuffs_base__io_buffer){
.data = wuffs_base__make_slice_u8(g_want_slice_u8.ptr, half_w),
});
wuffs_base__io_buffer want_md = ((wuffs_base__io_buffer){
.data = wuffs_base__make_slice_u8(g_want_slice_u8.ptr + half_w, half_w),
});
size_t want_md_pos[BROTLI_MD_MAX_BLOCKS] = {0};
brotli_md_want(&want_out, &want_md, want_md_pos, data.data.ptr);
static const uint32_t qualities[3] = {0, 5, 11};
static const size_t chunks[4][2] = {{4096, 1 << 30}, {1, 1}, {7, 3}, {3, 64}};
for (int qi = 0; qi < 3; qi++) {
// The decoder uses g_work_slice_u8 as its workbuf, so encode into the
// second half of g_have_slice_u8.
wuffs_base__io_buffer enc = ((wuffs_base__io_buffer){
.data = wuffs_base__make_slice_u8(g_have_slice_u8.ptr + half_h, half_h),
});
wuffs_base__io_buffer src = data;
src.meta.ri = 0;
uint32_t keys[2] = {BROTLI_ENC_PARAM_QUALITY, BROTLI_ENC_PARAM_LGWIN};
uint32_t values[2] = {qualities[qi], 18};
CHECK_STRING((*encode)(&enc, &src, keys, values, 2, g_brotli_md_step_ops,
g_brotli_md_step_lens, BROTLI_MD_NUM_STEPS, 0));
for (int ci = 0; ci < 4; ci++) {
// Decoded output and metadata share the first half of g_have_slice_u8.
wuffs_base__io_buffer have_out = ((wuffs_base__io_buffer){
.data = wuffs_base__make_slice_u8(g_have_slice_u8.ptr, half_h / 2),
});
wuffs_base__io_buffer have_md = ((wuffs_base__io_buffer){
.data = wuffs_base__make_slice_u8(g_have_slice_u8.ptr + half_h / 2,
half_h / 2),
});
size_t have_md_pos[BROTLI_MD_MAX_BLOCKS] = {0};
size_t num_md = 0;
const char* z = wuffs_brotli_decode_metadata(
&have_out, &have_md, have_md_pos, &num_md, enc.data.ptr, enc.meta.wi,
chunks[ci][0], chunks[ci][1]);
if (z) {
RETURN_FAIL("q=%u, chunks=%d: %s", qualities[qi], ci, z);
}
char s[64];
snprintf(s, sizeof s, "q=%u, chunks=%d: out", qualities[qi], ci);
CHECK_STRING(check_io_buffers_equal(s, &have_out, &want_out));
snprintf(s, sizeof s, "q=%u, chunks=%d: md", qualities[qi], ci);
CHECK_STRING(check_io_buffers_equal(s, &have_md, &want_md));
if (num_md != 4) {
RETURN_FAIL("q=%u, chunks=%d: num_md: have %zu, want 4",
qualities[qi], ci, num_md);
}
for (size_t i = 0; i < num_md; i++) {
if (have_md_pos[i] != want_md_pos[i]) {
RETURN_FAIL("q=%u, chunks=%d: md_pos[%zu]: have %zu, want %zu",
qualities[qi], ci, i, have_md_pos[i], want_md_pos[i]);
}
}
}
}
return NULL;
}
const char* //
test_wuffs_brotli_decode_metadata() {
CHECK_FOCUS(__func__);
return do_test_brotli_decode_metadata(wuffs_brotli_encode_plan);
}
#ifdef WUFFS_MIMIC
// mimic_brotli_md_start_func and mimic_brotli_md_chunk_func are libbrotlidec
// metadata callbacks that append to the wuffs_base__io_buffer opaque.
static void //
mimic_brotli_md_start_func(void* opaque, size_t size) {
wuffs_base__io_buffer* md = (wuffs_base__io_buffer*)opaque;
if (md->data.len - md->meta.wi >= 4) {
wuffs_base__poke_u32le__no_bounds_check(md->data.ptr + md->meta.wi,
(uint32_t)size);
md->meta.wi += 4;
} else {
md->meta.closed = true; // Overflow.
}
}
static void //
mimic_brotli_md_chunk_func(void* opaque, const uint8_t* data, size_t size) {
wuffs_base__io_buffer* md = (wuffs_base__io_buffer*)opaque;
if (md->data.len - md->meta.wi >= size) {
memcpy(md->data.ptr + md->meta.wi, data, size);
md->meta.wi += size;
} else {
md->meta.closed = true; // Overflow.
}
}
// test_mimic_brotli_decode_metadata checks that QUIRK_REPORT_METADATA reports
// the same metadata as libbrotlidec's metadata callbacks, for streams that
// libbrotlienc produces (and for the streams of the brotli test data, which
// have no metadata blocks), with a variety of input chunk sizes.
const char* //
test_mimic_brotli_decode_metadata() {
CHECK_FOCUS(__func__);
CHECK_STRING(do_test_brotli_decode_metadata(mimic_brotli_encode));
// Also compare libbrotlidec's callbacks with Wuffs on a libbrotlienc
// stream, fed to libbrotlidec in 5-byte pieces.
wuffs_base__io_buffer data = ((wuffs_base__io_buffer){
.data = g_mimiclib_scratch_slice_u8,
});
CHECK_STRING(read_file(&data, "test/data/romeo.txt"));
size_t half_h = g_have_slice_u8.len / 2;
size_t half_w = g_want_slice_u8.len / 2;
wuffs_base__io_buffer enc = ((wuffs_base__io_buffer){
.data = wuffs_base__make_slice_u8(g_have_slice_u8.ptr + half_h, half_h),
});
static const uint32_t ops[5] = {
BROTLI_ENC_OP_PROCESS, BROTLI_ENC_OP_EMIT_METADATA,
BROTLI_ENC_OP_EMIT_METADATA, BROTLI_ENC_OP_PROCESS,
BROTLI_ENC_OP_FINISH,
};
size_t lens[5] = {100, 200, 0, 300, 0};
lens[4] = data.meta.wi - 600;
uint32_t keys[1] = {BROTLI_ENC_PARAM_QUALITY};
uint32_t values[1] = {9};
CHECK_STRING(mimic_brotli_encode(&enc, &data, keys, values, 1, ops, lens, 5,
0));
wuffs_base__io_buffer c_out = ((wuffs_base__io_buffer){
.data = wuffs_base__make_slice_u8(g_want_slice_u8.ptr, half_w),
});
wuffs_base__io_buffer c_md = ((wuffs_base__io_buffer){
.data = wuffs_base__make_slice_u8(g_want_slice_u8.ptr + half_w, half_w),
});
BrotliDecoderState* state = BrotliDecoderCreateInstance(NULL, NULL, NULL);
if (!state) {
RETURN_FAIL("BrotliDecoderCreateInstance failed");
}
BrotliDecoderSetMetadataCallbacks(state, mimic_brotli_md_start_func,
mimic_brotli_md_chunk_func, &c_md);
size_t pos = 0;
BrotliDecoderResult r = BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT;
while (r != BROTLI_DECODER_RESULT_SUCCESS) {
size_t end = (enc.meta.wi - pos > 5) ? pos + 5 : enc.meta.wi;
size_t avail_in = end - pos;
const uint8_t* next_in = enc.data.ptr + pos;
size_t avail_out = c_out.data.len - c_out.meta.wi;
uint8_t* next_out = c_out.data.ptr + c_out.meta.wi;
r = BrotliDecoderDecompressStream(state, &avail_in, &next_in, &avail_out,
&next_out, NULL);
pos = end - avail_in;
c_out.meta.wi = c_out.data.len - avail_out;
if ((r == BROTLI_DECODER_RESULT_ERROR) ||
((r == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT) &&
(pos == enc.meta.wi))) {
BrotliDecoderDestroyInstance(state);
RETURN_FAIL("libbrotlidec failed");
}
}
BrotliDecoderDestroyInstance(state);
if (c_md.meta.closed) {
RETURN_FAIL("libbrotlidec metadata overflow");
}
wuffs_base__io_buffer have_out = ((wuffs_base__io_buffer){
.data = wuffs_base__make_slice_u8(g_have_slice_u8.ptr, half_h / 2),
});
wuffs_base__io_buffer have_md = ((wuffs_base__io_buffer){
.data = wuffs_base__make_slice_u8(g_have_slice_u8.ptr + half_h / 2,
half_h / 2),
});
size_t have_md_pos[BROTLI_MD_MAX_BLOCKS] = {0};
size_t num_md = 0;
CHECK_STRING(wuffs_brotli_decode_metadata(&have_out, &have_md, have_md_pos,
&num_md, enc.data.ptr, enc.meta.wi,
5, 5));
CHECK_STRING(check_io_buffers_equal("out", &have_out, &c_out));
CHECK_STRING(check_io_buffers_equal("md", &have_md, &c_md));
if ((num_md != 2) || (have_md_pos[0] != 100) || (have_md_pos[1] != 100)) {
RETURN_FAIL("md_pos: have %zu blocks at %zu, %zu; want 2 at 100, 100",
num_md, have_md_pos[0], have_md_pos[1]);
}
return NULL;
}
#endif // WUFFS_MIMIC
// ---------------- Encoder Mimic Tests
#ifdef WUFFS_MIMIC
// do_test_mimic_brotli_encode checks that Wuffs and libbrotlienc produce
// exactly the same bytes for encoding data[:data_len] with the config c.
const char* //
do_test_mimic_brotli_encode(const char* name,
const brotli_enc_config* c,
const uint8_t* data,
size_t data_len) {
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
.data = g_src_slice_u8,
});
CHECK_STRING(make_brotli_enc_plan(&src, c, data, data_len));
wuffs_base__io_buffer have = ((wuffs_base__io_buffer){
.data = g_have_slice_u8,
});
wuffs_base__io_buffer want = ((wuffs_base__io_buffer){
.data = g_want_slice_u8,
});
wuffs_base__io_buffer src0 = src;
const char* have_status = wuffs_brotli_encode_plan(
&have, &src0, c->param_keys, c->param_values, c->num_params,
g_brotli_enc_step_ops, g_brotli_enc_step_lens, g_brotli_enc_num_steps,
c->dst_chunk);
if (have_status) {
RETURN_FAIL("%s: wuffs: %s", name, have_status);
}
wuffs_base__io_buffer src1 = src;
const char* want_status = mimic_brotli_encode(
&want, &src1, c->param_keys, c->param_values, c->num_params,
g_brotli_enc_step_ops, g_brotli_enc_step_lens, g_brotli_enc_num_steps,
c->dst_chunk);
if (want_status) {
RETURN_FAIL("%s: mimic: %s", name, want_status);
}
const char* status = check_io_buffers_equal("", &have, &want);
if (status) {
RETURN_FAIL("%s: %s", name, status);
}
return NULL;
}
const char* brotli_enc_parity_filenames[] = {
"test/data/256.bytes",
"test/data/romeo.txt",
"test/data/brotli/alice29.txt",
"test/data/brotli/random_org_10k.bin",
};
const char* //
test_mimic_brotli_encode_parity_grid() {
CHECK_FOCUS(__func__);
static const uint32_t lgwins[] = {10, 16, 18, 22, 24};
wuffs_base__io_buffer data = ((wuffs_base__io_buffer){
.data = g_mimiclib_scratch_slice_u8,
});
for (size_t f = 0; f < WUFFS_TESTLIB_ARRAY_SIZE(brotli_enc_parity_filenames);
f++) {
data.meta.ri = 0;
data.meta.wi = 0;
data.meta.closed = false;
CHECK_STRING(read_file(&data, brotli_enc_parity_filenames[f]));
for (uint32_t q = 0; q <= 1; q++) {
for (size_t w = 0; w < WUFFS_TESTLIB_ARRAY_SIZE(lgwins); w++) {
for (uint32_t mode = 0; mode <= 2; mode++) {
for (uint32_t ch = 0; ch < BROTLI_ENC_NUM_CHUNKINGS; ch++) {
if ((ch == BROTLI_ENC_CHUNK_1) && (data.meta.wi > 4096)) {
continue;
}
for (uint32_t fl = 0; fl < BROTLI_ENC_NUM_FLUSHES; fl++) {
brotli_enc_config c = make_brotli_enc_config(q, lgwins[w]);
brotli_enc_config__add_param(&c, BROTLI_ENC_PARAM_MODE, mode);
c.chunking = ch;
c.flush = fl;
c.seed = (f << 16) | (w << 8) | (mode << 4) | fl;
// To keep this test fast, the 7-byte chunking only uses the
// first 4 KiB of the input. The
// script/manual-test-brotli-encoder-parity.c program runs
// the whole input.
size_t n = data.meta.wi;
if ((ch == BROTLI_ENC_CHUNK_7) && (n > 4096)) {
n = 4096;
}
char s[256];
snprintf(s, sizeof s,
"%s: q=%u lgwin=%u mode=%u chunking=%u flush=%u",
brotli_enc_parity_filenames[f], q, lgwins[w], mode, ch,
fl);
CHECK_STRING(
do_test_mimic_brotli_encode(s, &c, data.data.ptr, n));
}
}
}
}
}
}
return NULL;
}
const char* //
test_mimic_brotli_encode_parity_metadata() {
CHECK_FOCUS(__func__);
static const size_t lens[] = {0, 1, 100, 70000};
wuffs_base__io_buffer data = ((wuffs_base__io_buffer){
.data = g_mimiclib_scratch_slice_u8,
});
CHECK_STRING(read_file(&data, "test/data/romeo.txt"));
for (uint32_t q = 0; q <= 1; q++) {
for (size_t i = 0; i < WUFFS_TESTLIB_ARRAY_SIZE(lens); i++) {
for (uint32_t pos = 1; pos <= 3; pos++) {
for (uint32_t fl = 0; fl < BROTLI_ENC_NUM_FLUSHES; fl++) {
for (size_t dc = 0; dc < 2; dc++) {
brotli_enc_config c = make_brotli_enc_config(q, 16);
c.chunking = BROTLI_ENC_CHUNK_RANDOM;
c.flush = fl;
c.metadata_position = pos;
c.metadata_len = lens[i];
c.dst_chunk = dc ? 5 : 0;
c.seed = (i << 8) | pos;
char s[256];
snprintf(s, sizeof s,
"q=%u metadata_len=%zu pos=%u flush=%u dst_chunk=%zu", q,
lens[i], pos, fl, c.dst_chunk);
CHECK_STRING(do_test_mimic_brotli_encode(s, &c, data.data.ptr,
data.meta.wi));
}
}
}
}
}
return NULL;
}
const char* //
test_mimic_brotli_encode_parity_params() {
CHECK_FOCUS(__func__);
wuffs_base__io_buffer data = ((wuffs_base__io_buffer){
.data = g_mimiclib_scratch_slice_u8,
});
CHECK_STRING(read_file(&data, "test/data/brotli/alice29.txt"));
uint32_t n = (uint32_t)data.meta.wi;
static const struct {
uint32_t key;
uint32_t value;
} extras[] = {
{BROTLI_ENC_PARAM_SIZE_HINT, 0},
{BROTLI_ENC_PARAM_SIZE_HINT, 0xFFFFFFFF}, // Replaced by the exact size.
{BROTLI_ENC_PARAM_SIZE_HINT, 1u << 30},
{BROTLI_ENC_PARAM_LGBLOCK, 16},
{BROTLI_ENC_PARAM_LGBLOCK, 24},
{BROTLI_ENC_PARAM_DISABLE_LITERAL_CONTEXT_MODELING, 1},
{BROTLI_ENC_PARAM_LARGE_WINDOW, 1},
{BROTLI_ENC_PARAM_NPOSTFIX, 2},
{BROTLI_ENC_PARAM_NDIRECT, 12},
{BROTLI_ENC_PARAM_STREAM_OFFSET, 1},
{BROTLI_ENC_PARAM_STREAM_OFFSET, 1000},
{BROTLI_ENC_PARAM_STREAM_OFFSET, 1u << 30},
{BROTLI_ENC_PARAM_LGWIN, 0xFFFFFFFF}, // (int)-1.
{BROTLI_ENC_PARAM_LGWIN, 30},
{BROTLI_ENC_PARAM_QUALITY, 0xFFFFFFFF}, // (int)-1, so quality 0.
};
for (uint32_t q = 0; q <= 1; q++) {
for (size_t i = 0; i < WUFFS_TESTLIB_ARRAY_SIZE(extras); i++) {
for (uint32_t ch = 0; ch < BROTLI_ENC_NUM_CHUNKINGS; ch += 4) {
brotli_enc_config c = make_brotli_enc_config(q, 22);
uint32_t v = extras[i].value;
if ((extras[i].key == BROTLI_ENC_PARAM_SIZE_HINT) &&
(v == 0xFFFFFFFF)) {
v = n;
}
brotli_enc_config__add_param(&c, extras[i].key, v);
c.chunking = ch;
c.flush = BROTLI_ENC_FLUSH_MIDDLE;
c.seed = i;
char s[256];
snprintf(s, sizeof s, "q=%u param[%u]=%u chunking=%u", q,
extras[i].key, v, ch);
CHECK_STRING(
do_test_mimic_brotli_encode(s, &c, data.data.ptr, data.meta.wi));
}
}
}
return NULL;
}
// test_mimic_brotli_encode_parity_q2_grid is like
// test_mimic_brotli_encode_parity_grid but for qualities 2 to 9, with fewer
// lgwins and chunkings (to keep it fast).
const char* //
test_mimic_brotli_encode_parity_q2_grid() {
CHECK_FOCUS(__func__);
static const uint32_t lgwins[] = {10, 18, 24};
static const uint32_t chunkings[] = {
BROTLI_ENC_CHUNK_SINGLE,
BROTLI_ENC_CHUNK_7,
BROTLI_ENC_CHUNK_RANDOM,
};
wuffs_base__io_buffer data = ((wuffs_base__io_buffer){
.data = g_mimiclib_scratch_slice_u8,
});
for (size_t f = 0; f < WUFFS_TESTLIB_ARRAY_SIZE(brotli_enc_parity_filenames);
f++) {
data.meta.ri = 0;
data.meta.wi = 0;
data.meta.closed = false;
CHECK_STRING(read_file(&data, brotli_enc_parity_filenames[f]));
for (uint32_t q = 2; q <= 9; q++) {
for (size_t w = 0; w < WUFFS_TESTLIB_ARRAY_SIZE(lgwins); w++) {
for (uint32_t mode = 0; mode <= 2; mode++) {
for (size_t k = 0; k < WUFFS_TESTLIB_ARRAY_SIZE(chunkings); k++) {
uint32_t ch = chunkings[k];
for (uint32_t fl = 0; fl < BROTLI_ENC_NUM_FLUSHES; fl++) {
brotli_enc_config c = make_brotli_enc_config(q, lgwins[w]);
brotli_enc_config__add_param(&c, BROTLI_ENC_PARAM_MODE, mode);
c.chunking = ch;
c.flush = fl;
c.seed = (f << 16) | (w << 8) | (mode << 4) | fl | (q << 20);
// The chunked plans only use the first 4 KiB (7-byte chunks)
// or 64 KiB (random chunks) of the input.
size_t n = data.meta.wi;
if ((ch == BROTLI_ENC_CHUNK_7) && (n > 4096)) {
n = 4096;
} else if ((ch == BROTLI_ENC_CHUNK_RANDOM) && (n > 65536)) {
n = 65536;
}
char s[256];
snprintf(s, sizeof s,
"%s: q=%u lgwin=%u mode=%u chunking=%u flush=%u",
brotli_enc_parity_filenames[f], q, lgwins[w], mode, ch,
fl);
CHECK_STRING(
do_test_mimic_brotli_encode(s, &c, data.data.ptr, n));
}
}
}
}
}
}
return NULL;
}
const char* //
test_mimic_brotli_encode_parity_q2_metadata() {
CHECK_FOCUS(__func__);
static const size_t lens[] = {0, 1, 100, 70000};
wuffs_base__io_buffer data = ((wuffs_base__io_buffer){
.data = g_mimiclib_scratch_slice_u8,
});
CHECK_STRING(read_file(&data, "test/data/romeo.txt"));
for (uint32_t q = 2; q <= 9; q++) {
for (size_t i = 0; i < WUFFS_TESTLIB_ARRAY_SIZE(lens); i++) {
for (uint32_t pos = 1; pos <= 3; pos++) {
for (uint32_t fl = 0; fl < BROTLI_ENC_NUM_FLUSHES; fl++) {
for (size_t dc = 0; dc < 2; dc++) {
brotli_enc_config c = make_brotli_enc_config(q, 16);
c.chunking = dc ? BROTLI_ENC_CHUNK_RANDOM : BROTLI_ENC_CHUNK_7;
c.flush = fl;
c.metadata_position = pos;
c.metadata_len = lens[i];
c.dst_chunk = dc ? 5 : 0;
c.seed = (i << 8) | pos | (q << 12);
char s[256];
snprintf(s, sizeof s,
"q=%u metadata_len=%zu pos=%u flush=%u dst_chunk=%zu", q,
lens[i], pos, fl, c.dst_chunk);
CHECK_STRING(do_test_mimic_brotli_encode(s, &c, data.data.ptr,
data.meta.wi));
}
}
}
}
}
return NULL;
}
const char* //
test_mimic_brotli_encode_parity_q2_params() {
CHECK_FOCUS(__func__);
wuffs_base__io_buffer data = ((wuffs_base__io_buffer){
.data = g_mimiclib_scratch_slice_u8,
});
CHECK_STRING(read_file(&data, "test/data/brotli/alice29.txt"));
uint32_t n = (uint32_t)data.meta.wi;
// Each row is up to three (BrotliEncoderParameter, value) pairs. A zero key
// (other than in the first pair) ends the row.
static const uint32_t extras[][6] = {
{BROTLI_ENC_PARAM_SIZE_HINT, 0xFFFFFFFF}, // Replaced by the exact size.
{BROTLI_ENC_PARAM_SIZE_HINT, 1},
{BROTLI_ENC_PARAM_SIZE_HINT, 1u << 20},
{BROTLI_ENC_PARAM_SIZE_HINT, 1u << 30},
{BROTLI_ENC_PARAM_LGBLOCK, 16},
{BROTLI_ENC_PARAM_LGBLOCK, 17},
{BROTLI_ENC_PARAM_LGBLOCK, 20},
{BROTLI_ENC_PARAM_LGBLOCK, 24},
{BROTLI_ENC_PARAM_DISABLE_LITERAL_CONTEXT_MODELING, 1},
{BROTLI_ENC_PARAM_DISABLE_LITERAL_CONTEXT_MODELING, 1, //
BROTLI_ENC_PARAM_MODE, 1},
{BROTLI_ENC_PARAM_MODE, 1},
{BROTLI_ENC_PARAM_MODE, 2},
{BROTLI_ENC_PARAM_NPOSTFIX, 1},
{BROTLI_ENC_PARAM_NPOSTFIX, 2, BROTLI_ENC_PARAM_NDIRECT, 12},
{BROTLI_ENC_PARAM_NPOSTFIX, 3, BROTLI_ENC_PARAM_NDIRECT, 120},
{BROTLI_ENC_PARAM_NPOSTFIX, 0, BROTLI_ENC_PARAM_NDIRECT, 15},
{BROTLI_ENC_PARAM_MODE, 2, BROTLI_ENC_PARAM_NPOSTFIX, 3,
BROTLI_ENC_PARAM_NDIRECT, 30},
{BROTLI_ENC_PARAM_STREAM_OFFSET, 1},
{BROTLI_ENC_PARAM_STREAM_OFFSET, 1000},
{BROTLI_ENC_PARAM_LGWIN, 0xFFFFFFFF}, // (int)-1.
{BROTLI_ENC_PARAM_SIZE_HINT, 1u << 21, //
BROTLI_ENC_PARAM_LGBLOCK, 18},
};
for (uint32_t q = 2; q <= 9; q++) {
for (size_t i = 0; i < WUFFS_TESTLIB_ARRAY_SIZE(extras); i++) {
for (uint32_t ch = 0; ch < BROTLI_ENC_NUM_CHUNKINGS; ch += 4) {
brotli_enc_config c = make_brotli_enc_config(q, 22);
char s[256];
int k = snprintf(s, sizeof s, "q=%u chunking=%u", q, ch);
for (size_t j = 0; (j < 6) && ((j == 0) || extras[i][j]); j += 2) {
uint32_t v = extras[i][j + 1];
if ((extras[i][j] == BROTLI_ENC_PARAM_SIZE_HINT) &&
(v == 0xFFFFFFFF)) {
v = n;
}
brotli_enc_config__add_param(&c, extras[i][j], v);
if ((k > 0) && ((size_t)k < sizeof s)) {
k += snprintf(s + k, sizeof s - (size_t)k, " param[%u]=%u",
extras[i][j], v);
}
}
c.chunking = ch;
c.flush = BROTLI_ENC_FLUSH_MIDDLE;
c.seed = i | (q << 8);
CHECK_STRING(
do_test_mimic_brotli_encode(s, &c, data.data.ptr, data.meta.wi));
}
}
}
// Large-window brotli is supported (by the Wuffs encoder) at quality 2.
for (uint32_t w = 22; w <= 30; w += 4) {
brotli_enc_config c = make_brotli_enc_config(2, w);
brotli_enc_config__add_param(&c, BROTLI_ENC_PARAM_LARGE_WINDOW, 1);
c.chunking = BROTLI_ENC_CHUNK_RANDOM;
c.flush = BROTLI_ENC_FLUSH_MIDDLE;
c.seed = w;
char s[256];
snprintf(s, sizeof s, "q=2 large_window lgwin=%u", w);
CHECK_STRING(
do_test_mimic_brotli_encode(s, &c, data.data.ptr, data.meta.wi));
}
return NULL;
}
// test_mimic_brotli_encode_parity_q10_grid is like
// test_mimic_brotli_encode_parity_q2_grid but for qualities 10 and 11, with
// shorter inputs (to keep it fast).
const char* //
test_mimic_brotli_encode_parity_q10_grid() {
CHECK_FOCUS(__func__);
static const uint32_t lgwins[] = {10, 24};
static const uint32_t chunkings[] = {
BROTLI_ENC_CHUNK_SINGLE,
BROTLI_ENC_CHUNK_7,
BROTLI_ENC_CHUNK_RANDOM,
};
wuffs_base__io_buffer data = ((wuffs_base__io_buffer){
.data = g_mimiclib_scratch_slice_u8,
});
for (size_t f = 0; f < WUFFS_TESTLIB_ARRAY_SIZE(brotli_enc_parity_filenames);
f++) {
data.meta.ri = 0;
data.meta.wi = 0;
data.meta.closed = false;
CHECK_STRING(read_file(&data, brotli_enc_parity_filenames[f]));
for (uint32_t q = 10; q <= 11; q++) {
for (size_t w = 0; w < WUFFS_TESTLIB_ARRAY_SIZE(lgwins); w++) {
for (uint32_t mode = 0; mode <= 2; mode++) {
for (size_t k = 0; k < WUFFS_TESTLIB_ARRAY_SIZE(chunkings); k++) {
uint32_t ch = chunkings[k];
for (uint32_t fl = 0; fl < BROTLI_ENC_NUM_FLUSHES; fl++) {
brotli_enc_config c = make_brotli_enc_config(q, lgwins[w]);
brotli_enc_config__add_param(&c, BROTLI_ENC_PARAM_MODE, mode);
c.chunking = ch;
c.flush = fl;
c.seed = (f << 16) | (w << 8) | (mode << 4) | fl | (q << 20);
// Use the first 4 KiB (7-byte chunks) or 24 KiB (otherwise)
// of the input.
size_t n = data.meta.wi;
if ((ch == BROTLI_ENC_CHUNK_7) && (n > 4096)) {
n = 4096;
} else if (n > 24576) {
n = 24576;
}
char s[256];
snprintf(s, sizeof s,
"%s: q=%u lgwin=%u mode=%u chunking=%u flush=%u",
brotli_enc_parity_filenames[f], q, lgwins[w], mode, ch,
fl);
CHECK_STRING(
do_test_mimic_brotli_encode(s, &c, data.data.ptr, n));
}
}
}
}
}
}
return NULL;
}
const char* //
test_mimic_brotli_encode_parity_q10_metadata() {
CHECK_FOCUS(__func__);
static const size_t lens[] = {0, 1, 100, 70000};
wuffs_base__io_buffer data = ((wuffs_base__io_buffer){
.data = g_mimiclib_scratch_slice_u8,
});
CHECK_STRING(read_file(&data, "test/data/romeo.txt"));
for (uint32_t q = 10; q <= 11; q++) {
for (size_t i = 0; i < WUFFS_TESTLIB_ARRAY_SIZE(lens); i++) {
for (uint32_t pos = 1; pos <= 3; pos++) {
for (uint32_t fl = 0; fl < BROTLI_ENC_NUM_FLUSHES; fl++) {
for (size_t dc = 0; dc < 2; dc++) {
brotli_enc_config c = make_brotli_enc_config(q, 16);
c.chunking = dc ? BROTLI_ENC_CHUNK_RANDOM : BROTLI_ENC_CHUNK_7;
c.flush = fl;
c.metadata_position = pos;
c.metadata_len = lens[i];
c.dst_chunk = dc ? 5 : 0;
c.seed = (i << 8) | pos | (q << 12);
char s[256];
snprintf(s, sizeof s,
"q=%u metadata_len=%zu pos=%u flush=%u dst_chunk=%zu", q,
lens[i], pos, fl, c.dst_chunk);
CHECK_STRING(do_test_mimic_brotli_encode(s, &c, data.data.ptr,
data.meta.wi));
}
}
}
}
}
return NULL;
}
const char* //
test_mimic_brotli_encode_parity_q10_params() {
CHECK_FOCUS(__func__);
wuffs_base__io_buffer data = ((wuffs_base__io_buffer){
.data = g_mimiclib_scratch_slice_u8,
});
CHECK_STRING(read_file(&data, "test/data/brotli/alice29.txt"));
// Use the first 40 KiB, to keep it fast.
uint32_t n = (uint32_t)data.meta.wi;
if (n > 40960) {
n = 40960;
}
// Each row is up to three (BrotliEncoderParameter, value) pairs. A zero key
// (other than in the first pair) ends the row.
static const uint32_t extras[][6] = {
{BROTLI_ENC_PARAM_SIZE_HINT, 0xFFFFFFFF}, // Replaced by the exact size.
{BROTLI_ENC_PARAM_SIZE_HINT, 1},
{BROTLI_ENC_PARAM_SIZE_HINT, 1u << 30},
{BROTLI_ENC_PARAM_LGBLOCK, 16},
{BROTLI_ENC_PARAM_LGBLOCK, 17},
{BROTLI_ENC_PARAM_LGBLOCK, 24},
{BROTLI_ENC_PARAM_DISABLE_LITERAL_CONTEXT_MODELING, 1},
{BROTLI_ENC_PARAM_DISABLE_LITERAL_CONTEXT_MODELING, 1, //
BROTLI_ENC_PARAM_MODE, 1},
{BROTLI_ENC_PARAM_MODE, 1},
{BROTLI_ENC_PARAM_MODE, 2},
{BROTLI_ENC_PARAM_NPOSTFIX, 1},
{BROTLI_ENC_PARAM_NPOSTFIX, 2, BROTLI_ENC_PARAM_NDIRECT, 12},
{BROTLI_ENC_PARAM_NPOSTFIX, 3, BROTLI_ENC_PARAM_NDIRECT, 120},
{BROTLI_ENC_PARAM_NPOSTFIX, 0, BROTLI_ENC_PARAM_NDIRECT, 15},
{BROTLI_ENC_PARAM_MODE, 2, BROTLI_ENC_PARAM_NPOSTFIX, 3,
BROTLI_ENC_PARAM_NDIRECT, 30},
{BROTLI_ENC_PARAM_STREAM_OFFSET, 1},
{BROTLI_ENC_PARAM_STREAM_OFFSET, 1000},
{BROTLI_ENC_PARAM_STREAM_OFFSET, 1u << 30},
{BROTLI_ENC_PARAM_LGWIN, 0xFFFFFFFF}, // (int)-1.
{BROTLI_ENC_PARAM_SIZE_HINT, 1u << 21, //
BROTLI_ENC_PARAM_LGBLOCK, 18},
};
for (uint32_t q = 10; q <= 11; q++) {
for (size_t i = 0; i < WUFFS_TESTLIB_ARRAY_SIZE(extras); i++) {
for (uint32_t ch = 0; ch < BROTLI_ENC_NUM_CHUNKINGS; ch += 4) {
brotli_enc_config c = make_brotli_enc_config(q, 22);
char s[256];
int k = snprintf(s, sizeof s, "q=%u chunking=%u", q, ch);
for (size_t j = 0; (j < 6) && ((j == 0) || extras[i][j]); j += 2) {
uint32_t v = extras[i][j + 1];
if ((extras[i][j] == BROTLI_ENC_PARAM_SIZE_HINT) &&
(v == 0xFFFFFFFF)) {
v = n;
}
brotli_enc_config__add_param(&c, extras[i][j], v);
if ((k > 0) && ((size_t)k < sizeof s)) {
k += snprintf(s + k, sizeof s - (size_t)k, " param[%u]=%u",
extras[i][j], v);
}
}
c.chunking = ch;
c.flush = BROTLI_ENC_FLUSH_MIDDLE;
c.seed = i | (q << 8);
CHECK_STRING(do_test_mimic_brotli_encode(s, &c, data.data.ptr, n));
}
}
}
return NULL;
}
const char* //
test_mimic_brotli_encode_long_zero_run() {
CHECK_FOCUS(__func__);
uint8_t data[9200];
brotli_enc_long_zero_run(data);
brotli_enc_config c = make_brotli_enc_config(2, 16);
return do_test_mimic_brotli_encode("long zero run", &c, data, sizeof data);
}
// test_mimic_brotli_encode_metadata_stale_pointer checks a libbrotlienc bug
// that Wuffs does not mirror. At quality 1, a metadata block whose body goes
// through the C code's 16-byte tiny_buf_ (dst has no room) leaves next_out_
// just past tiny_buf_, at remaining_metadata_bytes_. Qualities 0 and 1 then
// compress in place without resetting next_out_, so the next flush's
// InjectBytePaddingBlock writes over remaining_metadata_bytes_ (its output is
// still right) and the next BrotliEncoderCompressStream call fails. Wuffs
// produces the same bytes and carries on, as libbrotlienc does when
// ProcessMetadata sets next_out_ to NULL when it ends. See "Encoder Operation
// Mapping" in std/brotli/README.md.
const char* //
test_mimic_brotli_encode_metadata_stale_pointer() {
CHECK_FOCUS(__func__);
uint8_t src[300];
static const char fox[] = "the quick brown fox jumps over the lazy dog ";
for (int i = 0; i < 300; i++) {
src[i] = (uint8_t)(fox[i % 44] ^ (i / 97));
}
// PROCESS 100 bytes, EMIT_METADATA 32 bytes (with 1 byte of dst per call),
// PROCESS 100 bytes, FLUSH, FINISH 68 bytes.
static const uint32_t ops[5] = {
BROTLI_ENC_OP_PROCESS, BROTLI_ENC_OP_EMIT_METADATA,
BROTLI_ENC_OP_PROCESS, BROTLI_ENC_OP_FLUSH, BROTLI_ENC_OP_FINISH,
};
static const size_t lens[5] = {100, 32, 100, 0, 68};
// libbrotlienc.
BrotliEncoderState* state = BrotliEncoderCreateInstance(NULL, NULL, NULL);
if (!state) {
RETURN_FAIL("BrotliEncoderCreateInstance failed");
}
BrotliEncoderSetParameter(state, BROTLI_PARAM_QUALITY, 1);
BrotliEncoderSetParameter(state, BROTLI_PARAM_LGWIN, 16);
const uint8_t* next_in = src;
size_t want_len = 0;
int failed_step = -1;
for (int i = 0; (i < 5) && (failed_step < 0); i++) {
size_t avail_in = lens[i];
while (true) {
size_t room = (ops[i] == BROTLI_ENC_OP_EMIT_METADATA) ? 1 : 1000;
size_t avail_out = room;
uint8_t* next_out = g_want_slice_u8.ptr + want_len;
if (!BrotliEncoderCompressStream(state, (BrotliEncoderOperation)ops[i],
&avail_in, &next_in, &avail_out,
&next_out, NULL)) {
failed_step = i;
break;
}
want_len += room - avail_out;
if ((ops[i] == BROTLI_ENC_OP_FINISH) ? BrotliEncoderIsFinished(state)
: ((avail_in == 0) &&
!BrotliEncoderHasMoreOutput(
state))) {
break;
}
}
}
BrotliEncoderDestroyInstance(state);
if (failed_step != 4) {
RETURN_FAIL("libbrotlienc: failed step: have %d, want 4", failed_step);
}
// Wuffs.
wuffs_base__io_buffer have = ((wuffs_base__io_buffer){
.data = g_have_slice_u8,
});
wuffs_base__io_buffer in = wuffs_base__ptr_u8__reader(src, 300, false);
uint32_t keys[2] = {BROTLI_ENC_PARAM_QUALITY, BROTLI_ENC_PARAM_LGWIN};
uint32_t values[2] = {1, 16};
CHECK_STRING(
wuffs_brotli_encode_plan(&have, &in, keys, values, 2, ops, lens, 5, 1));
if ((have.meta.wi <= want_len) ||
memcmp(have.data.ptr, g_want_slice_u8.ptr, want_len)) {
RETURN_FAIL("Wuffs output (%zu bytes) does not extend C's (%zu bytes)",
have.meta.wi, want_len);
}
// The Wuffs output decodes to the data (without the metadata).
size_t decoded_len = g_work_slice_u8.len;
if (BrotliDecoderDecompress(have.meta.wi, have.data.ptr, &decoded_len,
g_work_slice_u8.ptr) != BROTLI_DECODER_RESULT_SUCCESS) {
RETURN_FAIL("BrotliDecoderDecompress failed");
} else if ((decoded_len != 268) ||
memcmp(g_work_slice_u8.ptr, src, 100) ||
memcmp(g_work_slice_u8.ptr + 100, src + 132, 168)) {
RETURN_FAIL("round trip mismatch");
}
return NULL;
}
const char* //
do_test_mimic_brotli_encode_file(const char* name,
const uint8_t* data,
size_t len) {
for (uint32_t q = 0; q <= 11; q++) {
for (uint32_t w = 10; w <= 24; w += 14) {
brotli_enc_config c = make_brotli_enc_config(q, w);
c.chunking = (w == 10) ? BROTLI_ENC_CHUNK_SINGLE : BROTLI_ENC_CHUNK_RANDOM;
c.flush = (w == 10) ? BROTLI_ENC_FLUSH_NONE : BROTLI_ENC_FLUSH_MIDDLE;
c.seed = len;
char s[256];
snprintf(s, sizeof s, "%s: q=%u lgwin=%u", name, q, w);
CHECK_STRING(do_test_mimic_brotli_encode(s, &c, data, len));
}
}
return NULL;
}
const char* //
test_mimic_brotli_encode_parity_corpus() {
CHECK_FOCUS(__func__);
CHECK_STRING(
for_each_brotli_enc_corpus_file(do_test_mimic_brotli_encode_file));
uint8_t data[40];
brotli_enc_fill_random(data, 20, 0x9999);
memcpy(data + 20, data, 20);
for (size_t n = 0; n <= 40; n++) {
char s[64];
snprintf(s, sizeof s, "length %zu", n);
CHECK_STRING(do_test_mimic_brotli_encode_file(s, data, n));
}
for (int which = 0; which < 3; which++) {
size_t len = 0;
uint8_t* p = brotli_enc_synthetic(which, &len);
if (!p) {
RETURN_FAIL("malloc failed");
}
char s[64];
snprintf(s, sizeof s, "synthetic #%d", which);
const char* status = do_test_mimic_brotli_encode_file(s, p, len);
free(p);
if (status) {
return status;
}
}
return NULL;
}
#endif // WUFFS_MIMIC
// ---------------- Encoder Benches
const char* //
do_bench_brotli_encode(const char* (*codec_func)(wuffs_base__io_buffer*,
wuffs_base__io_buffer*,
const uint32_t*,
const uint32_t*,
size_t,
const uint32_t*,
const size_t*,
size_t,
size_t),
uint32_t quality,
uint64_t iters_unscaled) {
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
.data = g_src_slice_u8,
});
CHECK_STRING(read_file(&src, "test/data/brotli/alice29.txt"));
uint32_t keys[2] = {BROTLI_ENC_PARAM_QUALITY, BROTLI_ENC_PARAM_LGWIN};
uint32_t values[2] = {quality, 22};
uint32_t op = BROTLI_ENC_OP_FINISH;
size_t len = src.meta.wi;
bench_start();
uint64_t n_bytes = 0;
uint64_t i;
uint64_t iters = iters_unscaled * g_flags.iterscale;
for (i = 0; i < iters; i++) {
wuffs_base__io_buffer have = ((wuffs_base__io_buffer){
.data = g_have_slice_u8,
});
src.meta.ri = 0;
CHECK_STRING(codec_func(&have, &src, keys, values, 2, &op, &len, 1, 0));
n_bytes += len;
}
bench_finish(iters, n_bytes);
return NULL;
}
const char* //
bench_wuffs_brotli_encode_alice29_q0() {
CHECK_FOCUS(__func__);
return do_bench_brotli_encode(wuffs_brotli_encode_plan, 0, 30);
}
const char* //
bench_wuffs_brotli_encode_alice29_q1() {
CHECK_FOCUS(__func__);
return do_bench_brotli_encode(wuffs_brotli_encode_plan, 1, 30);
}
const char* //
bench_wuffs_brotli_encode_alice29_q5() {
CHECK_FOCUS(__func__);
return do_bench_brotli_encode(wuffs_brotli_encode_plan, 5, 10);
}
const char* //
bench_wuffs_brotli_encode_alice29_q9() {
CHECK_FOCUS(__func__);
return do_bench_brotli_encode(wuffs_brotli_encode_plan, 9, 3);
}
// ---------------- Encoder Mimic Benches
#ifdef WUFFS_MIMIC
const char* //
bench_mimic_brotli_encode_alice29_q0() {
CHECK_FOCUS(__func__);
return do_bench_brotli_encode(mimic_brotli_encode, 0, 30);
}
const char* //
bench_mimic_brotli_encode_alice29_q1() {
CHECK_FOCUS(__func__);
return do_bench_brotli_encode(mimic_brotli_encode, 1, 30);
}
const char* //
bench_mimic_brotli_encode_alice29_q5() {
CHECK_FOCUS(__func__);
return do_bench_brotli_encode(mimic_brotli_encode, 5, 10);
}
const char* //
bench_mimic_brotli_encode_alice29_q9() {
CHECK_FOCUS(__func__);
return do_bench_brotli_encode(mimic_brotli_encode, 9, 3);
}
#endif // WUFFS_MIMIC
// ---------------- Metablock Builder Tests
// These tests check the encoder's metablock builder (the ports of
// libbrotlienc's bit_cost.c, histogram.c, block_splitter.c, cluster.c and
// metablock.c, exposed through the test-only metablock_tester struct)
// against golden values computed with libbrotlienc v1.2.1. The much larger
// script/manual-test-brotli-metablock-parity.c compares them with the C
// functions directly.
static uint32_t //
mbt_log2_floor(uint64_t x) {
uint32_t n = 0;
while (x >>= 1) {
n++;
}
return n;
}
static uint64_t //
mbt_fnv(uint64_t h, const uint8_t* p, size_t n) {
size_t i;
for (i = 0; i < n; i++) {
h = (h ^ p[i]) * 0x100000001B3ull;
}
return h;
}
static uint64_t //
mbt_fnv_u64(uint64_t h, uint64_t v) {
uint8_t b[8];
int i;
for (i = 0; i < 8; i++) {
b[i] = (uint8_t)(v >> (8 * i));
}
return mbt_fnv(h, b, 8);
}
static void //
mbt_put_u32le(uint8_t* p, uint32_t v) {
p[0] = (uint8_t)(v >> 0);
p[1] = (uint8_t)(v >> 8);
p[2] = (uint8_t)(v >> 16);
p[3] = (uint8_t)(v >> 24);
}
// mbt_make_cmds writes the 16-byte commands (as command.h's InitCommand and
// InitInsertCommand would, for the given npostfix and ndirect) of a
// pseudo-random parse of total bytes to dst and returns how many there are.
// dst must have room for total + 1 commands.
static size_t //
mbt_make_cmds(uint8_t* dst, size_t total, uint32_t npostfix, uint32_t ndirect,
uint32_t seed) {
size_t n = 0;
size_t pos = 0;
uint32_t r = seed;
while (pos < total) {
size_t ins, copy, dcode, rem;
uint32_t inscode, copycode, dist_extra = 0;
uint16_t dist_prefix, cmd_prefix;
int use_last;
int insert_only = 0;
r = (r * 1103515245u) + 12345u;
ins = (r >> 16) % ((r & 0x300) ? 12 : 300);
r = (r * 1103515245u) + 12345u;
copy = 2 + ((r >> 16) % ((r & 0x700) ? 14 : 200));
rem = total - pos;
if (ins > rem) {
ins = rem;
}
if ((rem - ins) < copy) {
ins = rem;
copy = 0;
}
if (copy == 0) {
// InitInsertCommand.
insert_only = 1;
dist_prefix = 16;
copy = 4;
use_last = 0;
} else {
r = (r * 1103515245u) + 12345u;
if ((r & 0x30000) == 0) {
dcode = (r >> 18) & 15;
} else {
size_t max = pos + ins;
dcode = 16 + (((r >> 8) % (max ? max : 1)) % 3000);
}
// PrefixEncodeCopyDistance.
if (dcode < 16 + ndirect) {
dist_prefix = (uint16_t)dcode;
} else {
size_t d = ((size_t)1 << (npostfix + 2)) + (dcode - 16 - ndirect);
size_t bucket = mbt_log2_floor(d) - 1;
size_t postfix = d & ((1u << npostfix) - 1);
size_t prefix = (d >> bucket) & 1;
size_t offset = (2 + prefix) << bucket;
size_t nbits = bucket - npostfix;
dist_prefix = (uint16_t)((nbits << 10) |
(16 + ndirect +
((2 * (nbits - 1) + prefix) << npostfix) +
postfix));
dist_extra = (uint32_t)((d - offset) >> npostfix);
}
use_last = (dist_prefix & 0x3FF) == 0;
}
// GetInsertLengthCode.
if (ins < 6) {
inscode = (uint32_t)ins;
} else if (ins < 130) {
uint32_t nb = mbt_log2_floor(ins - 2) - 1;
inscode = (nb << 1) + (uint32_t)((ins - 2) >> nb) + 2;
} else if (ins < 2114) {
inscode = mbt_log2_floor(ins - 66) + 10;
} else if (ins < 6210) {
inscode = 21;
} else if (ins < 22594) {
inscode = 22;
} else {
inscode = 23;
}
// GetCopyLengthCode.
if (copy < 10) {
copycode = (uint32_t)(copy - 2);
} else if (copy < 134) {
uint32_t nb = mbt_log2_floor(copy - 6) - 1;
copycode = (nb << 1) + (uint32_t)((copy - 6) >> nb) + 4;
} else if (copy < 2118) {
copycode = mbt_log2_floor(copy - 70) + 12;
} else {
copycode = 23;
}
// CombineLengthCodes.
{
uint32_t bits64 = (copycode & 7) | ((inscode & 7) << 3);
if (use_last && (inscode < 8) && (copycode < 16)) {
cmd_prefix = (uint16_t)((copycode < 8) ? bits64 : (bits64 | 64));
} else {
uint32_t off = 2 * ((copycode >> 3) + 3 * (inscode >> 3));
off = (off << 5) + 0x40 + ((0x520D40u >> off) & 0xC0);
cmd_prefix = (uint16_t)(off | bits64);
}
}
mbt_put_u32le(dst + 16 * n + 0, (uint32_t)ins);
mbt_put_u32le(dst + 16 * n + 4, insert_only ? (4u << 25) : (uint32_t)copy);
mbt_put_u32le(dst + 16 * n + 8, dist_extra);
dst[16 * n + 12] = (uint8_t)cmd_prefix;
dst[16 * n + 13] = (uint8_t)(cmd_prefix >> 8);
dst[16 * n + 14] = (uint8_t)dist_prefix;
dst[16 * n + 15] = (uint8_t)(dist_prefix >> 8);
n++;
pos += ins + (insert_only ? 0 : copy);
}
return n;
}
typedef struct {
const char* filename; // NULL means the synthetic input.
uint32_t quality;
uint32_t mode;
uint32_t nolcm;
uint32_t npostfix;
uint32_t ndirect;
uint32_t large_window;
uint32_t seed;
uint64_t want_hash;
} mbt_case;
static const mbt_case g_mbt_cases[] = {
{"test/data/romeo.txt", 11, 2, 0, 0, 0, 0, 0x12345678,
0x91BCDB702326DA68},
{"test/data/midsummer.txt", 10, 3, 1, 1, 4, 0, 0x9ABCDEF0,
0xC1BC25672AFFE52C},
{NULL, 11, 0, 0, 2, 8, 1, 0x0BADF00D, 0x106C079EF30D8335},
{NULL, 10, 1, 0, 3, 120, 0, 0x00C0FFEE, 0x1CE5AB9FFAB4CEDB},
{"test/data/pi.txt", 11, 2, 0, 1, 2, 0, 0x31415926, 0x4434059D684D4F0E},
{"", 11, 2, 0, 0, 0, 0, 0x00000001, 0xAE70E50382B8388F},
};
const char* //
test_wuffs_brotli_metablock_build_meta_block() {
CHECK_FOCUS(__func__);
wuffs_brotli__metablock_tester* t = wuffs_brotli__metablock_tester__alloc();
if (!t) {
RETURN_FAIL("alloc failed");
}
const char* ret = NULL;
uint8_t* rb = NULL;
uint8_t* cmds = NULL;
uint8_t* wb = NULL;
size_t wb_len = 1 << 20;
size_t i;
for (i = 0; i < WUFFS_TESTLIB_ARRAY_SIZE(g_mbt_cases); i++) {
const mbt_case* tc = &g_mbt_cases[i];
size_t total = 0;
size_t rb_size = 1024;
size_t n, k;
int c;
uint64_t h = 0xCBF29CE484222325ull;
wuffs_base__status status;
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
.data = g_have_slice_u8,
});
if (!tc->filename) {
// A byte distribution that shifts every 700 bytes, 48 KiB.
uint32_t r = 1;
total = 49152;
for (k = 0; k < total; k++) {
r = (r * 1103515245u) + 12345u;
src.data.ptr[k] = (uint8_t)(((k / 700) * 37) +
((r >> 16) % (1 + (k / 700) % 23)));
}
} else if (*tc->filename) {
ret = read_file(&src, tc->filename);
if (ret) {
goto done;
}
total = src.meta.wi;
}
while (rb_size < total) {
rb_size *= 2;
}
free(rb);
free(cmds);
rb = calloc(rb_size, 1);
cmds = malloc(16 * (total + 1));
if (!rb || !cmds) {
ret = "out of memory";
goto done;
}
if (total) {
memcpy(rb, src.data.ptr, total);
}
n = mbt_make_cmds(cmds, total, tc->npostfix, tc->ndirect, tc->seed);
while (true) {
if (!wb) {
wb = malloc(wb_len);
if (!wb) {
ret = "out of memory";
goto done;
}
}
status = wuffs_brotli__metablock_tester__build_meta_block(
t, wuffs_base__make_slice_u8(wb, wb_len),
wuffs_base__make_slice_u8(rb, rb_size), 0, rb_size - 1, 0, 0,
wuffs_base__make_slice_u8(cmds, 16 * n), n, tc->mode, tc->quality,
tc->large_window, tc->nolcm, tc->npostfix, tc->ndirect);
if (status.repr != wuffs_base__error__bad_workbuf_length) {
break;
}
// The commands were rewritten, so regenerate them.
n = mbt_make_cmds(cmds, total, tc->npostfix, tc->ndirect, tc->seed);
free(wb);
wb = NULL;
wb_len *= 2;
}
if (status.repr) {
ret = status.repr;
goto done;
}
{
uint64_t ndc = wuffs_brotli__metablock_tester__get(t, 27);
wuffs_brotli__metablock_tester__optimize_histograms(
t, wuffs_base__make_slice_u8(wb, wb_len),
(uint32_t)((ndc < 544) ? ndc : 544));
}
for (c = 0; c < 3; c++) {
uint64_t num_blocks = wuffs_brotli__metablock_tester__get(t, 4 * c + 1);
uint64_t types = wuffs_brotli__metablock_tester__get(t, 4 * c + 2);
uint64_t lengths = wuffs_brotli__metablock_tester__get(t, 4 * c + 3);
h = mbt_fnv_u64(h, wuffs_brotli__metablock_tester__get(t, 4 * c + 0));
h = mbt_fnv_u64(h, num_blocks);
h = mbt_fnv(h, wb + types, num_blocks);
h = mbt_fnv(h, wb + lengths, 4 * num_blocks);
}
for (c = 0; c < 3; c += 2) {
uint64_t size = wuffs_brotli__metablock_tester__get(t, 15 + c);
h = mbt_fnv_u64(h, size);
h = mbt_fnv(h, wb + wuffs_brotli__metablock_tester__get(t, 12 + c),
4 * size);
}
for (c = 0; c < 3; c++) {
static const size_t hsizes[3] = {1040, 2832, 2192};
uint64_t size = wuffs_brotli__metablock_tester__get(t, 21 + c);
h = mbt_fnv_u64(h, size);
h = mbt_fnv(h, wb + wuffs_brotli__metablock_tester__get(t, 18 + c),
hsizes[c] * size);
}
for (c = 24; c < 29; c++) {
h = mbt_fnv_u64(h, wuffs_brotli__metablock_tester__get(t, c));
}
h = mbt_fnv(h, cmds, 16 * n);
if (h != tc->want_hash) {
RETURN_FAIL("case %zu: hash: have 0x%016" PRIX64 ", want 0x%016" PRIX64,
i, h, tc->want_hash);
}
if (wuffs_brotli__metablock_tester__get(t, 29) >
wuffs_brotli__metablock_tester__get(t, 30)) {
RETURN_FAIL("case %zu: workbuf used exceeds the worst case", i);
}
}
done:
free(rb);
free(cmds);
free(wb);
free(t);
return ret;
}
const char* //
test_wuffs_brotli_metablock_bit_cost() {
CHECK_FOCUS(__func__);
wuffs_brotli__metablock_tester* t = wuffs_brotli__metablock_tester__alloc();
if (!t) {
RETURN_FAIL("alloc failed");
}
// The C code's BrotliBitsEntropy and BrotliPopulationCost{Literal,Command,
// Distance} results, as binary64 bit patterns.
static const uint64_t want_pop[6][3] = {
{0x4028000000000000, 0x4028000000000000, 0x4028000000000000},
{0x4028000000000000, 0x4028000000000000, 0x4028000000000000},
{0x4055800000000000, 0x4055800000000000, 0x4055800000000000},
{0x4055800000000000, 0x406AC00000000000, 0x406AC00000000000},
{0x4055800000000000, 0x4076300000000000, 0x4076300000000000},
{0x40F842EE8C1F42B3, 0x4113DB811CA8812F, 0x410D76A1EA720A2C},
};
static const uint32_t data_sizes[3] = {256, 704, 544};
// Each histogram is data_size u32le counts, a u64le total_count_ and a
// binary64 bit_cost_.
uint8_t pop[4 * 704];
uint8_t histo[(4 * 704) + 16];
uint32_t counts[704];
size_t i;
int k, c;
for (i = 0; i < 704; i++) {
counts[i] = (uint32_t)((i * i * 7 + 3) % 101);
mbt_put_u32le(pop + 4 * i, counts[i]);
}
{
uint64_t have = wuffs_brotli__metablock_tester__bits_entropy(
t, wuffs_base__make_slice_u8(pop, sizeof(pop)), 256);
if (have != 0x40F79A0C92B03A26ull) {
free(t);
RETURN_FAIL("bits_entropy: have 0x%016" PRIX64, have);
}
}
for (k = 0; k < 6; k++) {
static const int nz[6] = {0, 1, 2, 3, 4, 300};
for (c = 0; c < 3; c++) {
uint64_t total = 0;
uint64_t have;
memset(histo, 0, sizeof(histo));
for (i = 0; i < data_sizes[c]; i++) {
int on = (nz[k] >= 300) ? (counts[i] != 0)
: (((i % 151) == 7) && ((int)(i / 151) < nz[k]));
if (on) {
mbt_put_u32le(histo + 4 * i, counts[i] + 1);
total += counts[i] + 1;
}
}
for (i = 0; i < 8; i++) {
histo[(4 * data_sizes[c]) + i] = (uint8_t)(total >> (8 * i));
}
have = wuffs_brotli__metablock_tester__population_cost(
t, wuffs_base__make_slice_u8(histo, (4 * data_sizes[c]) + 16),
data_sizes[c]);
if (have != want_pop[k][c]) {
free(t);
RETURN_FAIL("population_cost(k=%d, c=%d): have 0x%016" PRIX64
", want 0x%016" PRIX64,
k, c, have, want_pop[k][c]);
}
}
}
free(t);
return NULL;
}
// ---------------- Large Window and Dictionary Tests
// See test/data/brotli/README.md for where the large-window (lw), raw
// dictionary (raw) and serialized shared dictionary (sd) test files came from.
typedef struct {
uint32_t type; // WUFFS_BROTLI__DICTIONARY_TYPE_ETC.
const char* filename;
// lo and hi select the [lo .. hi) bytes of the file. A zero hi means the
// whole file.
size_t lo;
size_t hi;
} brotli_dict_spec;
#define BROTLI_DICT_SPECS_MAX 4
// g_brotli_dict_sets lists the sets of dictionaries (attached in order) that
// the test files use. A NULL filename terminates each set.
struct {
brotli_dict_spec specs[BROTLI_DICT_SPECS_MAX];
} g_brotli_dict_sets[] = {
#define BROTLI_DICT_SET_NONE 0
{{{0, NULL, 0, 0}}},
#define BROTLI_DICT_SET_RAW_ROMEO 1
{{{WUFFS_BROTLI__DICTIONARY_TYPE_RAW, "test/data/romeo.txt", 0, 0}}},
#define BROTLI_DICT_SET_RAW_ALICE_CHUNKS 2
{{
{WUFFS_BROTLI__DICTIONARY_TYPE_RAW, "test/data/brotli/alice29.txt", 0,
20000},
{WUFFS_BROTLI__DICTIONARY_TYPE_RAW, "test/data/brotli/alice29.txt",
20000, 20001},
{WUFFS_BROTLI__DICTIONARY_TYPE_RAW, "test/data/brotli/alice29.txt",
20001, 20001},
{WUFFS_BROTLI__DICTIONARY_TYPE_RAW, "test/data/brotli/alice29.txt",
20001, 60000},
}},
#define BROTLI_DICT_SET_RAW_ROMEO_CHUNKS 3
{{
{WUFFS_BROTLI__DICTIONARY_TYPE_RAW, "test/data/romeo.txt", 0, 300},
{WUFFS_BROTLI__DICTIONARY_TYPE_RAW, "test/data/romeo.txt", 300, 500},
{WUFFS_BROTLI__DICTIONARY_TYPE_RAW, "test/data/romeo.txt", 500, 942},
}},
#define BROTLI_DICT_SET_SD_FULL 4
{{{WUFFS_BROTLI__DICTIONARY_TYPE_SERIALIZED,
"test/data/brotli/shared-dictionary.full.bin", 0, 0}}},
#define BROTLI_DICT_SET_SD_WORDS 5
{{{WUFFS_BROTLI__DICTIONARY_TYPE_SERIALIZED,
"test/data/brotli/shared-dictionary.words.bin", 0, 0}}},
#define BROTLI_DICT_SET_SD_PREFIX_AND_RAW 6
{{
{WUFFS_BROTLI__DICTIONARY_TYPE_SERIALIZED,
"test/data/brotli/shared-dictionary.prefix.bin", 0, 0},
{WUFFS_BROTLI__DICTIONARY_TYPE_RAW, "test/data/romeo.txt", 0, 300},
}},
#define BROTLI_DICT_SET_SD_UTF8 7
{{{WUFFS_BROTLI__DICTIONARY_TYPE_SERIALIZED,
"test/data/brotli/shared-dictionary.utf8.bin", 0, 0}}},
};
// g_brotli_ext_vectors lists (compressed, decompressed) pairs of test files
// that need the large-window quirk or attached dictionaries.
struct {
const char* src_filename;
const char* want_filename;
bool large_window;
size_t dict_set;
} g_brotli_ext_vectors[] = {
{"test/data/brotli/alice29.txt.q11lw26.br", "test/data/brotli/alice29.txt",
true, BROTLI_DICT_SET_NONE},
{"test/data/brotli/alice29.txt.q5lw10.br", "test/data/brotli/alice29.txt",
true, BROTLI_DICT_SET_NONE},
{"test/data/brotli/romeo.txt.q11lw30.br", "test/data/romeo.txt", true,
BROTLI_DICT_SET_NONE},
{"test/data/brotli/romeo.txt.q9lw28.br", "test/data/romeo.txt", true,
BROTLI_DICT_SET_NONE},
{"test/data/brotli/romeo.txt.q11w22.raw-romeo.br", "test/data/romeo.txt",
false, BROTLI_DICT_SET_RAW_ROMEO},
{"test/data/brotli/romeo.txt.q11lw30.raw-romeo.br", "test/data/romeo.txt",
true, BROTLI_DICT_SET_RAW_ROMEO},
{"test/data/brotli/alice29.txt.q5w18.raw-chunks.br",
"test/data/brotli/alice29.txt", false, BROTLI_DICT_SET_RAW_ALICE_CHUNKS},
{"test/data/brotli/alice29.txt.q11w18.raw-chunks.br",
"test/data/brotli/alice29.txt", false, BROTLI_DICT_SET_RAW_ALICE_CHUNKS},
{"test/data/brotli/romeo.txt.q11w16.raw-chunks.br", "test/data/romeo.txt",
false, BROTLI_DICT_SET_RAW_ROMEO_CHUNKS},
{"test/data/brotli/shared-dictionary.text.q5w18.sd-full.br",
"test/data/brotli/shared-dictionary.text", false, BROTLI_DICT_SET_SD_FULL},
{"test/data/brotli/shared-dictionary.text.q11w18.sd-full.br",
"test/data/brotli/shared-dictionary.text", false, BROTLI_DICT_SET_SD_FULL},
{"test/data/brotli/shared-dictionary.text.q11lw24.sd-full.br",
"test/data/brotli/shared-dictionary.text", true, BROTLI_DICT_SET_SD_FULL},
{"test/data/brotli/alice29.txt.q11w22.sd-words.br",
"test/data/brotli/alice29.txt", false, BROTLI_DICT_SET_SD_WORDS},
{"test/data/brotli/asyoulik.txt.q11w18.sd-prefix-raw.br",
"test/data/brotli/asyoulik.txt", false, BROTLI_DICT_SET_SD_PREFIX_AND_RAW},
{"test/data/brotli/romeo-mutated.sd-utf8.br",
"test/data/brotli/romeo-mutated.sd-utf8.txt", false,
BROTLI_DICT_SET_SD_UTF8},
};
// g_brotli_ext_malformed is like g_brotli_malformed but for inputs that need
// the large-window quirk or attached dictionaries. The invalid.lw-*,
// invalid.raw-* and invalid.sd-* files were found by mutating the
// g_brotli_ext_vectors files (see test/data/brotli/README.md).
struct {
const char* src_filename;
const char* data;
size_t len;
bool large_window;
size_t dict_set;
const char* want_status;
} g_brotli_ext_malformed[] = {
// BROTLI_DECODER_ERROR_FORMAT_WINDOW_BITS: the large-window escape
// followed by a non-zero reserved bit.
{NULL, "\x91", 1, true, BROTLI_DICT_SET_NONE,
wuffs_brotli__error__bad_window_bits},
// BROTLI_DECODER_ERROR_FORMAT_WINDOW_BITS: large-window WBITS of 9 (too
// small) and 62 (too large).
{NULL, "\x11\x09", 2, true, BROTLI_DICT_SET_NONE,
wuffs_brotli__error__bad_window_bits},
{NULL, "\x11\x3E", 2, true, BROTLI_DICT_SET_NONE,
wuffs_brotli__error__bad_window_bits},
// BROTLI_DECODER_NEEDS_MORE_INPUT: a truncated large-window header.
{NULL, "\x11", 1, true, BROTLI_DICT_SET_NONE,
wuffs_brotli__error__truncated_input},
// BROTLI_DECODER_ERROR_FORMAT_WINDOW_BITS: a large-window stream without
// the large-window quirk.
{"test/data/brotli/romeo.txt.q11lw30.br", NULL, 0, false,
BROTLI_DICT_SET_NONE, wuffs_brotli__error__unsupported_large_window},
// BROTLI_DECODER_ERROR_FORMAT_DISTANCE.
{"test/data/brotli/invalid.lw-distance.br", NULL, 0, true,
BROTLI_DICT_SET_NONE, wuffs_brotli__error__bad_distance},
// BROTLI_DECODER_ERROR_FORMAT_DICTIONARY.
{"test/data/brotli/invalid.lw-dictionary.br", NULL, 0, true,
BROTLI_DICT_SET_NONE, wuffs_brotli__error__bad_dictionary},
// BROTLI_DECODER_ERROR_COMPOUND_DICTIONARY.
{"test/data/brotli/invalid.raw-compound-dictionary.br", NULL, 0, false,
BROTLI_DICT_SET_RAW_ROMEO_CHUNKS,
wuffs_brotli__error__bad_compound_dictionary},
// BROTLI_DECODER_ERROR_FORMAT_DICTIONARY.
{"test/data/brotli/invalid.raw-dictionary.br", NULL, 0, false,
BROTLI_DICT_SET_RAW_ROMEO_CHUNKS, wuffs_brotli__error__bad_dictionary},
// BROTLI_DECODER_ERROR_FORMAT_TRANSFORM.
{"test/data/brotli/invalid.raw-transform.br", NULL, 0, false,
BROTLI_DICT_SET_RAW_ROMEO_CHUNKS, wuffs_brotli__error__bad_transform},
// BROTLI_DECODER_ERROR_COMPOUND_DICTIONARY: a stream compressed without
// a dictionary refers to the static dictionary, but with a compound
// dictionary attached, the same distance refers to the compound
// dictionary.
{"test/data/brotli/romeo.txt.q11w22.br", NULL, 0, false,
BROTLI_DICT_SET_SD_FULL, wuffs_brotli__error__bad_compound_dictionary},
// BROTLI_DECODER_ERROR_COMPOUND_DICTIONARY.
{"test/data/brotli/invalid.sd-compound-dictionary.br", NULL, 0, false,
BROTLI_DICT_SET_SD_FULL, wuffs_brotli__error__bad_compound_dictionary},
// BROTLI_DECODER_ERROR_FORMAT_DICTIONARY.
{"test/data/brotli/invalid.sd-dictionary.br", NULL, 0, false,
BROTLI_DICT_SET_SD_FULL, wuffs_brotli__error__bad_dictionary},
// BROTLI_DECODER_ERROR_FORMAT_TRANSFORM.
{"test/data/brotli/invalid.sd-transform.br", NULL, 0, false,
BROTLI_DICT_SET_SD_FULL, wuffs_brotli__error__bad_transform},
{"test/data/brotli/invalid.lw-sd-transform.br", NULL, 0, true,
BROTLI_DICT_SET_SD_FULL, wuffs_brotli__error__bad_transform},
};
// g_brotli_dict_arrays holds the dictionaries' bytes.
uint8_t g_brotli_dict_arrays[BROTLI_DICT_SPECS_MAX][256 * 1024];
// load_brotli_dict_set loads the dict_set'th g_brotli_dict_sets entry.
const char* //
load_brotli_dict_set(size_t dict_set,
size_t* num_dicts,
uint32_t* dict_types,
wuffs_base__slice_u8* dicts) {
*num_dicts = 0;
if (dict_set >= WUFFS_TESTLIB_ARRAY_SIZE(g_brotli_dict_sets)) {
return "load_brotli_dict_set: dict_set out of range";
}
for (size_t i = 0; i < BROTLI_DICT_SPECS_MAX; i++) {
const brotli_dict_spec* spec = &g_brotli_dict_sets[dict_set].specs[i];
if (!spec->filename) {
break;
}
wuffs_base__io_buffer buf = ((wuffs_base__io_buffer){
.data = wuffs_base__make_slice_u8(g_brotli_dict_arrays[i],
sizeof g_brotli_dict_arrays[i]),
});
CHECK_STRING(read_file(&buf, spec->filename));
size_t hi = spec->hi ? spec->hi : buf.meta.wi;
if ((spec->lo > hi) || (hi > buf.meta.wi)) {
return "load_brotli_dict_set: bad lo/hi";
}
dict_types[i] = spec->type;
dicts[i] = wuffs_base__make_slice_u8(buf.data.ptr + spec->lo,
hi - spec->lo);
*num_dicts = i + 1;
}
return NULL;
}
// get_brotli_big_workbuf sets *workbuf to a malloc'ed buffer that is large
// enough for any large-window stream and 1 MiB of dictionaries. It is never
// freed. Only the parts of it that the decoder uses are touched, so the
// operating system should only need to commit as much memory as the stream's
// window size.
const char* //
get_brotli_big_workbuf(wuffs_base__slice_u8* workbuf) {
static uint8_t* ptr = NULL;
const size_t n = WUFFS_BROTLI__DECODER_WORKBUF_LEN_LARGE_WINDOW + (1 << 20);
if (!ptr) {
ptr = (uint8_t*)malloc(n);
if (!ptr) {
return "get_brotli_big_workbuf: out of memory";
}
}
*workbuf = wuffs_base__make_slice_u8(ptr, n);
return NULL;
}
// wuffs_brotli_decode_with_options is like wuffs_brotli_decode but it can
// also set the large-window quirk and attach dictionaries.
const char* //
wuffs_brotli_decode_with_options(wuffs_base__io_buffer* dst,
wuffs_base__io_buffer* src,
uint64_t wlimit,
uint64_t rlimit,
bool large_window,
size_t dict_set) {
size_t num_dicts = 0;
uint32_t dict_types[BROTLI_DICT_SPECS_MAX] = {0};
wuffs_base__slice_u8 dicts[BROTLI_DICT_SPECS_MAX] = {0};
CHECK_STRING(load_brotli_dict_set(dict_set, &num_dicts, dict_types, dicts));
wuffs_brotli__decoder dec;
CHECK_STATUS("initialize",
wuffs_brotli__decoder__initialize(
&dec, sizeof dec, WUFFS_VERSION,
WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED));
// The large-window workbuf_len (more than 1 GiB) is larger than
// g_work_slice_u8.
wuffs_base__slice_u8 workbuf = g_work_slice_u8;
if (large_window) {
CHECK_STATUS("set_quirk",
wuffs_brotli__decoder__set_quirk(
&dec, WUFFS_BROTLI__QUIRK_LARGE_WINDOW, 1));
CHECK_STRING(get_brotli_big_workbuf(&workbuf));
}
for (size_t i = 0; i < num_dicts; i++) {
CHECK_STATUS("add_dictionary",
wuffs_brotli__decoder__add_dictionary(
&dec, dict_types[i], dicts[i], workbuf));
}
wuffs_base__range_ii_u64 workbuf_len =
wuffs_brotli__decoder__workbuf_len(&dec);
if (workbuf_len.max_incl > workbuf.len) {
return "workbuf_len is too large";
}
workbuf.len = workbuf_len.max_incl;
while (true) {
wuffs_base__io_buffer limited_dst = make_limited_writer(*dst, wlimit);
wuffs_base__io_buffer limited_src = make_limited_reader(*src, rlimit);
wuffs_base__status status = wuffs_brotli__decoder__transform_io(
&dec, &limited_dst, &limited_src, workbuf);
dst->meta.wi += limited_dst.meta.wi;
src->meta.ri += limited_src.meta.ri;
if (((wlimit < UINT64_MAX) &&
(status.repr == wuffs_base__suspension__short_write)) ||
((rlimit < UINT64_MAX) &&
(status.repr == wuffs_base__suspension__short_read))) {
continue;
}
return status.repr;
}
}
const char* //
do_test_brotli_decode_ext_all(uint64_t wlimit, uint64_t rlimit) {
size_t n = WUFFS_TESTLIB_ARRAY_SIZE(g_brotli_ext_vectors);
for (size_t i = 0; i < n; i++) {
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
.data = g_src_slice_u8,
});
wuffs_base__io_buffer have = ((wuffs_base__io_buffer){
.data = g_have_slice_u8,
});
wuffs_base__io_buffer want = ((wuffs_base__io_buffer){
.data = g_want_slice_u8,
});
CHECK_STRING(read_file(&src, g_brotli_ext_vectors[i].src_filename));
CHECK_STRING(read_file(&want, g_brotli_ext_vectors[i].want_filename));
const char* status = wuffs_brotli_decode_with_options(
&have, &src, wlimit, rlimit, g_brotli_ext_vectors[i].large_window,
g_brotli_ext_vectors[i].dict_set);
if (status) {
RETURN_FAIL("%s: %s", g_brotli_ext_vectors[i].src_filename, status);
}
status = check_io_buffers_equal("", &have, &want);
if (status) {
RETURN_FAIL("%s: %s", g_brotli_ext_vectors[i].src_filename, status);
}
if (src.meta.ri != src.meta.wi) {
RETURN_FAIL("%s: ri: have %zu, want %zu",
g_brotli_ext_vectors[i].src_filename, src.meta.ri,
src.meta.wi);
}
}
return NULL;
}
const char* //
test_wuffs_brotli_decode_ext_all() {
CHECK_FOCUS(__func__);
return do_test_brotli_decode_ext_all(UINT64_MAX, UINT64_MAX);
}
const char* //
test_wuffs_brotli_decode_ext_all_with_io_limits() {
CHECK_FOCUS(__func__);
return do_test_brotli_decode_ext_all(1, 1);
}
// make_brotli_far_data writes the decompressed form of
// test/data/brotli/far17m.q9lw25.br: 64 KiB of xorshift32 output, 17 MiB of
// zeroes and the same 64 KiB again. Its second 64 KiB copy is more than
// (1 << 24) bytes after the first.
const char* //
make_brotli_far_data(wuffs_base__io_buffer* dst) {
const size_t a = 65536;
const size_t z = 17 << 20;
if ((dst->data.len - dst->meta.wi) < (a + z + a)) {
return "make_brotli_far_data: dst is too short";
}
uint8_t* p = dst->data.ptr + dst->meta.wi;
uint32_t x = 0x12345678;
for (size_t i = 0; i < a; i++) {
x ^= x << 13;
x ^= x >> 17;
x ^= x << 5;
p[i] = (uint8_t)(x >> 24);
}
memset(p + a, 0, z);
memcpy(p + a + z, p, a);
dst->meta.wi += a + z + a;
return NULL;
}
const char* //
do_test_brotli_decode_large_window_far(
const char* (*codec_func)(wuffs_base__io_buffer*,
wuffs_base__io_buffer*,
uint64_t,
uint64_t,
bool,
size_t),
uint64_t wlimit,
uint64_t rlimit) {
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
.data = g_src_slice_u8,
});
wuffs_base__io_buffer have = ((wuffs_base__io_buffer){
.data = g_have_slice_u8,
});
wuffs_base__io_buffer want = ((wuffs_base__io_buffer){
.data = g_want_slice_u8,
});
CHECK_STRING(read_file(&src, "test/data/brotli/far17m.q9lw25.br"));
CHECK_STRING(make_brotli_far_data(&want));
CHECK_STRING(
(*codec_func)(&have, &src, wlimit, rlimit, true, BROTLI_DICT_SET_NONE));
return check_io_buffers_equal("", &have, &want);
}
const char* //
test_wuffs_brotli_decode_large_window_far() {
CHECK_FOCUS(__func__);
CHECK_STRING(do_test_brotli_decode_large_window_far(
wuffs_brotli_decode_with_options, UINT64_MAX, UINT64_MAX));
return do_test_brotli_decode_large_window_far(
wuffs_brotli_decode_with_options, 4099, 113);
}
const char* //
test_wuffs_brotli_decode_large_window_quirk() {
CHECK_FOCUS(__func__);
// With the quirk set, standard (not large-window) streams still decode.
size_t n = WUFFS_TESTLIB_ARRAY_SIZE(g_brotli_vectors);
for (size_t i = 0; i < n; i++) {
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
.data = g_src_slice_u8,
});
wuffs_base__io_buffer have = ((wuffs_base__io_buffer){
.data = g_have_slice_u8,
});
wuffs_base__io_buffer want = ((wuffs_base__io_buffer){
.data = g_want_slice_u8,
});
CHECK_STRING(read_file(&src, g_brotli_vectors[i].src_filename));
CHECK_STRING(read_file(&want, g_brotli_vectors[i].want_filename));
const char* status = wuffs_brotli_decode_with_options(
&have, &src, UINT64_MAX, UINT64_MAX, true, BROTLI_DICT_SET_NONE);
if (status) {
RETURN_FAIL("%s: %s", g_brotli_vectors[i].src_filename, status);
}
CHECK_STRING(check_io_buffers_equal("", &have, &want));
}
// The quirk's value and the workbuf_len.
wuffs_brotli__decoder dec;
CHECK_STATUS("initialize", wuffs_brotli__decoder__initialize(
&dec, sizeof dec, WUFFS_VERSION,
WUFFS_INITIALIZE__DEFAULT_OPTIONS));
if (wuffs_brotli__decoder__get_quirk(&dec,
WUFFS_BROTLI__QUIRK_LARGE_WINDOW) != 0) {
RETURN_FAIL("get_quirk #0: have non-zero, want zero");
}
wuffs_base__range_ii_u64 r = wuffs_brotli__decoder__workbuf_len(&dec);
if ((r.min_incl != WUFFS_BROTLI__DECODER_WORKBUF_LEN_STANDARD) ||
(r.max_incl != WUFFS_BROTLI__DECODER_WORKBUF_LEN_STANDARD) ||
(r.max_incl != ((1 << 24) + 542))) {
RETURN_FAIL("workbuf_len #0: have %" PRIu64, r.max_incl);
}
CHECK_STATUS("set_quirk", wuffs_brotli__decoder__set_quirk(
&dec, WUFFS_BROTLI__QUIRK_LARGE_WINDOW, 1));
if (wuffs_brotli__decoder__get_quirk(&dec,
WUFFS_BROTLI__QUIRK_LARGE_WINDOW) != 1) {
RETURN_FAIL("get_quirk #1: have zero, want non-zero");
}
r = wuffs_brotli__decoder__workbuf_len(&dec);
if ((r.min_incl != r.max_incl) ||
(r.max_incl != WUFFS_BROTLI__DECODER_WORKBUF_LEN_LARGE_WINDOW) ||
(r.max_incl != ((1 << 30) + 542))) {
RETURN_FAIL("workbuf_len #1: have %" PRIu64, r.max_incl);
}
// Quirks cannot be set after decoding has started.
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
.data = g_src_slice_u8,
});
wuffs_base__io_buffer have = ((wuffs_base__io_buffer){
.data = g_have_slice_u8,
});
CHECK_STRING(read_file(&src, "test/data/brotli/alice29.txt.q5lw10.br"));
src.meta.wi = 3;
src.meta.closed = false;
wuffs_base__slice_u8 workbuf = g_work_slice_u8;
wuffs_base__status status =
wuffs_brotli__decoder__transform_io(&dec, &have, &src, workbuf);
if (status.repr != wuffs_base__suspension__short_read) {
RETURN_FAIL("transform_io: have \"%s\", want \"%s\"", status.repr,
wuffs_base__suspension__short_read);
}
status = wuffs_brotli__decoder__set_quirk(
&dec, WUFFS_BROTLI__QUIRK_LARGE_WINDOW, 0);
if (status.repr != wuffs_base__error__bad_call_sequence) {
RETURN_FAIL("set_quirk: have \"%s\", want \"%s\"", status.repr,
wuffs_base__error__bad_call_sequence);
}
return NULL;
}
const char* //
test_wuffs_brotli_decode_ext_malformed() {
CHECK_FOCUS(__func__);
size_t n = WUFFS_TESTLIB_ARRAY_SIZE(g_brotli_ext_malformed);
for (size_t i = 0; i < n; i++) {
for (int j = 0; j < 2; j++) {
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
.data = g_src_slice_u8,
});
wuffs_base__io_buffer have = ((wuffs_base__io_buffer){
.data = g_have_slice_u8,
});
if (g_brotli_ext_malformed[i].src_filename) {
CHECK_STRING(read_file(&src, g_brotli_ext_malformed[i].src_filename));
} else {
memcpy(src.data.ptr, g_brotli_ext_malformed[i].data,
g_brotli_ext_malformed[i].len);
src.meta.wi = g_brotli_ext_malformed[i].len;
src.meta.closed = true;
}
uint64_t limit = (j == 0) ? UINT64_MAX : 1;
const char* have_status = wuffs_brotli_decode_with_options(
&have, &src, limit, limit, g_brotli_ext_malformed[i].large_window,
g_brotli_ext_malformed[i].dict_set);
const char* want_status = g_brotli_ext_malformed[i].want_status;
if (have_status != want_status) {
RETURN_FAIL("i=%zu, j=%d: have \"%s\", want \"%s\"", i, j, have_status,
want_status);
}
}
}
return NULL;
}
const char* //
test_wuffs_brotli_add_dictionary() {
CHECK_FOCUS(__func__);
wuffs_base__io_buffer full = ((wuffs_base__io_buffer){
.data = wuffs_base__make_slice_u8(g_brotli_dict_arrays[0],
sizeof g_brotli_dict_arrays[0]),
});
wuffs_base__io_buffer words = ((wuffs_base__io_buffer){
.data = wuffs_base__make_slice_u8(g_brotli_dict_arrays[1],
sizeof g_brotli_dict_arrays[1]),
});
wuffs_base__io_buffer prefix = ((wuffs_base__io_buffer){
.data = wuffs_base__make_slice_u8(g_brotli_dict_arrays[2],
sizeof g_brotli_dict_arrays[2]),
});
CHECK_STRING(
read_file(&full, "test/data/brotli/shared-dictionary.full.bin"));
CHECK_STRING(
read_file(&words, "test/data/brotli/shared-dictionary.words.bin"));
CHECK_STRING(
read_file(&prefix, "test/data/brotli/shared-dictionary.prefix.bin"));
wuffs_base__slice_u8 full_s =
wuffs_base__make_slice_u8(full.data.ptr, full.meta.wi);
wuffs_base__slice_u8 words_s =
wuffs_base__make_slice_u8(words.data.ptr, words.meta.wi);
wuffs_base__slice_u8 prefix_s =
wuffs_base__make_slice_u8(prefix.data.ptr, prefix.meta.wi);
wuffs_base__slice_u8 raw_s =
wuffs_base__make_slice_u8(prefix.data.ptr, 100);
wuffs_base__slice_u8 workbuf = g_work_slice_u8;
wuffs_brotli__decoder dec;
wuffs_base__status status;
// An unknown dictionary type.
CHECK_STATUS("initialize", wuffs_brotli__decoder__initialize(
&dec, sizeof dec, WUFFS_VERSION,
WUFFS_INITIALIZE__DEFAULT_OPTIONS));
status = wuffs_brotli__decoder__add_dictionary(&dec, 2, raw_s, workbuf);
if (status.repr != wuffs_base__error__bad_argument) {
RETURN_FAIL("type 2: have \"%s\", want \"%s\"", status.repr,
wuffs_base__error__bad_argument);
}
// At most 15 compound (raw or prefix) dictionaries. The workbuf_len grows
// with each one.
for (int i = 0; i < 15; i++) {
CHECK_STATUS("add_dictionary #0",
wuffs_brotli__decoder__add_dictionary(
&dec, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, raw_s, workbuf));
wuffs_base__range_ii_u64 r = wuffs_brotli__decoder__workbuf_len(&dec);
uint64_t want = WUFFS_BROTLI__DECODER_WORKBUF_LEN_STANDARD +
(100 * ((uint64_t)i + 1));
if ((r.min_incl != want) || (r.max_incl != want)) {
RETURN_FAIL("workbuf_len: have %" PRIu64 ", want %" PRIu64, r.max_incl,
want);
}
}
status = wuffs_brotli__decoder__add_dictionary(
&dec, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, raw_s, workbuf);
if (status.repr != wuffs_brotli__error__too_many_compound_dictionaries) {
RETURN_FAIL("16th raw: have \"%s\", want \"%s\"", status.repr,
wuffs_brotli__error__too_many_compound_dictionaries);
}
status = wuffs_brotli__decoder__add_dictionary(
&dec, WUFFS_BROTLI__DICTIONARY_TYPE_SERIALIZED, prefix_s, workbuf);
if (status.repr != wuffs_brotli__error__too_many_compound_dictionaries) {
RETURN_FAIL("16th prefix: have \"%s\", want \"%s\"", status.repr,
wuffs_brotli__error__too_many_compound_dictionaries);
}
// A serialized dictionary without a prefix is still OK.
CHECK_STATUS("add_dictionary #1",
wuffs_brotli__decoder__add_dictionary(
&dec, WUFFS_BROTLI__DICTIONARY_TYPE_SERIALIZED, words_s,
workbuf));
// Only one serialized dictionary can have custom word or transform lists,
// but a serialized dictionary without them (only a prefix) is OK.
CHECK_STATUS("initialize", wuffs_brotli__decoder__initialize(
&dec, sizeof dec, WUFFS_VERSION,
WUFFS_INITIALIZE__DEFAULT_OPTIONS));
CHECK_STATUS("add_dictionary #2",
wuffs_brotli__decoder__add_dictionary(
&dec, WUFFS_BROTLI__DICTIONARY_TYPE_SERIALIZED, full_s,
workbuf));
status = wuffs_brotli__decoder__add_dictionary(
&dec, WUFFS_BROTLI__DICTIONARY_TYPE_SERIALIZED, words_s, workbuf);
if (status.repr !=
wuffs_brotli__error__unsupported_multiple_custom_static_dictionaries) {
RETURN_FAIL(
"words after full: have \"%s\", want \"%s\"", status.repr,
wuffs_brotli__error__unsupported_multiple_custom_static_dictionaries);
}
CHECK_STATUS("add_dictionary #3",
wuffs_brotli__decoder__add_dictionary(
&dec, WUFFS_BROTLI__DICTIONARY_TYPE_SERIALIZED, prefix_s,
workbuf));
// Attaching a prefix-only serialized dictionary resets the static
// dictionary to the built-in one, so custom lists can be attached again.
CHECK_STATUS("add_dictionary #4",
wuffs_brotli__decoder__add_dictionary(
&dec, WUFFS_BROTLI__DICTIONARY_TYPE_SERIALIZED, words_s,
workbuf));
// Invalid serialized dictionaries: every proper prefix of a valid one, and
// a bad magic number. A failed add_dictionary does not change the
// workbuf_len.
CHECK_STATUS("initialize", wuffs_brotli__decoder__initialize(
&dec, sizeof dec, WUFFS_VERSION,
WUFFS_INITIALIZE__DEFAULT_OPTIONS));
for (size_t i = 0; i < full_s.len; i++) {
status = wuffs_brotli__decoder__add_dictionary(
&dec, WUFFS_BROTLI__DICTIONARY_TYPE_SERIALIZED,
wuffs_base__make_slice_u8(full_s.ptr, i), workbuf);
if (status.repr != wuffs_brotli__error__bad_shared_dictionary) {
RETURN_FAIL("prefix length %zu: have \"%s\", want \"%s\"", i,
status.repr, wuffs_brotli__error__bad_shared_dictionary);
}
}
full_s.ptr[0] ^= 1;
status = wuffs_brotli__decoder__add_dictionary(
&dec, WUFFS_BROTLI__DICTIONARY_TYPE_SERIALIZED, full_s, workbuf);
full_s.ptr[0] ^= 1;
if (status.repr != wuffs_brotli__error__bad_shared_dictionary) {
RETURN_FAIL("bad magic: have \"%s\", want \"%s\"", status.repr,
wuffs_brotli__error__bad_shared_dictionary);
}
if (wuffs_brotli__decoder__workbuf_len(&dec).max_incl !=
WUFFS_BROTLI__DECODER_WORKBUF_LEN_STANDARD) {
RETURN_FAIL("workbuf_len changed");
}
// A too-short workbuf.
status = wuffs_brotli__decoder__add_dictionary(
&dec, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, raw_s,
wuffs_base__make_slice_u8(workbuf.ptr, 99));
if (status.repr != wuffs_base__error__bad_workbuf_length) {
RETURN_FAIL("short workbuf: have \"%s\", want \"%s\"", status.repr,
wuffs_base__error__bad_workbuf_length);
}
CHECK_STATUS("add_dictionary #5",
wuffs_brotli__decoder__add_dictionary(
&dec, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, raw_s,
wuffs_base__make_slice_u8(workbuf.ptr, 100)));
// A transform_io workbuf that is shorter than the dictionaries.
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
.data = g_src_slice_u8,
});
wuffs_base__io_buffer have = ((wuffs_base__io_buffer){
.data = g_have_slice_u8,
});
CHECK_STRING(read_file(&src, "test/data/brotli/romeo.txt.q11w22.br"));
status = wuffs_brotli__decoder__transform_io(
&dec, &have, &src, wuffs_base__make_slice_u8(workbuf.ptr, 99));
if (status.repr != wuffs_base__error__bad_workbuf_length) {
RETURN_FAIL("transform_io: have \"%s\", want \"%s\"", status.repr,
wuffs_base__error__bad_workbuf_length);
}
// Dictionaries cannot be added after decoding has started.
CHECK_STATUS("initialize", wuffs_brotli__decoder__initialize(
&dec, sizeof dec, WUFFS_VERSION,
WUFFS_INITIALIZE__DEFAULT_OPTIONS));
src.meta.ri = 0;
src.meta.wi = 0;
src.meta.closed = false;
// With no input, decoding has not started (the C code is still in its
// BROTLI_STATE_UNINITED state).
status = wuffs_brotli__decoder__transform_io(&dec, &have, &src, workbuf);
if (status.repr != wuffs_base__suspension__short_read) {
RETURN_FAIL("transform_io #0: have \"%s\", want \"%s\"", status.repr,
wuffs_base__suspension__short_read);
}
CHECK_STATUS("add_dictionary #6",
wuffs_brotli__decoder__add_dictionary(
&dec, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, raw_s, workbuf));
CHECK_STRING(read_file(&src, "test/data/brotli/romeo.txt.q11w22.br"));
src.meta.wi = 1;
src.meta.closed = false;
status = wuffs_brotli__decoder__transform_io(&dec, &have, &src, workbuf);
if (status.repr != wuffs_base__suspension__short_read) {
RETURN_FAIL("transform_io #1: have \"%s\", want \"%s\"", status.repr,
wuffs_base__suspension__short_read);
}
status = wuffs_brotli__decoder__add_dictionary(
&dec, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, raw_s, workbuf);
if (status.repr != wuffs_base__error__bad_call_sequence) {
RETURN_FAIL("after start: have \"%s\", want \"%s\"", status.repr,
wuffs_base__error__bad_call_sequence);
}
return NULL;
}
// ---------------- Large Window and Dictionary Mimic Tests
#ifdef WUFFS_MIMIC
// do_test_mimic_brotli_differential_with_options is like
// do_test_mimic_brotli_differential but with the large-window quirk and
// attached dictionaries. It sets *skipped (and returns NULL) if libbrotlidec
// cannot attach the dictionaries: by default, it does not support serialized
// dictionaries.
const char* //
do_test_mimic_brotli_differential_with_options(const char* name,
wuffs_base__io_buffer* src,
const char* want_filename,
bool want_error,
bool large_window,
size_t dict_set,
bool* skipped) {
*skipped = false;
wuffs_base__io_buffer have = ((wuffs_base__io_buffer){
.data = g_have_slice_u8,
});
wuffs_base__io_buffer want = ((wuffs_base__io_buffer){
.data = g_want_slice_u8,
});
wuffs_base__io_buffer src0 = *src;
const char* have_status = wuffs_brotli_decode_with_options(
&have, &src0, UINT64_MAX, UINT64_MAX, large_window, dict_set);
size_t num_dicts = 0;
uint32_t dict_types[BROTLI_DICT_SPECS_MAX] = {0};
wuffs_base__slice_u8 dicts[BROTLI_DICT_SPECS_MAX] = {0};
CHECK_STRING(load_brotli_dict_set(dict_set, &num_dicts, dict_types, dicts));
wuffs_base__io_buffer src1 = *src;
int error_code = 0;
const char* want_status = mimic_brotli_decode_with_options(
&want, &src1, large_window, num_dicts, dict_types, dicts, &error_code);
if (want_status == mimic_brotli_attach_dictionary_failed_message) {
*skipped = true;
return NULL;
}
if ((have_status == NULL) != (want_status == NULL)) {
RETURN_FAIL("%s: have \"%s\", want \"%s\" (error code %d)", name,
have_status, want_status, error_code);
} else if ((have_status != NULL) &&
(brotli_c_error_code(have_status) != error_code)) {
RETURN_FAIL("%s: have \"%s\", want error code %d", name, have_status,
error_code);
} else if ((have_status != NULL) != want_error) {
RETURN_FAIL("%s: have \"%s\", want_error %d", name, have_status,
(int)want_error);
} else if (!want_error && (src0.meta.ri != src1.meta.ri)) {
RETURN_FAIL("%s: ri: have %zu, want %zu", name, src0.meta.ri, src1.meta.ri);
}
if (want_error) {
if (have.meta.wi > want.meta.wi) {
have.meta.wi = want.meta.wi;
} else {
want.meta.wi = have.meta.wi;
}
}
const char* status = check_io_buffers_equal("", &have, &want);
if (status) {
RETURN_FAIL("%s: %s", name, status);
}
// Also check the golden file.
if (want_filename) {
wuffs_base__io_buffer golden = ((wuffs_base__io_buffer){
.data = g_want_slice_u8,
});
CHECK_STRING(read_file(&golden, want_filename));
status = check_io_buffers_equal("", &have, &golden);
if (status) {
RETURN_FAIL("%s: golden: %s", name, status);
}
}
return NULL;
}
const char* //
test_mimic_brotli_decode_ext_differential() {
CHECK_FOCUS(__func__);
size_t n = WUFFS_TESTLIB_ARRAY_SIZE(g_brotli_ext_vectors);
size_t num_skipped = 0;
for (size_t i = 0; i < n; i++) {
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
.data = g_src_slice_u8,
});
CHECK_STRING(read_file(&src, g_brotli_ext_vectors[i].src_filename));
bool skipped = false;
CHECK_STRING(do_test_mimic_brotli_differential_with_options(
g_brotli_ext_vectors[i].src_filename, &src,
g_brotli_ext_vectors[i].want_filename, false,
g_brotli_ext_vectors[i].large_window,
g_brotli_ext_vectors[i].dict_set, &skipped));
num_skipped += skipped ? 1 : 0;
}
// Only the serialized dictionary files (6 of them) can be skipped.
if (num_skipped > 6) {
RETURN_FAIL("too many skipped: %zu", num_skipped);
}
return NULL;
}
const char* //
test_mimic_brotli_decode_ext_malformed() {
CHECK_FOCUS(__func__);
size_t n = WUFFS_TESTLIB_ARRAY_SIZE(g_brotli_ext_malformed);
for (size_t i = 0; i < n; i++) {
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
.data = g_src_slice_u8,
});
if (g_brotli_ext_malformed[i].src_filename) {
CHECK_STRING(read_file(&src, g_brotli_ext_malformed[i].src_filename));
} else {
memcpy(src.data.ptr, g_brotli_ext_malformed[i].data,
g_brotli_ext_malformed[i].len);
src.meta.wi = g_brotli_ext_malformed[i].len;
src.meta.closed = true;
}
char name[64];
snprintf(name, sizeof name, "g_brotli_ext_malformed[%zu]", i);
bool skipped = false;
CHECK_STRING(do_test_mimic_brotli_differential_with_options(
name, &src, NULL, true, g_brotli_ext_malformed[i].large_window,
g_brotli_ext_malformed[i].dict_set, &skipped));
}
return NULL;
}
// mimic_brotli_decode_large_window adapts mimic_brotli_decode_with_options
// to do_test_brotli_decode_large_window_far's codec_func type.
const char* //
mimic_brotli_decode_large_window(wuffs_base__io_buffer* dst,
wuffs_base__io_buffer* src,
uint64_t wlimit,
uint64_t rlimit,
bool large_window,
size_t dict_set) {
if ((wlimit < UINT64_MAX) || (rlimit < UINT64_MAX) ||
(dict_set != BROTLI_DICT_SET_NONE)) {
return "unsupported option";
}
int error_code = 0;
return mimic_brotli_decode_with_options(dst, src, large_window, 0, NULL,
NULL, &error_code);
}
const char* //
test_mimic_brotli_decode_large_window_far() {
CHECK_FOCUS(__func__);
return do_test_brotli_decode_large_window_far(
mimic_brotli_decode_large_window, UINT64_MAX, UINT64_MAX);
}
#endif // WUFFS_MIMIC
// ---------------- Brotli Large Window Encoder Tests
// do_test_brotli_encode_large_window encodes data[:data_len] with the plan's
// config c (which should set the large-window quirk) and checks that
// decoding the result (with Wuffs' std/brotli decoder with the large-window
// quirk set, and, if mimic is true, with libbrotlidec) gives back the
// original data. If mimic is true, it also checks that libbrotlienc's output
// is the same. If max_compressed_len is non-zero, it checks that the
// compressed length is at most that. The decoded data goes in g_want.
const char* //
do_test_brotli_encode_large_window(const char* name,
const brotli_enc_config* c,
const uint8_t* data,
size_t data_len,
bool mimic,
size_t max_compressed_len) {
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
.data = g_src_slice_u8,
});
CHECK_STRING(make_brotli_enc_plan(&src, c, data, data_len));
wuffs_base__io_buffer have = ((wuffs_base__io_buffer){
.data = g_have_slice_u8,
});
wuffs_base__io_buffer src0 = src;
const char* status = wuffs_brotli_encode_plan(
&have, &src0, c->param_keys, c->param_values, c->num_params,
g_brotli_enc_step_ops, g_brotli_enc_step_lens, g_brotli_enc_num_steps,
c->dst_chunk);
if (status) {
RETURN_FAIL("%s: wuffs encode: %s", name, status);
} else if (max_compressed_len && (have.meta.wi > max_compressed_len)) {
RETURN_FAIL("%s: compressed length: have %zu, want at most %zu", name,
have.meta.wi, max_compressed_len);
}
have.meta.closed = true;
for (int decoder = 0; decoder < (mimic ? 3 : 1); decoder++) {
wuffs_base__io_buffer want = ((wuffs_base__io_buffer){
.data = g_want_slice_u8,
});
wuffs_base__io_buffer have1 = have;
if (decoder == 0) {
status = wuffs_brotli_decode_with_options(
&want, &have1, UINT64_MAX, UINT64_MAX, true, BROTLI_DICT_SET_NONE);
#ifdef WUFFS_MIMIC
} else if (decoder == 1) {
status = mimic_brotli_decode_large_window(
&want, &have1, UINT64_MAX, UINT64_MAX, true, BROTLI_DICT_SET_NONE);
} else {
wuffs_base__io_buffer src1 = src;
status = mimic_brotli_encode(
&want, &src1, c->param_keys, c->param_values, c->num_params,
g_brotli_enc_step_ops, g_brotli_enc_step_lens,
g_brotli_enc_num_steps, c->dst_chunk);
if (status) {
RETURN_FAIL("%s: mimic encode: %s", name, status);
}
status = check_io_buffers_equal("", &have, &want);
if (status) {
RETURN_FAIL("%s: %s", name, status);
}
continue;
#endif // WUFFS_MIMIC
}
const char* which = (decoder == 0) ? "wuffs" : "mimic";
if (status) {
RETURN_FAIL("%s: %s decode: %s", name, which, status);
} else if (have1.meta.ri != have1.meta.wi) {
RETURN_FAIL("%s: %s decode: trailing data", name, which);
} else if (want.meta.wi != data_len) {
RETURN_FAIL("%s: %s decoded length: have %zu, want %zu", name, which,
want.meta.wi, data_len);
} else if (memcmp(want.data.ptr, data, data_len)) {
RETURN_FAIL("%s: %s decoded bytes differ", name, which);
}
}
return NULL;
}
// do_test_brotli_encode_large_window_files runs a grid of large-window
// configurations (qualities 3 to 9, lgwin 18 to 30) over a few small files.
const char* //
do_test_brotli_encode_large_window_files(bool mimic) {
static const char* filenames[] = {
"test/data/romeo.txt",
"test/data/brotli/alice29.txt",
"test/data/brotli/random_org_10k.bin",
"test/data/brotli/quickfox_repeated",
};
static const uint32_t lgwins[] = {18, 25, 26, 28, 30};
wuffs_base__io_buffer data = ((wuffs_base__io_buffer){
.data = g_work_slice_u8,
});
for (size_t f = 0; f < WUFFS_TESTLIB_ARRAY_SIZE(filenames); f++) {
// g_work is free to use for the input: each config's workbuf (of more
// than 64 MiB) is malloc'ed.
data.meta.wi = 0;
data.meta.ri = 0;
data.meta.closed = false;
CHECK_STRING(read_file(&data, filenames[f]));
uint8_t* copy = malloc(data.meta.wi ? data.meta.wi : 1);
if (!copy) {
return "malloc failed";
}
memcpy(copy, data.data.ptr, data.meta.wi);
const char* ret = NULL;
for (uint32_t q = 3; (q <= 9) && !ret; q++) {
for (size_t w = 0; (w < 5) && !ret; w++) {
for (int k = 0; (k < 2) && !ret; k++) {
if ((k == 1) && (lgwins[w] != 26)) {
continue;
}
brotli_enc_config c = make_brotli_enc_config(q, lgwins[w]);
brotli_enc_config__add_param(&c, BROTLI_ENC_PARAM_LARGE_WINDOW, 1);
if (k == 1) {
// A big size hint selects H55 (at quality 4) and H65 (at
// qualities 5 to 9).
brotli_enc_config__add_param(&c, BROTLI_ENC_PARAM_SIZE_HINT,
1 << 20);
}
c.chunking = (w & 1) ? BROTLI_ENC_CHUNK_SINGLE
: BROTLI_ENC_CHUNK_RANDOM;
c.flush = (w & 1) ? BROTLI_ENC_FLUSH_NONE : BROTLI_ENC_FLUSH_MIDDLE;
c.seed = (q << 8) | w;
char s[256];
snprintf(s, sizeof s, "%s: q=%u lgwin=%u size_hint=%s",
filenames[f], q, lgwins[w], k ? "1<<20" : "0");
ret = do_test_brotli_encode_large_window(s, &c, copy, data.meta.wi,
mimic, 0);
}
}
}
free(copy);
if (ret) {
return ret;
}
}
return NULL;
}
// make_brotli_enc_far_data returns a malloc'ed buffer of pseudo-random bytes
// with two 2 MiB repeats at distances of (20 MiB + 64) and (40 MiB - 4096).
// The repeats' sources are not repeated elsewhere, so only an encoder with a
// large enough window (lgwin 25 or more for the first, 26 or more for the
// second) can compress it.
uint8_t* //
make_brotli_enc_far_data(size_t* len) {
const size_t mib = 1 << 20;
const size_t n = (46 * mib) + 12345;
uint8_t* p = malloc(n);
if (!p) {
return NULL;
}
brotli_enc_fill_random(p, n, 0xFA12);
memcpy(p + (21 * mib) + 64, p + (1 * mib), 2 * mib);
memcpy(p + (44 * mib) - 4096, p + (4 * mib), 2 * mib);
*len = n;
return p;
}
const char* //
do_test_brotli_encode_large_window_far(bool mimic) {
size_t len = 0;
uint8_t* data = make_brotli_enc_far_data(&len);
if (!data) {
return "malloc failed";
}
static const struct {
uint32_t quality;
uint32_t lgwin;
uint32_t size_hint;
uint32_t chunking;
size_t num_repeats;
} configs[] = {
// H35.
{3, 25, 0, BROTLI_ENC_CHUNK_SINGLE, 1},
{3, 30, 0, BROTLI_ENC_CHUNK_RANDOM, 2},
// H55.
{4, 26, 1 << 30, BROTLI_ENC_CHUNK_SINGLE, 2},
// H65.
{5, 30, 1 << 30, BROTLI_ENC_CHUNK_RANDOM, 2},
{7, 28, 0, BROTLI_ENC_CHUNK_SINGLE, 2},
{9, 26, 0, BROTLI_ENC_CHUNK_SINGLE, 2},
};
const char* ret = NULL;
for (size_t i = 0; (i < WUFFS_TESTLIB_ARRAY_SIZE(configs)) && !ret; i++) {
brotli_enc_config c =
make_brotli_enc_config(configs[i].quality, configs[i].lgwin);
brotli_enc_config__add_param(&c, BROTLI_ENC_PARAM_LARGE_WINDOW, 1);
if (configs[i].size_hint) {
brotli_enc_config__add_param(&c, BROTLI_ENC_PARAM_SIZE_HINT,
configs[i].size_hint);
}
c.chunking = configs[i].chunking;
c.flush = (configs[i].chunking == BROTLI_ENC_CHUNK_RANDOM)
? BROTLI_ENC_FLUSH_MIDDLE
: BROTLI_ENC_FLUSH_NONE;
c.seed = i;
char s[256];
snprintf(s, sizeof s, "far data: q=%u lgwin=%u", configs[i].quality,
configs[i].lgwin);
// Pseudo-random data is incompressible, except for the repeats (of
// 2 MiB each), which must be found (mostly) to meet this limit.
size_t max_compressed_len = len - (configs[i].num_repeats * (3 << 19));
ret = do_test_brotli_encode_large_window(s, &c, data, len, mimic,
max_compressed_len);
}
free(data);
return ret;
}
const char* //
test_wuffs_brotli_encode_large_window_files() {
CHECK_FOCUS(__func__);
return do_test_brotli_encode_large_window_files(false);
}
const char* //
test_wuffs_brotli_encode_large_window_far() {
CHECK_FOCUS(__func__);
return do_test_brotli_encode_large_window_far(false);
}
#ifdef WUFFS_MIMIC
const char* //
test_mimic_brotli_encode_large_window_files() {
CHECK_FOCUS(__func__);
return do_test_brotli_encode_large_window_files(true);
}
const char* //
test_mimic_brotli_encode_large_window_far() {
CHECK_FOCUS(__func__);
return do_test_brotli_encode_large_window_far(true);
}
#endif // WUFFS_MIMIC
// ---------------- Brotli Encoder Dictionary Tests
// brotli_enc_load_file returns a malloc'ed copy of the named file (of up to
// 1 MiB) and sets *len to its length.
uint8_t* //
brotli_enc_load_file(const char* filename, size_t* len) {
uint8_t* buf = malloc(1 << 20);
if (!buf) {
return NULL;
}
wuffs_base__io_buffer b = ((wuffs_base__io_buffer){
.data = wuffs_base__make_slice_u8(buf, 1 << 20),
});
if (read_file(&b, filename)) {
free(buf);
return NULL;
}
*len = b.meta.wi;
return buf;
}
// brotli_enc_config_has_param returns whether c sets the key param to a
// non-zero value.
bool //
brotli_enc_config_has_param(const brotli_enc_config* c, uint32_t key) {
bool ret = false;
for (size_t j = 0; j < c->num_params; j++) {
if (c->param_keys[j] == key) {
ret = c->param_values[j] != 0;
}
}
return ret;
}
// wuffs_brotli_decode_with_raw_dictionaries decodes src to dst with Wuffs'
// std/brotli decoder, with the given raw dictionaries attached.
const char* //
wuffs_brotli_decode_with_raw_dictionaries(wuffs_base__io_buffer* dst,
wuffs_base__io_buffer* src,
bool large_window,
size_t num_dicts,
const wuffs_base__slice_u8* dicts) {
wuffs_brotli__decoder dec;
CHECK_STATUS("initialize",
wuffs_brotli__decoder__initialize(
&dec, sizeof dec, WUFFS_VERSION,
WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED));
CHECK_STATUS("set_quirk",
wuffs_brotli__decoder__set_quirk(
&dec, WUFFS_BROTLI__QUIRK_LARGE_WINDOW, large_window));
size_t total = 0;
for (size_t j = 0; j < num_dicts; j++) {
total += dicts[j].len;
}
size_t n = total + (large_window ? (1 << 30) : (1 << 24)) + 542;
uint8_t* p = malloc(n);
if (!p) {
return "malloc failed";
}
wuffs_base__slice_u8 workbuf = wuffs_base__make_slice_u8(p, n);
const char* ret = NULL;
for (size_t j = 0; j < num_dicts; j++) {
ret = wuffs_brotli__decoder__add_dictionary(
&dec, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, dicts[j], workbuf)
.repr;
if (ret) {
goto cleanup;
}
}
if (wuffs_brotli__decoder__workbuf_len(&dec).max_incl > n) {
ret = "decoder workbuf_len is too large";
goto cleanup;
}
ret = wuffs_brotli__decoder__transform_io(&dec, dst, src, workbuf).repr;
cleanup:
free(p);
return ret;
}
// do_test_brotli_encode_dictionary encodes data[:data_len] with the plan's
// config c and the given raw dictionaries. Unless the config sets a stream
// offset (whose output is the continuation of another stream), it checks
// that Wuffs' decoder (and, if mimic is true, libbrotlidec), with the same
// dictionaries, decodes the result back to data. If mimic is true, it also
// checks that libbrotlienc's output is the same. It sets *compressed_len.
const char* //
do_test_brotli_encode_dictionary(const char* name,
const brotli_enc_config* c,
const uint8_t* data,
size_t data_len,
size_t num_dicts,
const wuffs_base__slice_u8* dicts,
bool mimic,
size_t* compressed_len) {
uint32_t dict_types[16] = {0};
for (size_t j = 0; j < 16; j++) {
dict_types[j] = WUFFS_BROTLI__DICTIONARY_TYPE_RAW;
}
if (num_dicts > 16) {
return "too many dictionaries";
}
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
.data = g_src_slice_u8,
});
CHECK_STRING(make_brotli_enc_plan(&src, c, data, data_len));
wuffs_base__io_buffer have = ((wuffs_base__io_buffer){
.data = g_have_slice_u8,
});
wuffs_base__io_buffer src0 = src;
const char* status = wuffs_brotli_encode_plan_with_dictionaries(
&have, &src0, c->param_keys, c->param_values, c->num_params, num_dicts,
dict_types, dicts, g_brotli_enc_step_ops, g_brotli_enc_step_lens,
g_brotli_enc_num_steps, c->dst_chunk);
if (status) {
RETURN_FAIL("%s: wuffs encode: %s", name, status);
}
have.meta.closed = true;
*compressed_len = have.meta.wi;
bool lw = brotli_enc_config_has_param(c, BROTLI_ENC_PARAM_LARGE_WINDOW);
bool so = brotli_enc_config_has_param(c, BROTLI_ENC_PARAM_STREAM_OFFSET);
for (int k = 0; k < (mimic ? 3 : 1); k++) {
wuffs_base__io_buffer want = ((wuffs_base__io_buffer){
.data = g_want_slice_u8,
});
wuffs_base__io_buffer have1 = have;
if (k == 0) {
if (so) {
continue;
}
status = wuffs_brotli_decode_with_raw_dictionaries(&want, &have1, lw,
num_dicts, dicts);
#ifdef WUFFS_MIMIC
} else if (k == 1) {
if (so) {
continue;
}
int error_code = 0;
status = mimic_brotli_decode_with_options(
&want, &have1, lw, num_dicts, dict_types, dicts, &error_code);
} else {
wuffs_base__io_buffer src1 = src;
status = mimic_brotli_encode_with_dictionaries(
&want, &src1, c->param_keys, c->param_values, c->num_params,
num_dicts, dict_types, dicts, g_brotli_enc_step_ops,
g_brotli_enc_step_lens, g_brotli_enc_num_steps, c->dst_chunk);
if (status) {
RETURN_FAIL("%s: mimic encode: %s", name, status);
}
status = check_io_buffers_equal("", &have, &want);
if (status) {
RETURN_FAIL("%s: %s", name, status);
}
continue;
#endif // WUFFS_MIMIC
}
const char* which = (k == 0) ? "wuffs" : "mimic";
if (status) {
RETURN_FAIL("%s: %s decode: %s", name, which, status);
} else if (have1.meta.ri != have1.meta.wi) {
RETURN_FAIL("%s: %s decode: trailing data", name, which);
} else if (want.meta.wi != data_len) {
RETURN_FAIL("%s: %s decoded length: have %zu, want %zu", name, which,
want.meta.wi, data_len);
} else if (memcmp(want.data.ptr, data, data_len)) {
RETURN_FAIL("%s: %s decoded bytes differ", name, which);
}
}
return NULL;
}
const char* //
test_wuffs_brotli_encode_add_dictionary() {
CHECK_FOCUS(__func__);
static const uint8_t text[] = "the quick brown fox jumps over the lazy dog";
wuffs_base__slice_u8 dict =
wuffs_base__make_slice_u8((uint8_t*)text, sizeof text - 1);
wuffs_base__slice_u8 workbuf = g_work_slice_u8;
wuffs_brotli__encoder enc;
CHECK_STATUS("initialize",
wuffs_brotli__encoder__initialize(
&enc, sizeof enc, WUFFS_VERSION,
WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED));
CHECK_STATUS("set_quirk", wuffs_brotli__encoder__set_quirk(
&enc, WUFFS_BROTLI__QUIRK_ENCODER_QUALITY, 5));
uint64_t wlen0 = wuffs_brotli__encoder__workbuf_len(&enc).max_incl;
// Serialized dictionaries are rejected (like libbrotlienc's default,
// non-experimental build) and so are unknown dictionary types.
const char* have = wuffs_brotli__encoder__add_dictionary(
&enc, WUFFS_BROTLI__DICTIONARY_TYPE_SERIALIZED, dict,
workbuf)
.repr;
if (have != wuffs_brotli__error__unsupported_serialized_encoder_dictionary) {
RETURN_FAIL("serialized: have \"%s\"", have);
}
have = wuffs_brotli__encoder__add_dictionary(&enc, 2, dict, workbuf).repr;
if (have != wuffs_base__error__bad_argument) {
RETURN_FAIL("unknown type: have \"%s\"", have);
}
// The add_dictionary_workbuf_len formula: for a dictionary of n bytes (and
// 17 bucket bits and 7 slot bits, for n up to 2 MiB), n bytes, (4 << 7)
// slot offsets, (2 << 17) heads and up to 4 * (n - 7) items, then (6 <<
// 17) + (4 * n) bytes of scratch space.
uint64_t want_wlen =
43 + (4 << 7) + (2 << 17) + (4 * 36) + (6 << 17) + (4 * 43);
uint64_t have_wlen =
wuffs_brotli__encoder__add_dictionary_workbuf_len(&enc, dict.len);
if (have_wlen != want_wlen) {
RETURN_FAIL("add_dictionary_workbuf_len: have %" PRIu64 ", want %" PRIu64,
have_wlen, want_wlen);
}
workbuf.len = (size_t)(have_wlen - 1);
have = wuffs_brotli__encoder__add_dictionary(
&enc, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, dict, workbuf)
.repr;
if (have != wuffs_base__error__bad_workbuf_length) {
RETURN_FAIL("short workbuf: have \"%s\"", have);
}
if (wuffs_brotli__encoder__workbuf_len(&enc).max_incl != wlen0) {
RETURN_FAIL("workbuf_len changed after failed add_dictionary calls");
}
// 15 dictionaries (the C code's SHARED_BROTLI_MAX_COMPOUND_DICTS), one of
// them empty, can be added, but not 16.
uint64_t dict_end = 0;
for (int i = 0; i < 16; i++) {
wuffs_base__slice_u8 d = dict;
if (i == 3) {
d.len = 0;
}
workbuf.len = g_work_slice_u8.len;
uint64_t n = wuffs_brotli__encoder__add_dictionary_workbuf_len(&enc, d.len);
if (n != (dict_end + ((i == 3) ? ((4 << 7) + (8 << 17)) : want_wlen))) {
RETURN_FAIL("i=%d: add_dictionary_workbuf_len: have %" PRIu64, i, n);
}
have = wuffs_brotli__encoder__add_dictionary(
&enc, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, d, workbuf)
.repr;
if (i == 15) {
if (have != wuffs_brotli__error__too_many_compound_dictionaries) {
RETURN_FAIL("i=%d: have \"%s\"", i, have);
}
break;
} else if (have) {
RETURN_FAIL("i=%d: have \"%s\"", i, have);
}
// The dictionary bytes, (4 << 7) slot offsets, (2 << 17) heads and 4
// bytes per item. Each of the (43 - 7) positions of "the quick brown fox
// etc" has a distinct hash, so there are 36 items.
dict_end += (i == 3) ? ((4 << 7) + (2 << 17))
: (43 + (4 << 7) + (2 << 17) + (4 * 36));
uint64_t wlen = wuffs_brotli__encoder__workbuf_len(&enc).max_incl;
if (wlen != (wlen0 + dict_end)) {
RETURN_FAIL("i=%d: workbuf_len: have %" PRIu64 ", want %" PRIu64, i,
wlen, wlen0 + dict_end);
}
}
// Quirks can still be set after add_dictionary, until transform_io.
CHECK_STATUS("set_quirk", wuffs_brotli__encoder__set_quirk(
&enc, WUFFS_BROTLI__QUIRK_LGWIN, 16));
uint64_t wlen = wuffs_brotli__encoder__workbuf_len(&enc).max_incl;
if (wlen <= dict_end) {
RETURN_FAIL("workbuf_len: have %" PRIu64, wlen);
}
// A workbuf shorter than the dictionaries fails.
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
.data = wuffs_base__make_slice_u8((uint8_t*)text, sizeof text - 1),
.meta = wuffs_base__make_io_buffer_meta(sizeof text - 1, 0, 0, true),
});
wuffs_base__io_buffer dst = ((wuffs_base__io_buffer){
.data = g_have_slice_u8,
});
// (An error disables the encoder, so try this on a copy.)
wuffs_brotli__encoder* copy = malloc(sizeof enc);
if (!copy) {
return "malloc failed";
}
memcpy(copy, &enc, sizeof enc);
workbuf.len = (size_t)(dict_end - 1);
have = wuffs_brotli__encoder__transform_io(copy, &dst, &src, workbuf).repr;
free(copy);
if (have != wuffs_base__error__bad_workbuf_length) {
RETURN_FAIL("transform_io (short workbuf): have \"%s\"", have);
}
workbuf.len = (size_t)wlen;
CHECK_STATUS("transform_io",
wuffs_brotli__encoder__transform_io(&enc, &dst, &src, workbuf));
if (dst.meta.wi >= (sizeof text - 1)) {
RETURN_FAIL("compressed length: have %zu", dst.meta.wi);
}
// After transform_io, add_dictionary (and set_quirk) fail. (The C code's
// BrotliEncoderAttachPreparedDictionary accepts a raw dictionary at any
// time, but attaching one after compression has started changes the
// compound dictionary's size mid-stream, which is not supported here.)
have = wuffs_brotli__encoder__add_dictionary(
&enc, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, dict, workbuf)
.repr;
if (have != wuffs_base__error__bad_call_sequence) {
RETURN_FAIL("after transform_io: have \"%s\"", have);
}
return NULL;
}
// brotli_enc_dict_set fills dicts with one of the dictionary test sets (for
// the alice29.txt, asyoulik.txt and romeo.txt files a, b and r, each of
// length a_len, b_len and r_len) and returns how many dictionaries it has.
size_t //
brotli_enc_dict_set(int which,
wuffs_base__slice_u8* dicts,
uint8_t* a,
size_t a_len,
uint8_t* b,
size_t b_len,
uint8_t* r,
size_t r_len) {
switch (which) {
case 0: // One small dictionary.
dicts[0] = wuffs_base__make_slice_u8(r, r_len);
return 1;
case 1: // The first half of alice29.txt.
dicts[0] = wuffs_base__make_slice_u8(a, a_len / 2);
return 1;
case 2: // Several dictionaries, including tiny and empty ones.
dicts[0] = wuffs_base__make_slice_u8(r, 0);
dicts[1] = wuffs_base__make_slice_u8(b + 1000, 5);
dicts[2] = wuffs_base__make_slice_u8(a + (a_len / 2), a_len / 4);
dicts[3] = wuffs_base__make_slice_u8(b, 4000);
dicts[4] = wuffs_base__make_slice_u8(r + 100, 300);
return 5;
case 3: // 15 (the maximum number of) pieces of alice29.txt.
for (size_t j = 0; j < 15; j++) {
dicts[j] = wuffs_base__make_slice_u8(a + (j * 10000), 9000);
}
return 15;
}
dicts[0] = wuffs_base__make_slice_u8(b, b_len);
return 1;
}
const char* //
do_test_brotli_encode_dictionary_grid(bool mimic) {
size_t a_len = 0;
size_t b_len = 0;
size_t r_len = 0;
uint8_t* a = brotli_enc_load_file("test/data/brotli/alice29.txt", &a_len);
uint8_t* b = brotli_enc_load_file("test/data/brotli/asyoulik.txt", &b_len);
uint8_t* r = brotli_enc_load_file("test/data/romeo.txt", &r_len);
const char* ret = NULL;
if (!a || !b || !r || (a_len < 150000)) {
ret = "could not load the test files";
goto cleanup;
}
// Each config is (lgwin, chunking, flush, metadata, extra param, value).
static const uint32_t configs[][6] = {
{22, BROTLI_ENC_CHUNK_SINGLE, BROTLI_ENC_FLUSH_NONE, 0, 0, 0},
{16, BROTLI_ENC_CHUNK_RANDOM, BROTLI_ENC_FLUSH_MIDDLE, 0, 0, 0},
{10, BROTLI_ENC_CHUNK_RANDOM, BROTLI_ENC_FLUSH_EVERY_CHUNK, 0, 0, 0},
{18, BROTLI_ENC_CHUNK_RANDOM, BROTLI_ENC_FLUSH_MIDDLE,
BROTLI_ENC_METADATA_MIDDLE, 0, 0},
{22, BROTLI_ENC_CHUNK_RANDOM, BROTLI_ENC_FLUSH_MIDDLE, 0,
BROTLI_ENC_PARAM_STREAM_OFFSET, 1000},
{24, BROTLI_ENC_CHUNK_SINGLE, BROTLI_ENC_FLUSH_NONE, 0,
BROTLI_ENC_PARAM_STREAM_OFFSET, 1 << 30},
{26, BROTLI_ENC_CHUNK_RANDOM, BROTLI_ENC_FLUSH_MIDDLE, 0,
BROTLI_ENC_PARAM_LARGE_WINDOW, 1},
{22, BROTLI_ENC_CHUNK_SINGLE, BROTLI_ENC_FLUSH_NONE, 0,
BROTLI_ENC_PARAM_SIZE_HINT, 1 << 20},
};
// The input is the second half of alice29.txt, except for dictionary set
// 4 (all of asyoulik.txt), whose input is romeo.txt.
for (int set = 0; (set < 5) && !ret; set++) {
wuffs_base__slice_u8 dicts[16];
size_t num_dicts = brotli_enc_dict_set(set, dicts, a, a_len, b, b_len, r,
r_len);
const uint8_t* data = (set == 4) ? r : (a + (a_len / 2));
size_t data_len = (set == 4) ? r_len : (a_len - (a_len / 2));
for (uint32_t q = 0; (q <= 11) && !ret; q++) {
for (size_t i = 0; (i < WUFFS_TESTLIB_ARRAY_SIZE(configs)) && !ret;
i++) {
if (!mimic && (i >= 2) && (q & 1)) {
continue;
} else if ((q >= 10) &&
((i == 2) || (i == 7) || (!mimic && (i != 0)) ||
((q == 11) && (i & 1)))) {
// Qualities 10 and 11 are slow, so they skip some configs.
continue;
}
brotli_enc_config c = make_brotli_enc_config(q, configs[i][0]);
if (configs[i][4]) {
brotli_enc_config__add_param(&c, configs[i][4], configs[i][5]);
}
c.chunking = configs[i][1];
c.flush = configs[i][2];
c.metadata_position = configs[i][3];
c.metadata_len = configs[i][3] ? 100 : 0;
c.seed = (set << 16) | (q << 8) | i;
char s[256];
snprintf(s, sizeof s, "set=%d q=%u config=%zu", set, q, i);
size_t have_len = 0;
ret = do_test_brotli_encode_dictionary(s, &c, data, data_len,
num_dicts, dicts, mimic,
&have_len);
if (ret || (set != 1) || (q < 5) || (i != 0)) {
continue;
}
// Encoding the second half of alice29.txt with the first half as a
// dictionary should be at least 5% better than without it.
size_t without_len = 0;
ret = do_test_brotli_encode_dictionary(s, &c, data, data_len, 0, NULL,
false, &without_len);
if (!ret && ((have_len * 100) >= (without_len * 95))) {
snprintf(g_fail_msg, sizeof g_fail_msg,
"%s: compressed length: have %zu, without dictionary %zu",
s, have_len, without_len);
ret = g_fail_msg;
}
}
}
}
cleanup:
free(a);
free(b);
free(r);
return ret;
}
const char* //
do_test_brotli_encode_dictionary_big(bool mimic) {
// Dictionaries of exactly 2 MiB (the largest with 17 bucket bits), one
// byte more (18 bucket bits) and 5 and 33 MiB (19 and 22 bucket bits).
// Each holds, at various offsets, pieces of the 1 MiB input.
static const size_t sizes[] = {2 << 20, (2 << 20) + 1, (5 << 20) + 3,
(33 << 20) + 5};
const size_t len = 1 << 20;
uint8_t* data = malloc(len);
uint8_t* dict = malloc(sizes[3]);
const char* ret = NULL;
if (!data || !dict) {
ret = "malloc failed";
goto cleanup;
}
brotli_enc_fill_random(data, len, 0xDA7A);
for (size_t i = 0; i < len; i++) {
// Make it compressible: mostly lowercase letters.
data[i] = 'a' + (data[i] % 13);
}
for (size_t k = 0; (k < WUFFS_TESTLIB_ARRAY_SIZE(sizes)) && !ret; k++) {
size_t dlen = sizes[k];
brotli_enc_fill_random(dict, dlen, 0xD1C7 + k);
for (size_t j = 0; j < 3; j++) {
size_t at = (j == 0) ? 999 : (j == 1) ? (dlen / 2) : (dlen - (65 << 10));
memcpy(dict + at, data + (j * (300 << 10)), 64 << 10);
}
wuffs_base__slice_u8 dicts[2] = {
wuffs_base__make_slice_u8(dict, dlen),
wuffs_base__make_slice_u8(data + (len - 1000), 1000),
};
static const uint32_t qs[] = {2, 4, 5, 9, 10, 11};
for (size_t i = 0; (i < WUFFS_TESTLIB_ARRAY_SIZE(qs)) && !ret; i++) {
// Without mimic, and for the largest dictionary, a smaller grid. The
// slow qualities 10 and 11 only run with mimic, for one dictionary
// each.
if (qs[i] >= 10) {
if (!mimic || (k != ((qs[i] == 10) ? 3 : 1))) {
continue;
}
} else if ((!mimic && ((k & 1) == 0)) ||
((!mimic || (k == 3)) && (i != 2))) {
continue;
}
brotli_enc_config c = make_brotli_enc_config(qs[i], 24);
if (i & 1) {
brotli_enc_config__add_param(&c, BROTLI_ENC_PARAM_SIZE_HINT, 1 << 20);
c.chunking = BROTLI_ENC_CHUNK_RANDOM;
c.flush = BROTLI_ENC_FLUSH_MIDDLE;
}
c.seed = (k << 8) | i;
char s[256];
snprintf(s, sizeof s, "dictionary of %zu bytes: q=%u", dlen, qs[i]);
size_t have_len = 0;
// At qualities 10 and 11, only the first 128 KiB of the input (which
// has the first piece) is encoded.
ret = do_test_brotli_encode_dictionary(
s, &c, data, (qs[i] >= 10) ? (128 << 10) : len, 2, dicts, mimic,
&have_len);
// The three 64 KiB pieces (of 3.7 bits of entropy per byte) should be
// found at quality 5 and above, even when they are more than 16 MiB
// (lgwin 24's window) into the dictionary. (At the slow qualities 10
// and 11, this is left to the parity check.)
size_t want_max = 0;
if (!ret && (qs[i] >= 5) && (qs[i] <= 9)) {
c = make_brotli_enc_config(qs[i], 24);
ret = do_test_brotli_encode_dictionary(s, &c, data, len, 0, NULL,
false, &want_max);
want_max -= 3 * (24 << 10);
if (!ret && (have_len > want_max)) {
snprintf(g_fail_msg, sizeof g_fail_msg,
"%s: compressed length: have %zu, want at most %zu", s,
have_len, want_max);
ret = g_fail_msg;
}
}
}
}
cleanup:
free(data);
free(dict);
return ret;
}
const char* //
test_wuffs_brotli_encode_dictionary_big() {
CHECK_FOCUS(__func__);
return do_test_brotli_encode_dictionary_big(false);
}
const char* //
test_wuffs_brotli_encode_dictionary_grid() {
CHECK_FOCUS(__func__);
return do_test_brotli_encode_dictionary_grid(false);
}
#ifdef WUFFS_MIMIC
const char* //
test_mimic_brotli_encode_dictionary_big() {
CHECK_FOCUS(__func__);
return do_test_brotli_encode_dictionary_big(true);
}
const char* //
test_mimic_brotli_encode_dictionary_grid() {
CHECK_FOCUS(__func__);
return do_test_brotli_encode_dictionary_grid(true);
}
// test_mimic_brotli_encode_dictionary_rejections checks that the dictionaries
// that libbrotlienc (a default, non-experimental build) rejects are also
// rejected by Wuffs, and that an empty dictionary is accepted by both.
const char* //
test_mimic_brotli_encode_dictionary_rejections() {
CHECK_FOCUS(__func__);
static const uint8_t text[] = "the quick brown fox jumps over the lazy dog";
// A serialized dictionary (an LZ77 prefix only), which an experimental
// libbrotlienc build would accept.
static const uint8_t serialized[] = {0x91, 0x00, 0x03, 'f', 'o', 'x', 0x00,
0x00};
static const struct {
size_t num_dicts;
uint32_t type; // The last dictionary's type.
size_t len; // The last dictionary's length.
const char* want_wuffs;
const char* want_mimic;
} cases[] = {
{1, WUFFS_BROTLI__DICTIONARY_TYPE_SERIALIZED, sizeof serialized,
wuffs_brotli__error__unsupported_serialized_encoder_dictionary,
mimic_brotli_prepare_dictionary_failed_message},
{16, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, 9,
wuffs_brotli__error__too_many_compound_dictionaries,
mimic_brotli_attach_prepared_dictionary_failed_message},
{15, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, 9, NULL, NULL},
{1, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, 0, NULL, NULL},
{3, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, 0, NULL, NULL},
};
// Quality 5 and the Zopfli qualities 10 and 11.
static const uint32_t qs[] = {5, 10, 11};
for (size_t iq = 0; iq < WUFFS_TESTLIB_ARRAY_SIZE(qs); iq++) {
for (size_t i = 0; i < WUFFS_TESTLIB_ARRAY_SIZE(cases); i++) {
uint32_t dict_types[16];
wuffs_base__slice_u8 dicts[16];
for (size_t j = 0; j < cases[i].num_dicts; j++) {
dict_types[j] = WUFFS_BROTLI__DICTIONARY_TYPE_RAW;
dicts[j] = wuffs_base__make_slice_u8((uint8_t*)text + j, 20);
}
size_t last = cases[i].num_dicts - 1;
dict_types[last] = cases[i].type;
dicts[last] = wuffs_base__make_slice_u8(
(uint8_t*)((cases[i].type == WUFFS_BROTLI__DICTIONARY_TYPE_RAW)
? text
: serialized),
cases[i].len);
brotli_enc_config c = make_brotli_enc_config(qs[iq], 22);
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
.data = g_src_slice_u8,
});
CHECK_STRING(make_brotli_enc_plan(&src, &c, text, sizeof text - 1));
wuffs_base__io_buffer dsts[2] = {
((wuffs_base__io_buffer){.data = g_have_slice_u8}),
((wuffs_base__io_buffer){.data = g_want_slice_u8}),
};
for (int k = 0; k < 2; k++) {
wuffs_base__io_buffer src1 = src;
const char* have =
(k ? mimic_brotli_encode_with_dictionaries
: wuffs_brotli_encode_plan_with_dictionaries)(
&dsts[k], &src1, c.param_keys, c.param_values, c.num_params,
cases[i].num_dicts, dict_types, dicts, g_brotli_enc_step_ops,
g_brotli_enc_step_lens, g_brotli_enc_num_steps, c.dst_chunk);
const char* want = k ? cases[i].want_mimic : cases[i].want_wuffs;
if (have != want) {
RETURN_FAIL("q=%u cases[%zu] (%s): have \"%s\", want \"%s\"",
qs[iq], i, k ? "mimic" : "wuffs", have ? have : "",
want ? want : "");
}
}
if (!cases[i].want_wuffs) {
CHECK_STRING(check_io_buffers_equal("", &dsts[0], &dsts[1]));
}
}
}
return NULL;
}
#endif // WUFFS_MIMIC
// ---------------- Brotli Benches
const char* //
bench_wuffs_brotli_decode_alice29() {
CHECK_FOCUS(__func__);
return do_bench_io_buffers(
wuffs_brotli_decode,
WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED, tcounter_dst,
&g_brotli_alice29_gt, UINT64_MAX, UINT64_MAX, 5);
}
// ---------------- Mimic Benches
#ifdef WUFFS_MIMIC
const char* //
bench_mimic_brotli_decode_alice29() {
CHECK_FOCUS(__func__);
return do_bench_io_buffers(
mimic_brotli_decode,
WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED, tcounter_dst,
&g_brotli_alice29_gt, UINT64_MAX, UINT64_MAX, 5);
}
#endif // WUFFS_MIMIC
// ---------------- Manifest
proc g_tests[] = {
test_wuffs_brotli_decode_alice29,
test_wuffs_brotli_decode_alice29_q11w10,
test_wuffs_brotli_decode_all,
test_wuffs_brotli_decode_all_with_io_limits,
test_wuffs_brotli_decode_interface,
test_wuffs_brotli_decode_metadata,
test_wuffs_brotli_decode_malformed,
test_wuffs_brotli_decode_romeo,
test_wuffs_brotli_decode_trailing_data,
test_wuffs_brotli_decode_truncated,
test_wuffs_brotli_decode_workbuf_length,
// Large window and dictionaries.
test_wuffs_brotli_add_dictionary,
test_wuffs_brotli_decode_ext_all,
test_wuffs_brotli_decode_ext_all_with_io_limits,
test_wuffs_brotli_decode_ext_malformed,
test_wuffs_brotli_decode_large_window_far,
test_wuffs_brotli_decode_large_window_quirk,
// Metablock builder tests.
test_wuffs_brotli_metablock_bit_cost,
test_wuffs_brotli_metablock_build_meta_block,
#ifdef WUFFS_MIMIC
test_mimic_brotli_decode_alice29,
test_mimic_brotli_decode_all,
test_mimic_brotli_decode_differential,
test_mimic_brotli_decode_malformed,
test_mimic_brotli_decode_metadata,
test_mimic_brotli_decode_romeo,
test_mimic_brotli_decode_split_block_length,
// Large window and dictionaries.
test_mimic_brotli_decode_ext_differential,
test_mimic_brotli_decode_ext_malformed,
test_mimic_brotli_decode_large_window_far,
#endif // WUFFS_MIMIC
// Encoder tests.
test_wuffs_brotli_encode_add_dictionary,
test_wuffs_brotli_encode_dictionary_big,
test_wuffs_brotli_encode_dictionary_grid,
test_wuffs_brotli_encode_interface,
test_wuffs_brotli_encode_large_window_far,
test_wuffs_brotli_encode_large_window_files,
test_wuffs_brotli_encode_long_zero_run,
test_wuffs_brotli_encode_metadata_round_trip,
test_wuffs_brotli_encode_parameters,
test_wuffs_brotli_encode_round_trip_corpus,
test_wuffs_brotli_encode_round_trip_lengths,
test_wuffs_brotli_encode_round_trip_synthetic,
#ifdef WUFFS_MIMIC
test_mimic_brotli_encode_dictionary_big,
test_mimic_brotli_encode_dictionary_grid,
test_mimic_brotli_encode_dictionary_rejections,
test_mimic_brotli_encode_large_window_far,
test_mimic_brotli_encode_large_window_files,
test_mimic_brotli_encode_long_zero_run,
test_mimic_brotli_encode_metadata_stale_pointer,
test_mimic_brotli_encode_parity_corpus,
test_mimic_brotli_encode_parity_grid,
test_mimic_brotli_encode_parity_metadata,
test_mimic_brotli_encode_parity_params,
test_mimic_brotli_encode_parity_q2_grid,
test_mimic_brotli_encode_parity_q2_metadata,
test_mimic_brotli_encode_parity_q2_params,
test_mimic_brotli_encode_parity_q10_grid,
test_mimic_brotli_encode_parity_q10_metadata,
test_mimic_brotli_encode_parity_q10_params,
#endif // WUFFS_MIMIC
NULL,
};
proc g_benches[] = {
bench_wuffs_brotli_decode_alice29,
#ifdef WUFFS_MIMIC
bench_mimic_brotli_decode_alice29,
#endif // WUFFS_MIMIC
// Encoder benches.
bench_wuffs_brotli_encode_alice29_q0,
bench_wuffs_brotli_encode_alice29_q1,
bench_wuffs_brotli_encode_alice29_q5,
bench_wuffs_brotli_encode_alice29_q9,
#ifdef WUFFS_MIMIC
bench_mimic_brotli_encode_alice29_q0,
bench_mimic_brotli_encode_alice29_q1,
bench_mimic_brotli_encode_alice29_q5,
bench_mimic_brotli_encode_alice29_q9,
#endif // WUFFS_MIMIC
NULL,
};
int //
main(int argc, char** argv) {
g_proc_package_name = "std/brotli";
return test_main(argc, argv, g_tests, g_benches);
}