mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-20 16:47:28 +01:00
std/brotli: add an encoder for quality levels 0 and 1
Adds a brotli encoder (std/brotli's encoder struct, a base.io_transformer) whose output is byte-identical to libbrotlienc v1.2.1 for quality levels 0 and 1, mirroring encode.c's BrotliEncoderCompressStream(Fast), compress_fragment.c and compress_fragment_two_pass.c, plus the entropy coding and bit stream helpers they need. Every double operation goes through std/softfloat, following the C evaluation order exactly. Parameters are quirks (encode_quirks.wuffs) that mirror BrotliEncoderSetParameter, sanitized at the first transform_io call. Each transform_io resumption corresponds to one C call with available_in equal to the readable src length; a closed src is FINISH; request_flush! and request_emit_metadata! make the next resumption a FLUSH or EMIT_METADATA call. Qualities 2 to 11 return "#unsupported encoder quality" for now. Encoder tables are generated by script/print-brotli-tables.go -encoder. Tests: round trips through the Wuffs decoder over the corpus and synthetic inputs; with -mimic, byte-exact comparisons against libbrotlienc over quality × window × mode × input chunking × flush schedule × metadata × size hint. script/manual-test-brotli-encoder-parity.c runs the full grid: 86,268 configurations, zero mismatches. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011NoosQNe7DKWnqDCBeqw1p
This commit is contained in:
17 files changed
+13990
-2772
No files matched your search
+1062
-1
@@ -33,7 +33,7 @@ the first "./a.out" with "./a.out -bench". Combine these changes with the
|
||||
"wuffs mimic cflags" to run the mimic benchmarks.
|
||||
*/
|
||||
|
||||
// ¿ wuffs mimic cflags: -DWUFFS_MIMIC -lbrotlidec -lbrotlicommon
|
||||
// ¿ wuffs mimic cflags: -DWUFFS_MIMIC -lbrotlienc -lbrotlidec -lbrotlicommon -lm
|
||||
|
||||
// Wuffs ships as a "single file C library" or "header file library" as per
|
||||
// https://github.com/nothings/stb/blob/master/docs/stb_howto.txt
|
||||
@@ -53,6 +53,7 @@ the first "./a.out" with "./a.out -bench". Combine these changes with the
|
||||
#define WUFFS_CONFIG__MODULES
|
||||
#define WUFFS_CONFIG__MODULE__BASE
|
||||
#define WUFFS_CONFIG__MODULE__BROTLI
|
||||
#define WUFFS_CONFIG__MODULE__SOFTFLOAT
|
||||
|
||||
// If building this program in an environment that doesn't easily accommodate
|
||||
// relative includes, you can use the script/inline-c-relative-includes.go
|
||||
@@ -633,6 +634,1036 @@ test_mimic_brotli_decode_malformed() {
|
||||
|
||||
#endif // WUFFS_MIMIC
|
||||
|
||||
// ---------------- Encoder Tests
|
||||
|
||||
// The encoder tests describe each encoding as a "plan": a sequence of steps.
|
||||
// Step i applies the BrotliEncoderOperation g_brotli_enc_step_ops[i] (0 =
|
||||
// PROCESS, 1 = FLUSH, 2 = FINISH, 3 = EMIT_METADATA) to the next
|
||||
// g_brotli_enc_step_lens[i] bytes of the source. The wuffs_brotli_encode_plan
|
||||
// and mimic_brotli_encode functions run a plan with Wuffs' std/brotli encoder
|
||||
// and libbrotlienc, respectively, so that their outputs can be compared.
|
||||
|
||||
#define BROTLI_ENC_OP_PROCESS 0
|
||||
#define BROTLI_ENC_OP_FLUSH 1
|
||||
#define BROTLI_ENC_OP_FINISH 2
|
||||
#define BROTLI_ENC_OP_EMIT_METADATA 3
|
||||
|
||||
// The BrotliEncoderParameter values.
|
||||
#define BROTLI_ENC_PARAM_MODE 0
|
||||
#define BROTLI_ENC_PARAM_QUALITY 1
|
||||
#define BROTLI_ENC_PARAM_LGWIN 2
|
||||
#define BROTLI_ENC_PARAM_LGBLOCK 3
|
||||
#define BROTLI_ENC_PARAM_DISABLE_LITERAL_CONTEXT_MODELING 4
|
||||
#define BROTLI_ENC_PARAM_SIZE_HINT 5
|
||||
#define BROTLI_ENC_PARAM_LARGE_WINDOW 6
|
||||
#define BROTLI_ENC_PARAM_NPOSTFIX 7
|
||||
#define BROTLI_ENC_PARAM_NDIRECT 8
|
||||
#define BROTLI_ENC_PARAM_STREAM_OFFSET 9
|
||||
|
||||
#define BROTLI_ENC_MAX_STEPS (1 << 19)
|
||||
#define BROTLI_ENC_MAX_PARAMS 16
|
||||
|
||||
uint32_t g_brotli_enc_step_ops[BROTLI_ENC_MAX_STEPS];
|
||||
size_t g_brotli_enc_step_lens[BROTLI_ENC_MAX_STEPS];
|
||||
size_t g_brotli_enc_num_steps;
|
||||
|
||||
// The chunkings (how the input is split into steps).
|
||||
#define BROTLI_ENC_CHUNK_SINGLE 0
|
||||
#define BROTLI_ENC_CHUNK_1 1
|
||||
#define BROTLI_ENC_CHUNK_7 2
|
||||
#define BROTLI_ENC_CHUNK_4096 3
|
||||
#define BROTLI_ENC_CHUNK_RANDOM 4
|
||||
#define BROTLI_ENC_NUM_CHUNKINGS 5
|
||||
|
||||
// The flush modes.
|
||||
#define BROTLI_ENC_FLUSH_NONE 0
|
||||
#define BROTLI_ENC_FLUSH_EVERY_CHUNK 1
|
||||
#define BROTLI_ENC_FLUSH_MIDDLE 2
|
||||
#define BROTLI_ENC_NUM_FLUSHES 3
|
||||
|
||||
// The metadata positions.
|
||||
#define BROTLI_ENC_METADATA_NONE 0
|
||||
#define BROTLI_ENC_METADATA_START 1
|
||||
#define BROTLI_ENC_METADATA_MIDDLE 2
|
||||
#define BROTLI_ENC_METADATA_END 3
|
||||
|
||||
typedef struct {
|
||||
uint32_t param_keys[BROTLI_ENC_MAX_PARAMS];
|
||||
uint32_t param_values[BROTLI_ENC_MAX_PARAMS];
|
||||
size_t num_params;
|
||||
uint32_t chunking;
|
||||
uint32_t flush;
|
||||
uint32_t metadata_position;
|
||||
size_t metadata_len;
|
||||
size_t dst_chunk;
|
||||
uint64_t seed;
|
||||
} brotli_enc_config;
|
||||
|
||||
void //
|
||||
brotli_enc_config__add_param(brotli_enc_config* c,
|
||||
uint32_t key,
|
||||
uint32_t value) {
|
||||
if (c->num_params < BROTLI_ENC_MAX_PARAMS) {
|
||||
c->param_keys[c->num_params] = key;
|
||||
c->param_values[c->num_params] = value;
|
||||
c->num_params++;
|
||||
}
|
||||
}
|
||||
|
||||
uint64_t //
|
||||
brotli_enc_rand(uint64_t* state) {
|
||||
// SplitMix64.
|
||||
uint64_t z = (*state += 0x9E3779B97F4A7C15ull);
|
||||
z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ull;
|
||||
z = (z ^ (z >> 27)) * 0x94D049BB133111EBull;
|
||||
return z ^ (z >> 31);
|
||||
}
|
||||
|
||||
void //
|
||||
brotli_enc_fill_random(uint8_t* p, size_t n, uint64_t seed) {
|
||||
uint64_t state = seed;
|
||||
for (size_t i = 0; i < n; i++) {
|
||||
p[i] = (uint8_t)(brotli_enc_rand(&state) >> 56);
|
||||
}
|
||||
}
|
||||
|
||||
const char* //
|
||||
brotli_enc_add_step(uint32_t op, size_t len) {
|
||||
if (g_brotli_enc_num_steps >= BROTLI_ENC_MAX_STEPS) {
|
||||
return "too many steps";
|
||||
}
|
||||
g_brotli_enc_step_ops[g_brotli_enc_num_steps] = op;
|
||||
g_brotli_enc_step_lens[g_brotli_enc_num_steps] = len;
|
||||
g_brotli_enc_num_steps++;
|
||||
return NULL;
|
||||
}
|
||||
|
||||
// make_brotli_enc_plan sets the g_brotli_enc_step_* plan for encoding the
|
||||
// data[:data_len] input with the config c. It also writes the plan's source
|
||||
// (the input, with any metadata bytes interleaved) to src.
|
||||
const char* //
|
||||
make_brotli_enc_plan(wuffs_base__io_buffer* src,
|
||||
const brotli_enc_config* c,
|
||||
const uint8_t* data,
|
||||
size_t data_len) {
|
||||
g_brotli_enc_num_steps = 0;
|
||||
src->meta.ri = 0;
|
||||
src->meta.wi = 0;
|
||||
src->meta.pos = 0;
|
||||
src->meta.closed = true;
|
||||
if ((data_len + c->metadata_len) > src->data.len) {
|
||||
return "make_brotli_enc_plan: src is too short";
|
||||
}
|
||||
uint64_t rng = c->seed;
|
||||
size_t metadata_at = SIZE_MAX;
|
||||
if (c->metadata_position == BROTLI_ENC_METADATA_START) {
|
||||
metadata_at = 0;
|
||||
} else if (c->metadata_position == BROTLI_ENC_METADATA_MIDDLE) {
|
||||
metadata_at = data_len / 2;
|
||||
} else if (c->metadata_position == BROTLI_ENC_METADATA_END) {
|
||||
metadata_at = data_len;
|
||||
}
|
||||
bool flushed_middle = false;
|
||||
|
||||
size_t i = 0;
|
||||
while (true) {
|
||||
if (i == metadata_at) {
|
||||
brotli_enc_fill_random(src->data.ptr + src->meta.wi, c->metadata_len,
|
||||
c->seed ^ 0x5A5A);
|
||||
src->meta.wi += c->metadata_len;
|
||||
CHECK_STRING(
|
||||
brotli_enc_add_step(BROTLI_ENC_OP_EMIT_METADATA, c->metadata_len));
|
||||
metadata_at = SIZE_MAX;
|
||||
}
|
||||
|
||||
size_t n = data_len - i;
|
||||
switch (c->chunking) {
|
||||
case BROTLI_ENC_CHUNK_1:
|
||||
n = 1;
|
||||
break;
|
||||
case BROTLI_ENC_CHUNK_7:
|
||||
n = 7;
|
||||
break;
|
||||
case BROTLI_ENC_CHUNK_4096:
|
||||
n = 4096;
|
||||
break;
|
||||
case BROTLI_ENC_CHUNK_RANDOM: {
|
||||
uint64_t r = brotli_enc_rand(&rng);
|
||||
// Mostly small chunks, with occasional large ones.
|
||||
n = (size_t)(1 + ((r & 0xFF) < 0xF0 ? ((r >> 8) % 300)
|
||||
: ((r >> 8) % 70000)));
|
||||
break;
|
||||
}
|
||||
}
|
||||
if ((metadata_at != SIZE_MAX) && (metadata_at > i)) {
|
||||
n = (n < (metadata_at - i)) ? n : (metadata_at - i);
|
||||
}
|
||||
if (n > (data_len - i)) {
|
||||
n = data_len - i;
|
||||
}
|
||||
|
||||
memcpy(src->data.ptr + src->meta.wi, data + i, n);
|
||||
src->meta.wi += n;
|
||||
i += n;
|
||||
bool last = (i == data_len) && (metadata_at == SIZE_MAX);
|
||||
|
||||
if (last) {
|
||||
CHECK_STRING(brotli_enc_add_step(BROTLI_ENC_OP_FINISH, n));
|
||||
break;
|
||||
} else if (c->flush == BROTLI_ENC_FLUSH_EVERY_CHUNK) {
|
||||
CHECK_STRING(brotli_enc_add_step(BROTLI_ENC_OP_FLUSH, n));
|
||||
} else {
|
||||
CHECK_STRING(brotli_enc_add_step(BROTLI_ENC_OP_PROCESS, n));
|
||||
if ((c->flush == BROTLI_ENC_FLUSH_MIDDLE) && !flushed_middle &&
|
||||
((2 * i) >= data_len)) {
|
||||
flushed_middle = true;
|
||||
CHECK_STRING(brotli_enc_add_step(BROTLI_ENC_OP_FLUSH, 0));
|
||||
}
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
// brotli_enc_quirk_key maps a BrotliEncoderParameter value to the
|
||||
// corresponding std/brotli quirk key (or 0 if there is none).
|
||||
uint32_t //
|
||||
brotli_enc_quirk_key(uint32_t param) {
|
||||
switch (param) {
|
||||
case 0:
|
||||
return WUFFS_BROTLI__QUIRK_MODE;
|
||||
case 1:
|
||||
return WUFFS_BROTLI__QUIRK_ENCODER_QUALITY;
|
||||
case 2:
|
||||
return WUFFS_BROTLI__QUIRK_LGWIN;
|
||||
case 3:
|
||||
return WUFFS_BROTLI__QUIRK_LGBLOCK;
|
||||
case 4:
|
||||
return WUFFS_BROTLI__QUIRK_DISABLE_LITERAL_CONTEXT_MODELING;
|
||||
case 5:
|
||||
return WUFFS_BROTLI__QUIRK_SIZE_HINT;
|
||||
case 6:
|
||||
return WUFFS_BROTLI__QUIRK_LARGE_WINDOW;
|
||||
case 7:
|
||||
return WUFFS_BROTLI__QUIRK_NPOSTFIX;
|
||||
case 8:
|
||||
return WUFFS_BROTLI__QUIRK_NDIRECT;
|
||||
case 9:
|
||||
return WUFFS_BROTLI__QUIRK_STREAM_OFFSET;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
// wuffs_brotli_encode_plan is the Wuffs equivalent of mimic_brotli_encode
|
||||
// (in test/c/mimiclib/brotli.c).
|
||||
const char* //
|
||||
wuffs_brotli_encode_plan(wuffs_base__io_buffer* dst,
|
||||
wuffs_base__io_buffer* src,
|
||||
const uint32_t* param_keys,
|
||||
const uint32_t* param_values,
|
||||
size_t num_params,
|
||||
const uint32_t* step_ops,
|
||||
const size_t* step_lens,
|
||||
size_t num_steps,
|
||||
size_t dst_chunk) {
|
||||
// The encoder struct is large (more than 1 MiB), mostly arrays that do not
|
||||
// need zero-initializing, so don't use wuffs_brotli__encoder__alloc.
|
||||
wuffs_brotli__encoder* enc = malloc(sizeof__wuffs_brotli__encoder());
|
||||
if (!enc) {
|
||||
return "malloc failed";
|
||||
}
|
||||
const char* ret = wuffs_brotli__encoder__initialize(
|
||||
enc, sizeof__wuffs_brotli__encoder(), WUFFS_VERSION,
|
||||
WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED)
|
||||
.repr;
|
||||
if (ret) {
|
||||
goto cleanup;
|
||||
}
|
||||
for (size_t j = 0; j < num_params; j++) {
|
||||
wuffs_base__status status = wuffs_brotli__encoder__set_quirk(
|
||||
enc, brotli_enc_quirk_key(param_keys[j]),
|
||||
param_values[j]);
|
||||
if (status.repr) {
|
||||
ret = status.repr;
|
||||
goto cleanup;
|
||||
}
|
||||
}
|
||||
wuffs_base__range_ii_u64 workbuf_len =
|
||||
wuffs_brotli__encoder__workbuf_len(enc);
|
||||
if (workbuf_len.max_incl > g_work_slice_u8.len) {
|
||||
ret = "workbuf_len is too large";
|
||||
goto cleanup;
|
||||
}
|
||||
wuffs_base__slice_u8 workbuf = wuffs_base__make_slice_u8(
|
||||
g_work_slice_u8.ptr, (size_t)workbuf_len.max_incl);
|
||||
|
||||
for (size_t i = 0; i < num_steps; i++) {
|
||||
size_t end = src->meta.ri + step_lens[i];
|
||||
if (end > src->meta.wi) {
|
||||
ret = "wuffs_brotli_encode_plan: step length exceeds the input";
|
||||
goto cleanup;
|
||||
}
|
||||
uint32_t op = step_ops[i];
|
||||
wuffs_base__status status = wuffs_base__make_status(NULL);
|
||||
if (op == BROTLI_ENC_OP_FLUSH) {
|
||||
status = wuffs_brotli__encoder__request_flush(enc);
|
||||
} else if (op == BROTLI_ENC_OP_EMIT_METADATA) {
|
||||
status = wuffs_brotli__encoder__request_emit_metadata(enc, step_lens[i]);
|
||||
}
|
||||
if (status.repr) {
|
||||
ret = status.repr;
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
while (true) {
|
||||
wuffs_base__io_buffer d = *dst;
|
||||
if ((dst_chunk > 0) && ((d.data.len - d.meta.wi) > dst_chunk)) {
|
||||
d.data.len = d.meta.wi + dst_chunk;
|
||||
}
|
||||
wuffs_base__io_buffer s = *src;
|
||||
s.meta.wi = end;
|
||||
s.meta.closed = (op == BROTLI_ENC_OP_FINISH);
|
||||
status = wuffs_brotli__encoder__transform_io(enc, &d, &s, workbuf);
|
||||
dst->meta.wi = d.meta.wi;
|
||||
src->meta.ri = s.meta.ri;
|
||||
|
||||
if (status.repr == NULL) {
|
||||
if (op != BROTLI_ENC_OP_FINISH) {
|
||||
ret = "wuffs_brotli_encode_plan: unexpected ok status";
|
||||
goto cleanup;
|
||||
}
|
||||
break;
|
||||
} else if (status.repr == wuffs_base__suspension__short_write) {
|
||||
if (dst->meta.wi == dst->data.len) {
|
||||
ret = "wuffs_brotli_encode_plan: dst is full";
|
||||
goto cleanup;
|
||||
}
|
||||
continue;
|
||||
} else if ((status.repr == wuffs_base__suspension__short_read) &&
|
||||
(op != BROTLI_ENC_OP_FINISH) && (src->meta.ri == end)) {
|
||||
break;
|
||||
}
|
||||
ret = status.repr;
|
||||
goto cleanup;
|
||||
}
|
||||
}
|
||||
|
||||
cleanup:
|
||||
free(enc);
|
||||
return ret;
|
||||
}
|
||||
|
||||
// wuffs_brotli_encode encodes src to dst in one go, at the given quality and
|
||||
// lgwin, with a dst limit of wlimit bytes per transform_io call.
|
||||
const char* //
|
||||
wuffs_brotli_encode(wuffs_base__io_buffer* dst,
|
||||
wuffs_base__io_buffer* src,
|
||||
uint32_t quality,
|
||||
uint32_t lgwin,
|
||||
size_t dst_chunk) {
|
||||
uint32_t keys[2] = {BROTLI_ENC_PARAM_QUALITY, BROTLI_ENC_PARAM_LGWIN};
|
||||
uint32_t values[2] = {quality, lgwin};
|
||||
uint32_t op = BROTLI_ENC_OP_FINISH;
|
||||
size_t len = wuffs_base__io_buffer__reader_length(src);
|
||||
return wuffs_brotli_encode_plan(dst, src, keys, values, 2, &op, &len, 1,
|
||||
dst_chunk);
|
||||
}
|
||||
|
||||
// do_test_brotli_encode_round_trip encodes data[:data_len] (with the plan's
|
||||
// config c) and then decodes it (with Wuffs' std/brotli decoder), checking
|
||||
// that the result is the original data.
|
||||
const char* //
|
||||
do_test_brotli_encode_round_trip(const char* name,
|
||||
const brotli_enc_config* c,
|
||||
const uint8_t* data,
|
||||
size_t data_len) {
|
||||
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
|
||||
.data = g_src_slice_u8,
|
||||
});
|
||||
CHECK_STRING(make_brotli_enc_plan(&src, c, data, data_len));
|
||||
wuffs_base__io_buffer have = ((wuffs_base__io_buffer){
|
||||
.data = g_have_slice_u8,
|
||||
});
|
||||
const char* status = wuffs_brotli_encode_plan(
|
||||
&have, &src, c->param_keys, c->param_values, c->num_params,
|
||||
g_brotli_enc_step_ops, g_brotli_enc_step_lens, g_brotli_enc_num_steps,
|
||||
c->dst_chunk);
|
||||
if (status) {
|
||||
RETURN_FAIL("%s: encode: %s", name, status);
|
||||
}
|
||||
have.meta.closed = true;
|
||||
|
||||
wuffs_base__io_buffer decoded = ((wuffs_base__io_buffer){
|
||||
.data = g_want_slice_u8,
|
||||
});
|
||||
status = wuffs_brotli_decode(&decoded, &have, WUFFS_INITIALIZE__DEFAULT_OPTIONS,
|
||||
UINT64_MAX, UINT64_MAX);
|
||||
if (status) {
|
||||
RETURN_FAIL("%s: decode: %s", name, status);
|
||||
} else if (have.meta.ri != have.meta.wi) {
|
||||
RETURN_FAIL("%s: decode: trailing data", name);
|
||||
} else if (decoded.meta.wi != data_len) {
|
||||
RETURN_FAIL("%s: decoded length: have %zu, want %zu", name,
|
||||
decoded.meta.wi, data_len);
|
||||
} else if (memcmp(decoded.data.ptr, data, data_len)) {
|
||||
RETURN_FAIL("%s: decoded bytes differ", name);
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
// brotli_enc_corpus_filenames lists the round trip (and parity) test inputs:
|
||||
// every file in test/data/brotli plus some others.
|
||||
const char* brotli_enc_corpus_filenames[] = {
|
||||
"test/data/256.bytes",
|
||||
"test/data/enwik5",
|
||||
"test/data/romeo.txt",
|
||||
"test/data/brotli/README.md",
|
||||
};
|
||||
|
||||
// for_each_brotli_enc_corpus_file calls f on every corpus file, loading each
|
||||
// one into a malloc'ed buffer.
|
||||
const char* //
|
||||
for_each_brotli_enc_corpus_file(
|
||||
const char* (*f)(const char* name, const uint8_t* data, size_t len)) {
|
||||
const char* names[256];
|
||||
size_t num_names = 0;
|
||||
for (size_t i = 0; i < WUFFS_TESTLIB_ARRAY_SIZE(brotli_enc_corpus_filenames);
|
||||
i++) {
|
||||
names[num_names++] = brotli_enc_corpus_filenames[i];
|
||||
}
|
||||
for (size_t i = 0; i < WUFFS_TESTLIB_ARRAY_SIZE(g_brotli_vectors); i++) {
|
||||
names[num_names++] = g_brotli_vectors[i].src_filename;
|
||||
names[num_names++] = g_brotli_vectors[i].want_filename;
|
||||
}
|
||||
for (size_t i = 0; i < WUFFS_TESTLIB_ARRAY_SIZE(g_brotli_malformed); i++) {
|
||||
if (g_brotli_malformed[i].src_filename) {
|
||||
names[num_names++] = g_brotli_malformed[i].src_filename;
|
||||
}
|
||||
}
|
||||
|
||||
uint8_t* buf = malloc(1 << 20);
|
||||
if (!buf) {
|
||||
return "malloc failed";
|
||||
}
|
||||
const char* ret = NULL;
|
||||
for (size_t i = 0; i < num_names; i++) {
|
||||
bool dup = false;
|
||||
for (size_t j = 0; j < i; j++) {
|
||||
if (!strcmp(names[i], names[j])) {
|
||||
dup = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (dup) {
|
||||
continue;
|
||||
}
|
||||
wuffs_base__io_buffer b = ((wuffs_base__io_buffer){
|
||||
.data = wuffs_base__make_slice_u8(buf, 1 << 20),
|
||||
});
|
||||
ret = read_file(&b, names[i]);
|
||||
if (!ret) {
|
||||
ret = f(names[i], b.data.ptr, b.meta.wi);
|
||||
}
|
||||
if (ret) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
free(buf);
|
||||
return ret;
|
||||
}
|
||||
|
||||
// brotli_enc_synthetic makes one of the synthetic test inputs: 0 is 1 MiB of
|
||||
// zeroes, 1 is 256 KiB of pseudo-random bytes and 2 is a 100-byte pattern
|
||||
// repeated to 64 KiB. It returns a malloc'ed buffer.
|
||||
uint8_t* //
|
||||
brotli_enc_synthetic(int which, size_t* len) {
|
||||
size_t n = (which == 0) ? (1 << 20) : (which == 1) ? (256 << 10) : (64 << 10);
|
||||
uint8_t* p = malloc(n);
|
||||
if (!p) {
|
||||
return NULL;
|
||||
}
|
||||
if (which == 0) {
|
||||
memset(p, 0, n);
|
||||
} else if (which == 1) {
|
||||
brotli_enc_fill_random(p, n, 0x1234);
|
||||
} else {
|
||||
uint8_t pattern[100];
|
||||
brotli_enc_fill_random(pattern, 100, 0x5678);
|
||||
for (size_t i = 0; i < n; i++) {
|
||||
p[i] = pattern[i % 100];
|
||||
}
|
||||
}
|
||||
*len = n;
|
||||
return p;
|
||||
}
|
||||
|
||||
brotli_enc_config //
|
||||
make_brotli_enc_config(uint32_t quality, uint32_t lgwin) {
|
||||
brotli_enc_config c;
|
||||
memset(&c, 0, sizeof c);
|
||||
brotli_enc_config__add_param(&c, BROTLI_ENC_PARAM_QUALITY, quality);
|
||||
brotli_enc_config__add_param(&c, BROTLI_ENC_PARAM_LGWIN, lgwin);
|
||||
return c;
|
||||
}
|
||||
|
||||
const char* //
|
||||
do_test_brotli_encode_round_trip_file(const char* name,
|
||||
const uint8_t* data,
|
||||
size_t len) {
|
||||
for (uint32_t q = 0; q <= 1; q++) {
|
||||
for (int k = 0; k < 3; k++) {
|
||||
brotli_enc_config c = make_brotli_enc_config(q, (k == 0) ? 10 : 22);
|
||||
if (k == 2) {
|
||||
c.chunking = BROTLI_ENC_CHUNK_RANDOM;
|
||||
c.flush = BROTLI_ENC_FLUSH_MIDDLE;
|
||||
c.dst_chunk = 97;
|
||||
c.seed = len;
|
||||
}
|
||||
char s[256];
|
||||
snprintf(s, sizeof s, "%s (q=%u, k=%d)", name, q, k);
|
||||
CHECK_STRING(do_test_brotli_encode_round_trip(s, &c, data, len));
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const char* //
|
||||
test_wuffs_brotli_encode_round_trip_corpus() {
|
||||
CHECK_FOCUS(__func__);
|
||||
return for_each_brotli_enc_corpus_file(do_test_brotli_encode_round_trip_file);
|
||||
}
|
||||
|
||||
const char* //
|
||||
test_wuffs_brotli_encode_round_trip_lengths() {
|
||||
CHECK_FOCUS(__func__);
|
||||
uint8_t data[40];
|
||||
brotli_enc_fill_random(data, 20, 0x9999);
|
||||
memcpy(data + 20, data, 20);
|
||||
for (size_t n = 0; n <= 40; n++) {
|
||||
char s[64];
|
||||
snprintf(s, sizeof s, "length %zu", n);
|
||||
CHECK_STRING(do_test_brotli_encode_round_trip_file(s, data, n));
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const char* //
|
||||
test_wuffs_brotli_encode_round_trip_synthetic() {
|
||||
CHECK_FOCUS(__func__);
|
||||
for (int which = 0; which < 3; which++) {
|
||||
size_t len = 0;
|
||||
uint8_t* data = brotli_enc_synthetic(which, &len);
|
||||
if (!data) {
|
||||
RETURN_FAIL("malloc failed");
|
||||
}
|
||||
char s[64];
|
||||
snprintf(s, sizeof s, "synthetic #%d", which);
|
||||
const char* status = do_test_brotli_encode_round_trip_file(s, data, len);
|
||||
free(data);
|
||||
if (status) {
|
||||
return status;
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const char* //
|
||||
test_wuffs_brotli_encode_metadata_round_trip() {
|
||||
CHECK_FOCUS(__func__);
|
||||
wuffs_base__io_buffer data = ((wuffs_base__io_buffer){
|
||||
.data = g_mimiclib_scratch_slice_u8,
|
||||
});
|
||||
CHECK_STRING(read_file(&data, "test/data/romeo.txt"));
|
||||
static const size_t lens[] = {0, 1, 100, 70000};
|
||||
for (uint32_t q = 0; q <= 1; q++) {
|
||||
for (size_t i = 0; i < WUFFS_TESTLIB_ARRAY_SIZE(lens); i++) {
|
||||
for (uint32_t pos = 1; pos <= 3; pos++) {
|
||||
brotli_enc_config c = make_brotli_enc_config(q, 18);
|
||||
c.chunking = BROTLI_ENC_CHUNK_7;
|
||||
c.metadata_position = pos;
|
||||
c.metadata_len = lens[i];
|
||||
c.dst_chunk = (pos == 2) ? 3 : 0;
|
||||
char s[64];
|
||||
snprintf(s, sizeof s, "q=%u, len=%zu, pos=%u", q, lens[i], pos);
|
||||
CHECK_STRING(do_test_brotli_encode_round_trip(s, &c, data.data.ptr,
|
||||
data.meta.wi));
|
||||
}
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const char* //
|
||||
test_wuffs_brotli_encode_parameters() {
|
||||
CHECK_FOCUS(__func__);
|
||||
wuffs_brotli__encoder* enc = wuffs_brotli__encoder__alloc();
|
||||
if (!enc) {
|
||||
RETURN_FAIL("alloc failed");
|
||||
}
|
||||
const char* ret = NULL;
|
||||
wuffs_base__status status;
|
||||
uint8_t buf[64];
|
||||
wuffs_base__io_buffer dst =
|
||||
wuffs_base__ptr_u8__writer(buf, sizeof buf);
|
||||
wuffs_base__io_buffer src = make_io_buffer_from_string("", 0);
|
||||
|
||||
#define BROTLI_ENC_CHECK(cond, msg) \
|
||||
if (!(cond)) { \
|
||||
ret = msg; \
|
||||
goto cleanup; \
|
||||
}
|
||||
|
||||
// Unknown keys and out-of-range values.
|
||||
status = wuffs_brotli__encoder__set_quirk(
|
||||
enc, WUFFS_BROTLI__QUIRK_STREAM_OFFSET + 1, 0);
|
||||
BROTLI_ENC_CHECK(status.repr == wuffs_base__error__unsupported_option,
|
||||
"unknown key");
|
||||
status = wuffs_brotli__encoder__set_quirk(enc, 12345, 0);
|
||||
BROTLI_ENC_CHECK(status.repr == wuffs_base__error__unsupported_option,
|
||||
"unknown key (2)");
|
||||
status = wuffs_brotli__encoder__set_quirk(
|
||||
enc, WUFFS_BROTLI__QUIRK_ENCODER_QUALITY, 0x100000000ull);
|
||||
BROTLI_ENC_CHECK(status.repr == wuffs_base__error__bad_argument,
|
||||
"value > 0xFFFFFFFF");
|
||||
status = wuffs_brotli__encoder__set_quirk(
|
||||
enc, WUFFS_BROTLI__QUIRK_DISABLE_LITERAL_CONTEXT_MODELING, 2);
|
||||
BROTLI_ENC_CHECK(status.repr == wuffs_base__error__bad_argument,
|
||||
"disable_lcm > 1");
|
||||
status = wuffs_brotli__encoder__set_quirk(
|
||||
enc, WUFFS_BROTLI__QUIRK_STREAM_OFFSET, (1 << 30) + 1);
|
||||
BROTLI_ENC_CHECK(status.repr == wuffs_base__error__bad_argument,
|
||||
"stream_offset > 1<<30");
|
||||
status = wuffs_brotli__encoder__request_emit_metadata(enc, (1 << 24) + 1);
|
||||
BROTLI_ENC_CHECK(status.repr == wuffs_base__error__bad_argument,
|
||||
"metadata length > 1<<24");
|
||||
|
||||
// Raw values are stored as is, and sanitized later.
|
||||
status = wuffs_brotli__encoder__set_quirk(
|
||||
enc, WUFFS_BROTLI__QUIRK_LGWIN, 99);
|
||||
BROTLI_ENC_CHECK(!status.repr, "set lgwin");
|
||||
BROTLI_ENC_CHECK(wuffs_brotli__encoder__get_quirk(
|
||||
enc, WUFFS_BROTLI__QUIRK_LGWIN) == 99,
|
||||
"get lgwin");
|
||||
// The default quality is 11, which is unsupported.
|
||||
BROTLI_ENC_CHECK(wuffs_brotli__encoder__workbuf_len(enc).max_incl == 0,
|
||||
"workbuf_len at q11");
|
||||
status = wuffs_brotli__encoder__set_quirk(
|
||||
enc, WUFFS_BROTLI__QUIRK_ENCODER_QUALITY, 0xFFFFFFFF); // (int)-1.
|
||||
BROTLI_ENC_CHECK(!status.repr, "set quality");
|
||||
// Quality -1 is sanitized to 0 and lgwin 99 to 24.
|
||||
BROTLI_ENC_CHECK(wuffs_brotli__encoder__workbuf_len(enc).max_incl ==
|
||||
((2u << 24) + 503 + 16),
|
||||
"workbuf_len at q0w24");
|
||||
status = wuffs_brotli__encoder__set_quirk(
|
||||
enc, WUFFS_BROTLI__QUIRK_LGWIN, 0xFFFFFFFF); // (int)-1.
|
||||
BROTLI_ENC_CHECK(!status.repr, "set lgwin (2)");
|
||||
BROTLI_ENC_CHECK(wuffs_brotli__encoder__workbuf_len(enc).max_incl ==
|
||||
((2u << 10) + 503 + 16),
|
||||
"workbuf_len at q0w10");
|
||||
|
||||
// max_compressed_size mirrors BrotliEncoderMaxCompressedSize.
|
||||
BROTLI_ENC_CHECK(wuffs_brotli__encoder__max_compressed_size(enc, 0) == 2,
|
||||
"max_compressed_size(0)");
|
||||
BROTLI_ENC_CHECK(
|
||||
wuffs_brotli__encoder__max_compressed_size(enc, 1 << 20) ==
|
||||
((1 << 20) + 6 + (4 * 64)),
|
||||
"max_compressed_size(1<<20)");
|
||||
BROTLI_ENC_CHECK(
|
||||
wuffs_brotli__encoder__max_compressed_size(enc, UINT64_MAX) == 0,
|
||||
"max_compressed_size(overflow)");
|
||||
|
||||
// A too-short workbuf.
|
||||
src.meta.closed = false;
|
||||
status = wuffs_brotli__encoder__transform_io(
|
||||
enc, &dst, &src, wuffs_base__make_slice_u8(g_work_slice_u8.ptr, 10));
|
||||
BROTLI_ENC_CHECK(status.repr == wuffs_base__suspension__short_read,
|
||||
"empty input");
|
||||
|
||||
// Parameters can't be changed after the first transform_io call.
|
||||
status = wuffs_brotli__encoder__set_quirk(
|
||||
enc, WUFFS_BROTLI__QUIRK_LGWIN, 16);
|
||||
BROTLI_ENC_CHECK(status.repr == wuffs_base__error__bad_call_sequence,
|
||||
"set_quirk after transform_io");
|
||||
|
||||
src.data.ptr = (uint8_t*)"abc";
|
||||
src.data.len = 3;
|
||||
src.meta.wi = 3;
|
||||
status = wuffs_brotli__encoder__transform_io(
|
||||
enc, &dst, &src, wuffs_base__make_slice_u8(g_work_slice_u8.ptr, 10));
|
||||
BROTLI_ENC_CHECK(status.repr == wuffs_base__error__bad_workbuf_length,
|
||||
"short workbuf");
|
||||
|
||||
// Qualities 2 to 11 are unsupported.
|
||||
for (uint32_t q = 2; q <= 12; q++) {
|
||||
free(enc);
|
||||
enc = wuffs_brotli__encoder__alloc();
|
||||
if (!enc) {
|
||||
RETURN_FAIL("alloc failed");
|
||||
}
|
||||
wuffs_brotli__encoder__set_quirk(enc, WUFFS_BROTLI__QUIRK_ENCODER_QUALITY,
|
||||
q);
|
||||
status = wuffs_brotli__encoder__transform_io(enc, &dst, &src,
|
||||
g_work_slice_u8);
|
||||
BROTLI_ENC_CHECK(
|
||||
status.repr == wuffs_brotli__error__unsupported_encoder_quality,
|
||||
"quality 2..11");
|
||||
}
|
||||
|
||||
// Requests.
|
||||
free(enc);
|
||||
enc = wuffs_brotli__encoder__alloc();
|
||||
if (!enc) {
|
||||
RETURN_FAIL("alloc failed");
|
||||
}
|
||||
status = wuffs_brotli__encoder__request_emit_metadata(enc, 1);
|
||||
BROTLI_ENC_CHECK(!status.repr, "request_emit_metadata");
|
||||
status = wuffs_brotli__encoder__request_flush(enc);
|
||||
BROTLI_ENC_CHECK(status.repr == wuffs_base__error__bad_call_sequence,
|
||||
"request_flush while metadata is pending");
|
||||
status = wuffs_brotli__encoder__request_emit_metadata(enc, 1);
|
||||
BROTLI_ENC_CHECK(status.repr == wuffs_base__error__bad_call_sequence,
|
||||
"request_emit_metadata twice");
|
||||
|
||||
// A closed src with too few metadata bytes.
|
||||
wuffs_brotli__encoder__set_quirk(enc, WUFFS_BROTLI__QUIRK_ENCODER_QUALITY, 1);
|
||||
src = make_io_buffer_from_string("", 0);
|
||||
status = wuffs_brotli__encoder__transform_io(enc, &dst, &src,
|
||||
g_work_slice_u8);
|
||||
BROTLI_ENC_CHECK(status.repr == wuffs_brotli__error__truncated_input,
|
||||
"truncated metadata");
|
||||
|
||||
#undef BROTLI_ENC_CHECK
|
||||
|
||||
cleanup:
|
||||
free(enc);
|
||||
if (ret) {
|
||||
RETURN_FAIL("%s: status \"%s\"", ret, status.repr);
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const char* //
|
||||
test_wuffs_brotli_encode_interface() {
|
||||
CHECK_FOCUS(__func__);
|
||||
wuffs_brotli__encoder* enc = wuffs_brotli__encoder__alloc();
|
||||
if (!enc) {
|
||||
RETURN_FAIL("alloc failed");
|
||||
}
|
||||
wuffs_brotli__encoder__set_quirk(enc, WUFFS_BROTLI__QUIRK_ENCODER_QUALITY, 1);
|
||||
// Encoding romeo.txt at q1w22 gives 575 bytes, the last of which is 0xC3.
|
||||
const char* status = do_test__wuffs_base__io_transformer(
|
||||
wuffs_brotli__encoder__upcast_as__wuffs_base__io_transformer(enc),
|
||||
"test/data/romeo.txt", 0, SIZE_MAX, 575, 0xC3);
|
||||
free(enc);
|
||||
return status;
|
||||
}
|
||||
|
||||
// ---------------- Encoder Mimic Tests
|
||||
|
||||
#ifdef WUFFS_MIMIC
|
||||
|
||||
// do_test_mimic_brotli_encode checks that Wuffs and libbrotlienc produce
|
||||
// exactly the same bytes for encoding data[:data_len] with the config c.
|
||||
const char* //
|
||||
do_test_mimic_brotli_encode(const char* name,
|
||||
const brotli_enc_config* c,
|
||||
const uint8_t* data,
|
||||
size_t data_len) {
|
||||
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
|
||||
.data = g_src_slice_u8,
|
||||
});
|
||||
CHECK_STRING(make_brotli_enc_plan(&src, c, data, data_len));
|
||||
wuffs_base__io_buffer have = ((wuffs_base__io_buffer){
|
||||
.data = g_have_slice_u8,
|
||||
});
|
||||
wuffs_base__io_buffer want = ((wuffs_base__io_buffer){
|
||||
.data = g_want_slice_u8,
|
||||
});
|
||||
|
||||
wuffs_base__io_buffer src0 = src;
|
||||
const char* have_status = wuffs_brotli_encode_plan(
|
||||
&have, &src0, c->param_keys, c->param_values, c->num_params,
|
||||
g_brotli_enc_step_ops, g_brotli_enc_step_lens, g_brotli_enc_num_steps,
|
||||
c->dst_chunk);
|
||||
if (have_status) {
|
||||
RETURN_FAIL("%s: wuffs: %s", name, have_status);
|
||||
}
|
||||
wuffs_base__io_buffer src1 = src;
|
||||
const char* want_status = mimic_brotli_encode(
|
||||
&want, &src1, c->param_keys, c->param_values, c->num_params,
|
||||
g_brotli_enc_step_ops, g_brotli_enc_step_lens, g_brotli_enc_num_steps,
|
||||
c->dst_chunk);
|
||||
if (want_status) {
|
||||
RETURN_FAIL("%s: mimic: %s", name, want_status);
|
||||
}
|
||||
const char* status = check_io_buffers_equal("", &have, &want);
|
||||
if (status) {
|
||||
RETURN_FAIL("%s: %s", name, status);
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const char* brotli_enc_parity_filenames[] = {
|
||||
"test/data/256.bytes",
|
||||
"test/data/romeo.txt",
|
||||
"test/data/brotli/alice29.txt",
|
||||
"test/data/brotli/random_org_10k.bin",
|
||||
};
|
||||
|
||||
const char* //
|
||||
test_mimic_brotli_encode_parity_grid() {
|
||||
CHECK_FOCUS(__func__);
|
||||
static const uint32_t lgwins[] = {10, 16, 18, 22, 24};
|
||||
wuffs_base__io_buffer data = ((wuffs_base__io_buffer){
|
||||
.data = g_mimiclib_scratch_slice_u8,
|
||||
});
|
||||
for (size_t f = 0; f < WUFFS_TESTLIB_ARRAY_SIZE(brotli_enc_parity_filenames);
|
||||
f++) {
|
||||
data.meta.ri = 0;
|
||||
data.meta.wi = 0;
|
||||
data.meta.closed = false;
|
||||
CHECK_STRING(read_file(&data, brotli_enc_parity_filenames[f]));
|
||||
for (uint32_t q = 0; q <= 1; q++) {
|
||||
for (size_t w = 0; w < WUFFS_TESTLIB_ARRAY_SIZE(lgwins); w++) {
|
||||
for (uint32_t mode = 0; mode <= 2; mode++) {
|
||||
for (uint32_t ch = 0; ch < BROTLI_ENC_NUM_CHUNKINGS; ch++) {
|
||||
if ((ch == BROTLI_ENC_CHUNK_1) && (data.meta.wi > 4096)) {
|
||||
continue;
|
||||
}
|
||||
for (uint32_t fl = 0; fl < BROTLI_ENC_NUM_FLUSHES; fl++) {
|
||||
brotli_enc_config c = make_brotli_enc_config(q, lgwins[w]);
|
||||
brotli_enc_config__add_param(&c, BROTLI_ENC_PARAM_MODE, mode);
|
||||
c.chunking = ch;
|
||||
c.flush = fl;
|
||||
c.seed = (f << 16) | (w << 8) | (mode << 4) | fl;
|
||||
// To keep this test fast, the 7-byte chunking only uses the
|
||||
// first 4 KiB of the input. The
|
||||
// script/manual-test-brotli-encoder-parity.c program runs
|
||||
// the whole input.
|
||||
size_t n = data.meta.wi;
|
||||
if ((ch == BROTLI_ENC_CHUNK_7) && (n > 4096)) {
|
||||
n = 4096;
|
||||
}
|
||||
char s[256];
|
||||
snprintf(s, sizeof s,
|
||||
"%s: q=%u lgwin=%u mode=%u chunking=%u flush=%u",
|
||||
brotli_enc_parity_filenames[f], q, lgwins[w], mode, ch,
|
||||
fl);
|
||||
CHECK_STRING(
|
||||
do_test_mimic_brotli_encode(s, &c, data.data.ptr, n));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const char* //
|
||||
test_mimic_brotli_encode_parity_metadata() {
|
||||
CHECK_FOCUS(__func__);
|
||||
static const size_t lens[] = {0, 1, 100, 70000};
|
||||
wuffs_base__io_buffer data = ((wuffs_base__io_buffer){
|
||||
.data = g_mimiclib_scratch_slice_u8,
|
||||
});
|
||||
CHECK_STRING(read_file(&data, "test/data/romeo.txt"));
|
||||
for (uint32_t q = 0; q <= 1; q++) {
|
||||
for (size_t i = 0; i < WUFFS_TESTLIB_ARRAY_SIZE(lens); i++) {
|
||||
for (uint32_t pos = 1; pos <= 3; pos++) {
|
||||
for (uint32_t fl = 0; fl < BROTLI_ENC_NUM_FLUSHES; fl++) {
|
||||
for (size_t dc = 0; dc < 2; dc++) {
|
||||
brotli_enc_config c = make_brotli_enc_config(q, 16);
|
||||
c.chunking = BROTLI_ENC_CHUNK_RANDOM;
|
||||
c.flush = fl;
|
||||
c.metadata_position = pos;
|
||||
c.metadata_len = lens[i];
|
||||
c.dst_chunk = dc ? 5 : 0;
|
||||
c.seed = (i << 8) | pos;
|
||||
char s[256];
|
||||
snprintf(s, sizeof s,
|
||||
"q=%u metadata_len=%zu pos=%u flush=%u dst_chunk=%zu", q,
|
||||
lens[i], pos, fl, c.dst_chunk);
|
||||
CHECK_STRING(do_test_mimic_brotli_encode(s, &c, data.data.ptr,
|
||||
data.meta.wi));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const char* //
|
||||
test_mimic_brotli_encode_parity_params() {
|
||||
CHECK_FOCUS(__func__);
|
||||
wuffs_base__io_buffer data = ((wuffs_base__io_buffer){
|
||||
.data = g_mimiclib_scratch_slice_u8,
|
||||
});
|
||||
CHECK_STRING(read_file(&data, "test/data/brotli/alice29.txt"));
|
||||
uint32_t n = (uint32_t)data.meta.wi;
|
||||
static const struct {
|
||||
uint32_t key;
|
||||
uint32_t value;
|
||||
} extras[] = {
|
||||
{BROTLI_ENC_PARAM_SIZE_HINT, 0},
|
||||
{BROTLI_ENC_PARAM_SIZE_HINT, 0xFFFFFFFF}, // Replaced by the exact size.
|
||||
{BROTLI_ENC_PARAM_SIZE_HINT, 1u << 30},
|
||||
{BROTLI_ENC_PARAM_LGBLOCK, 16},
|
||||
{BROTLI_ENC_PARAM_LGBLOCK, 24},
|
||||
{BROTLI_ENC_PARAM_DISABLE_LITERAL_CONTEXT_MODELING, 1},
|
||||
{BROTLI_ENC_PARAM_LARGE_WINDOW, 1},
|
||||
{BROTLI_ENC_PARAM_NPOSTFIX, 2},
|
||||
{BROTLI_ENC_PARAM_NDIRECT, 12},
|
||||
{BROTLI_ENC_PARAM_STREAM_OFFSET, 1},
|
||||
{BROTLI_ENC_PARAM_STREAM_OFFSET, 1000},
|
||||
{BROTLI_ENC_PARAM_STREAM_OFFSET, 1u << 30},
|
||||
{BROTLI_ENC_PARAM_LGWIN, 0xFFFFFFFF}, // (int)-1.
|
||||
{BROTLI_ENC_PARAM_LGWIN, 30},
|
||||
{BROTLI_ENC_PARAM_QUALITY, 0xFFFFFFFF}, // (int)-1, so quality 0.
|
||||
};
|
||||
for (uint32_t q = 0; q <= 1; q++) {
|
||||
for (size_t i = 0; i < WUFFS_TESTLIB_ARRAY_SIZE(extras); i++) {
|
||||
for (uint32_t ch = 0; ch < BROTLI_ENC_NUM_CHUNKINGS; ch += 4) {
|
||||
brotli_enc_config c = make_brotli_enc_config(q, 22);
|
||||
uint32_t v = extras[i].value;
|
||||
if ((extras[i].key == BROTLI_ENC_PARAM_SIZE_HINT) &&
|
||||
(v == 0xFFFFFFFF)) {
|
||||
v = n;
|
||||
}
|
||||
brotli_enc_config__add_param(&c, extras[i].key, v);
|
||||
c.chunking = ch;
|
||||
c.flush = BROTLI_ENC_FLUSH_MIDDLE;
|
||||
c.seed = i;
|
||||
char s[256];
|
||||
snprintf(s, sizeof s, "q=%u param[%u]=%u chunking=%u", q,
|
||||
extras[i].key, v, ch);
|
||||
CHECK_STRING(
|
||||
do_test_mimic_brotli_encode(s, &c, data.data.ptr, data.meta.wi));
|
||||
}
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const char* //
|
||||
do_test_mimic_brotli_encode_file(const char* name,
|
||||
const uint8_t* data,
|
||||
size_t len) {
|
||||
for (uint32_t q = 0; q <= 1; q++) {
|
||||
for (uint32_t w = 10; w <= 24; w += 14) {
|
||||
brotli_enc_config c = make_brotli_enc_config(q, w);
|
||||
c.chunking = (w == 10) ? BROTLI_ENC_CHUNK_SINGLE : BROTLI_ENC_CHUNK_RANDOM;
|
||||
c.flush = (w == 10) ? BROTLI_ENC_FLUSH_NONE : BROTLI_ENC_FLUSH_MIDDLE;
|
||||
c.seed = len;
|
||||
char s[256];
|
||||
snprintf(s, sizeof s, "%s: q=%u lgwin=%u", name, q, w);
|
||||
CHECK_STRING(do_test_mimic_brotli_encode(s, &c, data, len));
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const char* //
|
||||
test_mimic_brotli_encode_parity_corpus() {
|
||||
CHECK_FOCUS(__func__);
|
||||
CHECK_STRING(
|
||||
for_each_brotli_enc_corpus_file(do_test_mimic_brotli_encode_file));
|
||||
uint8_t data[40];
|
||||
brotli_enc_fill_random(data, 20, 0x9999);
|
||||
memcpy(data + 20, data, 20);
|
||||
for (size_t n = 0; n <= 40; n++) {
|
||||
char s[64];
|
||||
snprintf(s, sizeof s, "length %zu", n);
|
||||
CHECK_STRING(do_test_mimic_brotli_encode_file(s, data, n));
|
||||
}
|
||||
for (int which = 0; which < 3; which++) {
|
||||
size_t len = 0;
|
||||
uint8_t* p = brotli_enc_synthetic(which, &len);
|
||||
if (!p) {
|
||||
RETURN_FAIL("malloc failed");
|
||||
}
|
||||
char s[64];
|
||||
snprintf(s, sizeof s, "synthetic #%d", which);
|
||||
const char* status = do_test_mimic_brotli_encode_file(s, p, len);
|
||||
free(p);
|
||||
if (status) {
|
||||
return status;
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
#endif // WUFFS_MIMIC
|
||||
|
||||
// ---------------- Encoder Benches
|
||||
|
||||
const char* //
|
||||
do_bench_brotli_encode(const char* (*codec_func)(wuffs_base__io_buffer*,
|
||||
wuffs_base__io_buffer*,
|
||||
const uint32_t*,
|
||||
const uint32_t*,
|
||||
size_t,
|
||||
const uint32_t*,
|
||||
const size_t*,
|
||||
size_t,
|
||||
size_t),
|
||||
uint32_t quality,
|
||||
uint64_t iters_unscaled) {
|
||||
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
|
||||
.data = g_src_slice_u8,
|
||||
});
|
||||
CHECK_STRING(read_file(&src, "test/data/brotli/alice29.txt"));
|
||||
uint32_t keys[2] = {BROTLI_ENC_PARAM_QUALITY, BROTLI_ENC_PARAM_LGWIN};
|
||||
uint32_t values[2] = {quality, 22};
|
||||
uint32_t op = BROTLI_ENC_OP_FINISH;
|
||||
size_t len = src.meta.wi;
|
||||
|
||||
bench_start();
|
||||
uint64_t n_bytes = 0;
|
||||
uint64_t i;
|
||||
uint64_t iters = iters_unscaled * g_flags.iterscale;
|
||||
for (i = 0; i < iters; i++) {
|
||||
wuffs_base__io_buffer have = ((wuffs_base__io_buffer){
|
||||
.data = g_have_slice_u8,
|
||||
});
|
||||
src.meta.ri = 0;
|
||||
CHECK_STRING(codec_func(&have, &src, keys, values, 2, &op, &len, 1, 0));
|
||||
n_bytes += len;
|
||||
}
|
||||
bench_finish(iters, n_bytes);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const char* //
|
||||
bench_wuffs_brotli_encode_alice29_q0() {
|
||||
CHECK_FOCUS(__func__);
|
||||
return do_bench_brotli_encode(wuffs_brotli_encode_plan, 0, 30);
|
||||
}
|
||||
|
||||
const char* //
|
||||
bench_wuffs_brotli_encode_alice29_q1() {
|
||||
CHECK_FOCUS(__func__);
|
||||
return do_bench_brotli_encode(wuffs_brotli_encode_plan, 1, 30);
|
||||
}
|
||||
|
||||
// ---------------- Encoder Mimic Benches
|
||||
|
||||
#ifdef WUFFS_MIMIC
|
||||
|
||||
const char* //
|
||||
bench_mimic_brotli_encode_alice29_q0() {
|
||||
CHECK_FOCUS(__func__);
|
||||
return do_bench_brotli_encode(mimic_brotli_encode, 0, 30);
|
||||
}
|
||||
|
||||
const char* //
|
||||
bench_mimic_brotli_encode_alice29_q1() {
|
||||
CHECK_FOCUS(__func__);
|
||||
return do_bench_brotli_encode(mimic_brotli_encode, 1, 30);
|
||||
}
|
||||
|
||||
#endif // WUFFS_MIMIC
|
||||
|
||||
// ---------------- Brotli Benches
|
||||
|
||||
const char* //
|
||||
@@ -682,6 +1713,24 @@ proc g_tests[] = {
|
||||
test_mimic_brotli_decode_malformed,
|
||||
test_mimic_brotli_decode_romeo,
|
||||
|
||||
#endif // WUFFS_MIMIC
|
||||
|
||||
// Encoder tests.
|
||||
|
||||
test_wuffs_brotli_encode_interface,
|
||||
test_wuffs_brotli_encode_metadata_round_trip,
|
||||
test_wuffs_brotli_encode_parameters,
|
||||
test_wuffs_brotli_encode_round_trip_corpus,
|
||||
test_wuffs_brotli_encode_round_trip_lengths,
|
||||
test_wuffs_brotli_encode_round_trip_synthetic,
|
||||
|
||||
#ifdef WUFFS_MIMIC
|
||||
|
||||
test_mimic_brotli_encode_parity_corpus,
|
||||
test_mimic_brotli_encode_parity_grid,
|
||||
test_mimic_brotli_encode_parity_metadata,
|
||||
test_mimic_brotli_encode_parity_params,
|
||||
|
||||
#endif // WUFFS_MIMIC
|
||||
|
||||
NULL,
|
||||
@@ -695,6 +1744,18 @@ proc g_benches[] = {
|
||||
|
||||
bench_mimic_brotli_decode_alice29,
|
||||
|
||||
#endif // WUFFS_MIMIC
|
||||
|
||||
// Encoder benches.
|
||||
|
||||
bench_wuffs_brotli_encode_alice29_q0,
|
||||
bench_wuffs_brotli_encode_alice29_q1,
|
||||
|
||||
#ifdef WUFFS_MIMIC
|
||||
|
||||
bench_mimic_brotli_encode_alice29_q0,
|
||||
bench_mimic_brotli_encode_alice29_q1,
|
||||
|
||||
#endif // WUFFS_MIMIC
|
||||
|
||||
NULL,
|
||||
|
||||
Reference in new issue
Block a user