Files
wuffs/script/manual-test-brotli-encoder-parity.c
T

876 lines
27 KiB
C

// Copyright 2026 The Wuffs Authors.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// SPDX-License-Identifier: Apache-2.0 OR MIT
// ----------------
// manual-test-brotli-encoder-parity.c checks that Wuffs' std/brotli encoder
// produces exactly the same bytes as libbrotlienc v1.2.1 (at qualities 0 to
// 11) over a large parameter grid. For each test input and quality:
// - lgwin {10, 16, 18, 22, 24}
// - mode {0, 1, 2}
// - input chunking {single, 1-byte (inputs up to 4 KiB only), 7-byte,
// 4096-byte, pseudo-random}
// - flush {none, every chunk, middle}
// plus metadata blocks (of 0, 1, 100 and 70000 bytes, at the start, middle
// and end), size_hint {0, exact, 1 << 30, etc}, the other parameters
// (lgblock, disable literal context modeling, npostfix and ndirect, large
// window, stream offset) and limited dst lengths. At qualities 2 to 9,
// inputs longer than 512 KiB use a smaller grid (and at qualities 10 and 11,
// inputs longer than 64 KiB do).
//
// The test inputs are the files named on the command line (or, by default,
// every file in test/data/brotli plus test/data/{256.bytes,enwik5,romeo.txt,
// pi.txt,nobel-prizes.json,archive.iso,hibiscus.regular.bmp}) plus some
// synthetic inputs (1 MiB of zeroes, 256 KiB of pseudo-random bytes, a
// repeated 100-byte pattern, a 600 KiB mixture of text and binary data and
// pseudo-random lengths 0 to 40).
//
// Options (before any file names):
// -q=LO-HI only check qualities LO to HI (default 0-9, at most 0-11).
// -shard=K/N only check the inputs whose index i satisfies (i % N) == K,
// so that N processes can split the work.
//
// The test/c/std/brotli.c test program (run by "wuffs test -mimic
// std/brotli") checks a subset of this grid.
//
// Run it from the Wuffs root directory with:
//
// gcc -O2 -std=c99 script/manual-test-brotli-encoder-parity.c
// -lbrotlienc -lbrotlicommon -lm -o /tmp/brotli-enc-parity
//
// (as one command line).
// /tmp/brotli-enc-parity
//
// It prints each mismatching configuration (and the first differing offset)
// and exits with a non-zero status if there were any mismatches.
#define _POSIX_C_SOURCE 200809L
#include <dirent.h>
#include <inttypes.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include "brotli/encode.h"
#define WUFFS_IMPLEMENTATION
#define WUFFS_CONFIG__MODULES
#define WUFFS_CONFIG__MODULE__BASE
#define WUFFS_CONFIG__MODULE__BROTLI
#define WUFFS_CONFIG__MODULE__SOFTFLOAT
#include "../release/c/wuffs-unsupported-snapshot.c"
#define OP_PROCESS 0
#define OP_FLUSH 1
#define OP_FINISH 2
#define OP_EMIT_METADATA 3
#define MAX_STEPS (1 << 22)
#define MAX_PARAMS 16
#define MAX_INPUT (16 << 20)
#define MAX_OUTPUT (64 << 20)
static uint32_t g_step_ops[MAX_STEPS];
static size_t g_step_lens[MAX_STEPS];
static size_t g_num_steps;
static uint8_t* g_src; // The plan's source: input plus any metadata.
static size_t g_src_len;
static uint8_t* g_have; // Wuffs' output.
static uint8_t* g_want; // libbrotlienc's output.
static uint8_t* g_workbuf; // Wuffs' workbuf.
static size_t g_workbuf_len;
static uint64_t g_num_configs;
static uint64_t g_num_mismatches;
static uint32_t g_qmin = 0;
static uint32_t g_qmax = 9;
typedef struct {
uint32_t keys[MAX_PARAMS];
uint32_t values[MAX_PARAMS];
size_t num_params;
uint32_t chunking; // 0 single, 1 1-byte, 2 7-byte, 3 4096-byte, 4 random.
uint32_t flush; // 0 none, 1 every chunk, 2 middle.
uint32_t metadata_position; // 0 none, 1 start, 2 middle, 3 end.
size_t metadata_len;
size_t dst_chunk; // 0 means unlimited.
uint64_t seed;
} config;
static void //
add_param(config* c, uint32_t key, uint32_t value) {
if (c->num_params < MAX_PARAMS) {
c->keys[c->num_params] = key;
c->values[c->num_params] = value;
c->num_params++;
}
}
static uint64_t //
rand64(uint64_t* state) {
// SplitMix64.
uint64_t z = (*state += 0x9E3779B97F4A7C15ull);
z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ull;
z = (z ^ (z >> 27)) * 0x94D049BB133111EBull;
return z ^ (z >> 31);
}
static void //
fill_random(uint8_t* p, size_t n, uint64_t seed) {
uint64_t state = seed;
for (size_t i = 0; i < n; i++) {
p[i] = (uint8_t)(rand64(&state) >> 56);
}
}
static bool //
add_step(uint32_t op, size_t len) {
if (g_num_steps >= MAX_STEPS) {
return false;
}
g_step_ops[g_num_steps] = op;
g_step_lens[g_num_steps] = len;
g_num_steps++;
return true;
}
// make_plan mirrors make_brotli_enc_plan in test/c/std/brotli.c.
static const char* //
make_plan(const config* c, const uint8_t* data, size_t data_len) {
g_num_steps = 0;
g_src_len = 0;
if ((data_len + c->metadata_len) > MAX_INPUT) {
return "input is too long";
}
uint64_t rng = c->seed;
size_t metadata_at = SIZE_MAX;
if (c->metadata_position == 1) {
metadata_at = 0;
} else if (c->metadata_position == 2) {
metadata_at = data_len / 2;
} else if (c->metadata_position == 3) {
metadata_at = data_len;
}
bool flushed_middle = false;
size_t i = 0;
while (true) {
if (i == metadata_at) {
fill_random(g_src + g_src_len, c->metadata_len, c->seed ^ 0x5A5A);
g_src_len += c->metadata_len;
if (!add_step(OP_EMIT_METADATA, c->metadata_len)) {
return "too many steps";
}
metadata_at = SIZE_MAX;
}
size_t n = data_len - i;
switch (c->chunking) {
case 1:
n = 1;
break;
case 2:
n = 7;
break;
case 3:
n = 4096;
break;
case 4: {
uint64_t r = rand64(&rng);
n = (size_t)(1 + ((r & 0xFF) < 0xF0 ? ((r >> 8) % 300)
: ((r >> 8) % 70000)));
break;
}
}
if ((metadata_at != SIZE_MAX) && (metadata_at > i)) {
n = (n < (metadata_at - i)) ? n : (metadata_at - i);
}
if (n > (data_len - i)) {
n = data_len - i;
}
memcpy(g_src + g_src_len, data + i, n);
g_src_len += n;
i += n;
bool last = (i == data_len) && (metadata_at == SIZE_MAX);
bool ok = true;
if (last) {
ok = add_step(OP_FINISH, n);
} else if (c->flush == 1) {
ok = add_step(OP_FLUSH, n);
} else {
ok = add_step(OP_PROCESS, n);
if ((c->flush == 2) && !flushed_middle && ((2 * i) >= data_len)) {
flushed_middle = true;
ok = ok && add_step(OP_FLUSH, 0);
}
}
if (!ok) {
return "too many steps";
}
if (last) {
break;
}
}
return NULL;
}
// brotli_enc_quirk_key maps a BrotliEncoderParameter value to the
// corresponding std/brotli quirk key (or 0 if there is none).
static uint32_t //
brotli_enc_quirk_key(uint32_t param) {
switch (param) {
case 0:
return WUFFS_BROTLI__QUIRK_MODE;
case 1:
return WUFFS_BROTLI__QUIRK_ENCODER_QUALITY;
case 2:
return WUFFS_BROTLI__QUIRK_LGWIN;
case 3:
return WUFFS_BROTLI__QUIRK_LGBLOCK;
case 4:
return WUFFS_BROTLI__QUIRK_DISABLE_LITERAL_CONTEXT_MODELING;
case 5:
return WUFFS_BROTLI__QUIRK_SIZE_HINT;
case 6:
return WUFFS_BROTLI__QUIRK_LARGE_WINDOW;
case 7:
return WUFFS_BROTLI__QUIRK_NPOSTFIX;
case 8:
return WUFFS_BROTLI__QUIRK_NDIRECT;
case 9:
return WUFFS_BROTLI__QUIRK_STREAM_OFFSET;
}
return 0;
}
// encode_wuffs mirrors wuffs_brotli_encode_plan in test/c/std/brotli.c.
static const char* //
encode_wuffs(const config* c, size_t* have_len) {
wuffs_brotli__encoder* enc = wuffs_brotli__encoder__alloc();
if (!enc) {
return "alloc failed";
}
const char* ret = NULL;
wuffs_base__io_buffer dst = wuffs_base__ptr_u8__writer(g_have, MAX_OUTPUT);
wuffs_base__io_buffer src = wuffs_base__ptr_u8__reader(g_src, g_src_len,
false);
src.meta.wi = 0;
for (size_t j = 0; j < c->num_params; j++) {
wuffs_base__status status = wuffs_brotli__encoder__set_quirk(
enc, brotli_enc_quirk_key(c->keys[j]), c->values[j]);
if (status.repr) {
ret = status.repr;
goto cleanup;
}
}
// Use the minimum, so that the qualities 10 and 11 practical bound is
// exercised (at other qualities, min_incl == max_incl).
uint64_t wlen = wuffs_brotli__encoder__workbuf_len(enc).min_incl;
if (wlen > ((uint64_t)1 << 32)) {
ret = "workbuf_len is too large";
goto cleanup;
} else if (wlen > g_workbuf_len) {
free(g_workbuf);
g_workbuf = malloc((size_t)wlen);
g_workbuf_len = g_workbuf ? (size_t)wlen : 0;
if (!g_workbuf) {
ret = "malloc failed";
goto cleanup;
}
}
wuffs_base__slice_u8 workbuf =
wuffs_base__make_slice_u8(g_workbuf, (size_t)wlen);
size_t end = 0;
for (size_t i = 0; i < g_num_steps; i++) {
end += g_step_lens[i];
uint32_t op = g_step_ops[i];
wuffs_base__status status = wuffs_base__make_status(NULL);
if (op == OP_FLUSH) {
status = wuffs_brotli__encoder__request_flush(enc);
} else if (op == OP_EMIT_METADATA) {
status = wuffs_brotli__encoder__request_emit_metadata(enc, g_step_lens[i]);
}
if (status.repr) {
ret = status.repr;
goto cleanup;
}
while (true) {
wuffs_base__io_buffer d = dst;
if ((c->dst_chunk > 0) && ((d.data.len - d.meta.wi) > c->dst_chunk)) {
d.data.len = d.meta.wi + c->dst_chunk;
}
wuffs_base__io_buffer s = src;
s.meta.wi = end;
s.meta.closed = (op == OP_FINISH);
status = wuffs_brotli__encoder__transform_io(enc, &d, &s, workbuf);
dst.meta.wi = d.meta.wi;
src.meta.ri = s.meta.ri;
if (status.repr == NULL) {
if (op != OP_FINISH) {
ret = "unexpected ok";
goto cleanup;
}
break;
} else if (status.repr == wuffs_base__suspension__short_write) {
if (dst.meta.wi == dst.data.len) {
ret = "dst is full";
goto cleanup;
}
continue;
} else if ((status.repr == wuffs_base__suspension__short_read) &&
(op != OP_FINISH) && (src.meta.ri == end)) {
break;
}
ret = status.repr;
goto cleanup;
}
}
cleanup:
*have_len = dst.meta.wi;
free(enc);
return ret;
}
// encode_mimic mirrors mimic_brotli_encode in test/c/mimiclib/brotli.c.
static const char* //
encode_mimic(const config* c, size_t* want_len) {
*want_len = 0;
BrotliEncoderState* state = BrotliEncoderCreateInstance(NULL, NULL, NULL);
if (!state) {
return "BrotliEncoderCreateInstance failed";
}
const char* ret = NULL;
for (size_t j = 0; j < c->num_params; j++) {
if (!BrotliEncoderSetParameter(state, (BrotliEncoderParameter)(c->keys[j]),
c->values[j])) {
ret = "BrotliEncoderSetParameter failed";
goto cleanup;
}
}
const uint8_t* next_in = g_src;
for (size_t i = 0; i < g_num_steps; i++) {
size_t avail_in = g_step_lens[i];
BrotliEncoderOperation op = (BrotliEncoderOperation)(g_step_ops[i]);
while (true) {
size_t avail_out = MAX_OUTPUT - *want_len;
if ((c->dst_chunk > 0) && (avail_out > c->dst_chunk)) {
avail_out = c->dst_chunk;
}
size_t avail_out0 = avail_out;
uint8_t* next_out = g_want + *want_len;
if (!BrotliEncoderCompressStream(state, op, &avail_in, &next_in,
&avail_out, &next_out, NULL)) {
ret = "BrotliEncoderCompressStream failed";
goto cleanup;
}
*want_len += avail_out0 - avail_out;
if (op == BROTLI_OPERATION_FINISH) {
if (BrotliEncoderIsFinished(state)) {
break;
}
} else if ((avail_in == 0) && !BrotliEncoderHasMoreOutput(state)) {
break;
}
if (*want_len == MAX_OUTPUT) {
ret = "dst is full";
goto cleanup;
}
}
}
cleanup:
BrotliEncoderDestroyInstance(state);
return ret;
}
static void //
describe(char* buf, size_t n, const char* name, const config* c) {
int k = snprintf(buf, n, "%s:", name);
for (size_t j = 0; (j < c->num_params) && (k > 0) && ((size_t)k < n); j++) {
k += snprintf(buf + k, n - (size_t)k, " p%u=%u", c->keys[j], c->values[j]);
}
if ((k > 0) && ((size_t)k < n)) {
snprintf(buf + k, n - (size_t)k,
" chunking=%u flush=%u metadata=%u/%zu dst_chunk=%zu seed=0x%" PRIX64,
c->chunking, c->flush, c->metadata_position, c->metadata_len,
c->dst_chunk, c->seed);
}
}
static void //
check(const char* name, const config* c, const uint8_t* data, size_t len) {
char desc[512];
g_num_configs++;
const char* err = make_plan(c, data, len);
if (err) {
describe(desc, sizeof desc, name, c);
printf("SKIP %s: %s\n", desc, err);
return;
}
size_t have_len = 0;
size_t want_len = 0;
const char* have_err = encode_wuffs(c, &have_len);
const char* want_err = encode_mimic(c, &want_len);
if (have_err || want_err) {
g_num_mismatches++;
describe(desc, sizeof desc, name, c);
printf("MISMATCH %s: wuffs error \"%s\", mimic error \"%s\"\n", desc,
have_err ? have_err : "", want_err ? want_err : "");
fflush(stdout);
return;
}
size_t n = (have_len < want_len) ? have_len : want_len;
size_t i = 0;
while ((i < n) && (g_have[i] == g_want[i])) {
i++;
}
if ((i < n) || (have_len != want_len)) {
g_num_mismatches++;
describe(desc, sizeof desc, name, c);
printf("MISMATCH %s: lengths %zu (wuffs) vs %zu (mimic), first difference "
"at offset %zu\n",
desc, have_len, want_len, i);
fflush(stdout);
}
}
static config //
make_config(uint32_t quality, uint32_t lgwin) {
config c;
memset(&c, 0, sizeof c);
add_param(&c, 1, quality); // BROTLI_PARAM_QUALITY.
add_param(&c, 2, lgwin); // BROTLI_PARAM_LGWIN.
return c;
}
static void //
run_input_q01(const char* name, const uint8_t* data, size_t len, uint32_t q) {
static const uint32_t lgwins[] = {10, 16, 18, 22, 24};
static const size_t metadata_lens[] = {0, 1, 100, 70000};
// The main grid.
for (size_t w = 0; w < 5; w++) {
for (uint32_t mode = 0; mode <= 2; mode++) {
for (uint32_t ch = 0; ch < 5; ch++) {
if ((ch == 1) && (len > 4096)) {
continue;
}
for (uint32_t fl = 0; fl < 3; fl++) {
config c = make_config(q, lgwins[w]);
add_param(&c, 0, mode); // BROTLI_PARAM_MODE.
c.chunking = ch;
c.flush = fl;
c.seed = (w << 8) | (mode << 4) | fl | ((uint64_t)len << 16);
check(name, &c, data, len);
}
}
}
}
// Metadata, size_hint, other parameters and limited dst lengths.
for (size_t i = 0; i < 4; i++) {
for (uint32_t pos = 1; pos <= 3; pos++) {
for (uint32_t fl = 0; fl < 3; fl++) {
config c = make_config(q, 16);
c.chunking = 4;
c.flush = fl;
c.metadata_position = pos;
c.metadata_len = metadata_lens[i];
c.dst_chunk = (fl == 2) ? 5 : 0;
c.seed = (i << 8) | pos;
check(name, &c, data, len);
}
}
}
static const uint32_t extras[][2] = {
{5, 0}, // BROTLI_PARAM_SIZE_HINT.
{5, 0xFFFFFFFF}, // Replaced by the exact size.
{5, 1u << 30}, //
{3, 16}, // BROTLI_PARAM_LGBLOCK.
{3, 24}, //
{4, 1}, // BROTLI_PARAM_DISABLE_LITERAL_CONTEXT_MODELING.
{6, 1}, // BROTLI_PARAM_LARGE_WINDOW.
{7, 2}, // BROTLI_PARAM_NPOSTFIX.
{8, 12}, // BROTLI_PARAM_NDIRECT.
{9, 1}, // BROTLI_PARAM_STREAM_OFFSET.
{9, 1000}, //
{9, 1u << 30}, //
{2, 0xFFFFFFFF}, // LGWIN (int)-1.
{2, 30}, //
{1, 0xFFFFFFFF}, // QUALITY (int)-1, so quality 0.
};
for (size_t i = 0; i < sizeof extras / sizeof extras[0]; i++) {
for (uint32_t ch = 0; ch < 5; ch += 4) {
config c = make_config(q, 22);
uint32_t v = extras[i][1];
if ((extras[i][0] == 5) && (v == 0xFFFFFFFF)) {
v = (uint32_t)len;
}
add_param(&c, extras[i][0], v);
c.chunking = ch;
c.flush = 2;
c.seed = i;
check(name, &c, data, len);
}
}
if (len <= (256 << 10)) {
static const size_t dst_chunks[] = {1, 2, 3, 7, 64, 4096};
for (size_t i = 0; i < 6; i++) {
for (uint32_t fl = 0; fl < 3; fl++) {
config c = make_config(q, (i & 1) ? 10 : 24);
c.chunking = (fl == 0) ? 0 : 4;
c.flush = fl;
c.dst_chunk = dst_chunks[i];
c.seed = i;
check(name, &c, data, len);
}
}
}
}
// run_input_q2 runs the grid for a quality from 2 to 11.
static void //
run_input_q2(const char* name, const uint8_t* data, size_t len, uint32_t q) {
static const uint32_t lgwins[] = {10, 16, 18, 22, 24};
static const size_t metadata_lens[] = {0, 1, 100, 70000};
// At qualities 10 and 11 (which are much slower), inputs longer than 64
// KiB use the smaller grid.
bool big = len > ((q >= 10) ? (64 << 10) : (512 << 10));
// The main grid.
for (size_t w = 0; w < 5; w++) {
for (uint32_t mode = 0; mode <= 2; mode++) {
if (big && (mode == 1)) {
continue;
}
for (uint32_t ch = 0; ch < 5; ch++) {
if ((ch == 1) && (len > 4096)) {
continue;
} else if (big && ((ch == 2) || (ch == 3))) {
continue;
}
for (uint32_t fl = 0; fl < 3; fl++) {
if (big && (fl == 1)) {
continue;
}
config c = make_config(q, lgwins[w]);
add_param(&c, 0, mode); // BROTLI_PARAM_MODE.
c.chunking = ch;
c.flush = fl;
c.seed = (w << 8) | (mode << 4) | fl | ((uint64_t)len << 16) | q;
check(name, &c, data, len);
}
}
}
}
// Metadata blocks and limited dst lengths.
for (size_t i = 0; i < 4; i++) {
for (uint32_t pos = 1; pos <= 3; pos++) {
for (uint32_t fl = 0; fl < 3; fl++) {
if (big && (fl != 2)) {
continue;
}
config c = make_config(q, (pos == 2) ? 18 : 16);
c.chunking = (pos == 3) ? 0 : 4;
c.flush = fl;
c.metadata_position = pos;
c.metadata_len = metadata_lens[i];
c.dst_chunk = (fl == 2) ? 5 : 0;
c.seed = (i << 8) | pos | (q << 12);
check(name, &c, data, len);
}
}
}
// Other parameters, alone and in combination.
static const uint32_t extras[][6] = {
// Pairs of (BrotliEncoderParameter, value). A zero key ends the list.
{5, 0}, // BROTLI_PARAM_SIZE_HINT.
{5, 0xFFFFFFFF}, // Replaced by the exact size.
{5, 1}, //
{5, 1u << 20}, //
{5, 1u << 30}, //
{3, 16}, // BROTLI_PARAM_LGBLOCK.
{3, 17}, //
{3, 20}, //
{3, 24}, //
{3, 0xFFFFFFFF}, // LGBLOCK (int)-1.
{4, 1}, // DISABLE_LITERAL_CONTEXT_MODELING.
{4, 1, 0, 1}, // ... and TEXT mode.
{7, 1}, // BROTLI_PARAM_NPOSTFIX.
{7, 2, 8, 12}, // ... and BROTLI_PARAM_NDIRECT.
{7, 3, 8, 120}, //
{7, 0, 8, 15}, //
{7, 3, 8, 0}, //
{7, 1, 8, 4}, //
{0, 2}, // FONT mode.
{0, 2, 7, 3, 8, 30}, // FONT mode with explicit distance params.
{9, 1}, // BROTLI_PARAM_STREAM_OFFSET.
{9, 1000}, //
{9, 1u << 30}, //
{2, 0xFFFFFFFF}, // LGWIN (int)-1.
{2, 30}, //
{5, 0xFFFFFFFF, 3, 18}, // Exact size hint and LGBLOCK 18.
{5, 1u << 21, 4, 1}, // Big size hint, no context modeling.
};
for (size_t i = 0; i < sizeof extras / sizeof extras[0]; i++) {
for (uint32_t ch = 0; ch < 5; ch += 4) {
for (size_t w = 0; w < 3; w++) {
static const uint32_t ws[] = {16, 22, 24};
if (big && (w != 1)) {
continue;
}
config c = make_config(q, ws[w]);
for (size_t k = 0; (k < 6) && ((k == 0) || extras[i][k]); k += 2) {
uint32_t v = extras[i][k + 1];
if ((extras[i][k] == 5) && (v == 0xFFFFFFFF)) {
v = (uint32_t)len;
}
add_param(&c, extras[i][k], v);
}
c.chunking = ch;
c.flush = 2;
c.seed = i | (w << 8) | (q << 12);
check(name, &c, data, len);
}
}
}
if (q == 2) {
// Large window is supported (by the Wuffs encoder) only at quality 2.
for (uint32_t w = 22; w <= 30; w += 4) {
config c = make_config(q, w);
add_param(&c, 6, 1); // BROTLI_PARAM_LARGE_WINDOW.
c.chunking = 4;
c.flush = 2;
c.seed = w;
check(name, &c, data, len);
}
}
if (len <= (256 << 10)) {
static const size_t dst_chunks[] = {1, 2, 3, 7, 64, 4096};
for (size_t i = 0; i < 6; i++) {
for (uint32_t fl = 0; fl < 3; fl++) {
if ((dst_chunks[i] < 7) && (len > (32 << 10))) {
continue;
}
config c = make_config(q, (i & 1) ? 10 : 24);
c.chunking = (fl == 0) ? 0 : 4;
c.flush = fl;
c.dst_chunk = dst_chunks[i];
c.seed = i;
check(name, &c, data, len);
}
}
}
}
static void //
run_input(const char* name, const uint8_t* data, size_t len) {
uint64_t configs0 = g_num_configs;
uint64_t mismatches0 = g_num_mismatches;
time_t start = time(NULL);
for (uint32_t q = g_qmin; q <= g_qmax; q++) {
if (q <= 1) {
run_input_q01(name, data, len, q);
} else {
run_input_q2(name, data, len, q);
}
}
printf("%-50s %8zu bytes: %6" PRIu64 " configs, %" PRIu64
" mismatches (%lds)\n",
name, len, g_num_configs - configs0, g_num_mismatches - mismatches0,
(long)(time(NULL) - start));
fflush(stdout);
}
static uint8_t* //
read_whole_file(const char* path, size_t* len) {
FILE* f = fopen(path, "rb");
if (!f) {
return NULL;
}
uint8_t* p = malloc(MAX_INPUT);
if (p) {
*len = fread(p, 1, MAX_INPUT, f);
}
fclose(f);
return p;
}
static int //
compare_strings(const void* a, const void* b) {
return strcmp(*(const char* const*)a, *(const char* const*)b);
}
int //
main(int argc, char** argv) {
if (BrotliEncoderVersion() != 0x1002001) {
fprintf(stderr, "libbrotlienc version: have 0x%X, want 0x1002001\n",
BrotliEncoderVersion());
return 1;
}
g_src = malloc(MAX_INPUT);
g_have = malloc(MAX_OUTPUT);
g_want = malloc(MAX_OUTPUT);
g_workbuf_len = (2u << 24) + 503 + 16;
g_workbuf = malloc(g_workbuf_len); // It grows as needed.
if (!g_src || !g_have || !g_want || !g_workbuf) {
fprintf(stderr, "malloc failed\n");
return 1;
}
int shard_k = 0;
int shard_n = 1;
int argi = 1;
for (; argi < argc; argi++) {
unsigned a = 0;
unsigned b = 0;
if (sscanf(argv[argi], "-q=%u-%u", &a, &b) == 2) {
g_qmin = a;
g_qmax = (b < 11) ? b : 11;
} else if (sscanf(argv[argi], "-shard=%u/%u", &a, &b) == 2) {
shard_k = (int)a;
shard_n = (b > 0) ? (int)b : 1;
} else {
break;
}
}
char* names[1024];
int num_names = 0;
if (argi < argc) {
for (int i = argi; (i < argc) && (num_names < 1024); i++) {
names[num_names++] = strdup(argv[i]);
}
} else {
names[num_names++] = strdup("test/data/256.bytes");
names[num_names++] = strdup("test/data/enwik5");
names[num_names++] = strdup("test/data/romeo.txt");
names[num_names++] = strdup("test/data/pi.txt");
names[num_names++] = strdup("test/data/nobel-prizes.json");
names[num_names++] = strdup("test/data/archive.iso");
names[num_names++] = strdup("test/data/hibiscus.regular.bmp");
DIR* d = opendir("test/data/brotli");
if (!d) {
fprintf(stderr, "could not open test/data/brotli (run this program "
"from the Wuffs root directory)\n");
return 1;
}
struct dirent* e;
int first = num_names;
while ((e = readdir(d)) && (num_names < 1024)) {
if (e->d_name[0] == '.') {
continue;
}
char path[1024];
snprintf(path, sizeof path, "test/data/brotli/%s", e->d_name);
names[num_names++] = strdup(path);
}
closedir(d);
qsort(names + first, (size_t)(num_names - first), sizeof names[0],
compare_strings);
}
int input_index = 0;
for (int i = 0; i < num_names; i++) {
if (((input_index++) % shard_n) != shard_k) {
continue;
}
size_t len = 0;
uint8_t* data = read_whole_file(names[i], &len);
if (!data) {
fprintf(stderr, "could not read %s\n", names[i]);
return 1;
}
run_input(names[i], data, len);
free(data);
}
if (argi >= argc) {
// Synthetic inputs.
uint8_t* p = malloc(1 << 20);
if (!p) {
return 1;
}
if (((input_index++) % shard_n) == shard_k) {
memset(p, 0, 1 << 20);
run_input("synthetic: 1 MiB of zeroes", p, 1 << 20);
}
if (((input_index++) % shard_n) == shard_k) {
fill_random(p, 256 << 10, 0x1234);
run_input("synthetic: 256 KiB of pseudo-random bytes", p, 256 << 10);
}
if (((input_index++) % shard_n) == shard_k) {
uint8_t pattern[100];
fill_random(pattern, 100, 0x5678);
for (size_t i = 0; i < (64 << 10); i++) {
p[i] = pattern[i % 100];
}
run_input("synthetic: a repeated 100-byte pattern", p, 64 << 10);
}
if (((input_index++) % shard_n) == shard_k) {
// A mixture of text (from a small vocabulary, so that there are many
// matches), runs and binary noise.
static const char* words[] = {"the ", "brotli ", "wuffs ", "of ",
"and ", "data\n", "time ", "<a>",
"encoder", ", ", "zebra ", "1234"};
uint64_t state = 0xABCD;
size_t n = 0;
while (n < (600 << 10)) {
uint64_t r = rand64(&state);
if ((r & 0xFF) < 200) {
const char* w = words[(r >> 8) % 12];
size_t k = strlen(w);
while (k-- && (n < (600 << 10))) {
p[n++] = (uint8_t)(*w++);
}
} else if ((r & 0xFF) < 230) {
size_t k = (size_t)((r >> 8) % 300);
while (k-- && (n < (600 << 10))) {
p[n++] = (uint8_t)(r >> 40);
}
} else {
size_t k = (size_t)((r >> 8) % 2000);
while (k-- && (n < (600 << 10))) {
p[n++] = (uint8_t)(rand64(&state) >> 56);
}
}
}
run_input("synthetic: 600 KiB of mixed text and binary", p, n);
}
for (size_t n = 0; n <= 40; n++) {
if (((input_index++) % shard_n) != shard_k) {
continue;
}
fill_random(p, n, 0x9999 + n);
char name[64];
snprintf(name, sizeof name, "synthetic: %zu pseudo-random bytes", n);
run_input(name, p, n);
}
free(p);
}
printf("TOTAL: %" PRIu64 " configs, %" PRIu64 " mismatches\n", g_num_configs,
g_num_mismatches);
return g_num_mismatches ? 1 : 0;
}