diff --git a/script/manual-test-brotli-encoder-parity.c b/script/manual-test-brotli-encoder-parity.c
index c7347037..985034c4 100644
--- a/script/manual-test-brotli-encoder-parity.c
+++ b/script/manual-test-brotli-encoder-parity.c
@@ -35,6 +35,13 @@
// -q=LO-HI only check qualities LO to HI (default 0-9).
// -shard=K/N only check the inputs whose index i satisfies (i % N) == K,
// so that N processes can split the work.
+// -lw only run the large-window grid (BROTLI_PARAM_LARGE_WINDOW
+// with lgwin 10 to 30, see run_input_lw), at qualities 2 and
+// above.
+// -big instead of the usual inputs, check synthetic inputs of 3 to
+// 12 MiB (with the usual or, with -lw, the large-window grid)
+// and of 24 to 70 MiB (with a few large-window configurations
+// each), with repeats at distances of up to 40 MiB.
//
// The test/c/std/brotli.c test program (run by "wuffs test -mimic
// std/brotli") checks a subset of this grid.
@@ -77,8 +84,8 @@
#define MAX_STEPS (1 << 22)
#define MAX_PARAMS 16
-#define MAX_INPUT (16 << 20)
-#define MAX_OUTPUT (64 << 20)
+#define MAX_INPUT (80 << 20)
+#define MAX_OUTPUT (96 << 20)
static uint32_t g_step_ops[MAX_STEPS];
static size_t g_step_lens[MAX_STEPS];
@@ -92,6 +99,7 @@ static uint8_t* g_workbuf; // Wuffs' workbuf.
static size_t g_workbuf_len;
static uint64_t g_num_configs;
+static size_t g_last_want_len;
static uint64_t g_num_mismatches;
static uint32_t g_qmin = 0;
@@ -136,6 +144,77 @@ fill_random(uint8_t* p, size_t n, uint64_t seed) {
}
}
+// fill_mixed fills p[0 .. n] with a mixture of text (from a small
+// vocabulary, so that there are many matches), runs and binary noise.
+static void //
+fill_mixed(uint8_t* p, size_t n_max, uint64_t seed) {
+ static const char* words[] = {"the ", "brotli ", "wuffs ", "of ",
+ "and ", "data\n", "time ", "",
+ "encoder", ", ", "zebra ", "1234"};
+ uint64_t state = seed;
+ size_t n = 0;
+ while (n < n_max) {
+ uint64_t r = rand64(&state);
+ if ((r & 0xFF) < 200) {
+ const char* w = words[(r >> 8) % 12];
+ size_t k = strlen(w);
+ while (k-- && (n < n_max)) {
+ p[n++] = (uint8_t)(*w++);
+ }
+ } else if ((r & 0xFF) < 230) {
+ size_t k = (size_t)((r >> 8) % 300);
+ while (k-- && (n < n_max)) {
+ p[n++] = (uint8_t)(r >> 40);
+ }
+ } else {
+ size_t k = (size_t)((r >> 8) % 2000);
+ while (k-- && (n < n_max)) {
+ p[n++] = (uint8_t)(rand64(&state) >> 56);
+ }
+ }
+ }
+}
+
+// fill_far_repeats fills p[0 .. n] with 1 MiB blocks: most are fresh
+// pseudo-random bytes or fill_mixed data (with a per-block seed), but some
+// repeat (with sparse changes) the block that is near or far
+// (near_mib or far_mib MiB) before them, so that encoders with a large
+// enough window find matches at those distances. When far_mib is a multiple
+// of 4 and near_mib is not 2 modulo 4, the far copies' sources are fresh
+// blocks that are not also copied (or copies) nearer by.
+static void //
+fill_far_repeats(uint8_t* p, size_t n, size_t near_mib, size_t far_mib,
+ uint64_t seed) {
+ const size_t mib = 1 << 20;
+ for (size_t b = 0; (b * mib) < n; b++) {
+ uint8_t* q = p + (b * mib);
+ size_t k = ((n - (b * mib)) < mib) ? (n - (b * mib)) : mib;
+ size_t from = SIZE_MAX;
+ if ((b >= near_mib) && ((b % 4) == 1)) {
+ from = b - near_mib;
+ } else if ((b >= far_mib) && ((b % 4) == 3)) {
+ from = b - far_mib;
+ }
+ if (from != SIZE_MAX) {
+ // A shifted copy: the distance is not a multiple of 1 MiB (but, for
+ // odd b, it is a multiple of 4, which the HROLLING_FAST hasher, used
+ // by H35 and H55, needs to find it).
+ size_t shift = (size_t)(seed % 250) + (b * 7);
+ if (b & 1) {
+ shift *= 4;
+ }
+ memcpy(q, p + (from * mib) + shift, k);
+ for (size_t i = 4093; i < k; i += 65521) {
+ q[i] ^= 0x55;
+ }
+ } else if ((b % 3) == 0) {
+ fill_random(q, k, seed + b);
+ } else {
+ fill_mixed(q, k, seed + b);
+ }
+ }
+}
+
static bool //
add_step(uint32_t op, size_t len) {
if (g_num_steps >= MAX_STEPS) {
@@ -427,6 +506,7 @@ check(const char* name, const config* c, const uint8_t* data, size_t len) {
size_t want_len = 0;
const char* have_err = encode_wuffs(c, &have_len);
const char* want_err = encode_mimic(c, &want_len);
+ g_last_want_len = want_len;
if (have_err || want_err) {
g_num_mismatches++;
describe(desc, sizeof desc, name, c);
@@ -779,6 +859,40 @@ run_input(const char* name, const uint8_t* data, size_t len) {
fflush(stdout);
}
+// run_input_far checks a few large-window configurations (one-shot and
+// streaming) for a very long input.
+static void //
+run_input_far(const char* name, const uint8_t* data, size_t len) {
+ uint64_t configs0 = g_num_configs;
+ uint64_t mismatches0 = g_num_mismatches;
+ time_t start = time(NULL);
+ for (uint32_t q = (g_qmin > 2) ? g_qmin : 2; q <= g_qmax; q++) {
+ static const uint32_t lgwins[] = {24, 25, 26, 30};
+ for (size_t w = 0; w < 4; w++) {
+ config c = make_config(q, lgwins[w]);
+ add_param(&c, 6, 1); // BROTLI_PARAM_LARGE_WINDOW.
+ c.seed = 0x40000 | (w << 8) | (q << 12);
+ check(name, &c, data, len);
+ printf(" q=%u lgwin=%u: %zu compressed bytes\n", q, lgwins[w],
+ g_last_want_len);
+ }
+ // Streaming, with a flush and a size hint below 1 MiB (so H35 at
+ // quality 3 and H58 or H5 at qualities 5 to 9, but H3 at quality 4).
+ config c = make_config(q, 30);
+ add_param(&c, 6, 1); // BROTLI_PARAM_LARGE_WINDOW.
+ add_param(&c, 5, 12345); // BROTLI_PARAM_SIZE_HINT.
+ c.chunking = 4;
+ c.flush = 2;
+ c.seed = 0x50000 | (q << 12);
+ check(name, &c, data, len);
+ }
+ printf("%-50s %8zu bytes: %6" PRIu64 " configs, %" PRIu64
+ " mismatches (%lds)\n",
+ name, len, g_num_configs - configs0, g_num_mismatches - mismatches0,
+ (long)(time(NULL) - start));
+ fflush(stdout);
+}
+
static uint8_t* //
read_whole_file(const char* path, size_t* len) {
FILE* f = fopen(path, "rb");
@@ -817,6 +931,7 @@ main(int argc, char** argv) {
int shard_k = 0;
int shard_n = 1;
+ bool big = false;
int argi = 1;
for (; argi < argc; argi++) {
unsigned a = 0;
@@ -826,6 +941,8 @@ main(int argc, char** argv) {
g_qmax = (b < 9) ? b : 9;
} else if (strcmp(argv[argi], "-lw") == 0) {
g_lw_only = true;
+ } else if (strcmp(argv[argi], "-big") == 0) {
+ big = true;
} else if (sscanf(argv[argi], "-shard=%u/%u", &a, &b) == 2) {
shard_k = (int)a;
shard_n = (b > 0) ? (int)b : 1;
@@ -870,6 +987,42 @@ main(int argc, char** argv) {
}
int input_index = 0;
+ if (big) {
+ // Synthetic inputs of several MiB (with the usual grid) and of 24 to 70
+ // MiB (with run_input_far's configurations), with repeats at distances
+ // above 16 MiB (1 << 24, the largest non-large-window window).
+ num_names = 0;
+ uint8_t* p = malloc(70 << 20);
+ if (!p) {
+ return 1;
+ }
+ static const struct {
+ size_t mib;
+ size_t near_mib;
+ size_t far_mib;
+ bool far;
+ } bigs[] = {
+ {3, 2, 2, false}, {6, 2, 5, false}, {12, 3, 10, false},
+ {24, 3, 20, true}, {40, 3, 32, true}, {70, 5, 40, true},
+ };
+ for (size_t j = 0; j < sizeof bigs / sizeof bigs[0]; j++) {
+ if (((input_index++) % shard_n) != shard_k) {
+ continue;
+ }
+ size_t n = (bigs[j].mib << 20) - 12345;
+ fill_far_repeats(p, n, bigs[j].near_mib, bigs[j].far_mib, 0x777 + j);
+ char name[96];
+ snprintf(name, sizeof name,
+ "synthetic: %zu MiB with repeats at %zu and %zu MiB",
+ bigs[j].mib, bigs[j].near_mib, bigs[j].far_mib);
+ if (bigs[j].far) {
+ run_input_far(name, p, n);
+ } else {
+ run_input(name, p, n);
+ }
+ }
+ free(p);
+ }
for (int i = 0; i < num_names; i++) {
if (((input_index++) % shard_n) != shard_k) {
continue;
@@ -884,7 +1037,7 @@ main(int argc, char** argv) {
free(data);
}
- if (argi >= argc) {
+ if ((argi >= argc) && !big) {
// Synthetic inputs.
uint8_t* p = malloc(1 << 20);
if (!p) {
@@ -907,34 +1060,8 @@ main(int argc, char** argv) {
run_input("synthetic: a repeated 100-byte pattern", p, 64 << 10);
}
if (((input_index++) % shard_n) == shard_k) {
- // A mixture of text (from a small vocabulary, so that there are many
- // matches), runs and binary noise.
- static const char* words[] = {"the ", "brotli ", "wuffs ", "of ",
- "and ", "data\n", "time ", "",
- "encoder", ", ", "zebra ", "1234"};
- uint64_t state = 0xABCD;
- size_t n = 0;
- while (n < (600 << 10)) {
- uint64_t r = rand64(&state);
- if ((r & 0xFF) < 200) {
- const char* w = words[(r >> 8) % 12];
- size_t k = strlen(w);
- while (k-- && (n < (600 << 10))) {
- p[n++] = (uint8_t)(*w++);
- }
- } else if ((r & 0xFF) < 230) {
- size_t k = (size_t)((r >> 8) % 300);
- while (k-- && (n < (600 << 10))) {
- p[n++] = (uint8_t)(r >> 40);
- }
- } else {
- size_t k = (size_t)((r >> 8) % 2000);
- while (k-- && (n < (600 << 10))) {
- p[n++] = (uint8_t)(rand64(&state) >> 56);
- }
- }
- }
- run_input("synthetic: 600 KiB of mixed text and binary", p, n);
+ fill_mixed(p, 600 << 10, 0xABCD);
+ run_input("synthetic: 600 KiB of mixed text and binary", p, 600 << 10);
}
for (size_t n = 0; n <= 40; n++) {
if (((input_index++) % shard_n) != shard_k) {