diff --git a/script/manual-test-brotli-fuzz.c b/script/manual-test-brotli-fuzz.c new file mode 100644 index 00000000..47ec21a9 --- /dev/null +++ b/script/manual-test-brotli-fuzz.c @@ -0,0 +1,1937 @@ +// Copyright 2026 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. +// +// SPDX-License-Identifier: Apache-2.0 OR MIT + +// ---------------- + +// manual-test-brotli-fuzz.c is a differential fuzzer for std/brotli against +// the reference implementation (libbrotlidec and libbrotlienc v1.2.1). Every +// parameter is drawn from a pseudo-random number generator. +// +// Modes (the first argument): +// +// -dec decodes randomly mutated brotli streams with both libbrotlidec and +// Wuffs' decoder and checks that they produce the same output bytes, +// consume the same input (on success) and agree on the result: ok, +// truncated input, a full dst (needs more output) or the specific +// BROTLI_DECODER_ERROR_FORMAT_* code (see the "Status Mapping" in +// std/brotli/README.md). Seeds are the test/data/brotli files and +// streams that libbrotlienc or Wuffs' encoder produce (at random +// qualities, window sizes, large window and raw dictionaries) from +// random inputs. Mutations are bit flips, byte substitutions, +// truncations, insertions, deletions, splices of two streams and +// random bytes. Each stream is decoded one-shot (by both) and with +// random I/O chunking (the same absolute chunk boundaries for both, +// mostly 1 to 7 bytes), with a random large-window quirk, random +// (raw and serialized) dictionaries and a random workbuf length (at +// or above workbuf_len, or exactly the stream's minimum, or one byte +// less than that, which must fail). +// +// -enc encodes random inputs with random parameters (quality, lgwin, +// large window, lgblock, mode, size hint, npostfix, ndirect, +// disabling literal context modeling, stream offset, raw +// dictionaries) and random plans of PROCESS, FLUSH, EMIT_METADATA and +// FINISH steps with random chunk sizes and dst limits, with both +// libbrotlienc and Wuffs' encoder, and checks that the output is +// byte-exact and that both decoders round-trip it. +// +// Options (after the mode): +// -seed=S the PRNG seed (default 1). +// -n=N the number of iterations (default 1000). +// -secs=T stop after about T seconds. +// -q=LO-HI (-enc only) restrict the quality to LO ..= HI. +// -hq (-enc only) allow dictionaries at qualities 10 and 11. +// -out=DIR where to save failing cases (default "."). +// -data=DIR the test/data directory (default "test/data"). +// +// The libbrotlidec that it is linked with should be built with +// BROTLI_EXPERIMENTAL (so that it accepts serialized dictionaries, like +// Wuffs' decoder). Only c/common/shared_dictionary.c depends on it, so +// libbrotlienc can be the default build. For example, from the Wuffs root +// directory (as one command line): +// +// for f in $BROTLI/c/common/*.c $BROTLI/c/dec/*.c; do +// gcc -O2 -DBROTLI_EXPERIMENTAL -I$BROTLI/c/include -c $f +// -o /tmp/bd/$(basename $(dirname $f))_$(basename $f .c).o; done +// ar rcs /tmp/bd/libbrotlidecexp.a /tmp/bd/*.o +// gcc -O2 -std=c99 -I$BROTLI/c/include script/manual-test-brotli-fuzz.c +// $BROTLI_BUILD/libbrotlienc.a /tmp/bd/libbrotlidecexp.a -lm +// -o /tmp/brotli-fuzz +// /tmp/brotli-fuzz -dec -seed=1 -n=100000 +// +// It prints a summary and exits with a non-zero status if there were any +// divergences. Each divergence is saved (the stream, its dictionaries and a +// description) under the -out directory. + +#define _DEFAULT_SOURCE +#define _POSIX_C_SOURCE 200809L + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "brotli/decode.h" +#include "brotli/encode.h" +#include "brotli/shared_dictionary.h" + +#define WUFFS_IMPLEMENTATION +#define WUFFS_CONFIG__MODULES +#define WUFFS_CONFIG__MODULE__BASE +#define WUFFS_CONFIG__MODULE__BROTLI +#define WUFFS_CONFIG__MODULE__SOFTFLOAT +#include "../release/c/wuffs-unsupported-snapshot.c" + +// ---------------- PRNG + +static uint64_t g_rng_state = 1; +static uint64_t g_seed = 1; + +static uint64_t // +r64(void) { + // SplitMix64. + uint64_t z = (g_rng_state += 0x9E3779B97F4A7C15ull); + z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ull; + z = (z ^ (z >> 27)) * 0x94D049BB133111EBull; + return z ^ (z >> 31); +} + +// rn returns a pseudo-random number in [0 .. n), or 0 if n is 0. +static uint64_t // +rn(uint64_t n) { + return n ? (r64() % n) : 0; +} + +// chance returns true with probability (k / 1000). +static bool // +chance(uint64_t k) { + return rn(1000) < k; +} + +static uint64_t // +split64(uint64_t* state) { + uint64_t z = (*state += 0x9E3779B97F4A7C15ull); + z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ull; + z = (z ^ (z >> 27)) * 0x94D049BB133111EBull; + return z ^ (z >> 31); +} + +// ---------------- Globals + +static const char* g_out_dir = "."; +static const char* g_data_dir = "test/data"; +static uint64_t g_num_failures = 0; +static uint32_t g_qmin = 0; +static uint32_t g_qmax = 11; +static bool g_hq_dicts = false; + +static uint8_t* // +xmalloc(size_t n) { + uint8_t* p = malloc(n ? n : 1); + if (!p) { + fprintf(stderr, "out of memory (%zu bytes)\n", n); + exit(2); + } + return p; +} + +// big_alloc reserves (but does not touch) n bytes. +static uint8_t* // +big_alloc(size_t n) { + void* p = mmap(NULL, n, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0); + if (p == MAP_FAILED) { + fprintf(stderr, "mmap of %zu bytes failed\n", n); + exit(2); + } + return (uint8_t*)p; +} + +// big_release gives back the physical memory of a big_alloc region. +static void // +big_release(uint8_t* p, size_t n) { + madvise(p, n, MADV_DONTNEED); +} + +static uint8_t* // +read_file(const char* filename, size_t* len, size_t max_len) { + FILE* f = fopen(filename, "rb"); + if (!f) { + return NULL; + } + struct stat st; + if ((fstat(fileno(f), &st) != 0) || !S_ISREG(st.st_mode) || + ((size_t)st.st_size > max_len)) { + fclose(f); + return NULL; + } + size_t n = (size_t)st.st_size; + uint8_t* p = xmalloc(n + 1); + if (fread(p, 1, n, f) != n) { + fclose(f); + free(p); + return NULL; + } + fclose(f); + *len = n; + return p; +} + +static void // +write_file(const char* filename, const uint8_t* p, size_t n) { + FILE* f = fopen(filename, "wb"); + if (!f) { + fprintf(stderr, "could not write %s\n", filename); + return; + } + if (n) { + fwrite(p, 1, n, f); + } + fclose(f); +} + +// ---------------- Corpus (for inputs and dictionaries) + +typedef struct { + char name[256]; + uint8_t* ptr; + size_t len; +} blob; + +#define MAX_CORPUS 512 +static blob g_corpus[MAX_CORPUS]; +static size_t g_num_corpus; + +static int // +cmp_names(const void* a, const void* b) { + return strcmp(*(const char* const*)a, *(const char* const*)b); +} + +// list_dir returns the sorted names of the regular files in dir. +static size_t // +list_dir(const char* dir, char** names, size_t max) { + DIR* d = opendir(dir); + if (!d) { + return 0; + } + size_t n = 0; + struct dirent* e; + while ((e = readdir(d)) && (n < max)) { + if (e->d_name[0] == '.') { + continue; + } + names[n++] = strdup(e->d_name); + } + closedir(d); + qsort(names, n, sizeof names[0], cmp_names); + return n; +} + +static void // +load_corpus(void) { + static const char* subdirs[] = {"", "/brotli"}; + for (int s = 0; s < 2; s++) { + char dir[512]; + snprintf(dir, sizeof dir, "%s%s", g_data_dir, subdirs[s]); + char* names[1024]; + size_t n = list_dir(dir, names, 1024); + for (size_t i = 0; i < n; i++) { + size_t k = strlen(names[i]); + bool compressed = ((k > 3) && !strcmp(names[i] + k - 3, ".br")) || + strstr(names[i], ".compressed") || + strstr(names[i], ".bin"); + if ((s == 1) && compressed) { + free(names[i]); + continue; + } + char path[1024]; + snprintf(path, sizeof path, "%s/%s", dir, names[i]); + size_t len = 0; + uint8_t* p = read_file(path, &len, 4 << 20); + if (p && (len > 0) && (g_num_corpus < MAX_CORPUS)) { + blob* b = &g_corpus[g_num_corpus++]; + snprintf(b->name, sizeof b->name, "%s", path); + b->ptr = p; + b->len = len; + } else { + free(p); + } + free(names[i]); + } + } + if (g_num_corpus == 0) { + fprintf(stderr, "no corpus files in %s (use -data=DIR)\n", g_data_dir); + exit(2); + } +} + +// ---------------- Input generators + +// pick_len returns a pseudo-random length up to max, with an emphasis on +// short lengths. +static size_t // +pick_len(size_t max) { + uint64_t r = rn(1000); + size_t n; + if (r < 30) { + n = rn(4); + } else if (r < 300) { + n = rn(256); + } else if (r < 550) { + n = rn(4096); + } else if (r < 850) { + n = rn(65536 + 1); + } else if (r < 970) { + n = rn(1 << 20); + } else { + n = rn(4 << 20); + } + return (n < max) ? n : max; +} + +static void // +put_utf8(uint8_t* p, size_t* i, size_t n, uint32_t c) { + uint8_t b[4]; + size_t k; + if (c < 0x80) { + b[0] = (uint8_t)c; + k = 1; + } else if (c < 0x800) { + b[0] = (uint8_t)(0xC0 | (c >> 6)); + b[1] = (uint8_t)(0x80 | (c & 0x3F)); + k = 2; + } else if (c < 0x10000) { + b[0] = (uint8_t)(0xE0 | (c >> 12)); + b[1] = (uint8_t)(0x80 | ((c >> 6) & 0x3F)); + b[2] = (uint8_t)(0x80 | (c & 0x3F)); + k = 3; + } else { + b[0] = (uint8_t)(0xF0 | (c >> 18)); + b[1] = (uint8_t)(0x80 | ((c >> 12) & 0x3F)); + b[2] = (uint8_t)(0x80 | ((c >> 6) & 0x3F)); + b[3] = (uint8_t)(0x80 | (c & 0x3F)); + k = 4; + } + for (size_t j = 0; (j < k) && (*i < n); j++) { + p[(*i)++] = b[j]; + } +} + +// gen_segment fills p[0 .. n] with one kind of data. p[-dist] (for dist up to +// prefix) may be read, for repeats of earlier data. +static int // +gen_segment(uint8_t* p, size_t n, size_t prefix, int kind) { + switch (kind) { + case 0: { // Random bytes. + for (size_t i = 0; i < n; i++) { + p[i] = (uint8_t)r64(); + } + break; + } + case 1: { // A slice of a corpus file (text, images, binaries). + const blob* b = &g_corpus[rn(g_num_corpus)]; + size_t off = rn(b->len); + for (size_t i = 0; i < n; i++) { + p[i] = b->ptr[(off + i) % b->len]; + } + break; + } + case 2: { // A repeated pattern. + size_t period = 1 + rn(chance(500) ? 8 : 400); + for (size_t i = 0; i < n; i++) { + p[i] = (i < period) ? (uint8_t)r64() : p[i - period]; + } + break; + } + case 3: { // Text from a small vocabulary, runs and noise. + static const char* words[] = {"the ", "brotli ", "wuffs ", "of ", + "and ", "data\n", "time ", "", + "encoder", ", ", "zebra ", "1234", + "\xC3\xA9t\xC3\xA9 ", "\t", "{\"k\": ", + "}\n"}; + size_t i = 0; + while (i < n) { + uint64_t r = r64(); + if ((r & 0xFF) < 200) { + const char* w = words[(r >> 8) % 16]; + while (*w && (i < n)) { + p[i++] = (uint8_t)(*w++); + } + } else if ((r & 0xFF) < 230) { + size_t k = (size_t)((r >> 8) % 300); + while (k-- && (i < n)) { + p[i++] = (uint8_t)(r >> 40); + } + } else { + size_t k = (size_t)((r >> 8) % 2000); + while (k-- && (i < n)) { + p[i++] = (uint8_t)r64(); + } + } + } + break; + } + case 4: { // Repeats of earlier data (near and far), with changes. + if (prefix == 0) { + return gen_segment(p, n, prefix, 3); + } + size_t i = 0; + while (i < n) { + size_t avail = prefix + i; + size_t dist = 1 + (chance(300) ? rn(avail) : rn(avail < 64 ? avail : 64)); + size_t k = 4 + rn(chance(100) ? 70000 : 300); + for (size_t j = 0; (j < k) && (i < n); j++, i++) { + p[i] = p[(ptrdiff_t)i - (ptrdiff_t)dist]; + } + if (i < n) { + p[i++] = (uint8_t)r64(); + } + } + break; + } + case 5: { // UTF-8 text (valid, sometimes invalid). + size_t i = 0; + while (i < n) { + uint64_t r = r64(); + uint32_t c; + switch (r & 7) { + case 0: + case 1: + case 2: + c = 0x20 + (uint32_t)((r >> 8) % 0x5F); + break; + case 3: + case 4: + c = 0x80 + (uint32_t)((r >> 8) % 0x780); + break; + case 5: + c = 0x800 + (uint32_t)((r >> 8) % 0xD000); + break; + case 6: + c = 0x10000 + (uint32_t)((r >> 8) % 0x100000); + break; + default: + p[i++] = (uint8_t)(r >> 8); + continue; + } + put_utf8(p, &i, n, c); + } + break; + } + case 6: { // A small alphabet. + uint32_t k = 1 + (uint32_t)rn(4); + uint8_t base = (uint8_t)r64(); + for (size_t i = 0; i < n; i++) { + p[i] = (uint8_t)(base + (uint8_t)rn(k)); + } + break; + } + default: { // Zeroes. + memset(p, 0, n); + break; + } + } + return kind; +} + +// gen_input fills p[0 .. n] with a mixture of segments. +static void // +gen_input(uint8_t* p, size_t n) { + if (chance(400)) { + gen_segment(p, n, 0, (int)rn(8)); + return; + } + size_t i = 0; + while (i < n) { + size_t k = 1 + rn(chance(200) ? (n - i) : 4096); + if (k > (n - i)) { + k = n - i; + } + gen_segment(p + i, k, i, (int)rn(8)); + i += k; + } +} + +// ---------------- Dictionaries + +typedef struct { + uint32_t type; // 0 raw, 1 serialized. + const uint8_t* ptr; + size_t len; + bool owned; +} dict; + +#define MAX_DICTS 16 + +typedef struct { + dict d[MAX_DICTS]; + size_t n; +} dict_set; + +static void // +dict_set_free(dict_set* s) { + for (size_t i = 0; i < s->n; i++) { + if (s->d[i].owned) { + free((void*)s->d[i].ptr); + } + } + s->n = 0; +} + +static void // +dict_set_add(dict_set* s, uint32_t type, const uint8_t* ptr, size_t len, + bool copy) { + if (s->n >= MAX_DICTS) { + return; + } + dict* d = &s->d[s->n++]; + d->type = type; + d->len = len; + d->owned = copy; + if (copy) { + uint8_t* q = xmalloc(len); + if (len) { + memcpy(q, ptr, len); + } + d->ptr = q; + } else { + d->ptr = ptr; + } +} + +static void // +dict_set_copy(dict_set* dst, const dict_set* src) { + dst->n = 0; + for (size_t i = 0; i < src->n; i++) { + dict_set_add(dst, src->d[i].type, src->d[i].ptr, src->d[i].len, true); + } +} + +static size_t // +dict_set_total(const dict_set* s) { + size_t t = 0; + for (size_t i = 0; i < s->n; i++) { + t += s->d[i].len; + } + return t; +} + +// Well-known dictionaries (see test/data/brotli/README.md). +static uint8_t* g_romeo; +static size_t g_romeo_len; +static uint8_t* g_alice; +static size_t g_alice_len; +static uint8_t* g_sd[4]; // full, words, prefix, utf8. +static size_t g_sd_len[4]; + +static uint8_t* // +must_read(const char* rel, size_t* len) { + char path[1024]; + snprintf(path, sizeof path, "%s/%s", g_data_dir, rel); + uint8_t* p = read_file(path, len, 64 << 20); + if (!p) { + fprintf(stderr, "could not read %s\n", path); + exit(2); + } + return p; +} + +static void // +load_dicts(void) { + g_romeo = must_read("romeo.txt", &g_romeo_len); + g_alice = must_read("brotli/alice29.txt", &g_alice_len); + g_sd[0] = must_read("brotli/shared-dictionary.full.bin", &g_sd_len[0]); + g_sd[1] = must_read("brotli/shared-dictionary.words.bin", &g_sd_len[1]); + g_sd[2] = must_read("brotli/shared-dictionary.prefix.bin", &g_sd_len[2]); + g_sd[3] = must_read("brotli/shared-dictionary.utf8.bin", &g_sd_len[3]); +} + +// named_dict_set fills s with the dictionaries that the named test/data/brotli +// file was compressed with (and sets *lw for large-window files). +static void // +named_dict_set(const char* name, dict_set* s, bool* lw) { + s->n = 0; + *lw = strstr(name, "lw") != NULL; + if (strstr(name, "raw-romeo")) { + dict_set_add(s, 0, g_romeo, g_romeo_len, false); + } else if (strstr(name, "alice29.txt.") && strstr(name, "raw-chunks")) { + dict_set_add(s, 0, g_alice, 20000, false); + dict_set_add(s, 0, g_alice + 20000, 1, false); + dict_set_add(s, 0, g_alice + 20001, 0, false); + dict_set_add(s, 0, g_alice + 20001, 40000 - 1, false); + } else if (strstr(name, "raw-chunks") || strstr(name, "invalid.raw-")) { + dict_set_add(s, 0, g_romeo, 300, false); + dict_set_add(s, 0, g_romeo + 300, 200, false); + dict_set_add(s, 0, g_romeo + 500, g_romeo_len - 500, false); + } else if (strstr(name, "sd-full") || strstr(name, "invalid.sd-")) { + dict_set_add(s, 1, g_sd[0], g_sd_len[0], false); + } else if (strstr(name, "sd-words")) { + dict_set_add(s, 1, g_sd[1], g_sd_len[1], false); + } else if (strstr(name, "sd-prefix-raw")) { + dict_set_add(s, 1, g_sd[2], g_sd_len[2], false); + dict_set_add(s, 0, g_romeo, 300, false); + } else if (strstr(name, "sd-utf8")) { + dict_set_add(s, 1, g_sd[3], g_sd_len[3], false); + } +} + +// random_dict_bytes makes a random raw dictionary of len bytes, sometimes +// based on hint (e.g. the input that will be compressed). +static uint8_t* // +random_dict_bytes(size_t len, const uint8_t* hint, size_t hint_len) { + uint8_t* p = xmalloc(len); + if (hint && hint_len && chance(600)) { + size_t off = rn(hint_len); + for (size_t i = 0; i < len; i++) { + p[i] = hint[(off + i) % hint_len]; + } + // Sparse changes. + if (len && chance(500)) { + for (size_t k = rn(8); k > 0; k--) { + p[rn(len)] ^= (uint8_t)(1 + rn(255)); + } + } + } else { + gen_input(p, len); + } + return p; +} + +static size_t // +pick_dict_len(void) { + uint64_t r = rn(1000); + if (r < 50) { + return 0; + } else if (r < 400) { + return 1 + rn(64); + } else if (r < 800) { + return rn(4096); + } else if (r < 985) { + return rn(256 << 10); + } else if (r < 995) { + // Around the 2 MiB hash table size boundary. + return (2 << 20) - 8 + rn(17); + } + return rn(3 << 20); +} + +// random_raw_dicts fills s with k random raw dictionaries. +static void // +random_raw_dicts(dict_set* s, size_t k, const uint8_t* hint, size_t hint_len) { + s->n = 0; + for (size_t i = 0; i < k; i++) { + size_t len = pick_dict_len(); + uint8_t* p = random_dict_bytes(len, hint, hint_len); + dict_set_add(s, 0, p, len, false); + s->d[s->n - 1].owned = true; + } +} + +static size_t // +pick_num_dicts(void) { + uint64_t r = rn(1000); + if (r < 400) { + return 0; + } else if (r < 700) { + return 1; + } else if (r < 850) { + return 2 + rn(3); + } else if (r < 950) { + return 5 + rn(10); + } + return 15 + rn(2); // 15 or 16 (the 16th must be rejected). +} + +// ---------------- Wuffs encoder and libbrotlienc drivers + +#define OP_PROCESS 0 +#define OP_FLUSH 1 +#define OP_FINISH 2 +#define OP_EMIT_METADATA 3 + +#define MAX_PARAMS 16 + +typedef struct { + uint32_t keys[MAX_PARAMS]; // BrotliEncoderParameter values. + uint32_t values[MAX_PARAMS]; + size_t num_params; + size_t num_steps; + uint32_t* step_ops; + size_t* step_lens; + size_t steps_cap; + size_t dst_chunk; // 0 means unlimited. + bool dst_chunk_rand; // Each call gets 1 + rn(dst_chunk) bytes of dst. + uint64_t dst_seed; + uint64_t workbuf_extra; // Added to workbuf_len().min_incl (q10-11). + bool short_workbuf; // Use the shortest workbuf (q2-11, see below). +} enc_config; + +static void // +enc_add_param(enc_config* c, uint32_t key, uint32_t value) { + if (c->num_params < MAX_PARAMS) { + c->keys[c->num_params] = key; + c->values[c->num_params] = value; + c->num_params++; + } +} + +static uint32_t // +enc_get_param(const enc_config* c, uint32_t key, uint32_t dflt) { + uint32_t v = dflt; + for (size_t j = 0; j < c->num_params; j++) { + if (c->keys[j] == key) { + v = c->values[j]; + } + } + return v; +} + +static void // +enc_add_step(enc_config* c, uint32_t op, size_t len) { + if (c->num_steps == c->steps_cap) { + c->steps_cap = c->steps_cap ? (2 * c->steps_cap) : 64; + c->step_ops = realloc(c->step_ops, c->steps_cap * sizeof c->step_ops[0]); + c->step_lens = + realloc(c->step_lens, c->steps_cap * sizeof c->step_lens[0]); + if (!c->step_ops || !c->step_lens) { + fprintf(stderr, "out of memory\n"); + exit(2); + } + } + c->step_ops[c->num_steps] = op; + c->step_lens[c->num_steps] = len; + c->num_steps++; +} + +static uint32_t // +quirk_key(uint32_t param) { + switch (param) { + case 0: + return WUFFS_BROTLI__QUIRK_MODE; + case 1: + return WUFFS_BROTLI__QUIRK_ENCODER_QUALITY; + case 2: + return WUFFS_BROTLI__QUIRK_LGWIN; + case 3: + return WUFFS_BROTLI__QUIRK_LGBLOCK; + case 4: + return WUFFS_BROTLI__QUIRK_DISABLE_LITERAL_CONTEXT_MODELING; + case 5: + return WUFFS_BROTLI__QUIRK_SIZE_HINT; + case 6: + return WUFFS_BROTLI__QUIRK_LARGE_WINDOW; + case 7: + return WUFFS_BROTLI__QUIRK_NPOSTFIX; + case 8: + return WUFFS_BROTLI__QUIRK_NDIRECT; + case 9: + return WUFFS_BROTLI__QUIRK_STREAM_OFFSET; + } + return 0; +} + +static uint8_t* g_enc_wb; // The Wuffs encoder's workbuf (big_alloc). +#define ENC_WB_CAP ((size_t)3 << 30) +static size_t g_enc_wb_touched; +static const char g_too_large[] = "workbuf_len is too large"; + +static size_t // +next_dst_room(const enc_config* c, uint64_t* st) { + if (c->dst_chunk == 0) { + return SIZE_MAX; + } else if (c->dst_chunk_rand) { + return 1 + (size_t)(split64(st) % c->dst_chunk); + } + return c->dst_chunk; +} + +// short_workbuf_len mirrors the parity script's: the shortest workbuf (at +// qualities 2 to 11) that should work for an input of len bytes, given that +// workbuf_len().min_incl is wlen, or 0 if there is none. +static uint64_t // +short_workbuf_len(const enc_config* c, size_t len, uint64_t wlen) { + int32_t quality = (int32_t)enc_get_param(c, 1, 11); + int32_t lgwin = (int32_t)enc_get_param(c, 2, 22); + int32_t lgblock = (int32_t)enc_get_param(c, 3, 0); + bool large_window = enc_get_param(c, 6, 0) != 0; + quality = (quality < 0) ? 0 : (quality > 11) ? 11 : quality; + if (quality < 2) { + return 0; + } + large_window = large_window && (quality > 2); + int32_t max_lgwin = large_window ? 30 : 24; + lgwin = (lgwin < 10) ? 10 : (lgwin > max_lgwin) ? max_lgwin : lgwin; + if (quality < 4) { + lgblock = 14; + } else if (lgblock == 0) { + lgblock = 16; + if ((quality >= 9) && (lgwin > lgblock)) { + lgblock = (lgwin < 18) ? lgwin : 18; + } + } else { + lgblock = (lgblock < 16) ? 16 : (lgblock > 24) ? 24 : lgblock; + } + int32_t rb_bits = 1 + ((lgwin > lgblock) ? lgwin : lgblock); + uint64_t rb_size = (uint64_t)1 << rb_bits; + uint64_t rb_region = 2 + rb_size + ((uint64_t)1 << lgblock) + 7; + if (((len + 16) > rb_size) || (rb_region > wlen)) { + return 0; + } + uint64_t n = wlen - rb_region + 2 + len + 16; + return (n < wlen) ? n : 0; +} + +// encode_wuffs runs the plan with Wuffs' encoder. *wlen_out is the workbuf +// length used. short_delta (0 or 1) is subtracted from the short workbuf +// length. +static const char* // +encode_wuffs(const enc_config* c, const dict_set* dicts, const uint8_t* src_ptr, + size_t src_len, size_t input_len, uint8_t* out, size_t out_cap, + size_t* out_len, uint64_t short_delta, uint64_t* wlen_out) { + *out_len = 0; + wuffs_brotli__encoder* enc = wuffs_brotli__encoder__alloc(); + if (!enc) { + return "alloc failed"; + } + const char* ret = NULL; + for (size_t j = 0; j < dicts->n; j++) { + uint64_t n = wuffs_brotli__encoder__add_dictionary_workbuf_len( + enc, dicts->d[j].len); + if (n > ENC_WB_CAP) { + ret = g_too_large; + goto cleanup; + } + if (n > g_enc_wb_touched) { + g_enc_wb_touched = n; + } + wuffs_base__status status = wuffs_brotli__encoder__add_dictionary( + enc, dicts->d[j].type, + wuffs_base__make_slice_u8((uint8_t*)dicts->d[j].ptr, dicts->d[j].len), + wuffs_base__make_slice_u8(g_enc_wb, (size_t)n)); + if (status.repr) { + ret = status.repr; + goto cleanup; + } + } + for (size_t j = 0; j < c->num_params; j++) { + wuffs_base__status status = wuffs_brotli__encoder__set_quirk( + enc, quirk_key(c->keys[j]), c->values[j]); + if (status.repr) { + ret = status.repr; + goto cleanup; + } + } + wuffs_base__range_ii_u64 r = wuffs_brotli__encoder__workbuf_len(enc); + uint64_t wlen = r.min_incl; + if (c->workbuf_extra && (r.max_incl > r.min_incl)) { + uint64_t room = r.max_incl - r.min_incl; + wlen += (c->workbuf_extra < room) ? c->workbuf_extra : room; + } + if (c->short_workbuf) { + uint64_t s = short_workbuf_len(c, input_len, r.min_incl); + if (s > 0) { + wlen = s - short_delta; + } else if (short_delta) { + ret = "no short workbuf"; + goto cleanup; + } + } + if (wlen > ENC_WB_CAP) { + ret = g_too_large; + goto cleanup; + } + *wlen_out = wlen; + if (wlen > g_enc_wb_touched) { + g_enc_wb_touched = wlen; + } + wuffs_base__slice_u8 workbuf = + wuffs_base__make_slice_u8(g_enc_wb, (size_t)wlen); + + wuffs_base__io_buffer dst = wuffs_base__ptr_u8__writer(out, out_cap); + wuffs_base__io_buffer src = + wuffs_base__ptr_u8__reader((uint8_t*)src_ptr, src_len, false); + src.meta.wi = 0; + uint64_t dst_st = c->dst_seed; + + size_t end = 0; + for (size_t i = 0; i < c->num_steps; i++) { + end += c->step_lens[i]; + uint32_t op = c->step_ops[i]; + wuffs_base__status status = wuffs_base__make_status(NULL); + if (op == OP_FLUSH) { + status = wuffs_brotli__encoder__request_flush(enc); + } else if (op == OP_EMIT_METADATA) { + status = wuffs_brotli__encoder__request_emit_metadata(enc, + c->step_lens[i]); + } + if (status.repr) { + ret = status.repr; + goto cleanup; + } + uint64_t calls = 0; + while (true) { + if (++calls > (out_cap + src_len + 1000)) { + ret = "too many transform_io calls"; + goto cleanup; + } + wuffs_base__io_buffer d = dst; + size_t room = next_dst_room(c, &dst_st); + if ((d.data.len - d.meta.wi) > room) { + d.data.len = d.meta.wi + room; + } + wuffs_base__io_buffer s = src; + s.meta.wi = end; + s.meta.closed = (op == OP_FINISH); + status = wuffs_brotli__encoder__transform_io(enc, &d, &s, workbuf); + dst.meta.wi = d.meta.wi; + src.meta.ri = s.meta.ri; + if (status.repr == NULL) { + if (op != OP_FINISH) { + ret = "unexpected ok"; + goto cleanup; + } + break; + } else if (status.repr == wuffs_base__suspension__short_write) { + if (dst.meta.wi == dst.data.len) { + ret = "dst is full"; + goto cleanup; + } + continue; + } else if ((status.repr == wuffs_base__suspension__short_read) && + (op != OP_FINISH) && (src.meta.ri == end)) { + break; + } + ret = status.repr; + goto cleanup; + } + } + +cleanup: + *out_len = dst.meta.wi; + free(enc); + return ret; +} + +// encode_c runs the plan with libbrotlienc. +static const char* // +encode_c(const enc_config* c, const dict_set* dicts, const uint8_t* src_ptr, + uint8_t* out, size_t out_cap, size_t* out_len) { + *out_len = 0; + BrotliEncoderPreparedDictionary* prepared[MAX_DICTS] = {0}; + size_t num_prepared = 0; + BrotliEncoderState* state = BrotliEncoderCreateInstance(NULL, NULL, NULL); + if (!state) { + return "BrotliEncoderCreateInstance failed"; + } + const char* ret = NULL; + for (size_t j = 0; j < c->num_params; j++) { + if (!BrotliEncoderSetParameter(state, (BrotliEncoderParameter)(c->keys[j]), + c->values[j])) { + ret = "BrotliEncoderSetParameter failed"; + goto cleanup; + } + } + for (size_t j = 0; j < dicts->n; j++) { + prepared[j] = BrotliEncoderPrepareDictionary( + (BrotliSharedDictionaryType)dicts->d[j].type, dicts->d[j].len, + dicts->d[j].ptr, 11, NULL, NULL, NULL); + if (!prepared[j]) { + ret = "BrotliEncoderPrepareDictionary failed"; + goto cleanup; + } + num_prepared++; + if (!BrotliEncoderAttachPreparedDictionary(state, prepared[j])) { + ret = "BrotliEncoderAttachPreparedDictionary failed"; + goto cleanup; + } + } + const uint8_t* next_in = src_ptr; + uint64_t dst_st = c->dst_seed; + for (size_t i = 0; i < c->num_steps; i++) { + size_t avail_in = c->step_lens[i]; + BrotliEncoderOperation op = (BrotliEncoderOperation)(c->step_ops[i]); + uint64_t calls = 0; + while (true) { + if (++calls > (out_cap + c->step_lens[i] + 1000)) { + ret = "too many BrotliEncoderCompressStream calls"; + goto cleanup; + } + size_t avail_out = out_cap - *out_len; + size_t room = next_dst_room(c, &dst_st); + if (avail_out > room) { + avail_out = room; + } + size_t avail_out0 = avail_out; + uint8_t* next_out = out + *out_len; + if (!BrotliEncoderCompressStream(state, op, &avail_in, &next_in, + &avail_out, &next_out, NULL)) { + ret = "BrotliEncoderCompressStream failed"; + goto cleanup; + } + *out_len += avail_out0 - avail_out; + if (op == BROTLI_OPERATION_FINISH) { + if (BrotliEncoderIsFinished(state)) { + break; + } + } else if ((avail_in == 0) && !BrotliEncoderHasMoreOutput(state)) { + break; + } + if (*out_len == out_cap) { + ret = "dst is full"; + goto cleanup; + } + } + } + +cleanup: + BrotliEncoderDestroyInstance(state); + for (size_t j = 0; j < num_prepared; j++) { + BrotliEncoderDestroyPreparedDictionary(prepared[j]); + } + return ret; +} + +// ---------------- Decoder drivers + +// Result classes. +#define CLS_OK 1 +#define CLS_NEEDS_MORE_INPUT 2 +#define CLS_NEEDS_MORE_OUTPUT 3 +#define CLS_BUG (-1000) +// Error classes are the (negative) BrotliDecoderErrorCode values. + +typedef struct { + int cls; + size_t out_n; + size_t in_used; + const char* wuffs_status; + uint64_t calls; +} dec_result; + +static int // +wuffs_status_class(const char* s) { + static const struct { + const char* status; + int code; + } table[] = { + {wuffs_brotli__error__bad_exuberant_nibble, -1}, + {wuffs_brotli__error__bad_reserved_bit, -2}, + {wuffs_brotli__error__bad_exuberant_metadata_nibble, -3}, + {wuffs_brotli__error__bad_simple_huffman_alphabet, -4}, + {wuffs_brotli__error__bad_simple_huffman_symbol, -5}, + {wuffs_brotli__error__bad_code_length_code_lengths, -6}, + {wuffs_brotli__error__bad_huffman_space, -7}, + {wuffs_brotli__error__bad_context_map_repeat, -8}, + {wuffs_brotli__error__bad_block_length, -9}, + {wuffs_brotli__error__bad_metablock_length, -10}, + {wuffs_brotli__error__bad_transform, -11}, + {wuffs_brotli__error__bad_dictionary, -12}, + {wuffs_brotli__error__bad_window_bits, -13}, + {wuffs_brotli__error__unsupported_large_window, -13}, + {wuffs_brotli__error__bad_padding, -14}, + {wuffs_brotli__error__bad_final_padding, -15}, + {wuffs_brotli__error__bad_distance, -16}, + {wuffs_brotli__error__bad_block_switch, -17}, + {wuffs_brotli__error__bad_compound_dictionary, -18}, + {wuffs_brotli__error__truncated_input, CLS_NEEDS_MORE_INPUT}, + }; + if (!s) { + return CLS_OK; + } + for (size_t i = 0; i < sizeof table / sizeof table[0]; i++) { + if (table[i].status == s) { + return table[i].code; + } + } + return CLS_BUG; +} + +// A chunking schedule: when a decoder runs out of input (or output room), +// the next chunk boundary is the previous one plus next_chunk. Both decoders +// use the same sequence of boundaries. +typedef struct { + bool one_shot; + uint64_t in_seed; + uint64_t out_seed; + int in_style; + int out_style; +} chunking; + +static size_t // +next_chunk(uint64_t* st, int style) { + uint64_t r = split64(st); + switch (style) { + case 0: // 1 to 7 bytes. + return 1 + (size_t)((r >> 8) % 7); + case 1: // Exactly 1 byte. + return 1; + case 2: // Mostly small, sometimes large. + if ((r & 0xFF) < 0xE0) { + return 1 + (size_t)((r >> 8) % 7); + } + return 1 + (size_t)((r >> 8) % 70000); + default: // 1 to 300 bytes. + return 1 + (size_t)((r >> 8) % 300); + } +} + +typedef struct { + bool large_window; + dict_set dicts; // The dictionaries attached to both decoders. + uint64_t wb_len; // The Wuffs workbuf length. + bool expect_bad_wb; // wb_len is one byte too short for the stream. + bool uninit; // WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED. + size_t out_cap; +} dec_config; + +static uint8_t* g_dec_wb; // big_alloc. +#define DEC_WB_CAP (((size_t)1 << 30) + ((size_t)64 << 20)) +static wuffs_brotli__decoder* g_dec; + +static const char* // +wuffs_attach(wuffs_brotli__decoder* dec, const dec_config* dc, size_t* n_ok) { + wuffs_base__status status = wuffs_brotli__decoder__initialize( + dec, sizeof__wuffs_brotli__decoder(), WUFFS_VERSION, + dc->uninit ? WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED : 0); + if (status.repr) { + return status.repr; + } + if (dc->large_window) { + status = wuffs_brotli__decoder__set_quirk( + dec, WUFFS_BROTLI__QUIRK_LARGE_WINDOW, 1); + if (status.repr) { + return status.repr; + } + } + *n_ok = 0; + for (size_t i = 0; i < dc->dicts.n; i++) { + status = wuffs_brotli__decoder__add_dictionary( + dec, dc->dicts.d[i].type, + wuffs_base__make_slice_u8((uint8_t*)dc->dicts.d[i].ptr, + dc->dicts.d[i].len), + wuffs_base__make_slice_u8(g_dec_wb, DEC_WB_CAP)); + if (status.repr) { + return status.repr; + } + (*n_ok)++; + } + return NULL; +} + +static dec_result // +wuffs_decode(const dec_config* dc, + const chunking* ch, + const uint8_t* in, + size_t n, + uint8_t* out) { + dec_result res = {0}; + size_t n_ok = 0; + const char* err = wuffs_attach(g_dec, dc, &n_ok); + if (err) { + res.cls = CLS_BUG; + res.wuffs_status = err; + return res; + } + wuffs_base__slice_u8 wb = + wuffs_base__make_slice_u8(g_dec_wb, (size_t)dc->wb_len); + uint64_t in_st = ch->in_seed; + uint64_t out_st = ch->out_seed; + size_t lin = ch->one_shot ? n : next_chunk(&in_st, ch->in_style); + size_t lout = ch->one_shot ? dc->out_cap : next_chunk(&out_st, ch->out_style); + lin = (lin < n) ? lin : n; + lout = (lout < dc->out_cap) ? lout : dc->out_cap; + size_t ri = 0; + size_t wi = 0; + uint64_t max_calls = 3 * ((uint64_t)n + dc->out_cap) + 100; + while (true) { + if (++res.calls > max_calls) { + res.cls = CLS_BUG; + res.wuffs_status = "too many transform_io calls (a loop?)"; + break; + } + wuffs_base__io_buffer src = + wuffs_base__ptr_u8__reader((uint8_t*)in, lin, lin == n); + src.meta.ri = ri; + wuffs_base__io_buffer dst = wuffs_base__ptr_u8__writer(out, lout); + dst.meta.wi = wi; + wuffs_base__status status = + wuffs_brotli__decoder__transform_io(g_dec, &dst, &src, wb); + if ((src.meta.ri < ri) || (dst.meta.wi < wi)) { + res.cls = CLS_BUG; + res.wuffs_status = "went backwards"; + break; + } + ri = src.meta.ri; + wi = dst.meta.wi; + res.wuffs_status = status.repr; + if (status.repr == wuffs_base__suspension__short_read) { + if (lin < n) { + lin += next_chunk(&in_st, ch->in_style); + lin = (lin < n) ? lin : n; + continue; + } + res.cls = CLS_BUG; // A short read with a closed src. + break; + } else if (status.repr == wuffs_base__suspension__short_write) { + if (lout < dc->out_cap) { + lout += next_chunk(&out_st, ch->out_style); + lout = (lout < dc->out_cap) ? lout : dc->out_cap; + continue; + } + res.cls = CLS_NEEDS_MORE_OUTPUT; + break; + } + if (status.repr == wuffs_base__error__bad_workbuf_length) { + res.cls = -100; + } else { + res.cls = wuffs_status_class(status.repr); + } + break; + } + res.out_n = wi; + res.in_used = ri; + return res; +} + +static dec_result // +c_decode(const dec_config* dc, + const chunking* ch, + const uint8_t* in, + size_t n, + uint8_t* out, + size_t* c_attached) { + dec_result res = {0}; + BrotliDecoderState* s = BrotliDecoderCreateInstance(NULL, NULL, NULL); + if (!s) { + res.cls = CLS_BUG; + return res; + } + if (dc->large_window) { + BrotliDecoderSetParameter(s, BROTLI_DECODER_PARAM_LARGE_WINDOW, 1); + } + *c_attached = 0; + for (size_t i = 0; i < dc->dicts.n; i++) { + if (!BrotliDecoderAttachDictionary( + s, (BrotliSharedDictionaryType)dc->dicts.d[i].type, + dc->dicts.d[i].len, dc->dicts.d[i].ptr)) { + break; + } + (*c_attached)++; + } + if (*c_attached != dc->dicts.n) { + BrotliDecoderDestroyInstance(s); + res.cls = CLS_BUG; + return res; + } + uint64_t in_st = ch->in_seed; + uint64_t out_st = ch->out_seed; + size_t lin = ch->one_shot ? n : next_chunk(&in_st, ch->in_style); + size_t lout = ch->one_shot ? dc->out_cap : next_chunk(&out_st, ch->out_style); + lin = (lin < n) ? lin : n; + lout = (lout < dc->out_cap) ? lout : dc->out_cap; + size_t ri = 0; + size_t wi = 0; + while (true) { + res.calls++; + size_t avail_in = lin - ri; + const uint8_t* next_in = in + ri; + size_t avail_out = lout - wi; + uint8_t* next_out = out + wi; + BrotliDecoderResult r = BrotliDecoderDecompressStream( + s, &avail_in, &next_in, &avail_out, &next_out, NULL); + ri = lin - avail_in; + wi = lout - avail_out; + if (r == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT) { + if (lin < n) { + lin += next_chunk(&in_st, ch->in_style); + lin = (lin < n) ? lin : n; + continue; + } + res.cls = CLS_NEEDS_MORE_INPUT; + } else if (r == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) { + if (lout < dc->out_cap) { + lout += next_chunk(&out_st, ch->out_style); + lout = (lout < dc->out_cap) ? lout : dc->out_cap; + continue; + } + res.cls = CLS_NEEDS_MORE_OUTPUT; + } else if (r == BROTLI_DECODER_RESULT_SUCCESS) { + res.cls = CLS_OK; + } else { + res.cls = (int)BrotliDecoderGetErrorCode(s); + } + break; + } + BrotliDecoderDestroyInstance(s); + res.out_n = wi; + res.in_used = ri; + return res; +} + +// stream_window_bits parses a brotli stream header's WBITS, returning 0 if +// it is invalid or truncated. It mirrors DecodeWindowBits in decode.c. +static uint32_t // +stream_window_bits(const uint8_t* p, size_t n, bool large_window) { + uint64_t bits = 0; + for (size_t i = 0; (i < 8) && (i < n); i++) { + bits |= ((uint64_t)p[i]) << (8 * i); + } + size_t nbits = (n < 8) ? (8 * n) : 64; + if (nbits < 1) { + return 0; + } else if ((bits & 1) == 0) { + return 16; + } else if (nbits < 4) { + return 0; + } + uint32_t k = (uint32_t)((bits >> 1) & 7); + if (k != 0) { + return 17 + k; + } else if (nbits < 7) { + return 0; + } + k = (uint32_t)((bits >> 4) & 7); + if (k == 1) { + if (!large_window || (nbits < 14) || ((bits >> 7) & 1)) { + return 0; + } + k = (uint32_t)((bits >> 8) & 63); + return ((k < 10) || (k > 30)) ? 0 : k; + } else if (k != 0) { + return 8 + k; + } + return 17; +} + +// ---------------- Failure reporting + +static void // +save_failure(const char* kind, + const char* desc, + const uint8_t* stream, + size_t stream_len, + const dict_set* dicts) { + g_num_failures++; + char path[1024]; + snprintf(path, sizeof path, "%s/fail-%s-%" PRIu64 "-%d.br", g_out_dir, kind, + g_rng_state, (int)g_num_failures); + write_file(path, stream, stream_len); + char dpath[1100]; + for (size_t i = 0; dicts && (i < dicts->n); i++) { + snprintf(dpath, sizeof dpath, "%s.dict%zu.%s", path, i, + dicts->d[i].type ? "sd" : "raw"); + write_file(dpath, dicts->d[i].ptr, dicts->d[i].len); + } + snprintf(dpath, sizeof dpath, "%s.txt", path); + write_file(dpath, (const uint8_t*)desc, strlen(desc)); + printf("FAILURE (%s) %s\n saved %s\n", kind, desc, path); + fflush(stdout); +} + +// ---------------- Decoder campaign + +typedef struct { + char name[256]; + uint8_t* ptr; + size_t len; + dict_set dicts; + bool large_window; +} seed; + +#define MAX_FILE_SEEDS 256 +#define MAX_GEN_SEEDS 256 +static seed g_file_seeds[MAX_FILE_SEEDS]; +static size_t g_num_file_seeds; +static seed g_gen_seeds[MAX_GEN_SEEDS]; +static size_t g_num_gen_seeds; + +static uint64_t g_dec_inputs; +static uint64_t g_dec_decodes; +static uint64_t g_dec_cls_counts[2][32]; // [one-shot, chunked][class]. +static uint64_t g_dec_bad_wb_checks; +static uint64_t g_dec_exact_wb; +static uint64_t g_dec_dict_rejections; +static uint64_t g_dec_lw; +static uint64_t g_dec_with_dicts; +static uint64_t g_dec_gen_streams; +static uint64_t g_dec_gen_wuffs; +static uint64_t g_dec_wuffs_shorter_output; + +static int // +cls_index(int cls) { + if (cls >= 1 && cls <= 3) { + return cls; + } else if (cls <= -1 && cls >= -18) { + return 3 - cls; // 4 ..= 21. + } else if (cls == -100) { + return 22; + } + return 0; +} + +static void // +load_file_seeds(void) { + char dir[512]; + snprintf(dir, sizeof dir, "%s/brotli", g_data_dir); + char* names[1024]; + size_t n = list_dir(dir, names, 1024); + for (size_t i = 0; i < n; i++) { + size_t k = strlen(names[i]); + bool want = ((k > 3) && !strcmp(names[i] + k - 3, ".br")) || + strstr(names[i], ".compressed"); + if (want && (g_num_file_seeds < MAX_FILE_SEEDS)) { + char path[1024]; + snprintf(path, sizeof path, "%s/%s", dir, names[i]); + size_t len = 0; + uint8_t* p = read_file(path, &len, 16 << 20); + if (p) { + seed* s = &g_file_seeds[g_num_file_seeds++]; + snprintf(s->name, sizeof s->name, "%s", names[i]); + s->ptr = p; + s->len = len; + named_dict_set(names[i], &s->dicts, &s->large_window); + } + } + free(names[i]); + } +} + +// make_gen_seed compresses a random input with random parameters (with +// libbrotlienc or, sometimes, Wuffs' encoder) and checks that libbrotlidec +// decodes it. +static void // +make_gen_seed(seed* s, uint8_t* tmp_in, uint8_t* tmp_out, size_t tmp_cap) { + free(s->ptr); + dict_set_free(&s->dicts); + size_t len = pick_len(256 << 10); + gen_input(tmp_in, len); + + enc_config c; + memset(&c, 0, sizeof c); + uint32_t q = (uint32_t)rn(12); + if ((q >= 10) && (len > (64 << 10))) { + q = (uint32_t)rn(10); + } + bool lw = (q >= 3) && chance(300); + uint32_t lgwin = 10 + (uint32_t)rn(lw ? 21 : 15); + enc_add_param(&c, 1, q); + enc_add_param(&c, 2, lgwin); + if (lw) { + enc_add_param(&c, 6, 1); + } + if (chance(300)) { + enc_add_param(&c, 7, (uint32_t)rn(4)); + enc_add_param(&c, 8, (uint32_t)rn(16) << (uint32_t)rn(4)); + } + dict_set ds = {0}; + if ((q < 10) && chance(400)) { + random_raw_dicts(&ds, 1 + rn(chance(100) ? 15 : 3), tmp_in, len); + } + // A plan: random PROCESS chunks with occasional flushes and metadata. + size_t meta_total = 0; + size_t i = 0; + while (true) { + if (chance(30)) { + size_t m = rn(chance(100) ? 70000 : 64); + if ((len + meta_total + m) > tmp_cap) { + m = 0; + } + enc_add_step(&c, OP_EMIT_METADATA, m); + memmove(tmp_in + i + m, tmp_in + i, len + meta_total - i); + for (size_t j = 0; j < m; j++) { + tmp_in[i + j] = (uint8_t)r64(); + } + i += m; + meta_total += m; + } + size_t k = chance(300) ? (len + meta_total - i) : (1 + rn(5000)); + if (k > (len + meta_total - i)) { + k = len + meta_total - i; + } + i += k; + if (i == (len + meta_total)) { + enc_add_step(&c, OP_FINISH, k); + break; + } + enc_add_step(&c, chance(100) ? OP_FLUSH : OP_PROCESS, k); + } + + size_t out_len = 0; + const char* err; + bool use_wuffs = (q < 10) && chance(250); + if (use_wuffs) { + uint64_t wlen = 0; + err = encode_wuffs(&c, &ds, tmp_in, len + meta_total, len, tmp_out, + tmp_cap, &out_len, 0, &wlen); + g_dec_gen_wuffs++; + } else { + err = encode_c(&c, &ds, tmp_in, tmp_out, tmp_cap, &out_len); + } + free(c.step_ops); + free(c.step_lens); + if (err) { + fprintf(stderr, "make_gen_seed: %s encoder: %s\n", use_wuffs ? "Wuffs" : "C", + err); + exit(2); + } + g_dec_gen_streams++; + s->ptr = xmalloc(out_len); + memcpy(s->ptr, tmp_out, out_len); + s->len = out_len; + s->large_window = lw; + s->dicts = ds; + snprintf(s->name, sizeof s->name, "gen(q=%u lgwin=%u lw=%d dicts=%zu)", q, + lgwin, (int)lw, ds.n); +} + +static size_t // +mutate(uint8_t* m, size_t n, size_t cap, const seed* other) { + int k = 1 + (int)rn(chance(800) ? 3 : 12); + for (int j = 0; j < k; j++) { + switch (rn(10)) { + case 0: + case 1: + case 2: // Bit flip. + if (n) { + m[rn(n)] ^= (uint8_t)(1u << rn(8)); + } + break; + case 3: // Byte substitution. + if (n) { + m[rn(n)] = (uint8_t)r64(); + } + break; + case 4: // Truncation. + if (n) { + n = rn(n + 1); + } + break; + case 5: { // Insertion. + size_t len = 1 + rn(chance(800) ? 4 : 64); + if ((n + len) > cap) { + break; + } + size_t p = rn(n + 1); + memmove(m + p + len, m + p, n - p); + for (size_t i = 0; i < len; i++) { + m[p + i] = (uint8_t)r64(); + } + n += len; + break; + } + case 6: { // Deletion. + if (!n) { + break; + } + size_t p = rn(n); + size_t len = 1 + rn(chance(800) ? 4 : 64); + if (len > (n - p)) { + len = n - p; + } + memmove(m + p, m + p + len, n - p - len); + n -= len; + break; + } + case 7: { // Splice: our prefix and the other stream's suffix. + if (!other || !other->len) { + break; + } + size_t p = rn(n + 1); + size_t q = rn(other->len); + size_t len = other->len - q; + if ((p + len) > cap) { + len = cap - p; + } + memcpy(m + p, other->ptr + q, len); + n = p + len; + break; + } + case 8: { // Overwrite a range with random bytes. + if (!n) { + break; + } + size_t p = rn(n); + size_t len = 1 + rn(16); + for (size_t i = 0; (i < len) && ((p + i) < n); i++) { + m[p + i] = (uint8_t)r64(); + } + break; + } + default: { // Small arithmetic change. + if (n) { + m[rn(n)] += (uint8_t)(rn(2) ? 1 : 0xFF); + } + break; + } + } + } + return n; +} + +// pick_dec_dicts chooses the dictionaries to attach: usually the seed's own, +// sometimes random ones (raw and serialized, possibly mutated). +static void // +pick_dec_dicts(dict_set* out, const seed* s) { + out->n = 0; + if (chance(700)) { + dict_set_copy(out, &s->dicts); + return; + } + size_t k = pick_num_dicts(); + for (size_t i = 0; i < k; i++) { + if (chance(250)) { + size_t which = rn(4); + size_t len = g_sd_len[which]; + uint8_t* p = xmalloc(len + 64); + memcpy(p, g_sd[which], len); + if (chance(300)) { + len = mutate(p, len, len + 64, NULL); + } + dict_set_add(out, 1, p, len, false); + out->d[out->n - 1].owned = true; + } else if (chance(300)) { + size_t off = rn(g_romeo_len); + size_t len = rn(g_romeo_len - off + 1); + dict_set_add(out, 0, g_romeo + off, len, true); + } else { + size_t len = pick_dict_len(); + if (len > (256 << 10)) { + len = rn(256 << 10); + } + uint8_t* p = random_dict_bytes(len, NULL, 0); + dict_set_add(out, 0, p, len, false); + out->d[out->n - 1].owned = true; + } + } +} + +static void // +describe_dec(char* buf, size_t cap, const char* what, const seed* s, + const dec_config* dc, const chunking* ch, const dec_result* w, + const dec_result* c, size_t n) { + snprintf(buf, cap, + "%s: seed=%s stream_len=%zu lw=%d dicts=%zu (total %zu) wb_len=%" PRIu64 + " expect_bad_wb=%d uninit=%d out_cap=%zu one_shot=%d in_seed=0x%" PRIX64 + " in_style=%d out_seed=0x%" PRIX64 " out_style=%d | wuffs: cls=%d " + "status=%s out=%zu in=%zu calls=%" PRIu64 " | c: cls=%d out=%zu in=%zu\n", + what, s->name, n, (int)dc->large_window, dc->dicts.n, + dict_set_total(&dc->dicts), dc->wb_len, (int)dc->expect_bad_wb, + (int)dc->uninit, dc->out_cap, (int)ch->one_shot, ch->in_seed, + ch->in_style, ch->out_seed, ch->out_style, w ? w->cls : 0, + (w && w->wuffs_status) ? w->wuffs_status : "ok", w ? w->out_n : 0, + w ? w->in_used : 0, w ? w->calls : 0, c ? c->cls : 0, + c ? c->out_n : 0, c ? c->in_used : 0); +} + +// compare_dec returns NULL if the two results agree. +static const char* // +compare_dec(const dec_result* w, const dec_result* c, const uint8_t* wout, + const uint8_t* cout, size_t out_cap) { + if (w->cls == CLS_BUG) { + return "Wuffs returned an unexpected status"; + } else if (c->cls == CLS_BUG) { + return "libbrotlidec setup failed"; + } else if (w->cls != c->cls) { + return "different result classes"; + } + if ((w->cls == CLS_OK) || (w->cls == CLS_NEEDS_MORE_OUTPUT)) { + if (w->out_n != c->out_n) { + return "different output lengths"; + } else if (w->out_n && memcmp(wout, cout, w->out_n)) { + return "different output bytes"; + } else if ((w->cls == CLS_OK) && (w->in_used != c->in_used)) { + return "different input consumption"; + } else if ((w->cls == CLS_NEEDS_MORE_OUTPUT) && (w->out_n != out_cap)) { + return "needs more output but dst is not full"; + } + return NULL; + } + size_t m = (w->out_n < c->out_n) ? w->out_n : c->out_n; + if (m && memcmp(wout, cout, m)) { + return "different output bytes (common prefix)"; + } + if (w->out_n < c->out_n) { + g_dec_wuffs_shorter_output++; + } + return NULL; +} + +static int // +run_dec(uint64_t iters, double secs) { + load_corpus(); + load_dicts(); + load_file_seeds(); + g_dec_wb = big_alloc(DEC_WB_CAP); + g_enc_wb = big_alloc(ENC_WB_CAP); + g_dec = (wuffs_brotli__decoder*)xmalloc(sizeof__wuffs_brotli__decoder()); + const size_t tmp_cap = (8 << 20); + uint8_t* tmp_in = xmalloc(tmp_cap); + uint8_t* tmp_out = xmalloc(tmp_cap); + const size_t max_out = (2 << 20) + 4096; + uint8_t* wout = xmalloc(max_out); + uint8_t* cout = xmalloc(max_out); + const size_t mcap = (1 << 20); + uint8_t* m = xmalloc(mcap); + + // Fill the generated seed pool. + for (size_t i = 0; i < 32; i++) { + make_gen_seed(&g_gen_seeds[g_num_gen_seeds++], tmp_in, tmp_out, tmp_cap); + } + + struct timespec t0; + clock_gettime(CLOCK_MONOTONIC, &t0); + uint64_t it = 0; + for (; it < iters; it++) { + if ((it & 63) == 0) { + struct timespec t1; + clock_gettime(CLOCK_MONOTONIC, &t1); + double elapsed = + (double)(t1.tv_sec - t0.tv_sec) + 1e-9 * (double)(t1.tv_nsec - t0.tv_nsec); + if ((secs > 0) && (elapsed > secs)) { + break; + } + if (g_dec_wb && ((it & 4095) == 0)) { + big_release(g_dec_wb, DEC_WB_CAP); + big_release(g_enc_wb, ENC_WB_CAP); + } + } + if ((it % 16) == 0) { + size_t slot = (g_num_gen_seeds < MAX_GEN_SEEDS) ? g_num_gen_seeds++ + : (size_t)rn(MAX_GEN_SEEDS); + make_gen_seed(&g_gen_seeds[slot], tmp_in, tmp_out, tmp_cap); + } + + // Pick a seed and mutate it. + const seed* s = chance(500) ? &g_file_seeds[rn(g_num_file_seeds)] + : &g_gen_seeds[rn(g_num_gen_seeds)]; + const seed* other = chance(500) ? &g_file_seeds[rn(g_num_file_seeds)] + : &g_gen_seeds[rn(g_num_gen_seeds)]; + size_t n; + if (chance(20)) { + n = rn(chance(500) ? 32 : 4096); + for (size_t i = 0; i < n; i++) { + m[i] = (uint8_t)r64(); + } + } else { + n = (s->len < mcap) ? s->len : mcap; + memcpy(m, s->ptr, n); + if (!chance(30)) { // 3% are unmutated. + n = mutate(m, n, mcap, other); + } + } + g_dec_inputs++; + + dec_config dc; + memset(&dc, 0, sizeof dc); + dc.large_window = chance(800) ? s->large_window : chance(500); + g_dec_lw += dc.large_window; + pick_dec_dicts(&dc.dicts, s); + g_dec_with_dicts += (dc.dicts.n > 0); + dc.uninit = chance(500); + dc.out_cap = chance(900) ? ((size_t)1 << 20) + rn(1 << 20) : rn(4096); + + // Attach the dictionaries to both decoders, checking that they agree on + // which ones are accepted. libbrotlidec can be left in a modified state + // after a failure (see std/brotli/README.md), so only the dictionaries + // before the first failure are used. + { + size_t w_ok = 0; + const char* werr = wuffs_attach(g_dec, &dc, &w_ok); + BrotliDecoderState* cs = BrotliDecoderCreateInstance(NULL, NULL, NULL); + size_t c_ok = 0; + for (size_t i = 0; i < dc.dicts.n; i++) { + if (!BrotliDecoderAttachDictionary( + cs, (BrotliSharedDictionaryType)dc.dicts.d[i].type, + dc.dicts.d[i].len, dc.dicts.d[i].ptr)) { + break; + } + c_ok++; + } + BrotliDecoderDestroyInstance(cs); + if (w_ok != c_ok) { + char desc[2048]; + snprintf(desc, sizeof desc, + "add_dictionary: Wuffs accepted %zu (then \"%s\"), " + "libbrotlidec accepted %zu, of %zu dictionaries (seed %s)\n", + w_ok, werr ? werr : "", c_ok, dc.dicts.n, s->name); + save_failure("dict", desc, m, n, &dc.dicts); + } + if (w_ok < dc.dicts.n) { + g_dec_dict_rejections++; + size_t keep = (w_ok < c_ok) ? w_ok : c_ok; + for (size_t i = keep; i < dc.dicts.n; i++) { + if (dc.dicts.d[i].owned) { + free((void*)dc.dicts.d[i].ptr); + } + } + dc.dicts.n = keep; + } + } + + // The workbuf length. + { + size_t w_ok = 0; + wuffs_attach(g_dec, &dc, &w_ok); + uint64_t wlen = wuffs_brotli__decoder__workbuf_len(g_dec).max_incl; + uint32_t wbits = stream_window_bits(m, n, dc.large_window); + uint64_t exact = wbits ? (dict_set_total(&dc.dicts) + + ((uint64_t)1 << wbits) + 542) + : 0; + uint64_t r = rn(1000); + if (exact && (r < 100)) { + dc.wb_len = exact; + g_dec_exact_wb++; + } else if (exact && (r < 130)) { + dc.wb_len = exact - 1; + dc.expect_bad_wb = true; + g_dec_bad_wb_checks++; + } else if (r < 600) { + dc.wb_len = wlen; + } else { + dc.wb_len = wlen + rn(65536); + } + if (dc.wb_len > DEC_WB_CAP) { + dc.wb_len = wlen; + dc.expect_bad_wb = false; + } + } + + for (int pass = 0; pass < 2; pass++) { + chunking ch; + memset(&ch, 0, sizeof ch); + ch.one_shot = (pass == 0); + ch.in_seed = r64(); + ch.out_seed = r64(); + ch.in_style = (int)rn(4); + ch.out_style = (int)rn(4); + size_t c_attached = 0; + dec_result w = wuffs_decode(&dc, &ch, m, n, wout); + dec_result c = c_decode(&dc, &ch, m, n, cout, &c_attached); + g_dec_decodes += 2; + g_dec_cls_counts[pass][cls_index(c.cls)]++; + const char* why = NULL; + if (dc.expect_bad_wb) { + // The workbuf is one byte too short for the window. Wuffs must fail + // with "#bad workbuf length" unless the stream fails (or is + // truncated) before the window size is known, which can only be the + // case for a stream with an invalid or truncated header. Since + // stream_window_bits succeeded, the header is valid and complete. + if (w.cls != -100) { + why = "a too-short workbuf did not fail"; + } + } else { + why = compare_dec(&w, &c, wout, cout, dc.out_cap); + } + if (why) { + char desc[4096]; + describe_dec(desc, sizeof desc, why, s, &dc, &ch, &w, &c, n); + save_failure("dec", desc, m, n, &dc.dicts); + } + } + dict_set_free(&dc.dicts); + } + + printf("decoder campaign: seed=%" PRIu64 " inputs=%" PRIu64 " decodes=%" PRIu64 + " failures=%" PRIu64 "\n", + g_seed, g_dec_inputs, g_dec_decodes, g_num_failures); + printf(" generated seed streams=%" PRIu64 " (%" PRIu64 + " by Wuffs' encoder), large window=%" PRIu64 ", with dicts=%" PRIu64 + ", dict rejections=%" PRIu64 ", exact workbuf=%" PRIu64 + ", too-short workbuf=%" PRIu64 ", wuffs shorter output on error=%" PRIu64 + "\n", + g_dec_gen_streams, g_dec_gen_wuffs, g_dec_lw, g_dec_with_dicts, + g_dec_dict_rejections, g_dec_exact_wb, g_dec_bad_wb_checks, + g_dec_wuffs_shorter_output); + static const char* names[23] = { + "other", "ok", "needs-more-input", "needs-more-output", + "-1", "-2", "-3", "-4", + "-5", "-6", "-7", "-8", + "-9", "-10", "-11", "-12", + "-13", "-14", "-15", "-16", + "-17", "-18", "bad-workbuf"}; + for (int pass = 0; pass < 2; pass++) { + printf(" %s classes (C):", pass ? "chunked" : "one-shot"); + for (int i = 0; i < 23; i++) { + if (g_dec_cls_counts[pass][i]) { + printf(" %s=%" PRIu64, names[i], g_dec_cls_counts[pass][i]); + } + } + printf("\n"); + } + return g_num_failures ? 1 : 0; +} + +// ---------------- Encoder campaign + +static int run_enc(uint64_t iters, double secs); + +int // +main(int argc, char** argv) { + if (BrotliEncoderVersion() != 0x1002001 || + BrotliDecoderVersion() != 0x1002001) { + fprintf(stderr, "want libbrotli v1.2.1, have enc 0x%x dec 0x%x\n", + BrotliEncoderVersion(), BrotliDecoderVersion()); + return 2; + } + if (argc < 2) { + fprintf(stderr, "usage: %s -dec|-enc [options]\n", argv[0]); + return 2; + } + uint64_t seed = 1; + uint64_t iters = 1000; + double secs = 0; + for (int i = 2; i < argc; i++) { + if (!strncmp(argv[i], "-seed=", 6)) { + seed = strtoull(argv[i] + 6, NULL, 0); + } else if (!strncmp(argv[i], "-n=", 3)) { + iters = strtoull(argv[i] + 3, NULL, 0); + } else if (!strncmp(argv[i], "-secs=", 6)) { + secs = atof(argv[i] + 6); + } else if (!strncmp(argv[i], "-out=", 5)) { + g_out_dir = argv[i] + 5; + } else if (!strncmp(argv[i], "-data=", 6)) { + g_data_dir = argv[i] + 6; + } else if (!strncmp(argv[i], "-q=", 3)) { + unsigned lo = 0, hi = 0; + if (sscanf(argv[i] + 3, "%u-%u", &lo, &hi) != 2 || lo > hi || hi > 11) { + fprintf(stderr, "bad -q option\n"); + return 2; + } + g_qmin = lo; + g_qmax = hi; + } else if (!strcmp(argv[i], "-hq")) { + g_hq_dicts = true; + } else { + fprintf(stderr, "unknown option %s\n", argv[i]); + return 2; + } + } + g_seed = seed; + g_rng_state = seed * 0x9E3779B97F4A7C15ull; + if (!strcmp(argv[1], "-dec")) { + return run_dec(iters, secs); + } else if (!strcmp(argv[1], "-enc")) { + return run_enc(iters, secs); + } + fprintf(stderr, "unknown mode %s\n", argv[1]); + return 2; +} + +static int // +run_enc(uint64_t iters, double secs) { + (void)iters; + (void)secs; + return 2; +}