mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-17 16:32:11 +01:00
The decoder can now report metadata blocks, like libbrotlidec's BrotliDecoderSetMetadataCallbacks: transform_io returns "@metadata reported", metadata_length gives the block's length and the block's contents are written to dst. The differential fuzzer compares the reported metadata with libbrotlidec's callbacks. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UZ978Fjdsyig4v3wYAZRg5
3374 lines
111 KiB
C
3374 lines
111 KiB
C
// Copyright 2026 The Wuffs Authors.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
|
|
// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
|
|
// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
|
|
// option. This file may not be copied, modified, or distributed
|
|
// except according to those terms.
|
|
//
|
|
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
|
|
|
// ----------------
|
|
|
|
// manual-test-brotli-fuzz.c is a differential fuzzer for std/brotli against
|
|
// the reference implementation (libbrotlidec and libbrotlienc v1.2.1). Every
|
|
// parameter is drawn from a pseudo-random number generator.
|
|
//
|
|
// Modes (the first argument):
|
|
//
|
|
// -dec decodes randomly mutated brotli streams with both libbrotlidec and
|
|
// Wuffs' decoder and checks that they produce the same output bytes,
|
|
// consume the same input (on success) and agree on the result: ok,
|
|
// truncated input, a full dst (needs more output) or the specific
|
|
// BROTLI_DECODER_ERROR_FORMAT_* code (see the "Status Mapping" in
|
|
// std/brotli/README.md). Seeds are the test/data/brotli files and
|
|
// streams that libbrotlienc or Wuffs' encoder produce (at random
|
|
// qualities, window sizes, large window and raw dictionaries) from
|
|
// random inputs. Mutations are bit flips, byte substitutions,
|
|
// truncations, insertions, deletions, splices of two streams and
|
|
// random bytes. Each stream is decoded one-shot (by both) and with
|
|
// random I/O chunking (the same absolute chunk boundaries for both,
|
|
// mostly 1 to 7 bytes), with a random large-window quirk, random
|
|
// (raw and serialized) dictionaries and a random workbuf length (at
|
|
// or above workbuf_len, or exactly the stream's minimum, or one byte
|
|
// less than that, which must fail). Some decodes (with a large dst)
|
|
// also enable QUIRK_REPORT_METADATA and libbrotlidec's metadata
|
|
// callbacks and check that both report the same metadata blocks.
|
|
//
|
|
// -enc encodes random inputs with random parameters (quality, lgwin,
|
|
// large window, lgblock, mode, size hint, npostfix, ndirect,
|
|
// disabling literal context modeling, stream offset, raw
|
|
// dictionaries) and random plans of PROCESS, FLUSH, EMIT_METADATA and
|
|
// FINISH steps with random chunk sizes and dst limits, with both
|
|
// libbrotlienc and Wuffs' encoder, and checks that the output is
|
|
// byte-exact and that both decoders round-trip it.
|
|
//
|
|
// Options (after the mode):
|
|
// -seed=S the PRNG seed (default 1).
|
|
// -n=N the number of iterations (default 1000).
|
|
// -secs=T stop after about T seconds.
|
|
// -q=LO-HI (-enc only) restrict the quality to LO ..= HI.
|
|
// -nohq (-enc only) no dictionaries at qualities 10 and 11 (by
|
|
// default, 10% of the q10-11 configs get small inputs and
|
|
// dictionaries).
|
|
// -out=DIR where to save failing cases (default ".").
|
|
// -data=DIR the test/data directory (default "test/data").
|
|
//
|
|
// The libbrotlidec that it is linked with should be built with
|
|
// BROTLI_EXPERIMENTAL (so that it accepts serialized dictionaries, like
|
|
// Wuffs' decoder). Only c/common/shared_dictionary.c depends on it, so
|
|
// libbrotlienc can be the default build. For example, from the Wuffs root
|
|
// directory (as one command line):
|
|
//
|
|
// for f in $BROTLI/c/common/*.c $BROTLI/c/dec/*.c; do
|
|
// gcc -O2 -DBROTLI_EXPERIMENTAL -I$BROTLI/c/include -c $f
|
|
// -o /tmp/bd/$(basename $(dirname $f))_$(basename $f .c).o; done
|
|
// ar rcs /tmp/bd/libbrotlidecexp.a /tmp/bd/*.o
|
|
// gcc -O2 -std=c99 -I$BROTLI/c/include -I$BROTLI/c
|
|
// script/manual-test-brotli-fuzz.c
|
|
// $BROTLI_BUILD/libbrotlienc.a /tmp/bd/libbrotlidecexp.a -lm
|
|
// -o /tmp/brotli-fuzz
|
|
// /tmp/brotli-fuzz -dec -seed=1 -n=100000
|
|
//
|
|
// It prints a summary and exits with a non-zero status if there were any
|
|
// divergences. Each divergence is saved (the stream, its dictionaries and a
|
|
// description) under the -out directory.
|
|
|
|
#define _DEFAULT_SOURCE
|
|
#define _POSIX_C_SOURCE 200809L
|
|
|
|
#include <dirent.h>
|
|
#include <inttypes.h>
|
|
#include <stdbool.h>
|
|
#include <stdint.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
#include <sys/mman.h>
|
|
#include <sys/stat.h>
|
|
#include <time.h>
|
|
|
|
#include "brotli/decode.h"
|
|
#include "brotli/encode.h"
|
|
#include "brotli/shared_dictionary.h"
|
|
// libbrotlienc's internal state, to recognize a known C bug (see
|
|
// c_state_corrupted). This needs -I$BROTLI/c as well as -I$BROTLI/c/include.
|
|
#include "enc/state.h"
|
|
|
|
#define WUFFS_IMPLEMENTATION
|
|
#define WUFFS_CONFIG__MODULES
|
|
#define WUFFS_CONFIG__MODULE__BASE
|
|
#define WUFFS_CONFIG__MODULE__BROTLI
|
|
#define WUFFS_CONFIG__MODULE__SOFTFLOAT
|
|
#include "../release/c/wuffs-unsupported-snapshot.c"
|
|
|
|
// ---------------- PRNG
|
|
|
|
static uint64_t g_rng_state = 1;
|
|
static uint64_t g_seed = 1;
|
|
|
|
static uint64_t //
|
|
r64(void) {
|
|
// SplitMix64.
|
|
uint64_t z = (g_rng_state += 0x9E3779B97F4A7C15ull);
|
|
z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ull;
|
|
z = (z ^ (z >> 27)) * 0x94D049BB133111EBull;
|
|
return z ^ (z >> 31);
|
|
}
|
|
|
|
// rn returns a pseudo-random number in [0 .. n), or 0 if n is 0.
|
|
static uint64_t //
|
|
rn(uint64_t n) {
|
|
return n ? (r64() % n) : 0;
|
|
}
|
|
|
|
// chance returns true with probability (k / 1000).
|
|
static bool //
|
|
chance(uint64_t k) {
|
|
return rn(1000) < k;
|
|
}
|
|
|
|
static uint64_t //
|
|
split64(uint64_t* state) {
|
|
uint64_t z = (*state += 0x9E3779B97F4A7C15ull);
|
|
z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ull;
|
|
z = (z ^ (z >> 27)) * 0x94D049BB133111EBull;
|
|
return z ^ (z >> 31);
|
|
}
|
|
|
|
// ---------------- Globals
|
|
|
|
static const char* g_out_dir = ".";
|
|
static const char* g_data_dir = "test/data";
|
|
static uint64_t g_num_failures = 0;
|
|
static uint32_t g_qmin = 0;
|
|
static uint32_t g_qmax = 11;
|
|
static bool g_hq_dicts = true;
|
|
|
|
static uint8_t* //
|
|
xmalloc(size_t n) {
|
|
uint8_t* p = malloc(n ? n : 1);
|
|
if (!p) {
|
|
fprintf(stderr, "out of memory (%zu bytes)\n", n);
|
|
exit(2);
|
|
}
|
|
return p;
|
|
}
|
|
|
|
// big_alloc reserves (but does not touch) n bytes.
|
|
static uint8_t* //
|
|
big_alloc(size_t n) {
|
|
void* p = mmap(NULL, n, PROT_READ | PROT_WRITE,
|
|
MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0);
|
|
if (p == MAP_FAILED) {
|
|
fprintf(stderr, "mmap of %zu bytes failed\n", n);
|
|
exit(2);
|
|
}
|
|
return (uint8_t*)p;
|
|
}
|
|
|
|
// big_release gives back the physical memory of a big_alloc region.
|
|
static void //
|
|
big_release(uint8_t* p, size_t n) {
|
|
madvise(p, n, MADV_DONTNEED);
|
|
}
|
|
|
|
static uint8_t* //
|
|
read_file(const char* filename, size_t* len, size_t max_len) {
|
|
FILE* f = fopen(filename, "rb");
|
|
if (!f) {
|
|
return NULL;
|
|
}
|
|
struct stat st;
|
|
if ((fstat(fileno(f), &st) != 0) || !S_ISREG(st.st_mode) ||
|
|
((size_t)st.st_size > max_len)) {
|
|
fclose(f);
|
|
return NULL;
|
|
}
|
|
size_t n = (size_t)st.st_size;
|
|
uint8_t* p = xmalloc(n + 1);
|
|
if (fread(p, 1, n, f) != n) {
|
|
fclose(f);
|
|
free(p);
|
|
return NULL;
|
|
}
|
|
fclose(f);
|
|
*len = n;
|
|
return p;
|
|
}
|
|
|
|
static void //
|
|
write_file(const char* filename, const uint8_t* p, size_t n) {
|
|
FILE* f = fopen(filename, "wb");
|
|
if (!f) {
|
|
fprintf(stderr, "could not write %s\n", filename);
|
|
return;
|
|
}
|
|
if (n) {
|
|
fwrite(p, 1, n, f);
|
|
}
|
|
fclose(f);
|
|
}
|
|
|
|
// ---------------- Corpus (for inputs and dictionaries)
|
|
|
|
typedef struct {
|
|
char name[256];
|
|
uint8_t* ptr;
|
|
size_t len;
|
|
} blob;
|
|
|
|
#define MAX_CORPUS 512
|
|
static blob g_corpus[MAX_CORPUS];
|
|
static size_t g_num_corpus;
|
|
|
|
static int //
|
|
cmp_names(const void* a, const void* b) {
|
|
return strcmp(*(const char* const*)a, *(const char* const*)b);
|
|
}
|
|
|
|
// list_dir returns the sorted names of the regular files in dir.
|
|
static size_t //
|
|
list_dir(const char* dir, char** names, size_t max) {
|
|
DIR* d = opendir(dir);
|
|
if (!d) {
|
|
return 0;
|
|
}
|
|
size_t n = 0;
|
|
struct dirent* e;
|
|
while ((e = readdir(d)) && (n < max)) {
|
|
if (e->d_name[0] == '.') {
|
|
continue;
|
|
}
|
|
names[n++] = strdup(e->d_name);
|
|
}
|
|
closedir(d);
|
|
qsort(names, n, sizeof names[0], cmp_names);
|
|
return n;
|
|
}
|
|
|
|
static void //
|
|
load_corpus(void) {
|
|
static const char* subdirs[] = {"", "/brotli"};
|
|
for (int s = 0; s < 2; s++) {
|
|
char dir[512];
|
|
snprintf(dir, sizeof dir, "%s%s", g_data_dir, subdirs[s]);
|
|
char* names[1024];
|
|
size_t n = list_dir(dir, names, 1024);
|
|
for (size_t i = 0; i < n; i++) {
|
|
size_t k = strlen(names[i]);
|
|
bool compressed = ((k > 3) && !strcmp(names[i] + k - 3, ".br")) ||
|
|
strstr(names[i], ".compressed") ||
|
|
strstr(names[i], ".bin");
|
|
if ((s == 1) && compressed) {
|
|
free(names[i]);
|
|
continue;
|
|
}
|
|
char path[1024];
|
|
snprintf(path, sizeof path, "%s/%s", dir, names[i]);
|
|
size_t len = 0;
|
|
uint8_t* p = read_file(path, &len, 4 << 20);
|
|
if (p && (len > 0) && (g_num_corpus < MAX_CORPUS)) {
|
|
blob* b = &g_corpus[g_num_corpus++];
|
|
snprintf(b->name, sizeof b->name, "%s", path);
|
|
b->ptr = p;
|
|
b->len = len;
|
|
} else {
|
|
free(p);
|
|
}
|
|
free(names[i]);
|
|
}
|
|
}
|
|
if (g_num_corpus == 0) {
|
|
fprintf(stderr, "no corpus files in %s (use -data=DIR)\n", g_data_dir);
|
|
exit(2);
|
|
}
|
|
}
|
|
|
|
// ---------------- Input generators
|
|
|
|
// pick_len returns a pseudo-random length up to max, with an emphasis on
|
|
// short lengths.
|
|
static size_t //
|
|
pick_len(size_t max) {
|
|
uint64_t r = rn(1000);
|
|
size_t n;
|
|
if (r < 30) {
|
|
n = rn(4);
|
|
} else if (r < 300) {
|
|
n = rn(256);
|
|
} else if (r < 550) {
|
|
n = rn(4096);
|
|
} else if (r < 850) {
|
|
n = rn(65536 + 1);
|
|
} else if (r < 970) {
|
|
n = rn(1 << 20);
|
|
} else {
|
|
n = rn(4 << 20);
|
|
}
|
|
return (n < max) ? n : max;
|
|
}
|
|
|
|
static void //
|
|
put_utf8(uint8_t* p, size_t* i, size_t n, uint32_t c) {
|
|
uint8_t b[4];
|
|
size_t k;
|
|
if (c < 0x80) {
|
|
b[0] = (uint8_t)c;
|
|
k = 1;
|
|
} else if (c < 0x800) {
|
|
b[0] = (uint8_t)(0xC0 | (c >> 6));
|
|
b[1] = (uint8_t)(0x80 | (c & 0x3F));
|
|
k = 2;
|
|
} else if (c < 0x10000) {
|
|
b[0] = (uint8_t)(0xE0 | (c >> 12));
|
|
b[1] = (uint8_t)(0x80 | ((c >> 6) & 0x3F));
|
|
b[2] = (uint8_t)(0x80 | (c & 0x3F));
|
|
k = 3;
|
|
} else {
|
|
b[0] = (uint8_t)(0xF0 | (c >> 18));
|
|
b[1] = (uint8_t)(0x80 | ((c >> 12) & 0x3F));
|
|
b[2] = (uint8_t)(0x80 | ((c >> 6) & 0x3F));
|
|
b[3] = (uint8_t)(0x80 | (c & 0x3F));
|
|
k = 4;
|
|
}
|
|
for (size_t j = 0; (j < k) && (*i < n); j++) {
|
|
p[(*i)++] = b[j];
|
|
}
|
|
}
|
|
|
|
// gen_segment fills p[0 .. n] with one kind of data. p[-dist] (for dist up to
|
|
// prefix) may be read, for repeats of earlier data.
|
|
static int //
|
|
gen_segment(uint8_t* p, size_t n, size_t prefix, int kind) {
|
|
switch (kind) {
|
|
case 0: { // Random bytes.
|
|
for (size_t i = 0; i < n; i++) {
|
|
p[i] = (uint8_t)r64();
|
|
}
|
|
break;
|
|
}
|
|
case 1: { // A slice of a corpus file (text, images, binaries).
|
|
const blob* b = &g_corpus[rn(g_num_corpus)];
|
|
size_t off = rn(b->len);
|
|
for (size_t i = 0; i < n; i++) {
|
|
p[i] = b->ptr[(off + i) % b->len];
|
|
}
|
|
break;
|
|
}
|
|
case 2: { // A repeated pattern.
|
|
size_t period = 1 + rn(chance(500) ? 8 : 400);
|
|
for (size_t i = 0; i < n; i++) {
|
|
p[i] = (i < period) ? (uint8_t)r64() : p[i - period];
|
|
}
|
|
break;
|
|
}
|
|
case 3: { // Text from a small vocabulary, runs and noise.
|
|
static const char* words[] = {"the ", "brotli ", "wuffs ", "of ",
|
|
"and ", "data\n", "time ", "<a>",
|
|
"encoder", ", ", "zebra ", "1234",
|
|
"\xC3\xA9t\xC3\xA9 ", "\t", "{\"k\": ",
|
|
"}\n"};
|
|
size_t i = 0;
|
|
while (i < n) {
|
|
uint64_t r = r64();
|
|
if ((r & 0xFF) < 200) {
|
|
const char* w = words[(r >> 8) % 16];
|
|
while (*w && (i < n)) {
|
|
p[i++] = (uint8_t)(*w++);
|
|
}
|
|
} else if ((r & 0xFF) < 230) {
|
|
size_t k = (size_t)((r >> 8) % 300);
|
|
while (k-- && (i < n)) {
|
|
p[i++] = (uint8_t)(r >> 40);
|
|
}
|
|
} else {
|
|
size_t k = (size_t)((r >> 8) % 2000);
|
|
while (k-- && (i < n)) {
|
|
p[i++] = (uint8_t)r64();
|
|
}
|
|
}
|
|
}
|
|
break;
|
|
}
|
|
case 4: { // Repeats of earlier data (near and far), with changes.
|
|
if (prefix == 0) {
|
|
return gen_segment(p, n, prefix, 3);
|
|
}
|
|
size_t i = 0;
|
|
while (i < n) {
|
|
size_t avail = prefix + i;
|
|
size_t dist = 1 + (chance(300) ? rn(avail) : rn(avail < 64 ? avail : 64));
|
|
size_t k = 4 + rn(chance(100) ? 70000 : 300);
|
|
for (size_t j = 0; (j < k) && (i < n); j++, i++) {
|
|
p[i] = p[(ptrdiff_t)i - (ptrdiff_t)dist];
|
|
}
|
|
if (i < n) {
|
|
p[i++] = (uint8_t)r64();
|
|
}
|
|
}
|
|
break;
|
|
}
|
|
case 5: { // UTF-8 text (valid, sometimes invalid).
|
|
size_t i = 0;
|
|
while (i < n) {
|
|
uint64_t r = r64();
|
|
uint32_t c;
|
|
switch (r & 7) {
|
|
case 0:
|
|
case 1:
|
|
case 2:
|
|
c = 0x20 + (uint32_t)((r >> 8) % 0x5F);
|
|
break;
|
|
case 3:
|
|
case 4:
|
|
c = 0x80 + (uint32_t)((r >> 8) % 0x780);
|
|
break;
|
|
case 5:
|
|
c = 0x800 + (uint32_t)((r >> 8) % 0xD000);
|
|
break;
|
|
case 6:
|
|
c = 0x10000 + (uint32_t)((r >> 8) % 0x100000);
|
|
break;
|
|
default:
|
|
p[i++] = (uint8_t)(r >> 8);
|
|
continue;
|
|
}
|
|
put_utf8(p, &i, n, c);
|
|
}
|
|
break;
|
|
}
|
|
case 6: { // A small alphabet.
|
|
uint32_t k = 1 + (uint32_t)rn(4);
|
|
uint8_t base = (uint8_t)r64();
|
|
for (size_t i = 0; i < n; i++) {
|
|
p[i] = (uint8_t)(base + (uint8_t)rn(k));
|
|
}
|
|
break;
|
|
}
|
|
default: { // Zeroes.
|
|
memset(p, 0, n);
|
|
break;
|
|
}
|
|
}
|
|
return kind;
|
|
}
|
|
|
|
// gen_input fills p[0 .. n] with a mixture of segments.
|
|
static void //
|
|
gen_input(uint8_t* p, size_t n) {
|
|
if (chance(400)) {
|
|
gen_segment(p, n, 0, (int)rn(8));
|
|
return;
|
|
}
|
|
size_t i = 0;
|
|
while (i < n) {
|
|
size_t k = 1 + rn(chance(200) ? (n - i) : 4096);
|
|
if (k > (n - i)) {
|
|
k = n - i;
|
|
}
|
|
gen_segment(p + i, k, i, (int)rn(8));
|
|
i += k;
|
|
}
|
|
}
|
|
|
|
// ---------------- Dictionaries
|
|
|
|
typedef struct {
|
|
uint32_t type; // 0 raw, 1 serialized.
|
|
const uint8_t* ptr;
|
|
size_t len;
|
|
bool owned;
|
|
} dict;
|
|
|
|
#define MAX_DICTS 16
|
|
|
|
typedef struct {
|
|
dict d[MAX_DICTS];
|
|
size_t n;
|
|
} dict_set;
|
|
|
|
static void //
|
|
dict_set_free(dict_set* s) {
|
|
for (size_t i = 0; i < s->n; i++) {
|
|
if (s->d[i].owned) {
|
|
free((void*)s->d[i].ptr);
|
|
}
|
|
}
|
|
s->n = 0;
|
|
}
|
|
|
|
static void //
|
|
dict_set_add(dict_set* s, uint32_t type, const uint8_t* ptr, size_t len,
|
|
bool copy) {
|
|
if (s->n >= MAX_DICTS) {
|
|
return;
|
|
}
|
|
dict* d = &s->d[s->n++];
|
|
d->type = type;
|
|
d->len = len;
|
|
d->owned = copy;
|
|
if (copy) {
|
|
uint8_t* q = xmalloc(len);
|
|
if (len) {
|
|
memcpy(q, ptr, len);
|
|
}
|
|
d->ptr = q;
|
|
} else {
|
|
d->ptr = ptr;
|
|
}
|
|
}
|
|
|
|
static void //
|
|
dict_set_copy(dict_set* dst, const dict_set* src) {
|
|
dst->n = 0;
|
|
for (size_t i = 0; i < src->n; i++) {
|
|
dict_set_add(dst, src->d[i].type, src->d[i].ptr, src->d[i].len, true);
|
|
}
|
|
}
|
|
|
|
static size_t //
|
|
dict_set_total(const dict_set* s) {
|
|
size_t t = 0;
|
|
for (size_t i = 0; i < s->n; i++) {
|
|
t += s->d[i].len;
|
|
}
|
|
return t;
|
|
}
|
|
|
|
// Well-known dictionaries (see test/data/brotli/README.md).
|
|
static uint8_t* g_romeo;
|
|
static size_t g_romeo_len;
|
|
static uint8_t* g_alice;
|
|
static size_t g_alice_len;
|
|
static uint8_t* g_sd[4]; // full, words, prefix, utf8.
|
|
static size_t g_sd_len[4];
|
|
|
|
static uint8_t* //
|
|
must_read(const char* rel, size_t* len) {
|
|
char path[1024];
|
|
snprintf(path, sizeof path, "%s/%s", g_data_dir, rel);
|
|
uint8_t* p = read_file(path, len, 64 << 20);
|
|
if (!p) {
|
|
fprintf(stderr, "could not read %s\n", path);
|
|
exit(2);
|
|
}
|
|
return p;
|
|
}
|
|
|
|
static void //
|
|
load_dicts(void) {
|
|
g_romeo = must_read("romeo.txt", &g_romeo_len);
|
|
g_alice = must_read("brotli/alice29.txt", &g_alice_len);
|
|
g_sd[0] = must_read("brotli/shared-dictionary.full.bin", &g_sd_len[0]);
|
|
g_sd[1] = must_read("brotli/shared-dictionary.words.bin", &g_sd_len[1]);
|
|
g_sd[2] = must_read("brotli/shared-dictionary.prefix.bin", &g_sd_len[2]);
|
|
g_sd[3] = must_read("brotli/shared-dictionary.utf8.bin", &g_sd_len[3]);
|
|
}
|
|
|
|
// named_dict_set fills s with the dictionaries that the named test/data/brotli
|
|
// file was compressed with (and sets *lw for large-window files).
|
|
static void //
|
|
named_dict_set(const char* name, dict_set* s, bool* lw) {
|
|
s->n = 0;
|
|
*lw = strstr(name, "lw") != NULL;
|
|
if (strstr(name, "raw-romeo")) {
|
|
dict_set_add(s, 0, g_romeo, g_romeo_len, false);
|
|
} else if (strstr(name, "alice29.txt.") && strstr(name, "raw-chunks")) {
|
|
dict_set_add(s, 0, g_alice, 20000, false);
|
|
dict_set_add(s, 0, g_alice + 20000, 1, false);
|
|
dict_set_add(s, 0, g_alice + 20001, 0, false);
|
|
dict_set_add(s, 0, g_alice + 20001, 40000 - 1, false);
|
|
} else if (strstr(name, "raw-chunks") || strstr(name, "invalid.raw-")) {
|
|
dict_set_add(s, 0, g_romeo, 300, false);
|
|
dict_set_add(s, 0, g_romeo + 300, 200, false);
|
|
dict_set_add(s, 0, g_romeo + 500, g_romeo_len - 500, false);
|
|
} else if (strstr(name, "sd-full") || strstr(name, "invalid.sd-")) {
|
|
dict_set_add(s, 1, g_sd[0], g_sd_len[0], false);
|
|
} else if (strstr(name, "sd-words")) {
|
|
dict_set_add(s, 1, g_sd[1], g_sd_len[1], false);
|
|
} else if (strstr(name, "sd-prefix-raw")) {
|
|
dict_set_add(s, 1, g_sd[2], g_sd_len[2], false);
|
|
dict_set_add(s, 0, g_romeo, 300, false);
|
|
} else if (strstr(name, "sd-utf8")) {
|
|
dict_set_add(s, 1, g_sd[3], g_sd_len[3], false);
|
|
}
|
|
}
|
|
|
|
// random_dict_bytes makes a random raw dictionary of len bytes, sometimes
|
|
// based on hint (e.g. the input that will be compressed).
|
|
static uint8_t* //
|
|
random_dict_bytes(size_t len, const uint8_t* hint, size_t hint_len) {
|
|
uint8_t* p = xmalloc(len);
|
|
if (hint && hint_len && chance(600)) {
|
|
size_t off = rn(hint_len);
|
|
for (size_t i = 0; i < len; i++) {
|
|
p[i] = hint[(off + i) % hint_len];
|
|
}
|
|
// Sparse changes.
|
|
if (len && chance(500)) {
|
|
for (size_t k = rn(8); k > 0; k--) {
|
|
p[rn(len)] ^= (uint8_t)(1 + rn(255));
|
|
}
|
|
}
|
|
} else {
|
|
gen_input(p, len);
|
|
}
|
|
return p;
|
|
}
|
|
|
|
static size_t //
|
|
pick_dict_len(void) {
|
|
uint64_t r = rn(1000);
|
|
if (r < 50) {
|
|
return 0;
|
|
} else if (r < 400) {
|
|
return 1 + rn(64);
|
|
} else if (r < 800) {
|
|
return rn(4096);
|
|
} else if (r < 985) {
|
|
return rn(256 << 10);
|
|
} else if (r < 995) {
|
|
// Around the 2 MiB hash table size boundary.
|
|
return (2 << 20) - 8 + rn(17);
|
|
}
|
|
return rn(3 << 20);
|
|
}
|
|
|
|
// random_raw_dicts fills s with k random raw dictionaries.
|
|
static void //
|
|
random_raw_dicts(dict_set* s, size_t k, const uint8_t* hint, size_t hint_len) {
|
|
s->n = 0;
|
|
for (size_t i = 0; i < k; i++) {
|
|
size_t len = pick_dict_len();
|
|
uint8_t* p = random_dict_bytes(len, hint, hint_len);
|
|
dict_set_add(s, 0, p, len, false);
|
|
s->d[s->n - 1].owned = true;
|
|
}
|
|
}
|
|
|
|
static size_t //
|
|
pick_num_dicts(void) {
|
|
uint64_t r = rn(1000);
|
|
if (r < 400) {
|
|
return 0;
|
|
} else if (r < 700) {
|
|
return 1;
|
|
} else if (r < 850) {
|
|
return 2 + rn(3);
|
|
} else if (r < 950) {
|
|
return 5 + rn(10);
|
|
}
|
|
return 15 + rn(2); // 15 or 16 (the 16th must be rejected).
|
|
}
|
|
|
|
// ---------------- Wuffs encoder and libbrotlienc drivers
|
|
|
|
#define OP_PROCESS 0
|
|
#define OP_FLUSH 1
|
|
#define OP_FINISH 2
|
|
#define OP_EMIT_METADATA 3
|
|
|
|
#define MAX_PARAMS 16
|
|
|
|
typedef struct {
|
|
uint32_t keys[MAX_PARAMS]; // BrotliEncoderParameter values.
|
|
uint32_t values[MAX_PARAMS];
|
|
size_t num_params;
|
|
size_t num_steps;
|
|
uint32_t* step_ops;
|
|
size_t* step_lens;
|
|
size_t steps_cap;
|
|
size_t dst_chunk; // 0 means unlimited.
|
|
bool dst_chunk_rand; // Each call gets 1 + rn(dst_chunk) bytes of dst.
|
|
uint64_t dst_seed;
|
|
uint64_t workbuf_extra; // Added to workbuf_len().min_incl (q10-11).
|
|
bool short_workbuf; // Use the shortest workbuf (q2-11, see below).
|
|
} enc_config;
|
|
|
|
static void //
|
|
enc_add_param(enc_config* c, uint32_t key, uint32_t value) {
|
|
if (c->num_params < MAX_PARAMS) {
|
|
c->keys[c->num_params] = key;
|
|
c->values[c->num_params] = value;
|
|
c->num_params++;
|
|
}
|
|
}
|
|
|
|
static uint32_t //
|
|
enc_get_param(const enc_config* c, uint32_t key, uint32_t dflt) {
|
|
uint32_t v = dflt;
|
|
for (size_t j = 0; j < c->num_params; j++) {
|
|
if (c->keys[j] == key) {
|
|
v = c->values[j];
|
|
}
|
|
}
|
|
return v;
|
|
}
|
|
|
|
static void //
|
|
enc_add_step(enc_config* c, uint32_t op, size_t len) {
|
|
if (c->num_steps == c->steps_cap) {
|
|
c->steps_cap = c->steps_cap ? (2 * c->steps_cap) : 64;
|
|
c->step_ops = realloc(c->step_ops, c->steps_cap * sizeof c->step_ops[0]);
|
|
c->step_lens =
|
|
realloc(c->step_lens, c->steps_cap * sizeof c->step_lens[0]);
|
|
if (!c->step_ops || !c->step_lens) {
|
|
fprintf(stderr, "out of memory\n");
|
|
exit(2);
|
|
}
|
|
}
|
|
c->step_ops[c->num_steps] = op;
|
|
c->step_lens[c->num_steps] = len;
|
|
c->num_steps++;
|
|
}
|
|
|
|
static uint32_t //
|
|
quirk_key(uint32_t param) {
|
|
switch (param) {
|
|
case 0:
|
|
return WUFFS_BROTLI__QUIRK_MODE;
|
|
case 1:
|
|
return WUFFS_BROTLI__QUIRK_ENCODER_QUALITY;
|
|
case 2:
|
|
return WUFFS_BROTLI__QUIRK_LGWIN;
|
|
case 3:
|
|
return WUFFS_BROTLI__QUIRK_LGBLOCK;
|
|
case 4:
|
|
return WUFFS_BROTLI__QUIRK_DISABLE_LITERAL_CONTEXT_MODELING;
|
|
case 5:
|
|
return WUFFS_BROTLI__QUIRK_SIZE_HINT;
|
|
case 6:
|
|
return WUFFS_BROTLI__QUIRK_LARGE_WINDOW;
|
|
case 7:
|
|
return WUFFS_BROTLI__QUIRK_NPOSTFIX;
|
|
case 8:
|
|
return WUFFS_BROTLI__QUIRK_NDIRECT;
|
|
case 9:
|
|
return WUFFS_BROTLI__QUIRK_STREAM_OFFSET;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
static uint8_t* g_enc_wb; // The Wuffs encoder's workbuf (big_alloc).
|
|
#define ENC_WB_CAP ((size_t)3 << 30)
|
|
static size_t g_enc_wb_touched;
|
|
static const char g_too_large[] = "workbuf_len is too large";
|
|
|
|
static size_t //
|
|
next_dst_room(const enc_config* c, uint64_t* st) {
|
|
if (c->dst_chunk == 0) {
|
|
return SIZE_MAX;
|
|
} else if (c->dst_chunk_rand) {
|
|
return 1 + (size_t)(split64(st) % c->dst_chunk);
|
|
}
|
|
return c->dst_chunk;
|
|
}
|
|
|
|
// short_workbuf_len mirrors the parity script's: the shortest workbuf (at
|
|
// qualities 2 to 11) that should work for an input of len bytes, given that
|
|
// workbuf_len().min_incl is wlen, or 0 if there is none.
|
|
static uint64_t //
|
|
short_workbuf_len(const enc_config* c, size_t len, uint64_t wlen) {
|
|
int32_t quality = (int32_t)enc_get_param(c, 1, 11);
|
|
int32_t lgwin = (int32_t)enc_get_param(c, 2, 22);
|
|
int32_t lgblock = (int32_t)enc_get_param(c, 3, 0);
|
|
bool large_window = enc_get_param(c, 6, 0) != 0;
|
|
quality = (quality < 0) ? 0 : (quality > 11) ? 11 : quality;
|
|
if (quality < 2) {
|
|
return 0;
|
|
}
|
|
large_window = large_window && (quality > 2);
|
|
int32_t max_lgwin = large_window ? 30 : 24;
|
|
lgwin = (lgwin < 10) ? 10 : (lgwin > max_lgwin) ? max_lgwin : lgwin;
|
|
if (quality < 4) {
|
|
lgblock = 14;
|
|
} else if (lgblock == 0) {
|
|
lgblock = 16;
|
|
if ((quality >= 9) && (lgwin > lgblock)) {
|
|
lgblock = (lgwin < 18) ? lgwin : 18;
|
|
}
|
|
} else {
|
|
lgblock = (lgblock < 16) ? 16 : (lgblock > 24) ? 24 : lgblock;
|
|
}
|
|
int32_t rb_bits = 1 + ((lgwin > lgblock) ? lgwin : lgblock);
|
|
uint64_t rb_size = (uint64_t)1 << rb_bits;
|
|
uint64_t rb_region = 2 + rb_size + ((uint64_t)1 << lgblock) + 7;
|
|
if (((len + 16) > rb_size) || (rb_region > wlen)) {
|
|
return 0;
|
|
}
|
|
uint64_t n = wlen - rb_region + 2 + len + 16;
|
|
return (n < wlen) ? n : 0;
|
|
}
|
|
|
|
// encode_wuffs runs the plan with Wuffs' encoder. *wlen_out is the workbuf
|
|
// length used. short_delta (0 or 1) is subtracted from the short workbuf
|
|
// length.
|
|
static const char* //
|
|
encode_wuffs(const enc_config* c, const dict_set* dicts, const uint8_t* src_ptr,
|
|
size_t src_len, size_t input_len, uint8_t* out, size_t out_cap,
|
|
size_t* out_len, uint64_t short_delta, uint64_t* wlen_out) {
|
|
*out_len = 0;
|
|
wuffs_brotli__encoder* enc = wuffs_brotli__encoder__alloc();
|
|
if (!enc) {
|
|
return "alloc failed";
|
|
}
|
|
const char* ret = NULL;
|
|
for (size_t j = 0; j < dicts->n; j++) {
|
|
uint64_t n = wuffs_brotli__encoder__add_dictionary_workbuf_len(
|
|
enc, dicts->d[j].len);
|
|
if (n > ENC_WB_CAP) {
|
|
ret = g_too_large;
|
|
goto cleanup;
|
|
}
|
|
if (n > g_enc_wb_touched) {
|
|
g_enc_wb_touched = n;
|
|
}
|
|
wuffs_base__status status = wuffs_brotli__encoder__add_dictionary(
|
|
enc, dicts->d[j].type,
|
|
wuffs_base__make_slice_u8((uint8_t*)dicts->d[j].ptr, dicts->d[j].len),
|
|
wuffs_base__make_slice_u8(g_enc_wb, (size_t)n));
|
|
if (status.repr) {
|
|
ret = status.repr;
|
|
goto cleanup;
|
|
}
|
|
}
|
|
for (size_t j = 0; j < c->num_params; j++) {
|
|
wuffs_base__status status = wuffs_brotli__encoder__set_quirk(
|
|
enc, quirk_key(c->keys[j]), c->values[j]);
|
|
if (status.repr) {
|
|
ret = status.repr;
|
|
goto cleanup;
|
|
}
|
|
}
|
|
wuffs_base__range_ii_u64 r = wuffs_brotli__encoder__workbuf_len(enc);
|
|
uint64_t wlen = r.min_incl;
|
|
if (c->workbuf_extra && (r.max_incl > r.min_incl)) {
|
|
uint64_t room = r.max_incl - r.min_incl;
|
|
wlen += (c->workbuf_extra < room) ? c->workbuf_extra : room;
|
|
}
|
|
if (c->short_workbuf) {
|
|
uint64_t s = short_workbuf_len(c, input_len, r.min_incl);
|
|
if (s > 0) {
|
|
wlen = s - short_delta;
|
|
} else if (short_delta) {
|
|
ret = "no short workbuf";
|
|
goto cleanup;
|
|
}
|
|
}
|
|
if ((wlen > ENC_WB_CAP) && !c->short_workbuf && (r.min_incl <= ENC_WB_CAP)) {
|
|
wlen = ENC_WB_CAP;
|
|
}
|
|
if (wlen > ENC_WB_CAP) {
|
|
ret = g_too_large;
|
|
goto cleanup;
|
|
}
|
|
*wlen_out = wlen;
|
|
if (wlen > g_enc_wb_touched) {
|
|
g_enc_wb_touched = wlen;
|
|
}
|
|
wuffs_base__slice_u8 workbuf =
|
|
wuffs_base__make_slice_u8(g_enc_wb, (size_t)wlen);
|
|
|
|
wuffs_base__io_buffer dst = wuffs_base__ptr_u8__writer(out, out_cap);
|
|
wuffs_base__io_buffer src =
|
|
wuffs_base__ptr_u8__reader((uint8_t*)src_ptr, src_len, false);
|
|
src.meta.wi = 0;
|
|
uint64_t dst_st = c->dst_seed;
|
|
|
|
size_t end = 0;
|
|
for (size_t i = 0; i < c->num_steps; i++) {
|
|
end += c->step_lens[i];
|
|
uint32_t op = c->step_ops[i];
|
|
wuffs_base__status status = wuffs_base__make_status(NULL);
|
|
if (op == OP_FLUSH) {
|
|
status = wuffs_brotli__encoder__request_flush(enc);
|
|
} else if (op == OP_EMIT_METADATA) {
|
|
status = wuffs_brotli__encoder__request_emit_metadata(enc,
|
|
c->step_lens[i]);
|
|
}
|
|
if (status.repr) {
|
|
ret = status.repr;
|
|
goto cleanup;
|
|
}
|
|
uint64_t calls = 0;
|
|
while (true) {
|
|
if (++calls > (out_cap + src_len + 1000)) {
|
|
ret = "too many transform_io calls";
|
|
goto cleanup;
|
|
}
|
|
wuffs_base__io_buffer d = dst;
|
|
size_t room = next_dst_room(c, &dst_st);
|
|
if ((d.data.len - d.meta.wi) > room) {
|
|
d.data.len = d.meta.wi + room;
|
|
}
|
|
wuffs_base__io_buffer s = src;
|
|
s.meta.wi = end;
|
|
s.meta.closed = (op == OP_FINISH);
|
|
status = wuffs_brotli__encoder__transform_io(enc, &d, &s, workbuf);
|
|
dst.meta.wi = d.meta.wi;
|
|
src.meta.ri = s.meta.ri;
|
|
if (status.repr == NULL) {
|
|
if (op != OP_FINISH) {
|
|
ret = "unexpected ok";
|
|
goto cleanup;
|
|
}
|
|
break;
|
|
} else if (status.repr == wuffs_base__suspension__short_write) {
|
|
if (dst.meta.wi == dst.data.len) {
|
|
ret = "dst is full";
|
|
goto cleanup;
|
|
}
|
|
continue;
|
|
} else if ((status.repr == wuffs_base__suspension__short_read) &&
|
|
(op != OP_FINISH) && (src.meta.ri == end)) {
|
|
break;
|
|
}
|
|
ret = status.repr;
|
|
goto cleanup;
|
|
}
|
|
}
|
|
|
|
cleanup:
|
|
*out_len = dst.meta.wi;
|
|
free(enc);
|
|
return ret;
|
|
}
|
|
|
|
// encode_c runs the plan with libbrotlienc.
|
|
// g_c_state_corrupted is encode_c's error when libbrotlienc's
|
|
// BrotliEncoderCompressStream fails because of a C bug: in ProcessMetadata,
|
|
// when dst has no room, the metadata body goes through tiny_buf_ (16 bytes)
|
|
// and pushing it out leaves next_out_ pointing just past tiny_buf_, at
|
|
// remaining_metadata_bytes_. Back in the processing state, qualities 0 and 1
|
|
// compress in place (into the caller's dst) without resetting next_out_, so
|
|
// a later flush's InjectBytePaddingBlock writes the padding bits through the
|
|
// stale pointer, over remaining_metadata_bytes_. Its output is still right,
|
|
// but the next BrotliEncoderCompressStream call sees a metadata block in
|
|
// progress and fails. A C build that sets next_out_ to NULL when the
|
|
// metadata workflow ends produces the same bytes as Wuffs. The corruption
|
|
// shows as the processing state with remaining_metadata_bytes_ not
|
|
// BROTLI_UINT32_MAX, which never happens otherwise.
|
|
static const char g_c_state_corrupted[] =
|
|
"C state corrupted (stale next_out_ after metadata)";
|
|
|
|
static bool //
|
|
c_state_corrupted(BrotliEncoderState* state) {
|
|
return (state->stream_state_ == BROTLI_STREAM_PROCESSING) &&
|
|
(state->remaining_metadata_bytes_ != 0xFFFFFFFFu);
|
|
}
|
|
|
|
// g_c_zero_alloc makes encode_c give libbrotlienc zeroed memory (see
|
|
// g_enc_uninit_tail).
|
|
static bool g_c_zero_alloc;
|
|
|
|
static void* //
|
|
zero_alloc(void* opaque, size_t size) {
|
|
(void)opaque;
|
|
return calloc(1, size);
|
|
}
|
|
|
|
static void //
|
|
zero_free(void* opaque, void* address) {
|
|
(void)opaque;
|
|
free(address);
|
|
}
|
|
|
|
static const char* //
|
|
encode_c(const enc_config* c, const dict_set* dicts, const uint8_t* src_ptr,
|
|
uint8_t* out, size_t out_cap, size_t* out_len) {
|
|
*out_len = 0;
|
|
BrotliEncoderPreparedDictionary* prepared[MAX_DICTS] = {0};
|
|
size_t num_prepared = 0;
|
|
BrotliEncoderState* state =
|
|
g_c_zero_alloc ? BrotliEncoderCreateInstance(zero_alloc, zero_free, NULL)
|
|
: BrotliEncoderCreateInstance(NULL, NULL, NULL);
|
|
if (!state) {
|
|
return "BrotliEncoderCreateInstance failed";
|
|
}
|
|
const char* ret = NULL;
|
|
for (size_t j = 0; j < c->num_params; j++) {
|
|
if (!BrotliEncoderSetParameter(state, (BrotliEncoderParameter)(c->keys[j]),
|
|
c->values[j])) {
|
|
ret = "BrotliEncoderSetParameter failed";
|
|
goto cleanup;
|
|
}
|
|
}
|
|
for (size_t j = 0; j < dicts->n; j++) {
|
|
prepared[j] = BrotliEncoderPrepareDictionary(
|
|
(BrotliSharedDictionaryType)dicts->d[j].type, dicts->d[j].len,
|
|
dicts->d[j].ptr, 11, NULL, NULL, NULL);
|
|
if (!prepared[j]) {
|
|
ret = "BrotliEncoderPrepareDictionary failed";
|
|
goto cleanup;
|
|
}
|
|
num_prepared++;
|
|
if (!BrotliEncoderAttachPreparedDictionary(state, prepared[j])) {
|
|
ret = "BrotliEncoderAttachPreparedDictionary failed";
|
|
goto cleanup;
|
|
}
|
|
}
|
|
const uint8_t* next_in = src_ptr;
|
|
uint64_t dst_st = c->dst_seed;
|
|
for (size_t i = 0; i < c->num_steps; i++) {
|
|
size_t avail_in = c->step_lens[i];
|
|
BrotliEncoderOperation op = (BrotliEncoderOperation)(c->step_ops[i]);
|
|
uint64_t calls = 0;
|
|
while (true) {
|
|
if (++calls > (out_cap + c->step_lens[i] + 1000)) {
|
|
ret = "too many BrotliEncoderCompressStream calls";
|
|
goto cleanup;
|
|
}
|
|
size_t avail_out = out_cap - *out_len;
|
|
size_t room = next_dst_room(c, &dst_st);
|
|
if (avail_out > room) {
|
|
avail_out = room;
|
|
}
|
|
size_t avail_out0 = avail_out;
|
|
uint8_t* next_out = out + *out_len;
|
|
if (!BrotliEncoderCompressStream(state, op, &avail_in, &next_in,
|
|
&avail_out, &next_out, NULL)) {
|
|
ret = c_state_corrupted(state) ? g_c_state_corrupted
|
|
: "BrotliEncoderCompressStream failed";
|
|
goto cleanup;
|
|
}
|
|
*out_len += avail_out0 - avail_out;
|
|
if (op == BROTLI_OPERATION_FINISH) {
|
|
if (BrotliEncoderIsFinished(state)) {
|
|
break;
|
|
}
|
|
} else if ((avail_in == 0) && !BrotliEncoderHasMoreOutput(state)) {
|
|
break;
|
|
}
|
|
if (*out_len == out_cap) {
|
|
ret = "dst is full";
|
|
goto cleanup;
|
|
}
|
|
}
|
|
}
|
|
|
|
cleanup:
|
|
BrotliEncoderDestroyInstance(state);
|
|
for (size_t j = 0; j < num_prepared; j++) {
|
|
BrotliEncoderDestroyPreparedDictionary(prepared[j]);
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
// ---------------- Decoder drivers
|
|
|
|
// Result classes.
|
|
#define CLS_OK 1
|
|
#define CLS_NEEDS_MORE_INPUT 2
|
|
#define CLS_NEEDS_MORE_OUTPUT 3
|
|
#define CLS_BUG (-1000)
|
|
// Error classes are the (negative) BrotliDecoderErrorCode values.
|
|
|
|
typedef struct {
|
|
int cls;
|
|
size_t out_n;
|
|
size_t in_used;
|
|
const char* wuffs_status;
|
|
uint64_t calls;
|
|
size_t md_n; // With report_md: the metadata bytes (see g_wmd and g_cmd).
|
|
} dec_result;
|
|
|
|
// With dec_config.report_md, the Wuffs decoder (QUIRK_REPORT_METADATA) and
|
|
// libbrotlidec (BrotliDecoderSetMetadataCallbacks) collect each metadata
|
|
// block (its length as 4 little-endian bytes, then its contents) in g_wmd and
|
|
// g_cmd. A block's contents can be up to 16 MiB long.
|
|
#define MD_CAP ((size_t)40 << 20)
|
|
#define MAX_OUT (((size_t)2 << 20) + 4096)
|
|
static uint8_t* g_wmd;
|
|
static uint8_t* g_cmd;
|
|
static size_t g_cmd_n;
|
|
static bool g_cmd_overflow;
|
|
|
|
static void //
|
|
c_md_start(void* opaque, size_t size) {
|
|
if (MD_CAP - g_cmd_n < 4) {
|
|
g_cmd_overflow = true;
|
|
return;
|
|
}
|
|
for (int i = 0; i < 4; i++) {
|
|
g_cmd[g_cmd_n++] = (uint8_t)(size >> (8 * i));
|
|
}
|
|
}
|
|
|
|
static void //
|
|
c_md_chunk(void* opaque, const uint8_t* data, size_t size) {
|
|
if (MD_CAP - g_cmd_n < size) {
|
|
g_cmd_overflow = true;
|
|
return;
|
|
}
|
|
memcpy(g_cmd + g_cmd_n, data, size);
|
|
g_cmd_n += size;
|
|
}
|
|
|
|
static int //
|
|
wuffs_status_class(const char* s) {
|
|
static const struct {
|
|
const char* status;
|
|
int code;
|
|
} table[] = {
|
|
{wuffs_brotli__error__bad_exuberant_nibble, -1},
|
|
{wuffs_brotli__error__bad_reserved_bit, -2},
|
|
{wuffs_brotli__error__bad_exuberant_metadata_nibble, -3},
|
|
{wuffs_brotli__error__bad_simple_huffman_alphabet, -4},
|
|
{wuffs_brotli__error__bad_simple_huffman_symbol, -5},
|
|
{wuffs_brotli__error__bad_code_length_code_lengths, -6},
|
|
{wuffs_brotli__error__bad_huffman_space, -7},
|
|
{wuffs_brotli__error__bad_context_map_repeat, -8},
|
|
{wuffs_brotli__error__bad_block_length, -9},
|
|
{wuffs_brotli__error__bad_metablock_length, -10},
|
|
{wuffs_brotli__error__bad_transform, -11},
|
|
{wuffs_brotli__error__bad_dictionary, -12},
|
|
{wuffs_brotli__error__bad_window_bits, -13},
|
|
{wuffs_brotli__error__unsupported_large_window, -13},
|
|
{wuffs_brotli__error__bad_padding, -14},
|
|
{wuffs_brotli__error__bad_final_padding, -15},
|
|
{wuffs_brotli__error__bad_distance, -16},
|
|
{wuffs_brotli__error__bad_block_switch, -17},
|
|
{wuffs_brotli__error__bad_compound_dictionary, -18},
|
|
{wuffs_brotli__error__truncated_input, CLS_NEEDS_MORE_INPUT},
|
|
};
|
|
if (!s) {
|
|
return CLS_OK;
|
|
}
|
|
for (size_t i = 0; i < sizeof table / sizeof table[0]; i++) {
|
|
if (table[i].status == s) {
|
|
return table[i].code;
|
|
}
|
|
}
|
|
return CLS_BUG;
|
|
}
|
|
|
|
// A chunking schedule: when a decoder runs out of input (or output room),
|
|
// the next chunk boundary is the previous one plus next_chunk. Both decoders
|
|
// use the same sequence of boundaries.
|
|
typedef struct {
|
|
bool one_shot;
|
|
uint64_t in_seed;
|
|
uint64_t out_seed;
|
|
int in_style;
|
|
int out_style;
|
|
} chunking;
|
|
|
|
static size_t //
|
|
next_chunk(uint64_t* st, int style) {
|
|
uint64_t r = split64(st);
|
|
switch (style) {
|
|
case 0: // 1 to 7 bytes.
|
|
return 1 + (size_t)((r >> 8) % 7);
|
|
case 1: // Exactly 1 byte.
|
|
return 1;
|
|
case 2: // Mostly small, sometimes large.
|
|
if ((r & 0xFF) < 0xE0) {
|
|
return 1 + (size_t)((r >> 8) % 7);
|
|
}
|
|
return 1 + (size_t)((r >> 8) % 70000);
|
|
default: // 1 to 300 bytes.
|
|
return 1 + (size_t)((r >> 8) % 300);
|
|
}
|
|
}
|
|
|
|
typedef struct {
|
|
bool large_window;
|
|
dict_set dicts; // The dictionaries attached to both decoders.
|
|
uint64_t wb_len; // The Wuffs workbuf length.
|
|
bool expect_bad_wb; // wb_len is one byte too short for the stream.
|
|
bool uninit; // WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED.
|
|
bool report_md; // QUIRK_REPORT_METADATA and metadata callbacks.
|
|
size_t out_cap;
|
|
} dec_config;
|
|
|
|
static uint8_t* g_dec_wb; // big_alloc.
|
|
static bool g_trace = false;
|
|
#define DEC_WB_CAP (((size_t)1 << 30) + ((size_t)64 << 20))
|
|
static wuffs_brotli__decoder* g_dec;
|
|
|
|
static const char* //
|
|
wuffs_attach(wuffs_brotli__decoder* dec, const dec_config* dc, size_t* n_ok) {
|
|
wuffs_base__status status = wuffs_brotli__decoder__initialize(
|
|
dec, sizeof__wuffs_brotli__decoder(), WUFFS_VERSION,
|
|
dc->uninit ? WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED : 0);
|
|
if (status.repr) {
|
|
return status.repr;
|
|
}
|
|
if (dc->large_window) {
|
|
status = wuffs_brotli__decoder__set_quirk(
|
|
dec, WUFFS_BROTLI__QUIRK_LARGE_WINDOW, 1);
|
|
if (status.repr) {
|
|
return status.repr;
|
|
}
|
|
}
|
|
if (dc->report_md) {
|
|
status = wuffs_brotli__decoder__set_quirk(
|
|
dec, WUFFS_BROTLI__QUIRK_REPORT_METADATA, 1);
|
|
if (status.repr) {
|
|
return status.repr;
|
|
}
|
|
}
|
|
*n_ok = 0;
|
|
for (size_t i = 0; i < dc->dicts.n; i++) {
|
|
status = wuffs_brotli__decoder__add_dictionary(
|
|
dec, dc->dicts.d[i].type,
|
|
wuffs_base__make_slice_u8((uint8_t*)dc->dicts.d[i].ptr,
|
|
dc->dicts.d[i].len),
|
|
wuffs_base__make_slice_u8(g_dec_wb, DEC_WB_CAP));
|
|
if (status.repr) {
|
|
return status.repr;
|
|
}
|
|
(*n_ok)++;
|
|
}
|
|
return NULL;
|
|
}
|
|
|
|
static dec_result //
|
|
wuffs_decode(const dec_config* dc,
|
|
const chunking* ch,
|
|
const uint8_t* in,
|
|
size_t n,
|
|
uint8_t* out) {
|
|
dec_result res = {0};
|
|
size_t n_ok = 0;
|
|
const char* err = wuffs_attach(g_dec, dc, &n_ok);
|
|
if (err) {
|
|
res.cls = CLS_BUG;
|
|
res.wuffs_status = err;
|
|
return res;
|
|
}
|
|
wuffs_base__slice_u8 wb =
|
|
wuffs_base__make_slice_u8(g_dec_wb, (size_t)dc->wb_len);
|
|
uint64_t in_st = ch->in_seed;
|
|
uint64_t out_st = ch->out_seed;
|
|
size_t lin = ch->one_shot ? n : next_chunk(&in_st, ch->in_style);
|
|
size_t lout = ch->one_shot ? dc->out_cap : next_chunk(&out_st, ch->out_style);
|
|
lin = (lin < n) ? lin : n;
|
|
lout = (lout < dc->out_cap) ? lout : dc->out_cap;
|
|
size_t ri = 0;
|
|
size_t wi = 0;
|
|
uint64_t max_calls = 3 * ((uint64_t)n + dc->out_cap) + 100;
|
|
// With report_md, metadata bytes are written to dst (out) and then moved to
|
|
// g_wmd. While md_remaining > 0, dst gets up to md_remaining bytes of slack
|
|
// past lout (within the physical buffer's MAX_OUT bytes), which the
|
|
// metadata bytes fill before any decoded bytes, so that the decoded bytes
|
|
// still fit in lout (like libbrotlidec's, which uses no dst for metadata).
|
|
size_t md_remaining = 0;
|
|
while (true) {
|
|
if (++res.calls > max_calls) {
|
|
res.cls = CLS_BUG;
|
|
res.wuffs_status = "too many transform_io calls (a loop?)";
|
|
break;
|
|
}
|
|
wuffs_base__io_buffer src =
|
|
wuffs_base__ptr_u8__reader((uint8_t*)in, lin, lin == n);
|
|
src.meta.ri = ri;
|
|
size_t slack = 0;
|
|
if ((md_remaining > 0) && (lout < MAX_OUT)) {
|
|
slack = (md_remaining < (MAX_OUT - lout)) ? md_remaining
|
|
: (MAX_OUT - lout);
|
|
}
|
|
wuffs_base__io_buffer dst = wuffs_base__ptr_u8__writer(out, lout + slack);
|
|
dst.meta.wi = wi;
|
|
wuffs_base__status status =
|
|
wuffs_brotli__decoder__transform_io(g_dec, &dst, &src, wb);
|
|
if ((src.meta.ri < ri) || (dst.meta.wi < wi)) {
|
|
res.cls = CLS_BUG;
|
|
res.wuffs_status = "went backwards";
|
|
break;
|
|
}
|
|
if (md_remaining > 0) {
|
|
size_t k = dst.meta.wi - wi;
|
|
k = (k < md_remaining) ? k : md_remaining;
|
|
memcpy(g_wmd + res.md_n, out + wi, k);
|
|
res.md_n += k;
|
|
md_remaining -= k;
|
|
memmove(out + wi, out + wi + k, dst.meta.wi - wi - k);
|
|
dst.meta.wi -= k;
|
|
}
|
|
if (dst.meta.wi > lout) {
|
|
res.cls = CLS_BUG;
|
|
res.wuffs_status = "decoded bytes in the metadata slack";
|
|
break;
|
|
}
|
|
ri = src.meta.ri;
|
|
wi = dst.meta.wi;
|
|
res.wuffs_status = status.repr;
|
|
if (status.repr == wuffs_base__note__metadata_reported) {
|
|
uint64_t len = wuffs_brotli__decoder__metadata_length(g_dec);
|
|
if (!dc->report_md || (md_remaining > 0) || (len > (1 << 24)) ||
|
|
(MD_CAP - res.md_n < 4 + len)) {
|
|
res.cls = CLS_BUG;
|
|
res.wuffs_status = "unexpected metadata report";
|
|
break;
|
|
}
|
|
for (int i = 0; i < 4; i++) {
|
|
g_wmd[res.md_n++] = (uint8_t)(len >> (8 * i));
|
|
}
|
|
md_remaining = (size_t)len;
|
|
continue;
|
|
}
|
|
if (g_trace) {
|
|
printf("wuffs call %" PRIu64 ": lin=%zu lout=%zu -> ri=%zu wi=%zu %s\n",
|
|
res.calls, lin, lout, ri, wi, status.repr ? status.repr : "ok");
|
|
}
|
|
if (status.repr == wuffs_base__suspension__short_read) {
|
|
if (lin < n) {
|
|
lin += next_chunk(&in_st, ch->in_style);
|
|
lin = (lin < n) ? lin : n;
|
|
continue;
|
|
}
|
|
res.cls = CLS_BUG; // A short read with a closed src.
|
|
break;
|
|
} else if (status.repr == wuffs_base__suspension__short_write) {
|
|
if (lout < dc->out_cap) {
|
|
lout += next_chunk(&out_st, ch->out_style);
|
|
lout = (lout < dc->out_cap) ? lout : dc->out_cap;
|
|
continue;
|
|
}
|
|
res.cls = CLS_NEEDS_MORE_OUTPUT;
|
|
break;
|
|
}
|
|
if (status.repr == wuffs_base__error__bad_workbuf_length) {
|
|
res.cls = -100;
|
|
} else {
|
|
res.cls = wuffs_status_class(status.repr);
|
|
}
|
|
break;
|
|
}
|
|
res.out_n = wi;
|
|
res.in_used = ri;
|
|
return res;
|
|
}
|
|
|
|
static dec_result //
|
|
c_decode(const dec_config* dc,
|
|
const chunking* ch,
|
|
const uint8_t* in,
|
|
size_t n,
|
|
uint8_t* out,
|
|
size_t* c_attached) {
|
|
dec_result res = {0};
|
|
BrotliDecoderState* s = BrotliDecoderCreateInstance(NULL, NULL, NULL);
|
|
if (!s) {
|
|
res.cls = CLS_BUG;
|
|
return res;
|
|
}
|
|
if (dc->large_window) {
|
|
BrotliDecoderSetParameter(s, BROTLI_DECODER_PARAM_LARGE_WINDOW, 1);
|
|
}
|
|
g_cmd_n = 0;
|
|
g_cmd_overflow = false;
|
|
if (dc->report_md) {
|
|
BrotliDecoderSetMetadataCallbacks(s, c_md_start, c_md_chunk, NULL);
|
|
}
|
|
*c_attached = 0;
|
|
for (size_t i = 0; i < dc->dicts.n; i++) {
|
|
if (!BrotliDecoderAttachDictionary(
|
|
s, (BrotliSharedDictionaryType)dc->dicts.d[i].type,
|
|
dc->dicts.d[i].len, dc->dicts.d[i].ptr)) {
|
|
break;
|
|
}
|
|
(*c_attached)++;
|
|
}
|
|
if (*c_attached != dc->dicts.n) {
|
|
BrotliDecoderDestroyInstance(s);
|
|
res.cls = CLS_BUG;
|
|
return res;
|
|
}
|
|
uint64_t in_st = ch->in_seed;
|
|
uint64_t out_st = ch->out_seed;
|
|
size_t lin = ch->one_shot ? n : next_chunk(&in_st, ch->in_style);
|
|
size_t lout = ch->one_shot ? dc->out_cap : next_chunk(&out_st, ch->out_style);
|
|
lin = (lin < n) ? lin : n;
|
|
lout = (lout < dc->out_cap) ? lout : dc->out_cap;
|
|
size_t ri = 0;
|
|
size_t wi = 0;
|
|
while (true) {
|
|
res.calls++;
|
|
size_t avail_in = lin - ri;
|
|
const uint8_t* next_in = in + ri;
|
|
size_t avail_out = lout - wi;
|
|
uint8_t* next_out = out + wi;
|
|
BrotliDecoderResult r = BrotliDecoderDecompressStream(
|
|
s, &avail_in, &next_in, &avail_out, &next_out, NULL);
|
|
ri = lin - avail_in;
|
|
wi = lout - avail_out;
|
|
if (g_trace) {
|
|
printf("c call %" PRIu64 ": lin=%zu lout=%zu -> ri=%zu wi=%zu r=%d\n",
|
|
res.calls, lin, lout, ri, wi, (int)r);
|
|
}
|
|
if (r == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT) {
|
|
if (lin < n) {
|
|
lin += next_chunk(&in_st, ch->in_style);
|
|
lin = (lin < n) ? lin : n;
|
|
continue;
|
|
}
|
|
res.cls = CLS_NEEDS_MORE_INPUT;
|
|
} else if (r == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) {
|
|
if (lout < dc->out_cap) {
|
|
lout += next_chunk(&out_st, ch->out_style);
|
|
lout = (lout < dc->out_cap) ? lout : dc->out_cap;
|
|
continue;
|
|
}
|
|
res.cls = CLS_NEEDS_MORE_OUTPUT;
|
|
} else if (r == BROTLI_DECODER_RESULT_SUCCESS) {
|
|
res.cls = CLS_OK;
|
|
} else {
|
|
res.cls = (int)BrotliDecoderGetErrorCode(s);
|
|
}
|
|
break;
|
|
}
|
|
BrotliDecoderDestroyInstance(s);
|
|
res.out_n = wi;
|
|
res.in_used = ri;
|
|
res.md_n = g_cmd_overflow ? SIZE_MAX : g_cmd_n;
|
|
return res;
|
|
}
|
|
|
|
// stream_window_bits parses a brotli stream header's WBITS, returning 0 if
|
|
// it is invalid or truncated. It mirrors DecodeWindowBits in decode.c.
|
|
static uint32_t //
|
|
stream_window_bits(const uint8_t* p, size_t n, bool large_window) {
|
|
uint64_t bits = 0;
|
|
for (size_t i = 0; (i < 8) && (i < n); i++) {
|
|
bits |= ((uint64_t)p[i]) << (8 * i);
|
|
}
|
|
size_t nbits = (n < 8) ? (8 * n) : 64;
|
|
if (nbits < 1) {
|
|
return 0;
|
|
} else if ((bits & 1) == 0) {
|
|
return 16;
|
|
} else if (nbits < 4) {
|
|
return 0;
|
|
}
|
|
uint32_t k = (uint32_t)((bits >> 1) & 7);
|
|
if (k != 0) {
|
|
return 17 + k;
|
|
} else if (nbits < 7) {
|
|
return 0;
|
|
}
|
|
k = (uint32_t)((bits >> 4) & 7);
|
|
if (k == 1) {
|
|
if (!large_window || (nbits < 14) || ((bits >> 7) & 1)) {
|
|
return 0;
|
|
}
|
|
k = (uint32_t)((bits >> 8) & 63);
|
|
return ((k < 10) || (k > 30)) ? 0 : k;
|
|
} else if (k != 0) {
|
|
return 8 + k;
|
|
}
|
|
return 17;
|
|
}
|
|
|
|
// ---------------- Failure reporting
|
|
|
|
static void //
|
|
save_failure(const char* kind,
|
|
const char* desc,
|
|
const uint8_t* stream,
|
|
size_t stream_len,
|
|
const dict_set* dicts) {
|
|
g_num_failures++;
|
|
char path[1024];
|
|
snprintf(path, sizeof path, "%s/fail-%s-%" PRIu64 "-%d.br", g_out_dir, kind,
|
|
g_rng_state, (int)g_num_failures);
|
|
write_file(path, stream, stream_len);
|
|
char dpath[1100];
|
|
for (size_t i = 0; dicts && (i < dicts->n); i++) {
|
|
snprintf(dpath, sizeof dpath, "%s.dict%zu.%s", path, i,
|
|
dicts->d[i].type ? "sd" : "raw");
|
|
write_file(dpath, dicts->d[i].ptr, dicts->d[i].len);
|
|
}
|
|
snprintf(dpath, sizeof dpath, "%s.txt", path);
|
|
write_file(dpath, (const uint8_t*)desc, strlen(desc));
|
|
printf("FAILURE (%s) %s\n saved %s\n", kind, desc, path);
|
|
fflush(stdout);
|
|
}
|
|
|
|
// ---------------- Decoder campaign
|
|
|
|
typedef struct {
|
|
char name[256];
|
|
uint8_t* ptr;
|
|
size_t len;
|
|
dict_set dicts;
|
|
bool large_window;
|
|
} seed;
|
|
|
|
#define MAX_FILE_SEEDS 256
|
|
#define MAX_GEN_SEEDS 256
|
|
static seed g_file_seeds[MAX_FILE_SEEDS];
|
|
static size_t g_num_file_seeds;
|
|
static seed g_gen_seeds[MAX_GEN_SEEDS];
|
|
static size_t g_num_gen_seeds;
|
|
|
|
static uint64_t g_dec_inputs;
|
|
static uint64_t g_dec_decodes;
|
|
static uint64_t g_dec_cls_counts[2][32]; // [one-shot, chunked][class].
|
|
static uint64_t g_dec_bad_wb_checks;
|
|
static uint64_t g_dec_exact_wb;
|
|
static uint64_t g_dec_dict_rejections;
|
|
static uint64_t g_dec_lw;
|
|
static uint64_t g_dec_with_dicts;
|
|
static uint64_t g_dec_gen_streams;
|
|
static uint64_t g_dec_gen_wuffs;
|
|
static uint64_t g_dec_wuffs_shorter_output;
|
|
static uint64_t g_dec_known_block_length;
|
|
static uint64_t g_dec_known_saved;
|
|
static uint64_t g_dec_report_md;
|
|
static uint64_t g_dec_md_compared;
|
|
static uint64_t g_dec_md_blocks;
|
|
static uint64_t g_dec_md_inconclusive;
|
|
|
|
static int //
|
|
cls_index(int cls) {
|
|
if (cls >= 1 && cls <= 3) {
|
|
return cls;
|
|
} else if (cls <= -1 && cls >= -18) {
|
|
return 3 - cls; // 4 ..= 21.
|
|
} else if (cls == -100) {
|
|
return 22;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
static void //
|
|
load_file_seeds(void) {
|
|
char dir[512];
|
|
snprintf(dir, sizeof dir, "%s/brotli", g_data_dir);
|
|
char* names[1024];
|
|
size_t n = list_dir(dir, names, 1024);
|
|
for (size_t i = 0; i < n; i++) {
|
|
size_t k = strlen(names[i]);
|
|
bool want = ((k > 3) && !strcmp(names[i] + k - 3, ".br")) ||
|
|
strstr(names[i], ".compressed");
|
|
if (want && (g_num_file_seeds < MAX_FILE_SEEDS)) {
|
|
char path[1024];
|
|
snprintf(path, sizeof path, "%s/%s", dir, names[i]);
|
|
size_t len = 0;
|
|
uint8_t* p = read_file(path, &len, 16 << 20);
|
|
if (p) {
|
|
seed* s = &g_file_seeds[g_num_file_seeds++];
|
|
snprintf(s->name, sizeof s->name, "%s", names[i]);
|
|
s->ptr = p;
|
|
s->len = len;
|
|
named_dict_set(names[i], &s->dicts, &s->large_window);
|
|
}
|
|
}
|
|
free(names[i]);
|
|
}
|
|
}
|
|
|
|
// make_gen_seed compresses a random input with random parameters (with
|
|
// libbrotlienc or, sometimes, Wuffs' encoder) and checks that libbrotlidec
|
|
// decodes it.
|
|
static void //
|
|
make_gen_seed(seed* s, uint8_t* tmp_in, uint8_t* tmp_out, size_t tmp_cap) {
|
|
free(s->ptr);
|
|
dict_set_free(&s->dicts);
|
|
size_t len = pick_len(256 << 10);
|
|
gen_input(tmp_in, len);
|
|
|
|
enc_config c;
|
|
memset(&c, 0, sizeof c);
|
|
uint32_t q = (uint32_t)rn(12);
|
|
if ((q >= 10) && (len > (64 << 10))) {
|
|
q = (uint32_t)rn(10);
|
|
}
|
|
bool lw = (q >= 3) && chance(300);
|
|
uint32_t lgwin = 10 + (uint32_t)rn(lw ? 21 : 15);
|
|
enc_add_param(&c, 1, q);
|
|
enc_add_param(&c, 2, lgwin);
|
|
if (lw) {
|
|
enc_add_param(&c, 6, 1);
|
|
}
|
|
if (chance(300)) {
|
|
enc_add_param(&c, 7, (uint32_t)rn(4));
|
|
enc_add_param(&c, 8, (uint32_t)rn(16) << (uint32_t)rn(4));
|
|
}
|
|
dict_set ds = {0};
|
|
if ((q < 10) && chance(400)) {
|
|
random_raw_dicts(&ds, 1 + rn(chance(100) ? 15 : 3), tmp_in, len);
|
|
}
|
|
// A plan: random PROCESS chunks with occasional flushes and metadata.
|
|
size_t meta_total = 0;
|
|
size_t i = 0;
|
|
while (true) {
|
|
if (chance(30)) {
|
|
size_t m = rn(chance(100) ? 70000 : 64);
|
|
if ((len + meta_total + m) > tmp_cap) {
|
|
m = 0;
|
|
}
|
|
enc_add_step(&c, OP_EMIT_METADATA, m);
|
|
memmove(tmp_in + i + m, tmp_in + i, len + meta_total - i);
|
|
for (size_t j = 0; j < m; j++) {
|
|
tmp_in[i + j] = (uint8_t)r64();
|
|
}
|
|
i += m;
|
|
meta_total += m;
|
|
}
|
|
size_t k = chance(300) ? (len + meta_total - i) : (1 + rn(5000));
|
|
if (k > (len + meta_total - i)) {
|
|
k = len + meta_total - i;
|
|
}
|
|
i += k;
|
|
if (i == (len + meta_total)) {
|
|
enc_add_step(&c, OP_FINISH, k);
|
|
break;
|
|
}
|
|
enc_add_step(&c, chance(100) ? OP_FLUSH : OP_PROCESS, k);
|
|
}
|
|
|
|
size_t out_len = 0;
|
|
const char* err;
|
|
bool use_wuffs = (q < 10) && chance(250);
|
|
if (use_wuffs) {
|
|
uint64_t wlen = 0;
|
|
err = encode_wuffs(&c, &ds, tmp_in, len + meta_total, len, tmp_out,
|
|
tmp_cap, &out_len, 0, &wlen);
|
|
g_dec_gen_wuffs++;
|
|
} else {
|
|
err = encode_c(&c, &ds, tmp_in, tmp_out, tmp_cap, &out_len);
|
|
}
|
|
free(c.step_ops);
|
|
free(c.step_lens);
|
|
if (err) {
|
|
fprintf(stderr, "make_gen_seed: %s encoder: %s\n", use_wuffs ? "Wuffs" : "C",
|
|
err);
|
|
exit(2);
|
|
}
|
|
g_dec_gen_streams++;
|
|
s->ptr = xmalloc(out_len);
|
|
memcpy(s->ptr, tmp_out, out_len);
|
|
s->len = out_len;
|
|
s->large_window = lw;
|
|
s->dicts = ds;
|
|
snprintf(s->name, sizeof s->name, "gen(q=%u lgwin=%u lw=%d dicts=%zu)", q,
|
|
lgwin, (int)lw, ds.n);
|
|
}
|
|
|
|
static size_t //
|
|
mutate(uint8_t* m, size_t n, size_t cap, const seed* other) {
|
|
int k = 1 + (int)rn(chance(800) ? 3 : 12);
|
|
for (int j = 0; j < k; j++) {
|
|
switch (rn(10)) {
|
|
case 0:
|
|
case 1:
|
|
case 2: // Bit flip.
|
|
if (n) {
|
|
m[rn(n)] ^= (uint8_t)(1u << rn(8));
|
|
}
|
|
break;
|
|
case 3: // Byte substitution.
|
|
if (n) {
|
|
m[rn(n)] = (uint8_t)r64();
|
|
}
|
|
break;
|
|
case 4: // Truncation.
|
|
if (n) {
|
|
n = rn(n + 1);
|
|
}
|
|
break;
|
|
case 5: { // Insertion.
|
|
size_t len = 1 + rn(chance(800) ? 4 : 64);
|
|
if ((n + len) > cap) {
|
|
break;
|
|
}
|
|
size_t p = rn(n + 1);
|
|
memmove(m + p + len, m + p, n - p);
|
|
for (size_t i = 0; i < len; i++) {
|
|
m[p + i] = (uint8_t)r64();
|
|
}
|
|
n += len;
|
|
break;
|
|
}
|
|
case 6: { // Deletion.
|
|
if (!n) {
|
|
break;
|
|
}
|
|
size_t p = rn(n);
|
|
size_t len = 1 + rn(chance(800) ? 4 : 64);
|
|
if (len > (n - p)) {
|
|
len = n - p;
|
|
}
|
|
memmove(m + p, m + p + len, n - p - len);
|
|
n -= len;
|
|
break;
|
|
}
|
|
case 7: { // Splice: our prefix and the other stream's suffix.
|
|
if (!other || !other->len) {
|
|
break;
|
|
}
|
|
size_t p = rn(n + 1);
|
|
size_t q = rn(other->len);
|
|
size_t len = other->len - q;
|
|
if ((p + len) > cap) {
|
|
len = cap - p;
|
|
}
|
|
memcpy(m + p, other->ptr + q, len);
|
|
n = p + len;
|
|
break;
|
|
}
|
|
case 8: { // Overwrite a range with random bytes.
|
|
if (!n) {
|
|
break;
|
|
}
|
|
size_t p = rn(n);
|
|
size_t len = 1 + rn(16);
|
|
for (size_t i = 0; (i < len) && ((p + i) < n); i++) {
|
|
m[p + i] = (uint8_t)r64();
|
|
}
|
|
break;
|
|
}
|
|
default: { // Small arithmetic change.
|
|
if (n) {
|
|
m[rn(n)] += (uint8_t)(rn(2) ? 1 : 0xFF);
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
return n;
|
|
}
|
|
|
|
// pick_dec_dicts chooses the dictionaries to attach: usually the seed's own,
|
|
// sometimes random ones (raw and serialized, possibly mutated).
|
|
static void //
|
|
pick_dec_dicts(dict_set* out, const seed* s) {
|
|
out->n = 0;
|
|
if (chance(700)) {
|
|
dict_set_copy(out, &s->dicts);
|
|
return;
|
|
}
|
|
size_t k = pick_num_dicts();
|
|
for (size_t i = 0; i < k; i++) {
|
|
if (chance(250)) {
|
|
size_t which = rn(4);
|
|
size_t len = g_sd_len[which];
|
|
uint8_t* p = xmalloc(len + 64);
|
|
memcpy(p, g_sd[which], len);
|
|
if (chance(300)) {
|
|
len = mutate(p, len, len + 64, NULL);
|
|
}
|
|
dict_set_add(out, 1, p, len, false);
|
|
out->d[out->n - 1].owned = true;
|
|
} else if (chance(300)) {
|
|
size_t off = rn(g_romeo_len);
|
|
size_t len = rn(g_romeo_len - off + 1);
|
|
dict_set_add(out, 0, g_romeo + off, len, true);
|
|
} else {
|
|
size_t len = pick_dict_len();
|
|
if (len > (256 << 10)) {
|
|
len = rn(256 << 10);
|
|
}
|
|
uint8_t* p = random_dict_bytes(len, NULL, 0);
|
|
dict_set_add(out, 0, p, len, false);
|
|
out->d[out->n - 1].owned = true;
|
|
}
|
|
}
|
|
}
|
|
|
|
static void //
|
|
describe_dec(char* buf, size_t cap, const char* what, const seed* s,
|
|
const dec_config* dc, const chunking* ch, const dec_result* w,
|
|
const dec_result* c, size_t n) {
|
|
snprintf(buf, cap,
|
|
"%s: seed=%s stream_len=%zu lw=%d dicts=%zu (total %zu) wb_len=%" PRIu64
|
|
" expect_bad_wb=%d uninit=%d out_cap=%zu one_shot=%d in_seed=0x%" PRIX64
|
|
" in_style=%d out_seed=0x%" PRIX64 " out_style=%d | wuffs: cls=%d "
|
|
"status=%s out=%zu in=%zu calls=%" PRIu64 " | c: cls=%d out=%zu in=%zu\n",
|
|
what, s->name, n, (int)dc->large_window, dc->dicts.n,
|
|
dict_set_total(&dc->dicts), dc->wb_len, (int)dc->expect_bad_wb,
|
|
(int)dc->uninit, dc->out_cap, (int)ch->one_shot, ch->in_seed,
|
|
ch->in_style, ch->out_seed, ch->out_style, w ? w->cls : 0,
|
|
(w && w->wuffs_status) ? w->wuffs_status : "ok", w ? w->out_n : 0,
|
|
w ? w->in_used : 0, w ? w->calls : 0, c ? c->cls : 0,
|
|
c ? c->out_n : 0, c ? c->in_used : 0);
|
|
}
|
|
|
|
// compare_dec returns NULL if the two results agree.
|
|
static const char* //
|
|
compare_dec(const dec_result* w, const dec_result* c, const uint8_t* wout,
|
|
const uint8_t* cout, size_t out_cap) {
|
|
if (w->cls == CLS_BUG) {
|
|
return "Wuffs returned an unexpected status";
|
|
} else if (c->cls == CLS_BUG) {
|
|
return "libbrotlidec setup failed";
|
|
} else if (w->cls != c->cls) {
|
|
return "different result classes";
|
|
}
|
|
if ((w->cls == CLS_OK) || (w->cls == CLS_NEEDS_MORE_OUTPUT)) {
|
|
if (w->out_n != c->out_n) {
|
|
return "different output lengths";
|
|
} else if (w->out_n && memcmp(wout, cout, w->out_n)) {
|
|
return "different output bytes";
|
|
} else if ((w->cls == CLS_OK) && (w->in_used != c->in_used)) {
|
|
return "different input consumption";
|
|
} else if ((w->cls == CLS_NEEDS_MORE_OUTPUT) && (w->out_n != out_cap)) {
|
|
return "needs more output but dst is not full";
|
|
}
|
|
return NULL;
|
|
}
|
|
size_t m = (w->out_n < c->out_n) ? w->out_n : c->out_n;
|
|
if (m && memcmp(wout, cout, m)) {
|
|
return "different output bytes (common prefix)";
|
|
}
|
|
if (w->out_n < c->out_n) {
|
|
g_dec_wuffs_shorter_output++;
|
|
}
|
|
return NULL;
|
|
}
|
|
|
|
// known_block_length_deviation returns whether the results differ only in
|
|
// the documented way (see "Status Mapping" in std/brotli/README.md): with
|
|
// chunked input, libbrotlidec can report BROTLI_DECODER_ERROR_FORMAT_BLOCK_
|
|
// LENGTH_1 for an overrun metablock where Wuffs reports "#bad metablock
|
|
// length" (BLOCK_LENGTH_2), because libbrotlidec's internal input buffer
|
|
// (BrotliDecoderState.buffer, used after it rolls back a partially read
|
|
// command, distance or block switch) makes it run out of input, and check
|
|
// the metablock length, at points where Wuffs does not.
|
|
static bool //
|
|
known_block_length_deviation(const chunking* ch, const dec_result* w,
|
|
const dec_result* c, const uint8_t* wout,
|
|
const uint8_t* cout) {
|
|
if (ch->one_shot || (w->cls != -10) || (c->cls != -9)) {
|
|
return false;
|
|
}
|
|
size_t m = (w->out_n < c->out_n) ? w->out_n : c->out_n;
|
|
return !m || !memcmp(wout, cout, m);
|
|
}
|
|
|
|
static int //
|
|
run_dec(uint64_t iters, double secs) {
|
|
load_corpus();
|
|
load_dicts();
|
|
load_file_seeds();
|
|
g_dec_wb = big_alloc(DEC_WB_CAP);
|
|
g_enc_wb = big_alloc(ENC_WB_CAP);
|
|
g_dec = (wuffs_brotli__decoder*)xmalloc(sizeof__wuffs_brotli__decoder());
|
|
const size_t tmp_cap = (8 << 20);
|
|
uint8_t* tmp_in = xmalloc(tmp_cap);
|
|
uint8_t* tmp_out = xmalloc(tmp_cap);
|
|
uint8_t* wout = xmalloc(MAX_OUT);
|
|
uint8_t* cout = xmalloc(MAX_OUT);
|
|
g_wmd = xmalloc(MD_CAP);
|
|
g_cmd = xmalloc(MD_CAP);
|
|
const size_t mcap = (1 << 20);
|
|
uint8_t* m = xmalloc(mcap);
|
|
|
|
// Fill the generated seed pool.
|
|
for (size_t i = 0; i < 32; i++) {
|
|
make_gen_seed(&g_gen_seeds[g_num_gen_seeds++], tmp_in, tmp_out, tmp_cap);
|
|
}
|
|
|
|
struct timespec t0;
|
|
clock_gettime(CLOCK_MONOTONIC, &t0);
|
|
uint64_t it = 0;
|
|
for (; it < iters; it++) {
|
|
if ((it & 63) == 0) {
|
|
struct timespec t1;
|
|
clock_gettime(CLOCK_MONOTONIC, &t1);
|
|
double elapsed =
|
|
(double)(t1.tv_sec - t0.tv_sec) + 1e-9 * (double)(t1.tv_nsec - t0.tv_nsec);
|
|
if ((secs > 0) && (elapsed > secs)) {
|
|
break;
|
|
}
|
|
if (g_dec_wb && ((it & 4095) == 0)) {
|
|
big_release(g_dec_wb, DEC_WB_CAP);
|
|
big_release(g_enc_wb, ENC_WB_CAP);
|
|
}
|
|
}
|
|
if ((it % 16) == 0) {
|
|
size_t slot = (g_num_gen_seeds < MAX_GEN_SEEDS) ? g_num_gen_seeds++
|
|
: (size_t)rn(MAX_GEN_SEEDS);
|
|
make_gen_seed(&g_gen_seeds[slot], tmp_in, tmp_out, tmp_cap);
|
|
}
|
|
|
|
// Pick a seed and mutate it.
|
|
const seed* s = chance(500) ? &g_file_seeds[rn(g_num_file_seeds)]
|
|
: &g_gen_seeds[rn(g_num_gen_seeds)];
|
|
const seed* other = chance(500) ? &g_file_seeds[rn(g_num_file_seeds)]
|
|
: &g_gen_seeds[rn(g_num_gen_seeds)];
|
|
size_t n;
|
|
if (chance(20)) {
|
|
n = rn(chance(500) ? 32 : 4096);
|
|
for (size_t i = 0; i < n; i++) {
|
|
m[i] = (uint8_t)r64();
|
|
}
|
|
} else {
|
|
n = (s->len < mcap) ? s->len : mcap;
|
|
memcpy(m, s->ptr, n);
|
|
if (!chance(30)) { // 3% are unmutated.
|
|
n = mutate(m, n, mcap, other);
|
|
}
|
|
}
|
|
g_dec_inputs++;
|
|
|
|
dec_config dc;
|
|
memset(&dc, 0, sizeof dc);
|
|
dc.large_window = chance(800) ? s->large_window : chance(500);
|
|
g_dec_lw += dc.large_window;
|
|
pick_dec_dicts(&dc.dicts, s);
|
|
g_dec_with_dicts += (dc.dicts.n > 0);
|
|
dc.uninit = chance(500);
|
|
dc.out_cap = chance(900) ? ((size_t)1 << 20) + rn(1 << 20) : rn(4096);
|
|
// Only with a large dst: libbrotlidec reports metadata (via callbacks)
|
|
// even when its dst is full, but Wuffs needs dst space for it.
|
|
dc.report_md = (dc.out_cap >= ((size_t)1 << 20)) && chance(300);
|
|
g_dec_report_md += dc.report_md;
|
|
|
|
// Attach the dictionaries to both decoders, checking that they agree on
|
|
// which ones are accepted. libbrotlidec can be left in a modified state
|
|
// after a failure (see std/brotli/README.md), so only the dictionaries
|
|
// before the first failure are used.
|
|
{
|
|
size_t w_ok = 0;
|
|
const char* werr = wuffs_attach(g_dec, &dc, &w_ok);
|
|
BrotliDecoderState* cs = BrotliDecoderCreateInstance(NULL, NULL, NULL);
|
|
size_t c_ok = 0;
|
|
for (size_t i = 0; i < dc.dicts.n; i++) {
|
|
if (!BrotliDecoderAttachDictionary(
|
|
cs, (BrotliSharedDictionaryType)dc.dicts.d[i].type,
|
|
dc.dicts.d[i].len, dc.dicts.d[i].ptr)) {
|
|
break;
|
|
}
|
|
c_ok++;
|
|
}
|
|
BrotliDecoderDestroyInstance(cs);
|
|
if (w_ok != c_ok) {
|
|
char desc[2048];
|
|
snprintf(desc, sizeof desc,
|
|
"add_dictionary: Wuffs accepted %zu (then \"%s\"), "
|
|
"libbrotlidec accepted %zu, of %zu dictionaries (seed %s)\n",
|
|
w_ok, werr ? werr : "", c_ok, dc.dicts.n, s->name);
|
|
save_failure("dict", desc, m, n, &dc.dicts);
|
|
}
|
|
if (w_ok < dc.dicts.n) {
|
|
g_dec_dict_rejections++;
|
|
size_t keep = (w_ok < c_ok) ? w_ok : c_ok;
|
|
for (size_t i = keep; i < dc.dicts.n; i++) {
|
|
if (dc.dicts.d[i].owned) {
|
|
free((void*)dc.dicts.d[i].ptr);
|
|
}
|
|
}
|
|
dc.dicts.n = keep;
|
|
}
|
|
}
|
|
|
|
// The workbuf length.
|
|
{
|
|
size_t w_ok = 0;
|
|
wuffs_attach(g_dec, &dc, &w_ok);
|
|
uint64_t wlen = wuffs_brotli__decoder__workbuf_len(g_dec).max_incl;
|
|
uint32_t wbits = stream_window_bits(m, n, dc.large_window);
|
|
uint64_t exact = wbits ? (dict_set_total(&dc.dicts) +
|
|
((uint64_t)1 << wbits) + 542)
|
|
: 0;
|
|
uint64_t r = rn(1000);
|
|
if (exact && (r < 100)) {
|
|
dc.wb_len = exact;
|
|
g_dec_exact_wb++;
|
|
} else if (exact && (r < 130)) {
|
|
dc.wb_len = exact - 1;
|
|
dc.expect_bad_wb = true;
|
|
g_dec_bad_wb_checks++;
|
|
} else if (r < 600) {
|
|
dc.wb_len = wlen;
|
|
} else {
|
|
dc.wb_len = wlen + rn(65536);
|
|
}
|
|
if (dc.wb_len > DEC_WB_CAP) {
|
|
dc.wb_len = wlen;
|
|
dc.expect_bad_wb = false;
|
|
}
|
|
}
|
|
|
|
for (int pass = 0; pass < 2; pass++) {
|
|
chunking ch;
|
|
memset(&ch, 0, sizeof ch);
|
|
ch.one_shot = (pass == 0);
|
|
ch.in_seed = r64();
|
|
ch.out_seed = r64();
|
|
ch.in_style = (int)rn(4);
|
|
ch.out_style = (int)rn(4);
|
|
size_t c_attached = 0;
|
|
dec_result w = wuffs_decode(&dc, &ch, m, n, wout);
|
|
dec_result c = c_decode(&dc, &ch, m, n, cout, &c_attached);
|
|
if (dc.report_md && (w.cls == CLS_NEEDS_MORE_OUTPUT)) {
|
|
// A full dst stops Wuffs at a metadata block, but not libbrotlidec,
|
|
// so this is inconclusive. Compare without reporting instead.
|
|
g_dec_md_inconclusive++;
|
|
dc.report_md = false;
|
|
w = wuffs_decode(&dc, &ch, m, n, wout);
|
|
c = c_decode(&dc, &ch, m, n, cout, &c_attached);
|
|
dc.report_md = true;
|
|
} else if (dc.report_md) {
|
|
// Before an error, both have reported the same metadata, except that
|
|
// one can have more of the last block's contents (or also the next
|
|
// block's start). After success, they agree exactly.
|
|
size_t k = (w.md_n < c.md_n) ? w.md_n : c.md_n;
|
|
const char* mwhy = NULL;
|
|
if (c.md_n == SIZE_MAX) {
|
|
mwhy = "libbrotlidec metadata overflow";
|
|
} else if (k && memcmp(g_wmd, g_cmd, k)) {
|
|
mwhy = "different metadata";
|
|
} else if ((w.cls == CLS_OK) && (c.cls == CLS_OK) &&
|
|
(w.md_n != c.md_n)) {
|
|
mwhy = "different metadata lengths";
|
|
}
|
|
if ((w.cls == CLS_OK) && (c.cls == CLS_OK)) {
|
|
g_dec_md_compared++;
|
|
for (size_t j = 0; j + 4 <= w.md_n;) {
|
|
size_t len = (size_t)g_wmd[j] | ((size_t)g_wmd[j + 1] << 8) |
|
|
((size_t)g_wmd[j + 2] << 16) |
|
|
((size_t)g_wmd[j + 3] << 24);
|
|
g_dec_md_blocks++;
|
|
j += 4 + len;
|
|
}
|
|
}
|
|
if (mwhy) {
|
|
char desc[4096];
|
|
describe_dec(desc, sizeof desc, mwhy, s, &dc, &ch, &w, &c, n);
|
|
save_failure("md", desc, m, n, &dc.dicts);
|
|
}
|
|
}
|
|
g_dec_decodes += 2;
|
|
g_dec_cls_counts[pass][cls_index(c.cls)]++;
|
|
const char* why = NULL;
|
|
if (dc.expect_bad_wb) {
|
|
// The workbuf is one byte too short for the window. Wuffs must fail
|
|
// with "#bad workbuf length" unless the stream fails (or is
|
|
// truncated) before the window size is known, which can only be the
|
|
// case for a stream with an invalid or truncated header. Since
|
|
// stream_window_bits succeeded, the header is valid and complete.
|
|
if (w.cls != -100) {
|
|
why = "a too-short workbuf did not fail";
|
|
}
|
|
} else {
|
|
why = compare_dec(&w, &c, wout, cout, dc.out_cap);
|
|
if (why && known_block_length_deviation(&ch, &w, &c, wout, cout)) {
|
|
why = NULL;
|
|
g_dec_known_block_length++;
|
|
if (g_dec_known_saved < 20) {
|
|
g_dec_known_saved++;
|
|
char desc[4096];
|
|
describe_dec(desc, sizeof desc, "known BLOCK_LENGTH deviation", s,
|
|
&dc, &ch, &w, &c, n);
|
|
save_failure("known", desc, m, n, &dc.dicts);
|
|
g_num_failures--;
|
|
}
|
|
}
|
|
}
|
|
if (why) {
|
|
char desc[4096];
|
|
describe_dec(desc, sizeof desc, why, s, &dc, &ch, &w, &c, n);
|
|
save_failure("dec", desc, m, n, &dc.dicts);
|
|
}
|
|
}
|
|
dict_set_free(&dc.dicts);
|
|
}
|
|
|
|
printf("decoder campaign: seed=%" PRIu64 " inputs=%" PRIu64 " decodes=%" PRIu64
|
|
" failures=%" PRIu64 "\n",
|
|
g_seed, g_dec_inputs, g_dec_decodes, g_num_failures);
|
|
printf(" generated seed streams=%" PRIu64 " (%" PRIu64
|
|
" by Wuffs' encoder), large window=%" PRIu64 ", with dicts=%" PRIu64
|
|
", dict rejections=%" PRIu64 ", exact workbuf=%" PRIu64
|
|
", too-short workbuf=%" PRIu64 ", wuffs shorter output on error=%" PRIu64
|
|
"\n",
|
|
g_dec_gen_streams, g_dec_gen_wuffs, g_dec_lw, g_dec_with_dicts,
|
|
g_dec_dict_rejections, g_dec_exact_wb, g_dec_bad_wb_checks,
|
|
g_dec_wuffs_shorter_output);
|
|
printf(" known BLOCK_LENGTH_1/2 deviations (chunked)=%" PRIu64 "\n",
|
|
g_dec_known_block_length);
|
|
printf(" report metadata=%" PRIu64 " (compared after success=%" PRIu64
|
|
", blocks=%" PRIu64 ", inconclusive=%" PRIu64 ")\n",
|
|
g_dec_report_md, g_dec_md_compared, g_dec_md_blocks,
|
|
g_dec_md_inconclusive);
|
|
static const char* names[23] = {
|
|
"other", "ok", "needs-more-input", "needs-more-output",
|
|
"-1", "-2", "-3", "-4",
|
|
"-5", "-6", "-7", "-8",
|
|
"-9", "-10", "-11", "-12",
|
|
"-13", "-14", "-15", "-16",
|
|
"-17", "-18", "bad-workbuf"};
|
|
for (int pass = 0; pass < 2; pass++) {
|
|
printf(" %s classes (C):", pass ? "chunked" : "one-shot");
|
|
for (int i = 0; i < 23; i++) {
|
|
if (g_dec_cls_counts[pass][i]) {
|
|
printf(" %s=%" PRIu64, names[i], g_dec_cls_counts[pass][i]);
|
|
}
|
|
}
|
|
printf("\n");
|
|
}
|
|
return g_num_failures ? 1 : 0;
|
|
}
|
|
|
|
// replay_dec re-runs a saved decoder failure (see save_failure), printing
|
|
// every transform_io and BrotliDecoderDecompressStream call.
|
|
static int //
|
|
replay_dec(const char* path) {
|
|
g_dec_wb = big_alloc(DEC_WB_CAP);
|
|
g_dec = (wuffs_brotli__decoder*)xmalloc(sizeof__wuffs_brotli__decoder());
|
|
size_t n = 0;
|
|
uint8_t* m = read_file(path, &n, 64 << 20);
|
|
char tpath[1100];
|
|
snprintf(tpath, sizeof tpath, "%s.txt", path);
|
|
size_t tn = 0;
|
|
char* t = (char*)read_file(tpath, &tn, 1 << 20);
|
|
if (!m || !t) {
|
|
fprintf(stderr, "could not read %s or its .txt\n", path);
|
|
return 2;
|
|
}
|
|
t[tn] = 0;
|
|
dec_config dc;
|
|
memset(&dc, 0, sizeof dc);
|
|
chunking ch;
|
|
memset(&ch, 0, sizeof ch);
|
|
int lw = 0, ebw = 0, un = 0, os = 0;
|
|
char* q = strstr(t, " lw=");
|
|
if (!q || (sscanf(q, " lw=%d", &lw) != 1)) {
|
|
return 2;
|
|
}
|
|
q = strstr(t, " wb_len=");
|
|
if (!q || (sscanf(q,
|
|
" wb_len=%" SCNu64 " expect_bad_wb=%d uninit=%d out_cap=%zu "
|
|
"one_shot=%d in_seed=0x%" SCNx64 " in_style=%d out_seed=0x%" SCNx64
|
|
" out_style=%d",
|
|
&dc.wb_len, &ebw, &un, &dc.out_cap, &os, &ch.in_seed,
|
|
&ch.in_style, &ch.out_seed, &ch.out_style) != 9)) {
|
|
fprintf(stderr, "could not parse %s\n", tpath);
|
|
return 2;
|
|
}
|
|
dc.large_window = lw;
|
|
dc.expect_bad_wb = ebw;
|
|
dc.uninit = un;
|
|
ch.one_shot = os;
|
|
for (size_t i = 0; i < MAX_DICTS; i++) {
|
|
for (uint32_t type = 0; type < 2; type++) {
|
|
char dpath[1200];
|
|
snprintf(dpath, sizeof dpath, "%s.dict%zu.%s", path, i,
|
|
type ? "sd" : "raw");
|
|
size_t dn = 0;
|
|
uint8_t* d = read_file(dpath, &dn, 1 << 30);
|
|
if (d) {
|
|
dict_set_add(&dc.dicts, type, d, dn, false);
|
|
}
|
|
}
|
|
}
|
|
uint8_t* wout = xmalloc(dc.out_cap + 1);
|
|
uint8_t* cout = xmalloc(dc.out_cap + 1);
|
|
g_trace = true;
|
|
size_t att = 0;
|
|
dec_result w = wuffs_decode(&dc, &ch, m, n, wout);
|
|
dec_result c = c_decode(&dc, &ch, m, n, cout, &att);
|
|
printf("wuffs: cls=%d status=%s out=%zu in=%zu\nc: cls=%d out=%zu in=%zu\n",
|
|
w.cls, w.wuffs_status ? w.wuffs_status : "ok", w.out_n, w.in_used,
|
|
c.cls, c.out_n, c.in_used);
|
|
const char* why = compare_dec(&w, &c, wout, cout, dc.out_cap);
|
|
printf("verdict: %s\n", why ? why : "agree");
|
|
return why ? 1 : 0;
|
|
}
|
|
|
|
// ---------------- Encoder campaign
|
|
|
|
static int run_enc(uint64_t iters, double secs);
|
|
static int run_api(uint64_t iters, double secs);
|
|
|
|
int //
|
|
main(int argc, char** argv) {
|
|
if (BrotliEncoderVersion() != 0x1002001 ||
|
|
BrotliDecoderVersion() != 0x1002001) {
|
|
fprintf(stderr, "want libbrotli v1.2.1, have enc 0x%x dec 0x%x\n",
|
|
BrotliEncoderVersion(), BrotliDecoderVersion());
|
|
return 2;
|
|
}
|
|
if (argc < 2) {
|
|
fprintf(stderr, "usage: %s -dec|-enc [options]\n", argv[0]);
|
|
return 2;
|
|
}
|
|
uint64_t seed = 1;
|
|
uint64_t iters = 1000;
|
|
double secs = 0;
|
|
for (int i = 2; i < argc; i++) {
|
|
if (!strncmp(argv[i], "-seed=", 6)) {
|
|
seed = strtoull(argv[i] + 6, NULL, 0);
|
|
} else if (!strncmp(argv[i], "-n=", 3)) {
|
|
iters = strtoull(argv[i] + 3, NULL, 0);
|
|
} else if (!strncmp(argv[i], "-secs=", 6)) {
|
|
secs = atof(argv[i] + 6);
|
|
} else if (!strncmp(argv[i], "-out=", 5)) {
|
|
g_out_dir = argv[i] + 5;
|
|
} else if (!strncmp(argv[i], "-data=", 6)) {
|
|
g_data_dir = argv[i] + 6;
|
|
} else if (!strncmp(argv[i], "-q=", 3)) {
|
|
unsigned lo = 0, hi = 0;
|
|
if (sscanf(argv[i] + 3, "%u-%u", &lo, &hi) != 2 || lo > hi || hi > 11) {
|
|
fprintf(stderr, "bad -q option\n");
|
|
return 2;
|
|
}
|
|
g_qmin = lo;
|
|
g_qmax = hi;
|
|
} else if (!strcmp(argv[i], "-nohq")) {
|
|
g_hq_dicts = false;
|
|
} else {
|
|
fprintf(stderr, "unknown option %s\n", argv[i]);
|
|
return 2;
|
|
}
|
|
}
|
|
g_seed = seed;
|
|
// Hash the seed: SplitMix64 adds 0x9E3779B97F4A7C15 to its state per
|
|
// number, so a starting state of (seed * 0x9E3779B97F4A7C15) would make
|
|
// seed N+1's sequence that of seed N shifted by one number, and shards with
|
|
// consecutive seeds would soon generate the same cases.
|
|
g_rng_state = seed;
|
|
g_rng_state = r64();
|
|
if (!strncmp(argv[1], "-replay=", 8)) {
|
|
return replay_dec(argv[1] + 8);
|
|
} else if (!strcmp(argv[1], "-dec")) {
|
|
return run_dec(iters, secs);
|
|
} else if (!strcmp(argv[1], "-enc")) {
|
|
return run_enc(iters, secs);
|
|
} else if (!strcmp(argv[1], "-api")) {
|
|
return run_api(iters, secs);
|
|
}
|
|
fprintf(stderr, "unknown mode %s\n", argv[1]);
|
|
return 2;
|
|
}
|
|
|
|
// pick_quality picks a quality in [g_qmin ..= g_qmax], weighted towards the
|
|
// faster qualities.
|
|
static uint32_t //
|
|
pick_quality(void) {
|
|
for (int tries = 0; tries < 100; tries++) {
|
|
uint64_t r = rn(1000);
|
|
uint32_t q;
|
|
if (r < 550) {
|
|
q = (uint32_t)rn(5);
|
|
} else if (r < 930) {
|
|
q = 5 + (uint32_t)rn(5);
|
|
} else {
|
|
q = 10 + (uint32_t)rn(2);
|
|
}
|
|
if ((g_qmin <= q) && (q <= g_qmax)) {
|
|
return q;
|
|
}
|
|
}
|
|
return g_qmin + (uint32_t)rn(g_qmax - g_qmin + 1);
|
|
}
|
|
|
|
static uint32_t //
|
|
weird_u32(void) {
|
|
static const uint32_t v[] = {0, 1, 2, 9, 31, 32, 255, 0x7FFFFFFF, 0x80000000,
|
|
0xFFFFFFFE, 0xFFFFFFFF};
|
|
if (chance(500)) {
|
|
return v[rn(sizeof v / sizeof v[0])];
|
|
}
|
|
return (uint32_t)r64();
|
|
}
|
|
|
|
static void //
|
|
shuffle_params(enc_config* c) {
|
|
for (size_t i = c->num_params; i > 1; i--) {
|
|
size_t j = (size_t)rn(i);
|
|
uint32_t k = c->keys[i - 1];
|
|
uint32_t v = c->values[i - 1];
|
|
c->keys[i - 1] = c->keys[j];
|
|
c->values[i - 1] = c->values[j];
|
|
c->keys[j] = k;
|
|
c->values[j] = v;
|
|
}
|
|
}
|
|
|
|
static uint64_t g_enc_configs;
|
|
static uint64_t g_enc_by_quality[12];
|
|
static uint64_t g_enc_bytes_in;
|
|
static uint64_t g_enc_both_rejected;
|
|
static uint64_t g_enc_both_flint_failed;
|
|
static uint64_t g_enc_c_corrupted;
|
|
// g_enc_uninit_tail counts output mismatches that go away when both encoders
|
|
// get zeroed memory. libbrotlienc's RingBufferWrite allocates only the first
|
|
// write's bytes (when it is shorter than the tail) and, when it grows the ring
|
|
// buffer, copies them but leaves the tail (the copy of the buffer's start, past
|
|
// its end) uninitialized, which match finding can then read. Wuffs reads the
|
|
// corresponding workbuf bytes, so each output depends on what that memory
|
|
// held (malloc'd memory for C, the caller's workbuf for Wuffs), and the two
|
|
// agree when the memory does. Both outputs must still decode correctly.
|
|
static uint64_t g_enc_uninit_tail;
|
|
static uint64_t g_enc_skipped;
|
|
static uint64_t g_enc_roundtrips;
|
|
static uint64_t g_enc_with_dicts;
|
|
static uint64_t g_enc_hq_with_dicts;
|
|
static uint64_t g_enc_lw;
|
|
static uint64_t g_enc_short_checks;
|
|
static uint64_t g_enc_wb_extra;
|
|
static uint64_t g_enc_metadata;
|
|
static uint64_t g_enc_flushes;
|
|
static uint64_t g_enc_dst_limited;
|
|
static uint64_t g_enc_stream_offset;
|
|
static uint64_t g_enc_len_hist[5];
|
|
static double g_enc_time[3]; // C encoder, Wuffs encoder, round trips.
|
|
|
|
static double //
|
|
now_secs(void) {
|
|
struct timespec t;
|
|
clock_gettime(CLOCK_MONOTONIC, &t);
|
|
return (double)t.tv_sec + 1e-9 * (double)t.tv_nsec;
|
|
} // <=64Ki, <=256Ki, <=1Mi, <=4Mi, more.
|
|
|
|
static void //
|
|
describe_enc(char* buf, size_t cap, const char* what, const enc_config* c,
|
|
const dict_set* ds, size_t input_len, size_t src_len) {
|
|
int k = snprintf(buf, cap, "%s: input_len=%zu src_len=%zu params:", what,
|
|
input_len, src_len);
|
|
for (size_t j = 0; j < c->num_params; j++) {
|
|
k += snprintf(buf + k, cap - (size_t)k, " p%u=%u", c->keys[j], c->values[j]);
|
|
}
|
|
k += snprintf(buf + k, cap - (size_t)k,
|
|
" dst_chunk=%zu%s dst_seed=0x%" PRIX64 " workbuf_extra=%" PRIu64
|
|
" short_workbuf=%d dicts=%zu:",
|
|
c->dst_chunk, c->dst_chunk_rand ? "(rand)" : "", c->dst_seed,
|
|
c->workbuf_extra, (int)c->short_workbuf, ds->n);
|
|
for (size_t j = 0; (j < ds->n) && ((size_t)k < (cap - 64)); j++) {
|
|
k += snprintf(buf + k, cap - (size_t)k, " %zu", ds->d[j].len);
|
|
}
|
|
k += snprintf(buf + k, cap - (size_t)k, " steps=%zu:", c->num_steps);
|
|
for (size_t j = 0; (j < c->num_steps) && ((size_t)k < (cap - 64)); j++) {
|
|
k += snprintf(buf + k, cap - (size_t)k, " %c%zu", "PFXM"[c->step_ops[j]],
|
|
c->step_lens[j]);
|
|
}
|
|
snprintf(buf + k, cap - (size_t)k, "\n");
|
|
}
|
|
|
|
// save_enc_failure saves the input (the plan's source, including any
|
|
// metadata), the dictionaries and a description.
|
|
static void //
|
|
save_enc_failure(const char* what, const enc_config* c, const dict_set* ds,
|
|
const uint8_t* src, size_t src_len, size_t input_len) {
|
|
static char desc[1 << 16];
|
|
describe_enc(desc, sizeof desc, what, c, ds, input_len, src_len);
|
|
save_failure("enc", desc, src, src_len, ds);
|
|
}
|
|
|
|
// roundtrip_dec decodes stream with both decoders and checks that the result
|
|
// is want.
|
|
static const char* //
|
|
roundtrip_dec(const uint8_t* stream, size_t stream_len, const uint8_t* want,
|
|
size_t want_len, bool lw, const dict_set* ds, uint8_t* out) {
|
|
dec_config dc;
|
|
memset(&dc, 0, sizeof dc);
|
|
dc.large_window = lw;
|
|
dc.dicts = *ds; // Not owned.
|
|
dc.out_cap = want_len;
|
|
dc.uninit = chance(500);
|
|
chunking ch;
|
|
memset(&ch, 0, sizeof ch);
|
|
ch.one_shot = chance(800);
|
|
ch.in_seed = r64();
|
|
ch.out_seed = r64();
|
|
ch.in_style = 2 + (int)rn(2);
|
|
ch.out_style = 2 + (int)rn(2);
|
|
size_t attached = 0;
|
|
dec_result c = c_decode(&dc, &ch, stream, stream_len, out, &attached);
|
|
if (c.cls != CLS_OK) {
|
|
return "libbrotlidec round trip failed";
|
|
} else if ((c.out_n != want_len) || (want_len && memcmp(out, want, want_len))) {
|
|
return "libbrotlidec round trip mismatch";
|
|
} else if (c.in_used != stream_len) {
|
|
return "libbrotlidec did not consume the whole stream";
|
|
}
|
|
size_t n_ok = 0;
|
|
if (wuffs_attach(g_dec, &dc, &n_ok)) {
|
|
return "Wuffs add_dictionary failed";
|
|
}
|
|
dc.wb_len = wuffs_brotli__decoder__workbuf_len(g_dec).max_incl;
|
|
if (dc.wb_len > DEC_WB_CAP) {
|
|
return "decoder workbuf_len is too large";
|
|
}
|
|
dec_result w = wuffs_decode(&dc, &ch, stream, stream_len, out);
|
|
if (w.cls != CLS_OK) {
|
|
return "Wuffs decoder round trip failed";
|
|
} else if ((w.out_n != want_len) || (want_len && memcmp(out, want, want_len))) {
|
|
return "Wuffs decoder round trip mismatch";
|
|
} else if (w.in_used != stream_len) {
|
|
return "Wuffs decoder did not consume the whole stream";
|
|
}
|
|
return NULL;
|
|
}
|
|
|
|
static int //
|
|
run_enc(uint64_t iters, double secs) {
|
|
load_corpus();
|
|
load_dicts();
|
|
g_dec_wb = big_alloc(DEC_WB_CAP);
|
|
g_enc_wb = big_alloc(ENC_WB_CAP);
|
|
g_dec = (wuffs_brotli__decoder*)xmalloc(sizeof__wuffs_brotli__decoder());
|
|
const size_t in_cap = (4 << 20) + (40 << 20);
|
|
uint8_t* input = xmalloc(in_cap);
|
|
uint8_t* src = xmalloc(in_cap);
|
|
const size_t out_cap = in_cap + (1 << 20);
|
|
uint8_t* have = xmalloc(out_cap);
|
|
uint8_t* want = xmalloc(out_cap);
|
|
uint8_t* dec_out = xmalloc(in_cap);
|
|
enc_config c;
|
|
memset(&c, 0, sizeof c);
|
|
|
|
struct timespec t0;
|
|
clock_gettime(CLOCK_MONOTONIC, &t0);
|
|
uint64_t it = 0;
|
|
for (; it < iters; it++) {
|
|
if ((it & 15) == 0) {
|
|
struct timespec t1;
|
|
clock_gettime(CLOCK_MONOTONIC, &t1);
|
|
double elapsed = (double)(t1.tv_sec - t0.tv_sec) +
|
|
1e-9 * (double)(t1.tv_nsec - t0.tv_nsec);
|
|
if ((secs > 0) && (elapsed > secs)) {
|
|
break;
|
|
}
|
|
}
|
|
if (g_enc_wb_touched > ((size_t)512 << 20)) {
|
|
big_release(g_enc_wb, ENC_WB_CAP);
|
|
big_release(g_dec_wb, DEC_WB_CAP);
|
|
g_enc_wb_touched = 0;
|
|
}
|
|
|
|
// Parameters.
|
|
c.num_params = 0;
|
|
c.num_steps = 0;
|
|
uint32_t q = pick_quality();
|
|
bool hq = q >= 10;
|
|
if (chance(990) || (q != 11)) {
|
|
enc_add_param(&c, 1, q);
|
|
} // Otherwise the default quality, 11.
|
|
bool lw = chance(200);
|
|
if (lw) {
|
|
enc_add_param(&c, 6, chance(900) ? 1 : (uint32_t)(1 + rn(0xFFFFFFFE)));
|
|
}
|
|
uint32_t lgwin = 22;
|
|
{
|
|
uint64_t r = rn(1000);
|
|
if (r < 700) {
|
|
lgwin = 10 + (uint32_t)rn(15);
|
|
enc_add_param(&c, 2, lgwin);
|
|
} else if (r < 850) {
|
|
lgwin = 10 + (uint32_t)rn(hq ? 18 : 21);
|
|
enc_add_param(&c, 2, lgwin);
|
|
} else if (r < 900) {
|
|
lgwin = weird_u32();
|
|
if (hq && lw && (lgwin > 27) && (lgwin < 0x80000000)) {
|
|
lgwin = 27;
|
|
}
|
|
enc_add_param(&c, 2, lgwin);
|
|
} // Otherwise the default lgwin, 22.
|
|
}
|
|
if (chance(400)) {
|
|
enc_add_param(&c, 3, chance(800) ? 16 + (uint32_t)rn(9)
|
|
: (chance(500) ? (uint32_t)rn(31)
|
|
: weird_u32()));
|
|
}
|
|
if (chance(300)) {
|
|
enc_add_param(&c, 0, chance(950) ? (uint32_t)rn(3) : weird_u32());
|
|
}
|
|
if (chance(500)) {
|
|
uint64_t r = rn(3);
|
|
enc_add_param(&c, 5, (r == 0) ? 0 : (r == 1) ? 0xFFFFFFFF : weird_u32());
|
|
}
|
|
if (chance(300)) {
|
|
uint32_t np = chance(950) ? (uint32_t)rn(4) : (uint32_t)rn(8);
|
|
uint32_t nd = chance(900) ? ((uint32_t)rn(16) << np) : (uint32_t)rn(200);
|
|
enc_add_param(&c, 7, np);
|
|
enc_add_param(&c, 8, nd);
|
|
}
|
|
if (chance(200)) {
|
|
enc_add_param(&c, 4, (uint32_t)rn(2));
|
|
}
|
|
bool stream_offset = false;
|
|
if (chance(80)) {
|
|
uint32_t v = chance(500) ? (uint32_t)rn(100000)
|
|
: (uint32_t)rn((1u << 30) + 1);
|
|
if (chance(100)) {
|
|
v = (1u << 30) + (uint32_t)rn(2); // 1 << 30 is OK, one more is not.
|
|
}
|
|
enc_add_param(&c, 9, v);
|
|
stream_offset = v != 0;
|
|
}
|
|
shuffle_params(&c);
|
|
|
|
// Input.
|
|
// At q10-11, dictionaries go with small inputs (the Zopfli encoder is
|
|
// slow), and only in 10% of the configs.
|
|
bool with_dicts = hq ? (g_hq_dicts && chance(100)) : chance(250);
|
|
size_t len = pick_len((hq && with_dicts) ? (chance(950) ? (16 << 10) : (64 << 10))
|
|
: chance(800) ? (64 << 10)
|
|
: hq ? (chance(900) ? (256 << 10) : (1 << 20))
|
|
: (4 << 20));
|
|
gen_input(input, len);
|
|
// Exact size hints.
|
|
for (size_t j = 0; j < c.num_params; j++) {
|
|
if ((c.keys[j] == 5) && (c.values[j] == 0xFFFFFFFF) && chance(900)) {
|
|
c.values[j] = (uint32_t)len;
|
|
}
|
|
}
|
|
|
|
// Dictionaries.
|
|
dict_set ds = {0};
|
|
if (with_dicts) {
|
|
size_t k = pick_num_dicts();
|
|
k = (k == 0) ? 1 : (k > 15) ? 15 : k;
|
|
random_raw_dicts(&ds, k, input, len);
|
|
}
|
|
|
|
// The plan.
|
|
c.dst_chunk = 0;
|
|
c.dst_chunk_rand = false;
|
|
c.dst_seed = r64();
|
|
{
|
|
uint64_t r = rn(1000);
|
|
if ((r < 100) && (len <= (256 << 10))) {
|
|
c.dst_chunk = 1 + rn(7);
|
|
} else if (r < 200) {
|
|
c.dst_chunk = 1 + rn(chance(500) ? 64 : 70000);
|
|
c.dst_chunk_rand = chance(500);
|
|
}
|
|
}
|
|
int chunk_style = (int)rn(6);
|
|
if ((chunk_style == 1) && (len > 4096)) {
|
|
chunk_style = 2;
|
|
}
|
|
if ((chunk_style == 2) && (len > (256 << 10))) {
|
|
chunk_style = 3;
|
|
}
|
|
bool flush_every = chance(50);
|
|
uint32_t flush_rate = chance(300) ? (uint32_t)rn(300) : 0;
|
|
uint32_t meta_rate = chance(150) ? (uint32_t)rn(100) : 0;
|
|
size_t src_len = 0;
|
|
size_t i = 0;
|
|
bool any_meta = false;
|
|
bool any_flush = false;
|
|
while (true) {
|
|
if (meta_rate && chance(meta_rate)) {
|
|
uint64_t r = rn(1000);
|
|
size_t m = (r < 300) ? 0
|
|
: (r < 500) ? 1
|
|
: (r < 900) ? rn(300)
|
|
: (r < 995) ? rn(70000)
|
|
: ((1 << 24) - 1 + rn(2));
|
|
if ((src_len + m + (len - i)) > in_cap) {
|
|
m = 0;
|
|
}
|
|
for (size_t j = 0; j < m; j++) {
|
|
src[src_len + j] = (uint8_t)r64();
|
|
}
|
|
src_len += m;
|
|
enc_add_step(&c, OP_EMIT_METADATA, m);
|
|
any_meta = true;
|
|
}
|
|
size_t k;
|
|
switch (chunk_style) {
|
|
case 0:
|
|
k = len - i;
|
|
break;
|
|
case 1:
|
|
k = 1;
|
|
break;
|
|
case 2:
|
|
k = 1 + rn(7);
|
|
break;
|
|
case 3:
|
|
k = 1 + rn(chance(900) ? 300 : 70000);
|
|
break;
|
|
case 4:
|
|
k = rn(chance(900) ? 5000 : (1 << 20)); // Can be 0.
|
|
break;
|
|
default:
|
|
k = rn(len + 1);
|
|
break;
|
|
}
|
|
if (k > (len - i)) {
|
|
k = len - i;
|
|
}
|
|
memcpy(src + src_len, input + i, k);
|
|
src_len += k;
|
|
i += k;
|
|
bool last = (i == len) && (chance(800) || (c.num_steps > 100000));
|
|
if (last) {
|
|
enc_add_step(&c, OP_FINISH, k);
|
|
break;
|
|
} else if (flush_every || (flush_rate && chance(flush_rate))) {
|
|
enc_add_step(&c, OP_FLUSH, k);
|
|
any_flush = true;
|
|
} else {
|
|
enc_add_step(&c, OP_PROCESS, k);
|
|
}
|
|
}
|
|
if (hq && chance(500)) {
|
|
uint64_t r = rn(3);
|
|
c.workbuf_extra = (r == 0) ? rn(1 << 20)
|
|
: (r == 1) ? rn((uint64_t)1 << 32)
|
|
: UINT64_MAX;
|
|
} else {
|
|
c.workbuf_extra = 0;
|
|
}
|
|
c.short_workbuf = (q >= 2) && chance(50);
|
|
|
|
g_enc_configs++;
|
|
g_enc_by_quality[q]++;
|
|
g_enc_bytes_in += len;
|
|
g_enc_len_hist[(len <= (64 << 10)) ? 0
|
|
: (len <= (256 << 10)) ? 1
|
|
: (len <= (1 << 20)) ? 2
|
|
: (len <= (4 << 20)) ? 3
|
|
: 4]++;
|
|
g_enc_with_dicts += (ds.n > 0);
|
|
g_enc_hq_with_dicts += (hq && (ds.n > 0));
|
|
g_enc_lw += lw;
|
|
g_enc_metadata += any_meta;
|
|
g_enc_flushes += any_flush;
|
|
g_enc_dst_limited += (c.dst_chunk > 0);
|
|
g_enc_stream_offset += stream_offset;
|
|
g_enc_wb_extra += (c.workbuf_extra > 0);
|
|
|
|
size_t have_len = 0;
|
|
size_t want_len = 0;
|
|
uint64_t wlen = 0;
|
|
double tt0 = now_secs();
|
|
const char* want_err = encode_c(&c, &ds, src, want, out_cap, &want_len);
|
|
double tt1 = now_secs();
|
|
const char* have_err = encode_wuffs(&c, &ds, src, src_len, len, have,
|
|
out_cap, &have_len, 0, &wlen);
|
|
double tt2 = now_secs();
|
|
g_enc_time[0] += tt1 - tt0;
|
|
g_enc_time[1] += tt2 - tt1;
|
|
if (have_err == g_too_large) {
|
|
g_enc_skipped++;
|
|
if (getenv("BFUZZ_VERBOSE")) {
|
|
static char desc[1 << 16];
|
|
describe_enc(desc, sizeof desc, "skipped", &c, &ds, len, src_len);
|
|
printf("%s", desc);
|
|
}
|
|
dict_set_free(&ds);
|
|
continue;
|
|
}
|
|
if (have_err || want_err) {
|
|
bool both_rejected =
|
|
have_err && want_err &&
|
|
(have_err == wuffs_base__error__bad_argument) &&
|
|
!strcmp(want_err, "BrotliEncoderSetParameter failed");
|
|
// With a stream offset, the C code's "emit flint" flush can return
|
|
// early with input left while a flush is pending (when dst is too
|
|
// short to take the flush's output), and the next call then fails
|
|
// (BrotliEncoderCompressStream returns BROTLI_FALSE when the stream is
|
|
// not in its processing state and there is input). Wuffs mirrors this
|
|
// with "#base: bad call sequence". Their outputs before then must
|
|
// agree.
|
|
bool both_flint_failed =
|
|
stream_offset && have_err && want_err &&
|
|
(have_err == wuffs_base__error__bad_call_sequence) &&
|
|
!strcmp(want_err, "BrotliEncoderCompressStream failed") &&
|
|
(have_len == want_len) && !memcmp(have, want, have_len);
|
|
// See g_c_state_corrupted: C's output must be a prefix of Wuffs'.
|
|
bool c_corrupted = !have_err && (want_err == g_c_state_corrupted) &&
|
|
(want_len <= have_len) &&
|
|
!memcmp(have, want, want_len);
|
|
if (both_rejected) {
|
|
g_enc_both_rejected++;
|
|
} else if (both_flint_failed) {
|
|
g_enc_both_flint_failed++;
|
|
} else if (c_corrupted) {
|
|
g_enc_c_corrupted++;
|
|
if (g_enc_c_corrupted <= 5) {
|
|
static char desc[1 << 16];
|
|
describe_enc(desc, sizeof desc, g_c_state_corrupted, &c, &ds, len,
|
|
src_len);
|
|
save_failure("enc-known", desc, src, src_len, &ds);
|
|
g_num_failures--;
|
|
}
|
|
} else {
|
|
char what[512];
|
|
snprintf(what, sizeof what, "errors: wuffs \"%s\", C \"%s\"",
|
|
have_err ? have_err : "", want_err ? want_err : "");
|
|
save_enc_failure(what, &c, &ds, src, src_len, len);
|
|
}
|
|
dict_set_free(&ds);
|
|
continue;
|
|
}
|
|
if ((have_len != want_len) || memcmp(have, want, have_len)) {
|
|
// Check that Wuffs' output decodes, then rerun both encoders with
|
|
// zeroed memory (see g_enc_uninit_tail).
|
|
const char* rt_err =
|
|
stream_offset ? NULL
|
|
: roundtrip_dec(have, have_len, input, len, lw, &ds,
|
|
dec_out);
|
|
size_t want2_len = 0;
|
|
size_t have2_len = 0;
|
|
uint64_t wlen3 = 0;
|
|
g_c_zero_alloc = true;
|
|
const char* e1 = encode_c(&c, &ds, src, want, out_cap, &want2_len);
|
|
g_c_zero_alloc = false;
|
|
memset(g_enc_wb, 0, wlen);
|
|
const char* e2 = encode_wuffs(&c, &ds, src, src_len, len, have, out_cap,
|
|
&have2_len, 0, &wlen3);
|
|
if (!rt_err && !e1 && !e2 && (have2_len == want2_len) &&
|
|
!memcmp(have, want, have2_len)) {
|
|
g_enc_uninit_tail++;
|
|
if (g_enc_uninit_tail <= 5) {
|
|
static char desc[1 << 16];
|
|
describe_enc(desc, sizeof desc,
|
|
"outputs differ only with non-zeroed memory", &c, &ds,
|
|
len, src_len);
|
|
save_failure("enc-known", desc, src, src_len, &ds);
|
|
g_num_failures--;
|
|
}
|
|
dict_set_free(&ds);
|
|
continue;
|
|
}
|
|
have_len = have2_len;
|
|
want_len = want2_len;
|
|
size_t j = 0;
|
|
while ((j < have_len) && (j < want_len) && (have[j] == want[j])) {
|
|
j++;
|
|
}
|
|
char what[512];
|
|
snprintf(what, sizeof what,
|
|
"output mismatch: lengths %zu (wuffs) vs %zu (C), first "
|
|
"difference at %zu",
|
|
have_len, want_len, j);
|
|
save_enc_failure(what, &c, &ds, src, src_len, len);
|
|
dict_set_free(&ds);
|
|
continue;
|
|
}
|
|
if (c.short_workbuf) {
|
|
// One byte less than the shortest workbuf must fail.
|
|
size_t n2 = 0;
|
|
uint64_t wlen2 = 0;
|
|
const char* err2 = encode_wuffs(&c, &ds, src, src_len, len, have,
|
|
out_cap, &n2, 1, &wlen2);
|
|
if (err2 != (const char*)"no short workbuf") {
|
|
g_enc_short_checks++;
|
|
if (err2 != wuffs_base__error__bad_workbuf_length) {
|
|
char what[512];
|
|
snprintf(what, sizeof what,
|
|
"a workbuf of %" PRIu64 " bytes did not fail: %s", wlen2,
|
|
err2 ? err2 : "ok");
|
|
save_enc_failure(what, &c, &ds, src, src_len, len);
|
|
}
|
|
}
|
|
}
|
|
if (!stream_offset) {
|
|
g_enc_roundtrips++;
|
|
double tt3 = now_secs();
|
|
const char* err = roundtrip_dec(want, want_len, input, len, lw, &ds,
|
|
dec_out);
|
|
g_enc_time[2] += now_secs() - tt3;
|
|
if (err) {
|
|
save_enc_failure(err, &c, &ds, src, src_len, len);
|
|
}
|
|
}
|
|
dict_set_free(&ds);
|
|
}
|
|
|
|
printf("encoder campaign: seed=%" PRIu64 " q=%u-%u configs=%" PRIu64
|
|
" failures=%" PRIu64 " skipped(workbuf>3GiB)=%" PRIu64
|
|
" both_rejected=%" PRIu64 " both_flint_failed=%" PRIu64
|
|
" c_state_corrupted=%" PRIu64 " uninit_tail=%" PRIu64
|
|
" roundtrips=%" PRIu64 "\n",
|
|
g_seed, g_qmin, g_qmax, g_enc_configs, g_num_failures, g_enc_skipped,
|
|
g_enc_both_rejected, g_enc_both_flint_failed, g_enc_c_corrupted,
|
|
g_enc_uninit_tail, g_enc_roundtrips);
|
|
printf(" by quality:");
|
|
for (int q = 0; q < 12; q++) {
|
|
printf(" q%d=%" PRIu64, q, g_enc_by_quality[q]);
|
|
}
|
|
printf("\n input bytes=%" PRIu64 " lengths: <=64Ki=%" PRIu64
|
|
" <=256Ki=%" PRIu64 " <=1Mi=%" PRIu64 " <=4Mi=%" PRIu64 "\n",
|
|
g_enc_bytes_in, g_enc_len_hist[0], g_enc_len_hist[1],
|
|
g_enc_len_hist[2], g_enc_len_hist[3]);
|
|
printf(" with dicts=%" PRIu64 " (q10-11: %" PRIu64 ") large window=%" PRIu64 " metadata=%" PRIu64
|
|
" flushes=%" PRIu64 " dst limited=%" PRIu64 " stream offset=%" PRIu64
|
|
" workbuf above min=%" PRIu64 " short workbuf checks=%" PRIu64 "\n",
|
|
g_enc_with_dicts, g_enc_hq_with_dicts, g_enc_lw, g_enc_metadata, g_enc_flushes,
|
|
g_enc_dst_limited, g_enc_stream_offset, g_enc_wb_extra,
|
|
g_enc_short_checks);
|
|
printf(" seconds: C encoder %.1f, Wuffs encoder %.1f, round trips %.1f\n",
|
|
g_enc_time[0], g_enc_time[1], g_enc_time[2]);
|
|
return g_num_failures ? 1 : 0;
|
|
}
|
|
|
|
// ---------------- API edge cases
|
|
|
|
// run_steps_c and run_steps_wuffs run an arbitrary sequence of steps (unlike
|
|
// a plan, a FINISH step can be anywhere and be followed by other steps) and
|
|
// return the index of the first step that fails (or num_steps). A C step
|
|
// fails if BrotliEncoderCompressStream returns BROTLI_FALSE, a Wuffs step if
|
|
// request_flush, request_emit_metadata or transform_io returns an error.
|
|
static bool g_steps_c_corrupted;
|
|
|
|
static size_t //
|
|
run_steps_c(const enc_config* c, const uint8_t* src, uint8_t* out,
|
|
size_t out_cap, size_t* out_len) {
|
|
*out_len = 0;
|
|
g_steps_c_corrupted = false;
|
|
BrotliEncoderState* state = BrotliEncoderCreateInstance(NULL, NULL, NULL);
|
|
for (size_t j = 0; j < c->num_params; j++) {
|
|
BrotliEncoderSetParameter(state, (BrotliEncoderParameter)(c->keys[j]),
|
|
c->values[j]);
|
|
}
|
|
const uint8_t* next_in = src;
|
|
size_t i = 0;
|
|
for (; i < c->num_steps; i++) {
|
|
size_t avail_in = c->step_lens[i];
|
|
BrotliEncoderOperation op = (BrotliEncoderOperation)(c->step_ops[i]);
|
|
bool failed = false;
|
|
while (true) {
|
|
size_t avail_out = out_cap - *out_len;
|
|
if (c->dst_chunk && (avail_out > c->dst_chunk)) {
|
|
avail_out = c->dst_chunk;
|
|
}
|
|
size_t avail_out0 = avail_out;
|
|
uint8_t* next_out = out + *out_len;
|
|
if (!BrotliEncoderCompressStream(state, op, &avail_in, &next_in,
|
|
&avail_out, &next_out, NULL)) {
|
|
g_steps_c_corrupted = c_state_corrupted(state);
|
|
failed = true;
|
|
break;
|
|
}
|
|
*out_len += avail_out0 - avail_out;
|
|
if (op == BROTLI_OPERATION_FINISH) {
|
|
if (BrotliEncoderIsFinished(state)) {
|
|
break;
|
|
}
|
|
} else if ((avail_in == 0) && !BrotliEncoderHasMoreOutput(state)) {
|
|
break;
|
|
}
|
|
}
|
|
if (failed) {
|
|
break;
|
|
}
|
|
next_in += avail_in; // Skip any unconsumed input (there is none).
|
|
}
|
|
BrotliEncoderDestroyInstance(state);
|
|
return i;
|
|
}
|
|
|
|
static const char* g_steps_wuffs_err;
|
|
|
|
static size_t //
|
|
run_steps_wuffs(const enc_config* c, const uint8_t* src_ptr, size_t src_len,
|
|
uint8_t* out, size_t out_cap, size_t* out_len,
|
|
uint64_t zero_dst_rate) {
|
|
*out_len = 0;
|
|
g_steps_wuffs_err = NULL;
|
|
wuffs_brotli__encoder* enc = wuffs_brotli__encoder__alloc();
|
|
for (size_t j = 0; j < c->num_params; j++) {
|
|
wuffs_brotli__encoder__set_quirk(enc, quirk_key(c->keys[j]), c->values[j]);
|
|
}
|
|
uint64_t wlen = wuffs_brotli__encoder__workbuf_len(enc).min_incl;
|
|
if (wlen > ENC_WB_CAP) {
|
|
free(enc);
|
|
g_steps_wuffs_err = g_too_large;
|
|
return 0;
|
|
}
|
|
if (wlen > g_enc_wb_touched) {
|
|
g_enc_wb_touched = wlen;
|
|
}
|
|
wuffs_base__slice_u8 workbuf = wuffs_base__make_slice_u8(g_enc_wb, wlen);
|
|
wuffs_base__io_buffer dst = wuffs_base__ptr_u8__writer(out, out_cap);
|
|
wuffs_base__io_buffer src =
|
|
wuffs_base__ptr_u8__reader((uint8_t*)src_ptr, src_len, false);
|
|
src.meta.wi = 0;
|
|
size_t end = 0;
|
|
size_t i = 0;
|
|
for (; i < c->num_steps; i++) {
|
|
end += c->step_lens[i];
|
|
uint32_t op = c->step_ops[i];
|
|
wuffs_base__status status = wuffs_base__make_status(NULL);
|
|
if (op == OP_FLUSH) {
|
|
status = wuffs_brotli__encoder__request_flush(enc);
|
|
} else if (op == OP_EMIT_METADATA) {
|
|
status = wuffs_brotli__encoder__request_emit_metadata(enc,
|
|
c->step_lens[i]);
|
|
}
|
|
if (status.repr) {
|
|
g_steps_wuffs_err = status.repr;
|
|
break;
|
|
}
|
|
bool failed = false;
|
|
for (uint64_t calls = 0;; calls++) {
|
|
if (calls > (out_cap + src_len + 1000)) {
|
|
g_steps_wuffs_err = "too many transform_io calls";
|
|
failed = true;
|
|
break;
|
|
}
|
|
wuffs_base__io_buffer d = dst;
|
|
if (c->dst_chunk && ((d.data.len - d.meta.wi) > c->dst_chunk)) {
|
|
d.data.len = d.meta.wi + c->dst_chunk;
|
|
}
|
|
if (zero_dst_rate && chance(zero_dst_rate)) {
|
|
d.data.len = d.meta.wi;
|
|
}
|
|
wuffs_base__io_buffer s = src;
|
|
s.meta.wi = end;
|
|
s.meta.closed = (op == OP_FINISH);
|
|
status = wuffs_brotli__encoder__transform_io(enc, &d, &s, workbuf);
|
|
dst.meta.wi = d.meta.wi;
|
|
src.meta.ri = s.meta.ri;
|
|
if (status.repr == NULL) {
|
|
break;
|
|
} else if (status.repr == wuffs_base__suspension__short_write) {
|
|
continue;
|
|
} else if ((status.repr == wuffs_base__suspension__short_read) &&
|
|
(op != OP_FINISH) && (src.meta.ri == end)) {
|
|
break;
|
|
}
|
|
g_steps_wuffs_err = status.repr;
|
|
failed = true;
|
|
break;
|
|
}
|
|
if (failed) {
|
|
break;
|
|
}
|
|
src.meta.ri = end;
|
|
}
|
|
*out_len = dst.meta.wi;
|
|
free(enc);
|
|
return i;
|
|
}
|
|
|
|
static uint64_t g_api_checks;
|
|
static uint64_t g_api_both_failed_meta; // Too-long metadata.
|
|
static uint64_t g_api_both_failed_seq; // Bad call sequence.
|
|
static uint64_t g_api_c_corrupted; // See g_c_state_corrupted.
|
|
|
|
static void //
|
|
api_fail(const char* fmt, const char* a, uint64_t x, uint64_t y) {
|
|
char desc[1024];
|
|
snprintf(desc, sizeof desc, fmt, a, x, y);
|
|
strncat(desc, "\n", sizeof desc - strlen(desc) - 1);
|
|
save_failure("api", desc, NULL, 0, NULL);
|
|
}
|
|
|
|
// api_param_acceptance compares set_quirk's and BrotliEncoderSetParameter's
|
|
// acceptance of every parameter value (and of setting a parameter after the
|
|
// first call).
|
|
static void //
|
|
api_param_acceptance(void) {
|
|
static const uint32_t values[] = {0, 1, 2, 3,
|
|
4, 9, 10, 11,
|
|
12, 15, 16, 24,
|
|
25, 30, 31, 120,
|
|
121, 255, 1u << 24, 1u << 30,
|
|
(1u << 30) + 1, 0x7FFFFFFF, 0x80000000,
|
|
0xFFFFFFFE, 0xFFFFFFFF};
|
|
for (uint32_t param = 0; param <= 9; param++) {
|
|
for (size_t k = 0; k < sizeof values / sizeof values[0]; k++) {
|
|
g_api_checks++;
|
|
BrotliEncoderState* st = BrotliEncoderCreateInstance(NULL, NULL, NULL);
|
|
bool c_ok = BrotliEncoderSetParameter(st, (BrotliEncoderParameter)param,
|
|
values[k]);
|
|
BrotliEncoderDestroyInstance(st);
|
|
wuffs_brotli__encoder* enc = wuffs_brotli__encoder__alloc();
|
|
wuffs_base__status status =
|
|
wuffs_brotli__encoder__set_quirk(enc, quirk_key(param), values[k]);
|
|
bool w_ok = status.repr == NULL;
|
|
if (!w_ok && (status.repr != wuffs_base__error__bad_argument)) {
|
|
api_fail("set_quirk(param %s%" PRIu64 ", %" PRIu64 "): unexpected status",
|
|
"", param, values[k]);
|
|
}
|
|
if (c_ok != w_ok) {
|
|
api_fail("set_quirk(param %s%" PRIu64 ", %" PRIu64 "): acceptance differs",
|
|
"", param, values[k]);
|
|
}
|
|
// Values that do not fit in a u32.
|
|
status = wuffs_brotli__encoder__set_quirk(
|
|
enc, quirk_key(param), ((uint64_t)1 << 32) + values[k]);
|
|
if (status.repr != wuffs_base__error__bad_argument) {
|
|
api_fail("set_quirk(param %s%" PRIu64 ", (1<<32) + %" PRIu64
|
|
"): not rejected", "", param, values[k]);
|
|
}
|
|
free(enc);
|
|
}
|
|
// After the first call.
|
|
g_api_checks++;
|
|
BrotliEncoderState* st = BrotliEncoderCreateInstance(NULL, NULL, NULL);
|
|
size_t ai = 0, ao = 0;
|
|
const uint8_t* ni = NULL;
|
|
uint8_t* no = NULL;
|
|
BrotliEncoderCompressStream(st, BROTLI_OPERATION_PROCESS, &ai, &ni, &ao,
|
|
&no, NULL);
|
|
bool c_ok = BrotliEncoderSetParameter(st, (BrotliEncoderParameter)param, 0);
|
|
BrotliEncoderDestroyInstance(st);
|
|
wuffs_brotli__encoder* enc = wuffs_brotli__encoder__alloc();
|
|
wuffs_brotli__encoder__set_quirk(enc, WUFFS_BROTLI__QUIRK_ENCODER_QUALITY,
|
|
5);
|
|
wuffs_brotli__encoder__set_quirk(enc, WUFFS_BROTLI__QUIRK_LGWIN, 10);
|
|
uint64_t wlen = wuffs_brotli__encoder__workbuf_len(enc).max_incl;
|
|
uint8_t one = 0;
|
|
wuffs_base__io_buffer d = wuffs_base__ptr_u8__writer(&one, 0);
|
|
wuffs_base__io_buffer s = wuffs_base__ptr_u8__reader(&one, 0, false);
|
|
wuffs_brotli__encoder__transform_io(enc, &d, &s,
|
|
wuffs_base__make_slice_u8(g_enc_wb, wlen));
|
|
wuffs_base__status status =
|
|
wuffs_brotli__encoder__set_quirk(enc, quirk_key(param), 0);
|
|
if (c_ok || (status.repr != wuffs_base__error__bad_call_sequence)) {
|
|
api_fail("set_quirk after the first call (param %s%" PRIu64
|
|
"): C %" PRIu64 ", Wuffs differs", "", param, c_ok);
|
|
}
|
|
free(enc);
|
|
}
|
|
// An unknown parameter.
|
|
g_api_checks++;
|
|
BrotliEncoderState* st = BrotliEncoderCreateInstance(NULL, NULL, NULL);
|
|
bool c_ok = BrotliEncoderSetParameter(st, (BrotliEncoderParameter)10, 0);
|
|
BrotliEncoderDestroyInstance(st);
|
|
wuffs_brotli__encoder* enc = wuffs_brotli__encoder__alloc();
|
|
wuffs_base__status status = wuffs_brotli__encoder__set_quirk(
|
|
enc, WUFFS_BROTLI__QUIRK_STREAM_OFFSET + 1, 0);
|
|
if (c_ok || (status.repr != wuffs_base__error__unsupported_option)) {
|
|
api_fail("unknown parameter: %sC %" PRIu64 " %" PRIu64, "", c_ok, 0);
|
|
}
|
|
free(enc);
|
|
}
|
|
|
|
// api_compare_steps runs c's steps with both libraries and compares the
|
|
// failing step and the output before it.
|
|
static void //
|
|
api_compare_steps(const char* what, const enc_config* c, const uint8_t* src,
|
|
size_t src_len, uint8_t* have, uint8_t* want, size_t cap,
|
|
uint64_t zero_dst_rate) {
|
|
g_api_checks++;
|
|
size_t want_len = 0;
|
|
size_t have_len = 0;
|
|
size_t cf = run_steps_c(c, src, want, cap, &want_len);
|
|
size_t wf = run_steps_wuffs(c, src, src_len, have, cap, &have_len,
|
|
zero_dst_rate);
|
|
if (g_steps_wuffs_err == g_too_large) {
|
|
return;
|
|
}
|
|
bool bad = (cf != wf);
|
|
if (!bad && (cf < c->num_steps)) {
|
|
// Which error Wuffs returns.
|
|
uint32_t op = c->step_ops[cf];
|
|
const char* e = g_steps_wuffs_err;
|
|
bool meta_too_long = (op == OP_EMIT_METADATA) &&
|
|
(c->step_lens[cf] > (1u << 24));
|
|
if (meta_too_long) {
|
|
g_api_both_failed_meta++;
|
|
} else {
|
|
g_api_both_failed_seq++;
|
|
}
|
|
if (meta_too_long ? (e != wuffs_base__error__bad_argument)
|
|
: (e != wuffs_base__error__bad_call_sequence)) {
|
|
bad = true;
|
|
}
|
|
}
|
|
if (!bad) {
|
|
// C's output on failure is whatever it wrote before the failing call.
|
|
if ((have_len != want_len) || memcmp(have, want, have_len)) {
|
|
bad = true;
|
|
}
|
|
}
|
|
if (bad && g_steps_c_corrupted && (cf < wf) && (want_len <= have_len) &&
|
|
!memcmp(have, want, want_len)) {
|
|
// See g_c_state_corrupted.
|
|
g_api_c_corrupted++;
|
|
return;
|
|
}
|
|
if (bad) {
|
|
static char desc[1 << 16];
|
|
char what2[512];
|
|
snprintf(what2, sizeof what2,
|
|
"%s: failing step C %zu, Wuffs %zu (%s); output lengths C %zu, "
|
|
"Wuffs %zu",
|
|
what, cf, wf, g_steps_wuffs_err ? g_steps_wuffs_err : "",
|
|
want_len, have_len);
|
|
dict_set ds = {0};
|
|
describe_enc(desc, sizeof desc, what2, c, &ds, 0, src_len);
|
|
save_failure("api", desc, src, src_len < (1 << 20) ? src_len : 0, NULL);
|
|
}
|
|
}
|
|
|
|
// api_metadata checks metadata blocks of 0, 1, (1 << 24) - 1, 1 << 24 and
|
|
// (1 << 24) + 1 bytes (the last must be rejected), at the start, after some
|
|
// data, after a flush and after the end of the stream.
|
|
static void //
|
|
api_metadata(uint8_t* src, uint8_t* have, uint8_t* want, size_t cap) {
|
|
static const size_t lens[] = {0, 1, (1u << 24) - 1, 1u << 24,
|
|
(1u << 24) + 1};
|
|
static const uint32_t qs[] = {0, 1, 2, 4, 5, 9, 10, 11};
|
|
for (size_t qi = 0; qi < sizeof qs / sizeof qs[0]; qi++) {
|
|
for (size_t li = 0; li < 5; li++) {
|
|
for (int where = 0; where < 4; where++) {
|
|
enc_config c;
|
|
memset(&c, 0, sizeof c);
|
|
enc_add_param(&c, 1, qs[qi]);
|
|
enc_add_param(&c, 2, 16);
|
|
size_t n = 0;
|
|
size_t data = 3000;
|
|
gen_segment(src, data, 0, 3);
|
|
n += data;
|
|
if (where == 0) {
|
|
enc_add_step(&c, OP_EMIT_METADATA, lens[li]);
|
|
enc_add_step(&c, OP_FINISH, data);
|
|
} else if (where == 1) {
|
|
enc_add_step(&c, OP_PROCESS, data);
|
|
enc_add_step(&c, OP_EMIT_METADATA, lens[li]);
|
|
enc_add_step(&c, OP_FINISH, 0);
|
|
} else if (where == 2) {
|
|
enc_add_step(&c, OP_FLUSH, data);
|
|
enc_add_step(&c, OP_EMIT_METADATA, lens[li]);
|
|
enc_add_step(&c, OP_FINISH, 0);
|
|
} else {
|
|
enc_add_step(&c, OP_FINISH, data);
|
|
enc_add_step(&c, OP_EMIT_METADATA, lens[li]);
|
|
}
|
|
size_t at = (where == 0) ? 0 : data;
|
|
memmove(src + at + lens[li], src + at, n - at);
|
|
memset(src + at, 0xA5, lens[li]);
|
|
n += lens[li];
|
|
char what[128];
|
|
snprintf(what, sizeof what, "metadata q=%u len=%zu where=%d", qs[qi],
|
|
lens[li], where);
|
|
api_compare_steps(what, &c, src, n, have, want, cap, 0);
|
|
free(c.step_ops);
|
|
free(c.step_lens);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// api_dictionaries checks add_dictionary's edge cases against
|
|
// BrotliEncoderPrepareDictionary and BrotliEncoderAttachPreparedDictionary.
|
|
static void //
|
|
api_dictionaries(void) {
|
|
static const uint8_t bytes[] = "some dictionary bytes, some dictionary bytes";
|
|
// Up to 16 dictionaries (the 16th must be rejected by both), of various
|
|
// types.
|
|
for (uint32_t type = 0; type < 4; type++) {
|
|
g_api_checks++;
|
|
BrotliEncoderState* st = BrotliEncoderCreateInstance(NULL, NULL, NULL);
|
|
wuffs_brotli__encoder* enc = wuffs_brotli__encoder__alloc();
|
|
BrotliEncoderPreparedDictionary* pd[17] = {0};
|
|
for (int k = 0; k < 17; k++) {
|
|
size_t len = (k * 7) % (sizeof bytes);
|
|
uint32_t t = (k == 3) ? type : 0;
|
|
pd[k] = BrotliEncoderPrepareDictionary((BrotliSharedDictionaryType)t,
|
|
len, bytes, 11, NULL, NULL, NULL);
|
|
bool c_ok = pd[k] && BrotliEncoderAttachPreparedDictionary(st, pd[k]);
|
|
uint64_t wl =
|
|
wuffs_brotli__encoder__add_dictionary_workbuf_len(enc, len);
|
|
wuffs_base__status status = wuffs_brotli__encoder__add_dictionary(
|
|
enc, t, wuffs_base__make_slice_u8((uint8_t*)bytes, len),
|
|
wuffs_base__make_slice_u8(g_enc_wb, wl));
|
|
const char* want_status =
|
|
c_ok ? NULL
|
|
: (t == 1) ? wuffs_brotli__error__unsupported_serialized_encoder_dictionary
|
|
: (t > 1) ? wuffs_base__error__bad_argument
|
|
: wuffs_brotli__error__too_many_compound_dictionaries;
|
|
if (status.repr != want_status) {
|
|
api_fail("add_dictionary #%s%" PRIu64 " (type %" PRIu64 "): differs", "",
|
|
(uint64_t)k, t);
|
|
}
|
|
if (wl > g_enc_wb_touched) {
|
|
g_enc_wb_touched = wl;
|
|
}
|
|
}
|
|
// A too-short workbuf.
|
|
{
|
|
wuffs_brotli__encoder* e2 = wuffs_brotli__encoder__alloc();
|
|
uint64_t wl = wuffs_brotli__encoder__add_dictionary_workbuf_len(e2, 10);
|
|
wuffs_base__status status = wuffs_brotli__encoder__add_dictionary(
|
|
e2, 0, wuffs_base__make_slice_u8((uint8_t*)bytes, 10),
|
|
wuffs_base__make_slice_u8(g_enc_wb, wl - 1));
|
|
if (status.repr != wuffs_base__error__bad_workbuf_length) {
|
|
api_fail("add_dictionary with a short workbuf: %s%" PRIu64 "%" PRIu64,
|
|
status.repr ? status.repr : "ok", 0, 0);
|
|
}
|
|
free(e2);
|
|
}
|
|
BrotliEncoderDestroyInstance(st);
|
|
for (int k = 0; k < 17; k++) {
|
|
if (pd[k]) {
|
|
BrotliEncoderDestroyPreparedDictionary(pd[k]);
|
|
}
|
|
}
|
|
free(enc);
|
|
}
|
|
// A transform_io workbuf shorter than the dictionaries.
|
|
g_api_checks++;
|
|
{
|
|
wuffs_brotli__encoder* enc = wuffs_brotli__encoder__alloc();
|
|
uint64_t wl = wuffs_brotli__encoder__add_dictionary_workbuf_len(enc, 40);
|
|
wuffs_brotli__encoder__add_dictionary(
|
|
enc, 0, wuffs_base__make_slice_u8((uint8_t*)bytes, 40),
|
|
wuffs_base__make_slice_u8(g_enc_wb, wl));
|
|
uint64_t d_end = wuffs_brotli__encoder__workbuf_len(enc).min_incl;
|
|
uint8_t one = 0;
|
|
wuffs_base__io_buffer d = wuffs_base__ptr_u8__writer(&one, 1);
|
|
wuffs_base__io_buffer s = wuffs_base__ptr_u8__reader(&one, 0, true);
|
|
(void)d_end;
|
|
wuffs_base__status status = wuffs_brotli__encoder__transform_io(
|
|
enc, &d, &s, wuffs_base__make_slice_u8(g_enc_wb, 50));
|
|
if (status.repr != wuffs_base__error__bad_workbuf_length) {
|
|
api_fail("transform_io with a workbuf shorter than the dictionaries: %s%" PRIu64
|
|
"%" PRIu64, status.repr ? status.repr : "ok", 0, 0);
|
|
}
|
|
// add_dictionary after transform_io (the C code accepts this, see the
|
|
// README).
|
|
status = wuffs_brotli__encoder__add_dictionary(
|
|
enc, 0, wuffs_base__make_slice_u8((uint8_t*)bytes, 4),
|
|
wuffs_base__make_slice_u8(g_enc_wb, wl * 2));
|
|
if (status.repr != NULL) {
|
|
// The failed transform_io did not start compressing.
|
|
}
|
|
free(enc);
|
|
}
|
|
}
|
|
|
|
// api_short_workbuf_q01 checks that, at qualities 0 and 1, an input block of
|
|
// n bytes needs exactly (2 * n) + 519 bytes of workbuf.
|
|
static void //
|
|
api_short_workbuf_q01(uint8_t* src, uint8_t* out, size_t cap) {
|
|
static const size_t ns[] = {0, 1, 2, 100, 1000, 65536, 100000};
|
|
for (uint32_t q = 0; q <= 1; q++) {
|
|
for (size_t k = 0; k < sizeof ns / sizeof ns[0]; k++) {
|
|
size_t n = ns[k];
|
|
gen_segment(src, n, 0, 3);
|
|
for (int delta = -1; delta <= 0; delta++) {
|
|
g_api_checks++;
|
|
wuffs_brotli__encoder* enc = wuffs_brotli__encoder__alloc();
|
|
wuffs_brotli__encoder__set_quirk(enc, WUFFS_BROTLI__QUIRK_ENCODER_QUALITY,
|
|
q);
|
|
uint64_t wl = (2 * n) + 519 + delta;
|
|
wuffs_base__io_buffer d = wuffs_base__ptr_u8__writer(out, cap);
|
|
wuffs_base__io_buffer s = wuffs_base__ptr_u8__reader(src, n, true);
|
|
wuffs_base__status status = wuffs_base__make_status(NULL);
|
|
for (int calls = 0; calls < 100; calls++) {
|
|
status = wuffs_brotli__encoder__transform_io(
|
|
enc, &d, &s, wuffs_base__make_slice_u8(g_enc_wb, wl));
|
|
if (status.repr != wuffs_base__suspension__short_write) {
|
|
break;
|
|
}
|
|
}
|
|
bool want_ok = delta == 0;
|
|
if (want_ok ? (status.repr != NULL)
|
|
: (status.repr != wuffs_base__error__bad_workbuf_length)) {
|
|
api_fail("q0/1 workbuf for a block of n bytes: %s (n=%" PRIu64
|
|
" delta=%" PRIu64 ")",
|
|
status.repr ? status.repr : "ok", n, (uint64_t)(-delta));
|
|
}
|
|
free(enc);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// random_steps makes a random sequence of steps, where FINISH can be
|
|
// anywhere and metadata can be of odd lengths.
|
|
static size_t //
|
|
random_steps(enc_config* c, uint8_t* src, size_t cap) {
|
|
size_t n = 0;
|
|
size_t k = 1 + rn(chance(800) ? 6 : 30);
|
|
for (size_t i = 0; i < k; i++) {
|
|
uint32_t op = (uint32_t)rn(4);
|
|
size_t len = 0;
|
|
if (op == OP_EMIT_METADATA) {
|
|
uint64_t r = rn(1000);
|
|
len = (r < 300) ? 0 : (r < 500) ? 1 : (r < 990) ? rn(300)
|
|
: ((1 << 24) - 1 + rn(3));
|
|
} else if (!chance(300)) {
|
|
len = rn(chance(900) ? 200 : 70000);
|
|
}
|
|
if ((n + len) > cap) {
|
|
len = 0;
|
|
}
|
|
gen_input(src + n, len);
|
|
n += len;
|
|
enc_add_step(c, op, len);
|
|
}
|
|
return n;
|
|
}
|
|
|
|
static int //
|
|
run_api(uint64_t iters, double secs) {
|
|
load_corpus();
|
|
g_enc_wb = big_alloc(ENC_WB_CAP);
|
|
const size_t cap = (48 << 20);
|
|
uint8_t* src = xmalloc(cap);
|
|
uint8_t* have = xmalloc(cap + (1 << 20));
|
|
uint8_t* want = xmalloc(cap + (1 << 20));
|
|
|
|
api_param_acceptance();
|
|
api_dictionaries();
|
|
api_short_workbuf_q01(src, have, cap);
|
|
api_metadata(src, have, want, cap);
|
|
uint64_t fixed = g_api_checks;
|
|
|
|
// Empty input (an immediately closed src) at every quality and some
|
|
// lgwins, with and without large window.
|
|
for (uint32_t q = 0; q <= 11; q++) {
|
|
for (uint32_t lgwin = 10; lgwin <= 30; lgwin += 4) {
|
|
for (uint32_t lw = 0; lw <= 1; lw++) {
|
|
enc_config c;
|
|
memset(&c, 0, sizeof c);
|
|
enc_add_param(&c, 1, q);
|
|
enc_add_param(&c, 2, lgwin);
|
|
enc_add_param(&c, 6, lw);
|
|
enc_add_step(&c, OP_FINISH, 0);
|
|
api_compare_steps("empty input", &c, src, 0, have, want, cap, 300);
|
|
free(c.step_ops);
|
|
free(c.step_lens);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Random sequences of steps.
|
|
struct timespec t0;
|
|
clock_gettime(CLOCK_MONOTONIC, &t0);
|
|
uint64_t it = 0;
|
|
for (; it < iters; it++) {
|
|
if ((it & 15) == 0) {
|
|
struct timespec t1;
|
|
clock_gettime(CLOCK_MONOTONIC, &t1);
|
|
if ((secs > 0) && (((double)(t1.tv_sec - t0.tv_sec)) > secs)) {
|
|
break;
|
|
}
|
|
}
|
|
if (g_enc_wb_touched > ((size_t)512 << 20)) {
|
|
big_release(g_enc_wb, ENC_WB_CAP);
|
|
g_enc_wb_touched = 0;
|
|
}
|
|
enc_config c;
|
|
memset(&c, 0, sizeof c);
|
|
uint32_t q = pick_quality();
|
|
enc_add_param(&c, 1, q);
|
|
enc_add_param(&c, 2, 10 + (uint32_t)rn(15));
|
|
bool stream_offset = chance(200);
|
|
if (stream_offset) {
|
|
enc_add_param(&c, 9, (uint32_t)rn(1000));
|
|
}
|
|
c.dst_chunk = chance(300) ? 1 + rn(100) : 0;
|
|
size_t n = random_steps(&c, src, cap);
|
|
// The zero-length dst calls are Wuffs only (run_steps_c gives every C
|
|
// call some dst). A Wuffs call with no dst room while the stream offset
|
|
// flush is pending is like a C call with avail_out == 0 there, after
|
|
// which the next call with input fails in both libraries (see "Encoder
|
|
// Operation Mapping" in std/brotli/README.md), so they are not used with
|
|
// a stream offset.
|
|
uint64_t zero_dst_rate = chance(300) ? 300 : 0;
|
|
api_compare_steps("random steps", &c, src, n, have, want, cap,
|
|
stream_offset ? 0 : zero_dst_rate);
|
|
free(c.step_ops);
|
|
free(c.step_lens);
|
|
}
|
|
printf("api campaign: seed=%" PRIu64 " fixed checks=%" PRIu64
|
|
" random step sequences=%" PRIu64 " total checks=%" PRIu64
|
|
" failures=%" PRIu64 "\n",
|
|
g_seed, fixed, it, g_api_checks, g_num_failures);
|
|
printf(" both rejected: too-long metadata=%" PRIu64
|
|
", bad call sequence=%" PRIu64 "; C state corrupted=%" PRIu64 "\n",
|
|
g_api_both_failed_meta, g_api_both_failed_seq, g_api_c_corrupted);
|
|
return g_num_failures ? 1 : 0;
|
|
}
|