diff --git a/fuzz/c/std/brotli_fuzzer.c b/fuzz/c/std/brotli_fuzzer.c index 7095de15..2fa7228e 100644 --- a/fuzz/c/std/brotli_fuzzer.c +++ b/fuzz/c/std/brotli_fuzzer.c @@ -26,6 +26,11 @@ gcc -DWUFFS_CONFIG__FUZZLIB_MAIN brotli_fuzzer.c rm -f ./a.out It should print "PASS", amongst other information, and exit(0). + +As well as decoding its input (with or without the large-window quirk and +dictionaries), it also encodes the input (with encoder parameters, chunking, +flushes, metadata blocks and dst limits derived from the input's hash), +decodes the result and checks that the round trip reproduces the input. */ #pragma clang diagnostic pop @@ -74,12 +79,17 @@ It should print "PASS", amongst other information, and exit(0). // Wuffs allows either statically or dynamically allocated work buffers. This // program exercises static allocation. // -// We hard-code 64 MiB, as WUFFS_BROTLI__DECODER_WORKBUF_LEN_MAX_INCL_WORST_CASE +// We hard-code 128 MiB, as WUFFS_BROTLI__DECODER_WORKBUF_LEN_MAX_INCL_WORST_CASE // is over 3 GiB and even the large-window workbuf_len is over 1 GiB. Using // those can cause "relocation truncated to fit: R_X86_64_PC32 against symbol -// etc" compiler errors. Large-window streams with a window of 64 MiB or more +// etc" compiler errors. Large-window streams with a window of 128 MiB or more // are rejected with "#base: bad workbuf length", but smaller ones are fine. -#define WORK_BUFFER_ARRAY_SIZE 67108864 +// +// The encoder (see fuzz_encoder) uses the same work buffer, after decoding. +// Its workbuf_len depends on the parameters: about 92 MiB at qualities 10 and +// 11 with lgwin 16 (and less at lower qualities), so encoder parameters whose +// workbuf_len is larger are scaled down. +#define WORK_BUFFER_ARRAY_SIZE 134217728 #if WORK_BUFFER_ARRAY_SIZE > 0 uint8_t g_work_buffer_array[WORK_BUFFER_ARRAY_SIZE]; #else @@ -115,8 +125,257 @@ static const uint8_t g_serialized_dictionary[] = { 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, }; +// ---------------- Encoder round trip + +// The encoder's input is capped (a prefix of the fuzz input), as is the input +// at qualities 10 and 11, which are slow. +#define ENC_MAX_INPUT 262144 +#define ENC_MAX_INPUT_HQ 32768 +#define ENC_DST_ARRAY_SIZE (2 * ENC_MAX_INPUT + 65536) + +static wuffs_brotli__encoder g_encoder; +static wuffs_brotli__decoder g_rt_decoder; +static uint8_t g_enc_dst[ENC_DST_ARRAY_SIZE]; +static uint8_t g_enc_want[ENC_MAX_INPUT]; // The input minus any metadata. +static uint8_t g_enc_have[ENC_MAX_INPUT + 1]; + +static uint64_t // +mix64(uint64_t x) { + x ^= x >> 31; + x *= 0x7FB5D329728EA185ull; + x ^= x >> 27; + x *= 0x81DADEF4BC2DD44Dull; + x ^= x >> 33; + return x; +} + +static void // +fail_encoder(const char* what, const char* status) { + fprintf(stderr, "brotli encoder round trip: %s: %s\n", what, + status ? status : "ok"); + intentional_segfault(); +} + +// fuzz_encoder encodes data[0 .. n] with parameters derived from hash and +// checks that decoding the result gives back the input (minus the bytes used +// as metadata). +static void // +fuzz_encoder(const uint8_t* data, size_t n, uint64_t hash) { + uint64_t h = mix64(hash ^ 0x5DEECE66Dull); + uint64_t h2 = mix64(h); + uint32_t quality = (uint32_t)((h >> 0) % 12); + uint32_t lgwin = 10 + (uint32_t)((h >> 4) % 15); + bool large_window = ((h >> 8) & 7) == 0; + bool use_dict = ((h >> 11) & 3) == 0; + if (large_window && (quality >= 3)) { + lgwin = 10 + (uint32_t)((h >> 4) % 17); // Up to 26. + } + if (quality >= 10) { + if (n > ENC_MAX_INPUT_HQ) { + n = ENC_MAX_INPUT_HQ; + } + // Compound dictionaries are not supported at qualities 10 and 11 yet. + use_dict = false; + } else if (n > ENC_MAX_INPUT) { + n = ENC_MAX_INPUT; + } + + wuffs_base__status status = wuffs_brotli__encoder__initialize( + &g_encoder, sizeof g_encoder, WUFFS_VERSION, + (hash & 1) ? WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED : 0); + if (!wuffs_base__status__is_ok(&status)) { + fail_encoder("initialize", status.repr); + } + wuffs_base__slice_u8 workbuf = + wuffs_base__make_slice_u8(g_work_buffer_array, WORK_BUFFER_ARRAY_SIZE); + if (use_dict) { + status = wuffs_brotli__encoder__add_dictionary( + &g_encoder, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, + wuffs_base__make_slice_u8((uint8_t*)g_raw_dictionary, + sizeof g_raw_dictionary - 1), + workbuf); + if (!wuffs_base__status__is_ok(&status)) { + fail_encoder("add_dictionary", status.repr); + } + } + + // Set the parameters. Each has a (mostly) default value for some inputs. + struct { + uint32_t key; + uint64_t value; + bool enabled; + } quirks[] = { + {WUFFS_BROTLI__QUIRK_ENCODER_QUALITY, quality, true}, + {WUFFS_BROTLI__QUIRK_LGWIN, lgwin, true}, + {WUFFS_BROTLI__QUIRK_LARGE_WINDOW, 1, large_window}, + {WUFFS_BROTLI__QUIRK_MODE, (h >> 13) % 3, ((h >> 15) & 1) != 0}, + {WUFFS_BROTLI__QUIRK_LGBLOCK, 16 + ((h >> 16) % 9), + ((h >> 20) & 3) == 0}, + {WUFFS_BROTLI__QUIRK_DISABLE_LITERAL_CONTEXT_MODELING, 1, + ((h >> 22) & 3) == 0}, + {WUFFS_BROTLI__QUIRK_SIZE_HINT, ((h >> 24) & 1) ? n : (h >> 32), + ((h >> 25) & 1) != 0}, + {WUFFS_BROTLI__QUIRK_NPOSTFIX, (h >> 26) & 3, ((h >> 28) & 3) == 0}, + {WUFFS_BROTLI__QUIRK_NDIRECT, ((h >> 30) & 15) << ((h >> 26) & 3), + ((h >> 28) & 3) == 0}, + }; + for (size_t i = 0; i < sizeof quirks / sizeof quirks[0]; i++) { + if (quirks[i].enabled) { + status = wuffs_brotli__encoder__set_quirk(&g_encoder, quirks[i].key, + quirks[i].value); + if (!wuffs_base__status__is_ok(&status)) { + fail_encoder("set_quirk", status.repr); + } + } + } + // Scale the parameters down until the workbuf fits. + bool lgblock_set = quirks[4].enabled; + while (wuffs_brotli__encoder__workbuf_len(&g_encoder).min_incl > + WORK_BUFFER_ARRAY_SIZE) { + if (lgwin > 16) { + lgwin = 16; + wuffs_brotli__encoder__set_quirk(&g_encoder, WUFFS_BROTLI__QUIRK_LGWIN, + lgwin); + } else if (lgblock_set) { + lgblock_set = false; + wuffs_brotli__encoder__set_quirk(&g_encoder, WUFFS_BROTLI__QUIRK_LGBLOCK, + 0); + } else if (quality > 9) { + quality = 9; + wuffs_brotli__encoder__set_quirk( + &g_encoder, WUFFS_BROTLI__QUIRK_ENCODER_QUALITY, quality); + } else { + fail_encoder("workbuf_len is too large", NULL); + } + } + wuffs_base__range_ii_u64 wlen = wuffs_brotli__encoder__workbuf_len(&g_encoder); + uint64_t wl = wlen.min_incl; + if ((h2 & 1) && (wlen.max_incl > wl)) { + wl = (wlen.max_incl < WORK_BUFFER_ARRAY_SIZE) ? wlen.max_incl + : WORK_BUFFER_ARRAY_SIZE; + } + workbuf.len = (size_t)wl; + + // The plan: chunks of up to chunk_max bytes, a flush after some of them and + // some metadata blocks (whose contents are input bytes). + size_t chunk_max = ((h2 >> 1) & 1) ? n : (1 + ((h2 >> 2) % 4096)); + uint32_t flush_mask = (uint32_t)((h2 >> 14) & 7); // Flush if (i & 7) < it. + uint32_t meta_mask = (uint32_t)((h2 >> 17) & 15); // Metadata if == 15. + size_t dst_limit = ((h2 >> 21) & 3) ? SIZE_MAX : (1 + ((h2 >> 23) % 64)); + uint64_t rng = h2; + + wuffs_base__io_buffer dst = + wuffs_base__ptr_u8__writer(g_enc_dst, ENC_DST_ARRAY_SIZE); + wuffs_base__io_buffer src = + wuffs_base__ptr_u8__reader((uint8_t*)data, n, false); + src.meta.wi = 0; + size_t want_len = 0; + for (uint32_t i = 0; src.meta.wi < n; i++) { + rng = mix64(rng); + size_t k = 1 + (size_t)(rng % chunk_max); + if (k > (n - src.meta.wi)) { + k = n - src.meta.wi; + } + bool meta = (meta_mask == 15) && ((rng >> 40) & 1) && (k <= 600); + bool last = (src.meta.wi + k) == n; + status = wuffs_base__make_status(NULL); + if (meta) { + status = wuffs_brotli__encoder__request_emit_metadata(&g_encoder, k); + } else if (!last && ((i & 7) < flush_mask)) { + status = wuffs_brotli__encoder__request_flush(&g_encoder); + } + if (!wuffs_base__status__is_ok(&status)) { + fail_encoder("request", status.repr); + } + if (!meta) { + memcpy(g_enc_want + want_len, data + src.meta.wi, k); + want_len += k; + } + src.meta.wi += k; + src.meta.closed = last && !meta; + while (true) { + wuffs_base__io_buffer d = dst; + if ((d.data.len - d.meta.wi) > dst_limit) { + d.data.len = d.meta.wi + dst_limit; + } + status = wuffs_brotli__encoder__transform_io(&g_encoder, &d, &src, + workbuf); + dst.meta.wi = d.meta.wi; + if (status.repr == wuffs_base__suspension__short_write) { + if (dst.meta.wi == dst.data.len) { + fail_encoder("dst is full", NULL); + } + continue; + } else if ((status.repr == wuffs_base__suspension__short_read) && + !src.meta.closed && (src.meta.ri == src.meta.wi)) { + break; + } else if (wuffs_base__status__is_ok(&status) && src.meta.closed) { + break; + } + fail_encoder("transform_io", status.repr); + } + } + if (!src.meta.closed) { + // The input ended with a metadata block (or is empty): finish. + src.meta.closed = true; + while (true) { + wuffs_base__io_buffer d = dst; + if ((d.data.len - d.meta.wi) > dst_limit) { + d.data.len = d.meta.wi + dst_limit; + } + status = wuffs_brotli__encoder__transform_io(&g_encoder, &d, &src, + workbuf); + dst.meta.wi = d.meta.wi; + if (status.repr == wuffs_base__suspension__short_write) { + continue; + } else if (wuffs_base__status__is_ok(&status)) { + break; + } + fail_encoder("transform_io (finish)", status.repr); + } + } + + // Decode it again. + wuffs_brotli__decoder* dec = &g_rt_decoder; + status = wuffs_brotli__decoder__initialize(dec, sizeof *dec, WUFFS_VERSION, + 0); + if (!wuffs_base__status__is_ok(&status)) { + fail_encoder("decoder initialize", status.repr); + } + wuffs_brotli__decoder__set_quirk(dec, WUFFS_BROTLI__QUIRK_LARGE_WINDOW, + large_window); + workbuf.len = WORK_BUFFER_ARRAY_SIZE; + if (use_dict) { + status = wuffs_brotli__decoder__add_dictionary( + dec, WUFFS_BROTLI__DICTIONARY_TYPE_RAW, + wuffs_base__make_slice_u8((uint8_t*)g_raw_dictionary, + sizeof g_raw_dictionary - 1), + workbuf); + if (!wuffs_base__status__is_ok(&status)) { + fail_encoder("decoder add_dictionary", status.repr); + } + } + wuffs_base__io_buffer have = + wuffs_base__ptr_u8__writer(g_enc_have, sizeof g_enc_have); + wuffs_base__io_buffer enc_src = + wuffs_base__ptr_u8__reader(g_enc_dst, dst.meta.wi, true); + status = wuffs_brotli__decoder__transform_io(dec, &have, &enc_src, workbuf); + if (!wuffs_base__status__is_ok(&status)) { + fail_encoder("decoder transform_io", status.repr); + } else if ((have.meta.wi != want_len) || + memcmp(g_enc_have, g_enc_want, want_len) || + (enc_src.meta.ri != enc_src.meta.wi)) { + fail_encoder("round trip mismatch", NULL); + } +} + +// ---------------- Decoder + const char* // fuzz(wuffs_base__io_buffer* src, uint64_t hash) { + fuzz_encoder(wuffs_base__io_buffer__reader_pointer(src), + wuffs_base__io_buffer__reader_length(src), hash); + wuffs_brotli__decoder dec; wuffs_base__status status = wuffs_brotli__decoder__initialize( &dec, sizeof dec, WUFFS_VERSION, diff --git a/fuzz/c/std/seed_corpora.txt b/fuzz/c/std/seed_corpora.txt index 4499525c..046fd35d 100644 --- a/fuzz/c/std/seed_corpora.txt +++ b/fuzz/c/std/seed_corpora.txt @@ -15,7 +15,7 @@ # https://github.com/google/oss-fuzz/blob/master/projects/wuffs/Dockerfile bmp: test/data/*.bmp ../bmpsuite_corpus/*.bmp -brotli: test/data/brotli/*.br test/data/brotli/*.compressed* +brotli: test/data/brotli/*.br test/data/brotli/*.compressed* test/data/brotli/*.txt test/data/*.txt bzip2: test/data/*.bz2 cbor: test/data/*.cbor gif: test/data/*.gif test/data/artificial-gif/*.gif