diff --git a/fuzz/c/std/cbor_fuzzer.c b/fuzz/c/std/cbor_fuzzer.c new file mode 100644 index 00000000..fe02c202 --- /dev/null +++ b/fuzz/c/std/cbor_fuzzer.c @@ -0,0 +1,410 @@ +// Copyright 2020 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// ---------------- + +// Silence the nested slash-star warning for the next comment's command line. +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wcomment" + +/* +This fuzzer (the fuzz function) is typically run indirectly, by a framework +such as https://github.com/google/oss-fuzz calling LLVMFuzzerTestOneInput. + +When working on the fuzz implementation, or as a sanity check, defining +WUFFS_CONFIG__FUZZLIB_MAIN will let you manually run fuzz over a set of files: + +gcc -DWUFFS_CONFIG__FUZZLIB_MAIN cbor_fuzzer.c +./a.out ../../../test/data/*.cbor +rm -f ./a.out + +It should print "PASS", amongst other information, and exit(0). +*/ + +#pragma clang diagnostic pop + +// Wuffs ships as a "single file C library" or "header file library" as per +// https://github.com/nothings/stb/blob/master/docs/stb_howto.txt +// +// To use that single file as a "foo.c"-like implementation, instead of a +// "foo.h"-like header, #define WUFFS_IMPLEMENTATION before #include'ing or +// compiling it. +#define WUFFS_IMPLEMENTATION + +// Defining the WUFFS_CONFIG__MODULE* macros are optional, but it lets users of +// release/c/etc.c whitelist which parts of Wuffs to build. That file contains +// the entire Wuffs standard library, implementing a variety of codecs and file +// formats. Without this macro definition, an optimizing compiler or linker may +// very well discard Wuffs code for unused codecs, but listing the Wuffs +// modules we use makes that process explicit. Preprocessing means that such +// code simply isn't compiled. +#define WUFFS_CONFIG__MODULES +#define WUFFS_CONFIG__MODULE__BASE +#define WUFFS_CONFIG__MODULE__CBOR + +// If building this program in an environment that doesn't easily accommodate +// relative includes, you can use the script/inline-c-relative-includes.go +// program to generate a stand-alone C file. +#include "../../../release/c/wuffs-unsupported-snapshot.c" +#include "../fuzzlib/fuzzlib.c" + +#define TOK_BUFFER_ARRAY_SIZE 4096 +#define STACK_SIZE (WUFFS_CBOR__DECODER_DEPTH_MAX_INCL + 1) + +// Wuffs allows either statically or dynamically allocated work buffers. This +// program exercises static allocation. +#define WORK_BUFFER_ARRAY_SIZE \ + WUFFS_CBOR__DECODER_WORKBUF_LEN_MAX_INCL_WORST_CASE +#if WORK_BUFFER_ARRAY_SIZE > 0 +uint8_t g_work_buffer_array[WORK_BUFFER_ARRAY_SIZE]; +#else +// Not all C/C++ compilers support 0-length arrays. +uint8_t g_work_buffer_array[1]; +#endif + +// Each stack element is 1 byte. The low 7 bits denote the container: +// - 0x01 means no container: we are at the top level. +// - 0x02 means a [] list. +// - 0x04 means a {} dictionary. +// +// The high 0x80 bit holds the even/odd-ness of the number of elements in that +// container. A valid dictionary contains key-value pairs and should therefore +// contain an even number of elements. +typedef uint8_t stack_element; + +bool // +is_cbor_tag(wuffs_base__token t) { + return (wuffs_base__token__value_major(&t) == + WUFFS_CBOR__TOKEN_VALUE_MAJOR) && + (wuffs_base__token__value_minor(&t) & + WUFFS_CBOR__TOKEN_VALUE_MINOR__TAG); +} + +const char* // +fuzz_one_token(wuffs_base__token t, + wuffs_base__token prev_token, + wuffs_base__io_buffer* src, + size_t* ti, + stack_element* stack, + size_t* depth) { + uint64_t len = wuffs_base__token__length(&t); + if (len > 0xFFFF) { + return "fuzz: internal error: length too long (vs 0xFFFF)"; + } else if (len > (src->meta.wi - *ti)) { + return "fuzz: internal error: length too long (vs wi - ti)"; + } + *ti += len; + + if ((wuffs_base__token__value_extension(&t) >= 0) && + !wuffs_base__token__continued(&prev_token)) { + return "fuzz: internal error: extended token not after continued token"; + } + + int64_t vbc = wuffs_base__token__value_base_category(&t); + uint64_t vbd = wuffs_base__token__value_base_detail(&t); + + switch (vbc) { + case WUFFS_BASE__TOKEN__VBC__STRUCTURE: { + bool from_consistent = false; + if (vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_NONE) { + from_consistent = stack[*depth] & 0x01; + } else if (vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST) { + from_consistent = stack[*depth] & 0x02; + } else if (vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_DICT) { + from_consistent = stack[*depth] & 0x04; + } + if (!from_consistent) { + return "fuzz: internal error: inconsistent VBD__STRUCTURE__FROM_ETC"; + } + + if (vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__PUSH) { + (*depth)++; + if ((*depth >= STACK_SIZE) || (*depth == 0)) { + return "fuzz: internal error: depth too large"; + } + + if (vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE) { + return "fuzz: internal error: push to the 'none' container"; + } else if (vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST) { + stack[*depth] = 0x02; + } else if (vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_DICT) { + stack[*depth] = 0x04; + } else { + return "fuzz: internal error: unrecognized VBD__STRUCTURE__TO_ETC"; + } + + } else if (vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP) { + if ((vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_DICT) && + (0 != (0x80 & stack[*depth]))) { + return "fuzz: internal error: dictionary had an incomplete key/value " + "pair"; + } + + if (*depth <= 0) { + return "fuzz: internal error: depth too small"; + } + (*depth)--; + + bool to_consistent = false; + if (vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE) { + to_consistent = stack[*depth] & 0x01; + } else if (vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST) { + to_consistent = stack[*depth] & 0x02; + } else if (vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_DICT) { + to_consistent = stack[*depth] & 0x04; + } + if (!to_consistent) { + return "fuzz: internal error: inconsistent VBD__STRUCTURE__TO_ETC"; + } + + } else { + return "fuzz: internal error: unrecognized VBC__STRUCTURE"; + } + break; + } + + case WUFFS_BASE__TOKEN__VBC__STRING: { + if (vbd & WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY) { + wuffs_base__slice_u8 s = + wuffs_base__make_slice_u8(src->data.ptr + *ti - len, len); + if ((vbd & WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_UTF_8) && + (s.len != wuffs_base__utf_8__longest_valid_prefix(s.ptr, s.len))) { + return "fuzz: internal error: invalid UTF-8"; + } + if ((vbd & WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_ASCII) && + (s.len != wuffs_base__ascii__longest_valid_prefix(s.ptr, s.len))) { + return "fuzz: internal error: invalid ASCII"; + } + } + break; + } + + case WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT: { + if ((WUFFS_BASE__UNICODE_SURROGATE__MIN_INCL <= vbd) && + (vbd <= WUFFS_BASE__UNICODE_SURROGATE__MAX_INCL)) { + return "fuzz: internal error: invalid Unicode surrogate"; + } else if (WUFFS_BASE__UNICODE_CODE_POINT__MAX_INCL < vbd) { + return "fuzz: internal error: invalid Unicode code point"; + } + break; + } + + default: + break; + } + + // After a complete CBOR value, update the parity (even/odd count) of the + // container. + if (!wuffs_base__token__continued(&t) && + (vbc != WUFFS_BASE__TOKEN__VBC__FILLER) && + ((vbc != WUFFS_BASE__TOKEN__VBC__STRUCTURE) || + (vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP)) && + !is_cbor_tag(t)) { + stack[*depth] ^= 0x80; + } + + return NULL; +} + +uint64_t // +buffer_limit(uint32_t hash_6_bits, uint64_t min, uint64_t max) { + uint64_t n; + if (hash_6_bits < 0x20) { + n = min + hash_6_bits; + } else { + n = max - (0x3F - hash_6_bits); + } + if (n < min) { + return min; + } else if (n > max) { + return max; + } + return n; +} + +const char* // +fuzz_complex(wuffs_base__io_buffer* full_src, uint32_t hash_24_bits) { + uint64_t tok_limit = buffer_limit( + hash_24_bits & 0x3F, WUFFS_CBOR__DECODER_DST_TOKEN_BUFFER_LENGTH_MIN_INCL, + TOK_BUFFER_ARRAY_SIZE); + uint32_t hash_18_bits = hash_24_bits >> 6; + + uint64_t src_limit = + buffer_limit(hash_18_bits & 0x3F, + WUFFS_CBOR__DECODER_SRC_IO_BUFFER_LENGTH_MIN_INCL, 4096); + + // ---- + + wuffs_cbor__decoder dec; + wuffs_base__status status = wuffs_cbor__decoder__initialize( + &dec, sizeof dec, WUFFS_VERSION, + WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED); + if (!wuffs_base__status__is_ok(&status)) { + return wuffs_base__status__message(&status); + } + + wuffs_base__token tok_array[TOK_BUFFER_ARRAY_SIZE]; + wuffs_base__token_buffer tok = ((wuffs_base__token_buffer){ + .data = ((wuffs_base__slice_token){ + .ptr = tok_array, + .len = (tok_limit < TOK_BUFFER_ARRAY_SIZE) ? tok_limit + : TOK_BUFFER_ARRAY_SIZE, + }), + }); + + wuffs_base__token prev_token = wuffs_base__make_token(0); + uint32_t no_progress_count = 0; + + stack_element stack[STACK_SIZE]; + stack[0] = 0x01; // We start in the 'none' container. + size_t depth = 0; + + // ---- + + while (true) { // Outer loop. + wuffs_base__io_buffer src = make_limited_reader(*full_src, src_limit); + + size_t old_tok_wi = tok.meta.wi; + size_t old_tok_ri = tok.meta.ri; + size_t old_src_wi = src.meta.wi; + size_t old_src_ri = src.meta.ri; + size_t ti = old_src_ri; + + status = wuffs_cbor__decoder__decode_tokens( + &dec, &tok, &src, + wuffs_base__make_slice_u8(g_work_buffer_array, WORK_BUFFER_ARRAY_SIZE)); + if ((tok.data.len < tok.meta.wi) || // + (tok.meta.wi < tok.meta.ri) || // + (tok.meta.ri != old_tok_ri)) { + return "fuzz: internal error: inconsistent tok indexes"; + } else if ((src.data.len < src.meta.wi) || // + (src.meta.wi < src.meta.ri) || // + (src.meta.wi != old_src_wi)) { + return "fuzz: internal error: inconsistent src indexes"; + } + full_src->meta.ri += src.meta.ri - old_src_ri; + + if ((tok.meta.wi > old_tok_wi) || (src.meta.ri > old_src_ri) || + !wuffs_base__status__is_suspension(&status)) { + no_progress_count = 0; + } else if (no_progress_count < 999) { + no_progress_count++; + } else { + return "fuzz: internal error: no progress"; + } + + // ---- + + while (tok.meta.ri < tok.meta.wi) { // Inner loop. + wuffs_base__token t = tok.data.ptr[tok.meta.ri++]; + const char* z = + fuzz_one_token(t, prev_token, &src, &ti, &stack[0], &depth); + if (z != NULL) { + return z; + } + prev_token = t; + } // Inner loop. + + // ---- + + // Check that, starting from old_src_ri, summing the token lengths brings + // us to the new src.meta.ri. + if (ti != src.meta.ri) { + return "fuzz: internal error: ti != ri"; + } + + if (status.repr == NULL) { + break; + + } else if (status.repr == wuffs_base__suspension__short_read) { + // Some Wuffs packages can yield "$short read" for a closed io_reader, + // but Wuffs' cbor package does not. + if (src.meta.closed) { + return "fuzz: internal error: short read on a closed io_reader"; + } + // We don't compact full_src as it may be mmap'ed read-only. + continue; + + } else if (status.repr == wuffs_base__suspension__short_write) { + wuffs_base__token_buffer__compact(&tok); + continue; + } + + return wuffs_base__status__message(&status); + } // Outer loop. + + // ---- + + if (depth != 0) { + return "fuzz: internal error: decoded OK but final depth was not zero"; + } else if (wuffs_base__token__continued(&prev_token)) { + return "fuzz: internal error: decoded OK but final token was continued"; + } + return NULL; +} + +const char* // +fuzz_simple(wuffs_base__io_buffer* full_src) { + wuffs_cbor__decoder dec; + wuffs_base__status status = + wuffs_cbor__decoder__initialize(&dec, sizeof dec, WUFFS_VERSION, 0); + if (!wuffs_base__status__is_ok(&status)) { + return wuffs_base__status__message(&status); + } + + wuffs_base__token tok_array[TOK_BUFFER_ARRAY_SIZE]; + wuffs_base__token_buffer tok = ((wuffs_base__token_buffer){ + .data = ((wuffs_base__slice_token){ + .ptr = tok_array, + .len = TOK_BUFFER_ARRAY_SIZE, + }), + }); + + while (true) { + status = wuffs_cbor__decoder__decode_tokens( + &dec, &tok, full_src, + wuffs_base__make_slice_u8(g_work_buffer_array, WORK_BUFFER_ARRAY_SIZE)); + if (status.repr == NULL) { + break; + + } else if (status.repr == wuffs_base__suspension__short_write) { + tok.meta.ri = tok.meta.wi; + wuffs_base__token_buffer__compact(&tok); + continue; + } + + return wuffs_base__status__message(&status); + } + + return NULL; +} + +const char* // +fuzz(wuffs_base__io_buffer* full_src, uint32_t hash) { + // Send 99.6% of inputs to fuzz_complex and the remainder to fuzz_simple. The + // 0xA5 constant is arbitrary but non-zero. If the hash function maps the + // empty input to 0, this still sends the empty input to fuzz_complex. + // + // The fuzz_simple implementation shows how easy decoding with Wuffs is when + // all you want is to run LLVMFuzzerTestOneInput's built-in (Wuffs API + // agnostic) checks (e.g. the ASan address sanitizer) and you don't really + // care what the output is, just that it doesn't crash. + // + // The fuzz_complex implementation adds many more Wuffs API specific checks + // (e.g. that the sum of the tokens' lengths do not exceed the input length). + if ((hash & 0xFF) != 0xA5) { + return fuzz_complex(full_src, hash >> 8); + } + return fuzz_simple(full_src); +} diff --git a/fuzz/c/std/json_fuzzer.c b/fuzz/c/std/json_fuzzer.c index b705e9ce..6cb02700 100644 --- a/fuzz/c/std/json_fuzzer.c +++ b/fuzz/c/std/json_fuzzer.c @@ -85,6 +85,7 @@ typedef uint8_t stack_element; const char* // fuzz_one_token(wuffs_base__token t, + wuffs_base__token prev_token, wuffs_base__io_buffer* src, size_t* ti, stack_element* stack, @@ -97,8 +98,9 @@ fuzz_one_token(wuffs_base__token t, } *ti += len; - if ((t.repr >> 63) != 0) { - return "fuzz: internal error: token high bit was not zero"; + if ((wuffs_base__token__value_extension(&t) >= 0) && + !wuffs_base__token__continued(&prev_token)) { + return "fuzz: internal error: extended token not after continued token"; } int64_t vbc = wuffs_base__token__value_base_category(&t); @@ -285,7 +287,7 @@ fuzz_complex(wuffs_base__io_buffer* full_src, uint32_t hash_24_bits) { }), }); - wuffs_base__token final_token = wuffs_base__make_token(0); + wuffs_base__token prev_token = wuffs_base__make_token(0); uint32_t no_progress_count = 0; stack_element stack[STACK_SIZE]; @@ -330,11 +332,12 @@ fuzz_complex(wuffs_base__io_buffer* full_src, uint32_t hash_24_bits) { while (tok.meta.ri < tok.meta.wi) { // Inner loop. wuffs_base__token t = tok.data.ptr[tok.meta.ri++]; - const char* z = fuzz_one_token(t, &src, &ti, &stack[0], &depth); + const char* z = + fuzz_one_token(t, prev_token, &src, &ti, &stack[0], &depth); if (z != NULL) { return z; } - final_token = t; + prev_token = t; } // Inner loop. // ---- @@ -369,7 +372,7 @@ fuzz_complex(wuffs_base__io_buffer* full_src, uint32_t hash_24_bits) { if (depth != 0) { return "fuzz: internal error: decoded OK but final depth was not zero"; - } else if (wuffs_base__token__continued(&final_token)) { + } else if (wuffs_base__token__continued(&prev_token)) { return "fuzz: internal error: decoded OK but final token was continued"; } return NULL; diff --git a/fuzz/c/std/seed_corpora.txt b/fuzz/c/std/seed_corpora.txt index 3ecc8667..ad886a2a 100644 --- a/fuzz/c/std/seed_corpora.txt +++ b/fuzz/c/std/seed_corpora.txt @@ -15,6 +15,7 @@ # https://github.com/google/oss-fuzz/blob/master/projects/wuffs/Dockerfile bmp: test/data/*.bmp ../bmpsuite_corpus/*.bmp +cbor: test/data/*.cbor gif: test/data/*.gif test/data/artificial/*.gif gzip: test/data/*.gz test/data/artificial/*.gz json: test/data/*.json ../rapidjson_corpus/* ../simdjson_corpus/* ../JSONTestSuite/test_*/*.json diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index f862238d..dc52b2bc 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -5650,6 +5650,12 @@ extern const char wuffs_cbor__error__unsupported_recursion_depth[]; #define WUFFS_CBOR__DECODER_WORKBUF_LEN_MAX_INCL_WORST_CASE 0 +#define WUFFS_CBOR__DECODER_DEPTH_MAX_INCL 1024 + +#define WUFFS_CBOR__DECODER_DST_TOKEN_BUFFER_LENGTH_MIN_INCL 2 + +#define WUFFS_CBOR__DECODER_SRC_IO_BUFFER_LENGTH_MIN_INCL 9 + #define WUFFS_CBOR__TOKEN_VALUE_MAJOR 787997 #define WUFFS_CBOR__TOKEN_VALUE_MINOR__DETAIL_MASK 262143 diff --git a/std/cbor/decode_cbor.wuffs b/std/cbor/decode_cbor.wuffs index 9db6de3a..c58234f6 100644 --- a/std/cbor/decode_cbor.wuffs +++ b/std/cbor/decode_cbor.wuffs @@ -18,8 +18,29 @@ pub status "#unsupported recursion depth" pri status "#internal error: inconsistent I/O" pri status "#internal error: inconsistent token length" +// -------- + +// DECODER_WORKBUF_LEN_MAX_INCL_WORST_CASE is the largest workbuf length that a +// decoder will request. pub const DECODER_WORKBUF_LEN_MAX_INCL_WORST_CASE : base.u64 = 0 +// DECODER_DEPTH_MAX_INCL is the maximum supported recursion depth: how deeply +// nested [] arrays and {} maps can be. +// +// The CBOR spec itself does not define a limit, but allows implementations to +// set their own limits. +pub const DECODER_DEPTH_MAX_INCL : base.u64 = 1024 + +// DECODER_DST_TOKEN_BUFFER_LENGTH_MIN_INCL is the minimum length of the dst +// wuffs_base__token_buffer passed to the decoder. +pub const DECODER_DST_TOKEN_BUFFER_LENGTH_MIN_INCL : base.u64 = 2 + +// DECODER_SRC_IO_BUFFER_LENGTH_MIN_INCL is the minimum length of the src +// wuffs_base__io_buffer passed to the decoder. +pub const DECODER_SRC_IO_BUFFER_LENGTH_MIN_INCL : base.u64 = 9 + +// -------- + // TOKEN_VALUE_MAJOR is the base-38 encoding of "cbor". pub const TOKEN_VALUE_MAJOR : base.u32 = 0x0C_061D @@ -53,6 +74,8 @@ pub const TOKEN_VALUE_MINOR__SIMPLE_VALUE : base.u32 = 0x080_0000 // token has zero length. pub const TOKEN_VALUE_MINOR__TAG : base.u32 = 0x040_0000 +// -------- + pri const LITERALS : array[4] base.u32[..= 0x1FF_FFFF] = [ (base.TOKEN__VBC__LITERAL << 21) | base.TOKEN__VBD__LITERAL__FALSE, (base.TOKEN__VBC__LITERAL << 21) | base.TOKEN__VBD__LITERAL__TRUE, diff --git a/std/json/common_consts.wuffs b/std/json/common_consts.wuffs index b0724d96..1365d9e1 100644 --- a/std/json/common_consts.wuffs +++ b/std/json/common_consts.wuffs @@ -22,6 +22,10 @@ pub status "#unsupported recursion depth" pri status "#internal error: inconsistent I/O" +// -------- + +// DECODER_WORKBUF_LEN_MAX_INCL_WORST_CASE is the largest workbuf length that a +// decoder will request. pub const DECODER_WORKBUF_LEN_MAX_INCL_WORST_CASE : base.u64 = 0 // DECODER_DEPTH_MAX_INCL is the maximum supported recursion depth: how deeply