diff --git a/tests/fuzz/block_decompress.c b/tests/fuzz/block_decompress.c index 3cccc32f4..a904b4462 100644 --- a/tests/fuzz/block_decompress.c +++ b/tests/fuzz/block_decompress.c @@ -28,8 +28,6 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { size_t const neededBufSize = ZSTD_BLOCKSIZE_MAX; - FUZZ_seed(&src, &size); - /* Allocate all buffers and contexts if not already allocated */ if (neededBufSize > bufSize) { free(rBuf); diff --git a/tests/fuzz/block_round_trip.c b/tests/fuzz/block_round_trip.c index 64ca5fc40..0f948d4fc 100644 --- a/tests/fuzz/block_round_trip.c +++ b/tests/fuzz/block_round_trip.c @@ -20,21 +20,20 @@ #include #include "fuzz_helpers.h" #include "zstd.h" - -static const int kMaxClevel = 19; +#include "zstd_helpers.h" +#include "fuzz_data_producer.h" static ZSTD_CCtx *cctx = NULL; static ZSTD_DCtx *dctx = NULL; static void* cBuf = NULL; static void* rBuf = NULL; static size_t bufSize = 0; -static uint32_t seed; static size_t roundTripTest(void *result, size_t resultCapacity, void *compressed, size_t compressedCapacity, - const void *src, size_t srcSize) + const void *src, size_t srcSize, + int cLevel) { - int const cLevel = FUZZ_rand(&seed) % kMaxClevel; ZSTD_parameters const params = ZSTD_getParams(cLevel, srcSize, 0); size_t ret = ZSTD_compressBegin_advanced(cctx, NULL, 0, params, srcSize); FUZZ_ZASSERT(ret); @@ -52,10 +51,14 @@ static size_t roundTripTest(void *result, size_t resultCapacity, int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { - size_t neededBufSize; + /* Give a random portion of src data to the producer, to use for + parameter generation. The rest will be used for (de)compression */ + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + size = FUZZ_dataProducer_reserveDataPrefix(producer); - seed = FUZZ_seed(&src, &size); - neededBufSize = size; + int const cLevel = FUZZ_dataProducer_int32Range(producer, kMinClevel, kMaxClevel); + + size_t neededBufSize = size; if (size > ZSTD_BLOCKSIZE_MAX) return 0; @@ -79,11 +82,13 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { size_t const result = - roundTripTest(rBuf, neededBufSize, cBuf, neededBufSize, src, size); + roundTripTest(rBuf, neededBufSize, cBuf, neededBufSize, src, size, + cLevel); FUZZ_ZASSERT(result); FUZZ_ASSERT_MSG(result == size, "Incorrect regenerated size"); FUZZ_ASSERT_MSG(!memcmp(src, rBuf, size), "Corruption!"); } + FUZZ_dataProducer_free(producer); #ifndef STATEFUL_FUZZING ZSTD_freeCCtx(cctx); cctx = NULL; ZSTD_freeDCtx(dctx); dctx = NULL; diff --git a/tests/fuzz/dictionary_decompress.c b/tests/fuzz/dictionary_decompress.c index e900054f5..9cc69fa37 100644 --- a/tests/fuzz/dictionary_decompress.c +++ b/tests/fuzz/dictionary_decompress.c @@ -18,33 +18,37 @@ #include #include "fuzz_helpers.h" #include "zstd_helpers.h" +#include "fuzz_data_producer.h" static ZSTD_DCtx *dctx = NULL; int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { - uint32_t seed = FUZZ_seed(&src, &size); + /* Give a random portion of src data to the producer, to use for + parameter generation. The rest will be used for (de)compression */ + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + size = FUZZ_dataProducer_reserveDataPrefix(producer); + FUZZ_dict_t dict; ZSTD_DDict* ddict = NULL; - int i; if (!dctx) { dctx = ZSTD_createDCtx(); FUZZ_ASSERT(dctx); } - dict = FUZZ_train(src, size, &seed); - if (FUZZ_rand32(&seed, 0, 1) == 0) { + dict = FUZZ_train(src, size, producer); + if (FUZZ_dataProducer_uint32Range(producer, 0, 1) == 0) { ddict = ZSTD_createDDict(dict.buff, dict.size); FUZZ_ASSERT(ddict); } else { FUZZ_ZASSERT(ZSTD_DCtx_loadDictionary_advanced( dctx, dict.buff, dict.size, - (ZSTD_dictLoadMethod_e)FUZZ_rand32(&seed, 0, 1), - (ZSTD_dictContentType_e)FUZZ_rand32(&seed, 0, 2))); + (ZSTD_dictLoadMethod_e)FUZZ_dataProducer_uint32Range(producer, 0, 1), + (ZSTD_dictContentType_e)FUZZ_dataProducer_uint32Range(producer, 0, 2))); } - /* Run it 10 times over 10 output sizes. Reuse the context and dict. */ - for (i = 0; i < 10; ++i) { - size_t const bufSize = FUZZ_rand32(&seed, 0, 2 * size); + + { + size_t const bufSize = FUZZ_dataProducer_uint32Range(producer, 0, 10 * size); void* rBuf = malloc(bufSize); FUZZ_ASSERT(rBuf); if (ddict) { @@ -55,6 +59,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) free(rBuf); } free(dict.buff); + FUZZ_dataProducer_free(producer); ZSTD_freeDDict(ddict); #ifndef STATEFUL_FUZZING ZSTD_freeDCtx(dctx); dctx = NULL; diff --git a/tests/fuzz/dictionary_round_trip.c b/tests/fuzz/dictionary_round_trip.c index e28c65c98..9411b50a7 100644 --- a/tests/fuzz/dictionary_round_trip.c +++ b/tests/fuzz/dictionary_round_trip.c @@ -19,22 +19,21 @@ #include #include "fuzz_helpers.h" #include "zstd_helpers.h" - -static const int kMaxClevel = 19; +#include "fuzz_data_producer.h" static ZSTD_CCtx *cctx = NULL; static ZSTD_DCtx *dctx = NULL; -static uint32_t seed; static size_t roundTripTest(void *result, size_t resultCapacity, void *compressed, size_t compressedCapacity, - const void *src, size_t srcSize) + const void *src, size_t srcSize, + FUZZ_dataProducer_t *producer) { ZSTD_dictContentType_e dictContentType = ZSTD_dct_auto; - FUZZ_dict_t dict = FUZZ_train(src, srcSize, &seed); + FUZZ_dict_t dict = FUZZ_train(src, srcSize, producer); size_t cSize; - if ((FUZZ_rand(&seed) & 15) == 0) { - int const cLevel = FUZZ_rand(&seed) % kMaxClevel; + if (FUZZ_dataProducer_uint32Range(producer, 0, 15) == 0) { + int const cLevel = FUZZ_dataProducer_int32Range(producer, kMinClevel, kMaxClevel); cSize = ZSTD_compress_usingDict(cctx, compressed, compressedCapacity, @@ -42,20 +41,20 @@ static size_t roundTripTest(void *result, size_t resultCapacity, dict.buff, dict.size, cLevel); } else { - dictContentType = FUZZ_rand32(&seed, 0, 2); - FUZZ_setRandomParameters(cctx, srcSize, &seed); + dictContentType = FUZZ_dataProducer_uint32Range(producer, 0, 2); + FUZZ_setRandomParameters(cctx, srcSize, producer); /* Disable checksum so we can use sizes smaller than compress bound. */ FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 0)); FUZZ_ZASSERT(ZSTD_CCtx_loadDictionary_advanced( cctx, dict.buff, dict.size, - (ZSTD_dictLoadMethod_e)FUZZ_rand32(&seed, 0, 1), + (ZSTD_dictLoadMethod_e)FUZZ_dataProducer_uint32Range(producer, 0, 1), dictContentType)); cSize = ZSTD_compress2(cctx, compressed, compressedCapacity, src, srcSize); } FUZZ_ZASSERT(cSize); FUZZ_ZASSERT(ZSTD_DCtx_loadDictionary_advanced( dctx, dict.buff, dict.size, - (ZSTD_dictLoadMethod_e)FUZZ_rand32(&seed, 0, 1), + (ZSTD_dictLoadMethod_e)FUZZ_dataProducer_uint32Range(producer, 0, 1), dictContentType)); { size_t const ret = ZSTD_decompressDCtx( @@ -67,17 +66,20 @@ static size_t roundTripTest(void *result, size_t resultCapacity, int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { + /* Give a random portion of src data to the producer, to use for + parameter generation. The rest will be used for (de)compression */ + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + size = FUZZ_dataProducer_reserveDataPrefix(producer); + size_t const rBufSize = size; void* rBuf = malloc(rBufSize); size_t cBufSize = ZSTD_compressBound(size); - void* cBuf; - - seed = FUZZ_seed(&src, &size); + void *cBuf; /* Half of the time fuzz with a 1 byte smaller output size. * This will still succeed because we force the checksum to be disabled, * giving us 4 bytes of overhead. */ - cBufSize -= FUZZ_rand32(&seed, 0, 1); + cBufSize -= FUZZ_dataProducer_uint32Range(producer, 0, 1); cBuf = malloc(cBufSize); if (!cctx) { @@ -91,13 +93,14 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { size_t const result = - roundTripTest(rBuf, rBufSize, cBuf, cBufSize, src, size); + roundTripTest(rBuf, rBufSize, cBuf, cBufSize, src, size, producer); FUZZ_ZASSERT(result); FUZZ_ASSERT_MSG(result == size, "Incorrect regenerated size"); FUZZ_ASSERT_MSG(!memcmp(src, rBuf, size), "Corruption!"); } free(rBuf); free(cBuf); + FUZZ_dataProducer_free(producer); #ifndef STATEFUL_FUZZING ZSTD_freeCCtx(cctx); cctx = NULL; ZSTD_freeDCtx(dctx); dctx = NULL; diff --git a/tests/fuzz/fuzz.h b/tests/fuzz/fuzz.h index 8850025b0..6d53aa6d5 100644 --- a/tests/fuzz/fuzz.h +++ b/tests/fuzz/fuzz.h @@ -17,12 +17,6 @@ * test code paths which are only executed when contexts are reused. * WARNING: Makes reproducing crashes much harder. * Default: Not defined. - * @param FUZZ_RNG_SEED_SIZE: - * The number of bytes of the source to look at when constructing a seed - * for the deterministic RNG. These bytes are discarded before passing - * the data to zstd functions. Every fuzzer initializes the RNG exactly - * once before doing anything else, even if it is unused. - * Default: 4. * @param DEBUGLEVEL: * This is a parameter for the zstd library. Defining `DEBUGLEVEL=1` * enables assert() statements in the zstd library. Higher levels enable @@ -42,10 +36,6 @@ #ifndef FUZZ_H #define FUZZ_H -#ifndef FUZZ_RNG_SEED_SIZE -# define FUZZ_RNG_SEED_SIZE 4 -#endif - #include #include diff --git a/tests/fuzz/fuzz.py b/tests/fuzz/fuzz.py index faf8ce8ae..c790c8373 100755 --- a/tests/fuzz/fuzz.py +++ b/tests/fuzz/fuzz.py @@ -740,10 +740,8 @@ def gen(args): for name in os.listdir(samples): samplename = abs_join(samples, name) outname = abs_join(seed, name) - rng_seed = os.urandom(args.fuzz_rng_seed_size) with open(samplename, 'rb') as sample: with open(outname, 'wb') as out: - out.write(rng_seed) CHUNK_SIZE = 131072 chunk = sample.read(CHUNK_SIZE) while len(chunk) > 0: diff --git a/tests/fuzz/fuzz_data_producer.c b/tests/fuzz/fuzz_data_producer.c index a083f6362..b465337ea 100644 --- a/tests/fuzz/fuzz_data_producer.c +++ b/tests/fuzz/fuzz_data_producer.c @@ -49,9 +49,37 @@ uint32_t FUZZ_dataProducer_uint32Range(FUZZ_dataProducer_t *producer, uint32_t m } uint32_t FUZZ_dataProducer_uint32(FUZZ_dataProducer_t *producer) { - return FUZZ_dataProducer_uint32Range(producer, 0, 0xffffffff); + return FUZZ_dataProducer_uint32Range(producer, 0, 0xffffffff); +} + +int32_t FUZZ_dataProducer_int32Range(FUZZ_dataProducer_t *producer, + int32_t min, int32_t max) +{ + FUZZ_ASSERT(min <= max); + + if (min < 0) + return (int)FUZZ_dataProducer_uint32Range(producer, 0, max - min) + min; + + return FUZZ_dataProducer_uint32Range(producer, min, max); } size_t FUZZ_dataProducer_remainingBytes(FUZZ_dataProducer_t *producer){ return producer->size; } + +size_t FUZZ_dataProducer_contract(FUZZ_dataProducer_t *producer, size_t newSize) +{ + newSize = newSize > producer->size ? producer->size : newSize; + + size_t remaining = producer->size - newSize; + producer->data = producer->data + remaining; + producer->size = newSize; + return remaining; +} + +size_t FUZZ_dataProducer_reserveDataPrefix(FUZZ_dataProducer_t *producer) +{ + size_t producerSliceSize = FUZZ_dataProducer_uint32Range( + producer, 0, producer->size); + return FUZZ_dataProducer_contract(producer, producerSliceSize); +} diff --git a/tests/fuzz/fuzz_data_producer.h b/tests/fuzz/fuzz_data_producer.h index 4fcf6fd41..f2b609677 100644 --- a/tests/fuzz/fuzz_data_producer.h +++ b/tests/fuzz/fuzz_data_producer.h @@ -41,7 +41,20 @@ uint32_t FUZZ_dataProducer_uint32Range(FUZZ_dataProducer_t *producer, uint32_t m /* Returns a uint32 value */ uint32_t FUZZ_dataProducer_uint32(FUZZ_dataProducer_t *producer); +/* Returns a signed value between [min, max] */ +int32_t FUZZ_dataProducer_int32Range(FUZZ_dataProducer_t *producer, + int32_t min, int32_t max); + /* Returns the size of the remaining bytes of data in the producer */ size_t FUZZ_dataProducer_remainingBytes(FUZZ_dataProducer_t *producer); +/* Restricts the producer to only the last newSize bytes of data. +If newSize > current data size, nothing happens. Returns the number of bytes +the producer won't use anymore, after contracting. */ +size_t FUZZ_dataProducer_contract(FUZZ_dataProducer_t *producer, size_t newSize); + +/* Restricts the producer to use only the last X bytes of data, where X is + a random number in the interval [0, data_size]. Returns the size of the + remaining data the producer won't use anymore (the prefix). */ +size_t FUZZ_dataProducer_reserveDataPrefix(FUZZ_dataProducer_t *producer); #endif // FUZZ_DATA_PRODUCER_H diff --git a/tests/fuzz/fuzz_helpers.h b/tests/fuzz/fuzz_helpers.h index 0ee85fc7e..3de917fd1 100644 --- a/tests/fuzz/fuzz_helpers.h +++ b/tests/fuzz/fuzz_helpers.h @@ -55,37 +55,6 @@ extern "C" { #define FUZZ_STATIC static #endif -/** - * Deterministically constructs a seed based on the fuzz input. - * Consumes up to the first FUZZ_RNG_SEED_SIZE bytes of the input. - */ -FUZZ_STATIC uint32_t FUZZ_seed(uint8_t const **src, size_t* size) { - uint8_t const *data = *src; - size_t const toHash = MIN(FUZZ_RNG_SEED_SIZE, *size); - *size -= toHash; - *src += toHash; - return XXH32(data, toHash, 0); -} - -#define FUZZ_rotl32(x, r) (((x) << (r)) | ((x) >> (32 - (r)))) - -FUZZ_STATIC uint32_t FUZZ_rand(uint32_t *state) { - static const uint32_t prime1 = 2654435761U; - static const uint32_t prime2 = 2246822519U; - uint32_t rand32 = *state; - rand32 *= prime1; - rand32 += prime2; - rand32 = FUZZ_rotl32(rand32, 13); - *state = rand32; - return rand32 >> 5; -} - -/* Returns a random numer in the range [min, max]. */ -FUZZ_STATIC uint32_t FUZZ_rand32(uint32_t *state, uint32_t min, uint32_t max) { - uint32_t random = FUZZ_rand(state); - return min + (random % (max - min + 1)); -} - #ifdef __cplusplus } #endif diff --git a/tests/fuzz/simple_compress.c b/tests/fuzz/simple_compress.c index aaed40357..487be3a35 100644 --- a/tests/fuzz/simple_compress.c +++ b/tests/fuzz/simple_compress.c @@ -18,28 +18,33 @@ #include #include "fuzz_helpers.h" #include "zstd.h" +#include "zstd_helpers.h" +#include "fuzz_data_producer.h" static ZSTD_CCtx *cctx = NULL; int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { - uint32_t seed = FUZZ_seed(&src, &size); + /* Give a random portion of src data to the producer, to use for + parameter generation. The rest will be used for (de)compression */ + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + size = FUZZ_dataProducer_reserveDataPrefix(producer); + size_t const maxSize = ZSTD_compressBound(size); - int i; + size_t const bufSize = FUZZ_dataProducer_uint32Range(producer, 0, maxSize); + + int const cLevel = FUZZ_dataProducer_int32Range(producer, kMinClevel, kMaxClevel); + if (!cctx) { cctx = ZSTD_createCCtx(); FUZZ_ASSERT(cctx); } - /* Run it 10 times over 10 output sizes. Reuse the context. */ - for (i = 0; i < 10; ++i) { - int const level = (int)FUZZ_rand32(&seed, 0, 19 + 3) - 3; /* [-3, 19] */ - size_t const bufSize = FUZZ_rand32(&seed, 0, maxSize); - void* rBuf = malloc(bufSize); - FUZZ_ASSERT(rBuf); - ZSTD_compressCCtx(cctx, rBuf, bufSize, src, size, level); - free(rBuf); - } + void *rBuf = malloc(bufSize); + FUZZ_ASSERT(rBuf); + ZSTD_compressCCtx(cctx, rBuf, bufSize, src, size, cLevel); + free(rBuf); + FUZZ_dataProducer_free(producer); #ifndef STATEFUL_FUZZING ZSTD_freeCCtx(cctx); cctx = NULL; #endif diff --git a/tests/fuzz/simple_decompress.c b/tests/fuzz/simple_decompress.c index a68813ee5..6182746a1 100644 --- a/tests/fuzz/simple_decompress.c +++ b/tests/fuzz/simple_decompress.c @@ -23,26 +23,24 @@ static ZSTD_DCtx *dctx = NULL; int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { - FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + /* Give a random portion of src data to the producer, to use for + parameter generation. The rest will be used for (de)compression */ + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + size = FUZZ_dataProducer_reserveDataPrefix(producer); - int i; - if (!dctx) { - dctx = ZSTD_createDCtx(); - FUZZ_ASSERT(dctx); - } + if (!dctx) { + dctx = ZSTD_createDCtx(); + FUZZ_ASSERT(dctx); + } - size_t const bufSize = FUZZ_dataProducer_uint32Range(producer, 0, 10 * size); - void *rBuf = malloc(bufSize); - FUZZ_ASSERT(rBuf); + size_t const bufSize = FUZZ_dataProducer_uint32Range(producer, 0, 10 * size); + void *rBuf = malloc(bufSize); + FUZZ_ASSERT(rBuf); - /* Restrict to remaining data. If we run out of data while generating params, - we should still continue and let decompression happen on empty data. */ - size = FUZZ_dataProducer_remainingBytes(producer); + ZSTD_decompressDCtx(dctx, rBuf, bufSize, src, size); + free(rBuf); - ZSTD_decompressDCtx(dctx, rBuf, bufSize, src, size); - free(rBuf); - - FUZZ_dataProducer_free(producer); + FUZZ_dataProducer_free(producer); #ifndef STATEFUL_FUZZING ZSTD_freeDCtx(dctx); dctx = NULL; diff --git a/tests/fuzz/simple_round_trip.c b/tests/fuzz/simple_round_trip.c index 7e3b66098..2d1d0598a 100644 --- a/tests/fuzz/simple_round_trip.c +++ b/tests/fuzz/simple_round_trip.c @@ -20,23 +20,23 @@ #include #include "fuzz_helpers.h" #include "zstd_helpers.h" - -static const int kMaxClevel = 19; +#include "fuzz_data_producer.h" static ZSTD_CCtx *cctx = NULL; static ZSTD_DCtx *dctx = NULL; -static uint32_t seed; static size_t roundTripTest(void *result, size_t resultCapacity, void *compressed, size_t compressedCapacity, - const void *src, size_t srcSize) + const void *src, size_t srcSize, + FUZZ_dataProducer_t *producer) { size_t cSize; - if (FUZZ_rand(&seed) & 1) { - FUZZ_setRandomParameters(cctx, srcSize, &seed); + if (FUZZ_dataProducer_uint32Range(producer, 0, 1)) { + FUZZ_setRandomParameters(cctx, srcSize, producer); cSize = ZSTD_compress2(cctx, compressed, compressedCapacity, src, srcSize); } else { - int const cLevel = FUZZ_rand(&seed) % kMaxClevel; + int const cLevel = FUZZ_dataProducer_int32Range(producer, kMinClevel, kMaxClevel); + cSize = ZSTD_compressCCtx( cctx, compressed, compressedCapacity, src, srcSize, cLevel); } @@ -51,12 +51,17 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) size_t cBufSize = ZSTD_compressBound(size); void* cBuf; - seed = FUZZ_seed(&src, &size); + /* Give a random portion of src data to the producer, to use for + parameter generation. The rest will be used for (de)compression */ + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + size = FUZZ_dataProducer_reserveDataPrefix(producer); + /* Half of the time fuzz with a 1 byte smaller output size. * This will still succeed because we don't use a dictionary, so the dictID * field is empty, giving us 4 bytes of overhead. */ - cBufSize -= FUZZ_rand32(&seed, 0, 1); + cBufSize -= FUZZ_dataProducer_uint32Range(producer, 0, 1); + cBuf = malloc(cBufSize); FUZZ_ASSERT(cBuf && rBuf); @@ -72,13 +77,14 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { size_t const result = - roundTripTest(rBuf, rBufSize, cBuf, cBufSize, src, size); + roundTripTest(rBuf, rBufSize, cBuf, cBufSize, src, size, producer); FUZZ_ZASSERT(result); FUZZ_ASSERT_MSG(result == size, "Incorrect regenerated size"); FUZZ_ASSERT_MSG(!memcmp(src, rBuf, size), "Corruption!"); } free(rBuf); free(cBuf); + FUZZ_dataProducer_free(producer); #ifndef STATEFUL_FUZZING ZSTD_freeCCtx(cctx); cctx = NULL; ZSTD_freeDCtx(dctx); dctx = NULL; diff --git a/tests/fuzz/stream_decompress.c b/tests/fuzz/stream_decompress.c index 68e120d7e..c71cc9d3e 100644 --- a/tests/fuzz/stream_decompress.c +++ b/tests/fuzz/stream_decompress.c @@ -19,6 +19,7 @@ #include #include "fuzz_helpers.h" #include "zstd.h" +#include "fuzz_data_producer.h" static size_t const kBufSize = ZSTD_BLOCKSIZE_MAX; @@ -26,22 +27,23 @@ static ZSTD_DStream *dstream = NULL; static void* buf = NULL; uint32_t seed; -static ZSTD_outBuffer makeOutBuffer(void) +static ZSTD_outBuffer makeOutBuffer(FUZZ_dataProducer_t *producer) { ZSTD_outBuffer buffer = { buf, 0, 0 }; - buffer.size = (FUZZ_rand(&seed) % kBufSize) + 1; + buffer.size = (FUZZ_dataProducer_uint32Range(producer, 1, kBufSize)); FUZZ_ASSERT(buffer.size <= kBufSize); return buffer; } -static ZSTD_inBuffer makeInBuffer(const uint8_t **src, size_t *size) +static ZSTD_inBuffer makeInBuffer(const uint8_t **src, size_t *size, + FUZZ_dataProducer_t *producer) { ZSTD_inBuffer buffer = { *src, 0, 0 }; FUZZ_ASSERT(*size > 0); - buffer.size = (FUZZ_rand(&seed) % *size) + 1; + buffer.size = (FUZZ_dataProducer_uint32Range(producer, 1, *size)); FUZZ_ASSERT(buffer.size <= *size); *src += buffer.size; *size -= buffer.size; @@ -51,13 +53,16 @@ static ZSTD_inBuffer makeInBuffer(const uint8_t **src, size_t *size) int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { - seed = FUZZ_seed(&src, &size); + /* Give a random portion of src data to the producer, to use for + parameter generation. The rest will be used for (de)compression */ + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + size = FUZZ_dataProducer_reserveDataPrefix(producer); /* Allocate all buffers and contexts if not already allocated */ if (!buf) { buf = malloc(kBufSize); - FUZZ_ASSERT(buf); - } + FUZZ_ASSERT(buf); + } if (!dstream) { dstream = ZSTD_createDStream(); @@ -67,9 +72,9 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) } while (size > 0) { - ZSTD_inBuffer in = makeInBuffer(&src, &size); + ZSTD_inBuffer in = makeInBuffer(&src, &size, producer); while (in.pos != in.size) { - ZSTD_outBuffer out = makeOutBuffer(); + ZSTD_outBuffer out = makeOutBuffer(producer); size_t const rc = ZSTD_decompressStream(dstream, &out, &in); if (ZSTD_isError(rc)) goto error; } @@ -79,5 +84,6 @@ error: #ifndef STATEFUL_FUZZING ZSTD_freeDStream(dstream); dstream = NULL; #endif + FUZZ_dataProducer_free(producer); return 0; } diff --git a/tests/fuzz/stream_round_trip.c b/tests/fuzz/stream_round_trip.c index d13c2dbe7..c534a904a 100644 --- a/tests/fuzz/stream_round_trip.c +++ b/tests/fuzz/stream_round_trip.c @@ -20,31 +20,33 @@ #include #include "fuzz_helpers.h" #include "zstd_helpers.h" +#include "fuzz_data_producer.h" ZSTD_CCtx *cctx = NULL; static ZSTD_DCtx *dctx = NULL; static uint8_t* cBuf = NULL; static uint8_t* rBuf = NULL; static size_t bufSize = 0; -static uint32_t seed; -static ZSTD_outBuffer makeOutBuffer(uint8_t *dst, size_t capacity) +static ZSTD_outBuffer makeOutBuffer(uint8_t *dst, size_t capacity, + FUZZ_dataProducer_t *producer) { ZSTD_outBuffer buffer = { dst, 0, 0 }; FUZZ_ASSERT(capacity > 0); - buffer.size = (FUZZ_rand(&seed) % capacity) + 1; + buffer.size = (FUZZ_dataProducer_uint32Range(producer, 1, capacity)); FUZZ_ASSERT(buffer.size <= capacity); return buffer; } -static ZSTD_inBuffer makeInBuffer(const uint8_t **src, size_t *size) +static ZSTD_inBuffer makeInBuffer(const uint8_t **src, size_t *size, + FUZZ_dataProducer_t *producer) { ZSTD_inBuffer buffer = { *src, 0, 0 }; FUZZ_ASSERT(*size > 0); - buffer.size = (FUZZ_rand(&seed) % *size) + 1; + buffer.size = (FUZZ_dataProducer_uint32Range(producer, 1, *size)); FUZZ_ASSERT(buffer.size <= *size); *src += buffer.size; *size -= buffer.size; @@ -53,23 +55,24 @@ static ZSTD_inBuffer makeInBuffer(const uint8_t **src, size_t *size) } static size_t compress(uint8_t *dst, size_t capacity, - const uint8_t *src, size_t srcSize) + const uint8_t *src, size_t srcSize, + FUZZ_dataProducer_t *producer) { size_t dstSize = 0; ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); - FUZZ_setRandomParameters(cctx, srcSize, &seed); + FUZZ_setRandomParameters(cctx, srcSize, producer); while (srcSize > 0) { - ZSTD_inBuffer in = makeInBuffer(&src, &srcSize); + ZSTD_inBuffer in = makeInBuffer(&src, &srcSize, producer); /* Mode controls the action. If mode == -1 we pick a new mode */ int mode = -1; while (in.pos < in.size || mode != -1) { - ZSTD_outBuffer out = makeOutBuffer(dst, capacity); + ZSTD_outBuffer out = makeOutBuffer(dst, capacity, producer); /* Previous action finished, pick a new mode. */ - if (mode == -1) mode = FUZZ_rand(&seed) % 10; + if (mode == -1) mode = FUZZ_dataProducer_uint32Range(producer, 0, 9); switch (mode) { - case 0: /* fall-though */ - case 1: /* fall-though */ + case 0: /* fall-through */ + case 1: /* fall-through */ case 2: { size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush); @@ -85,9 +88,9 @@ static size_t compress(uint8_t *dst, size_t capacity, /* Reset the compressor when the frame is finished */ if (ret == 0) { ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); - if ((FUZZ_rand(&seed) & 7) == 0) { + if (FUZZ_dataProducer_uint32Range(producer, 0, 7) == 0) { size_t const remaining = in.size - in.pos; - FUZZ_setRandomParameters(cctx, remaining, &seed); + FUZZ_setRandomParameters(cctx, remaining, producer); } mode = -1; } @@ -107,7 +110,7 @@ static size_t compress(uint8_t *dst, size_t capacity, } for (;;) { ZSTD_inBuffer in = {NULL, 0, 0}; - ZSTD_outBuffer out = makeOutBuffer(dst, capacity); + ZSTD_outBuffer out = makeOutBuffer(dst, capacity, producer); size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end); FUZZ_ZASSERT(ret); @@ -122,10 +125,13 @@ static size_t compress(uint8_t *dst, size_t capacity, int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { - size_t neededBufSize; + /* Give a random portion of src data to the producer, to use for + parameter generation. The rest will be used for (de)compression */ + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + size = FUZZ_dataProducer_reserveDataPrefix(producer); - seed = FUZZ_seed(&src, &size); - neededBufSize = ZSTD_compressBound(size) * 2; + size_t neededBufSize; + neededBufSize = ZSTD_compressBound(size) * 5; /* Allocate all buffers and contexts if not already allocated */ if (neededBufSize > bufSize) { @@ -146,7 +152,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) } { - size_t const cSize = compress(cBuf, neededBufSize, src, size); + size_t const cSize = compress(cBuf, neededBufSize, src, size, producer); size_t const rSize = ZSTD_decompressDCtx(dctx, rBuf, neededBufSize, cBuf, cSize); FUZZ_ZASSERT(rSize); @@ -154,6 +160,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) FUZZ_ASSERT_MSG(!memcmp(src, rBuf, size), "Corruption!"); } + FUZZ_dataProducer_free(producer); #ifndef STATEFUL_FUZZING ZSTD_freeCCtx(cctx); cctx = NULL; ZSTD_freeDCtx(dctx); dctx = NULL; diff --git a/tests/fuzz/zstd_frame_info.c b/tests/fuzz/zstd_frame_info.c index 7512d5f49..359cf128f 100644 --- a/tests/fuzz/zstd_frame_info.c +++ b/tests/fuzz/zstd_frame_info.c @@ -21,10 +21,6 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { ZSTD_frameHeader zfh; - /* Consume the seed to be compatible with the corpora of other decompression - * fuzzers. - */ - FUZZ_seed(&src, &size); /* You can fuzz any helper functions here that are fast, and take zstd * compressed data as input. E.g. don't expect the input to be a dictionary, * so don't fuzz ZSTD_getDictID_fromDict(). diff --git a/tests/fuzz/zstd_helpers.c b/tests/fuzz/zstd_helpers.c index 5ff057b8c..90bf1a156 100644 --- a/tests/fuzz/zstd_helpers.c +++ b/tests/fuzz/zstd_helpers.c @@ -17,53 +17,56 @@ #include "zstd.h" #include "zdict.h" +const int kMinClevel = -3; +const int kMaxClevel = 19; + static void set(ZSTD_CCtx *cctx, ZSTD_cParameter param, int value) { FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, param, value)); } static void setRand(ZSTD_CCtx *cctx, ZSTD_cParameter param, unsigned min, - unsigned max, uint32_t *state) { - unsigned const value = FUZZ_rand32(state, min, max); + unsigned max, FUZZ_dataProducer_t *producer) { + unsigned const value = FUZZ_dataProducer_uint32Range(producer, min, max); set(cctx, param, value); } -ZSTD_compressionParameters FUZZ_randomCParams(size_t srcSize, uint32_t *state) +ZSTD_compressionParameters FUZZ_randomCParams(size_t srcSize, FUZZ_dataProducer_t *producer) { /* Select compression parameters */ ZSTD_compressionParameters cParams; - cParams.windowLog = FUZZ_rand32(state, ZSTD_WINDOWLOG_MIN, 15); - cParams.hashLog = FUZZ_rand32(state, ZSTD_HASHLOG_MIN, 15); - cParams.chainLog = FUZZ_rand32(state, ZSTD_CHAINLOG_MIN, 16); - cParams.searchLog = FUZZ_rand32(state, ZSTD_SEARCHLOG_MIN, 9); - cParams.minMatch = FUZZ_rand32(state, ZSTD_MINMATCH_MIN, + cParams.windowLog = FUZZ_dataProducer_uint32Range(producer, ZSTD_WINDOWLOG_MIN, 15); + cParams.hashLog = FUZZ_dataProducer_uint32Range(producer, ZSTD_HASHLOG_MIN, 15); + cParams.chainLog = FUZZ_dataProducer_uint32Range(producer, ZSTD_CHAINLOG_MIN, 16); + cParams.searchLog = FUZZ_dataProducer_uint32Range(producer, ZSTD_SEARCHLOG_MIN, 9); + cParams.minMatch = FUZZ_dataProducer_uint32Range(producer, ZSTD_MINMATCH_MIN, ZSTD_MINMATCH_MAX); - cParams.targetLength = FUZZ_rand32(state, 0, 512); - cParams.strategy = FUZZ_rand32(state, ZSTD_STRATEGY_MIN, ZSTD_STRATEGY_MAX); + cParams.targetLength = FUZZ_dataProducer_uint32Range(producer, 0, 512); + cParams.strategy = FUZZ_dataProducer_uint32Range(producer, ZSTD_STRATEGY_MIN, ZSTD_STRATEGY_MAX); return ZSTD_adjustCParams(cParams, srcSize, 0); } -ZSTD_frameParameters FUZZ_randomFParams(uint32_t *state) +ZSTD_frameParameters FUZZ_randomFParams(FUZZ_dataProducer_t *producer) { /* Select frame parameters */ ZSTD_frameParameters fParams; - fParams.contentSizeFlag = FUZZ_rand32(state, 0, 1); - fParams.checksumFlag = FUZZ_rand32(state, 0, 1); - fParams.noDictIDFlag = FUZZ_rand32(state, 0, 1); + fParams.contentSizeFlag = FUZZ_dataProducer_uint32Range(producer, 0, 1); + fParams.checksumFlag = FUZZ_dataProducer_uint32Range(producer, 0, 1); + fParams.noDictIDFlag = FUZZ_dataProducer_uint32Range(producer, 0, 1); return fParams; } -ZSTD_parameters FUZZ_randomParams(size_t srcSize, uint32_t *state) +ZSTD_parameters FUZZ_randomParams(size_t srcSize, FUZZ_dataProducer_t *producer) { ZSTD_parameters params; - params.cParams = FUZZ_randomCParams(srcSize, state); - params.fParams = FUZZ_randomFParams(state); + params.cParams = FUZZ_randomCParams(srcSize, producer); + params.fParams = FUZZ_randomFParams(producer); return params; } -void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, uint32_t *state) +void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer_t *producer) { - ZSTD_compressionParameters cParams = FUZZ_randomCParams(srcSize, state); + ZSTD_compressionParameters cParams = FUZZ_randomCParams(srcSize, producer); set(cctx, ZSTD_c_windowLog, cParams.windowLog); set(cctx, ZSTD_c_hashLog, cParams.hashLog); set(cctx, ZSTD_c_chainLog, cParams.chainLog); @@ -72,30 +75,30 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, uint32_t *state) set(cctx, ZSTD_c_targetLength, cParams.targetLength); set(cctx, ZSTD_c_strategy, cParams.strategy); /* Select frame parameters */ - setRand(cctx, ZSTD_c_contentSizeFlag, 0, 1, state); - setRand(cctx, ZSTD_c_checksumFlag, 0, 1, state); - setRand(cctx, ZSTD_c_dictIDFlag, 0, 1, state); + setRand(cctx, ZSTD_c_contentSizeFlag, 0, 1, producer); + setRand(cctx, ZSTD_c_checksumFlag, 0, 1, producer); + setRand(cctx, ZSTD_c_dictIDFlag, 0, 1, producer); /* Select long distance matching parameters */ - setRand(cctx, ZSTD_c_enableLongDistanceMatching, 0, 1, state); - setRand(cctx, ZSTD_c_ldmHashLog, ZSTD_HASHLOG_MIN, 16, state); + setRand(cctx, ZSTD_c_enableLongDistanceMatching, 0, 1, producer); + setRand(cctx, ZSTD_c_ldmHashLog, ZSTD_HASHLOG_MIN, 16, producer); setRand(cctx, ZSTD_c_ldmMinMatch, ZSTD_LDM_MINMATCH_MIN, - ZSTD_LDM_MINMATCH_MAX, state); + ZSTD_LDM_MINMATCH_MAX, producer); setRand(cctx, ZSTD_c_ldmBucketSizeLog, 0, ZSTD_LDM_BUCKETSIZELOG_MAX, - state); + producer); setRand(cctx, ZSTD_c_ldmHashRateLog, ZSTD_LDM_HASHRATELOG_MIN, - ZSTD_LDM_HASHRATELOG_MAX, state); + ZSTD_LDM_HASHRATELOG_MAX, producer); /* Set misc parameters */ - setRand(cctx, ZSTD_c_nbWorkers, 0, 2, state); - setRand(cctx, ZSTD_c_rsyncable, 0, 1, state); - setRand(cctx, ZSTD_c_forceMaxWindow, 0, 1, state); - setRand(cctx, ZSTD_c_literalCompressionMode, 0, 2, state); - setRand(cctx, ZSTD_c_forceAttachDict, 0, 2, state); - if (FUZZ_rand32(state, 0, 1) == 0) { - setRand(cctx, ZSTD_c_srcSizeHint, ZSTD_SRCSIZEHINT_MIN, 2 * srcSize, state); + setRand(cctx, ZSTD_c_nbWorkers, 0, 2, producer); + setRand(cctx, ZSTD_c_rsyncable, 0, 1, producer); + setRand(cctx, ZSTD_c_forceMaxWindow, 0, 1, producer); + setRand(cctx, ZSTD_c_literalCompressionMode, 0, 2, producer); + setRand(cctx, ZSTD_c_forceAttachDict, 0, 2, producer); + if (FUZZ_dataProducer_uint32Range(producer, 0, 1) == 0) { + setRand(cctx, ZSTD_c_srcSizeHint, ZSTD_SRCSIZEHINT_MIN, 2 * srcSize, producer); } } -FUZZ_dict_t FUZZ_train(void const* src, size_t srcSize, uint32_t *state) +FUZZ_dict_t FUZZ_train(void const* src, size_t srcSize, FUZZ_dataProducer_t *producer) { size_t const dictSize = MAX(srcSize / 8, 1024); size_t const totalSampleSize = dictSize * 11; @@ -110,7 +113,7 @@ FUZZ_dict_t FUZZ_train(void const* src, size_t srcSize, uint32_t *state) for (sample = 0; sample < nbSamples; ++sample) { size_t const remaining = totalSampleSize - pos; - size_t const offset = FUZZ_rand32(state, 0, MAX(srcSize, 1) - 1); + size_t const offset = FUZZ_dataProducer_uint32Range(producer, 0, MAX(srcSize, 1) - 1); size_t const limit = MIN(srcSize - offset, remaining); size_t const toCopy = MIN(limit, remaining / (nbSamples - sample)); memcpy(samples + pos, src + offset, toCopy); diff --git a/tests/fuzz/zstd_helpers.h b/tests/fuzz/zstd_helpers.h index 457e6e995..2210bcaf8 100644 --- a/tests/fuzz/zstd_helpers.h +++ b/tests/fuzz/zstd_helpers.h @@ -17,17 +17,21 @@ #define ZSTD_STATIC_LINKING_ONLY #include "zstd.h" +#include "fuzz_data_producer.h" #include #ifdef __cplusplus extern "C" { #endif -void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, uint32_t *state); +extern const int kMinClevel; +extern const int kMaxClevel; -ZSTD_compressionParameters FUZZ_randomCParams(size_t srcSize, uint32_t *state); -ZSTD_frameParameters FUZZ_randomFParams(uint32_t *state); -ZSTD_parameters FUZZ_randomParams(size_t srcSize, uint32_t *state); +void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer_t *producer); + +ZSTD_compressionParameters FUZZ_randomCParams(size_t srcSize, FUZZ_dataProducer_t *producer); +ZSTD_frameParameters FUZZ_randomFParams(FUZZ_dataProducer_t *producer); +ZSTD_parameters FUZZ_randomParams(size_t srcSize, FUZZ_dataProducer_t *producer); typedef struct { void* buff; @@ -38,8 +42,7 @@ typedef struct { * NOTE: Don't use this to train production dictionaries, it is only optimized * for speed, and doesn't care about dictionary quality. */ -FUZZ_dict_t FUZZ_train(void const* src, size_t srcSize, uint32_t *state); - +FUZZ_dict_t FUZZ_train(void const* src, size_t srcSize, FUZZ_dataProducer_t *producer); #ifdef __cplusplus }