From 0630d084cb0e0ea829630cc4fb1ebdb40af91652 Mon Sep 17 00:00:00 2001 From: Dario Pavlovic Date: Tue, 10 Sep 2019 16:14:43 -0700 Subject: [PATCH 1/6] [Fuzz] Improve data generation #1723 Converting the rest of the tests to use the new data producer. --- tests/fuzz/block_decompress.c | 2 - tests/fuzz/block_round_trip.c | 17 ++++--- tests/fuzz/dictionary_decompress.c | 14 +++--- tests/fuzz/dictionary_round_trip.c | 27 ++++++----- tests/fuzz/fuzz_data_producer.c | 12 ++++- tests/fuzz/fuzz_data_producer.h | 6 +++ tests/fuzz/simple_compress.c | 25 ++++++----- tests/fuzz/simple_decompress.c | 3 +- tests/fuzz/simple_round_trip.c | 18 ++++---- tests/fuzz/stream_decompress.c | 25 +++++++---- tests/fuzz/stream_round_trip.c | 44 ++++++++++-------- tests/fuzz/zstd_helpers.c | 72 +++++++++++++++--------------- tests/fuzz/zstd_helpers.h | 11 ++--- 13 files changed, 159 insertions(+), 117 deletions(-) diff --git a/tests/fuzz/block_decompress.c b/tests/fuzz/block_decompress.c index 3cccc32f4..a904b4462 100644 --- a/tests/fuzz/block_decompress.c +++ b/tests/fuzz/block_decompress.c @@ -28,8 +28,6 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { size_t const neededBufSize = ZSTD_BLOCKSIZE_MAX; - FUZZ_seed(&src, &size); - /* Allocate all buffers and contexts if not already allocated */ if (neededBufSize > bufSize) { free(rBuf); diff --git a/tests/fuzz/block_round_trip.c b/tests/fuzz/block_round_trip.c index 64ca5fc40..326d5b247 100644 --- a/tests/fuzz/block_round_trip.c +++ b/tests/fuzz/block_round_trip.c @@ -20,6 +20,7 @@ #include #include "fuzz_helpers.h" #include "zstd.h" +#include "fuzz_data_producer.h" static const int kMaxClevel = 19; @@ -28,13 +29,12 @@ static ZSTD_DCtx *dctx = NULL; static void* cBuf = NULL; static void* rBuf = NULL; static size_t bufSize = 0; -static uint32_t seed; static size_t roundTripTest(void *result, size_t resultCapacity, void *compressed, size_t compressedCapacity, - const void *src, size_t srcSize) + const void *src, size_t srcSize, + int cLevel) { - int const cLevel = FUZZ_rand(&seed) % kMaxClevel; ZSTD_parameters const params = ZSTD_getParams(cLevel, srcSize, 0); size_t ret = ZSTD_compressBegin_advanced(cctx, NULL, 0, params, srcSize); FUZZ_ZASSERT(ret); @@ -52,10 +52,11 @@ static size_t roundTripTest(void *result, size_t resultCapacity, int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { - size_t neededBufSize; + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + int cLevel = FUZZ_dataProducer_uint32(producer) % kMaxClevel; + size = FUZZ_dataProducer_remainingBytes(producer); - seed = FUZZ_seed(&src, &size); - neededBufSize = size; + size_t neededBufSize = size; if (size > ZSTD_BLOCKSIZE_MAX) return 0; @@ -79,11 +80,13 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { size_t const result = - roundTripTest(rBuf, neededBufSize, cBuf, neededBufSize, src, size); + roundTripTest(rBuf, neededBufSize, cBuf, neededBufSize, src, size, + cLevel); FUZZ_ZASSERT(result); FUZZ_ASSERT_MSG(result == size, "Incorrect regenerated size"); FUZZ_ASSERT_MSG(!memcmp(src, rBuf, size), "Corruption!"); } + FUZZ_dataProducer_free(producer); #ifndef STATEFUL_FUZZING ZSTD_freeCCtx(cctx); cctx = NULL; ZSTD_freeDCtx(dctx); dctx = NULL; diff --git a/tests/fuzz/dictionary_decompress.c b/tests/fuzz/dictionary_decompress.c index e900054f5..5f660ac52 100644 --- a/tests/fuzz/dictionary_decompress.c +++ b/tests/fuzz/dictionary_decompress.c @@ -18,12 +18,13 @@ #include #include "fuzz_helpers.h" #include "zstd_helpers.h" +#include "fuzz_data_producer.h" static ZSTD_DCtx *dctx = NULL; int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { - uint32_t seed = FUZZ_seed(&src, &size); + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); FUZZ_dict_t dict; ZSTD_DDict* ddict = NULL; int i; @@ -32,19 +33,19 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) dctx = ZSTD_createDCtx(); FUZZ_ASSERT(dctx); } - dict = FUZZ_train(src, size, &seed); - if (FUZZ_rand32(&seed, 0, 1) == 0) { + dict = FUZZ_train(src, size, producer); + if (FUZZ_dataProducer_uint32Range(producer, 0, 1) == 0) { ddict = ZSTD_createDDict(dict.buff, dict.size); FUZZ_ASSERT(ddict); } else { FUZZ_ZASSERT(ZSTD_DCtx_loadDictionary_advanced( dctx, dict.buff, dict.size, - (ZSTD_dictLoadMethod_e)FUZZ_rand32(&seed, 0, 1), - (ZSTD_dictContentType_e)FUZZ_rand32(&seed, 0, 2))); + (ZSTD_dictLoadMethod_e)FUZZ_dataProducer_uint32Range(producer, 0, 1), + (ZSTD_dictContentType_e)FUZZ_dataProducer_uint32Range(producer, 0, 2))); } /* Run it 10 times over 10 output sizes. Reuse the context and dict. */ for (i = 0; i < 10; ++i) { - size_t const bufSize = FUZZ_rand32(&seed, 0, 2 * size); + size_t const bufSize = FUZZ_dataProducer_uint32Range(producer, 0, 2 * size); void* rBuf = malloc(bufSize); FUZZ_ASSERT(rBuf); if (ddict) { @@ -55,6 +56,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) free(rBuf); } free(dict.buff); + FUZZ_dataProducer_free(producer); ZSTD_freeDDict(ddict); #ifndef STATEFUL_FUZZING ZSTD_freeDCtx(dctx); dctx = NULL; diff --git a/tests/fuzz/dictionary_round_trip.c b/tests/fuzz/dictionary_round_trip.c index e28c65c98..fe0a217b2 100644 --- a/tests/fuzz/dictionary_round_trip.c +++ b/tests/fuzz/dictionary_round_trip.c @@ -19,22 +19,23 @@ #include #include "fuzz_helpers.h" #include "zstd_helpers.h" +#include "fuzz_data_producer.h" static const int kMaxClevel = 19; static ZSTD_CCtx *cctx = NULL; static ZSTD_DCtx *dctx = NULL; -static uint32_t seed; static size_t roundTripTest(void *result, size_t resultCapacity, void *compressed, size_t compressedCapacity, - const void *src, size_t srcSize) + const void *src, size_t srcSize, + FUZZ_dataProducer_t *producer) { ZSTD_dictContentType_e dictContentType = ZSTD_dct_auto; - FUZZ_dict_t dict = FUZZ_train(src, srcSize, &seed); + FUZZ_dict_t dict = FUZZ_train(src, srcSize, producer); size_t cSize; - if ((FUZZ_rand(&seed) & 15) == 0) { - int const cLevel = FUZZ_rand(&seed) % kMaxClevel; + if ((FUZZ_dataProducer_uint32(producer) & 15) == 0) { + int const cLevel = FUZZ_dataProducer_uint32(producer) % kMaxClevel; cSize = ZSTD_compress_usingDict(cctx, compressed, compressedCapacity, @@ -42,20 +43,20 @@ static size_t roundTripTest(void *result, size_t resultCapacity, dict.buff, dict.size, cLevel); } else { - dictContentType = FUZZ_rand32(&seed, 0, 2); - FUZZ_setRandomParameters(cctx, srcSize, &seed); + dictContentType = FUZZ_dataProducer_uint32Range(producer, 0, 2); + FUZZ_setRandomParameters(cctx, srcSize, producer); /* Disable checksum so we can use sizes smaller than compress bound. */ FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 0)); FUZZ_ZASSERT(ZSTD_CCtx_loadDictionary_advanced( cctx, dict.buff, dict.size, - (ZSTD_dictLoadMethod_e)FUZZ_rand32(&seed, 0, 1), + (ZSTD_dictLoadMethod_e)FUZZ_dataProducer_uint32Range(producer, 0, 1), dictContentType)); cSize = ZSTD_compress2(cctx, compressed, compressedCapacity, src, srcSize); } FUZZ_ZASSERT(cSize); FUZZ_ZASSERT(ZSTD_DCtx_loadDictionary_advanced( dctx, dict.buff, dict.size, - (ZSTD_dictLoadMethod_e)FUZZ_rand32(&seed, 0, 1), + (ZSTD_dictLoadMethod_e)FUZZ_dataProducer_uint32Range(producer, 0, 1), dictContentType)); { size_t const ret = ZSTD_decompressDCtx( @@ -72,12 +73,13 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) size_t cBufSize = ZSTD_compressBound(size); void* cBuf; - seed = FUZZ_seed(&src, &size); + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + /* Half of the time fuzz with a 1 byte smaller output size. * This will still succeed because we force the checksum to be disabled, * giving us 4 bytes of overhead. */ - cBufSize -= FUZZ_rand32(&seed, 0, 1); + cBufSize -= FUZZ_dataProducer_uint32Range(producer, 0, 1); cBuf = malloc(cBufSize); if (!cctx) { @@ -91,13 +93,14 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { size_t const result = - roundTripTest(rBuf, rBufSize, cBuf, cBufSize, src, size); + roundTripTest(rBuf, rBufSize, cBuf, cBufSize, src, size, producer); FUZZ_ZASSERT(result); FUZZ_ASSERT_MSG(result == size, "Incorrect regenerated size"); FUZZ_ASSERT_MSG(!memcmp(src, rBuf, size), "Corruption!"); } free(rBuf); free(cBuf); + FUZZ_dataProducer_free(producer); #ifndef STATEFUL_FUZZING ZSTD_freeCCtx(cctx); cctx = NULL; ZSTD_freeDCtx(dctx); dctx = NULL; diff --git a/tests/fuzz/fuzz_data_producer.c b/tests/fuzz/fuzz_data_producer.c index a083f6362..6dcc1413d 100644 --- a/tests/fuzz/fuzz_data_producer.c +++ b/tests/fuzz/fuzz_data_producer.c @@ -49,9 +49,19 @@ uint32_t FUZZ_dataProducer_uint32Range(FUZZ_dataProducer_t *producer, uint32_t m } uint32_t FUZZ_dataProducer_uint32(FUZZ_dataProducer_t *producer) { - return FUZZ_dataProducer_uint32Range(producer, 0, 0xffffffff); + return FUZZ_dataProducer_uint32Range(producer, 0, 0xffffffff); } size_t FUZZ_dataProducer_remainingBytes(FUZZ_dataProducer_t *producer){ return producer->size; } + +size_t FUZZ_dataProducer_contract(FUZZ_dataProducer_t *producer, size_t newSize) +{ + newSize = newSize > producer->size ? producer->size : newSize; + + size_t remaining = producer->size - newSize; + producer->data = producer->data + remaining; + producer->size = newSize; + return remaining; +} diff --git a/tests/fuzz/fuzz_data_producer.h b/tests/fuzz/fuzz_data_producer.h index 4fcf6fd41..668c87f29 100644 --- a/tests/fuzz/fuzz_data_producer.h +++ b/tests/fuzz/fuzz_data_producer.h @@ -44,4 +44,10 @@ uint32_t FUZZ_dataProducer_uint32(FUZZ_dataProducer_t *producer); /* Returns the size of the remaining bytes of data in the producer */ size_t FUZZ_dataProducer_remainingBytes(FUZZ_dataProducer_t *producer); +/* Tells the producer to contract to newSize bytes of data it currently uses, +counted from the end, and forget about the rest. If newSize > current data size, +nothing happens. Returns the number of bytes the producer won't use anymore, +after contracting. */ +size_t FUZZ_dataProducer_contract(FUZZ_dataProducer_t *producer, size_t newSize); + #endif // FUZZ_DATA_PRODUCER_H diff --git a/tests/fuzz/simple_compress.c b/tests/fuzz/simple_compress.c index aaed40357..29de4701f 100644 --- a/tests/fuzz/simple_compress.c +++ b/tests/fuzz/simple_compress.c @@ -18,28 +18,31 @@ #include #include "fuzz_helpers.h" #include "zstd.h" +#include "fuzz_data_producer.h" static ZSTD_CCtx *cctx = NULL; int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { - uint32_t seed = FUZZ_seed(&src, &size); + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + + int const level = (int)FUZZ_dataProducer_uint32Range( + producer, 0, 19 + 3) - 3; /* [-3, 19] */ size_t const maxSize = ZSTD_compressBound(size); - int i; + size_t const bufSize = FUZZ_dataProducer_uint32Range(producer, 0, maxSize); + + size = FUZZ_dataProducer_remainingBytes(producer); + if (!cctx) { cctx = ZSTD_createCCtx(); FUZZ_ASSERT(cctx); } - /* Run it 10 times over 10 output sizes. Reuse the context. */ - for (i = 0; i < 10; ++i) { - int const level = (int)FUZZ_rand32(&seed, 0, 19 + 3) - 3; /* [-3, 19] */ - size_t const bufSize = FUZZ_rand32(&seed, 0, maxSize); - void* rBuf = malloc(bufSize); - FUZZ_ASSERT(rBuf); - ZSTD_compressCCtx(cctx, rBuf, bufSize, src, size, level); - free(rBuf); - } + void *rBuf = malloc(bufSize); + FUZZ_ASSERT(rBuf); + ZSTD_compressCCtx(cctx, rBuf, bufSize, src, size, level); + free(rBuf); + FUZZ_dataProducer_free(producer); #ifndef STATEFUL_FUZZING ZSTD_freeCCtx(cctx); cctx = NULL; #endif diff --git a/tests/fuzz/simple_decompress.c b/tests/fuzz/simple_decompress.c index a68813ee5..0ab634fa0 100644 --- a/tests/fuzz/simple_decompress.c +++ b/tests/fuzz/simple_decompress.c @@ -25,7 +25,6 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); - int i; if (!dctx) { dctx = ZSTD_createDCtx(); FUZZ_ASSERT(dctx); @@ -37,7 +36,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) /* Restrict to remaining data. If we run out of data while generating params, we should still continue and let decompression happen on empty data. */ - size = FUZZ_dataProducer_remainingBytes(producer); + size = FUZZ_dataProducer_remainingBytes(producer); ZSTD_decompressDCtx(dctx, rBuf, bufSize, src, size); free(rBuf); diff --git a/tests/fuzz/simple_round_trip.c b/tests/fuzz/simple_round_trip.c index 7e3b66098..b88d404f9 100644 --- a/tests/fuzz/simple_round_trip.c +++ b/tests/fuzz/simple_round_trip.c @@ -20,23 +20,24 @@ #include #include "fuzz_helpers.h" #include "zstd_helpers.h" +#include "fuzz_data_producer.h" static const int kMaxClevel = 19; static ZSTD_CCtx *cctx = NULL; static ZSTD_DCtx *dctx = NULL; -static uint32_t seed; static size_t roundTripTest(void *result, size_t resultCapacity, void *compressed, size_t compressedCapacity, - const void *src, size_t srcSize) + const void *src, size_t srcSize, + FUZZ_dataProducer_t *producer) { size_t cSize; - if (FUZZ_rand(&seed) & 1) { - FUZZ_setRandomParameters(cctx, srcSize, &seed); + if (FUZZ_dataProducer_uint32(producer) & 1) { + FUZZ_setRandomParameters(cctx, srcSize, producer); cSize = ZSTD_compress2(cctx, compressed, compressedCapacity, src, srcSize); } else { - int const cLevel = FUZZ_rand(&seed) % kMaxClevel; + int const cLevel = FUZZ_dataProducer_uint32(producer) % kMaxClevel; cSize = ZSTD_compressCCtx( cctx, compressed, compressedCapacity, src, srcSize, cLevel); } @@ -51,12 +52,12 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) size_t cBufSize = ZSTD_compressBound(size); void* cBuf; - seed = FUZZ_seed(&src, &size); + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); /* Half of the time fuzz with a 1 byte smaller output size. * This will still succeed because we don't use a dictionary, so the dictID * field is empty, giving us 4 bytes of overhead. */ - cBufSize -= FUZZ_rand32(&seed, 0, 1); + cBufSize -= FUZZ_dataProducer_uint32Range(producer, 0, 1); cBuf = malloc(cBufSize); FUZZ_ASSERT(cBuf && rBuf); @@ -72,13 +73,14 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { size_t const result = - roundTripTest(rBuf, rBufSize, cBuf, cBufSize, src, size); + roundTripTest(rBuf, rBufSize, cBuf, cBufSize, src, size, producer); FUZZ_ZASSERT(result); FUZZ_ASSERT_MSG(result == size, "Incorrect regenerated size"); FUZZ_ASSERT_MSG(!memcmp(src, rBuf, size), "Corruption!"); } free(rBuf); free(cBuf); + FUZZ_dataProducer_free(producer); #ifndef STATEFUL_FUZZING ZSTD_freeCCtx(cctx); cctx = NULL; ZSTD_freeDCtx(dctx); dctx = NULL; diff --git a/tests/fuzz/stream_decompress.c b/tests/fuzz/stream_decompress.c index 68e120d7e..4d5c49966 100644 --- a/tests/fuzz/stream_decompress.c +++ b/tests/fuzz/stream_decompress.c @@ -19,6 +19,7 @@ #include #include "fuzz_helpers.h" #include "zstd.h" +#include "fuzz_data_producer.h" static size_t const kBufSize = ZSTD_BLOCKSIZE_MAX; @@ -26,22 +27,23 @@ static ZSTD_DStream *dstream = NULL; static void* buf = NULL; uint32_t seed; -static ZSTD_outBuffer makeOutBuffer(void) +static ZSTD_outBuffer makeOutBuffer(FUZZ_dataProducer_t *producer) { ZSTD_outBuffer buffer = { buf, 0, 0 }; - buffer.size = (FUZZ_rand(&seed) % kBufSize) + 1; + buffer.size = (FUZZ_dataProducer_uint32(producer) % kBufSize) + 1; FUZZ_ASSERT(buffer.size <= kBufSize); return buffer; } -static ZSTD_inBuffer makeInBuffer(const uint8_t **src, size_t *size) +static ZSTD_inBuffer makeInBuffer(const uint8_t **src, size_t *size, + FUZZ_dataProducer_t *producer) { ZSTD_inBuffer buffer = { *src, 0, 0 }; FUZZ_ASSERT(*size > 0); - buffer.size = (FUZZ_rand(&seed) % *size) + 1; + buffer.size = (FUZZ_dataProducer_uint32(producer) % *size) + 1; FUZZ_ASSERT(buffer.size <= *size); *src += buffer.size; *size -= buffer.size; @@ -51,13 +53,17 @@ static ZSTD_inBuffer makeInBuffer(const uint8_t **src, size_t *size) int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { - seed = FUZZ_seed(&src, &size); + /* Give a random portion of src data to the producer, to use for + parameter generation. The rest will be used for (de)compression */ + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + size_t producerSliceSize = FUZZ_dataProducer_uint32Range(producer, 0, size); + size = FUZZ_dataProducer_contract(producer, producerSliceSize); /* Allocate all buffers and contexts if not already allocated */ if (!buf) { buf = malloc(kBufSize); - FUZZ_ASSERT(buf); - } + FUZZ_ASSERT(buf); + } if (!dstream) { dstream = ZSTD_createDStream(); @@ -67,9 +73,9 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) } while (size > 0) { - ZSTD_inBuffer in = makeInBuffer(&src, &size); + ZSTD_inBuffer in = makeInBuffer(&src, &size, producer); while (in.pos != in.size) { - ZSTD_outBuffer out = makeOutBuffer(); + ZSTD_outBuffer out = makeOutBuffer(producer); size_t const rc = ZSTD_decompressStream(dstream, &out, &in); if (ZSTD_isError(rc)) goto error; } @@ -79,5 +85,6 @@ error: #ifndef STATEFUL_FUZZING ZSTD_freeDStream(dstream); dstream = NULL; #endif + FUZZ_dataProducer_free(producer); return 0; } diff --git a/tests/fuzz/stream_round_trip.c b/tests/fuzz/stream_round_trip.c index d13c2dbe7..4569222f1 100644 --- a/tests/fuzz/stream_round_trip.c +++ b/tests/fuzz/stream_round_trip.c @@ -20,31 +20,33 @@ #include #include "fuzz_helpers.h" #include "zstd_helpers.h" +#include "fuzz_data_producer.h" ZSTD_CCtx *cctx = NULL; static ZSTD_DCtx *dctx = NULL; static uint8_t* cBuf = NULL; static uint8_t* rBuf = NULL; static size_t bufSize = 0; -static uint32_t seed; -static ZSTD_outBuffer makeOutBuffer(uint8_t *dst, size_t capacity) +static ZSTD_outBuffer makeOutBuffer(uint8_t *dst, size_t capacity, + FUZZ_dataProducer_t *producer) { ZSTD_outBuffer buffer = { dst, 0, 0 }; FUZZ_ASSERT(capacity > 0); - buffer.size = (FUZZ_rand(&seed) % capacity) + 1; + buffer.size = (FUZZ_dataProducer_uint32(producer) % capacity) + 1; FUZZ_ASSERT(buffer.size <= capacity); return buffer; } -static ZSTD_inBuffer makeInBuffer(const uint8_t **src, size_t *size) +static ZSTD_inBuffer makeInBuffer(const uint8_t **src, size_t *size, + FUZZ_dataProducer_t *producer) { ZSTD_inBuffer buffer = { *src, 0, 0 }; FUZZ_ASSERT(*size > 0); - buffer.size = (FUZZ_rand(&seed) % *size) + 1; + buffer.size = (FUZZ_dataProducer_uint32(producer) % *size) + 1; FUZZ_ASSERT(buffer.size <= *size); *src += buffer.size; *size -= buffer.size; @@ -53,23 +55,24 @@ static ZSTD_inBuffer makeInBuffer(const uint8_t **src, size_t *size) } static size_t compress(uint8_t *dst, size_t capacity, - const uint8_t *src, size_t srcSize) + const uint8_t *src, size_t srcSize, + FUZZ_dataProducer_t *producer) { size_t dstSize = 0; ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); - FUZZ_setRandomParameters(cctx, srcSize, &seed); + FUZZ_setRandomParameters(cctx, srcSize, producer); while (srcSize > 0) { - ZSTD_inBuffer in = makeInBuffer(&src, &srcSize); + ZSTD_inBuffer in = makeInBuffer(&src, &srcSize, producer); /* Mode controls the action. If mode == -1 we pick a new mode */ int mode = -1; while (in.pos < in.size || mode != -1) { - ZSTD_outBuffer out = makeOutBuffer(dst, capacity); + ZSTD_outBuffer out = makeOutBuffer(dst, capacity, producer); /* Previous action finished, pick a new mode. */ - if (mode == -1) mode = FUZZ_rand(&seed) % 10; + if (mode == -1) mode = FUZZ_dataProducer_uint32(producer) % 10; switch (mode) { - case 0: /* fall-though */ - case 1: /* fall-though */ + case 0: /* fall-through */ + case 1: /* fall-through */ case 2: { size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush); @@ -85,9 +88,9 @@ static size_t compress(uint8_t *dst, size_t capacity, /* Reset the compressor when the frame is finished */ if (ret == 0) { ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); - if ((FUZZ_rand(&seed) & 7) == 0) { + if ((FUZZ_dataProducer_uint32(producer) & 7) == 0) { size_t const remaining = in.size - in.pos; - FUZZ_setRandomParameters(cctx, remaining, &seed); + FUZZ_setRandomParameters(cctx, remaining, producer); } mode = -1; } @@ -107,7 +110,7 @@ static size_t compress(uint8_t *dst, size_t capacity, } for (;;) { ZSTD_inBuffer in = {NULL, 0, 0}; - ZSTD_outBuffer out = makeOutBuffer(dst, capacity); + ZSTD_outBuffer out = makeOutBuffer(dst, capacity, producer); size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end); FUZZ_ZASSERT(ret); @@ -123,9 +126,13 @@ static size_t compress(uint8_t *dst, size_t capacity, int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { size_t neededBufSize; + neededBufSize = ZSTD_compressBound(size) * 5; - seed = FUZZ_seed(&src, &size); - neededBufSize = ZSTD_compressBound(size) * 2; + /* Give a random portion of src data to the producer, to use for + parameter generation. The rest will be used for (de)compression */ + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + size_t producerSliceSize = FUZZ_dataProducer_uint32Range(producer, 0, size); + size = FUZZ_dataProducer_contract(producer, producerSliceSize); /* Allocate all buffers and contexts if not already allocated */ if (neededBufSize > bufSize) { @@ -146,7 +153,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) } { - size_t const cSize = compress(cBuf, neededBufSize, src, size); + size_t const cSize = compress(cBuf, neededBufSize, src, size, producer); size_t const rSize = ZSTD_decompressDCtx(dctx, rBuf, neededBufSize, cBuf, cSize); FUZZ_ZASSERT(rSize); @@ -154,6 +161,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) FUZZ_ASSERT_MSG(!memcmp(src, rBuf, size), "Corruption!"); } + FUZZ_dataProducer_free(producer); #ifndef STATEFUL_FUZZING ZSTD_freeCCtx(cctx); cctx = NULL; ZSTD_freeDCtx(dctx); dctx = NULL; diff --git a/tests/fuzz/zstd_helpers.c b/tests/fuzz/zstd_helpers.c index 5ff057b8c..2635de4cc 100644 --- a/tests/fuzz/zstd_helpers.c +++ b/tests/fuzz/zstd_helpers.c @@ -23,47 +23,47 @@ static void set(ZSTD_CCtx *cctx, ZSTD_cParameter param, int value) } static void setRand(ZSTD_CCtx *cctx, ZSTD_cParameter param, unsigned min, - unsigned max, uint32_t *state) { - unsigned const value = FUZZ_rand32(state, min, max); + unsigned max, FUZZ_dataProducer_t *producer) { + unsigned const value = FUZZ_dataProducer_uint32Range(producer, min, max); set(cctx, param, value); } -ZSTD_compressionParameters FUZZ_randomCParams(size_t srcSize, uint32_t *state) +ZSTD_compressionParameters FUZZ_randomCParams(size_t srcSize, FUZZ_dataProducer_t *producer) { /* Select compression parameters */ ZSTD_compressionParameters cParams; - cParams.windowLog = FUZZ_rand32(state, ZSTD_WINDOWLOG_MIN, 15); - cParams.hashLog = FUZZ_rand32(state, ZSTD_HASHLOG_MIN, 15); - cParams.chainLog = FUZZ_rand32(state, ZSTD_CHAINLOG_MIN, 16); - cParams.searchLog = FUZZ_rand32(state, ZSTD_SEARCHLOG_MIN, 9); - cParams.minMatch = FUZZ_rand32(state, ZSTD_MINMATCH_MIN, + cParams.windowLog = FUZZ_dataProducer_uint32Range(producer, ZSTD_WINDOWLOG_MIN, 15); + cParams.hashLog = FUZZ_dataProducer_uint32Range(producer, ZSTD_HASHLOG_MIN, 15); + cParams.chainLog = FUZZ_dataProducer_uint32Range(producer, ZSTD_CHAINLOG_MIN, 16); + cParams.searchLog = FUZZ_dataProducer_uint32Range(producer, ZSTD_SEARCHLOG_MIN, 9); + cParams.minMatch = FUZZ_dataProducer_uint32Range(producer, ZSTD_MINMATCH_MIN, ZSTD_MINMATCH_MAX); - cParams.targetLength = FUZZ_rand32(state, 0, 512); - cParams.strategy = FUZZ_rand32(state, ZSTD_STRATEGY_MIN, ZSTD_STRATEGY_MAX); + cParams.targetLength = FUZZ_dataProducer_uint32Range(producer, 0, 512); + cParams.strategy = FUZZ_dataProducer_uint32Range(producer, ZSTD_STRATEGY_MIN, ZSTD_STRATEGY_MAX); return ZSTD_adjustCParams(cParams, srcSize, 0); } -ZSTD_frameParameters FUZZ_randomFParams(uint32_t *state) +ZSTD_frameParameters FUZZ_randomFParams(FUZZ_dataProducer_t *producer) { /* Select frame parameters */ ZSTD_frameParameters fParams; - fParams.contentSizeFlag = FUZZ_rand32(state, 0, 1); - fParams.checksumFlag = FUZZ_rand32(state, 0, 1); - fParams.noDictIDFlag = FUZZ_rand32(state, 0, 1); + fParams.contentSizeFlag = FUZZ_dataProducer_uint32Range(producer, 0, 1); + fParams.checksumFlag = FUZZ_dataProducer_uint32Range(producer, 0, 1); + fParams.noDictIDFlag = FUZZ_dataProducer_uint32Range(producer, 0, 1); return fParams; } -ZSTD_parameters FUZZ_randomParams(size_t srcSize, uint32_t *state) +ZSTD_parameters FUZZ_randomParams(size_t srcSize, FUZZ_dataProducer_t *producer) { ZSTD_parameters params; - params.cParams = FUZZ_randomCParams(srcSize, state); - params.fParams = FUZZ_randomFParams(state); + params.cParams = FUZZ_randomCParams(srcSize, producer); + params.fParams = FUZZ_randomFParams(producer); return params; } -void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, uint32_t *state) +void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer_t *producer) { - ZSTD_compressionParameters cParams = FUZZ_randomCParams(srcSize, state); + ZSTD_compressionParameters cParams = FUZZ_randomCParams(srcSize, producer); set(cctx, ZSTD_c_windowLog, cParams.windowLog); set(cctx, ZSTD_c_hashLog, cParams.hashLog); set(cctx, ZSTD_c_chainLog, cParams.chainLog); @@ -72,30 +72,30 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, uint32_t *state) set(cctx, ZSTD_c_targetLength, cParams.targetLength); set(cctx, ZSTD_c_strategy, cParams.strategy); /* Select frame parameters */ - setRand(cctx, ZSTD_c_contentSizeFlag, 0, 1, state); - setRand(cctx, ZSTD_c_checksumFlag, 0, 1, state); - setRand(cctx, ZSTD_c_dictIDFlag, 0, 1, state); + setRand(cctx, ZSTD_c_contentSizeFlag, 0, 1, producer); + setRand(cctx, ZSTD_c_checksumFlag, 0, 1, producer); + setRand(cctx, ZSTD_c_dictIDFlag, 0, 1, producer); /* Select long distance matching parameters */ - setRand(cctx, ZSTD_c_enableLongDistanceMatching, 0, 1, state); - setRand(cctx, ZSTD_c_ldmHashLog, ZSTD_HASHLOG_MIN, 16, state); + setRand(cctx, ZSTD_c_enableLongDistanceMatching, 0, 1, producer); + setRand(cctx, ZSTD_c_ldmHashLog, ZSTD_HASHLOG_MIN, 16, producer); setRand(cctx, ZSTD_c_ldmMinMatch, ZSTD_LDM_MINMATCH_MIN, - ZSTD_LDM_MINMATCH_MAX, state); + ZSTD_LDM_MINMATCH_MAX, producer); setRand(cctx, ZSTD_c_ldmBucketSizeLog, 0, ZSTD_LDM_BUCKETSIZELOG_MAX, - state); + producer); setRand(cctx, ZSTD_c_ldmHashRateLog, ZSTD_LDM_HASHRATELOG_MIN, - ZSTD_LDM_HASHRATELOG_MAX, state); + ZSTD_LDM_HASHRATELOG_MAX, producer); /* Set misc parameters */ - setRand(cctx, ZSTD_c_nbWorkers, 0, 2, state); - setRand(cctx, ZSTD_c_rsyncable, 0, 1, state); - setRand(cctx, ZSTD_c_forceMaxWindow, 0, 1, state); - setRand(cctx, ZSTD_c_literalCompressionMode, 0, 2, state); - setRand(cctx, ZSTD_c_forceAttachDict, 0, 2, state); - if (FUZZ_rand32(state, 0, 1) == 0) { - setRand(cctx, ZSTD_c_srcSizeHint, ZSTD_SRCSIZEHINT_MIN, 2 * srcSize, state); + setRand(cctx, ZSTD_c_nbWorkers, 0, 2, producer); + setRand(cctx, ZSTD_c_rsyncable, 0, 1, producer); + setRand(cctx, ZSTD_c_forceMaxWindow, 0, 1, producer); + setRand(cctx, ZSTD_c_literalCompressionMode, 0, 2, producer); + setRand(cctx, ZSTD_c_forceAttachDict, 0, 2, producer); + if (FUZZ_dataProducer_uint32Range(producer, 0, 1) == 0) { + setRand(cctx, ZSTD_c_srcSizeHint, ZSTD_SRCSIZEHINT_MIN, 2 * srcSize, producer); } } -FUZZ_dict_t FUZZ_train(void const* src, size_t srcSize, uint32_t *state) +FUZZ_dict_t FUZZ_train(void const* src, size_t srcSize, FUZZ_dataProducer_t *producer) { size_t const dictSize = MAX(srcSize / 8, 1024); size_t const totalSampleSize = dictSize * 11; @@ -110,7 +110,7 @@ FUZZ_dict_t FUZZ_train(void const* src, size_t srcSize, uint32_t *state) for (sample = 0; sample < nbSamples; ++sample) { size_t const remaining = totalSampleSize - pos; - size_t const offset = FUZZ_rand32(state, 0, MAX(srcSize, 1) - 1); + size_t const offset = FUZZ_dataProducer_uint32Range(producer, 0, MAX(srcSize, 1) - 1); size_t const limit = MIN(srcSize - offset, remaining); size_t const toCopy = MIN(limit, remaining / (nbSamples - sample)); memcpy(samples + pos, src + offset, toCopy); diff --git a/tests/fuzz/zstd_helpers.h b/tests/fuzz/zstd_helpers.h index 457e6e995..f2001f8b2 100644 --- a/tests/fuzz/zstd_helpers.h +++ b/tests/fuzz/zstd_helpers.h @@ -17,17 +17,18 @@ #define ZSTD_STATIC_LINKING_ONLY #include "zstd.h" +#include "fuzz_data_producer.h" #include #ifdef __cplusplus extern "C" { #endif -void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, uint32_t *state); +void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer_t *producer); -ZSTD_compressionParameters FUZZ_randomCParams(size_t srcSize, uint32_t *state); -ZSTD_frameParameters FUZZ_randomFParams(uint32_t *state); -ZSTD_parameters FUZZ_randomParams(size_t srcSize, uint32_t *state); +ZSTD_compressionParameters FUZZ_randomCParams(size_t srcSize, FUZZ_dataProducer_t *producer); +ZSTD_frameParameters FUZZ_randomFParams(FUZZ_dataProducer_t *producer); +ZSTD_parameters FUZZ_randomParams(size_t srcSize, FUZZ_dataProducer_t *producer); typedef struct { void* buff; @@ -38,7 +39,7 @@ typedef struct { * NOTE: Don't use this to train production dictionaries, it is only optimized * for speed, and doesn't care about dictionary quality. */ -FUZZ_dict_t FUZZ_train(void const* src, size_t srcSize, uint32_t *state); +FUZZ_dict_t FUZZ_train(void const* src, size_t srcSize, FUZZ_dataProducer_t *producer); #ifdef __cplusplus From 23cc2d851083fdc1c1af15fa4190ce74e29b73dd Mon Sep 17 00:00:00 2001 From: Dario Pavlovic Date: Tue, 10 Sep 2019 16:52:38 -0700 Subject: [PATCH 2/6] All tests should give some portion of data to the producer and use the rest. --- tests/fuzz/block_round_trip.c | 6 +++++- tests/fuzz/dictionary_decompress.c | 9 +++++++-- tests/fuzz/dictionary_round_trip.c | 11 +++++++---- tests/fuzz/simple_compress.c | 9 ++++++--- tests/fuzz/simple_decompress.c | 30 +++++++++++++++--------------- tests/fuzz/simple_round_trip.c | 7 +++++++ tests/fuzz/stream_round_trip.c | 6 +++--- tests/fuzz/zstd_frame_info.c | 4 ---- 8 files changed, 50 insertions(+), 32 deletions(-) diff --git a/tests/fuzz/block_round_trip.c b/tests/fuzz/block_round_trip.c index 326d5b247..9ead57a5a 100644 --- a/tests/fuzz/block_round_trip.c +++ b/tests/fuzz/block_round_trip.c @@ -52,9 +52,13 @@ static size_t roundTripTest(void *result, size_t resultCapacity, int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { + /* Give a random portion of src data to the producer, to use for + parameter generation. The rest will be used for (de)compression */ FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + size_t producerSliceSize = FUZZ_dataProducer_uint32Range(producer, 0, size); + size = FUZZ_dataProducer_contract(producer, producerSliceSize); + int cLevel = FUZZ_dataProducer_uint32(producer) % kMaxClevel; - size = FUZZ_dataProducer_remainingBytes(producer); size_t neededBufSize = size; if (size > ZSTD_BLOCKSIZE_MAX) diff --git a/tests/fuzz/dictionary_decompress.c b/tests/fuzz/dictionary_decompress.c index 5f660ac52..54903f1f6 100644 --- a/tests/fuzz/dictionary_decompress.c +++ b/tests/fuzz/dictionary_decompress.c @@ -24,7 +24,12 @@ static ZSTD_DCtx *dctx = NULL; int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { + /* Give a random portion of src data to the producer, to use for + parameter generation. The rest will be used for (de)compression */ FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + size_t producerSliceSize = FUZZ_dataProducer_uint32Range(producer, 0, size); + size = FUZZ_dataProducer_contract(producer, producerSliceSize); + FUZZ_dict_t dict; ZSTD_DDict* ddict = NULL; int i; @@ -43,8 +48,8 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) (ZSTD_dictLoadMethod_e)FUZZ_dataProducer_uint32Range(producer, 0, 1), (ZSTD_dictContentType_e)FUZZ_dataProducer_uint32Range(producer, 0, 2))); } - /* Run it 10 times over 10 output sizes. Reuse the context and dict. */ - for (i = 0; i < 10; ++i) { + + { size_t const bufSize = FUZZ_dataProducer_uint32Range(producer, 0, 2 * size); void* rBuf = malloc(bufSize); FUZZ_ASSERT(rBuf); diff --git a/tests/fuzz/dictionary_round_trip.c b/tests/fuzz/dictionary_round_trip.c index fe0a217b2..cd1ca38f7 100644 --- a/tests/fuzz/dictionary_round_trip.c +++ b/tests/fuzz/dictionary_round_trip.c @@ -68,13 +68,16 @@ static size_t roundTripTest(void *result, size_t resultCapacity, int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { + /* Give a random portion of src data to the producer, to use for + parameter generation. The rest will be used for (de)compression */ + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + size_t producerSliceSize = FUZZ_dataProducer_uint32Range(producer, 0, size); + size = FUZZ_dataProducer_contract(producer, producerSliceSize); + size_t const rBufSize = size; void* rBuf = malloc(rBufSize); size_t cBufSize = ZSTD_compressBound(size); - void* cBuf; - - FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); - + void *cBuf; /* Half of the time fuzz with a 1 byte smaller output size. * This will still succeed because we force the checksum to be disabled, * giving us 4 bytes of overhead. diff --git a/tests/fuzz/simple_compress.c b/tests/fuzz/simple_compress.c index 29de4701f..b8c6394cb 100644 --- a/tests/fuzz/simple_compress.c +++ b/tests/fuzz/simple_compress.c @@ -24,14 +24,17 @@ static ZSTD_CCtx *cctx = NULL; int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { + /* Give a random portion of src data to the producer, to use for + parameter generation. The rest will be used for (de)compression */ FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + size_t producerSliceSize = FUZZ_dataProducer_uint32Range(producer, 0, size); + size = FUZZ_dataProducer_contract(producer, producerSliceSize); - int const level = (int)FUZZ_dataProducer_uint32Range( - producer, 0, 19 + 3) - 3; /* [-3, 19] */ size_t const maxSize = ZSTD_compressBound(size); size_t const bufSize = FUZZ_dataProducer_uint32Range(producer, 0, maxSize); - size = FUZZ_dataProducer_remainingBytes(producer); + int const level = (int)FUZZ_dataProducer_uint32Range( + producer, 0, 19 + 3) - 3; /* [-3, 19] */ if (!cctx) { cctx = ZSTD_createCCtx(); diff --git a/tests/fuzz/simple_decompress.c b/tests/fuzz/simple_decompress.c index 0ab634fa0..5c7680bf0 100644 --- a/tests/fuzz/simple_decompress.c +++ b/tests/fuzz/simple_decompress.c @@ -23,25 +23,25 @@ static ZSTD_DCtx *dctx = NULL; int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { - FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + /* Give a random portion of src data to the producer, to use for + parameter generation. The rest will be used for (de)compression */ + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + size_t producerSliceSize = FUZZ_dataProducer_uint32Range(producer, 0, size); + size = FUZZ_dataProducer_contract(producer, producerSliceSize); - if (!dctx) { - dctx = ZSTD_createDCtx(); - FUZZ_ASSERT(dctx); - } + if (!dctx) { + dctx = ZSTD_createDCtx(); + FUZZ_ASSERT(dctx); + } - size_t const bufSize = FUZZ_dataProducer_uint32Range(producer, 0, 10 * size); - void *rBuf = malloc(bufSize); - FUZZ_ASSERT(rBuf); + size_t const bufSize = FUZZ_dataProducer_uint32Range(producer, 0, 10 * size); + void *rBuf = malloc(bufSize); + FUZZ_ASSERT(rBuf); - /* Restrict to remaining data. If we run out of data while generating params, - we should still continue and let decompression happen on empty data. */ - size = FUZZ_dataProducer_remainingBytes(producer); + ZSTD_decompressDCtx(dctx, rBuf, bufSize, src, size); + free(rBuf); - ZSTD_decompressDCtx(dctx, rBuf, bufSize, src, size); - free(rBuf); - - FUZZ_dataProducer_free(producer); + FUZZ_dataProducer_free(producer); #ifndef STATEFUL_FUZZING ZSTD_freeDCtx(dctx); dctx = NULL; diff --git a/tests/fuzz/simple_round_trip.c b/tests/fuzz/simple_round_trip.c index b88d404f9..f19e37a35 100644 --- a/tests/fuzz/simple_round_trip.c +++ b/tests/fuzz/simple_round_trip.c @@ -52,12 +52,19 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) size_t cBufSize = ZSTD_compressBound(size); void* cBuf; + /* Give a random portion of src data to the producer, to use for + parameter generation. The rest will be used for (de)compression */ FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + size_t producerSliceSize = FUZZ_dataProducer_uint32Range(producer, 0, size); + size = FUZZ_dataProducer_contract(producer, producerSliceSize); + /* Half of the time fuzz with a 1 byte smaller output size. * This will still succeed because we don't use a dictionary, so the dictID * field is empty, giving us 4 bytes of overhead. */ cBufSize -= FUZZ_dataProducer_uint32Range(producer, 0, 1); + size = FUZZ_dataProducer_remainingBytes(producer); + cBuf = malloc(cBufSize); FUZZ_ASSERT(cBuf && rBuf); diff --git a/tests/fuzz/stream_round_trip.c b/tests/fuzz/stream_round_trip.c index 4569222f1..02bb26daa 100644 --- a/tests/fuzz/stream_round_trip.c +++ b/tests/fuzz/stream_round_trip.c @@ -125,15 +125,15 @@ static size_t compress(uint8_t *dst, size_t capacity, int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { - size_t neededBufSize; - neededBufSize = ZSTD_compressBound(size) * 5; - /* Give a random portion of src data to the producer, to use for parameter generation. The rest will be used for (de)compression */ FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); size_t producerSliceSize = FUZZ_dataProducer_uint32Range(producer, 0, size); size = FUZZ_dataProducer_contract(producer, producerSliceSize); + size_t neededBufSize; + neededBufSize = ZSTD_compressBound(size) * 5; + /* Allocate all buffers and contexts if not already allocated */ if (neededBufSize > bufSize) { free(cBuf); diff --git a/tests/fuzz/zstd_frame_info.c b/tests/fuzz/zstd_frame_info.c index 7512d5f49..359cf128f 100644 --- a/tests/fuzz/zstd_frame_info.c +++ b/tests/fuzz/zstd_frame_info.c @@ -21,10 +21,6 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { ZSTD_frameHeader zfh; - /* Consume the seed to be compatible with the corpora of other decompression - * fuzzers. - */ - FUZZ_seed(&src, &size); /* You can fuzz any helper functions here that are fast, and take zstd * compressed data as input. E.g. don't expect the input to be a dictionary, * so don't fuzz ZSTD_getDictID_fromDict(). From b5b24c2a0d68aee159e01e8d0ad2f657f7c10b5c Mon Sep 17 00:00:00 2001 From: Dario Pavlovic Date: Wed, 11 Sep 2019 10:09:29 -0700 Subject: [PATCH 3/6] Combining fuzz_data_producer restrict calls into a single function --- tests/fuzz/block_round_trip.c | 3 +-- tests/fuzz/dictionary_decompress.c | 3 +-- tests/fuzz/dictionary_round_trip.c | 3 +-- tests/fuzz/fuzz_data_producer.c | 7 +++++++ tests/fuzz/fuzz_data_producer.h | 11 +++++++---- tests/fuzz/simple_compress.c | 3 +-- tests/fuzz/simple_decompress.c | 3 +-- tests/fuzz/simple_round_trip.c | 3 +-- tests/fuzz/stream_decompress.c | 3 +-- tests/fuzz/stream_round_trip.c | 3 +-- 10 files changed, 22 insertions(+), 20 deletions(-) diff --git a/tests/fuzz/block_round_trip.c b/tests/fuzz/block_round_trip.c index 9ead57a5a..a0079d352 100644 --- a/tests/fuzz/block_round_trip.c +++ b/tests/fuzz/block_round_trip.c @@ -55,8 +55,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) /* Give a random portion of src data to the producer, to use for parameter generation. The rest will be used for (de)compression */ FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); - size_t producerSliceSize = FUZZ_dataProducer_uint32Range(producer, 0, size); - size = FUZZ_dataProducer_contract(producer, producerSliceSize); + size = FUZZ_dataProducer_reserveDataPrefix(producer); int cLevel = FUZZ_dataProducer_uint32(producer) % kMaxClevel; diff --git a/tests/fuzz/dictionary_decompress.c b/tests/fuzz/dictionary_decompress.c index 54903f1f6..3bbd9bf5b 100644 --- a/tests/fuzz/dictionary_decompress.c +++ b/tests/fuzz/dictionary_decompress.c @@ -27,8 +27,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) /* Give a random portion of src data to the producer, to use for parameter generation. The rest will be used for (de)compression */ FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); - size_t producerSliceSize = FUZZ_dataProducer_uint32Range(producer, 0, size); - size = FUZZ_dataProducer_contract(producer, producerSliceSize); + size = FUZZ_dataProducer_reserveDataPrefix(producer); FUZZ_dict_t dict; ZSTD_DDict* ddict = NULL; diff --git a/tests/fuzz/dictionary_round_trip.c b/tests/fuzz/dictionary_round_trip.c index cd1ca38f7..5a4b9503a 100644 --- a/tests/fuzz/dictionary_round_trip.c +++ b/tests/fuzz/dictionary_round_trip.c @@ -71,8 +71,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) /* Give a random portion of src data to the producer, to use for parameter generation. The rest will be used for (de)compression */ FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); - size_t producerSliceSize = FUZZ_dataProducer_uint32Range(producer, 0, size); - size = FUZZ_dataProducer_contract(producer, producerSliceSize); + size = FUZZ_dataProducer_reserveDataPrefix(producer); size_t const rBufSize = size; void* rBuf = malloc(rBufSize); diff --git a/tests/fuzz/fuzz_data_producer.c b/tests/fuzz/fuzz_data_producer.c index 6dcc1413d..d6893e4f9 100644 --- a/tests/fuzz/fuzz_data_producer.c +++ b/tests/fuzz/fuzz_data_producer.c @@ -65,3 +65,10 @@ size_t FUZZ_dataProducer_contract(FUZZ_dataProducer_t *producer, size_t newSize) producer->size = newSize; return remaining; } + +size_t FUZZ_dataProducer_reserveDataPrefix(FUZZ_dataProducer_t *producer) +{ + size_t producerSliceSize = FUZZ_dataProducer_uint32Range( + producer, 0, producer->size); + return FUZZ_dataProducer_contract(producer, producerSliceSize); +} diff --git a/tests/fuzz/fuzz_data_producer.h b/tests/fuzz/fuzz_data_producer.h index 668c87f29..8eea1e256 100644 --- a/tests/fuzz/fuzz_data_producer.h +++ b/tests/fuzz/fuzz_data_producer.h @@ -44,10 +44,13 @@ uint32_t FUZZ_dataProducer_uint32(FUZZ_dataProducer_t *producer); /* Returns the size of the remaining bytes of data in the producer */ size_t FUZZ_dataProducer_remainingBytes(FUZZ_dataProducer_t *producer); -/* Tells the producer to contract to newSize bytes of data it currently uses, -counted from the end, and forget about the rest. If newSize > current data size, -nothing happens. Returns the number of bytes the producer won't use anymore, -after contracting. */ +/* Restricts the producer to only the last newSize bytes of data. +If newSize > current data size, nothing happens. Returns the number of bytes +the producer won't use anymore, after contracting. */ size_t FUZZ_dataProducer_contract(FUZZ_dataProducer_t *producer, size_t newSize); +/* Restricts the producer to use only the last X bytes of data, where X is + a random number in the interval [0, data_size]. Returns the size of the + remaining data the producer won't use anymore (the prefix). */ +size_t FUZZ_dataProducer_reserveDataPrefix(FUZZ_dataProducer_t *producer); #endif // FUZZ_DATA_PRODUCER_H diff --git a/tests/fuzz/simple_compress.c b/tests/fuzz/simple_compress.c index b8c6394cb..74ab0d631 100644 --- a/tests/fuzz/simple_compress.c +++ b/tests/fuzz/simple_compress.c @@ -27,8 +27,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) /* Give a random portion of src data to the producer, to use for parameter generation. The rest will be used for (de)compression */ FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); - size_t producerSliceSize = FUZZ_dataProducer_uint32Range(producer, 0, size); - size = FUZZ_dataProducer_contract(producer, producerSliceSize); + size = FUZZ_dataProducer_reserveDataPrefix(producer); size_t const maxSize = ZSTD_compressBound(size); size_t const bufSize = FUZZ_dataProducer_uint32Range(producer, 0, maxSize); diff --git a/tests/fuzz/simple_decompress.c b/tests/fuzz/simple_decompress.c index 5c7680bf0..6182746a1 100644 --- a/tests/fuzz/simple_decompress.c +++ b/tests/fuzz/simple_decompress.c @@ -26,8 +26,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) /* Give a random portion of src data to the producer, to use for parameter generation. The rest will be used for (de)compression */ FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); - size_t producerSliceSize = FUZZ_dataProducer_uint32Range(producer, 0, size); - size = FUZZ_dataProducer_contract(producer, producerSliceSize); + size = FUZZ_dataProducer_reserveDataPrefix(producer); if (!dctx) { dctx = ZSTD_createDCtx(); diff --git a/tests/fuzz/simple_round_trip.c b/tests/fuzz/simple_round_trip.c index f19e37a35..4a07d16a4 100644 --- a/tests/fuzz/simple_round_trip.c +++ b/tests/fuzz/simple_round_trip.c @@ -55,8 +55,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) /* Give a random portion of src data to the producer, to use for parameter generation. The rest will be used for (de)compression */ FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); - size_t producerSliceSize = FUZZ_dataProducer_uint32Range(producer, 0, size); - size = FUZZ_dataProducer_contract(producer, producerSliceSize); + size = FUZZ_dataProducer_reserveDataPrefix(producer); /* Half of the time fuzz with a 1 byte smaller output size. * This will still succeed because we don't use a dictionary, so the dictID diff --git a/tests/fuzz/stream_decompress.c b/tests/fuzz/stream_decompress.c index 4d5c49966..70582e11b 100644 --- a/tests/fuzz/stream_decompress.c +++ b/tests/fuzz/stream_decompress.c @@ -56,8 +56,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) /* Give a random portion of src data to the producer, to use for parameter generation. The rest will be used for (de)compression */ FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); - size_t producerSliceSize = FUZZ_dataProducer_uint32Range(producer, 0, size); - size = FUZZ_dataProducer_contract(producer, producerSliceSize); + size = FUZZ_dataProducer_reserveDataPrefix(producer); /* Allocate all buffers and contexts if not already allocated */ if (!buf) { diff --git a/tests/fuzz/stream_round_trip.c b/tests/fuzz/stream_round_trip.c index 02bb26daa..08a4927a6 100644 --- a/tests/fuzz/stream_round_trip.c +++ b/tests/fuzz/stream_round_trip.c @@ -128,8 +128,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) /* Give a random portion of src data to the producer, to use for parameter generation. The rest will be used for (de)compression */ FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); - size_t producerSliceSize = FUZZ_dataProducer_uint32Range(producer, 0, size); - size = FUZZ_dataProducer_contract(producer, producerSliceSize); + size = FUZZ_dataProducer_reserveDataPrefix(producer); size_t neededBufSize; neededBufSize = ZSTD_compressBound(size) * 5; From 92c58c4d5d36f6ecbd2f3d2be8d7a4dc7d4d3900 Mon Sep 17 00:00:00 2001 From: Dario Pavlovic Date: Thu, 12 Sep 2019 12:40:12 -0700 Subject: [PATCH 4/6] Use range instead of the generic uint32 method to use less bytes when generating necessary numbers. --- tests/fuzz/block_round_trip.c | 5 ++--- tests/fuzz/dictionary_decompress.c | 3 +-- tests/fuzz/dictionary_round_trip.c | 6 ++---- tests/fuzz/fuzz_data_producer.c | 11 +++++++++++ tests/fuzz/fuzz_data_producer.h | 4 ++++ tests/fuzz/simple_compress.c | 6 +++--- tests/fuzz/simple_round_trip.c | 8 +++----- tests/fuzz/stream_decompress.c | 4 ++-- tests/fuzz/stream_round_trip.c | 8 ++++---- tests/fuzz/zstd_helpers.c | 3 +++ tests/fuzz/zstd_helpers.h | 4 +++- 11 files changed, 38 insertions(+), 24 deletions(-) diff --git a/tests/fuzz/block_round_trip.c b/tests/fuzz/block_round_trip.c index a0079d352..0f948d4fc 100644 --- a/tests/fuzz/block_round_trip.c +++ b/tests/fuzz/block_round_trip.c @@ -20,10 +20,9 @@ #include #include "fuzz_helpers.h" #include "zstd.h" +#include "zstd_helpers.h" #include "fuzz_data_producer.h" -static const int kMaxClevel = 19; - static ZSTD_CCtx *cctx = NULL; static ZSTD_DCtx *dctx = NULL; static void* cBuf = NULL; @@ -57,7 +56,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); size = FUZZ_dataProducer_reserveDataPrefix(producer); - int cLevel = FUZZ_dataProducer_uint32(producer) % kMaxClevel; + int const cLevel = FUZZ_dataProducer_int32Range(producer, kMinClevel, kMaxClevel); size_t neededBufSize = size; if (size > ZSTD_BLOCKSIZE_MAX) diff --git a/tests/fuzz/dictionary_decompress.c b/tests/fuzz/dictionary_decompress.c index 3bbd9bf5b..9cc69fa37 100644 --- a/tests/fuzz/dictionary_decompress.c +++ b/tests/fuzz/dictionary_decompress.c @@ -31,7 +31,6 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) FUZZ_dict_t dict; ZSTD_DDict* ddict = NULL; - int i; if (!dctx) { dctx = ZSTD_createDCtx(); @@ -49,7 +48,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) } { - size_t const bufSize = FUZZ_dataProducer_uint32Range(producer, 0, 2 * size); + size_t const bufSize = FUZZ_dataProducer_uint32Range(producer, 0, 10 * size); void* rBuf = malloc(bufSize); FUZZ_ASSERT(rBuf); if (ddict) { diff --git a/tests/fuzz/dictionary_round_trip.c b/tests/fuzz/dictionary_round_trip.c index 5a4b9503a..9411b50a7 100644 --- a/tests/fuzz/dictionary_round_trip.c +++ b/tests/fuzz/dictionary_round_trip.c @@ -21,8 +21,6 @@ #include "zstd_helpers.h" #include "fuzz_data_producer.h" -static const int kMaxClevel = 19; - static ZSTD_CCtx *cctx = NULL; static ZSTD_DCtx *dctx = NULL; @@ -34,8 +32,8 @@ static size_t roundTripTest(void *result, size_t resultCapacity, ZSTD_dictContentType_e dictContentType = ZSTD_dct_auto; FUZZ_dict_t dict = FUZZ_train(src, srcSize, producer); size_t cSize; - if ((FUZZ_dataProducer_uint32(producer) & 15) == 0) { - int const cLevel = FUZZ_dataProducer_uint32(producer) % kMaxClevel; + if (FUZZ_dataProducer_uint32Range(producer, 0, 15) == 0) { + int const cLevel = FUZZ_dataProducer_int32Range(producer, kMinClevel, kMaxClevel); cSize = ZSTD_compress_usingDict(cctx, compressed, compressedCapacity, diff --git a/tests/fuzz/fuzz_data_producer.c b/tests/fuzz/fuzz_data_producer.c index d6893e4f9..b465337ea 100644 --- a/tests/fuzz/fuzz_data_producer.c +++ b/tests/fuzz/fuzz_data_producer.c @@ -52,6 +52,17 @@ uint32_t FUZZ_dataProducer_uint32(FUZZ_dataProducer_t *producer) { return FUZZ_dataProducer_uint32Range(producer, 0, 0xffffffff); } +int32_t FUZZ_dataProducer_int32Range(FUZZ_dataProducer_t *producer, + int32_t min, int32_t max) +{ + FUZZ_ASSERT(min <= max); + + if (min < 0) + return (int)FUZZ_dataProducer_uint32Range(producer, 0, max - min) + min; + + return FUZZ_dataProducer_uint32Range(producer, min, max); +} + size_t FUZZ_dataProducer_remainingBytes(FUZZ_dataProducer_t *producer){ return producer->size; } diff --git a/tests/fuzz/fuzz_data_producer.h b/tests/fuzz/fuzz_data_producer.h index 8eea1e256..8153c0667 100644 --- a/tests/fuzz/fuzz_data_producer.h +++ b/tests/fuzz/fuzz_data_producer.h @@ -41,6 +41,10 @@ uint32_t FUZZ_dataProducer_uint32Range(FUZZ_dataProducer_t *producer, uint32_t m /* Returns a uint32 value */ uint32_t FUZZ_dataProducer_uint32(FUZZ_dataProducer_t *producer); +/* Returns a signed value between [min, ,max] */ +int32_t FUZZ_dataProducer_int32Range(FUZZ_dataProducer_t *producer, + int32_t min, int32_t max); + /* Returns the size of the remaining bytes of data in the producer */ size_t FUZZ_dataProducer_remainingBytes(FUZZ_dataProducer_t *producer); diff --git a/tests/fuzz/simple_compress.c b/tests/fuzz/simple_compress.c index 74ab0d631..487be3a35 100644 --- a/tests/fuzz/simple_compress.c +++ b/tests/fuzz/simple_compress.c @@ -18,6 +18,7 @@ #include #include "fuzz_helpers.h" #include "zstd.h" +#include "zstd_helpers.h" #include "fuzz_data_producer.h" static ZSTD_CCtx *cctx = NULL; @@ -32,8 +33,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) size_t const maxSize = ZSTD_compressBound(size); size_t const bufSize = FUZZ_dataProducer_uint32Range(producer, 0, maxSize); - int const level = (int)FUZZ_dataProducer_uint32Range( - producer, 0, 19 + 3) - 3; /* [-3, 19] */ + int const cLevel = FUZZ_dataProducer_int32Range(producer, kMinClevel, kMaxClevel); if (!cctx) { cctx = ZSTD_createCCtx(); @@ -42,7 +42,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) void *rBuf = malloc(bufSize); FUZZ_ASSERT(rBuf); - ZSTD_compressCCtx(cctx, rBuf, bufSize, src, size, level); + ZSTD_compressCCtx(cctx, rBuf, bufSize, src, size, cLevel); free(rBuf); FUZZ_dataProducer_free(producer); #ifndef STATEFUL_FUZZING diff --git a/tests/fuzz/simple_round_trip.c b/tests/fuzz/simple_round_trip.c index 4a07d16a4..2d1d0598a 100644 --- a/tests/fuzz/simple_round_trip.c +++ b/tests/fuzz/simple_round_trip.c @@ -22,8 +22,6 @@ #include "zstd_helpers.h" #include "fuzz_data_producer.h" -static const int kMaxClevel = 19; - static ZSTD_CCtx *cctx = NULL; static ZSTD_DCtx *dctx = NULL; @@ -33,11 +31,12 @@ static size_t roundTripTest(void *result, size_t resultCapacity, FUZZ_dataProducer_t *producer) { size_t cSize; - if (FUZZ_dataProducer_uint32(producer) & 1) { + if (FUZZ_dataProducer_uint32Range(producer, 0, 1)) { FUZZ_setRandomParameters(cctx, srcSize, producer); cSize = ZSTD_compress2(cctx, compressed, compressedCapacity, src, srcSize); } else { - int const cLevel = FUZZ_dataProducer_uint32(producer) % kMaxClevel; + int const cLevel = FUZZ_dataProducer_int32Range(producer, kMinClevel, kMaxClevel); + cSize = ZSTD_compressCCtx( cctx, compressed, compressedCapacity, src, srcSize, cLevel); } @@ -62,7 +61,6 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) * field is empty, giving us 4 bytes of overhead. */ cBufSize -= FUZZ_dataProducer_uint32Range(producer, 0, 1); - size = FUZZ_dataProducer_remainingBytes(producer); cBuf = malloc(cBufSize); diff --git a/tests/fuzz/stream_decompress.c b/tests/fuzz/stream_decompress.c index 70582e11b..c71cc9d3e 100644 --- a/tests/fuzz/stream_decompress.c +++ b/tests/fuzz/stream_decompress.c @@ -31,7 +31,7 @@ static ZSTD_outBuffer makeOutBuffer(FUZZ_dataProducer_t *producer) { ZSTD_outBuffer buffer = { buf, 0, 0 }; - buffer.size = (FUZZ_dataProducer_uint32(producer) % kBufSize) + 1; + buffer.size = (FUZZ_dataProducer_uint32Range(producer, 1, kBufSize)); FUZZ_ASSERT(buffer.size <= kBufSize); return buffer; @@ -43,7 +43,7 @@ static ZSTD_inBuffer makeInBuffer(const uint8_t **src, size_t *size, ZSTD_inBuffer buffer = { *src, 0, 0 }; FUZZ_ASSERT(*size > 0); - buffer.size = (FUZZ_dataProducer_uint32(producer) % *size) + 1; + buffer.size = (FUZZ_dataProducer_uint32Range(producer, 1, *size)); FUZZ_ASSERT(buffer.size <= *size); *src += buffer.size; *size -= buffer.size; diff --git a/tests/fuzz/stream_round_trip.c b/tests/fuzz/stream_round_trip.c index 08a4927a6..c534a904a 100644 --- a/tests/fuzz/stream_round_trip.c +++ b/tests/fuzz/stream_round_trip.c @@ -34,7 +34,7 @@ static ZSTD_outBuffer makeOutBuffer(uint8_t *dst, size_t capacity, ZSTD_outBuffer buffer = { dst, 0, 0 }; FUZZ_ASSERT(capacity > 0); - buffer.size = (FUZZ_dataProducer_uint32(producer) % capacity) + 1; + buffer.size = (FUZZ_dataProducer_uint32Range(producer, 1, capacity)); FUZZ_ASSERT(buffer.size <= capacity); return buffer; @@ -46,7 +46,7 @@ static ZSTD_inBuffer makeInBuffer(const uint8_t **src, size_t *size, ZSTD_inBuffer buffer = { *src, 0, 0 }; FUZZ_ASSERT(*size > 0); - buffer.size = (FUZZ_dataProducer_uint32(producer) % *size) + 1; + buffer.size = (FUZZ_dataProducer_uint32Range(producer, 1, *size)); FUZZ_ASSERT(buffer.size <= *size); *src += buffer.size; *size -= buffer.size; @@ -69,7 +69,7 @@ static size_t compress(uint8_t *dst, size_t capacity, while (in.pos < in.size || mode != -1) { ZSTD_outBuffer out = makeOutBuffer(dst, capacity, producer); /* Previous action finished, pick a new mode. */ - if (mode == -1) mode = FUZZ_dataProducer_uint32(producer) % 10; + if (mode == -1) mode = FUZZ_dataProducer_uint32Range(producer, 0, 9); switch (mode) { case 0: /* fall-through */ case 1: /* fall-through */ @@ -88,7 +88,7 @@ static size_t compress(uint8_t *dst, size_t capacity, /* Reset the compressor when the frame is finished */ if (ret == 0) { ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); - if ((FUZZ_dataProducer_uint32(producer) & 7) == 0) { + if (FUZZ_dataProducer_uint32Range(producer, 0, 7) == 0) { size_t const remaining = in.size - in.pos; FUZZ_setRandomParameters(cctx, remaining, producer); } diff --git a/tests/fuzz/zstd_helpers.c b/tests/fuzz/zstd_helpers.c index 2635de4cc..90bf1a156 100644 --- a/tests/fuzz/zstd_helpers.c +++ b/tests/fuzz/zstd_helpers.c @@ -17,6 +17,9 @@ #include "zstd.h" #include "zdict.h" +const int kMinClevel = -3; +const int kMaxClevel = 19; + static void set(ZSTD_CCtx *cctx, ZSTD_cParameter param, int value) { FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, param, value)); diff --git a/tests/fuzz/zstd_helpers.h b/tests/fuzz/zstd_helpers.h index f2001f8b2..2210bcaf8 100644 --- a/tests/fuzz/zstd_helpers.h +++ b/tests/fuzz/zstd_helpers.h @@ -24,6 +24,9 @@ extern "C" { #endif +extern const int kMinClevel; +extern const int kMaxClevel; + void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer_t *producer); ZSTD_compressionParameters FUZZ_randomCParams(size_t srcSize, FUZZ_dataProducer_t *producer); @@ -41,7 +44,6 @@ typedef struct { */ FUZZ_dict_t FUZZ_train(void const* src, size_t srcSize, FUZZ_dataProducer_t *producer); - #ifdef __cplusplus } #endif From 47bb4c6a23439813b2d20f668f69313d519304d7 Mon Sep 17 00:00:00 2001 From: Dario Pavlovic Date: Thu, 12 Sep 2019 12:45:28 -0700 Subject: [PATCH 5/6] Update tests/fuzz/fuzz_data_producer.h --- tests/fuzz/fuzz_data_producer.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fuzz/fuzz_data_producer.h b/tests/fuzz/fuzz_data_producer.h index 8153c0667..f2b609677 100644 --- a/tests/fuzz/fuzz_data_producer.h +++ b/tests/fuzz/fuzz_data_producer.h @@ -41,7 +41,7 @@ uint32_t FUZZ_dataProducer_uint32Range(FUZZ_dataProducer_t *producer, uint32_t m /* Returns a uint32 value */ uint32_t FUZZ_dataProducer_uint32(FUZZ_dataProducer_t *producer); -/* Returns a signed value between [min, ,max] */ +/* Returns a signed value between [min, max] */ int32_t FUZZ_dataProducer_int32Range(FUZZ_dataProducer_t *producer, int32_t min, int32_t max); From cd8588077e4119ad9a328463b760edd59b529430 Mon Sep 17 00:00:00 2001 From: Dario Pavlovic Date: Thu, 12 Sep 2019 13:10:34 -0700 Subject: [PATCH 6/6] It's time for all of rng seed code to go. Goodbye --- tests/fuzz/fuzz.h | 10 ---------- tests/fuzz/fuzz.py | 2 -- tests/fuzz/fuzz_helpers.h | 31 ------------------------------- 3 files changed, 43 deletions(-) diff --git a/tests/fuzz/fuzz.h b/tests/fuzz/fuzz.h index 8850025b0..6d53aa6d5 100644 --- a/tests/fuzz/fuzz.h +++ b/tests/fuzz/fuzz.h @@ -17,12 +17,6 @@ * test code paths which are only executed when contexts are reused. * WARNING: Makes reproducing crashes much harder. * Default: Not defined. - * @param FUZZ_RNG_SEED_SIZE: - * The number of bytes of the source to look at when constructing a seed - * for the deterministic RNG. These bytes are discarded before passing - * the data to zstd functions. Every fuzzer initializes the RNG exactly - * once before doing anything else, even if it is unused. - * Default: 4. * @param DEBUGLEVEL: * This is a parameter for the zstd library. Defining `DEBUGLEVEL=1` * enables assert() statements in the zstd library. Higher levels enable @@ -42,10 +36,6 @@ #ifndef FUZZ_H #define FUZZ_H -#ifndef FUZZ_RNG_SEED_SIZE -# define FUZZ_RNG_SEED_SIZE 4 -#endif - #include #include diff --git a/tests/fuzz/fuzz.py b/tests/fuzz/fuzz.py index faf8ce8ae..c790c8373 100755 --- a/tests/fuzz/fuzz.py +++ b/tests/fuzz/fuzz.py @@ -740,10 +740,8 @@ def gen(args): for name in os.listdir(samples): samplename = abs_join(samples, name) outname = abs_join(seed, name) - rng_seed = os.urandom(args.fuzz_rng_seed_size) with open(samplename, 'rb') as sample: with open(outname, 'wb') as out: - out.write(rng_seed) CHUNK_SIZE = 131072 chunk = sample.read(CHUNK_SIZE) while len(chunk) > 0: diff --git a/tests/fuzz/fuzz_helpers.h b/tests/fuzz/fuzz_helpers.h index 0ee85fc7e..3de917fd1 100644 --- a/tests/fuzz/fuzz_helpers.h +++ b/tests/fuzz/fuzz_helpers.h @@ -55,37 +55,6 @@ extern "C" { #define FUZZ_STATIC static #endif -/** - * Deterministically constructs a seed based on the fuzz input. - * Consumes up to the first FUZZ_RNG_SEED_SIZE bytes of the input. - */ -FUZZ_STATIC uint32_t FUZZ_seed(uint8_t const **src, size_t* size) { - uint8_t const *data = *src; - size_t const toHash = MIN(FUZZ_RNG_SEED_SIZE, *size); - *size -= toHash; - *src += toHash; - return XXH32(data, toHash, 0); -} - -#define FUZZ_rotl32(x, r) (((x) << (r)) | ((x) >> (32 - (r)))) - -FUZZ_STATIC uint32_t FUZZ_rand(uint32_t *state) { - static const uint32_t prime1 = 2654435761U; - static const uint32_t prime2 = 2246822519U; - uint32_t rand32 = *state; - rand32 *= prime1; - rand32 += prime2; - rand32 = FUZZ_rotl32(rand32, 13); - *state = rand32; - return rand32 >> 5; -} - -/* Returns a random numer in the range [min, max]. */ -FUZZ_STATIC uint32_t FUZZ_rand32(uint32_t *state, uint32_t min, uint32_t max) { - uint32_t random = FUZZ_rand(state); - return min + (random % (max - min + 1)); -} - #ifdef __cplusplus } #endif