Merge pull request #1781 from darxsys/improvDataGen

Improve data generation

tests/fuzz/block_decompress.c

@@ -28,8 +28,6 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
 {
     size_t const neededBufSize = ZSTD_BLOCKSIZE_MAX;
 
-    FUZZ_seed(&src, &size);
-
     /* Allocate all buffers and contexts if not already allocated */
     if (neededBufSize > bufSize) {
         free(rBuf);

tests/fuzz/block_round_trip.c

@@ -20,21 +20,20 @@
 #include <string.h>
 #include "fuzz_helpers.h"
 #include "zstd.h"
-
+#include "zstd_helpers.h"
-static const int kMaxClevel = 19;
+#include "fuzz_data_producer.h"
 
 static ZSTD_CCtx *cctx = NULL;
 static ZSTD_DCtx *dctx = NULL;
 static void* cBuf = NULL;
 static void* rBuf = NULL;
 static size_t bufSize = 0;
-static uint32_t seed;
 
 static size_t roundTripTest(void *result, size_t resultCapacity,
                             void *compressed, size_t compressedCapacity,
-                            const void *src, size_t srcSize)
+                            const void *src, size_t srcSize,
+                            int cLevel)
 {
-    int const cLevel = FUZZ_rand(&seed) % kMaxClevel;
     ZSTD_parameters const params = ZSTD_getParams(cLevel, srcSize, 0);
     size_t ret = ZSTD_compressBegin_advanced(cctx, NULL, 0, params, srcSize);
     FUZZ_ZASSERT(ret);
@@ -52,10 +51,14 @@ static size_t roundTripTest(void *result, size_t resultCapacity,
 
 int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
 {
-    size_t neededBufSize;
+    /* Give a random portion of src data to the producer, to use for
+    parameter generation. The rest will be used for (de)compression */
+    FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size);
+    size = FUZZ_dataProducer_reserveDataPrefix(producer);
 
-    seed = FUZZ_seed(&src, &size);
+    int const cLevel = FUZZ_dataProducer_int32Range(producer, kMinClevel, kMaxClevel);
-    neededBufSize = size;
+
+    size_t neededBufSize = size;
     if (size > ZSTD_BLOCKSIZE_MAX)
         return 0;
 
@@ -79,11 +82,13 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
 
     {
         size_t const result =
-            roundTripTest(rBuf, neededBufSize, cBuf, neededBufSize, src, size);
+            roundTripTest(rBuf, neededBufSize, cBuf, neededBufSize, src, size,
+              cLevel);
         FUZZ_ZASSERT(result);
         FUZZ_ASSERT_MSG(result == size, "Incorrect regenerated size");
         FUZZ_ASSERT_MSG(!memcmp(src, rBuf, size), "Corruption!");
     }
+    FUZZ_dataProducer_free(producer);
 #ifndef STATEFUL_FUZZING
     ZSTD_freeCCtx(cctx); cctx = NULL;
     ZSTD_freeDCtx(dctx); dctx = NULL;

tests/fuzz/dictionary_decompress.c

@@ -18,33 +18,37 @@
 #include <stdio.h>
 #include "fuzz_helpers.h"
 #include "zstd_helpers.h"
+#include "fuzz_data_producer.h"
 
 static ZSTD_DCtx *dctx = NULL;
 
 int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
 {
-    uint32_t seed = FUZZ_seed(&src, &size);
+    /* Give a random portion of src data to the producer, to use for
+    parameter generation. The rest will be used for (de)compression */
+    FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size);
+    size = FUZZ_dataProducer_reserveDataPrefix(producer);
+
     FUZZ_dict_t dict;
     ZSTD_DDict* ddict = NULL;
-    int i;
 
     if (!dctx) {
         dctx = ZSTD_createDCtx();
         FUZZ_ASSERT(dctx);
     }
-    dict = FUZZ_train(src, size, &seed);
+    dict = FUZZ_train(src, size, producer);
-    if (FUZZ_rand32(&seed, 0, 1) == 0) {
+    if (FUZZ_dataProducer_uint32Range(producer, 0, 1) == 0) {
         ddict = ZSTD_createDDict(dict.buff, dict.size);
         FUZZ_ASSERT(ddict);
     } else {
         FUZZ_ZASSERT(ZSTD_DCtx_loadDictionary_advanced(
                 dctx, dict.buff, dict.size,
-                (ZSTD_dictLoadMethod_e)FUZZ_rand32(&seed, 0, 1),
+                (ZSTD_dictLoadMethod_e)FUZZ_dataProducer_uint32Range(producer, 0, 1),
-                (ZSTD_dictContentType_e)FUZZ_rand32(&seed, 0, 2)));
+                (ZSTD_dictContentType_e)FUZZ_dataProducer_uint32Range(producer, 0, 2)));
     }
-    /* Run it 10 times over 10 output sizes. Reuse the context and dict. */
+
-    for (i = 0; i < 10; ++i) {
+    {
-        size_t const bufSize = FUZZ_rand32(&seed, 0, 2 * size);
+        size_t const bufSize = FUZZ_dataProducer_uint32Range(producer, 0, 10 * size);
         void* rBuf = malloc(bufSize);
         FUZZ_ASSERT(rBuf);
         if (ddict) {
@@ -55,6 +59,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
         free(rBuf);
     }
     free(dict.buff);
+    FUZZ_dataProducer_free(producer);
     ZSTD_freeDDict(ddict);
 #ifndef STATEFUL_FUZZING
     ZSTD_freeDCtx(dctx); dctx = NULL;

tests/fuzz/dictionary_round_trip.c

@@ -19,22 +19,21 @@
 #include <string.h>
 #include "fuzz_helpers.h"
 #include "zstd_helpers.h"
-
+#include "fuzz_data_producer.h"
-static const int kMaxClevel = 19;
 
 static ZSTD_CCtx *cctx = NULL;
 static ZSTD_DCtx *dctx = NULL;
-static uint32_t seed;
 
 static size_t roundTripTest(void *result, size_t resultCapacity,
                             void *compressed, size_t compressedCapacity,
-                            const void *src, size_t srcSize)
+                            const void *src, size_t srcSize,
+                            FUZZ_dataProducer_t *producer)
 {
     ZSTD_dictContentType_e dictContentType = ZSTD_dct_auto;
-    FUZZ_dict_t dict = FUZZ_train(src, srcSize, &seed);
+    FUZZ_dict_t dict = FUZZ_train(src, srcSize, producer);
     size_t cSize;
-    if ((FUZZ_rand(&seed) & 15) == 0) {
+    if (FUZZ_dataProducer_uint32Range(producer, 0, 15) == 0) {
-        int const cLevel = FUZZ_rand(&seed) % kMaxClevel;
+        int const cLevel = FUZZ_dataProducer_int32Range(producer, kMinClevel, kMaxClevel);
 
         cSize = ZSTD_compress_usingDict(cctx,
                 compressed, compressedCapacity,
@@ -42,20 +41,20 @@ static size_t roundTripTest(void *result, size_t resultCapacity,
                 dict.buff, dict.size,
                 cLevel);
     } else {
-        dictContentType = FUZZ_rand32(&seed, 0, 2);
+        dictContentType = FUZZ_dataProducer_uint32Range(producer, 0, 2);
-        FUZZ_setRandomParameters(cctx, srcSize, &seed);
+        FUZZ_setRandomParameters(cctx, srcSize, producer);
         /* Disable checksum so we can use sizes smaller than compress bound. */
         FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 0));
         FUZZ_ZASSERT(ZSTD_CCtx_loadDictionary_advanced(
                 cctx, dict.buff, dict.size,
-                (ZSTD_dictLoadMethod_e)FUZZ_rand32(&seed, 0, 1),
+                (ZSTD_dictLoadMethod_e)FUZZ_dataProducer_uint32Range(producer, 0, 1),
                 dictContentType));
         cSize = ZSTD_compress2(cctx, compressed, compressedCapacity, src, srcSize);
     }
     FUZZ_ZASSERT(cSize);
     FUZZ_ZASSERT(ZSTD_DCtx_loadDictionary_advanced(
         dctx, dict.buff, dict.size,
-        (ZSTD_dictLoadMethod_e)FUZZ_rand32(&seed, 0, 1),
+        (ZSTD_dictLoadMethod_e)FUZZ_dataProducer_uint32Range(producer, 0, 1),
         dictContentType));
     {
         size_t const ret = ZSTD_decompressDCtx(
@@ -67,17 +66,20 @@ static size_t roundTripTest(void *result, size_t resultCapacity,
 
 int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
 {
+    /* Give a random portion of src data to the producer, to use for
+    parameter generation. The rest will be used for (de)compression */
+    FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size);
+    size = FUZZ_dataProducer_reserveDataPrefix(producer);
+
     size_t const rBufSize = size;
     void* rBuf = malloc(rBufSize);
     size_t cBufSize = ZSTD_compressBound(size);
-    void* cBuf;
+    void *cBuf;
-
-    seed = FUZZ_seed(&src, &size);
     /* Half of the time fuzz with a 1 byte smaller output size.
      * This will still succeed because we force the checksum to be disabled,
      * giving us 4 bytes of overhead.
      */
-    cBufSize -= FUZZ_rand32(&seed, 0, 1);
+    cBufSize -= FUZZ_dataProducer_uint32Range(producer, 0, 1);
     cBuf = malloc(cBufSize);
 
     if (!cctx) {
@@ -91,13 +93,14 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
 
     {
         size_t const result =
-            roundTripTest(rBuf, rBufSize, cBuf, cBufSize, src, size);
+            roundTripTest(rBuf, rBufSize, cBuf, cBufSize, src, size, producer);
         FUZZ_ZASSERT(result);
         FUZZ_ASSERT_MSG(result == size, "Incorrect regenerated size");
         FUZZ_ASSERT_MSG(!memcmp(src, rBuf, size), "Corruption!");
     }
     free(rBuf);
     free(cBuf);
+    FUZZ_dataProducer_free(producer);
 #ifndef STATEFUL_FUZZING
     ZSTD_freeCCtx(cctx); cctx = NULL;
     ZSTD_freeDCtx(dctx); dctx = NULL;

tests/fuzz/fuzz.h

@@ -17,12 +17,6 @@
  *        test code paths which are only executed when contexts are reused.
  *        WARNING: Makes reproducing crashes much harder.
  *        Default: Not defined.
- * @param FUZZ_RNG_SEED_SIZE:
- *        The number of bytes of the source to look at when constructing a seed
- *        for the deterministic RNG. These bytes are discarded before passing
- *        the data to zstd functions. Every fuzzer initializes the RNG exactly
- *        once before doing anything else, even if it is unused.
- *        Default: 4.
  * @param DEBUGLEVEL:
  *        This is a parameter for the zstd library. Defining `DEBUGLEVEL=1`
  *        enables assert() statements in the zstd library. Higher levels enable
@@ -42,10 +36,6 @@
 #ifndef FUZZ_H
 #define FUZZ_H
 
-#ifndef FUZZ_RNG_SEED_SIZE
-#  define FUZZ_RNG_SEED_SIZE 4
-#endif
-
 #include <stddef.h>
 #include <stdint.h>
 

tests/fuzz/fuzz.py

@@ -740,10 +740,8 @@ def gen(args):
             for name in os.listdir(samples):
                 samplename = abs_join(samples, name)
                 outname = abs_join(seed, name)
-                rng_seed = os.urandom(args.fuzz_rng_seed_size)
                 with open(samplename, 'rb') as sample:
                     with open(outname, 'wb') as out:
-                        out.write(rng_seed)
                         CHUNK_SIZE = 131072
                         chunk = sample.read(CHUNK_SIZE)
                         while len(chunk) > 0:

tests/fuzz/fuzz_data_producer.c

@@ -49,9 +49,37 @@ uint32_t FUZZ_dataProducer_uint32Range(FUZZ_dataProducer_t *producer, uint32_t m
 }
 
 uint32_t FUZZ_dataProducer_uint32(FUZZ_dataProducer_t *producer) {
-  return FUZZ_dataProducer_uint32Range(producer, 0, 0xffffffff);
+    return FUZZ_dataProducer_uint32Range(producer, 0, 0xffffffff);
+}
+
+int32_t FUZZ_dataProducer_int32Range(FUZZ_dataProducer_t *producer,
+                                    int32_t min, int32_t max)
+{
+    FUZZ_ASSERT(min <= max);
+
+    if (min < 0)
+      return (int)FUZZ_dataProducer_uint32Range(producer, 0, max - min) + min;
+
+    return FUZZ_dataProducer_uint32Range(producer, min, max);
 }
 
 size_t FUZZ_dataProducer_remainingBytes(FUZZ_dataProducer_t *producer){
     return producer->size;
 }
+
+size_t FUZZ_dataProducer_contract(FUZZ_dataProducer_t *producer, size_t newSize)
+{
+    newSize = newSize > producer->size ? producer->size : newSize;
+
+    size_t remaining = producer->size - newSize;
+    producer->data = producer->data + remaining;
+    producer->size = newSize;
+    return remaining;
+}
+
+size_t FUZZ_dataProducer_reserveDataPrefix(FUZZ_dataProducer_t *producer)
+{
+    size_t producerSliceSize = FUZZ_dataProducer_uint32Range(
+                                  producer, 0, producer->size);
+    return FUZZ_dataProducer_contract(producer, producerSliceSize);
+}

tests/fuzz/fuzz_data_producer.h

@@ -41,7 +41,20 @@ uint32_t FUZZ_dataProducer_uint32Range(FUZZ_dataProducer_t *producer, uint32_t m
 /* Returns a uint32 value */
 uint32_t FUZZ_dataProducer_uint32(FUZZ_dataProducer_t *producer);
 
+/* Returns a signed value between [min, max] */
+int32_t FUZZ_dataProducer_int32Range(FUZZ_dataProducer_t *producer,
+                                    int32_t min, int32_t max);
+
 /* Returns the size of the remaining bytes of data in the producer */
 size_t FUZZ_dataProducer_remainingBytes(FUZZ_dataProducer_t *producer);
 
+/* Restricts the producer to only the last newSize bytes of data.
+If newSize > current data size, nothing happens. Returns the number of bytes
+the producer won't use anymore, after contracting. */
+size_t FUZZ_dataProducer_contract(FUZZ_dataProducer_t *producer, size_t newSize);
+
+/* Restricts the producer to use only the last X bytes of data, where X is
+ a random number in the interval [0, data_size]. Returns the size of the
+ remaining data the producer won't use anymore (the prefix). */
+size_t FUZZ_dataProducer_reserveDataPrefix(FUZZ_dataProducer_t *producer);
 #endif // FUZZ_DATA_PRODUCER_H

tests/fuzz/fuzz_helpers.h

@@ -55,37 +55,6 @@ extern "C" {
 #define FUZZ_STATIC static
 #endif
 
-/**
- * Deterministically constructs a seed based on the fuzz input.
- * Consumes up to the first FUZZ_RNG_SEED_SIZE bytes of the input.
- */
-FUZZ_STATIC uint32_t FUZZ_seed(uint8_t const **src, size_t* size) {
-    uint8_t const *data = *src;
-    size_t const toHash = MIN(FUZZ_RNG_SEED_SIZE, *size);
-    *size -= toHash;
-    *src += toHash;
-    return XXH32(data, toHash, 0);
-}
-
-#define FUZZ_rotl32(x, r) (((x) << (r)) | ((x) >> (32 - (r))))
-
-FUZZ_STATIC uint32_t FUZZ_rand(uint32_t *state) {
-    static const uint32_t prime1 = 2654435761U;
-    static const uint32_t prime2 = 2246822519U;
-    uint32_t rand32 = *state;
-    rand32 *= prime1;
-    rand32 += prime2;
-    rand32 = FUZZ_rotl32(rand32, 13);
-    *state = rand32;
-    return rand32 >> 5;
-}
-
-/* Returns a random numer in the range [min, max]. */
-FUZZ_STATIC uint32_t FUZZ_rand32(uint32_t *state, uint32_t min, uint32_t max) {
-    uint32_t random = FUZZ_rand(state);
-    return min + (random % (max - min + 1));
-}
-
 #ifdef __cplusplus
 }
 #endif

tests/fuzz/simple_compress.c

@@ -18,28 +18,33 @@
 #include <stdio.h>
 #include "fuzz_helpers.h"
 #include "zstd.h"
+#include "zstd_helpers.h"
+#include "fuzz_data_producer.h"
 
 static ZSTD_CCtx *cctx = NULL;
 
 int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
 {
-    uint32_t seed = FUZZ_seed(&src, &size);
+    /* Give a random portion of src data to the producer, to use for
+    parameter generation. The rest will be used for (de)compression */
+    FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size);
+    size = FUZZ_dataProducer_reserveDataPrefix(producer);
+
     size_t const maxSize = ZSTD_compressBound(size);
-    int i;
+    size_t const bufSize = FUZZ_dataProducer_uint32Range(producer, 0, maxSize);
+
+    int const cLevel = FUZZ_dataProducer_int32Range(producer, kMinClevel, kMaxClevel);
+
     if (!cctx) {
         cctx = ZSTD_createCCtx();
         FUZZ_ASSERT(cctx);
     }
-    /* Run it 10 times over 10 output sizes. Reuse the context. */
-    for (i = 0; i < 10; ++i) {
-        int const level = (int)FUZZ_rand32(&seed, 0, 19 + 3) - 3; /* [-3, 19] */
-        size_t const bufSize = FUZZ_rand32(&seed, 0, maxSize);
-        void* rBuf = malloc(bufSize);
-        FUZZ_ASSERT(rBuf);
-        ZSTD_compressCCtx(cctx, rBuf, bufSize, src, size, level);
-        free(rBuf);
-    }
 
+    void *rBuf = malloc(bufSize);
+    FUZZ_ASSERT(rBuf);
+    ZSTD_compressCCtx(cctx, rBuf, bufSize, src, size, cLevel);
+    free(rBuf);
+    FUZZ_dataProducer_free(producer);
 #ifndef STATEFUL_FUZZING
     ZSTD_freeCCtx(cctx); cctx = NULL;
 #endif

tests/fuzz/simple_decompress.c

@@ -23,26 +23,24 @@ static ZSTD_DCtx *dctx = NULL;
 
 int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
 {
-  FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size);
+    /* Give a random portion of src data to the producer, to use for
+    parameter generation. The rest will be used for (de)compression */
+    FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size);
+    size = FUZZ_dataProducer_reserveDataPrefix(producer);
 
-  int i;
+    if (!dctx) {
-  if (!dctx) {
+        dctx = ZSTD_createDCtx();
-      dctx = ZSTD_createDCtx();
+        FUZZ_ASSERT(dctx);
-      FUZZ_ASSERT(dctx);
+    }
-  }
 
-  size_t const bufSize = FUZZ_dataProducer_uint32Range(producer, 0, 10 * size);
+    size_t const bufSize = FUZZ_dataProducer_uint32Range(producer, 0, 10 * size);
-  void *rBuf = malloc(bufSize);
+    void *rBuf = malloc(bufSize);
-  FUZZ_ASSERT(rBuf);
+    FUZZ_ASSERT(rBuf);
 
-  /* Restrict to remaining data. If we run out of data while generating params,
+    ZSTD_decompressDCtx(dctx, rBuf, bufSize, src, size);
-   we should still continue and let decompression happen on empty data. */
+    free(rBuf);
-  size = FUZZ_dataProducer_remainingBytes(producer);
 
-  ZSTD_decompressDCtx(dctx, rBuf, bufSize, src, size);
+    FUZZ_dataProducer_free(producer);
-  free(rBuf);
-
-  FUZZ_dataProducer_free(producer);
 
 #ifndef STATEFUL_FUZZING
     ZSTD_freeDCtx(dctx); dctx = NULL;

tests/fuzz/simple_round_trip.c

@@ -20,23 +20,23 @@
 #include <string.h>
 #include "fuzz_helpers.h"
 #include "zstd_helpers.h"
-
+#include "fuzz_data_producer.h"
-static const int kMaxClevel = 19;
 
 static ZSTD_CCtx *cctx = NULL;
 static ZSTD_DCtx *dctx = NULL;
-static uint32_t seed;
 
 static size_t roundTripTest(void *result, size_t resultCapacity,
                             void *compressed, size_t compressedCapacity,
-                            const void *src, size_t srcSize)
+                            const void *src, size_t srcSize,
+                            FUZZ_dataProducer_t *producer)
 {
     size_t cSize;
-    if (FUZZ_rand(&seed) & 1) {
+    if (FUZZ_dataProducer_uint32Range(producer, 0, 1)) {
-        FUZZ_setRandomParameters(cctx, srcSize, &seed);
+        FUZZ_setRandomParameters(cctx, srcSize, producer);
         cSize = ZSTD_compress2(cctx, compressed, compressedCapacity, src, srcSize);
     } else {
-        int const cLevel = FUZZ_rand(&seed) % kMaxClevel;
+      int const cLevel = FUZZ_dataProducer_int32Range(producer, kMinClevel, kMaxClevel);
+
         cSize = ZSTD_compressCCtx(
             cctx, compressed, compressedCapacity, src, srcSize, cLevel);
     }
@@ -51,12 +51,17 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
     size_t cBufSize = ZSTD_compressBound(size);
     void* cBuf;
 
-    seed = FUZZ_seed(&src, &size);
+    /* Give a random portion of src data to the producer, to use for
+    parameter generation. The rest will be used for (de)compression */
+    FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size);
+    size = FUZZ_dataProducer_reserveDataPrefix(producer);
+
     /* Half of the time fuzz with a 1 byte smaller output size.
      * This will still succeed because we don't use a dictionary, so the dictID
      * field is empty, giving us 4 bytes of overhead.
      */
-    cBufSize -= FUZZ_rand32(&seed, 0, 1);
+    cBufSize -= FUZZ_dataProducer_uint32Range(producer, 0, 1);
+
     cBuf = malloc(cBufSize);
 
     FUZZ_ASSERT(cBuf && rBuf);
@@ -72,13 +77,14 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
 
     {
         size_t const result =
-            roundTripTest(rBuf, rBufSize, cBuf, cBufSize, src, size);
+            roundTripTest(rBuf, rBufSize, cBuf, cBufSize, src, size, producer);
         FUZZ_ZASSERT(result);
         FUZZ_ASSERT_MSG(result == size, "Incorrect regenerated size");
         FUZZ_ASSERT_MSG(!memcmp(src, rBuf, size), "Corruption!");
     }
     free(rBuf);
     free(cBuf);
+    FUZZ_dataProducer_free(producer);
 #ifndef STATEFUL_FUZZING
     ZSTD_freeCCtx(cctx); cctx = NULL;
     ZSTD_freeDCtx(dctx); dctx = NULL;

tests/fuzz/stream_decompress.c

@@ -19,6 +19,7 @@
 #include <stdio.h>
 #include "fuzz_helpers.h"
 #include "zstd.h"
+#include "fuzz_data_producer.h"
 
 static size_t const kBufSize = ZSTD_BLOCKSIZE_MAX;
 
@@ -26,22 +27,23 @@ static ZSTD_DStream *dstream = NULL;
 static void* buf = NULL;
 uint32_t seed;
 
-static ZSTD_outBuffer makeOutBuffer(void)
+static ZSTD_outBuffer makeOutBuffer(FUZZ_dataProducer_t *producer)
 {
   ZSTD_outBuffer buffer = { buf, 0, 0 };
 
-  buffer.size = (FUZZ_rand(&seed) % kBufSize) + 1;
+  buffer.size = (FUZZ_dataProducer_uint32Range(producer, 1, kBufSize));
   FUZZ_ASSERT(buffer.size <= kBufSize);
 
   return buffer;
 }
 
-static ZSTD_inBuffer makeInBuffer(const uint8_t **src, size_t *size)
+static ZSTD_inBuffer makeInBuffer(const uint8_t **src, size_t *size,
+                                  FUZZ_dataProducer_t *producer)
 {
   ZSTD_inBuffer buffer = { *src, 0, 0 };
 
   FUZZ_ASSERT(*size > 0);
-  buffer.size = (FUZZ_rand(&seed) % *size) + 1;
+  buffer.size = (FUZZ_dataProducer_uint32Range(producer, 1, *size));
   FUZZ_ASSERT(buffer.size <= *size);
   *src += buffer.size;
   *size -= buffer.size;
@@ -51,13 +53,16 @@ static ZSTD_inBuffer makeInBuffer(const uint8_t **src, size_t *size)
 
 int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
 {
-    seed = FUZZ_seed(&src, &size);
+    /* Give a random portion of src data to the producer, to use for
+    parameter generation. The rest will be used for (de)compression */
+    FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size);
+    size = FUZZ_dataProducer_reserveDataPrefix(producer);
 
     /* Allocate all buffers and contexts if not already allocated */
     if (!buf) {
       buf = malloc(kBufSize);
-      FUZZ_ASSERT(buf);
+        FUZZ_ASSERT(buf);
-    }
+      }
 
     if (!dstream) {
         dstream = ZSTD_createDStream();
@@ -67,9 +72,9 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
     }
 
     while (size > 0) {
-        ZSTD_inBuffer in = makeInBuffer(&src, &size);
+        ZSTD_inBuffer in = makeInBuffer(&src, &size, producer);
         while (in.pos != in.size) {
-            ZSTD_outBuffer out = makeOutBuffer();
+            ZSTD_outBuffer out = makeOutBuffer(producer);
             size_t const rc = ZSTD_decompressStream(dstream, &out, &in);
             if (ZSTD_isError(rc)) goto error;
         }
@@ -79,5 +84,6 @@ error:
 #ifndef STATEFUL_FUZZING
     ZSTD_freeDStream(dstream); dstream = NULL;
 #endif
+    FUZZ_dataProducer_free(producer);
     return 0;
 }

tests/fuzz/stream_round_trip.c

@@ -20,31 +20,33 @@
 #include <string.h>
 #include "fuzz_helpers.h"
 #include "zstd_helpers.h"
+#include "fuzz_data_producer.h"
 
 ZSTD_CCtx *cctx = NULL;
 static ZSTD_DCtx *dctx = NULL;
 static uint8_t* cBuf = NULL;
 static uint8_t* rBuf = NULL;
 static size_t bufSize = 0;
-static uint32_t seed;
 
-static ZSTD_outBuffer makeOutBuffer(uint8_t *dst, size_t capacity)
+static ZSTD_outBuffer makeOutBuffer(uint8_t *dst, size_t capacity,
+                                    FUZZ_dataProducer_t *producer)
 {
     ZSTD_outBuffer buffer = { dst, 0, 0 };
 
     FUZZ_ASSERT(capacity > 0);
-    buffer.size = (FUZZ_rand(&seed) % capacity) + 1;
+    buffer.size = (FUZZ_dataProducer_uint32Range(producer, 1, capacity));
     FUZZ_ASSERT(buffer.size <= capacity);
 
     return buffer;
 }
 
-static ZSTD_inBuffer makeInBuffer(const uint8_t **src, size_t *size)
+static ZSTD_inBuffer makeInBuffer(const uint8_t **src, size_t *size,
+                                  FUZZ_dataProducer_t *producer)
 {
     ZSTD_inBuffer buffer = { *src, 0, 0 };
 
     FUZZ_ASSERT(*size > 0);
-    buffer.size = (FUZZ_rand(&seed) % *size) + 1;
+    buffer.size = (FUZZ_dataProducer_uint32Range(producer, 1, *size));
     FUZZ_ASSERT(buffer.size <= *size);
     *src += buffer.size;
     *size -= buffer.size;
@@ -53,23 +55,24 @@ static ZSTD_inBuffer makeInBuffer(const uint8_t **src, size_t *size)
 }
 
 static size_t compress(uint8_t *dst, size_t capacity,
-                       const uint8_t *src, size_t srcSize)
+                       const uint8_t *src, size_t srcSize,
+                     FUZZ_dataProducer_t *producer)
 {
     size_t dstSize = 0;
     ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
-    FUZZ_setRandomParameters(cctx, srcSize, &seed);
+    FUZZ_setRandomParameters(cctx, srcSize, producer);
 
     while (srcSize > 0) {
-        ZSTD_inBuffer in = makeInBuffer(&src, &srcSize);
+        ZSTD_inBuffer in = makeInBuffer(&src, &srcSize, producer);
         /* Mode controls the action. If mode == -1 we pick a new mode */
         int mode = -1;
         while (in.pos < in.size || mode != -1) {
-            ZSTD_outBuffer out = makeOutBuffer(dst, capacity);
+            ZSTD_outBuffer out = makeOutBuffer(dst, capacity, producer);
             /* Previous action finished, pick a new mode. */
-            if (mode == -1) mode = FUZZ_rand(&seed) % 10;
+            if (mode == -1) mode = FUZZ_dataProducer_uint32Range(producer, 0, 9);
             switch (mode) {
-                case 0: /* fall-though */
+                case 0: /* fall-through */
-                case 1: /* fall-though */
+                case 1: /* fall-through */
                 case 2: {
                     size_t const ret =
                         ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush);
@@ -85,9 +88,9 @@ static size_t compress(uint8_t *dst, size_t capacity,
                     /* Reset the compressor when the frame is finished */
                     if (ret == 0) {
                         ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
-                        if ((FUZZ_rand(&seed) & 7) == 0) {
+                        if (FUZZ_dataProducer_uint32Range(producer, 0, 7) == 0) {
                             size_t const remaining = in.size - in.pos;
-                            FUZZ_setRandomParameters(cctx, remaining, &seed);
+                            FUZZ_setRandomParameters(cctx, remaining, producer);
                         }
                         mode = -1;
                     }
@@ -107,7 +110,7 @@ static size_t compress(uint8_t *dst, size_t capacity,
     }
     for (;;) {
         ZSTD_inBuffer in = {NULL, 0, 0};
-        ZSTD_outBuffer out = makeOutBuffer(dst, capacity);
+        ZSTD_outBuffer out = makeOutBuffer(dst, capacity, producer);
         size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
         FUZZ_ZASSERT(ret);
 
@@ -122,10 +125,13 @@ static size_t compress(uint8_t *dst, size_t capacity,
 
 int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
 {
-    size_t neededBufSize;
+    /* Give a random portion of src data to the producer, to use for
+    parameter generation. The rest will be used for (de)compression */
+    FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size);
+    size = FUZZ_dataProducer_reserveDataPrefix(producer);
 
-    seed = FUZZ_seed(&src, &size);
+    size_t neededBufSize;
-    neededBufSize = ZSTD_compressBound(size) * 2;
+    neededBufSize = ZSTD_compressBound(size) * 5;
 
     /* Allocate all buffers and contexts if not already allocated */
     if (neededBufSize > bufSize) {
@@ -146,7 +152,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
     }
 
     {
-        size_t const cSize = compress(cBuf, neededBufSize, src, size);
+        size_t const cSize = compress(cBuf, neededBufSize, src, size, producer);
         size_t const rSize =
             ZSTD_decompressDCtx(dctx, rBuf, neededBufSize, cBuf, cSize);
         FUZZ_ZASSERT(rSize);
@@ -154,6 +160,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
         FUZZ_ASSERT_MSG(!memcmp(src, rBuf, size), "Corruption!");
     }
 
+    FUZZ_dataProducer_free(producer);
 #ifndef STATEFUL_FUZZING
     ZSTD_freeCCtx(cctx); cctx = NULL;
     ZSTD_freeDCtx(dctx); dctx = NULL;

tests/fuzz/zstd_frame_info.c

@@ -21,10 +21,6 @@
 int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
 {
     ZSTD_frameHeader zfh;
-    /* Consume the seed to be compatible with the corpora of other decompression
-     * fuzzers.
-     */
-    FUZZ_seed(&src, &size);
     /* You can fuzz any helper functions here that are fast, and take zstd
      * compressed data as input. E.g. don't expect the input to be a dictionary,
      * so don't fuzz ZSTD_getDictID_fromDict().

tests/fuzz/zstd_helpers.c

@@ -17,53 +17,56 @@
 #include "zstd.h"
 #include "zdict.h"
 
+const int kMinClevel = -3;
+const int kMaxClevel = 19;
+
 static void set(ZSTD_CCtx *cctx, ZSTD_cParameter param, int value)
 {
     FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, param, value));
 }
 
 static void setRand(ZSTD_CCtx *cctx, ZSTD_cParameter param, unsigned min,
-                    unsigned max, uint32_t *state) {
+                    unsigned max, FUZZ_dataProducer_t *producer) {
-    unsigned const value = FUZZ_rand32(state, min, max);
+    unsigned const value = FUZZ_dataProducer_uint32Range(producer, min, max);
     set(cctx, param, value);
 }
 
-ZSTD_compressionParameters FUZZ_randomCParams(size_t srcSize, uint32_t *state)
+ZSTD_compressionParameters FUZZ_randomCParams(size_t srcSize, FUZZ_dataProducer_t *producer)
 {
     /* Select compression parameters */
     ZSTD_compressionParameters cParams;
-    cParams.windowLog = FUZZ_rand32(state, ZSTD_WINDOWLOG_MIN, 15);
+    cParams.windowLog = FUZZ_dataProducer_uint32Range(producer, ZSTD_WINDOWLOG_MIN, 15);
-    cParams.hashLog = FUZZ_rand32(state, ZSTD_HASHLOG_MIN, 15);
+    cParams.hashLog = FUZZ_dataProducer_uint32Range(producer, ZSTD_HASHLOG_MIN, 15);
-    cParams.chainLog = FUZZ_rand32(state, ZSTD_CHAINLOG_MIN, 16);
+    cParams.chainLog = FUZZ_dataProducer_uint32Range(producer, ZSTD_CHAINLOG_MIN, 16);
-    cParams.searchLog = FUZZ_rand32(state, ZSTD_SEARCHLOG_MIN, 9);
+    cParams.searchLog = FUZZ_dataProducer_uint32Range(producer, ZSTD_SEARCHLOG_MIN, 9);
-    cParams.minMatch = FUZZ_rand32(state, ZSTD_MINMATCH_MIN,
+    cParams.minMatch = FUZZ_dataProducer_uint32Range(producer, ZSTD_MINMATCH_MIN,
                                           ZSTD_MINMATCH_MAX);
-    cParams.targetLength = FUZZ_rand32(state, 0, 512);
+    cParams.targetLength = FUZZ_dataProducer_uint32Range(producer, 0, 512);
-    cParams.strategy = FUZZ_rand32(state, ZSTD_STRATEGY_MIN, ZSTD_STRATEGY_MAX);
+    cParams.strategy = FUZZ_dataProducer_uint32Range(producer, ZSTD_STRATEGY_MIN, ZSTD_STRATEGY_MAX);
     return ZSTD_adjustCParams(cParams, srcSize, 0);
 }
 
-ZSTD_frameParameters FUZZ_randomFParams(uint32_t *state)
+ZSTD_frameParameters FUZZ_randomFParams(FUZZ_dataProducer_t *producer)
 {
     /* Select frame parameters */
     ZSTD_frameParameters fParams;
-    fParams.contentSizeFlag = FUZZ_rand32(state, 0, 1);
+    fParams.contentSizeFlag = FUZZ_dataProducer_uint32Range(producer, 0, 1);
-    fParams.checksumFlag = FUZZ_rand32(state, 0, 1);
+    fParams.checksumFlag = FUZZ_dataProducer_uint32Range(producer, 0, 1);
-    fParams.noDictIDFlag = FUZZ_rand32(state, 0, 1);
+    fParams.noDictIDFlag = FUZZ_dataProducer_uint32Range(producer, 0, 1);
     return fParams;
 }
 
-ZSTD_parameters FUZZ_randomParams(size_t srcSize, uint32_t *state)
+ZSTD_parameters FUZZ_randomParams(size_t srcSize, FUZZ_dataProducer_t *producer)
 {
     ZSTD_parameters params;
-    params.cParams = FUZZ_randomCParams(srcSize, state);
+    params.cParams = FUZZ_randomCParams(srcSize, producer);
-    params.fParams = FUZZ_randomFParams(state);
+    params.fParams = FUZZ_randomFParams(producer);
     return params;
 }
 
-void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, uint32_t *state)
+void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer_t *producer)
 {
-    ZSTD_compressionParameters cParams = FUZZ_randomCParams(srcSize, state);
+    ZSTD_compressionParameters cParams = FUZZ_randomCParams(srcSize, producer);
     set(cctx, ZSTD_c_windowLog, cParams.windowLog);
     set(cctx, ZSTD_c_hashLog, cParams.hashLog);
     set(cctx, ZSTD_c_chainLog, cParams.chainLog);
@@ -72,30 +75,30 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, uint32_t *state)
     set(cctx, ZSTD_c_targetLength, cParams.targetLength);
     set(cctx, ZSTD_c_strategy, cParams.strategy);
     /* Select frame parameters */
-    setRand(cctx, ZSTD_c_contentSizeFlag, 0, 1, state);
+    setRand(cctx, ZSTD_c_contentSizeFlag, 0, 1, producer);
-    setRand(cctx, ZSTD_c_checksumFlag, 0, 1, state);
+    setRand(cctx, ZSTD_c_checksumFlag, 0, 1, producer);
-    setRand(cctx, ZSTD_c_dictIDFlag, 0, 1, state);
+    setRand(cctx, ZSTD_c_dictIDFlag, 0, 1, producer);
     /* Select long distance matching parameters */
-    setRand(cctx, ZSTD_c_enableLongDistanceMatching, 0, 1, state);
+    setRand(cctx, ZSTD_c_enableLongDistanceMatching, 0, 1, producer);
-    setRand(cctx, ZSTD_c_ldmHashLog, ZSTD_HASHLOG_MIN, 16, state);
+    setRand(cctx, ZSTD_c_ldmHashLog, ZSTD_HASHLOG_MIN, 16, producer);
     setRand(cctx, ZSTD_c_ldmMinMatch, ZSTD_LDM_MINMATCH_MIN,
-            ZSTD_LDM_MINMATCH_MAX, state);
+            ZSTD_LDM_MINMATCH_MAX, producer);
     setRand(cctx, ZSTD_c_ldmBucketSizeLog, 0, ZSTD_LDM_BUCKETSIZELOG_MAX,
-            state);
+            producer);
     setRand(cctx, ZSTD_c_ldmHashRateLog, ZSTD_LDM_HASHRATELOG_MIN,
-            ZSTD_LDM_HASHRATELOG_MAX, state);
+            ZSTD_LDM_HASHRATELOG_MAX, producer);
     /* Set misc parameters */
-    setRand(cctx, ZSTD_c_nbWorkers, 0, 2, state);
+    setRand(cctx, ZSTD_c_nbWorkers, 0, 2, producer);
-    setRand(cctx, ZSTD_c_rsyncable, 0, 1, state);
+    setRand(cctx, ZSTD_c_rsyncable, 0, 1, producer);
-    setRand(cctx, ZSTD_c_forceMaxWindow, 0, 1, state);
+    setRand(cctx, ZSTD_c_forceMaxWindow, 0, 1, producer);
-    setRand(cctx, ZSTD_c_literalCompressionMode, 0, 2, state);
+    setRand(cctx, ZSTD_c_literalCompressionMode, 0, 2, producer);
-    setRand(cctx, ZSTD_c_forceAttachDict, 0, 2, state);
+    setRand(cctx, ZSTD_c_forceAttachDict, 0, 2, producer);
-    if (FUZZ_rand32(state, 0, 1) == 0) {
+    if (FUZZ_dataProducer_uint32Range(producer, 0, 1) == 0) {
-      setRand(cctx, ZSTD_c_srcSizeHint, ZSTD_SRCSIZEHINT_MIN, 2 * srcSize, state);
+      setRand(cctx, ZSTD_c_srcSizeHint, ZSTD_SRCSIZEHINT_MIN, 2 * srcSize, producer);
     }
 }
 
-FUZZ_dict_t FUZZ_train(void const* src, size_t srcSize, uint32_t *state)
+FUZZ_dict_t FUZZ_train(void const* src, size_t srcSize, FUZZ_dataProducer_t *producer)
 {
     size_t const dictSize = MAX(srcSize / 8, 1024);
     size_t const totalSampleSize = dictSize * 11;
@@ -110,7 +113,7 @@ FUZZ_dict_t FUZZ_train(void const* src, size_t srcSize, uint32_t *state)
 
     for (sample = 0; sample < nbSamples; ++sample) {
       size_t const remaining = totalSampleSize - pos;
-      size_t const offset = FUZZ_rand32(state, 0, MAX(srcSize, 1) - 1);
+      size_t const offset = FUZZ_dataProducer_uint32Range(producer, 0, MAX(srcSize, 1) - 1);
       size_t const limit = MIN(srcSize - offset, remaining);
       size_t const toCopy = MIN(limit, remaining / (nbSamples - sample));
       memcpy(samples + pos, src + offset, toCopy);

tests/fuzz/zstd_helpers.h

@@ -17,17 +17,21 @@
 #define ZSTD_STATIC_LINKING_ONLY
 
 #include "zstd.h"
+#include "fuzz_data_producer.h"
 #include <stdint.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, uint32_t *state);
+extern const int kMinClevel;
+extern const int kMaxClevel;
 
-ZSTD_compressionParameters FUZZ_randomCParams(size_t srcSize, uint32_t *state);
+void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer_t *producer);
-ZSTD_frameParameters FUZZ_randomFParams(uint32_t *state);
+
-ZSTD_parameters FUZZ_randomParams(size_t srcSize, uint32_t *state);
+ZSTD_compressionParameters FUZZ_randomCParams(size_t srcSize, FUZZ_dataProducer_t *producer);
+ZSTD_frameParameters FUZZ_randomFParams(FUZZ_dataProducer_t *producer);
+ZSTD_parameters FUZZ_randomParams(size_t srcSize, FUZZ_dataProducer_t *producer);
 
 typedef struct {
   void* buff;
@@ -38,8 +42,7 @@ typedef struct {
  * NOTE: Don't use this to train production dictionaries, it is only optimized
  * for speed, and doesn't care about dictionary quality.
  */
-FUZZ_dict_t FUZZ_train(void const* src, size_t srcSize, uint32_t *state);
+FUZZ_dict_t FUZZ_train(void const* src, size_t srcSize, FUZZ_dataProducer_t *producer);
-
 
 #ifdef __cplusplus
 }