ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[fuzzer] Fuzz long range matching & new API

Browse Source
This commit is contained in:
Nick Terrell
2017-09-14 14:48:08 -07:00
parent 9712d5ebe6
commit 39357c41cb
6 changed files with 209 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/fuzz/Makefile
+23 -18
View File
@@ -33,14 +33,19 @@ FUZZ_LDFLAGS := $(LDFLAGS)
FUZZ_ARFLAGS := $(ARFLAGS) FUZZ_ARFLAGS := $(ARFLAGS)
FUZZ_TARGET_FLAGS = $(FUZZ_CPPFLAGS) $(FUZZ_CXXFLAGS) $(FUZZ_LDFLAGS) FUZZ_TARGET_FLAGS = $(FUZZ_CPPFLAGS) $(FUZZ_CXXFLAGS) $(FUZZ_LDFLAGS)
FUZZ_HEADERS := fuzz_helpers.h fuzz.h FUZZ_HEADERS := fuzz_helpers.h fuzz.h zstd_helpers.h
FUZZ_SRC := zstd_helpers.c
ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c ZSTDCOMMON_SRC := $(ZSTDDIR)/common/*.c
ZSTDCOMP_FILES := $(ZSTDDIR)/compress/*.c ZSTDCOMP_SRC := $(ZSTDDIR)/compress/*.c
ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/*.c ZSTDDECOMP_SRC := $(ZSTDDIR)/decompress/*.c
ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) FUZZ_SRC := \
$(FUZZ_SRC) \
$(ZSTDDECOMP_SRC) \
$(ZSTDCOMMON_SRC) \
$(ZSTDCOMP_SRC)
ZSTD_OBJ := $(patsubst %.c,%.o, $(wildcard $(ZSTD_FILES))) FUZZ_OBJ := $(patsubst %.c,%.o, $(wildcard $(FUZZ_SRC)))
.PHONY: default all clean .PHONY: default all clean
@@ -58,23 +63,23 @@ all: \
%.o: %.c %.o: %.c
$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $^ -c -o $@ $(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $^ -c -o $@
simple_round_trip: $(FUZZ_HEADERS) $(ZSTD_OBJ) simple_round_trip.o simple_round_trip: $(FUZZ_HEADERS) $(FUZZ_OBJ) simple_round_trip.o
$(CXX) $(FUZZ_TARGET_FLAGS) $(ZSTD_OBJ) simple_round_trip.o $(LIB_FUZZING_ENGINE) -o $@ $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_OBJ) simple_round_trip.o $(LIB_FUZZING_ENGINE) -o $@
stream_round_trip: $(FUZZ_HEADERS) $(ZSTD_OBJ) stream_round_trip.o stream_round_trip: $(FUZZ_HEADERS) $(FUZZ_OBJ) stream_round_trip.o
$(CXX) $(FUZZ_TARGET_FLAGS) $(ZSTD_OBJ) stream_round_trip.o $(LIB_FUZZING_ENGINE) -o $@ $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_OBJ) stream_round_trip.o $(LIB_FUZZING_ENGINE) -o $@
block_round_trip: $(FUZZ_HEADERS) $(ZSTD_OBJ) block_round_trip.o block_round_trip: $(FUZZ_HEADERS) $(FUZZ_OBJ) block_round_trip.o
$(CXX) $(FUZZ_TARGET_FLAGS) $(ZSTD_OBJ) block_round_trip.o $(LIB_FUZZING_ENGINE) -o $@ $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_OBJ) block_round_trip.o $(LIB_FUZZING_ENGINE) -o $@
simple_decompress: $(FUZZ_HEADERS) $(ZSTD_OBJ) simple_decompress.o simple_decompress: $(FUZZ_HEADERS) $(FUZZ_OBJ) simple_decompress.o
$(CXX) $(FUZZ_TARGET_FLAGS) $(ZSTD_OBJ) simple_decompress.o $(LIB_FUZZING_ENGINE) -o $@ $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_OBJ) simple_decompress.o $(LIB_FUZZING_ENGINE) -o $@
stream_decompress: $(FUZZ_HEADERS) $(ZSTD_OBJ) stream_decompress.o stream_decompress: $(FUZZ_HEADERS) $(FUZZ_OBJ) stream_decompress.o
$(CXX) $(FUZZ_TARGET_FLAGS) $(ZSTD_OBJ) stream_decompress.o $(LIB_FUZZING_ENGINE) -o $@ $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_OBJ) stream_decompress.o $(LIB_FUZZING_ENGINE) -o $@
block_decompress: $(FUZZ_HEADERS) $(ZSTD_OBJ) block_decompress.o block_decompress: $(FUZZ_HEADERS) $(FUZZ_OBJ) block_decompress.o
$(CXX) $(FUZZ_TARGET_FLAGS) $(ZSTD_OBJ) block_decompress.o $(LIB_FUZZING_ENGINE) -o $@ $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_OBJ) block_decompress.o $(LIB_FUZZING_ENGINE) -o $@
libregression.a: $(FUZZ_HEADERS) $(PRGDIR)/util.h regression_driver.o libregression.a: $(FUZZ_HEADERS) $(PRGDIR)/util.h regression_driver.o
$(AR) $(FUZZ_ARFLAGS) $@ regression_driver.o $(AR) $(FUZZ_ARFLAGS) $@ regression_driver.o
tests/fuzz/fuzz_helpers.h
+24 -13
View File
@@ -16,8 +16,10 @@
#include "fuzz.h" #include "fuzz.h"
#include "xxhash.h" #include "xxhash.h"
#include "zstd.h"
#include <stdint.h> #include <stdint.h>
#include <stdio.h> #include <stdio.h>
#include <stdlib.h>
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
@@ -38,6 +40,8 @@ extern "C" {
__LINE__, FUZZ_QUOTE(cond), (msg)), \ __LINE__, FUZZ_QUOTE(cond), (msg)), \
abort())) abort()))
#define FUZZ_ASSERT(cond) FUZZ_ASSERT_MSG((cond), ""); #define FUZZ_ASSERT(cond) FUZZ_ASSERT_MSG((cond), "");
#define FUZZ_ZASSERT(code) \
FUZZ_ASSERT_MSG(!ZSTD_isError(code), ZSTD_getErrorName(code))
#if defined(__GNUC__) #if defined(__GNUC__)
#define FUZZ_STATIC static __inline __attribute__((unused)) #define FUZZ_STATIC static __inline __attribute__((unused))
@@ -55,23 +59,30 @@ extern "C" {
* Consumes up to the first FUZZ_RNG_SEED_SIZE bytes of the input. * Consumes up to the first FUZZ_RNG_SEED_SIZE bytes of the input.
*/ */
FUZZ_STATIC uint32_t FUZZ_seed(uint8_t const **src, size_t* size) { FUZZ_STATIC uint32_t FUZZ_seed(uint8_t const **src, size_t* size) {
uint8_t const *data = *src; uint8_t const *data = *src;
size_t const toHash = MIN(FUZZ_RNG_SEED_SIZE, *size); size_t const toHash = MIN(FUZZ_RNG_SEED_SIZE, *size);
*size -= toHash; *size -= toHash;
*src += toHash; *src += toHash;
return XXH32(data, toHash, 0); return XXH32(data, toHash, 0);
} }
#define FUZZ_rotl32(x, r) (((x) << (r)) | ((x) >> (32 - (r)))) #define FUZZ_rotl32(x, r) (((x) << (r)) | ((x) >> (32 - (r))))
FUZZ_STATIC uint32_t FUZZ_rand(uint32_t *state) { FUZZ_STATIC uint32_t FUZZ_rand(uint32_t *state) {
static const uint32_t prime1 = 2654435761U; static const uint32_t prime1 = 2654435761U;
static const uint32_t prime2 = 2246822519U; static const uint32_t prime2 = 2246822519U;
uint32_t rand32 = *state; uint32_t rand32 = *state;
rand32 *= prime1; rand32 *= prime1;
rand32 += prime2; rand32 += prime2;
rand32 = FUZZ_rotl32(rand32, 13); rand32 = FUZZ_rotl32(rand32, 13);
*state = rand32; *state = rand32;
return rand32 >> 5; return rand32 >> 5;
}
/* Returns a random numer in the range [min, max]. */
FUZZ_STATIC uint32_t FUZZ_rand32(uint32_t *state, uint32_t min, uint32_t max) {
uint32_t random = FUZZ_rand(state);
return min + (random % (max - min + 1));
} }
#ifdef __cplusplus #ifdef __cplusplus
tests/fuzz/simple_round_trip.c
+25 -9
View File
@@ -12,12 +12,14 @@
* compares the result with the original, and calls abort() on corruption. * compares the result with the original, and calls abort() on corruption.
*/ */
#define ZSTD_STATIC_LINKING_ONLY
#include <stddef.h> #include <stddef.h>
#include <stdlib.h> #include <stdlib.h>
#include <stdio.h> #include <stdio.h>
#include <string.h> #include <string.h>
#include "fuzz_helpers.h" #include "fuzz_helpers.h"
#include "zstd.h" #include "zstd_helpers.h"
static const int kMaxClevel = 19; static const int kMaxClevel = 19;
@@ -32,14 +34,28 @@ static size_t roundTripTest(void *result, size_t resultCapacity,
void *compressed, size_t compressedCapacity, void *compressed, size_t compressedCapacity,
const void *src, size_t srcSize) const void *src, size_t srcSize)
{ {
int const cLevel = FUZZ_rand(&seed) % kMaxClevel; size_t cSize;
size_t const cSize = ZSTD_compressCCtx(cctx, compressed, compressedCapacity, if (FUZZ_rand(&seed) & 1) {
src, srcSize, cLevel); ZSTD_inBuffer in = {src, srcSize, 0};
if (ZSTD_isError(cSize)) { ZSTD_outBuffer out = {compressed, compressedCapacity, 0};
fprintf(stderr, "Compression error: %s\n", ZSTD_getErrorName(cSize));
return cSize; ZSTD_CCtx_reset(cctx);
} FUZZ_setRandomParameters(cctx, &seed);
return ZSTD_decompressDCtx(dctx, result, resultCapacity, compressed, cSize); size_t const err = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
if (err != 0) {
return err;
}
cSize = out.pos;
} else {
int const cLevel = FUZZ_rand(&seed) % kMaxClevel;
cSize = ZSTD_compressCCtx(
cctx, compressed, compressedCapacity, src, srcSize, cLevel);
}
if (ZSTD_isError(cSize)) {
fprintf(stderr, "Compression error: %s\n", ZSTD_getErrorName(cSize));
return cSize;
}
return ZSTD_decompressDCtx(dctx, result, resultCapacity, compressed, cSize);
} }
int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
tests/fuzz/stream_round_trip.c
+60 -55
View File
@@ -12,16 +12,16 @@
* compares the result with the original, and calls abort() on corruption. * compares the result with the original, and calls abort() on corruption.
*/ */
#define ZSTD_STATIC_LINKING_ONLY
#include <stddef.h> #include <stddef.h>
#include <stdlib.h> #include <stdlib.h>
#include <stdio.h> #include <stdio.h>
#include <string.h> #include <string.h>
#include "fuzz_helpers.h" #include "fuzz_helpers.h"
#include "zstd.h" #include "zstd_helpers.h"
static const int kMaxClevel = 19; ZSTD_CCtx *cctx = NULL;
static ZSTD_CStream *cstream = NULL;
static ZSTD_DCtx *dctx = NULL; static ZSTD_DCtx *dctx = NULL;
static uint8_t* cBuf = NULL; static uint8_t* cBuf = NULL;
static uint8_t* rBuf = NULL; static uint8_t* rBuf = NULL;
@@ -30,84 +30,89 @@ static uint32_t seed;
static ZSTD_outBuffer makeOutBuffer(uint8_t *dst, size_t capacity) static ZSTD_outBuffer makeOutBuffer(uint8_t *dst, size_t capacity)
{ {
ZSTD_outBuffer buffer = { dst, 0, 0 }; ZSTD_outBuffer buffer = { dst, 0, 0 };
FUZZ_ASSERT(capacity > 0); FUZZ_ASSERT(capacity > 0);
buffer.size = (FUZZ_rand(&seed) % capacity) + 1; buffer.size = (FUZZ_rand(&seed) % capacity) + 1;
FUZZ_ASSERT(buffer.size <= capacity); FUZZ_ASSERT(buffer.size <= capacity);
return buffer; return buffer;
} }
static ZSTD_inBuffer makeInBuffer(const uint8_t **src, size_t *size) static ZSTD_inBuffer makeInBuffer(const uint8_t **src, size_t *size)
{ {
ZSTD_inBuffer buffer = { *src, 0, 0 }; ZSTD_inBuffer buffer = { *src, 0, 0 };
FUZZ_ASSERT(*size > 0); FUZZ_ASSERT(*size > 0);
buffer.size = (FUZZ_rand(&seed) % *size) + 1; buffer.size = (FUZZ_rand(&seed) % *size) + 1;
FUZZ_ASSERT(buffer.size <= *size); FUZZ_ASSERT(buffer.size <= *size);
*src += buffer.size; *src += buffer.size;
*size -= buffer.size; *size -= buffer.size;
return buffer; return buffer;
} }
static size_t compress(uint8_t *dst, size_t capacity, static size_t compress(uint8_t *dst, size_t capacity,
const uint8_t *src, size_t srcSize) const uint8_t *src, size_t srcSize)
{ {
int cLevel = FUZZ_rand(&seed) % kMaxClevel;
size_t dstSize = 0; size_t dstSize = 0;
FUZZ_ASSERT(!ZSTD_isError(ZSTD_initCStream(cstream, cLevel))); ZSTD_CCtx_reset(cctx);
FUZZ_setRandomParameters(cctx, &seed);
while (srcSize > 0) { while (srcSize > 0) {
ZSTD_inBuffer in = makeInBuffer(&src, &srcSize); ZSTD_inBuffer in = makeInBuffer(&src, &srcSize);
/* Mode controls the action. If mode == -1 we pick a new mode */ /* Mode controls the action. If mode == -1 we pick a new mode */
int mode = -1; int mode = -1;
while (in.pos < in.size) { while (in.pos < in.size) {
ZSTD_outBuffer out = makeOutBuffer(dst, capacity); ZSTD_outBuffer out = makeOutBuffer(dst, capacity);
/* Previous action finished, pick a new mode. */ /* Previous action finished, pick a new mode. */
if (mode == -1) mode = FUZZ_rand(&seed) % 10; if (mode == -1) mode = FUZZ_rand(&seed) % 10;
switch (mode) { switch (mode) {
case 0: /* fall-though */ case 0: /* fall-though */
case 1: /* fall-though */ case 1: /* fall-though */
case 2: { case 2: {
size_t const ret = ZSTD_flushStream(cstream, &out); size_t const ret =
FUZZ_ASSERT_MSG(!ZSTD_isError(ret), ZSTD_getErrorName(ret)); ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_flush);
if (ret == 0) mode = -1; FUZZ_ZASSERT(ret);
break; if (ret == 0)
} mode = -1;
case 3: { break;
size_t ret = ZSTD_endStream(cstream, &out); }
FUZZ_ASSERT_MSG(!ZSTD_isError(ret), ZSTD_getErrorName(ret)); case 3: {
/* Reset the compressor when the frame is finished */ size_t ret =
if (ret == 0) { ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
cLevel = FUZZ_rand(&seed) % kMaxClevel; FUZZ_ZASSERT(ret);
ret = ZSTD_initCStream(cstream, cLevel); /* Reset the compressor when the frame is finished */
FUZZ_ASSERT(!ZSTD_isError(ret)); if (ret == 0) {
ZSTD_CCtx_reset(cctx);
FUZZ_setRandomParameters(cctx, &seed);
mode = -1;
}
break;
}
default: {
size_t const ret =
ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_continue);
FUZZ_ZASSERT(ret);
mode = -1; mode = -1;
} }
break;
} }
default: { dst += out.pos;
size_t const ret = ZSTD_compressStream(cstream, &out, &in); dstSize += out.pos;
FUZZ_ASSERT_MSG(!ZSTD_isError(ret), ZSTD_getErrorName(ret)); capacity -= out.pos;
mode = -1;
}
}
dst += out.pos;
dstSize += out.pos;
capacity -= out.pos;
} }
} }
for (;;) { for (;;) {
ZSTD_inBuffer in = {NULL, 0, 0};
ZSTD_outBuffer out = makeOutBuffer(dst, capacity); ZSTD_outBuffer out = makeOutBuffer(dst, capacity);
size_t const ret = ZSTD_endStream(cstream, &out); size_t const ret = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
FUZZ_ASSERT_MSG(!ZSTD_isError(ret), ZSTD_getErrorName(ret)); FUZZ_ZASSERT(ret);
dst += out.pos; dst += out.pos;
dstSize += out.pos; dstSize += out.pos;
capacity -= out.pos; capacity -= out.pos;
if (ret == 0) break; if (ret == 0)
break;
} }
return dstSize; return dstSize;
} }
@@ -128,9 +133,9 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
bufSize = neededBufSize; bufSize = neededBufSize;
FUZZ_ASSERT(cBuf && rBuf); FUZZ_ASSERT(cBuf && rBuf);
} }
if (!cstream) { if (!cctx) {
cstream = ZSTD_createCStream(); cctx = ZSTD_createCCtx();
FUZZ_ASSERT(cstream); FUZZ_ASSERT(cctx);
} }
if (!dctx) { if (!dctx) {
dctx = ZSTD_createDCtx(); dctx = ZSTD_createDCtx();
@@ -147,7 +152,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
} }
#ifndef STATEFUL_FUZZING #ifndef STATEFUL_FUZZING
ZSTD_freeCStream(cstream); cstream = NULL; ZSTD_freeCCtx(cctx); cctx = NULL;
ZSTD_freeDCtx(dctx); dctx = NULL; ZSTD_freeDCtx(dctx); dctx = NULL;
#endif #endif
return 0; return 0;
tests/fuzz/zstd_helpers.c
+46
View File
@@ -0,0 +1,46 @@
/*
* Copyright (c) 2016-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
*/
#define ZSTD_STATIC_LINKING_ONLY
#include "zstd_helpers.h"
#include "fuzz_helpers.h"
#include "zstd.h"
static void setRand(ZSTD_CCtx *cctx, ZSTD_cParameter param, unsigned min,
unsigned max, uint32_t *state) {
unsigned const value = FUZZ_rand32(state, min, max);
FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, param, value));
}
void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, uint32_t *state)
{
setRand(cctx, ZSTD_p_windowLog, ZSTD_WINDOWLOG_MIN, 23, state);
setRand(cctx, ZSTD_p_hashLog, ZSTD_HASHLOG_MIN, 23, state);
setRand(cctx, ZSTD_p_chainLog, ZSTD_CHAINLOG_MIN, 24, state);
setRand(cctx, ZSTD_p_searchLog, ZSTD_SEARCHLOG_MIN, 9, state);
setRand(cctx, ZSTD_p_minMatch, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX,
state);
setRand(cctx, ZSTD_p_targetLength, ZSTD_TARGETLENGTH_MIN,
ZSTD_TARGETLENGTH_MAX, state);
setRand(cctx, ZSTD_p_compressionStrategy, ZSTD_fast, ZSTD_btultra, state);
/* Select frame parameters */
setRand(cctx, ZSTD_p_contentSizeFlag, 0, 1, state);
setRand(cctx, ZSTD_p_checksumFlag, 0, 1, state);
setRand(cctx, ZSTD_p_dictIDFlag, 0, 1, state);
/* Select long distance matchig parameters */
setRand(cctx, ZSTD_p_enableLongDistanceMatching, 0, 1, state);
setRand(cctx, ZSTD_p_ldmHashLog, ZSTD_HASHLOG_MIN, 24, state);
setRand(cctx, ZSTD_p_ldmMinMatch, ZSTD_LDM_MINMATCH_MIN,
ZSTD_LDM_MINMATCH_MAX, state);
setRand(cctx, ZSTD_p_ldmBucketSizeLog, 0, ZSTD_LDM_BUCKETSIZELOG_MAX,
state);
setRand(cctx, ZSTD_p_ldmHashEveryLog, 0,
ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN, state);
}
tests/fuzz/zstd_helpers.h
+31
View File
@@ -0,0 +1,31 @@
/*
* Copyright (c) 2016-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
*/
/**
* Helper functions for fuzzing.
*/
#ifndef ZSTD_HELPERS_H
#define ZSTD_HELPERS_H
#include "zstd.h"
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
#endif
void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, uint32_t *state);
#ifdef __cplusplus
}
#endif
#endif /* ZSTD_HELPERS_H */