ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fixed fuzzer dictionary test

Browse Source
This commit is contained in:
Yann Collet
2016-05-29 23:09:51 +02:00
parent fcf2087a15
commit 33341de7d7
2 changed files with 34 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/fuzzer.c
+29 -29
View File
@@ -87,7 +87,7 @@ static clock_t FUZ_clockSpan(clock_t cStart)
# define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r))) # define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
unsigned int FUZ_rand(unsigned int* src) unsigned FUZ_rand(unsigned* src)
{ {
static const U32 prime1 = 2654435761U; static const U32 prime1 = 2654435761U;
static const U32 prime2 = 2246822519U; static const U32 prime2 = 2246822519U;
@@ -115,17 +115,14 @@ static unsigned FUZ_highbit32(U32 v32)
static int basicUnitTests(U32 seed, double compressibility) static int basicUnitTests(U32 seed, double compressibility)
{ {
int testResult = 0; int testResult = 0;
void* CNBuffer; void* const CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
void* compressedBuffer; void* const compressedBuffer = malloc(ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH));
void* decodedBuffer; void* const decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
U32 randState = seed; U32 randState = seed;
size_t result, cSize; size_t result, cSize;
U32 testNb=0; U32 testNb=0;
/* Create compressible test buffer */ /* Create compressible test buffer */
CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
compressedBuffer = malloc(ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH));
decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
if (!CNBuffer || !compressedBuffer || !decodedBuffer) { if (!CNBuffer || !compressedBuffer || !decodedBuffer) {
DISPLAY("Not enough memory, aborting\n"); DISPLAY("Not enough memory, aborting\n");
testResult = 1; testResult = 1;
@@ -227,19 +224,16 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(4, "test%3i : check content size on duplicated context : ", testNb++); DISPLAYLEVEL(4, "test%3i : check content size on duplicated context : ", testNb++);
{ size_t const testSize = COMPRESSIBLE_NOISE_LENGTH / 3; { size_t const testSize = COMPRESSIBLE_NOISE_LENGTH / 3;
{ ZSTD_parameters p; { ZSTD_parameters const p = (ZSTD_parameters) { ZSTD_getCParams(2, testSize, dictSize), { 1, 0 } };
p.cParams = ZSTD_getCParams(2, testSize, dictSize); size_t const initResult = ZSTD_compressBegin_advanced(ctxOrig, CNBuffer, dictSize, p, testSize-1);
p.fParams.contentSizeFlag = 1;
{ size_t const initResult = ZSTD_compressBegin_advanced(ctxOrig, CNBuffer, dictSize, p, testSize-1);
if (ZSTD_isError(initResult)) goto _output_error; if (ZSTD_isError(initResult)) goto _output_error;
} } }
{ size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig); { size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig);
if (ZSTD_isError(copyResult)) goto _output_error; } if (ZSTD_isError(copyResult)) goto _output_error; }
cSize = ZSTD_compressContinue(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize), (const char*)CNBuffer + dictSize, COMPRESSIBLE_NOISE_LENGTH - dictSize); cSize = ZSTD_compressContinue(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize), (const char*)CNBuffer + dictSize, COMPRESSIBLE_NOISE_LENGTH - dictSize);
if (ZSTD_isError(cSize)) goto _output_error; if (ZSTD_isError(cSize)) goto _output_error;
{ ZSTD_frameParams fp; { ZSTD_frameParams fp;
size_t const gfpResult = ZSTD_getFrameParams(&fp, compressedBuffer, cSize); if (ZSTD_getFrameParams(&fp, compressedBuffer, cSize)) goto _output_error;
if (gfpResult!=0) goto _output_error;
if ((fp.frameContentSize != testSize) && (fp.frameContentSize != 0)) goto _output_error; if ((fp.frameContentSize != testSize) && (fp.frameContentSize != 0)) goto _output_error;
} } } }
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
@@ -258,20 +252,20 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(4, "test%3i : Check magic Number : ", testNb++); DISPLAYLEVEL(4, "test%3i : Check magic Number : ", testNb++);
((char*)(CNBuffer))[0] = 1; ((char*)(CNBuffer))[0] = 1;
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, CNBuffer, 4); { size_t const r = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, CNBuffer, 4);
if (!ZSTD_isError(result)) goto _output_error; if (!ZSTD_isError(r)) goto _output_error; }
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
/* block API tests */ /* block API tests */
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx(); { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
ZSTD_DCtx* const dctx = ZSTD_createDCtx(); ZSTD_DCtx* const dctx = ZSTD_createDCtx();
const size_t blockSize = 100 KB; static const size_t blockSize = 100 KB;
const size_t dictSize = 16 KB; static const size_t dictSize = 16 KB;
/* basic block compression */ /* basic block compression */
DISPLAYLEVEL(4, "test%3i : Block compression test : ", testNb++); DISPLAYLEVEL(4, "test%3i : Block compression test : ", testNb++);
result = ZSTD_compressBegin(cctx, 5); { size_t const r = ZSTD_compressBegin(cctx, 5);
if (ZSTD_isError(result)) goto _output_error; if (ZSTD_isError(r)) goto _output_error; }
cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize); cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize);
if (ZSTD_isError(cSize)) goto _output_error; if (ZSTD_isError(cSize)) goto _output_error;
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
@@ -575,13 +569,12 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
} }
if (pos <= cSize) break; if (pos <= cSize) break;
/* add noise */ /* add noise */
{ U32 nbBits = FUZ_rand(&lseed) % maxNbBits; { U32 const nbBitsCodes = FUZ_rand(&lseed) % maxNbBits;
size_t mask, noiseStart, noiseLength; U32 const nbBits = nbBitsCodes ? nbBitsCodes-1 : 0;
if (nbBits>0) nbBits--; size_t const mask = (1<<nbBits) - 1;
mask = (1<<nbBits) - 1; size_t const rNoiseLength = (FUZ_rand(&lseed) & mask) + 1;
noiseLength = (FUZ_rand(&lseed) & mask) + 1; size_t const noiseLength = MIN(rNoiseLength, cSize-pos);
if ( pos+noiseLength > cSize ) noiseLength = cSize-pos; size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseLength);
noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseLength);
memcpy(cBuffer + pos, srcBuffer + noiseStart, noiseLength); memcpy(cBuffer + pos, srcBuffer + noiseStart, noiseLength);
pos += noiseLength; pos += noiseLength;
} } } } } }
@@ -610,8 +603,14 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
dict = srcBuffer + sampleStart; dict = srcBuffer + sampleStart;
dictSize = sampleSize; dictSize = sampleSize;
{ size_t const errorCode = ZSTD_compressBegin_usingDict(refCtx, dict, dictSize, cLevel); if (FUZ_rand(&lseed) & 15) {
CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_usingDict error : %s", ZSTD_getErrorName(errorCode)); } size_t const errorCode = ZSTD_compressBegin_usingDict(refCtx, dict, dictSize, cLevel);
CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_usingDict error : %s", ZSTD_getErrorName(errorCode));
} else {
ZSTD_parameters p = (ZSTD_parameters) { ZSTD_getCParams(cLevel, 0, dictSize), { 0, 0 } };
size_t const errorCode = ZSTD_compressBegin_advanced(refCtx, dict, dictSize, p, 0);
CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_advanced error : %s", ZSTD_getErrorName(errorCode));
}
{ size_t const errorCode = ZSTD_copyCCtx(ctx, refCtx); { size_t const errorCode = ZSTD_copyCCtx(ctx, refCtx);
CHECK (ZSTD_isError(errorCode), "ZSTD_copyCCtx error : %s", ZSTD_getErrorName(errorCode)); } CHECK (ZSTD_isError(errorCode), "ZSTD_copyCCtx error : %s", ZSTD_getErrorName(errorCode)); }
} }
@@ -641,6 +640,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
crcOrig = XXH64_digest(&xxhState); crcOrig = XXH64_digest(&xxhState);
/* streaming decompression test */ /* streaming decompression test */
if (dictSize<8) dictSize=0, dict=NULL; /* disable dictionary */
{ size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, dict, dictSize); { size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, dict, dictSize);
CHECK (ZSTD_isError(errorCode), "cannot init DCtx : %s", ZSTD_getErrorName(errorCode)); } CHECK (ZSTD_isError(errorCode), "cannot init DCtx : %s", ZSTD_getErrorName(errorCode)); }
totalCSize = 0; totalCSize = 0;
programs/tests/playTests.sh
+4 -2
View File
@@ -114,7 +114,7 @@ $ZSTD -d -v -f tmpSparseCompressed -o tmpSparseRegenerated
$ZSTD -d -v -f tmpSparseCompressed -c >> tmpSparseRegenerated $ZSTD -d -v -f tmpSparseCompressed -c >> tmpSparseRegenerated
ls -ls tmpSparse* ls -ls tmpSparse*
diff tmpSparse2M tmpSparseRegenerated diff tmpSparse2M tmpSparseRegenerated
# rm tmpSparse* rm tmpSparse*
$ECHO "\n**** dictionary tests **** " $ECHO "\n**** dictionary tests **** "
@@ -123,10 +123,12 @@ $ECHO "\n**** dictionary tests **** "
./datagen -g1M | md5sum > tmp1 ./datagen -g1M | md5sum > tmp1
./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | md5sum > tmp2 ./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | md5sum > tmp2
diff -q tmp1 tmp2 diff -q tmp1 tmp2
$ZSTD --train *.c *.h -o tmpDict $ZSTD --train *.c -o tmpDict
$ZSTD zstdcli.c -D tmpDict -of tmp $ZSTD zstdcli.c -D tmpDict -of tmp
$ZSTD -d tmp -D tmpDict -of result $ZSTD -d tmp -D tmpDict -of result
diff zstdcli.c result diff zstdcli.c result
$ZSTD --train *.c *.h -o tmpDictC
$ZSTD -d tmp -D tmpDictC -of result && die "wrong dictionary not detected!"
$ECHO "\n**** multiple files tests **** " $ECHO "\n**** multiple files tests **** "