ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge from Cyan4973/dev060

Browse Source
This commit is contained in:
inikep
2016-04-04 12:10:00 +02:00
parent 1c556a3838 1535414927
commit 19bd48f516
22 changed files with 1326 additions and 1195 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/bench.c
+77 -71
View File
@@ -58,16 +58,16 @@
# include <sys/resource.h> /* setpriority */
# define BMK_sleep(s) sleep(s)
# define mili_sleep(mili) { struct timespec t; t.tv_sec=0; t.tv_nsec=mili*1000000ULL; nanosleep(&t, NULL); }
# define setHighPriority() setpriority(PRIO_PROCESS, 0, -20)
# define SET_HIGH_PRIORITY setpriority(PRIO_PROCESS, 0, -20)
#elif defined(_WIN32)
# include <windows.h>
# define BMK_sleep(s) Sleep(1000*s)
# define mili_sleep(mili) Sleep(mili)
# define setHighPriority() SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS)
# define SET_HIGH_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS)
#else
# define BMK_sleep(s) /* disabled */
# define mili_sleep(mili) /* disabled */
# define setHighPriority() /* disabled */
# define SET_HIGH_PRIORITY /* disabled */
#endif
#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)))
@@ -97,6 +97,7 @@
#include "xxhash.h"
/* *************************************
* Compiler specifics
***************************************/
@@ -238,18 +239,16 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
const size_t* fileSizes, U32 nbFiles,
const void* dictBuffer, size_t dictBufferSize, benchResult_t *result)
{
const size_t blockSize = (g_blockSize ? g_blockSize : srcSize) + (!srcSize); /* avoid div by 0 */
const U32 maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
size_t const blockSize = (g_blockSize ? g_blockSize : srcSize) + (!srcSize); /* avoid div by 0 */
U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t));
const size_t maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */
size_t const maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */
void* const compressedBuffer = malloc(maxCompressedSize);
void* const resultBuffer = malloc(srcSize);
ZSTD_CCtx* refCtx = ZSTD_createCCtx();
ZSTD_CCtx* ctx = ZSTD_createCCtx();
ZSTD_DCtx* refDCtx = ZSTD_createDCtx();
ZSTD_DCtx* dctx = ZSTD_createDCtx();
U64 const crcOrig = XXH64(srcBuffer, srcSize, 0);
U32 nbBlocks;
BMK_time_t ticksPerSecond;
@@ -287,20 +286,19 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1);
/* Bench */
{ size_t cSize = 0;
double fastestC = 100000000., fastestD = 100000000.;
double ratio = 0.;
{ double fastestC = 100000000., fastestD = 100000000.;
U64 const crcOrig = XXH64(srcBuffer, srcSize, 0);
U64 crcCheck = 0;
BMK_time_t coolTime;
U32 testNb;
size_t cSize = 0;
double ratio = 0.;
BMK_getTime(coolTime);
DISPLAYLEVEL(2, "\r%79s\r", "");
for (testNb = 1; testNb <= (g_nbIterations + !g_nbIterations); testNb++) {
int nbLoops;
BMK_time_t clockStart, clockEnd;
U64 clockSpan;
U64 const clockLoop = g_nbIterations ? TIMELOOP_S*1000000ULL : 10;
U64 clockLoop = g_nbIterations ? TIMELOOP_S*1000000ULL : 10;
/* overheat protection */
if (BMK_clockSpan(coolTime, ticksPerSecond) > ACTIVEPERIOD_S*1000000ULL) {
@@ -319,19 +317,22 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
while (BMK_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) == 0);
BMK_getTime(clockStart);
for (nbLoops = 0 ; BMK_clockSpan(clockStart, ticksPerSecond) < clockLoop ; nbLoops++) {
U32 blockNb;
ZSTD_compressBegin_targetSrcSize(refCtx, dictBuffer, dictBufferSize, blockSize, cLevel);
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
size_t const rSize = ZSTD_compress_usingPreparedCCtx(ctx, refCtx,
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize);
if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_compress_usingPreparedCCtx() failed : %s", ZSTD_getErrorName(rSize));
blockTable[blockNb].cSize = rSize;
{ U32 nbLoops;
for (nbLoops = 0 ; BMK_clockSpan(clockStart, ticksPerSecond) < clockLoop ; nbLoops++) {
U32 blockNb;
ZSTD_compressBegin_targetSrcSize(refCtx, dictBuffer, dictBufferSize, blockSize, cLevel);
// ZSTD_compressBegin_usingDict(refCtx, dictBuffer, dictBufferSize, cLevel);
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
size_t const rSize = ZSTD_compress_usingPreparedCCtx(ctx, refCtx,
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize);
if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_compress_usingPreparedCCtx() failed : %s", ZSTD_getErrorName(rSize));
blockTable[blockNb].cSize = rSize;
} }
{ U64 const clockSpan = BMK_clockSpan(clockStart, ticksPerSecond);
if ((double)clockSpan < fastestC*nbLoops) fastestC = (double)clockSpan / nbLoops;
} }
clockSpan = BMK_clockSpan(clockStart, ticksPerSecond);
if ((double)clockSpan < fastestC*nbLoops) fastestC = (double)clockSpan / nbLoops;
cSize = 0;
{ U32 blockNb; for (blockNb=0; blockNb<nbBlocks; blockNb++) cSize += blockTable[blockNb].cSize; }
ratio = (double)srcSize / (double)cSize;
@@ -339,6 +340,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
testNb, displayName, (U32)srcSize, (U32)cSize, ratio,
(double)srcSize / fastestC );
(void)fastestD; (void)crcOrig; /* unused when decompression disabled */
#if 1
/* Decompression */
memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */
@@ -349,53 +351,55 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
while (BMK_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) == 0);
BMK_getTime(clockStart);
for (nbLoops = 0 ; BMK_clockSpan(clockStart, ticksPerSecond) < clockLoop ; nbLoops++) {
U32 blockNb;
ZSTD_decompressBegin_usingDict(refDCtx, dictBuffer, dictBufferSize);
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
size_t regenSize = ZSTD_decompress_usingPreparedDCtx(dctx, refDCtx,
blockTable[blockNb].resPtr, blockTable[blockNb].srcSize,
blockTable[blockNb].cPtr, blockTable[blockNb].cSize);
if (ZSTD_isError(regenSize)) {
DISPLAY("ZSTD_decompress_usingPreparedDCtx() failed on block %u : %s",
blockNb, ZSTD_getErrorName(regenSize));
goto _findError;
}
blockTable[blockNb].resSize = regenSize;
{ U32 nbLoops;
for (nbLoops = 0 ; BMK_clockSpan(clockStart, ticksPerSecond) < clockLoop ; nbLoops++) {
U32 blockNb;
ZSTD_decompressBegin_usingDict(refDCtx, dictBuffer, dictBufferSize);
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
size_t const regenSize = ZSTD_decompress_usingPreparedDCtx(dctx, refDCtx,
blockTable[blockNb].resPtr, blockTable[blockNb].srcSize,
blockTable[blockNb].cPtr, blockTable[blockNb].cSize);
if (ZSTD_isError(regenSize)) {
DISPLAY("ZSTD_decompress_usingPreparedDCtx() failed on block %u : %s \n",
blockNb, ZSTD_getErrorName(regenSize));
clockLoop = 0; /* force immediate test end */
break;
}
blockTable[blockNb].resSize = regenSize;
} }
{ U64 const clockSpan = BMK_clockSpan(clockStart, ticksPerSecond);
if ((double)clockSpan < fastestD*nbLoops) fastestD = (double)clockSpan / nbLoops;
} }
clockSpan = BMK_clockSpan(clockStart, ticksPerSecond);
if ((double)clockSpan < fastestD*nbLoops) fastestD = (double)clockSpan / nbLoops;
DISPLAYLEVEL(2, "%2i-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r",
testNb, displayName, (U32)srcSize, (U32)cSize, ratio,
(double)srcSize / fastestC,
(double)srcSize / fastestD );
/* CRC Checking */
_findError:
crcCheck = XXH64(resultBuffer, srcSize, 0);
if (crcOrig!=crcCheck) {
size_t u;
DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", displayName, (unsigned)crcOrig, (unsigned)crcCheck);
for (u=0; u<srcSize; u++) {
if (((const BYTE*)srcBuffer)[u] != ((const BYTE*)resultBuffer)[u]) {
U32 segNb, bNb, pos;
size_t bacc = 0;
printf("Decoding error at pos %u ", (U32)u);
for (segNb = 0; segNb < nbBlocks; segNb++) {
if (bacc + blockTable[segNb].srcSize > u) break;
bacc += blockTable[segNb].srcSize;
{ crcCheck = XXH64(resultBuffer, srcSize, 0);
if (crcOrig!=crcCheck) {
size_t u;
DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck);
for (u=0; u<srcSize; u++) {
if (((const BYTE*)srcBuffer)[u] != ((const BYTE*)resultBuffer)[u]) {
U32 segNb, bNb, pos;
size_t bacc = 0;
DISPLAY("Decoding error at pos %u ", (U32)u);
for (segNb = 0; segNb < nbBlocks; segNb++) {
if (bacc + blockTable[segNb].srcSize > u) break;
bacc += blockTable[segNb].srcSize;
}
pos = (U32)(u - bacc);
bNb = pos / (128 KB);
DISPLAY("(block %u, sub %u, pos %u) \n", segNb, bNb, pos);
break;
}
pos = (U32)(u - bacc);
bNb = pos / (128 KB);
printf("(block %u, sub %u, pos %u) \n", segNb, bNb, pos);
break;
}
if (u==srcSize-1) { /* should never happen */
printf("no difference detected\n");
} }
break;
} /* if (crcOrig!=crcCheck) */
if (u==srcSize-1) { /* should never happen */
DISPLAY("no difference detected\n");
} }
break;
} } /* CRC Checking */
#endif
} /* for (testNb = 1; testNb <= (g_nbIterations + !g_nbIterations); testNb++) */
@@ -445,7 +449,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
benchResult_t result, total;
int l;
setHighPriority();
SET_HIGH_PRIORITY;
const char* pch = strrchr(displayName, '\\'); /* Windows */
if (!pch) pch = strrchr(displayName, '/'); /* Linux */
@@ -494,23 +498,25 @@ static U64 BMK_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles)
return total;
}
/*! BMK_loadFiles() :
Loads `buffer` with content of files listed within `fileNamesTable`.
At most, fills `buffer` entirely */
static void BMK_loadFiles(void* buffer, size_t bufferSize,
size_t* fileSizes,
const char** fileNamesTable, unsigned const nbFiles)
const char** fileNamesTable, unsigned nbFiles)
{
size_t pos = 0;
unsigned n;
for (n=0; n<nbFiles; n++) {
size_t readSize;
U64 fileSize = BMK_getFileSize(fileNamesTable[n]);
FILE* f = fopen(fileNamesTable[n], "rb");
FILE* const f = fopen(fileNamesTable[n], "rb");
if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]);
DISPLAYLEVEL(2, "Loading %s... \r", fileNamesTable[n]);
if (fileSize > bufferSize-pos) fileSize = bufferSize-pos;
readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f);
if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]);
pos += readSize;
if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */
{ size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f);
if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]);
pos += readSize; }
fileSizes[n] = (size_t)fileSize;
fclose(f);
}
programs/fileio.c
+16 -17
View File
@@ -333,28 +333,28 @@ static int FIO_compressFilename_internal(cRess_t ress,
{
FILE* srcFile = ress.srcFile;
FILE* dstFile = ress.dstFile;
U64 filesize = 0;
U64 readsize = 0;
U64 compressedfilesize = 0;
size_t dictSize = ress.dictBufferSize;
size_t sizeCheck, errorCode;
size_t sizeCheck;
ZSTD_parameters params;
U64 const fileSize = FIO_getFileSize(srcFileName);
/* init */
filesize = MAX(FIO_getFileSize(srcFileName),dictSize);
params = ZSTD_getParams(cLevel, filesize);
params.srcSize = filesize;
if (g_maxWLog) if (params.windowLog > g_maxWLog) params.windowLog = g_maxWLog;
errorCode = ZBUFF_compressInit_advanced(ress.ctx, ress.dictBuffer, ress.dictBufferSize, params);
if (ZBUFF_isError(errorCode)) EXM_THROW(21, "Error initializing compression : %s", ZBUFF_getErrorName(errorCode));
params.cParams = ZSTD_getCParams(cLevel, fileSize, dictSize);
params.fParams.contentSizeFlag = 1;
if (g_maxWLog) if (params.cParams.windowLog > g_maxWLog) params.cParams.windowLog = g_maxWLog;
{ size_t const errorCode = ZBUFF_compressInit_advanced(ress.ctx, ress.dictBuffer, ress.dictBufferSize, params, fileSize);
if (ZBUFF_isError(errorCode)) EXM_THROW(21, "Error initializing compression : %s", ZBUFF_getErrorName(errorCode)); }
/* Main compression loop */
filesize = 0;
readsize = 0;
while (1) {
/* Fill input Buffer */
size_t inSize = fread(ress.srcBuffer, (size_t)1, ress.srcBufferSize, srcFile);
size_t const inSize = fread(ress.srcBuffer, (size_t)1, ress.srcBufferSize, srcFile);
if (inSize==0) break;
filesize += inSize;
DISPLAYUPDATE(2, "\rRead : %u MB ", (U32)(filesize>>20));
readsize += inSize;
DISPLAYUPDATE(2, "\rRead : %u MB ", (U32)(readsize>>20));
{ /* Compress using buffered streaming */
size_t usedInSize = inSize;
@@ -371,13 +371,12 @@ static int FIO_compressFilename_internal(cRess_t ress,
if (sizeCheck!=cSize) EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName);
compressedfilesize += cSize;
}
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (U32)(filesize>>20), (double)compressedfilesize/filesize*100);
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (U32)(readsize>>20), (double)compressedfilesize/readsize*100);
}
/* End of Frame */
{
size_t cSize = ress.dstBufferSize;
size_t result = ZBUFF_compressEnd(ress.ctx, ress.dstBuffer, &cSize);
{ size_t cSize = ress.dstBufferSize;
size_t const result = ZBUFF_compressEnd(ress.ctx, ress.dstBuffer, &cSize);
if (result!=0) EXM_THROW(26, "Compression error : cannot create frame end");
sizeCheck = fwrite(ress.dstBuffer, 1, cSize, dstFile);
@@ -388,7 +387,7 @@ static int FIO_compressFilename_internal(cRess_t ress,
/* Status */
DISPLAYLEVEL(2, "\r%79s\r", "");
DISPLAYLEVEL(2,"Compressed %llu bytes into %llu bytes ==> %.2f%%\n",
(unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100);
(unsigned long long)readsize, (unsigned long long) compressedfilesize, (double)compressedfilesize/readsize*100);
return 0;
}
programs/fullbench.c
+3 -5
View File
@@ -118,7 +118,7 @@ static clock_t BMK_clockSpan( clock_t clockStart )
static size_t BMK_findMaxMem(U64 requiredMem)
{
const size_t step = 64 MB;
size_t const step = 64 MB;
void* testmem = NULL;
requiredMem = (((requiredMem >> 26) + 1) << 26);
@@ -182,15 +182,13 @@ size_t local_ZSTD_decodeLiteralsBlock(void* dst, size_t dstSize, void* buff2, co
}
extern size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr);
extern size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr, FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb, const void* src, size_t srcSize);
extern size_t ZSTD_decodeSeqHeaders(int* nbSeq, FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb, const void* src, size_t srcSize);
size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
{
U32 DTableML[FSE_DTABLE_SIZE_U32(10)], DTableLL[FSE_DTABLE_SIZE_U32(10)], DTableOffb[FSE_DTABLE_SIZE_U32(9)]; /* MLFSELog, LLFSELog and OffFSELog are not public values */
const BYTE* dumps;
size_t length;
int nbSeq;
(void)src; (void)srcSize; (void)dst; (void)dstSize;
return ZSTD_decodeSeqHeaders(&nbSeq, &dumps, &length, DTableLL, DTableML, DTableOffb, buff2, g_cSize);
return ZSTD_decodeSeqHeaders(&nbSeq, DTableLL, DTableML, DTableOffb, buff2, g_cSize);
}
programs/fuzzer.c
+2 -2
View File
@@ -488,12 +488,12 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 maxDuration, doub
if (cSize > 3) {
const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
const size_t tooSmallSize = cSize - missing;
static const U32 endMark = 0x4DC2B1A9;
const U32 endMark = 0x4DC2B1A9;
memcpy(dstBuffer+tooSmallSize, &endMark, 4);
errorCode = ZSTD_compressCCtx(ctx, dstBuffer, tooSmallSize, sampleBuffer, sampleSize, cLevel);
CHECK(!ZSTD_isError(errorCode), "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (U32)tooSmallSize, (U32)cSize);
{ U32 endCheck; memcpy(&endCheck, dstBuffer+tooSmallSize, 4);
CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow"); }
CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow"); }
}
/* frame header decompression test */
programs/paramgrill.c
+78 -70
View File
@@ -126,7 +126,7 @@ static U32 g_rand = 1;
static U32 g_singleRun = 0;
static U32 g_target = 0;
static U32 g_noSeed = 0;
static ZSTD_parameters g_params = { 0, 0, 0, 0, 0, 0, 0, ZSTD_greedy };
static ZSTD_compressionParameters g_params = { 0, 0, 0, 0, 0, 0, ZSTD_greedy };
void BMK_SetNbIterations(int nbLoops)
{
@@ -251,7 +251,7 @@ typedef struct
static size_t BMK_benchParam(BMK_result_t* resultPtr,
const void* srcBuffer, size_t srcSize,
ZSTD_CCtx* ctx,
const ZSTD_parameters params)
const ZSTD_compressionParameters cParams)
{
const size_t blockSize = g_blockSize ? g_blockSize : srcSize;
const U32 nbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize);
@@ -259,13 +259,14 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
const size_t maxCompressedSize = (size_t)nbBlocks * ZSTD_compressBound(blockSize);
void* const compressedBuffer = malloc(maxCompressedSize);
void* const resultBuffer = malloc(srcSize);
U32 Wlog = params.windowLog;
U32 Clog = params.contentLog;
U32 Hlog = params.hashLog;
U32 Slog = params.searchLog;
U32 Slength = params.searchLength;
U32 Tlength = params.targetLength;
ZSTD_strategy strat = params.strategy;
ZSTD_parameters params;
U32 Wlog = cParams.windowLog;
U32 Clog = cParams.contentLog;
U32 Hlog = cParams.hashLog;
U32 Slog = cParams.searchLog;
U32 Slength = cParams.searchLength;
U32 Tlength = cParams.targetLength;
ZSTD_strategy strat = cParams.strategy;
char name[30] = { 0 };
U64 crcOrig;
@@ -315,6 +316,8 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
const int startTime =BMK_GetMilliStart();
DISPLAY("\r%79s\r", "");
params.cParams = cParams;
params.fParams.contentSizeFlag = 0;
for (loopNb = 1; loopNb <= g_nbIterations; loopNb++) {
int nbLoops;
int milliTime;
@@ -407,11 +410,11 @@ const char* g_stratName[] = { "ZSTD_fast ",
"ZSTD_btlazy2",
"ZSTD_btopt " };
static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_parameters params, size_t srcSize)
static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_compressionParameters params, size_t srcSize)
{
DISPLAY("\r%79s\r", "");
fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u,%3u, %s }, ",
0, params.windowLog, params.contentLog, params.hashLog, params.searchLog, params.searchLength,
fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ",
params.windowLog, params.contentLog, params.hashLog, params.searchLog, params.searchLength,
params.targetLength, g_stratName[(U32)(params.strategy)]);
fprintf(f,
"/* level %2u */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
@@ -423,7 +426,7 @@ static U32 g_cSpeedTarget[NB_LEVELS_TRACKED] = { 0 }; /* NB_LEVELS_TRACKED : c
typedef struct {
BMK_result_t result;
ZSTD_parameters params;
ZSTD_compressionParameters params;
} winnerInfo_t;
static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize)
@@ -431,7 +434,7 @@ static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSi
unsigned cLevel;
fprintf(f, "\n /* Proposed configurations : */ \n");
fprintf(f, " /* l, W, C, H, S, L, T, strat */ \n");
fprintf(f, " /* W, C, H, S, L, T, strat */ \n");
for (cLevel=0; cLevel <= ZSTD_maxCLevel(); cLevel++)
BMK_printWinner(f, cLevel, winners[cLevel].result, winners[cLevel].params, srcSize);
@@ -446,9 +449,9 @@ static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, size_t srcSiz
BMK_printWinners2(stdout, winners, srcSize);
}
size_t ZSTD_sizeofCCtx(ZSTD_parameters params); /* hidden interface, declared here */
size_t ZSTD_sizeofCCtx(ZSTD_compressionParameters params); /* hidden interface, declared here */
static int BMK_seed(winnerInfo_t* winners, const ZSTD_parameters params,
static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters params,
const void* srcBuffer, size_t srcSize,
ZSTD_CCtx* ctx)
{
@@ -541,7 +544,7 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_parameters params,
/* nullified useless params, to ensure count stats */
static ZSTD_parameters* sanitizeParams(ZSTD_parameters params)
static ZSTD_compressionParameters* sanitizeParams(ZSTD_compressionParameters params)
{
g_params = params;
if (params.strategy == ZSTD_fast)
@@ -552,44 +555,50 @@ static ZSTD_parameters* sanitizeParams(ZSTD_parameters params)
}
static void paramVariation(ZSTD_parameters* p)
static void paramVariation(ZSTD_compressionParameters* ptr)
{
U32 nbChanges = (FUZ_rand(&g_rand) & 3) + 1;
for (; nbChanges; nbChanges--) {
const U32 changeID = FUZ_rand(&g_rand) % 14;
switch(changeID)
{
case 0:
p->contentLog++; break;
case 1:
p->contentLog--; break;
case 2:
p->hashLog++; break;
case 3:
p->hashLog--; break;
case 4:
p->searchLog++; break;
case 5:
p->searchLog--; break;
case 6:
p->windowLog++; break;
case 7:
p->windowLog--; break;
case 8:
p->searchLength++; break;
case 9:
p->searchLength--; break;
case 10:
p->strategy = (ZSTD_strategy)(((U32)p->strategy)+1); break;
case 11:
p->strategy = (ZSTD_strategy)(((U32)p->strategy)-1); break;
case 12:
p->targetLength *= 1 + ((double)(FUZ_rand(&g_rand)&255)) / 256.; break;
case 13:
p->targetLength /= 1 + ((double)(FUZ_rand(&g_rand)&255)) / 256.; break;
ZSTD_compressionParameters p;
U32 validated = 0;
while (!validated) {
U32 nbChanges = (FUZ_rand(&g_rand) & 3) + 1;
p = *ptr;
for ( ; nbChanges ; nbChanges--) {
const U32 changeID = FUZ_rand(&g_rand) % 14;
switch(changeID)
{
case 0:
p.contentLog++; break;
case 1:
p.contentLog--; break;
case 2:
p.hashLog++; break;
case 3:
p.hashLog--; break;
case 4:
p.searchLog++; break;
case 5:
p.searchLog--; break;
case 6:
p.windowLog++; break;
case 7:
p.windowLog--; break;
case 8:
p.searchLength++; break;
case 9:
p.searchLength--; break;
case 10:
p.strategy = (ZSTD_strategy)(((U32)p.strategy)+1); break;
case 11:
p.strategy = (ZSTD_strategy)(((U32)p.strategy)-1); break;
case 12:
p.targetLength *= 1 + ((double)(FUZ_rand(&g_rand)&255)) / 256.; break;
case 13:
p.targetLength /= 1 + ((double)(FUZ_rand(&g_rand)&255)) / 256.; break;
}
}
validated = !ZSTD_isError(ZSTD_checkCParams(p));
}
ZSTD_validateParams(p);
*ptr = p;
}
@@ -605,7 +614,7 @@ static BYTE g_alreadyTested[PARAMTABLESIZE] = {0}; /* init to zero */
#define MAX(a,b) ( (a) > (b) ? (a) : (b) )
static void playAround(FILE* f, winnerInfo_t* winners,
ZSTD_parameters params,
ZSTD_compressionParameters params,
const void* srcBuffer, size_t srcSize,
ZSTD_CCtx* ctx)
{
@@ -613,7 +622,7 @@ static void playAround(FILE* f, winnerInfo_t* winners,
const int startTime = BMK_GetMilliStart();
while (BMK_GetMilliSpan(startTime) < g_maxVariationTime) {
ZSTD_parameters p = params;
ZSTD_compressionParameters p = params;
if (nbVariations++ > g_maxNbVariations) break;
paramVariation(&p);
@@ -634,10 +643,12 @@ static void playAround(FILE* f, winnerInfo_t* winners,
}
static void potentialRandomParams(ZSTD_parameters* p, U32 inverseChance)
static void potentialRandomParams(ZSTD_compressionParameters* p, U32 inverseChance)
{
U32 chance = (FUZ_rand(&g_rand) % (inverseChance+1));
if (!chance) {
U32 validated = 0;
if (!chance)
while (!validated) {
/* totally random entry */
p->contentLog = FUZ_rand(&g_rand) % (ZSTD_CONTENTLOG_MAX+1 - ZSTD_CONTENTLOG_MIN) + ZSTD_CONTENTLOG_MIN;
p->hashLog = FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN) + ZSTD_HASHLOG_MIN;
@@ -646,7 +657,7 @@ static void potentialRandomParams(ZSTD_parameters* p, U32 inverseChance)
p->searchLength=FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN) + ZSTD_SEARCHLENGTH_MIN;
p->targetLength=FUZ_rand(&g_rand) % (ZSTD_TARGETLENGTH_MAX+1 - ZSTD_TARGETLENGTH_MIN) + ZSTD_TARGETLENGTH_MIN;
p->strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btopt +1));
ZSTD_validateParams(p);
validated = !ZSTD_isError(ZSTD_checkCParams(*p));
}
}
@@ -658,10 +669,9 @@ static void BMK_selectRandomStart(
U32 id = (FUZ_rand(&g_rand) % (ZSTD_maxCLevel()+1));
if ((id==0) || (winners[id].params.windowLog==0)) {
/* totally random entry */
ZSTD_parameters p;
ZSTD_compressionParameters p;
potentialRandomParams(&p, 1);
p.srcSize = srcSize;
ZSTD_validateParams(&p);
ZSTD_adjustCParams(&p, srcSize, 0);
playAround(f, winners, p, srcBuffer, srcSize, ctx);
}
else
@@ -672,7 +682,7 @@ static void BMK_selectRandomStart(
static void BMK_benchMem(void* srcBuffer, size_t srcSize)
{
ZSTD_CCtx* ctx = ZSTD_createCCtx();
ZSTD_parameters params;
ZSTD_compressionParameters params;
winnerInfo_t winners[NB_LEVELS_TRACKED];
int i;
unsigned u;
@@ -682,8 +692,7 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize)
if (g_singleRun) {
BMK_result_t testResult;
g_params.srcSize = blockSize;
ZSTD_validateParams(&g_params);
ZSTD_adjustCParams(&g_params, srcSize, 0);
BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, g_params);
DISPLAY("\n");
return;
@@ -699,7 +708,7 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize)
else {
/* baseline config for level 1 */
BMK_result_t testResult;
params = ZSTD_getParams(1, blockSize);
params = ZSTD_getCParams(1, blockSize, 0);
BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, params);
g_cSpeedTarget[1] = (testResult.cSpeed * 31) >> 5;
}
@@ -712,8 +721,7 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize)
{
const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel();
for (i=1; i<=maxSeeds; i++) {
params = ZSTD_getParams(i, blockSize);
ZSTD_validateParams(&params);
params = ZSTD_getCParams(i, blockSize, 0);
BMK_seed(winners, params, srcBuffer, srcSize, ctx);
}
}
@@ -864,7 +872,7 @@ int optimizeForSize(char* inFileName)
{
ZSTD_CCtx* ctx = ZSTD_createCCtx();
ZSTD_parameters params;
ZSTD_compressionParameters params;
winnerInfo_t winner;
BMK_result_t candidate;
const size_t blockSize = g_blockSize ? g_blockSize : benchedSize;
@@ -878,7 +886,7 @@ int optimizeForSize(char* inFileName)
{
const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel();
for (i=1; i<=maxSeeds; i++) {
params = ZSTD_getParams(i, blockSize);
params = ZSTD_getCParams(i, blockSize, 0);
BMK_benchParam(&candidate, origBuff, benchedSize, ctx, params);
if ( (candidate.cSize < winner.result.cSize)
||((candidate.cSize == winner.result.cSize) && (candidate.cSpeed > winner.result.cSpeed)) )
@@ -1028,7 +1036,7 @@ int main(int argc, char** argv)
case 'S':
g_singleRun = 1;
argument++;
g_params = ZSTD_getParams(2, g_blockSize);
g_params = ZSTD_getCParams(2, g_blockSize, 0);
for ( ; ; ) {
switch(*argument)
{
@@ -1079,7 +1087,7 @@ int main(int argc, char** argv)
argument++;
while ((*argument>= '0') && (*argument<='9'))
cLevel *= 10, cLevel += *argument++ - '0';
g_params = ZSTD_getParams(cLevel, g_blockSize);
g_params = ZSTD_getCParams(cLevel, g_blockSize, 0);
continue;
}
default : ;
programs/playTests.sh
+12 -1
View File
@@ -25,7 +25,9 @@ roundTripTest() {
echo "\n**** simple tests **** "
./datagen > tmp
$ZSTD tmp
echo -n "trivial compression : "
$ZSTD -f tmp
echo "OK"
$ZSTD -99 tmp && die "too large compression level undetected"
$ZSTD tmp -c > tmpCompressed
$ZSTD tmp --stdout > tmpCompressed
@@ -71,6 +73,11 @@ echo "\n**** dictionary tests **** "
./datagen -g1M | md5sum > tmp1
./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | md5sum > tmp2
diff -q tmp1 tmp2
$ZSTD --train *.c *.h -o tmpDict
$ZSTD xxhash.c -D tmpDict -of tmp
$ZSTD -d tmp -D tmpDict -of result
diff xxhash.c result
echo "\n**** multiple files tests **** "
@@ -106,6 +113,10 @@ $ZSTD -t * && die "bad files not detected !"
echo "\n**** zstd round-trip tests **** "
roundTripTest
roundTripTest -g15K # TableID==3
roundTripTest -g127K # TableID==2
roundTripTest -g255K # TableID==1
roundTripTest -g513K # TableID==0
roundTripTest -g512K 6 # greedy, hash chain
roundTripTest -g512K 16 # btlazy2
roundTripTest -g512K 19 # btopt
programs/zbufftest.c
+160 -195
View File
@@ -1,6 +1,6 @@
/*
Fuzzer test tool for zstd_buffered
Copyright (C) Yann Collet 2105
Copyright (C) Yann Collet 2015-2016
GPL v2 License
@@ -19,11 +19,10 @@
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You can contact the author at :
- ZSTD source repository : https://github.com/Cyan4973/zstd
- ZSTD public forum : https://groups.google.com/forum/#!forum/lz4c
- ZSTD homepage : https://www.zstd.net/
*/
/**************************************
/*-************************************
* Compiler specific
**************************************/
#ifdef _MSC_VER /* Visual Studio */
@@ -33,7 +32,7 @@
#endif
/**************************************
/*-************************************
* Includes
**************************************/
#include <stdlib.h> /* free */
@@ -42,13 +41,13 @@
#include <string.h> /* strcmp */
#include "mem.h"
#include "zbuff.h"
#include "zstd.h" /* ZSTD_compressBound() */
#include "zstd_static.h" /* ZSTD_compressBound(), ZSTD_maxCLevel() */
#include "datagen.h" /* RDG_genBuffer */
#include "xxhash.h" /* XXH64 */
/**************************************
Constants
/*-************************************
* Constants
**************************************/
#ifndef ZSTD_VERSION
# define ZSTD_VERSION ""
@@ -66,7 +65,7 @@ static const U32 prime2 = 2246822519U;
/**************************************
/*-************************************
* Display Macros
**************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
@@ -83,7 +82,7 @@ static U32 g_displayTime = 0;
static U32 g_testTime = 0;
/*********************************************************
/*-*******************************************************
* Fuzzer functions
*********************************************************/
#define MAX(a,b) ((a)>(b)?(a):(b))
@@ -107,15 +106,17 @@ static U32 FUZ_GetMilliSpan(U32 nTimeStart)
return nSpan;
}
/*! FUZ_rand() :
@return : a 27 bits random value, from a 32-bits `seed`.
`seed` is also modified */
# define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
unsigned int FUZ_rand(unsigned int* src)
unsigned int FUZ_rand(unsigned int* seedPtr)
{
U32 rand32 = *src;
U32 rand32 = *seedPtr;
rand32 *= prime1;
rand32 += prime2;
rand32 = FUZ_rotl32(rand32, 13);
*src = rand32;
*seedPtr = rand32;
return rand32 >> 5;
}
@@ -133,12 +134,12 @@ static unsigned FUZ_highbit32(U32 v32)
static int basicUnitTests(U32 seed, double compressibility)
{
int testResult = 0;
void* CNBuffer;
size_t CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
void* compressedBuffer;
size_t compressedBufferSize = ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
void* decodedBuffer;
size_t decodedBufferSize = CNBufferSize;
void* CNBuffer = malloc(CNBufferSize);
size_t const compressedBufferSize = ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
void* compressedBuffer = malloc(compressedBufferSize);
size_t const decodedBufferSize = CNBufferSize;
void* decodedBuffer = malloc(decodedBufferSize);
U32 randState = seed;
size_t result, cSize, readSize, genSize;
U32 testNb=0;
@@ -146,11 +147,7 @@ static int basicUnitTests(U32 seed, double compressibility)
ZBUFF_DCtx* zd = ZBUFF_createDCtx();
/* Create compressible test buffer */
CNBuffer = malloc(CNBufferSize);
compressedBuffer = malloc(compressedBufferSize);
decodedBuffer = malloc(decodedBufferSize);
if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd)
{
if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd) {
DISPLAY("Not enough memory, aborting\n");
goto _output_error;
}
@@ -183,11 +180,9 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(4, "OK \n");
/* check regenerated data is byte exact */
{
size_t i;
{ size_t i;
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
for (i=0; i<CNBufferSize; i++)
{
for (i=0; i<CNBufferSize; i++) {
if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;;
}
DISPLAYLEVEL(4, "OK \n");
@@ -213,8 +208,7 @@ static size_t findDiff(const void* buf1, const void* buf2, size_t max)
const BYTE* b1 = (const BYTE*)buf1;
const BYTE* b2 = (const BYTE*)buf2;
size_t i;
for (i=0; i<max; i++)
{
for (i=0; i<max; i++) {
if (b1[i] != b2[i]) break;
}
return i;
@@ -225,28 +219,39 @@ static size_t findDiff(const void* buf1, const void* buf2, size_t max)
#define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; }
static const U32 maxSrcLog = 24;
static const U32 maxSampleLog = 19;
int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility)
static size_t FUZ_rLogLength(U32* seed, U32 logLength)
{
size_t const lengthMask = ((size_t)1 << logLength) - 1;
return (lengthMask+1) + (FUZ_rand(seed) & lengthMask);
}
static size_t FUZ_randomLength(U32* seed, U32 maxLog)
{
U32 const logLength = FUZ_rand(seed) % maxLog;
return FUZ_rLogLength(seed, logLength);
}
static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility)
{
static const U32 maxSrcLog = 24;
static const U32 maxSampleLog = 19;
BYTE* cNoiseBuffer[5];
BYTE* srcBuffer;
size_t srcBufferSize = (size_t)1<<maxSrcLog;
BYTE* copyBuffer;
size_t copyBufferSize = srcBufferSize + (1<<maxSampleLog);
size_t copyBufferSize= srcBufferSize + (1<<maxSampleLog);
BYTE* cBuffer;
size_t cBufferSize = ZSTD_compressBound(srcBufferSize);
BYTE* dstBuffer;
size_t dstBufferSize = srcBufferSize;
U32 result = 0;
U32 testNb = 0;
U32 coreSeed = seed, lseed = 0;
U32 coreSeed = seed;
ZBUFF_CCtx* zc;
ZBUFF_DCtx* zd;
U32 startTime = FUZ_GetMilliStart();
/* allocation */
/* allocations */
zc = ZBUFF_createCCtx();
zd = ZBUFF_createDCtx();
cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
@@ -267,172 +272,144 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
srcBuffer = cNoiseBuffer[2];
memset(copyBuffer, 0x65, copyBufferSize);
memcpy(copyBuffer, srcBuffer, MIN(copyBufferSize,srcBufferSize)); /* make copyBuffer considered initialized */
memset(copyBuffer, 0x65, copyBufferSize); /* make copyBuffer considered initialized */
/* catch up testNb */
for (testNb=1; testNb < startTest; testNb++)
FUZ_rand(&coreSeed);
/* test loop */
for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime); testNb++ )
{
size_t sampleSize, sampleStart;
for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime) ; testNb++ ) {
U32 lseed;
const BYTE* srcBuffer;
const BYTE* dict;
size_t cSize, dictSize;
size_t maxTestSize, totalTestSize, readSize, totalCSize, genSize, totalGenSize;
size_t maxTestSize, dictSize;
size_t cSize, totalTestSize, totalCSize, totalGenSize;
size_t errorCode;
U32 sampleSizeLog, buffNb, n, nbChunks;
U32 n, nbChunks;
XXH64_CREATESTATE_STATIC(xxh64);
U64 crcOrig, crcDest;
U64 crcOrig;
/* init */
DISPLAYUPDATE(2, "\r%6u", testNb);
if (nbTests >= testNb) DISPLAYUPDATE(2, "/%6u ", nbTests);
FUZ_rand(&coreSeed);
lseed = coreSeed ^ prime1;
buffNb = FUZ_rand(&lseed) & 127;
if (buffNb & 7) buffNb=2; /* select buffer */
else
{
buffNb >>= 3;
if (buffNb & 7)
{
const U32 tnb[2] = { 1, 3 };
buffNb = tnb[buffNb >> 3];
}
else
{
const U32 tnb[2] = { 0, 4 };
buffNb = tnb[buffNb >> 3];
}
}
srcBuffer = cNoiseBuffer[buffNb];
/* Multi - segments compression test */
/* state total reset */
/* some problems only happen when states are re-used in a specific order */
if ((FUZ_rand(&lseed) & 0xFF) == 131) { ZBUFF_freeCCtx(zc); zc = ZBUFF_createCCtx(); }
if ((FUZ_rand(&lseed) & 0xFF) == 132) { ZBUFF_freeDCtx(zd); zd = ZBUFF_createDCtx(); }
/* srcBuffer selection [0-4] */
{ U32 buffNb = FUZ_rand(&lseed) & 0x7F;
if (buffNb & 7) buffNb=2; /* most common : compressible (P) */
else {
buffNb >>= 3;
if (buffNb & 7) {
const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */
buffNb = tnb[buffNb >> 3];
} else {
const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */
buffNb = tnb[buffNb >> 3];
} }
srcBuffer = cNoiseBuffer[buffNb];
}
/* compression init */
{ U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (testLog/3))) + 1;
maxTestSize = FUZ_rLogLength(&lseed, testLog);
/* random dictionary selection */
{ size_t dictStart;
dictSize = (FUZ_rand(&lseed)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0;
dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
dict = srcBuffer + dictStart;
}
{ size_t const initError = ZBUFF_compressInitDictionary(zc, dict, dictSize, cLevel);
CHECK (ZBUFF_isError(initError),"init error : %s", ZBUFF_getErrorName(initError));
} }
/* multi-segments compression test */
XXH64_reset(xxh64, 0);
nbChunks = (FUZ_rand(&lseed) & 127) + 2;
sampleSizeLog = FUZ_rand(&lseed) % maxSrcLog;
maxTestSize = (size_t)1 << sampleSizeLog;
maxTestSize += FUZ_rand(&lseed) & (maxTestSize-1);
nbChunks = (FUZ_rand(&lseed) & 127) + 2;
for (n=0, cSize=0, totalTestSize=0 ; (n<nbChunks) && (totalTestSize < maxTestSize) ; n++) {
/* compress random chunk into random size dst buffer */
{ size_t readChunkSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - readChunkSize);
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
dict = srcBuffer + sampleStart;
dictSize = sampleSize;
ZBUFF_compressInitDictionary(zc, dict, dictSize, (FUZ_rand(&lseed) % (20 - (sampleSizeLog/3))) + 1);
size_t const compressionError = ZBUFF_compressContinue(zc, cBuffer+cSize, &dstBuffSize, srcBuffer+srcStart, &readChunkSize);
CHECK (ZBUFF_isError(compressionError), "compression error : %s", ZBUFF_getErrorName(compressionError));
totalTestSize = 0;
cSize = 0;
for (n=0; n<nbChunks; n++)
{
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
readSize = sampleSize;
/* random size output buffer */
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
genSize = MIN (cBufferSize - cSize, sampleSize);
errorCode = ZBUFF_compressContinue(zc, cBuffer+cSize, &genSize, srcBuffer+sampleStart, &readSize);
CHECK (ZBUFF_isError(errorCode), "compression error : %s", ZBUFF_getErrorName(errorCode));
XXH64_update(xxh64, srcBuffer+sampleStart, readSize);
memcpy(copyBuffer+totalTestSize, srcBuffer+sampleStart, readSize);
cSize += genSize;
totalTestSize += readSize;
if ((FUZ_rand(&lseed) & 15) == 0)
{
/* add a few random flushes operations, to mess around */
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
genSize = MIN (cBufferSize - cSize, sampleSize);
errorCode = ZBUFF_compressFlush(zc, cBuffer+cSize, &genSize);
CHECK (ZBUFF_isError(errorCode), "flush error : %s", ZBUFF_getErrorName(errorCode));
cSize += genSize;
XXH64_update(xxh64, srcBuffer+srcStart, readChunkSize);
memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, readChunkSize);
cSize += dstBuffSize;
totalTestSize += readChunkSize;
}
if (totalTestSize > maxTestSize) break;
/* random flush operation, to mess around */
if ((FUZ_rand(&lseed) & 15) == 0) {
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
size_t const flushError = ZBUFF_compressFlush(zc, cBuffer+cSize, &dstBuffSize);
CHECK (ZBUFF_isError(flushError), "flush error : %s", ZBUFF_getErrorName(flushError));
cSize += dstBuffSize;
} }
/* final frame epilogue */
{ size_t dstBuffSize = cBufferSize - cSize;
size_t const flushError = ZBUFF_compressEnd(zc, cBuffer+cSize, &dstBuffSize);
CHECK (ZBUFF_isError(flushError), "flush error : %s", ZBUFF_getErrorName(flushError));
cSize += dstBuffSize;
}
genSize = cBufferSize - cSize;
errorCode = ZBUFF_compressEnd(zc, cBuffer+cSize, &genSize);
CHECK (ZBUFF_isError(errorCode), "compression error : %s", ZBUFF_getErrorName(errorCode));
CHECK (errorCode != 0, "frame epilogue not fully consumed");
cSize += genSize;
crcOrig = XXH64_digest(xxh64);
/* multi - fragments decompression test */
ZBUFF_decompressInitDictionary(zd, dict, dictSize);
totalCSize = 0;
totalGenSize = 0;
while (totalCSize < cSize)
{
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
readSize = sampleSize;
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
genSize = MIN(sampleSize, dstBufferSize - totalGenSize);
errorCode = ZBUFF_decompressContinue(zd, dstBuffer+totalGenSize, &genSize, cBuffer+totalCSize, &readSize);
CHECK (ZBUFF_isError(errorCode), "decompression error : %s", ZBUFF_getErrorName(errorCode));
totalGenSize += genSize;
totalCSize += readSize;
for (totalCSize = 0, totalGenSize = 0 ; totalCSize < cSize ; ) {
size_t readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
size_t const decompressError = ZBUFF_decompressContinue(zd, dstBuffer+totalGenSize, &dstBuffSize, cBuffer+totalCSize, &readCSrcSize);
CHECK (ZBUFF_isError(decompressError), "decompression error : %s", ZBUFF_getErrorName(decompressError));
totalGenSize += dstBuffSize;
totalCSize += readCSrcSize;
errorCode = decompressError; /* needed for != 0 last test */
}
CHECK (errorCode != 0, "frame not fully decoded");
CHECK (totalGenSize != totalTestSize, "decompressed data : wrong size")
CHECK (totalCSize != cSize, "compressed data should be fully read")
crcDest = XXH64(dstBuffer, totalTestSize, 0);
if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize);
CHECK (crcDest!=crcOrig, "decompressed data corrupted");
{ U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize);
CHECK (crcDest!=crcOrig, "decompressed data corrupted"); }
/*===== noisy/erroneous src decompression test =====*/
/* noisy/erroneous src decompression test */
/* add some noise */
nbChunks = (FUZ_rand(&lseed) & 7) + 2;
for (n=0; n<nbChunks; n++)
{
size_t cStart;
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
if (sampleSize > cSize/3) sampleSize = cSize/3;
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
cStart = FUZ_rand(&lseed) % (cSize - sampleSize);
memcpy(cBuffer+cStart, srcBuffer+sampleStart, sampleSize);
}
{ U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2;
U32 nn; for (nn=0; nn<nbNoiseChunks; nn++) {
size_t const randomNoiseSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const noiseSize = MIN((cSize/3) , randomNoiseSize);
size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseSize);
size_t const cStart = FUZ_rand(&lseed) % (cSize - noiseSize);
memcpy(cBuffer+cStart, srcBuffer+noiseStart, noiseSize);
} }
/* try decompression on noisy data */
ZBUFF_decompressInit(zd);
totalCSize = 0;
totalGenSize = 0;
while ( (totalCSize < cSize) && (totalGenSize < dstBufferSize) )
{
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
readSize = sampleSize;
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
genSize = MIN(sampleSize, dstBufferSize - totalGenSize);
errorCode = ZBUFF_decompressContinue(zd, dstBuffer+totalGenSize, &genSize, cBuffer+totalCSize, &readSize);
if (ZBUFF_isError(errorCode)) break; /* error correctly detected */
totalGenSize += genSize;
totalCSize += readSize;
}
}
while ( (totalCSize < cSize) && (totalGenSize < dstBufferSize) ) {
size_t readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
size_t const decompressError = ZBUFF_decompressContinue(zd, dstBuffer+totalGenSize, &dstBuffSize, cBuffer+totalCSize, &readCSrcSize);
if (ZBUFF_isError(decompressError)) break; /* error correctly detected */
totalGenSize += dstBuffSize;
totalCSize += readCSrcSize;
} }
DISPLAY("\r%u fuzzer tests completed \n", testNb);
_cleanup:
@@ -454,10 +431,10 @@ _output_error:
}
/*********************************************************
/*-*******************************************************
* Command line
*********************************************************/
int FUZ_usage(char* programName)
int FUZ_usage(const char* programName)
{
DISPLAY( "Usage :\n");
DISPLAY( " %s [args]\n", programName);
@@ -474,7 +451,7 @@ int FUZ_usage(char* programName)
}
int main(int argc, char** argv)
int main(int argc, const char** argv)
{
U32 seed=0;
int seedset=0;
@@ -484,23 +461,18 @@ int main(int argc, char** argv)
int proba = FUZ_COMPRESSIBILITY_DEFAULT;
int result=0;
U32 mainPause = 0;
char* programName;
const char* programName = argv[0];
/* Check command line */
programName = argv[0];
for(argNb=1; argNb<argc; argNb++)
{
char* argument = argv[argNb];
for(argNb=1; argNb<argc; argNb++) {
const char* argument = argv[argNb];
if(!argument) continue; /* Protection if argument empty */
/* Handle commands. Aggregated commands are allowed */
if (argument[0]=='-')
{
/* Parsing commands. Aggregated commands are allowed */
if (argument[0]=='-') {
argument++;
while (*argument!=0)
{
while (*argument!=0) {
switch(*argument)
{
case 'h':
@@ -521,8 +493,7 @@ int main(int argc, char** argv)
case 'i':
argument++;
nbTests=0; g_testTime=0;
while ((*argument>='0') && (*argument<='9'))
{
while ((*argument>='0') && (*argument<='9')) {
nbTests *= 10;
nbTests += *argument - '0';
argument++;
@@ -532,8 +503,7 @@ int main(int argc, char** argv)
case 'T':
argument++;
nbTests=0; g_testTime=0;
while ((*argument>='0') && (*argument<='9'))
{
while ((*argument>='0') && (*argument<='9')) {
g_testTime *= 10;
g_testTime += *argument - '0';
argument++;
@@ -547,8 +517,7 @@ int main(int argc, char** argv)
argument++;
seed=0;
seedset=1;
while ((*argument>='0') && (*argument<='9'))
{
while ((*argument>='0') && (*argument<='9')) {
seed *= 10;
seed += *argument - '0';
argument++;
@@ -558,8 +527,7 @@ int main(int argc, char** argv)
case 't':
argument++;
testNb=0;
while ((*argument>='0') && (*argument<='9'))
{
while ((*argument>='0') && (*argument<='9')) {
testNb *= 10;
testNb += *argument - '0';
argument++;
@@ -569,8 +537,7 @@ int main(int argc, char** argv)
case 'P': /* compressibility % */
argument++;
proba=0;
while ((*argument>='0') && (*argument<='9'))
{
while ((*argument>='0') && (*argument<='9')) {
proba *= 10;
proba += *argument - '0';
argument++;
@@ -582,9 +549,7 @@ int main(int argc, char** argv)
default:
return FUZ_usage(programName);
}
}
}
}
} } } /* for(argNb=1; argNb<argc; argNb++) */
/* Get Seed */
DISPLAY("Starting zstd_buffered tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION);
@@ -598,8 +563,8 @@ int main(int argc, char** argv)
if (testNb==0) result = basicUnitTests(0, ((double)proba) / 100); /* constant seed for predictability */
if (!result)
result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100);
if (mainPause)
{
if (mainPause) {
int unused;
DISPLAY("Press Enter \n");
unused = getchar();