ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/dev' into btopt

Browse Source 
* upstream/dev: (305 commits)
  added test for ZSTD_estimateCStreamSize()
  changed variable name, for clarity
  fixed ZSTD_estimateCStreamSize()
  shortened ZSTD_createCStream_Advanced()
  fixed symbols test
  added ZSTD_estimateDStreamSize()
  changed name frameParams into frameHeader
  regroup memory usage function declarations
  separated ZSTD_estimateCStreamSize() from ZSTD_estimateCCtxSize()
  bumped version number
  added ZSTD_estimateCDictSize() and ZSTD_estimateDDictSize()
  Updated ZSTD_freeCCtx()
  updated ZSTD_estimateCCtxSize()
  Updated ZSTD_sizeof_CCtx()
  merged CCtx and CStream as a single same object
  cli : -d and -t do not stop after a failed decompression
  added dev branch CircleCI badge
  added dev branch Appveyor badge
  keep dev branch status only
  creates a binary archive without the `programs` directory
  ...
This commit is contained in:
Nick Terrell
2017-05-10 16:49:58 -07:00
parent f35ef5c8e9 ba41b26405
commit 5f2c7213c7
149 changed files with 22690 additions and 2821 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/Makefile
+19 -13
View File
@@ -26,9 +26,12 @@ PYTHON ?= python3
TESTARTEFACT := versionsTest namespaceTest
CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR)
DEBUGFLAGS=-g -DZSTD_DEBUG=1
CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
-I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) \
$(DEBUGFLAG)
CFLAGS ?= -O3
CFLAGS += -g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
-Wstrict-prototypes -Wundef -Wformat-security
CFLAGS += $(MOREFLAGS)
@@ -60,23 +63,23 @@ endif
MULTITHREAD = $(MULTITHREAD_CPP) $(MULTITHREAD_LD)
VOID = /dev/null
ZSTREAM_TESTTIME = -T2mn
ZSTREAM_TESTTIME ?= -T2mn
FUZZERTEST ?= -T5mn
ZSTDRTTEST = --test-large-data
DECODECORPUS_TESTTIME = -T30
DECODECORPUS_TESTTIME ?= -T30
.PHONY: default all all32 dll clean test test32 test-all namespaceTest versionsTest
.PHONY: default all all32 allnothread dll clean test test32 test-all namespaceTest versionsTest
default: fullbench
all: fullbench fuzzer zstreamtest paramgrill datagen zbufftest
all: fullbench fuzzer zstreamtest paramgrill datagen zbufftest decodecorpus
all32: fullbench32 fuzzer32 zstreamtest32 zbufftest32
allnothread: fullbench fuzzer paramgrill datagen zbufftest decodecorpus
dll: fuzzer-dll zstreamtest-dll zbufftest-dll
zstd:
$(MAKE) -C $(PRGDIR) $@
@@ -154,6 +157,7 @@ zstreamtest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c zstreamtest.c
$(MAKE) -C $(ZSTDDIR) libzstd
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@$(EXT)
paramgrill : DEBUGFLAG =
paramgrill : $(ZSTD_FILES) $(PRGDIR)/datagen.c paramgrill.c
$(CC) $(FLAGS) $^ -lm -o $@$(EXT)
@@ -169,7 +173,7 @@ longmatch : $(ZSTD_FILES) longmatch.c
invalidDictionaries : $(ZSTD_FILES) invalidDictionaries.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
legacy : CFLAGS+= -DZSTD_LEGACY_SUPPORT=1
legacy : CFLAGS+= -DZSTD_LEGACY_SUPPORT=4
legacy : CPPFLAGS+= -I$(ZSTDDIR)/legacy
legacy : $(ZSTD_FILES) $(wildcard $(ZSTDDIR)/legacy/*.c) legacy.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
@@ -257,6 +261,8 @@ zstd-playTests: datagen
shortest: ZSTDRTTEST=
shortest: test-zstd
fuzztest: test-fuzzer test-zstream test-decodecorpus
test: test-zstd test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-legacy test-decodecorpus
ifeq ($(QEMU_SYS),)
test: test-pool
@@ -300,10 +306,10 @@ test-fullbench32: fullbench32 datagen
$(QEMU_SYS) ./fullbench32 -i1 -P0
test-fuzzer: fuzzer
$(QEMU_SYS) ./fuzzer $(FUZZERTEST)
$(QEMU_SYS) ./fuzzer $(FUZZERTEST) $(FUZZER_FLAGS)
test-fuzzer32: fuzzer32
$(QEMU_SYS) ./fuzzer32 $(FUZZERTEST)
$(QEMU_SYS) ./fuzzer32 $(FUZZERTEST) $(FUZZER_FLAGS)
test-zbuff: zbufftest
$(QEMU_SYS) ./zbufftest $(ZSTREAM_TESTTIME)
@@ -312,10 +318,10 @@ test-zbuff32: zbufftest32
$(QEMU_SYS) ./zbufftest32 $(ZSTREAM_TESTTIME)
test-zstream: zstreamtest
$(QEMU_SYS) ./zstreamtest $(ZSTREAM_TESTTIME)
$(QEMU_SYS) ./zstreamtest $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
test-zstream32: zstreamtest32
$(QEMU_SYS) ./zstreamtest32 $(ZSTREAM_TESTTIME)
$(QEMU_SYS) ./zstreamtest32 $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
test-longmatch: longmatch
$(QEMU_SYS) ./longmatch
tests/decodecorpus.c
+30 -30
View File
@@ -98,7 +98,7 @@ static void RAND_bufferMaxSymb(U32* seed, void* ptr, size_t size, int maxSymb)
BYTE* op = ptr;
for (i = 0; i < size; i++) {
op[i] = RAND(seed) % (maxSymb + 1);
op[i] = (BYTE) (RAND(seed) % (maxSymb + 1));
}
}
@@ -134,8 +134,8 @@ static void RAND_genDist(U32* seed, BYTE* dist, double weight)
{
size_t i = 0;
size_t statesLeft = DISTSIZE;
BYTE symb = RAND(seed) % 256;
BYTE step = (RAND(seed) % 256) | 1; /* force it to be odd so it's relatively prime to 256 */
BYTE symb = (BYTE) (RAND(seed) % 256);
BYTE step = (BYTE) ((RAND(seed) % 256) | 1); /* force it to be odd so it's relatively prime to 256 */
while (i < DISTSIZE) {
size_t states = ((size_t)(weight * statesLeft)) + 1;
@@ -259,7 +259,7 @@ static void writeFrameHeader(U32* seed, frame_t* frame)
/* Follow window algorithm from specification */
int const exponent = RAND(seed) % (MAX_WINDOW_LOG - 10);
int const mantissa = RAND(seed) % 8;
windowByte = (exponent << 3) | mantissa;
windowByte = (BYTE) ((exponent << 3) | mantissa);
fh.windowSize = (1U << (exponent + 10));
fh.windowSize += fh.windowSize / 8 * mantissa;
}
@@ -284,7 +284,7 @@ static void writeFrameHeader(U32* seed, frame_t* frame)
if (contentSizeFlag && (fh.contentSize == 0 || !(RAND(seed) & 7))) {
/* do single segment sometimes */
fh.windowSize = fh.contentSize;
fh.windowSize = (U32) fh.contentSize;
singleSegment = 1;
}
}
@@ -307,7 +307,7 @@ static void writeFrameHeader(U32* seed, frame_t* frame)
{
BYTE const frameHeaderDescriptor =
(fcsCode << 6) | (singleSegment << 5) | (1 << 2);
(BYTE) ((fcsCode << 6) | (singleSegment << 5) | (1 << 2));
op[pos++] = frameHeaderDescriptor;
}
@@ -318,14 +318,14 @@ static void writeFrameHeader(U32* seed, frame_t* frame)
if (contentSizeFlag) {
switch (fcsCode) {
default: /* Impossible */
case 0: op[pos++] = fh.contentSize; break;
case 1: MEM_writeLE16(op + pos, fh.contentSize - 256); pos += 2; break;
case 2: MEM_writeLE32(op + pos, fh.contentSize); pos += 4; break;
case 3: MEM_writeLE64(op + pos, fh.contentSize); pos += 8; break;
case 0: op[pos++] = (BYTE) fh.contentSize; break;
case 1: MEM_writeLE16(op + pos, (U16) (fh.contentSize - 256)); pos += 2; break;
case 2: MEM_writeLE32(op + pos, (U32) fh.contentSize); pos += 4; break;
case 3: MEM_writeLE64(op + pos, (U64) fh.contentSize); pos += 8; break;
}
}
DISPLAYLEVEL(2, " frame content size:\t%zu\n", fh.contentSize);
DISPLAYLEVEL(2, " frame content size:\t%u\n", (U32)fh.contentSize);
DISPLAYLEVEL(2, " frame window size:\t%u\n", fh.windowSize);
DISPLAYLEVEL(2, " content size flag:\t%d\n", contentSizeFlag);
DISPLAYLEVEL(2, " single segment flag:\t%d\n", singleSegment);
@@ -385,7 +385,7 @@ static size_t writeLiteralsBlockSimple(U32* seed, frame_t* frame, size_t content
op += litSize;
} else {
/* RLE literals */
BYTE const symb = RAND(seed) % 256;
BYTE const symb = (BYTE) (RAND(seed) % 256);
DISPLAYLEVEL(4, " rle literals: 0x%02x\n", (U32)symb);
@@ -542,8 +542,8 @@ static size_t writeLiteralsBlockCompressed(U32* seed, frame_t* frame, size_t con
op += compressedSize;
compressedSize += hufHeaderSize;
DISPLAYLEVEL(5, " regenerated size: %zu\n", litSize);
DISPLAYLEVEL(5, " compressed size: %zu\n", compressedSize);
DISPLAYLEVEL(5, " regenerated size: %u\n", (U32)litSize);
DISPLAYLEVEL(5, " compressed size: %u\n", (U32)compressedSize);
if (compressedSize >= litSize) {
DISPLAYLEVEL(5, " trying again\n");
/* if we have to try again, reset the stats so we don't accidentally
@@ -611,7 +611,7 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore,
size_t const remainingMatch = contentSize - literalsSize;
size_t excessMatch = 0;
U32 numSequences = 0;
U32 i;
@@ -628,7 +628,7 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore,
excessMatch = remainingMatch - numSequences * MIN_SEQ_LEN;
}
DISPLAYLEVEL(5, " total match lengths: %zu\n", remainingMatch);
DISPLAYLEVEL(5, " total match lengths: %u\n", (U32)remainingMatch);
for (i = 0; i < numSequences; i++) {
/* Generate match and literal lengths by exponential distribution to
@@ -647,10 +647,10 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore,
U32 offset, offsetCode, repIndex;
/* bounds checks */
matchLen = MIN(matchLen, excessMatch + MIN_SEQ_LEN);
literalLen = MIN(literalLen, literalsSize);
matchLen = (U32) MIN(matchLen, excessMatch + MIN_SEQ_LEN);
literalLen = MIN(literalLen, (U32) literalsSize);
if (i == 0 && srcPtr == frame->srcStart && literalLen == 0) literalLen = 1;
if (i + 1 == numSequences) matchLen = MIN_SEQ_LEN + excessMatch;
if (i + 1 == numSequences) matchLen = MIN_SEQ_LEN + (U32) excessMatch;
memcpy(srcPtr, literals, literalLen);
srcPtr += literalLen;
@@ -694,8 +694,8 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore,
}
DISPLAYLEVEL(6, " LL: %5u OF: %5u ML: %5u", literalLen, offset, matchLen);
DISPLAYLEVEL(7, " srcPos: %8tu seqNb: %3u",
(BYTE*)srcPtr - (BYTE*)frame->srcStart, i);
DISPLAYLEVEL(7, " srcPos: %8u seqNb: %3u",
(U32)((BYTE*)srcPtr - (BYTE*)frame->srcStart), i);
DISPLAYLEVEL(6, "\n");
if (offsetCode < 3) {
DISPLAYLEVEL(7, " repeat offset: %d\n", repIndex);
@@ -711,8 +711,8 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore,
memcpy(srcPtr, literals, literalsSize);
srcPtr += literalsSize;
DISPLAYLEVEL(6, " excess literals: %5zu", literalsSize);
DISPLAYLEVEL(7, " srcPos: %8tu", (BYTE*)srcPtr - (BYTE*)frame->srcStart);
DISPLAYLEVEL(6, " excess literals: %5u", (U32)literalsSize);
DISPLAYLEVEL(7, " srcPos: %8u", (U32)((BYTE*)srcPtr - (BYTE*)frame->srcStart));
DISPLAYLEVEL(6, "\n");
return numSequences;
@@ -957,11 +957,11 @@ static size_t writeCompressedBlock(U32* seed, frame_t* frame, size_t contentSize
literalsSize = writeLiteralsBlock(seed, frame, contentSize);
DISPLAYLEVEL(4, " literals size: %zu\n", literalsSize);
DISPLAYLEVEL(4, " literals size: %u\n", (U32)literalsSize);
nbSeq = writeSequencesBlock(seed, frame, contentSize, literalsSize);
DISPLAYLEVEL(4, " number of sequences: %zu\n", nbSeq);
DISPLAYLEVEL(4, " number of sequences: %u\n", (U32)nbSeq);
return (BYTE*)frame->data - blockStart;
}
@@ -977,7 +977,7 @@ static void writeBlock(U32* seed, frame_t* frame, size_t contentSize,
BYTE *op = header + 3;
DISPLAYLEVEL(3, " block:\n");
DISPLAYLEVEL(3, " block content size: %zu\n", contentSize);
DISPLAYLEVEL(3, " block content size: %u\n", (U32)contentSize);
DISPLAYLEVEL(3, " last block: %s\n", lastBlock ? "yes" : "no");
if (blockTypeDesc == 0) {
@@ -1025,10 +1025,10 @@ static void writeBlock(U32* seed, frame_t* frame, size_t contentSize,
frame->src = (BYTE*)frame->src + contentSize;
DISPLAYLEVEL(3, " block type: %s\n", BLOCK_TYPES[blockType]);
DISPLAYLEVEL(3, " block size field: %zu\n", blockSize);
DISPLAYLEVEL(3, " block size field: %u\n", (U32)blockSize);
header[0] = (lastBlock | (blockType << 1) | (blockSize << 3)) & 0xff;
MEM_writeLE16(header + 1, blockSize >> 5);
header[0] = (BYTE) ((lastBlock | (blockType << 1) | (blockSize << 3)) & 0xff);
MEM_writeLE16(header + 1, (U16) (blockSize >> 5));
frame->data = op;
}
@@ -1300,7 +1300,7 @@ static int generateCorpus(U32 seed, unsigned numFiles, const char* const path,
*********************************************************/
static U32 makeSeed(void)
{
U32 t = time(NULL);
U32 t = (U32) time(NULL);
return XXH32(&t, sizeof(t), 0) % 65536;
}
tests/fullbench.c
+6 -6
View File
@@ -251,14 +251,14 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
case 13:
benchFunction = local_ZSTD_decompressContinue; benchName = "ZSTD_decompressContinue";
break;
case 31:
case 31:
benchFunction = local_ZSTD_decodeLiteralsBlock; benchName = "ZSTD_decodeLiteralsBlock";
break;
case 32:
benchFunction = local_ZSTD_decodeSeqHeaders; benchName = "ZSTD_decodeSeqHeaders";
break;
#endif
case 41:
case 41:
benchFunction = local_ZSTD_compressStream; benchName = "ZSTD_compressStream";
break;
case 42:
@@ -297,10 +297,10 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
case 31: /* ZSTD_decodeLiteralsBlock */
if (g_zdc==NULL) g_zdc = ZSTD_createDCtx();
{ blockProperties_t bp;
ZSTD_frameParams zfp;
ZSTD_frameHeader zfp;
size_t frameHeaderSize, skippedSize;
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1);
frameHeaderSize = ZSTD_getFrameParams(&zfp, dstBuff, ZSTD_frameHeaderSize_min);
frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_frameHeaderSize_min);
if (frameHeaderSize==0) frameHeaderSize = ZSTD_frameHeaderSize_min;
ZSTD_getcBlockSize(dstBuff+frameHeaderSize, dstBuffSize, &bp); /* Get 1st block type */
if (bp.blockType != bt_compressed) {
@@ -315,13 +315,13 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
case 32: /* ZSTD_decodeSeqHeaders */
if (g_zdc==NULL) g_zdc = ZSTD_createDCtx();
{ blockProperties_t bp;
ZSTD_frameParams zfp;
ZSTD_frameHeader zfp;
const BYTE* ip = dstBuff;
const BYTE* iend;
size_t frameHeaderSize, cBlockSize;
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1); /* it would be better to use direct block compression here */
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1);
frameHeaderSize = ZSTD_getFrameParams(&zfp, dstBuff, ZSTD_frameHeaderSize_min);
frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_frameHeaderSize_min);
if (frameHeaderSize==0) frameHeaderSize = ZSTD_frameHeaderSize_min;
ip += frameHeaderSize; /* Skip frame Header */
cBlockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); /* Get 1st block type */
tests/fuzzer.c
+215 -67
View File
@@ -28,6 +28,7 @@
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */
#include "zstd.h" /* ZSTD_VERSION_STRING */
#include "zstd_errors.h" /* ZSTD_getErrorCode */
#include "zstdmt_compress.h"
#define ZDICT_STATIC_LINKING_ONLY
#include "zdict.h" /* ZDICT_trainFromBuffer */
#include "datagen.h" /* RDG_genBuffer */
@@ -57,7 +58,7 @@ static U32 g_displayLevel = 2;
#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
if ((FUZ_clockSpan(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
{ g_displayClock = clock(); DISPLAY(__VA_ARGS__); \
if (g_displayLevel>=4) fflush(stdout); } }
if (g_displayLevel>=4) fflush(stderr); } }
static const clock_t g_refreshRate = CLOCKS_PER_SEC / 6;
static clock_t g_displayClock = 0;
@@ -66,6 +67,7 @@ static clock_t g_displayClock = 0;
* Fuzzer functions
*********************************************************/
#define MIN(a,b) ((a)<(b)?(a):(b))
#define MAX(a,b) ((a)>(b)?(a):(b))
static clock_t FUZ_clockSpan(clock_t cStart)
{
@@ -132,13 +134,21 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(4, "OK : %s \n", errorString);
}
DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, (U32)CNBuffSize);
CHECKPLUS(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(CNBuffSize),
CNBuffer, CNBuffSize, 1),
cSize=r );
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
DISPLAYLEVEL(4, "test%3i : decompressed size test : ", testNb++);
DISPLAYLEVEL(4, "test%3i : ZSTD_getFrameContentSize test : ", testNb++);
{ unsigned long long const rSize = ZSTD_getFrameContentSize(compressedBuffer, cSize);
if (rSize != CNBuffSize) goto _output_error;
}
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : ZSTD_findDecompressedSize test : ", testNb++);
{ unsigned long long const rSize = ZSTD_findDecompressedSize(compressedBuffer, cSize);
if (rSize != CNBuffSize) goto _output_error;
}
@@ -156,6 +166,7 @@ static int basicUnitTests(U32 seed, double compressibility)
} }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : decompress with null dict : ", testNb++);
{ size_t const r = ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, NULL, 0);
if (r != CNBuffSize) goto _output_error; }
@@ -178,6 +189,49 @@ static int basicUnitTests(U32 seed, double compressibility)
if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
/* ZSTDMT simple MT compression test */
DISPLAYLEVEL(4, "test%3i : create ZSTDMT CCtx : ", testNb++);
{ ZSTDMT_CCtx* mtctx = ZSTDMT_createCCtx(2);
if (mtctx==NULL) {
DISPLAY("mtctx : mot enough memory, aborting \n");
testResult = 1;
goto _end;
}
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : compress %u bytes with 2 threads : ", testNb++, (U32)CNBuffSize);
CHECKPLUS(r, ZSTDMT_compressCCtx(mtctx,
compressedBuffer, ZSTD_compressBound(CNBuffSize),
CNBuffer, CNBuffSize,
1),
cSize=r );
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
DISPLAYLEVEL(4, "test%3i : decompressed size test : ", testNb++);
{ unsigned long long const rSize = ZSTD_getFrameContentSize(compressedBuffer, cSize);
if (rSize != CNBuffSize) {
DISPLAY("ZSTD_getFrameContentSize incorrect : %u != %u \n", (U32)rSize, (U32)CNBuffSize);
goto _output_error;
} }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, (U32)CNBuffSize);
{ size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
if (r != CNBuffSize) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
{ size_t u;
for (u=0; u<CNBuffSize; u++) {
if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u]) goto _output_error;;
} }
DISPLAYLEVEL(4, "OK \n");
ZSTDMT_freeCCtx(mtctx);
}
/* Simple API multiframe test */
DISPLAYLEVEL(4, "test%3i : compress multiple frames : ", testNb++);
{ size_t off = 0;
@@ -285,8 +339,8 @@ static int basicUnitTests(U32 seed, double compressibility)
CHECKPLUS(r, ZSTD_compressEnd(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize),
(const char*)CNBuffer + dictSize, testSize),
cSize = r);
{ ZSTD_frameParams fp;
if (ZSTD_getFrameParams(&fp, compressedBuffer, cSize)) goto _output_error;
{ ZSTD_frameHeader fp;
if (ZSTD_getFrameHeader(&fp, compressedBuffer, cSize)) goto _output_error;
if ((fp.frameContentSize != testSize) && (fp.frameContentSize != 0)) goto _output_error;
} }
DISPLAYLEVEL(4, "OK \n");
@@ -350,7 +404,64 @@ static int basicUnitTests(U32 seed, double compressibility)
if (r != CNBuffSize) goto _output_error);
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : compress without dictID : ", testNb++);
DISPLAYLEVEL(4, "test%3i : estimate CDict size : ", testNb++);
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
size_t const estimatedSize = ZSTD_estimateCDictSize(cParams, dictSize);
DISPLAYLEVEL(4, "OK : %u \n", (U32)estimatedSize);
}
DISPLAYLEVEL(4, "test%3i : compress with preprocessed dictionary : ", testNb++);
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
ZSTD_customMem customMem = { NULL, NULL, NULL };
ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, 1, cParams, customMem);
cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize),
CNBuffer, CNBuffSize, cdict);
ZSTD_freeCDict(cdict);
if (ZSTD_isError(cSize)) goto _output_error;
}
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
DISPLAYLEVEL(4, "test%3i : retrieve dictID from frame : ", testNb++);
{ U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */
}
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
decodedBuffer, CNBuffSize,
compressedBuffer, cSize,
dictBuffer, dictSize),
if (r != CNBuffSize) goto _output_error);
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : ZSTD_compress_usingCDict_advanced, no contentSize, no dictID : ", testNb++);
{ ZSTD_frameParameters const fParams = { 0 /* frameSize */, 1 /* checksum */, 1 /* noDictID*/ };
ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
ZSTD_customMem const customMem = { NULL, NULL, NULL };
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, 1, cParams, customMem);
cSize = ZSTD_compress_usingCDict_advanced(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize),
CNBuffer, CNBuffSize, cdict, fParams);
ZSTD_freeCDict(cdict);
if (ZSTD_isError(cSize)) goto _output_error;
}
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
DISPLAYLEVEL(4, "test%3i : try retrieving contentSize from frame : ", testNb++);
{ U64 const contentSize = ZSTD_getFrameContentSize(compressedBuffer, cSize);
if (contentSize != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error;
}
DISPLAYLEVEL(4, "OK (unknown)\n");
DISPLAYLEVEL(4, "test%3i : frame built without dictID should be decompressible : ", testNb++);
CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
decodedBuffer, CNBuffSize,
compressedBuffer, cSize,
dictBuffer, dictSize),
if (r != CNBuffSize) goto _output_error);
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : ZSTD_compress_advanced, no dictID : ", testNb++);
{ ZSTD_parameters p = ZSTD_getParams(3, CNBuffSize, dictSize);
p.fParams.noDictIDFlag = 1;
cSize = ZSTD_compress_advanced(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize),
@@ -437,9 +548,9 @@ static int basicUnitTests(U32 seed, double compressibility)
/* Decompression defense tests */
DISPLAYLEVEL(4, "test%3i : Check input length for magic number : ", testNb++);
{ size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 3);
{ size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 3); /* too small input */
if (!ZSTD_isError(r)) goto _output_error;
if (r != (size_t)-ZSTD_error_srcSize_wrong) goto _output_error; }
if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : Check magic Number : ", testNb++);
@@ -448,6 +559,24 @@ static int basicUnitTests(U32 seed, double compressibility)
if (!ZSTD_isError(r)) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
/* content size verification test */
DISPLAYLEVEL(4, "test%3i : Content size verification : ", testNb++);
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
size_t const srcSize = 5000;
size_t const wrongSrcSize = (srcSize + 1000);
ZSTD_parameters params = ZSTD_getParams(1, wrongSrcSize, 0);
params.fParams.contentSizeFlag = 1;
{ size_t const result = ZSTD_compressBegin_advanced(cctx, NULL, 0, params, wrongSrcSize);
if (ZSTD_isError(result)) goto _output_error;
}
{ size_t const result = ZSTD_compressEnd(cctx, decodedBuffer, CNBuffSize, CNBuffer, srcSize);
if (!ZSTD_isError(result)) goto _output_error;
if (ZSTD_getErrorCode(result) != ZSTD_error_srcSize_wrong) goto _output_error;
DISPLAYLEVEL(4, "OK : %s \n", ZSTD_getErrorName(result));
}
ZSTD_freeCCtx(cctx);
}
/* block API tests */
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
static const size_t dictSize = 65 KB;
@@ -599,6 +728,14 @@ static size_t findDiff(const void* buf1, const void* buf2, size_t max)
}
static ZSTD_parameters FUZ_makeParams(ZSTD_compressionParameters cParams, ZSTD_frameParameters fParams)
{
ZSTD_parameters params;
params.cParams = cParams;
params.fParams = fParams;
return params;
}
static size_t FUZ_rLogLength(U32* seed, U32 logLength)
{
size_t const lengthMask = ((size_t)1 << logLength) - 1;
@@ -615,7 +752,7 @@ static size_t FUZ_randomLength(U32* seed, U32 maxLog)
#define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; }
static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxDurationS, double compressibility)
static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxDurationS, double compressibility, int bigTests)
{
static const U32 maxSrcLog = 23;
static const U32 maxSampleLog = 22;
@@ -635,6 +772,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
U32 coreSeed = seed, lseed = 0;
clock_t const startClock = clock();
clock_t const maxClockSpan = maxDurationS * CLOCKS_PER_SEC;
int const cLevelLimiter = bigTests ? 3 : 2;
/* allocation */
cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
@@ -661,7 +799,6 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
for ( ; (testNb <= nbTests) || (FUZ_clockSpan(startClock) < maxClockSpan); testNb++ ) {
size_t sampleSize, maxTestSize, totalTestSize;
size_t cSize, totalCSize, totalGenSize;
XXH64_state_t xxhState;
U64 crcOrig;
BYTE* sampleBuffer;
const BYTE* dict;
@@ -700,7 +837,10 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
crcOrig = XXH64(sampleBuffer, sampleSize, 0);
/* compression tests */
{ unsigned const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (FUZ_highbit32((U32)sampleSize)/3))) + 1;
{ unsigned const cLevel =
( FUZ_rand(&lseed) %
(ZSTD_maxCLevel() - (FUZ_highbit32((U32)sampleSize) / cLevelLimiter)) )
+ 1;
cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel);
CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed : %s", ZSTD_getErrorName(cSize));
@@ -722,8 +862,8 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
}
/* frame header decompression test */
{ ZSTD_frameParams dParams;
size_t const check = ZSTD_getFrameParams(&dParams, cBuffer, cSize);
{ ZSTD_frameHeader dParams;
size_t const check = ZSTD_getFrameHeader(&dParams, cBuffer, cSize);
CHECK(ZSTD_isError(check), "Frame Parameters extraction failed");
CHECK(dParams.frameContentSize != sampleSize, "Frame content size incorrect");
}
@@ -799,11 +939,15 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
/*===== Streaming compression test, scattered segments and dictionary =====*/
{ U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
int const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (testLog/3))) + 1;
U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
int const cLevel = (FUZ_rand(&lseed) %
(ZSTD_maxCLevel() -
(MAX(testLog, dictLog) / cLevelLimiter))) +
1;
maxTestSize = FUZ_rLogLength(&lseed, testLog);
if (maxTestSize >= dstBufferSize) maxTestSize = dstBufferSize-1;
dictSize = FUZ_randomLength(&lseed, maxSampleLog); /* needed also for decompression */
dictSize = FUZ_rLogLength(&lseed, dictLog); /* needed also for decompression */
dict = srcBuffer + (FUZ_rand(&lseed) % (srcBufferSize - dictSize));
if (FUZ_rand(&lseed) & 0xF) {
@@ -811,22 +955,22 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_usingDict error : %s", ZSTD_getErrorName(errorCode));
} else {
ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, 0, dictSize);
ZSTD_frameParameters const fpar = { FUZ_rand(&lseed)&1 /* contentSizeFlag */,
ZSTD_frameParameters const fPar = { FUZ_rand(&lseed)&1 /* contentSizeFlag */,
!(FUZ_rand(&lseed)&3) /* contentChecksumFlag*/,
0 /*NodictID*/ }; /* note : since dictionary is fake, dictIDflag has no impact */
ZSTD_parameters p;
size_t errorCode;
p.cParams = cPar; p.fParams = fpar;
errorCode = ZSTD_compressBegin_advanced(refCtx, dict, dictSize, p, 0);
ZSTD_parameters const p = FUZ_makeParams(cPar, fPar);
size_t const errorCode = ZSTD_compressBegin_advanced(refCtx, dict, dictSize, p, 0);
CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_advanced error : %s", ZSTD_getErrorName(errorCode));
}
{ size_t const errorCode = ZSTD_copyCCtx(ctx, refCtx, 0);
CHECK (ZSTD_isError(errorCode), "ZSTD_copyCCtx error : %s", ZSTD_getErrorName(errorCode));
} }
XXH64_reset(&xxhState, 0);
ZSTD_setCCtxParameter(ctx, ZSTD_p_forceWindow, FUZ_rand(&lseed) & 1);
{ U32 const nbChunks = (FUZ_rand(&lseed) & 127) + 2;
U32 n;
XXH64_state_t xxhState;
XXH64_reset(&xxhState, 0);
for (totalTestSize=0, cSize=0, n=0 ; n<nbChunks ; n++) {
size_t const segmentSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const segmentStart = FUZ_rand(&lseed) % (srcBufferSize - segmentSize);
@@ -841,13 +985,14 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
XXH64_update(&xxhState, srcBuffer+segmentStart, segmentSize);
memcpy(mirrorBuffer + totalTestSize, srcBuffer+segmentStart, segmentSize);
totalTestSize += segmentSize;
} }
}
{ size_t const flushResult = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize, NULL, 0);
CHECK (ZSTD_isError(flushResult), "multi-segments epilogue error : %s", ZSTD_getErrorName(flushResult));
cSize += flushResult;
{ size_t const flushResult = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize, NULL, 0);
CHECK (ZSTD_isError(flushResult), "multi-segments epilogue error : %s", ZSTD_getErrorName(flushResult));
cSize += flushResult;
}
crcOrig = XXH64_digest(&xxhState);
}
crcOrig = XXH64_digest(&xxhState);
/* streaming decompression test */
if (dictSize<8) dictSize=0, dict=NULL; /* disable dictionary */
@@ -897,7 +1042,7 @@ _output_error:
/*_*******************************************************
* Command line
*********************************************************/
int FUZ_usage(const char* programName)
static int FUZ_usage(const char* programName)
{
DISPLAY( "Usage :\n");
DISPLAY( " %s [args]\n", programName);
@@ -913,19 +1058,39 @@ int FUZ_usage(const char* programName)
return 0;
}
/*! readU32FromChar() :
@return : unsigned integer value read from input in `char` format
allows and interprets K, KB, KiB, M, MB and MiB suffix.
Will also modify `*stringPtr`, advancing it to position where it stopped reading.
Note : function result can overflow if digit string > MAX_UINT */
static unsigned readU32FromChar(const char** stringPtr)
{
unsigned result = 0;
while ((**stringPtr >='0') && (**stringPtr <='9'))
result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
if ((**stringPtr=='K') || (**stringPtr=='M')) {
result <<= 10;
if (**stringPtr=='M') result <<= 10;
(*stringPtr)++ ;
if (**stringPtr=='i') (*stringPtr)++;
if (**stringPtr=='B') (*stringPtr)++;
}
return result;
}
int main(int argc, const char** argv)
{
U32 seed=0;
int seedset=0;
U32 seed = 0;
int seedset = 0;
int argNb;
int nbTests = nbTestsDefault;
int testNb = 0;
U32 proba = FUZ_compressibility_default;
int result=0;
int result = 0;
U32 mainPause = 0;
U32 maxDuration = 0;
const char* programName = argv[0];
int bigTests = 1;
const char* const programName = argv[0];
/* Check command line */
for (argNb=1; argNb<argc; argNb++) {
@@ -934,81 +1099,64 @@ int main(int argc, const char** argv)
/* Handle commands. Aggregated commands are allowed */
if (argument[0]=='-') {
if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; }
argument++;
while (*argument!=0) {
switch(*argument)
{
case 'h':
return FUZ_usage(programName);
case 'v':
argument++;
g_displayLevel=4;
g_displayLevel = 4;
break;
case 'q':
argument++;
g_displayLevel--;
break;
case 'p': /* pause at the end */
argument++;
mainPause = 1;
break;
case 'i':
argument++; maxDuration=0;
nbTests=0;
while ((*argument>='0') && (*argument<='9')) {
nbTests *= 10;
nbTests += *argument - '0';
argument++;
}
argument++; maxDuration = 0;
nbTests = readU32FromChar(&argument);
break;
case 'T':
argument++;
nbTests=0; maxDuration=0;
while ((*argument>='0') && (*argument<='9')) {
maxDuration *= 10;
maxDuration += *argument - '0';
argument++;
}
if (*argument=='m') maxDuration *=60, argument++;
nbTests = 0;
maxDuration = readU32FromChar(&argument);
if (*argument=='s') argument++; /* seconds */
if (*argument=='m') maxDuration *= 60, argument++; /* minutes */
if (*argument=='n') argument++;
break;
case 's':
argument++;
seed=0;
seedset=1;
while ((*argument>='0') && (*argument<='9')) {
seed *= 10;
seed += *argument - '0';
argument++;
}
seedset = 1;
seed = readU32FromChar(&argument);
break;
case 't':
argument++;
testNb=0;
while ((*argument>='0') && (*argument<='9')) {
testNb *= 10;
testNb += *argument - '0';
argument++;
}
testNb = readU32FromChar(&argument);
break;
case 'P': /* compressibility % */
argument++;
proba=0;
while ((*argument>='0') && (*argument<='9')) {
proba *= 10;
proba += *argument - '0';
argument++;
}
if (proba>100) proba=100;
proba = readU32FromChar(&argument);
if (proba>100) proba = 100;
break;
default:
return FUZ_usage(programName);
return (FUZ_usage(programName), 1);
} } } } /* for (argNb=1; argNb<argc; argNb++) */
/* Get Seed */
@@ -1028,7 +1176,7 @@ int main(int argc, const char** argv)
if (testNb==0)
result = basicUnitTests(0, ((double)proba) / 100); /* constant seed for predictability */
if (!result)
result = fuzzerTests(seed, nbTests, testNb, maxDuration, ((double)proba) / 100);
result = fuzzerTests(seed, nbTests, testNb, maxDuration, ((double)proba) / 100, bigTests);
if (mainPause) {
int unused;
DISPLAY("Press Enter \n");
tests/paramgrill.c
+52 -30
View File
@@ -58,6 +58,11 @@ static const int g_maxNbVariations = 64;
**************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#undef MIN
#undef MAX
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
#define MAX(a,b) ( (a) > (b) ? (a) : (b) )
/*-************************************
* Benchmark Parameters
@@ -106,7 +111,11 @@ static size_t BMK_findMaxMem(U64 requiredMem)
}
# define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
static U32 FUZ_rotl32(U32 x, U32 r)
{
return ((x << r) | (x >> (32 - r)));
}
U32 FUZ_rand(U32* src)
{
const U32 prime1 = 2654435761U;
@@ -125,7 +134,7 @@ U32 FUZ_rand(U32* src)
*********************************************************/
typedef struct {
size_t cSize;
double cSpeed;
double cSpeed; /* bytes / sec */
double dSpeed;
} BMK_result_t;
@@ -141,8 +150,6 @@ typedef struct
} blockParam_t;
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
static size_t BMK_benchParam(BMK_result_t* resultPtr,
const void* srcBuffer, size_t srcSize,
ZSTD_CCtx* ctx,
@@ -165,6 +172,11 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
char name[30] = { 0 };
U64 crcOrig;
/* init result for early exit */
resultPtr->cSize = srcSize;
resultPtr->cSpeed = 0.;
resultPtr->dSpeed = 0.;
/* Memory allocation & restrictions */
snprintf(name, 30, "Sw%02uc%02uh%02us%02ul%1ut%03uS%1u", Wlog, Clog, Hlog, Slog, Slength, Tlength, strat);
if (!compressedBuffer || !resultBuffer || !blockTable) {
@@ -206,7 +218,6 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
size_t cSize = 0;
double fastestC = 100000000., fastestD = 100000000.;
double ratio = 0.;
U64 crcCheck = 0;
clock_t const benchStart = clock();
DISPLAY("\r%79s\r", "");
@@ -242,8 +253,8 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
cSize = 0;
for (blockNb=0; blockNb<nbBlocks; blockNb++)
cSize += blockTable[blockNb].cSize;
if ((double)roundClock < fastestC * CLOCKS_PER_SEC * nbLoops) fastestC = ((double)roundClock / CLOCKS_PER_SEC) / nbLoops;
ratio = (double)srcSize / (double)cSize;
if ((double)roundClock < fastestC * CLOCKS_PER_SEC * nbLoops) fastestC = ((double)roundClock / CLOCKS_PER_SEC) / nbLoops;
DISPLAY("\r");
DISPLAY("%1u-%s : %9u ->", loopNb, name, (U32)srcSize);
DISPLAY(" %9u (%4.3f),%7.1f MB/s", (U32)cSize, ratio, (double)srcSize / fastestC / 1000000.);
@@ -273,18 +284,18 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
resultPtr->dSpeed = (double)srcSize / fastestD;
/* CRC Checking */
crcCheck = XXH64(resultBuffer, srcSize, 0);
if (crcOrig!=crcCheck) {
unsigned u;
unsigned eBlockSize = (unsigned)(MIN(65536*2, blockSize));
DISPLAY("\n!!! WARNING !!! Invalid Checksum : %x != %x\n", (unsigned)crcOrig, (unsigned)crcCheck);
for (u=0; u<srcSize; u++) {
if (((const BYTE*)srcBuffer)[u] != ((BYTE*)resultBuffer)[u]) {
printf("Decoding error at pos %u (block %u, pos %u) \n", u, u / eBlockSize, u % eBlockSize);
break;
} }
break;
}
{ U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
if (crcOrig!=crcCheck) {
unsigned u;
unsigned eBlockSize = (unsigned)(MIN(65536*2, blockSize));
DISPLAY("\n!!! WARNING !!! Invalid Checksum : %x != %x\n", (unsigned)crcOrig, (unsigned)crcCheck);
for (u=0; u<srcSize; u++) {
if (((const BYTE*)srcBuffer)[u] != ((BYTE*)resultBuffer)[u]) {
printf("Decoding error at pos %u (block %u, pos %u) \n", u, u / eBlockSize, u % eBlockSize);
break;
} }
break;
} }
#endif
} }
@@ -505,8 +516,6 @@ static BYTE g_alreadyTested[PARAMTABLESIZE] = {0}; /* init to zero */
g_alreadyTested[(XXH64(sanitizeParams(p), sizeof(p), 0) >> 3) & PARAMTABLEMASK]
#define MAX(a,b) ( (a) > (b) ? (a) : (b) )
static void playAround(FILE* f, winnerInfo_t* winners,
ZSTD_compressionParameters params,
const void* srcBuffer, size_t srcSize,
@@ -711,6 +720,14 @@ int benchFiles(const char** fileNamesTable, int nbFiles)
}
static void BMK_translateAdvancedParams(ZSTD_compressionParameters params)
{
DISPLAY("--zstd=windowLog=%u,chainLog=%u,hashLog=%u,searchLog=%u,searchLength=%u,targetLength=%u,strategy=%u \n",
params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, params.targetLength, (U32)(params.strategy));
}
/* optimizeForSize():
* targetSpeed : expressed in MB/s */
int optimizeForSize(const char* inFileName, U32 targetSpeed)
{
FILE* const inFile = fopen( inFileName, "rb" );
@@ -723,8 +740,11 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed)
/* Memory allocation & restrictions */
if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
if (benchedSize < inFileSize)
DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20));
if (benchedSize < inFileSize) {
DISPLAY("Not enough memory for '%s' \n", inFileName);
fclose(inFile);
return 11;
}
/* Alloc */
origBuff = malloc(benchedSize);
@@ -747,10 +767,9 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed)
/* bench */
DISPLAY("\r%79s\r", "");
DISPLAY("optimizing for %s - limit speed %u MB/s \n", inFileName, targetSpeed);
targetSpeed *= 1000;
targetSpeed *= 1000000;
{ ZSTD_CCtx* const ctx = ZSTD_createCCtx();
ZSTD_compressionParameters params;
winnerInfo_t winner;
BMK_result_t candidate;
const size_t blockSize = g_blockSize ? g_blockSize : benchedSize;
@@ -764,26 +783,28 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed)
{ const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel();
int i;
for (i=1; i<=maxSeeds; i++) {
params = ZSTD_getCParams(i, blockSize, 0);
BMK_benchParam(&candidate, origBuff, benchedSize, ctx, params);
ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, 0);
BMK_benchParam(&candidate, origBuff, benchedSize, ctx, CParams);
if (candidate.cSpeed < targetSpeed)
break;
if ( (candidate.cSize < winner.result.cSize)
| ((candidate.cSize == winner.result.cSize) & (candidate.cSpeed > winner.result.cSpeed)) )
{
winner.params = params;
winner.params = CParams;
winner.result = candidate;
BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize);
} }
}
BMK_printWinner(stdout, 99, winner.result, winner.params, benchedSize);
BMK_translateAdvancedParams(winner.params);
/* start tests */
{ time_t const grillStart = time(NULL);
do {
params = winner.params;
ZSTD_compressionParameters params = winner.params;
paramVariation(&params);
if ((FUZ_rand(&g_rand) & 15) == 3) params = randomParams();
if ((FUZ_rand(&g_rand) & 31) == 3) params = randomParams(); /* totally random config to improve search space */
params = ZSTD_adjustCParams(params, blockSize, 0);
/* exclude faster if already played set of params */
if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(params))-1)) continue;
@@ -800,6 +821,7 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed)
winner.params = params;
winner.result = candidate;
BMK_printWinner(stdout, 99, winner.result, winner.params, benchedSize);
BMK_translateAdvancedParams(winner.params);
}
} while (BMK_timeSpan(grillStart) < g_grillDuration_s);
}
@@ -833,7 +855,7 @@ static int usage_advanced(void)
DISPLAY( " -T# : set level 1 speed objective \n");
DISPLAY( " -B# : cut input into blocks of size # (default : single block) \n");
DISPLAY( " -i# : iteration loops [1-9](default : %i) \n", NBLOOPS);
DISPLAY( " -O# : find Optimized parameters for # target speed (default : 0) \n");
DISPLAY( " -O# : find Optimized parameters for # MB/s compression speed (default : 0) \n");
DISPLAY( " -S : Single run \n");
DISPLAY( " -P# : generated sample compressibility (default : %.1f%%) \n", COMPRESSIBILITY_DEFAULT * 100);
return 0;
tests/playTests.sh
+224 -21
View File
@@ -11,6 +11,7 @@ roundTripTest() {
local_p="$2"
else
local_c="$2"
local_p=""
fi
rm -f tmp1 tmp2
@@ -20,13 +21,36 @@ roundTripTest() {
$DIFF -q tmp1 tmp2
}
fileRoundTripTest() {
if [ -n "$3" ]; then
local_c="$3"
local_p="$2"
else
local_c="$2"
local_p=""
fi
rm -f tmp.zstd tmp.md5.1 tmp.md5.2
$ECHO "fileRoundTripTest: ./datagen $1 $local_p > tmp && $ZSTD -v$local_c -c tmp | $ZSTD -d"
./datagen $1 $local_p > tmp
cat tmp | $MD5SUM > tmp.md5.1
$ZSTD --ultra -v$local_c -c tmp | $ZSTD -d | $MD5SUM > tmp.md5.2
$DIFF -q tmp.md5.1 tmp.md5.2
}
isTerminal=false
if [ -t 0 ] && [ -t 1 ]
then
isTerminal=true
fi
isWindows=false
ECHO="echo"
ECHO="echo -e"
INTOVOID="/dev/null"
case "$OS" in
Windows*)
isWindows=true
ECHO="echo -e"
INTOVOID="NUL"
;;
esac
@@ -42,11 +66,17 @@ case "$UNAME" in
SunOS) DIFF="gdiff" ;;
esac
$ECHO "\nStarting playTests.sh isWindows=$isWindows ZSTD='$ZSTD'"
[ -n "$ZSTD" ] || die "ZSTD variable must be defined!"
if [ -n "$(echo hello | $ZSTD -v -T2 2>&1 > $INTOVOID | grep 'multi-threading is disabled')" ]
then
hasMT=""
else
hasMT="true"
fi
$ECHO "\n**** simple tests **** "
./datagen > tmp
@@ -62,7 +92,7 @@ $ZSTD tmp --stdout > tmpCompressed # long command format
$ECHO "test : compress to named file"
rm tmpCompressed
$ZSTD tmp -o tmpCompressed
ls tmpCompressed # must work
test -f tmpCompressed # file must be created
$ECHO "test : -o must be followed by filename (must fail)"
$ZSTD tmp -of tmpCompressed && die "-o must be followed by filename "
$ECHO "test : force write, correct order"
@@ -72,6 +102,12 @@ cp tmp tmp2
$ZSTD tmp2 -fo && die "-o must be followed by filename "
$ECHO "test : implied stdout when input is stdin"
$ECHO bob | $ZSTD | $ZSTD -d
if [ "$isTerminal" = true ]; then
$ECHO "test : compressed data to terminal"
$ECHO bob | $ZSTD && die "should have refused : compressed data to terminal"
$ECHO "test : compressed data from terminal (a hang here is a test fail, zstd is wrongly waiting on data from terminal)"
$ZSTD -d > $INTOVOID && die "should have refused : compressed data from terminal"
fi
$ECHO "test : null-length file roundtrip"
$ECHO -n '' | $ZSTD - --stdout | $ZSTD -d --stdout
$ECHO "test : decompress file with wrong suffix (must fail)"
@@ -96,23 +132,31 @@ $ZSTD -q tmp && die "overwrite check failed!"
$ECHO "test : force overwrite"
$ZSTD -q -f tmp
$ZSTD -q --force tmp
$ECHO "test : overwrite readonly file"
rm -f tmpro tmpro.zst
$ECHO foo > tmpro.zst
$ECHO foo > tmpro
chmod 400 tmpro.zst
$ZSTD -q tmpro && die "should have refused to overwrite read-only file"
$ZSTD -q -f tmpro
rm -f tmpro tmpro.zst
$ECHO "test : file removal"
$ZSTD -f --rm tmp
ls tmp && die "tmp should no longer be present"
test ! -f tmp # tmp should no longer be present
$ZSTD -f -d --rm tmp.zst
ls tmp.zst && die "tmp.zst should no longer be present"
test ! -f tmp.zst # tmp.zst should no longer be present
$ECHO "test : --rm on stdin"
$ECHO a | $ZSTD --rm > $INTOVOID # --rm should remain silent
rm tmp
$ZSTD -f tmp && die "tmp not present : should have failed"
ls tmp.zst && die "tmp.zst should not be created"
test ! -f tmp.zst # tmp.zst should not be created
$ECHO "\n**** Advanced compression parameters **** "
$ECHO "Hello world!" | $ZSTD --zstd=windowLog=21, - -o tmp.zst && die "wrong parameters not detected!"
$ECHO "Hello world!" | $ZSTD --zstd=windowLo=21 - -o tmp.zst && die "wrong parameters not detected!"
$ECHO "Hello world!" | $ZSTD --zstd=windowLog=21,slog - -o tmp.zst && die "wrong parameters not detected!"
ls tmp.zst && die "tmp.zst should not be created"
test ! -f tmp.zst # tmp.zst should not be created
roundTripTest -g512K
roundTripTest -g512K " --zstd=slen=3,tlen=48,strat=6"
roundTripTest -g512K " --zstd=strat=6,wlog=23,clog=23,hlog=22,slog=6"
@@ -156,6 +200,20 @@ $ECHO "$ECHO foo | $ZSTD > /dev/full"
$ECHO foo | $ZSTD > /dev/full && die "write error not detected!"
$ECHO "$ECHO foo | $ZSTD | $ZSTD -d > /dev/full"
$ECHO foo | $ZSTD | $ZSTD -d > /dev/full && die "write error not detected!"
$ECHO "\n**** symbolic link test **** "
rm -f hello.tmp world.tmp hello.tmp.zst world.tmp.zst
$ECHO "hello world" > hello.tmp
ln -s hello.tmp world.tmp
$ZSTD world.tmp hello.tmp
test -f hello.tmp.zst # regular file should have been compressed!
test ! -f world.tmp.zst # symbolic link should not have been compressed!
$ZSTD world.tmp hello.tmp -f
test -f world.tmp.zst # symbolic link should have been compressed with --force
rm -f hello.tmp world.tmp hello.tmp.zst world.tmp.zst
fi
@@ -168,10 +226,10 @@ $ZSTD tmpSparse -c | $ZSTD -dv --sparse -c > tmpOutSparse
$DIFF -s tmpSparse tmpOutSparse
$ZSTD tmpSparse -c | $ZSTD -dv --no-sparse -c > tmpOutNoSparse
$DIFF -s tmpSparse tmpOutNoSparse
ls -ls tmpSparse*
ls -ls tmpSparse* # look at file size and block size on disk
./datagen -s1 -g1200007 -P100 | $ZSTD | $ZSTD -dv --sparse -c > tmpSparseOdd # Odd size file (to not finish on an exact nb of blocks)
./datagen -s1 -g1200007 -P100 | $DIFF -s - tmpSparseOdd
ls -ls tmpSparseOdd
ls -ls tmpSparseOdd # look at file size and block size on disk
$ECHO "\n Sparse Compatibility with Console :"
$ECHO "Hello World 1 !" | $ZSTD | $ZSTD -d -c
$ECHO "Hello World 2 !" | $ZSTD | $ZSTD -d | cat
@@ -181,7 +239,7 @@ cat tmpSparse1M tmpSparse1M > tmpSparse2M
$ZSTD -v -f tmpSparse1M -o tmpSparseCompressed
$ZSTD -d -v -f tmpSparseCompressed -o tmpSparseRegenerated
$ZSTD -d -v -f tmpSparseCompressed -c >> tmpSparseRegenerated
ls -ls tmpSparse*
ls -ls tmpSparse* # look at file size and block size on disk
$DIFF tmpSparse2M tmpSparseRegenerated
rm tmpSparse*
@@ -200,11 +258,11 @@ $ZSTD -df *.zst
ls -ls tmp*
$ECHO "compress tmp* into stdout > tmpall : "
$ZSTD -c tmp1 tmp2 tmp3 > tmpall
ls -ls tmp*
ls -ls tmp* # check size of tmpall (should be tmp1.zst + tmp2.zst + tmp3.zst)
$ECHO "decompress tmpall* into stdout > tmpdec : "
cp tmpall tmpall2
$ZSTD -dc tmpall* > tmpdec
ls -ls tmp*
ls -ls tmp* # check size of tmpdec (should be 2*(tmp1 + tmp2 + tmp3))
$ECHO "compress multiple files including a missing one (notHere) : "
$ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!"
@@ -227,12 +285,12 @@ $ECHO "- Create second (different) dictionary "
$ZSTD --train *.c ../programs/*.c ../programs/*.h -o tmpDictC
$ZSTD -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!"
$ECHO "- Create dictionary with short dictID"
$ZSTD --train *.c ../programs/*.c --dictID 1 -o tmpDict1
$ZSTD --train *.c ../programs/*.c --dictID=1 -o tmpDict1
cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
$ECHO "- Create dictionary with wrong dictID parameter order (must fail)"
$ZSTD --train *.c ../programs/*.c --dictID -o 1 tmpDict1 && die "wrong order : --dictID must be followed by argument "
$ECHO "- Create dictionary with size limit"
$ZSTD --train *.c ../programs/*.c -o tmpDict2 --maxdict 4K -v
$ZSTD --train *.c ../programs/*.c -o tmpDict2 --maxdict=4K -v
$ECHO "- Create dictionary with wrong parameter order (must fail)"
$ZSTD --train *.c ../programs/*.c -o tmpDict2 --maxdict -v 4K && die "wrong order : --maxdict must be followed by argument "
$ECHO "- Compress without dictID"
@@ -240,7 +298,7 @@ $ZSTD -f tmp -D tmpDict1 --no-dictID
$ZSTD -d tmp.zst -D tmpDict -fo result
$DIFF $TESTFILE result
$ECHO "- Compress with wrong argument order (must fail)"
$ZSTD tmp -Df tmpDict1 -c > /dev/null && die "-D must be followed by dictionary name "
$ZSTD tmp -Df tmpDict1 -c > $INTOVOID && die "-D must be followed by dictionary name "
$ECHO "- Compress multiple files with dictionary"
rm -rf dirTestDict
mkdir dirTestDict
@@ -255,6 +313,11 @@ case "$UNAME" in
*) $MD5SUM -c tmph1 ;;
esac
rm -rf dirTestDict
$ECHO "- dictionary builder on bogus input"
$ECHO "Hello World" > tmp
$ZSTD --train-legacy -q tmp && die "Dictionary training should fail : not enough input source"
./datagen -P0 -g10M > tmp
$ZSTD --train-legacy -q tmp && die "Dictionary training should fail : source is pure noise"
rm tmp*
@@ -263,19 +326,39 @@ $ECHO "\n**** cover dictionary tests **** "
TESTFILE=../programs/zstdcli.c
./datagen > tmpDict
$ECHO "- Create first dictionary"
$ZSTD --train --cover=k=46,d=8 *.c ../programs/*.c -o tmpDict
$ZSTD --train-cover=k=46,d=8 *.c ../programs/*.c -o tmpDict
cp $TESTFILE tmp
$ZSTD -f tmp -D tmpDict
$ZSTD -d tmp.zst -D tmpDict -fo result
$DIFF $TESTFILE result
$ECHO "- Create second (different) dictionary"
$ZSTD --train --cover=k=56,d=8 *.c ../programs/*.c ../programs/*.h -o tmpDictC
$ZSTD --train-cover=k=56,d=8 *.c ../programs/*.c ../programs/*.h -o tmpDictC
$ZSTD -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!"
$ECHO "- Create dictionary with short dictID"
$ZSTD --train --cover=k=46,d=8 *.c ../programs/*.c --dictID 1 -o tmpDict1
$ZSTD --train-cover=k=46,d=8 *.c ../programs/*.c --dictID=1 -o tmpDict1
cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
$ECHO "- Create dictionary with size limit"
$ZSTD --train --optimize-cover=steps=8 *.c ../programs/*.c -o tmpDict2 --maxdict 4K
$ZSTD --train-cover=steps=8 *.c ../programs/*.c -o tmpDict2 --maxdict=4K
rm tmp*
$ECHO "\n**** legacy dictionary tests **** "
TESTFILE=../programs/zstdcli.c
./datagen > tmpDict
$ECHO "- Create first dictionary"
$ZSTD --train-legacy=selectivity=8 *.c ../programs/*.c -o tmpDict
cp $TESTFILE tmp
$ZSTD -f tmp -D tmpDict
$ZSTD -d tmp.zst -D tmpDict -fo result
$DIFF $TESTFILE result
$ECHO "- Create second (different) dictionary"
$ZSTD --train-legacy=s=5 *.c ../programs/*.c ../programs/*.h -o tmpDictC
$ZSTD -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!"
$ECHO "- Create dictionary with short dictID"
$ZSTD --train-legacy -s5 *.c ../programs/*.c --dictID=1 -o tmpDict1
cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
$ECHO "- Create dictionary with size limit"
$ZSTD --train-legacy -s9 *.c ../programs/*.c -o tmpDict2 --maxdict=4K
rm tmp*
@@ -297,7 +380,9 @@ $ZSTD -t tmp2.zst && die "bad file not detected !"
$ZSTD -t tmp3 && die "bad file not detected !" # detects 0-sized files as bad
$ECHO "test --rm and --test combined "
$ZSTD -t --rm tmp1.zst
ls -ls tmp1.zst # check file is still present
test -f tmp1.zst # check file is still present
split -b16384 tmp1.zst tmpSplit.
$ZSTD -t tmpSplit.* && die "bad file not detected !"
$ECHO "\n**** benchmark mode tests **** "
@@ -325,6 +410,7 @@ if [ $GZIPMODE -eq 1 ]; then
gzip -t -v tmp.gz
gzip -f tmp
$ZSTD -d -f -v tmp.gz
rm tmp*
else
$ECHO "gzip binary not detected"
fi
@@ -333,6 +419,98 @@ else
fi
$ECHO "\n**** gzip frame tests **** "
if [ $GZIPMODE -eq 1 ]; then
./datagen > tmp
$ZSTD -f --format=gzip tmp
$ZSTD -f tmp
cat tmp.gz tmp.zst tmp.gz tmp.zst | $ZSTD -d -f -o tmp
head -c -1 tmp.gz | $ZSTD -t > $INTOVOID && die "incomplete frame not detected !"
rm tmp*
else
$ECHO "gzip mode not supported"
fi
$ECHO "\n**** xz compatibility tests **** "
LZMAMODE=1
$ZSTD --format=xz -V || LZMAMODE=0
if [ $LZMAMODE -eq 1 ]; then
$ECHO "xz support detected"
XZEXE=1
xz -V && lzma -V || XZEXE=0
if [ $XZEXE -eq 1 ]; then
./datagen > tmp
$ZSTD --format=lzma -f tmp
$ZSTD --format=xz -f tmp
xz -t -v tmp.xz
xz -t -v tmp.lzma
xz -f -k tmp
lzma -f -k --lzma1 tmp
$ZSTD -d -f -v tmp.xz
$ZSTD -d -f -v tmp.lzma
rm tmp*
else
$ECHO "xz binary not detected"
fi
else
$ECHO "xz mode not supported"
fi
$ECHO "\n**** xz frame tests **** "
if [ $LZMAMODE -eq 1 ]; then
./datagen > tmp
$ZSTD -f --format=xz tmp
$ZSTD -f --format=lzma tmp
$ZSTD -f tmp
cat tmp.xz tmp.lzma tmp.zst tmp.lzma tmp.xz tmp.zst | $ZSTD -d -f -o tmp
head -c -1 tmp.xz | $ZSTD -t > $INTOVOID && die "incomplete frame not detected !"
head -c -1 tmp.lzma | $ZSTD -t > $INTOVOID && die "incomplete frame not detected !"
rm tmp*
else
$ECHO "xz mode not supported"
fi
$ECHO "\n**** lz4 compatibility tests **** "
LZ4MODE=1
$ZSTD --format=lz4 -V || LZ4MODE=0
if [ $LZ4MODE -eq 1 ]; then
$ECHO "lz4 support detected"
LZ4EXE=1
lz4 -V || LZ4EXE=0
if [ $LZ4EXE -eq 1 ]; then
./datagen > tmp
$ZSTD --format=lz4 -f tmp
lz4 -t -v tmp.lz4
lz4 -f tmp
$ZSTD -d -f -v tmp.lz4
rm tmp*
else
$ECHO "lz4 binary not detected"
fi
else
$ECHO "lz4 mode not supported"
fi
$ECHO "\n**** lz4 frame tests **** "
if [ $LZ4MODE -eq 1 ]; then
./datagen > tmp
$ZSTD -f --format=lz4 tmp
$ZSTD -f tmp
cat tmp.lz4 tmp.zst tmp.lz4 tmp.zst | $ZSTD -d -f -o tmp
head -c -1 tmp.lz4 | $ZSTD -t > $INTOVOID && die "incomplete frame not detected !"
rm tmp*
else
$ECHO "lz4 mode not supported"
fi
$ECHO "\n**** zstd round-trip tests **** "
roundTripTest
@@ -344,6 +522,19 @@ roundTripTest -g519K 6 # greedy, hash chain
roundTripTest -g517K 16 # btlazy2
roundTripTest -g516K 19 # btopt
fileRoundTripTest -g500K
if [ -n "$hasMT" ]
then
$ECHO "\n**** zstdmt round-trip tests **** "
roundTripTest -g4M "1 -T0"
roundTripTest -g8M "3 -T2"
roundTripTest -g8000K "2 --threads=2"
fileRoundTripTest -g4M "19 -T2 -B1M"
else
$ECHO "\n**** no multithreading, skipping zstdmt tests **** "
fi
rm tmp*
if [ "$1" != "--test-large-data" ]; then
@@ -379,4 +570,16 @@ roundTripTest -g50000000 -P94 19
roundTripTest -g99000000 -P99 20
roundTripTest -g6000000000 -P99 1
fileRoundTripTest -g4193M -P99 1
if [ -n "$hasMT" ]
then
$ECHO "\n**** zstdmt long round-trip tests **** "
roundTripTest -g99000000 -P99 "20 -T2"
roundTripTest -g6000000000 -P99 "1 -T2"
fileRoundTripTest -g4193M -P98 " -T0"
else
$ECHO "\n**** no multithreading, skipping zstdmt tests **** "
fi
rm tmp*
tests/symbols.c
+1 -1
View File
@@ -88,7 +88,7 @@ static const void *symbols[] = {
&ZSTD_copyCCtx,
&ZSTD_compressContinue,
&ZSTD_compressEnd,
&ZSTD_getFrameParams,
&ZSTD_getFrameHeader,
&ZSTD_decompressBegin,
&ZSTD_decompressBegin_usingDict,
&ZSTD_copyDCtx,
tests/test-zstd-speed.py
+13 -7
View File
@@ -14,14 +14,15 @@
# - dir1/zstd and dir2/zstd will be merged in a single results file
import argparse
import os
import os # getloadavg
import string
import subprocess
import time
import time # strftime
import traceback
import hashlib
import platform # system
script_version = 'v1.1.1 (2016-10-28)'
script_version = 'v1.1.2 (2017-03-26)'
default_repo_url = 'https://github.com/facebook/zstd.git'
working_dir_name = 'speedTest'
working_path = os.getcwd() + '/' + working_dir_name # /path/to/zstd/tests/speedTest
@@ -152,10 +153,15 @@ def benchmark_and_compare(branch, commit, last_commit, args, executableName, md5
% (os.getloadavg()[0], args.maxLoadAvg, sleepTime))
time.sleep(sleepTime)
start_load = str(os.getloadavg())
if args.dictionary:
result = execute('programs/%s -rqi5b1e%s -D %s %s' % (executableName, args.lastCLevel, args.dictionary, testFilePath), print_output=True)
osType = platform.system()
if osType == 'Linux':
cpuSelector = "taskset --cpu-list 0"
else:
result = execute('programs/%s -rqi5b1e%s %s' % (executableName, args.lastCLevel, testFilePath), print_output=True)
cpuSelector = ""
if args.dictionary:
result = execute('%s programs/%s -rqi5b1e%s -D %s %s' % (cpuSelector, executableName, args.lastCLevel, args.dictionary, testFilePath), print_output=True)
else:
result = execute('%s programs/%s -rqi5b1e%s %s' % (cpuSelector, executableName, args.lastCLevel, testFilePath), print_output=True)
end_load = str(os.getloadavg())
linesExpected = args.lastCLevel + 1
if len(result) != linesExpected:
@@ -291,7 +297,7 @@ if __name__ == '__main__':
log("ERROR: e-mail senders 'mail' or 'mutt' not found")
exit(1)
clang_version = execute("clang -v 2>&1 | grep 'clang version' | sed -e 's:.*version \\([0-9.]*\\).*:\\1:' -e 's:\\.\\([0-9][0-9]\\):\\1:g'", verbose)[0];
clang_version = execute("clang -v 2>&1 | grep ' version ' | sed -e 's:.*version \\([0-9.]*\\).*:\\1:' -e 's:\\.\\([0-9][0-9]\\):\\1:g'", verbose)[0];
gcc_version = execute("gcc -dumpversion", verbose)[0];
if verbose:
tests/zbufftest.c
+1 -1
View File
@@ -60,7 +60,7 @@ static U32 g_displayLevel = 2;
#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
if ((FUZ_GetClockSpan(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
{ g_displayClock = clock(); DISPLAY(__VA_ARGS__); \
if (g_displayLevel>=4) fflush(stdout); } }
if (g_displayLevel>=4) fflush(stderr); } }
static const clock_t g_refreshRate = CLOCKS_PER_SEC * 15 / 100;
static clock_t g_displayClock = 0;
tests/zstreamtest.c
+143 -28
View File
@@ -53,13 +53,15 @@ static const U32 prime32 = 2654435761U;
* Display Macros
**************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { \
DISPLAY(__VA_ARGS__); \
if (g_displayLevel>=4) fflush(stderr); }
static U32 g_displayLevel = 2;
#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
if ((FUZ_GetClockSpan(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
{ g_displayClock = clock(); DISPLAY(__VA_ARGS__); \
if (g_displayLevel>=4) fflush(stdout); } }
if (g_displayLevel>=4) fflush(stderr); } }
static const clock_t g_refreshRate = CLOCKS_PER_SEC / 6;
static clock_t g_displayClock = 0;
@@ -131,7 +133,7 @@ static buffer_t FUZ_createDictionary(const void* src, size_t srcSize, size_t blo
}
{ size_t const dictSize = ZDICT_trainFromBuffer(dict.start, requestedDictSize, src, blockSizes, (unsigned)nbBlocks);
free(blockSizes);
if (ZDICT_isError(dictSize)) { free(dict.start); return (buffer_t){ NULL, 0, 0 }; }
if (ZDICT_isError(dictSize)) { free(dict.start); return g_nullBuffer; }
dict.size = requestedDictSize;
dict.filled = dictSize;
return dict; /* how to return dictSize ? */
@@ -155,12 +157,13 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
void* decodedBuffer = malloc(decodedBufferSize);
size_t cSize;
int testResult = 0;
U32 testNb=0;
U32 testNb = 1;
ZSTD_CStream* zc = ZSTD_createCStream_advanced(customMem);
ZSTD_DStream* zd = ZSTD_createDStream_advanced(customMem);
ZSTD_inBuffer inBuff, inBuff2;
ZSTD_outBuffer outBuff;
buffer_t dictionary = g_nullBuffer;
size_t const dictSize = 128 KB;
unsigned dictID = 0;
/* Create compressible test buffer */
@@ -186,7 +189,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
/* Basic compression test */
DISPLAYLEVEL(3, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
ZSTD_initCStream_usingDict(zc, CNBuffer, 128 KB, 1);
ZSTD_initCStream_usingDict(zc, CNBuffer, dictSize, 1);
outBuff.dst = (char*)(compressedBuffer)+cSize;
outBuff.size = compressedBufferSize;
outBuff.pos = 0;
@@ -201,15 +204,34 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
cSize += outBuff.pos;
DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
DISPLAYLEVEL(3, "test%3i : check CStream size : ", testNb++);
/* context size functions */
DISPLAYLEVEL(3, "test%3i : estimate CStream size : ", testNb++);
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictSize);
size_t const s = ZSTD_estimateCStreamSize(cParams)
+ ZSTD_estimateCDictSize(cParams, dictSize); /* uses ZSTD_initCStream_usingDict() */
if (ZSTD_isError(s)) goto _output_error;
DISPLAYLEVEL(3, "OK (%u bytes) \n", (U32)s);
}
DISPLAYLEVEL(3, "test%3i : check actual CStream size : ", testNb++);
{ size_t const s = ZSTD_sizeof_CStream(zc);
if (ZSTD_isError(s)) goto _output_error;
DISPLAYLEVEL(3, "OK (%u bytes) \n", (U32)s);
}
/* Attempt bad compression parameters */
DISPLAYLEVEL(3, "test%3i : use bad compression parameters : ", testNb++);
{ size_t r;
ZSTD_parameters params = ZSTD_getParams(1, 0, 0);
params.cParams.searchLength = 2;
r = ZSTD_initCStream_advanced(zc, NULL, 0, params, 0);
if (!ZSTD_isError(r)) goto _output_error;
DISPLAYLEVEL(3, "init error : %s \n", ZSTD_getErrorName(r));
}
/* skippable frame test */
DISPLAYLEVEL(3, "test%3i : decompress skippable frame : ", testNb++);
ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
inBuff.src = compressedBuffer;
inBuff.size = cSize;
inBuff.pos = 0;
@@ -224,7 +246,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
/* Basic decompression test */
inBuff2 = inBuff;
DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
{ size_t const r = ZSTD_setDStreamParameter(zd, DStream_p_maxWindowSize, 1000000000); /* large limit */
if (ZSTD_isError(r)) goto _output_error; }
{ size_t const remaining = ZSTD_decompressStream(zd, &outBuff, &inBuff);
@@ -250,7 +272,20 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
} }
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : check DStream size : ", testNb++);
/* context size functions */
DISPLAYLEVEL(3, "test%3i : estimate DStream size : ", testNb++);
{ ZSTD_frameHeader fhi;
const void* cStart = (char*)compressedBuffer + (skippableFrameSize + 8);
size_t const gfhError = ZSTD_getFrameHeader(&fhi, cStart, cSize);
if (gfhError!=0) goto _output_error;
DISPLAYLEVEL(5, " (windowSize : %u) ", fhi.windowSize);
{ size_t const s = ZSTD_estimateDStreamSize(fhi)
+ ZSTD_estimateDDictSize(dictSize); /* uses ZSTD_initDStream_usingDict() */
if (ZSTD_isError(s)) goto _output_error;
DISPLAYLEVEL(3, "OK (%u bytes) \n", (U32)s);
} }
DISPLAYLEVEL(3, "test%3i : check actual DStream size : ", testNb++);
{ size_t const s = ZSTD_sizeof_DStream(zd);
if (ZSTD_isError(s)) goto _output_error;
DISPLAYLEVEL(3, "OK (%u bytes) \n", (U32)s);
@@ -260,7 +295,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
DISPLAYLEVEL(3, "test%3i : decompress byte-by-byte : ", testNb++);
{ /* skippable frame */
size_t r = 1;
ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
inBuff.src = compressedBuffer;
outBuff.dst = decodedBuffer;
inBuff.pos = 0;
@@ -272,7 +307,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
if (ZSTD_isError(r)) goto _output_error;
}
/* normal frame */
ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
r=1;
while (r) {
inBuff.size = inBuff.pos + 1;
@@ -334,6 +369,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
if (zc==NULL) goto _output_error; /* memory allocation issue */
/* use 1 */
{ size_t const inSize = 513;
DISPLAYLEVEL(5, "use1 ");
ZSTD_initCStream_advanced(zc, NULL, 0, ZSTD_getParams(19, inSize, 0), inSize); /* needs btopt + search3 to trigger hashLog3 */
inBuff.src = CNBuffer;
inBuff.size = inSize;
@@ -341,14 +377,17 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
outBuff.dst = (char*)(compressedBuffer)+cSize;
outBuff.size = ZSTD_compressBound(inSize);
outBuff.pos = 0;
DISPLAYLEVEL(5, "compress1 ");
{ size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
if (ZSTD_isError(r)) goto _output_error; }
if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
DISPLAYLEVEL(5, "end1 ");
{ size_t const r = ZSTD_endStream(zc, &outBuff);
if (r != 0) goto _output_error; } /* error, or some data not flushed */
}
/* use 2 */
{ size_t const inSize = 1025; /* will not continue, because tables auto-adjust and are therefore different size */
DISPLAYLEVEL(5, "use2 ");
ZSTD_initCStream_advanced(zc, NULL, 0, ZSTD_getParams(19, inSize, 0), inSize); /* needs btopt + search3 to trigger hashLog3 */
inBuff.src = CNBuffer;
inBuff.size = inSize;
@@ -356,9 +395,11 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
outBuff.dst = (char*)(compressedBuffer)+cSize;
outBuff.size = ZSTD_compressBound(inSize);
outBuff.pos = 0;
DISPLAYLEVEL(5, "compress2 ");
{ size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
if (ZSTD_isError(r)) goto _output_error; }
if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
DISPLAYLEVEL(5, "end2 ");
{ size_t const r = ZSTD_endStream(zc, &outBuff);
if (r != 0) goto _output_error; } /* error, or some data not flushed */
}
@@ -425,7 +466,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
/* Memory restriction */
DISPLAYLEVEL(3, "test%3i : maxWindowSize < frame requirement : ", testNb++);
ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
{ size_t const r = ZSTD_setDStreamParameter(zd, DStream_p_maxWindowSize, 1000); /* too small limit */
if (ZSTD_isError(r)) goto _output_error; }
inBuff.src = compressedBuffer;
@@ -438,11 +479,64 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
if (!ZSTD_isError(r)) goto _output_error; /* must fail : frame requires > 100 bytes */
DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r)); }
/* Unknown srcSize */
DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_usingCDict_advanced with masked dictID : ", testNb++);
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictionary.filled);
ZSTD_frameParameters const fParams = { 1 /* contentSize */, 1 /* checksum */, 1 /* noDictID */};
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, 1 /* byReference */, cParams, customMem);
size_t const initError = ZSTD_initCStream_usingCDict_advanced(zc, cdict, CNBufferSize, fParams);
if (ZSTD_isError(initError)) goto _output_error;
cSize = 0;
outBuff.dst = compressedBuffer;
outBuff.size = compressedBufferSize;
outBuff.pos = 0;
inBuff.src = CNBuffer;
inBuff.size = CNBufferSize;
inBuff.pos = 0;
{ size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
if (ZSTD_isError(r)) goto _output_error; }
if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
{ size_t const r = ZSTD_endStream(zc, &outBuff);
if (r != 0) goto _output_error; } /* error, or some data not flushed */
cSize = outBuff.pos;
ZSTD_freeCDict(cdict);
DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBufferSize*100);
}
DISPLAYLEVEL(3, "test%3i : try retrieving dictID from frame : ", testNb++);
{ U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
if (did != 0) goto _output_error;
}
DISPLAYLEVEL(3, "OK (not detected) \n");
DISPLAYLEVEL(3, "test%3i : decompress without dictionary : ", testNb++);
{ size_t const r = ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize);
if (!ZSTD_isError(r)) goto _output_error; /* must fail : dictionary not used */
DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r));
}
/* Empty srcSize */
DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_advanced with pledgedSrcSize=0 and dict : ", testNb++);
{ ZSTD_parameters params = ZSTD_getParams(5, 0, 0);
params.fParams.contentSizeFlag = 1;
ZSTD_initCStream_advanced(zc, dictionary.start, dictionary.filled, params, 0);
} /* cstream advanced shall write content size = 0 */
inBuff.src = CNBuffer;
inBuff.size = 0;
inBuff.pos = 0;
outBuff.dst = compressedBuffer;
outBuff.size = compressedBufferSize;
outBuff.pos = 0;
if (ZSTD_isError(ZSTD_compressStream(zc, &outBuff, &inBuff))) goto _output_error;
if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
cSize = outBuff.pos;
if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != 0) goto _output_error;
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : pledgedSrcSize == 0 behaves properly : ", testNb++);
{ ZSTD_parameters params = ZSTD_getParams(5, 0, 0);
params.fParams.contentSizeFlag = 1;
ZSTD_initCStream_advanced(zc, NULL, 0, params, 0); } /* cstream advanced should write the 0 size field */
ZSTD_initCStream_advanced(zc, NULL, 0, params, 0);
} /* cstream advanced shall write content size = 0 */
inBuff.src = CNBuffer;
inBuff.size = 0;
inBuff.pos = 0;
@@ -552,7 +646,7 @@ static size_t FUZ_randomLength(U32* seed, U32 maxLog)
#define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; }
static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility)
static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility, int bigTests)
{
static const U32 maxSrcLog = 24;
static const U32 maxSampleLog = 19;
@@ -574,6 +668,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
const BYTE* dict=NULL; /* can keep same dict on 2 consecutive tests */
size_t dictSize = 0;
U32 oldTestLog = 0;
int const cLevelLimiter = bigTests ? 3 : 2;
/* allocations */
cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
@@ -638,18 +733,23 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
if ((FUZ_rand(&lseed)&1) /* at beginning, to keep same nb of rand */
&& oldTestLog /* at least one test happened */ && resetAllowed) {
maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2);
if (maxTestSize >= srcBufferSize) maxTestSize = srcBufferSize-1;
if (maxTestSize >= srcBufferSize)
maxTestSize = srcBufferSize-1;
{ U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? 0 : maxTestSize;
size_t const resetError = ZSTD_resetCStream(zc, pledgedSrcSize);
CHECK(ZSTD_isError(resetError), "ZSTD_resetCStream error : %s", ZSTD_getErrorName(resetError));
}
} else {
U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (testLog/3))) + 1;
U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
U32 const cLevel = ( FUZ_rand(&lseed) %
(ZSTD_maxCLevel() -
(MAX(testLog, dictLog) / cLevelLimiter)))
+ 1;
maxTestSize = FUZ_rLogLength(&lseed, testLog);
oldTestLog = testLog;
/* random dictionary selection */
dictSize = ((FUZ_rand(&lseed)&63)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0;
dictSize = ((FUZ_rand(&lseed)&1)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0;
{ size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
dict = srcBuffer + dictStart;
}
@@ -701,7 +801,9 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
outBuff.size = outBuff.pos + adjustedDstSize;
remainingToFlush = ZSTD_endStream(zc, &outBuff);
CHECK (ZSTD_isError(remainingToFlush), "flush error : %s", ZSTD_getErrorName(remainingToFlush));
CHECK (enoughDstSize && remainingToFlush, "ZSTD_endStream() not fully flushed (%u remaining), but enough space available", (U32)remainingToFlush);
CHECK (enoughDstSize && remainingToFlush,
"ZSTD_endStream() not fully flushed (%u remaining), but enough space available (%u)",
(U32)remainingToFlush, (U32)adjustedDstSize);
} }
crcOrig = XXH64_digest(&xxhState);
cSize = outBuff.pos;
@@ -784,7 +886,7 @@ _output_error:
/* Multi-threading version of fuzzer Tests */
static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double compressibility)
static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double compressibility, int bigTests)
{
static const U32 maxSrcLog = 24;
static const U32 maxSampleLog = 19;
@@ -806,6 +908,7 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
const BYTE* dict=NULL; /* can keep same dict on 2 consecutive tests */
size_t dictSize = 0;
U32 oldTestLog = 0;
int const cLevelLimiter = bigTests ? 3 : 2;
/* allocations */
cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
@@ -850,6 +953,7 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
/* some issues can only happen when reusing states */
if ((FUZ_rand(&lseed) & 0xFF) == 131) {
U32 const nbThreads = (FUZ_rand(&lseed) % 6) + 1;
DISPLAYLEVEL(5, "Creating new context with %u threads \n", nbThreads);
ZSTDMT_freeCCtx(zc);
zc = ZSTDMT_createCCtx(nbThreads);
resetAllowed=0;
@@ -886,19 +990,26 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
}
} else {
U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (testLog/3))) + 1;
U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
U32 const cLevel = (FUZ_rand(&lseed) %
(ZSTD_maxCLevel() -
(MAX(testLog, dictLog) / cLevelLimiter))) +
1;
maxTestSize = FUZ_rLogLength(&lseed, testLog);
oldTestLog = testLog;
/* random dictionary selection */
dictSize = ((FUZ_rand(&lseed)&63)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0;
dictSize = ((FUZ_rand(&lseed)&63)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0;
{ size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
dict = srcBuffer + dictStart;
}
{ U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? 0 : maxTestSize;
ZSTD_parameters params = ZSTD_getParams(cLevel, pledgedSrcSize, dictSize);
DISPLAYLEVEL(5, "Init with windowLog = %u \n", params.cParams.windowLog);
DISPLAYLEVEL(5, "Init with windowLog = %u and pledgedSrcSize = %u \n",
params.cParams.windowLog, (U32)pledgedSrcSize);
params.fParams.checksumFlag = FUZ_rand(&lseed) & 1;
params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1;
params.fParams.contentSizeFlag = pledgedSrcSize>0;
DISPLAYLEVEL(5, "checksumFlag : %u \n", params.fParams.checksumFlag);
{ size_t const initError = ZSTDMT_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize);
CHECK (ZSTD_isError(initError),"ZSTDMT_initCStream_advanced error : %s", ZSTD_getErrorName(initError)); }
ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_overlapSectionLog, FUZ_rand(&lseed) % 12);
@@ -936,7 +1047,7 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
outBuff.size = outBuff.pos + adjustedDstSize;
DISPLAYLEVEL(5, "Flushing into dst buffer of size %u \n", (U32)adjustedDstSize);
{ size_t const flushError = ZSTDMT_flushStream(zc, &outBuff);
CHECK (ZSTD_isError(flushError), "flush error : %s", ZSTD_getErrorName(flushError));
CHECK (ZSTD_isError(flushError), "ZSTDMT_flushStream error : %s", ZSTD_getErrorName(flushError));
} } }
/* final frame epilogue */
@@ -947,12 +1058,12 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
outBuff.size = outBuff.pos + adjustedDstSize;
DISPLAYLEVEL(5, "Ending into dst buffer of size %u \n", (U32)adjustedDstSize);
remainingToFlush = ZSTDMT_endStream(zc, &outBuff);
CHECK (ZSTD_isError(remainingToFlush), "flush error : %s", ZSTD_getErrorName(remainingToFlush));
CHECK (ZSTD_isError(remainingToFlush), "ZSTDMT_endStream error : %s", ZSTD_getErrorName(remainingToFlush));
DISPLAYLEVEL(5, "endStream : remainingToFlush : %u \n", (U32)remainingToFlush);
} }
DISPLAYLEVEL(5, "Frame completed \n");
crcOrig = XXH64_digest(&xxhState);
cSize = outBuff.pos;
DISPLAYLEVEL(5, "Frame completed : %u bytes \n", (U32)cSize);
}
/* multi - fragments decompression test */
@@ -970,8 +1081,10 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
inBuff.size = inBuff.pos + readCSrcSize;
outBuff.size = inBuff.pos + dstBuffSize;
DISPLAYLEVEL(5, "ZSTD_decompressStream input %u bytes \n", (U32)readCSrcSize);
decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff);
CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult));
DISPLAYLEVEL(5, "inBuff.pos = %u \n", (U32)readCSrcSize);
}
CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)", (U32)outBuff.pos, (U32)totalTestSize);
CHECK (inBuff.pos != cSize, "compressed data should be fully read (%u != %u)", (U32)inBuff.pos, (U32)cSize);
@@ -1061,6 +1174,7 @@ int main(int argc, const char** argv)
int result=0;
int mainPause = 0;
int mtOnly = 0;
int bigTests = 1;
const char* const programName = argv[0];
ZSTD_customMem const customMem = { allocFunction, freeFunction, NULL };
ZSTD_customMem const customNULL = { NULL, NULL, NULL };
@@ -1074,6 +1188,7 @@ int main(int argc, const char** argv)
if (argument[0]=='-') {
if (!strcmp(argument, "--mt")) { mtOnly=1; continue; }
if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; }
argument++;
while (*argument!=0) {
@@ -1179,8 +1294,8 @@ int main(int argc, const char** argv)
result = basicUnitTests(0, ((double)proba) / 100, customMem); /* use custom memory allocation functions */
} }
if (!result && !mtOnly) result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100);
if (!result) result = fuzzerTests_MT(seed, nbTests, testNb, ((double)proba) / 100);
if (!result && !mtOnly) result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100, bigTests);
if (!result) result = fuzzerTests_MT(seed, nbTests, testNb, ((double)proba) / 100, bigTests);
if (mainPause) {
int unused;