+9
-9
@@ -41,7 +41,7 @@ Additional remarks:
|
||||
The example usage with two test files, one e-mail address, and with an additional message:
|
||||
```
|
||||
./test-zstd-speed.py "silesia.tar calgary.tar" "email@gmail.com" --message "tested on my laptop" --sleepTime 60
|
||||
```
|
||||
```
|
||||
|
||||
To run the script in background please use:
|
||||
```
|
||||
@@ -100,19 +100,19 @@ Full list of arguments
|
||||
h# - hashLog
|
||||
c# - chainLog
|
||||
s# - searchLog
|
||||
l# - searchLength
|
||||
l# - minMatch
|
||||
t# - targetLength
|
||||
S# - strategy
|
||||
L# - level
|
||||
--zstd= : Single run, parameter selection syntax same as zstdcli with more parameters
|
||||
(Added forceAttachDictionary / fadt)
|
||||
When invoked with --optimize, this represents the sample to exceed.
|
||||
(Added forceAttachDictionary / fadt)
|
||||
When invoked with --optimize, this represents the sample to exceed.
|
||||
--optimize= : find parameters to maximize compression ratio given parameters
|
||||
Can use all --zstd= commands to constrain the type of solution found in addition to the following constraints
|
||||
cSpeed= : Minimum compression speed
|
||||
dSpeed= : Minimum decompression speed
|
||||
cMem= : Maximum compression memory
|
||||
lvl= : Searches for solutions which are strictly better than that compression lvl in ratio and cSpeed,
|
||||
lvl= : Searches for solutions which are strictly better than that compression lvl in ratio and cSpeed,
|
||||
stc= : When invoked with lvl=, represents percentage slack in ratio/cSpeed allowed for a solution to be considered (Default 100%)
|
||||
: In normal operation, represents percentage slack in choosing viable starting strategy selection in choosing the default parameters
|
||||
(Lower value will begin with stronger strategies) (Default 90%)
|
||||
@@ -121,13 +121,13 @@ Full list of arguments
|
||||
when determining overall winner (default 5 (1% ratio = 5% speed)).
|
||||
tries= : Maximum number of random restarts on a single strategy before switching (Default 5)
|
||||
Higher values will make optimizer run longer, more chances to find better solution.
|
||||
memLog : Limits the log of the size of each memotable (1 per strategy). Will use hash tables when state space is larger than max size.
|
||||
Setting memLog = 0 turns off memoization
|
||||
memLog : Limits the log of the size of each memotable (1 per strategy). Will use hash tables when state space is larger than max size.
|
||||
Setting memLog = 0 turns off memoization
|
||||
--display= : specifiy which parameters are included in the output
|
||||
can use all --zstd parameter names and 'cParams' as a shorthand for all parameters used in ZSTD_compressionParameters
|
||||
can use all --zstd parameter names and 'cParams' as a shorthand for all parameters used in ZSTD_compressionParameters
|
||||
(Default: display all params available)
|
||||
-P# : generated sample compressibility (when no file is provided)
|
||||
-t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours ))
|
||||
-t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours ))
|
||||
-v : Prints Benchmarking output
|
||||
-D : Next argument dictionary file
|
||||
-s : Benchmark all files separately
|
||||
|
||||
+31
-49
@@ -171,17 +171,8 @@ local_ZSTD_compress_generic_end(const void* src, size_t srcSize,
|
||||
void* dst, size_t dstCapacity,
|
||||
void* buff2)
|
||||
{
|
||||
ZSTD_outBuffer buffOut;
|
||||
ZSTD_inBuffer buffIn;
|
||||
(void)buff2;
|
||||
buffOut.dst = dst;
|
||||
buffOut.size = dstCapacity;
|
||||
buffOut.pos = 0;
|
||||
buffIn.src = src;
|
||||
buffIn.size = srcSize;
|
||||
buffIn.pos = 0;
|
||||
ZSTD_compress_generic(g_cstream, &buffOut, &buffIn, ZSTD_e_end);
|
||||
return buffOut.pos;
|
||||
return ZSTD_compress2(g_cstream, dst, dstCapacity, src, srcSize);
|
||||
}
|
||||
|
||||
static size_t
|
||||
@@ -198,8 +189,8 @@ local_ZSTD_compress_generic_continue(const void* src, size_t srcSize,
|
||||
buffIn.src = src;
|
||||
buffIn.size = srcSize;
|
||||
buffIn.pos = 0;
|
||||
ZSTD_compress_generic(g_cstream, &buffOut, &buffIn, ZSTD_e_continue);
|
||||
ZSTD_compress_generic(g_cstream, &buffOut, &buffIn, ZSTD_e_end);
|
||||
ZSTD_compressStream2(g_cstream, &buffOut, &buffIn, ZSTD_e_continue);
|
||||
ZSTD_compressStream2(g_cstream, &buffOut, &buffIn, ZSTD_e_end);
|
||||
return buffOut.pos;
|
||||
}
|
||||
|
||||
@@ -208,18 +199,9 @@ local_ZSTD_compress_generic_T2_end(const void* src, size_t srcSize,
|
||||
void* dst, size_t dstCapacity,
|
||||
void* buff2)
|
||||
{
|
||||
ZSTD_outBuffer buffOut;
|
||||
ZSTD_inBuffer buffIn;
|
||||
(void)buff2;
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_nbWorkers, 2);
|
||||
buffOut.dst = dst;
|
||||
buffOut.size = dstCapacity;
|
||||
buffOut.pos = 0;
|
||||
buffIn.src = src;
|
||||
buffIn.size = srcSize;
|
||||
buffIn.pos = 0;
|
||||
while (ZSTD_compress_generic(g_cstream, &buffOut, &buffIn, ZSTD_e_end)) {}
|
||||
return buffOut.pos;
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_nbWorkers, 2);
|
||||
return ZSTD_compress2(g_cstream, dst, dstCapacity, src, srcSize);
|
||||
}
|
||||
|
||||
static size_t
|
||||
@@ -230,15 +212,15 @@ local_ZSTD_compress_generic_T2_continue(const void* src, size_t srcSize,
|
||||
ZSTD_outBuffer buffOut;
|
||||
ZSTD_inBuffer buffIn;
|
||||
(void)buff2;
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_nbWorkers, 2);
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_nbWorkers, 2);
|
||||
buffOut.dst = dst;
|
||||
buffOut.size = dstCapacity;
|
||||
buffOut.pos = 0;
|
||||
buffIn.src = src;
|
||||
buffIn.size = srcSize;
|
||||
buffIn.pos = 0;
|
||||
ZSTD_compress_generic(g_cstream, &buffOut, &buffIn, ZSTD_e_continue);
|
||||
while(ZSTD_compress_generic(g_cstream, &buffOut, &buffIn, ZSTD_e_end)) {}
|
||||
ZSTD_compressStream2(g_cstream, &buffOut, &buffIn, ZSTD_e_continue);
|
||||
while(ZSTD_compressStream2(g_cstream, &buffOut, &buffIn, ZSTD_e_end)) {}
|
||||
return buffOut.pos;
|
||||
}
|
||||
|
||||
@@ -408,28 +390,28 @@ static size_t benchMem(U32 benchNb,
|
||||
if (g_cstream==NULL) g_cstream = ZSTD_createCStream();
|
||||
if (g_dstream==NULL) g_dstream = ZSTD_createDStream();
|
||||
|
||||
/* DISPLAY("params: cLevel %d, wlog %d hlog %d clog %d slog %d slen %d tlen %d strat %d \n",
|
||||
/* DISPLAY("params: cLevel %d, wlog %d hlog %d clog %d slog %d mml %d tlen %d strat %d \n",
|
||||
cLevel, cparams->windowLog, cparams->hashLog, cparams->chainLog, cparams->searchLog,
|
||||
cparams->searchLength, cparams->targetLength, cparams->strategy); */
|
||||
cparams->minMatch, cparams->targetLength, cparams->strategy); */
|
||||
|
||||
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_compressionLevel, cLevel);
|
||||
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_windowLog, cparams.windowLog);
|
||||
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_hashLog, cparams.hashLog);
|
||||
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_chainLog, cparams.chainLog);
|
||||
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_searchLog, cparams.searchLog);
|
||||
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_minMatch, cparams.searchLength);
|
||||
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_targetLength, cparams.targetLength);
|
||||
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_compressionStrategy, cparams.strategy);
|
||||
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_compressionLevel, cLevel);
|
||||
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_windowLog, cparams.windowLog);
|
||||
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_hashLog, cparams.hashLog);
|
||||
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_chainLog, cparams.chainLog);
|
||||
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_searchLog, cparams.searchLog);
|
||||
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_minMatch, cparams.minMatch);
|
||||
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_targetLength, cparams.targetLength);
|
||||
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_compressionStrategy, cparams.strategy);
|
||||
|
||||
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionLevel, cLevel);
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_windowLog, cparams.windowLog);
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_hashLog, cparams.hashLog);
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_chainLog, cparams.chainLog);
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_searchLog, cparams.searchLog);
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_minMatch, cparams.searchLength);
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_targetLength, cparams.targetLength);
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionStrategy, cparams.strategy);
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_compressionLevel, cLevel);
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_windowLog, cparams.windowLog);
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_hashLog, cparams.hashLog);
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_chainLog, cparams.chainLog);
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_searchLog, cparams.searchLog);
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_minMatch, cparams.minMatch);
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_targetLength, cparams.targetLength);
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_compressionStrategy, cparams.strategy);
|
||||
|
||||
/* Preparation */
|
||||
switch(benchNb)
|
||||
@@ -455,8 +437,8 @@ static size_t benchMem(U32 benchNb,
|
||||
ZSTD_frameHeader zfp;
|
||||
size_t frameHeaderSize, skippedSize;
|
||||
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel);
|
||||
frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_frameHeaderSize_min);
|
||||
if (frameHeaderSize==0) frameHeaderSize = ZSTD_frameHeaderSize_min;
|
||||
frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_FRAMEHEADERSIZE_MIN);
|
||||
if (frameHeaderSize==0) frameHeaderSize = ZSTD_FRAMEHEADERSIZE_MIN;
|
||||
ZSTD_getcBlockSize(dstBuff+frameHeaderSize, dstBuffSize, &bp); /* Get 1st block type */
|
||||
if (bp.blockType != bt_compressed) {
|
||||
DISPLAY("ZSTD_decodeLiteralsBlock : impossible to test on this sample (not compressible)\n");
|
||||
@@ -476,8 +458,8 @@ static size_t benchMem(U32 benchNb,
|
||||
size_t frameHeaderSize, cBlockSize;
|
||||
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel); /* it would be better to use direct block compression here */
|
||||
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel);
|
||||
frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_frameHeaderSize_min);
|
||||
if (frameHeaderSize==0) frameHeaderSize = ZSTD_frameHeaderSize_min;
|
||||
frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_FRAMEHEADERSIZE_MIN);
|
||||
if (frameHeaderSize==0) frameHeaderSize = ZSTD_FRAMEHEADERSIZE_MIN;
|
||||
ip += frameHeaderSize; /* Skip frame Header */
|
||||
cBlockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); /* Get 1st block type */
|
||||
if (bp.blockType != bt_compressed) {
|
||||
@@ -752,7 +734,7 @@ int main(int argc, const char** argv)
|
||||
if (longCommandWArg(&argument, "chainLog=") || longCommandWArg(&argument, "clog=")) { cparams.chainLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
|
||||
if (longCommandWArg(&argument, "hashLog=") || longCommandWArg(&argument, "hlog=")) { cparams.hashLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
|
||||
if (longCommandWArg(&argument, "searchLog=") || longCommandWArg(&argument, "slog=")) { cparams.searchLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
|
||||
if (longCommandWArg(&argument, "searchLength=") || longCommandWArg(&argument, "slen=")) { cparams.searchLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
|
||||
if (longCommandWArg(&argument, "minMatch=") || longCommandWArg(&argument, "mml=")) { cparams.minMatch = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
|
||||
if (longCommandWArg(&argument, "targetLength=") || longCommandWArg(&argument, "tlen=")) { cparams.targetLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
|
||||
if (longCommandWArg(&argument, "strategy=") || longCommandWArg(&argument, "strat=")) { cparams.strategy = (ZSTD_strategy)(readU32FromChar(&argument)); if (argument[0]==',') { argument++; continue; } else break; }
|
||||
if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevel = (int)readU32FromChar(&argument); cparams = ZSTD_getCParams(cLevel, 0, 0); if (argument[0]==',') { argument++; continue; } else break; }
|
||||
|
||||
@@ -40,9 +40,9 @@ static size_t roundTripTest(void *result, size_t resultCapacity,
|
||||
ZSTD_outBuffer out = {compressed, compressedCapacity, 0};
|
||||
size_t err;
|
||||
|
||||
ZSTD_CCtx_reset(cctx);
|
||||
ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
|
||||
FUZZ_setRandomParameters(cctx, srcSize, &seed);
|
||||
err = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
|
||||
err = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
|
||||
FUZZ_ZASSERT(err);
|
||||
FUZZ_ASSERT(err == 0);
|
||||
cSize = out.pos;
|
||||
|
||||
@@ -56,7 +56,7 @@ static size_t compress(uint8_t *dst, size_t capacity,
|
||||
const uint8_t *src, size_t srcSize)
|
||||
{
|
||||
size_t dstSize = 0;
|
||||
ZSTD_CCtx_reset(cctx);
|
||||
ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
|
||||
FUZZ_setRandomParameters(cctx, srcSize, &seed);
|
||||
|
||||
while (srcSize > 0) {
|
||||
@@ -72,7 +72,7 @@ static size_t compress(uint8_t *dst, size_t capacity,
|
||||
case 1: /* fall-though */
|
||||
case 2: {
|
||||
size_t const ret =
|
||||
ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_flush);
|
||||
ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush);
|
||||
FUZZ_ZASSERT(ret);
|
||||
if (ret == 0)
|
||||
mode = -1;
|
||||
@@ -80,11 +80,11 @@ static size_t compress(uint8_t *dst, size_t capacity,
|
||||
}
|
||||
case 3: {
|
||||
size_t ret =
|
||||
ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
|
||||
ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
|
||||
FUZZ_ZASSERT(ret);
|
||||
/* Reset the compressor when the frame is finished */
|
||||
if (ret == 0) {
|
||||
ZSTD_CCtx_reset(cctx);
|
||||
ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
|
||||
if ((FUZZ_rand(&seed) & 7) == 0) {
|
||||
size_t const remaining = in.size - in.pos;
|
||||
FUZZ_setRandomParameters(cctx, remaining, &seed);
|
||||
@@ -95,7 +95,7 @@ static size_t compress(uint8_t *dst, size_t capacity,
|
||||
}
|
||||
default: {
|
||||
size_t const ret =
|
||||
ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_continue);
|
||||
ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue);
|
||||
FUZZ_ZASSERT(ret);
|
||||
mode = -1;
|
||||
}
|
||||
@@ -108,7 +108,7 @@ static size_t compress(uint8_t *dst, size_t capacity,
|
||||
for (;;) {
|
||||
ZSTD_inBuffer in = {NULL, 0, 0};
|
||||
ZSTD_outBuffer out = makeOutBuffer(dst, capacity);
|
||||
size_t const ret = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
|
||||
size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
|
||||
FUZZ_ZASSERT(ret);
|
||||
|
||||
dst += out.pos;
|
||||
|
||||
+19
-19
@@ -32,8 +32,8 @@ ZSTD_compressionParameters FUZZ_randomCParams(size_t srcSize, uint32_t *state)
|
||||
cParams.hashLog = FUZZ_rand32(state, ZSTD_HASHLOG_MIN, 15);
|
||||
cParams.chainLog = FUZZ_rand32(state, ZSTD_CHAINLOG_MIN, 16);
|
||||
cParams.searchLog = FUZZ_rand32(state, ZSTD_SEARCHLOG_MIN, 9);
|
||||
cParams.searchLength = FUZZ_rand32(state, ZSTD_SEARCHLENGTH_MIN,
|
||||
ZSTD_SEARCHLENGTH_MAX);
|
||||
cParams.minMatch = FUZZ_rand32(state, ZSTD_MINMATCH_MIN,
|
||||
ZSTD_MINMATCH_MAX);
|
||||
cParams.targetLength = FUZZ_rand32(state, 0, 512);
|
||||
cParams.strategy = FUZZ_rand32(state, ZSTD_fast, ZSTD_btultra);
|
||||
return ZSTD_adjustCParams(cParams, srcSize, 0);
|
||||
@@ -60,25 +60,25 @@ ZSTD_parameters FUZZ_randomParams(size_t srcSize, uint32_t *state)
|
||||
void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, uint32_t *state)
|
||||
{
|
||||
ZSTD_compressionParameters cParams = FUZZ_randomCParams(srcSize, state);
|
||||
set(cctx, ZSTD_p_windowLog, cParams.windowLog);
|
||||
set(cctx, ZSTD_p_hashLog, cParams.hashLog);
|
||||
set(cctx, ZSTD_p_chainLog, cParams.chainLog);
|
||||
set(cctx, ZSTD_p_searchLog, cParams.searchLog);
|
||||
set(cctx, ZSTD_p_minMatch, cParams.searchLength);
|
||||
set(cctx, ZSTD_p_targetLength, cParams.targetLength);
|
||||
set(cctx, ZSTD_p_compressionStrategy, cParams.strategy);
|
||||
set(cctx, ZSTD_c_windowLog, cParams.windowLog);
|
||||
set(cctx, ZSTD_c_hashLog, cParams.hashLog);
|
||||
set(cctx, ZSTD_c_chainLog, cParams.chainLog);
|
||||
set(cctx, ZSTD_c_searchLog, cParams.searchLog);
|
||||
set(cctx, ZSTD_c_minMatch, cParams.minMatch);
|
||||
set(cctx, ZSTD_c_targetLength, cParams.targetLength);
|
||||
set(cctx, ZSTD_c_compressionStrategy, cParams.strategy);
|
||||
/* Select frame parameters */
|
||||
setRand(cctx, ZSTD_p_contentSizeFlag, 0, 1, state);
|
||||
setRand(cctx, ZSTD_p_checksumFlag, 0, 1, state);
|
||||
setRand(cctx, ZSTD_p_dictIDFlag, 0, 1, state);
|
||||
setRand(cctx, ZSTD_p_forceAttachDict, 0, 2, state);
|
||||
setRand(cctx, ZSTD_c_contentSizeFlag, 0, 1, state);
|
||||
setRand(cctx, ZSTD_c_checksumFlag, 0, 1, state);
|
||||
setRand(cctx, ZSTD_c_dictIDFlag, 0, 1, state);
|
||||
setRand(cctx, ZSTD_c_forceAttachDict, 0, 2, state);
|
||||
/* Select long distance matchig parameters */
|
||||
setRand(cctx, ZSTD_p_enableLongDistanceMatching, 0, 1, state);
|
||||
setRand(cctx, ZSTD_p_ldmHashLog, ZSTD_HASHLOG_MIN, 16, state);
|
||||
setRand(cctx, ZSTD_p_ldmMinMatch, ZSTD_LDM_MINMATCH_MIN,
|
||||
setRand(cctx, ZSTD_c_enableLongDistanceMatching, 0, 1, state);
|
||||
setRand(cctx, ZSTD_c_ldmHashLog, ZSTD_HASHLOG_MIN, 16, state);
|
||||
setRand(cctx, ZSTD_c_ldmMinMatch, ZSTD_LDM_MINMATCH_MIN,
|
||||
ZSTD_LDM_MINMATCH_MAX, state);
|
||||
setRand(cctx, ZSTD_p_ldmBucketSizeLog, 0, ZSTD_LDM_BUCKETSIZELOG_MAX,
|
||||
setRand(cctx, ZSTD_c_ldmBucketSizeLog, 0, ZSTD_LDM_BUCKETSIZELOG_MAX,
|
||||
state);
|
||||
setRand(cctx, ZSTD_p_ldmHashEveryLog, 0,
|
||||
ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN, state);
|
||||
setRand(cctx, ZSTD_c_ldmHashRateLog, ZSTD_LDM_HASHRATELOG_MIN,
|
||||
ZSTD_LDM_HASHRATELOG_MAX, state);
|
||||
}
|
||||
|
||||
+173
-98
@@ -233,11 +233,9 @@ static int FUZ_mallocTests_internal(unsigned seed, double compressibility, unsig
|
||||
mallocCounter_t malcount = INIT_MALLOC_COUNTER;
|
||||
ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
|
||||
ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
|
||||
ZSTD_outBuffer out = { outBuffer, outSize, 0 };
|
||||
ZSTD_inBuffer in = { inBuffer, inSize, 0 };
|
||||
CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, (U32)compressionLevel) );
|
||||
CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_p_nbWorkers, nbThreads) );
|
||||
while ( ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end) ) {}
|
||||
CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) );
|
||||
CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads) );
|
||||
CHECK_Z( ZSTD_compress2(cctx, outBuffer, outSize, inBuffer, inSize) );
|
||||
ZSTD_freeCCtx(cctx);
|
||||
DISPLAYLEVEL(3, "compress_generic,-T%u,end level %i : ",
|
||||
nbThreads, compressionLevel);
|
||||
@@ -255,10 +253,10 @@ static int FUZ_mallocTests_internal(unsigned seed, double compressibility, unsig
|
||||
ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
|
||||
ZSTD_outBuffer out = { outBuffer, outSize, 0 };
|
||||
ZSTD_inBuffer in = { inBuffer, inSize, 0 };
|
||||
CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, (U32)compressionLevel) );
|
||||
CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_p_nbWorkers, nbThreads) );
|
||||
CHECK_Z( ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_continue) );
|
||||
while ( ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end) ) {}
|
||||
CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) );
|
||||
CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads) );
|
||||
CHECK_Z( ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue) );
|
||||
while ( ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end) ) {}
|
||||
ZSTD_freeCCtx(cctx);
|
||||
DISPLAYLEVEL(3, "compress_generic,-T%u,continue level %i : ",
|
||||
nbThreads, compressionLevel);
|
||||
@@ -311,7 +309,6 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize);
|
||||
void* const compressedBuffer = malloc(compressedBufferSize);
|
||||
void* const decodedBuffer = malloc(CNBuffSize);
|
||||
ZSTD_DCtx* dctx = ZSTD_createDCtx();
|
||||
int testResult = 0;
|
||||
U32 testNb=0;
|
||||
size_t cSize;
|
||||
@@ -383,13 +380,27 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : decompress with null dict : ", testNb++);
|
||||
{ size_t const r = ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, NULL, 0);
|
||||
if (r != CNBuffSize) goto _output_error; }
|
||||
{ ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
|
||||
{ size_t const r = ZSTD_decompress_usingDict(dctx,
|
||||
decodedBuffer, CNBuffSize,
|
||||
compressedBuffer, cSize,
|
||||
NULL, 0);
|
||||
if (r != CNBuffSize) goto _output_error;
|
||||
}
|
||||
ZSTD_freeDCtx(dctx);
|
||||
}
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : decompress with null DDict : ", testNb++);
|
||||
{ size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, NULL);
|
||||
if (r != CNBuffSize) goto _output_error; }
|
||||
{ ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
|
||||
{ size_t const r = ZSTD_decompress_usingDDict(dctx,
|
||||
decodedBuffer, CNBuffSize,
|
||||
compressedBuffer, cSize,
|
||||
NULL);
|
||||
if (r != CNBuffSize) goto _output_error;
|
||||
}
|
||||
ZSTD_freeDCtx(dctx);
|
||||
}
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : decompress with 1 missing byte : ", testNb++);
|
||||
@@ -504,39 +515,39 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
|
||||
ZSTD_outBuffer out = {NULL, 0, 0};
|
||||
ZSTD_inBuffer in = {NULL, 0, 0};
|
||||
unsigned value;
|
||||
int value;
|
||||
|
||||
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_compressionLevel, &value));
|
||||
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
|
||||
CHECK_EQ(value, 3);
|
||||
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_hashLog, &value));
|
||||
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
|
||||
CHECK_EQ(value, 0);
|
||||
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_p_hashLog, ZSTD_HASHLOG_MIN));
|
||||
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_compressionLevel, &value));
|
||||
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, ZSTD_HASHLOG_MIN));
|
||||
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
|
||||
CHECK_EQ(value, 3);
|
||||
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_hashLog, &value));
|
||||
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
|
||||
CHECK_EQ(value, ZSTD_HASHLOG_MIN);
|
||||
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, 7));
|
||||
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_compressionLevel, &value));
|
||||
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7));
|
||||
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
|
||||
CHECK_EQ(value, 7);
|
||||
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_hashLog, &value));
|
||||
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
|
||||
CHECK_EQ(value, ZSTD_HASHLOG_MIN);
|
||||
/* Start a compression job */
|
||||
ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_continue);
|
||||
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_compressionLevel, &value));
|
||||
ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue);
|
||||
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
|
||||
CHECK_EQ(value, 7);
|
||||
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_hashLog, &value));
|
||||
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
|
||||
CHECK_EQ(value, ZSTD_HASHLOG_MIN);
|
||||
/* Reset the CCtx */
|
||||
ZSTD_CCtx_reset(cctx);
|
||||
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_compressionLevel, &value));
|
||||
ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
|
||||
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
|
||||
CHECK_EQ(value, 7);
|
||||
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_hashLog, &value));
|
||||
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
|
||||
CHECK_EQ(value, ZSTD_HASHLOG_MIN);
|
||||
/* Reset the parameters */
|
||||
ZSTD_CCtx_resetParameters(cctx);
|
||||
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_compressionLevel, &value));
|
||||
ZSTD_CCtx_reset(cctx, ZSTD_reset_parameters);
|
||||
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
|
||||
CHECK_EQ(value, 3);
|
||||
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_hashLog, &value));
|
||||
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
|
||||
CHECK_EQ(value, 0);
|
||||
|
||||
ZSTD_freeCCtx(cctx);
|
||||
@@ -546,8 +557,8 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
/* this test is really too long, and should be made faster */
|
||||
DISPLAYLEVEL(3, "test%3d : overflow protection with large windowLog : ", testNb++);
|
||||
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
|
||||
ZSTD_parameters params = ZSTD_getParams(-9, ZSTD_CONTENTSIZE_UNKNOWN, 0);
|
||||
size_t const nbCompressions = ((1U << 31) / CNBuffSize) + 1; /* ensure U32 overflow protection is triggered */
|
||||
ZSTD_parameters params = ZSTD_getParams(-999, ZSTD_CONTENTSIZE_UNKNOWN, 0);
|
||||
size_t const nbCompressions = ((1U << 31) / CNBuffSize) + 2; /* ensure U32 overflow protection is triggered */
|
||||
size_t cnb;
|
||||
assert(cctx != NULL);
|
||||
params.fParams.contentSizeFlag = 0;
|
||||
@@ -753,7 +764,7 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
const U32 skipLen = 129 KB;
|
||||
MEM_writeLE32((BYTE*)compressedBuffer + off, ZSTD_MAGIC_SKIPPABLE_START);
|
||||
MEM_writeLE32((BYTE*)compressedBuffer + off + 4, skipLen);
|
||||
off += skipLen + ZSTD_skippableHeaderSize;
|
||||
off += skipLen + ZSTD_SKIPPABLEHEADERSIZE;
|
||||
}
|
||||
}
|
||||
cSize = off;
|
||||
@@ -777,7 +788,9 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
/* Dictionary and CCtx Duplication tests */
|
||||
{ ZSTD_CCtx* const ctxOrig = ZSTD_createCCtx();
|
||||
ZSTD_CCtx* const ctxDuplicated = ZSTD_createCCtx();
|
||||
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
|
||||
static const size_t dictSize = 551;
|
||||
assert(dctx != NULL); assert(ctxOrig != NULL); assert(ctxDuplicated != NULL);
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : copy context too soon : ", testNb++);
|
||||
{ size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0);
|
||||
@@ -861,6 +874,7 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
|
||||
ZSTD_freeCCtx(ctxOrig);
|
||||
ZSTD_freeCCtx(ctxDuplicated);
|
||||
ZSTD_freeDCtx(dctx);
|
||||
}
|
||||
|
||||
/* Dictionary and dictBuilder tests */
|
||||
@@ -905,7 +919,7 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
coverParams.nbThreads = 4;
|
||||
dictSize = ZDICT_optimizeTrainFromBuffer_cover(
|
||||
dictBuffer, dictBufferCapacity,
|
||||
CNBuffer, samplesSizes, nbSamples,
|
||||
CNBuffer, samplesSizes, nbSamples/8, /* less samples for faster tests */
|
||||
&coverParams);
|
||||
if (ZDICT_isError(dictSize)) goto _output_error;
|
||||
}
|
||||
@@ -950,11 +964,14 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
|
||||
CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
|
||||
decodedBuffer, CNBuffSize,
|
||||
compressedBuffer, cSize,
|
||||
dictBuffer, dictSize),
|
||||
if (r != CNBuffSize) goto _output_error);
|
||||
{ ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
|
||||
CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
|
||||
decodedBuffer, CNBuffSize,
|
||||
compressedBuffer, cSize,
|
||||
dictBuffer, dictSize),
|
||||
if (r != CNBuffSize) goto _output_error);
|
||||
ZSTD_freeDCtx(dctx);
|
||||
}
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : estimate CDict size : ", testNb++);
|
||||
@@ -983,11 +1000,14 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
|
||||
CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
|
||||
decodedBuffer, CNBuffSize,
|
||||
compressedBuffer, cSize,
|
||||
dictBuffer, dictSize),
|
||||
if (r != CNBuffSize) goto _output_error);
|
||||
{ ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
|
||||
CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
|
||||
decodedBuffer, CNBuffSize,
|
||||
compressedBuffer, cSize,
|
||||
dictBuffer, dictSize),
|
||||
if (r != CNBuffSize) goto _output_error);
|
||||
ZSTD_freeDCtx(dctx);
|
||||
}
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : compress with static CDict : ", testNb++);
|
||||
@@ -1036,11 +1056,14 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
DISPLAYLEVEL(3, "OK (unknown)\n");
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : frame built without dictID should be decompressible : ", testNb++);
|
||||
CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
|
||||
decodedBuffer, CNBuffSize,
|
||||
compressedBuffer, cSize,
|
||||
dictBuffer, dictSize),
|
||||
if (r != CNBuffSize) goto _output_error);
|
||||
{ ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
|
||||
CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
|
||||
decodedBuffer, CNBuffSize,
|
||||
compressedBuffer, cSize,
|
||||
dictBuffer, dictSize),
|
||||
if (r != CNBuffSize) goto _output_error);
|
||||
ZSTD_freeDCtx(dctx);
|
||||
}
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : ZSTD_compress_advanced, no dictID : ", testNb++);
|
||||
@@ -1054,19 +1077,26 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : frame built without dictID should be decompressible : ", testNb++);
|
||||
CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
|
||||
decodedBuffer, CNBuffSize,
|
||||
compressedBuffer, cSize,
|
||||
dictBuffer, dictSize),
|
||||
if (r != CNBuffSize) goto _output_error);
|
||||
{ ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
|
||||
CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
|
||||
decodedBuffer, CNBuffSize,
|
||||
compressedBuffer, cSize,
|
||||
dictBuffer, dictSize),
|
||||
if (r != CNBuffSize) goto _output_error);
|
||||
ZSTD_freeDCtx(dctx);
|
||||
}
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : dictionary containing only header should return error : ", testNb++);
|
||||
{
|
||||
const size_t ret = ZSTD_decompress_usingDict(
|
||||
dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize,
|
||||
"\x37\xa4\x30\xec\x11\x22\x33\x44", 8);
|
||||
if (ZSTD_getErrorCode(ret) != ZSTD_error_dictionary_corrupted) goto _output_error;
|
||||
{ ZSTD_DCtx* const dctx = ZSTD_createDCtx();
|
||||
assert(dctx != NULL);
|
||||
{ const size_t ret = ZSTD_decompress_usingDict(
|
||||
dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize,
|
||||
"\x37\xa4\x30\xec\x11\x22\x33\x44", 8);
|
||||
if (ZSTD_getErrorCode(ret) != ZSTD_error_dictionary_corrupted)
|
||||
goto _output_error;
|
||||
}
|
||||
ZSTD_freeDCtx(dctx);
|
||||
}
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
@@ -1135,10 +1165,12 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
*/
|
||||
{ size_t dSize;
|
||||
BYTE data[1024];
|
||||
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
|
||||
ZSTD_compressionParameters const cParams = ZSTD_getCParams(19, CNBuffSize, dictSize);
|
||||
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize,
|
||||
ZSTD_dlm_byRef, ZSTD_dct_auto,
|
||||
cParams, ZSTD_defaultCMem);
|
||||
assert(dctx != NULL); assert(cdict != NULL);
|
||||
memset(data, 'x', sizeof(data));
|
||||
cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize,
|
||||
data, sizeof(data), cdict);
|
||||
@@ -1147,6 +1179,7 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
dSize = ZSTD_decompress_usingDict(dctx, decodedBuffer, sizeof(data), compressedBuffer, cSize, dictBuffer, dictSize);
|
||||
if (ZSTD_isError(dSize)) { DISPLAYLEVEL(5, "Decompression error %s : ", ZSTD_getErrorName(dSize)); goto _output_error; }
|
||||
if (memcmp(data, decodedBuffer, sizeof(data))) { DISPLAYLEVEL(5, "Data corruption : "); goto _output_error; }
|
||||
ZSTD_freeDCtx(dctx);
|
||||
}
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
@@ -1252,13 +1285,13 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
params);
|
||||
if (ZSTD_isError(cSize_1pass)) goto _output_error;
|
||||
|
||||
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, (unsigned)compressionLevel) );
|
||||
{ ZSTD_inBuffer in = { CNBuffer, srcSize, 0 };
|
||||
ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 };
|
||||
size_t const compressionResult = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
|
||||
DISPLAYLEVEL(5, "simple=%zu vs %zu=advanced : ", cSize_1pass, out.pos);
|
||||
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) );
|
||||
{ size_t const compressionResult = ZSTD_compress2(cctx,
|
||||
compressedBuffer, compressedBufferSize,
|
||||
CNBuffer, srcSize);
|
||||
DISPLAYLEVEL(5, "simple=%zu vs %zu=advanced : ", cSize_1pass, compressionResult);
|
||||
if (ZSTD_isError(compressionResult)) goto _output_error;
|
||||
if (out.pos != cSize_1pass) goto _output_error;
|
||||
if (compressionResult != cSize_1pass) goto _output_error;
|
||||
} }
|
||||
ZSTD_freeCCtx(cctx);
|
||||
}
|
||||
@@ -1271,16 +1304,15 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
|
||||
DISPLAYLEVEL(3, "test%3i : parameters in order : ", testNb++);
|
||||
assert(cctx != NULL);
|
||||
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, 2) );
|
||||
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_enableLongDistanceMatching, 1) );
|
||||
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_windowLog, 18) );
|
||||
{ ZSTD_inBuffer in = { CNBuffer, inputSize, 0 };
|
||||
ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(inputSize), 0 };
|
||||
size_t const result = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
|
||||
if (result != 0) goto _output_error;
|
||||
if (in.pos != in.size) goto _output_error;
|
||||
cSize = out.pos;
|
||||
xxh64 = XXH64(out.dst, out.pos, 0);
|
||||
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) );
|
||||
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1) );
|
||||
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) );
|
||||
{ size_t const compressedSize = ZSTD_compress2(cctx,
|
||||
compressedBuffer, ZSTD_compressBound(inputSize),
|
||||
CNBuffer, inputSize);
|
||||
CHECK(compressedSize);
|
||||
cSize = compressedSize;
|
||||
xxh64 = XXH64(compressedBuffer, compressedSize, 0);
|
||||
}
|
||||
DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (U32)inputSize, (U32)cSize);
|
||||
ZSTD_freeCCtx(cctx);
|
||||
@@ -1288,32 +1320,65 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
|
||||
{ ZSTD_CCtx* cctx = ZSTD_createCCtx();
|
||||
DISPLAYLEVEL(3, "test%3i : parameters disordered : ", testNb++);
|
||||
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_windowLog, 18) );
|
||||
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_enableLongDistanceMatching, 1) );
|
||||
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, 2) );
|
||||
{ ZSTD_inBuffer in = { CNBuffer, inputSize, 0 };
|
||||
ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(inputSize), 0 };
|
||||
size_t const result = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
|
||||
if (result != 0) goto _output_error;
|
||||
if (in.pos != in.size) goto _output_error;
|
||||
if (out.pos != cSize) goto _output_error; /* must result in same compressed result, hence same size */
|
||||
if (XXH64(out.dst, out.pos, 0) != xxh64) goto _output_error; /* must result in exactly same content, hence same hash */
|
||||
DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (U32)inputSize, (U32)out.pos);
|
||||
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) );
|
||||
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1) );
|
||||
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) );
|
||||
{ size_t const result = ZSTD_compress2(cctx,
|
||||
compressedBuffer, ZSTD_compressBound(inputSize),
|
||||
CNBuffer, inputSize);
|
||||
CHECK(result);
|
||||
if (result != cSize) goto _output_error; /* must result in same compressed result, hence same size */
|
||||
if (XXH64(compressedBuffer, result, 0) != xxh64) goto _output_error; /* must result in exactly same content, hence same hash */
|
||||
DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (U32)inputSize, (U32)result);
|
||||
}
|
||||
ZSTD_freeCCtx(cctx);
|
||||
}
|
||||
}
|
||||
|
||||
/* advanced parameters for decompression */
|
||||
{ ZSTD_DCtx* const dctx = ZSTD_createDCtx();
|
||||
assert(dctx != NULL);
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : get dParameter bounds ", testNb++);
|
||||
{ ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax);
|
||||
CHECK(bounds.error);
|
||||
}
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : wrong dParameter : ", testNb++);
|
||||
{ size_t const sr = ZSTD_DCtx_setParameter(dctx, (ZSTD_dParameter)999999, 0);
|
||||
if (!ZSTD_isError(sr)) goto _output_error;
|
||||
}
|
||||
{ ZSTD_bounds const bounds = ZSTD_dParam_getBounds((ZSTD_dParameter)999998);
|
||||
if (!ZSTD_isError(bounds.error)) goto _output_error;
|
||||
}
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : out of bound dParameter : ", testNb++);
|
||||
{ size_t const sr = ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 9999);
|
||||
if (!ZSTD_isError(sr)) goto _output_error;
|
||||
}
|
||||
{ size_t const sr = ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, (ZSTD_format_e)888);
|
||||
if (!ZSTD_isError(sr)) goto _output_error;
|
||||
}
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
ZSTD_freeDCtx(dctx);
|
||||
}
|
||||
|
||||
|
||||
/* custom formats tests */
|
||||
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
|
||||
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
|
||||
size_t const inputSize = CNBuffSize / 2; /* won't cause pb with small dict size */
|
||||
assert(dctx != NULL); assert(cctx != NULL);
|
||||
|
||||
/* basic block compression */
|
||||
DISPLAYLEVEL(3, "test%3i : magic-less format test : ", testNb++);
|
||||
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_format, ZSTD_f_zstd1_magicless) );
|
||||
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
|
||||
{ ZSTD_inBuffer in = { CNBuffer, inputSize, 0 };
|
||||
ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(inputSize), 0 };
|
||||
size_t const result = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
|
||||
size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
|
||||
if (result != 0) goto _output_error;
|
||||
if (in.pos != in.size) goto _output_error;
|
||||
cSize = out.pos;
|
||||
@@ -1327,29 +1392,38 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
}
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : decompress of magic-less frame : ", testNb++);
|
||||
ZSTD_DCtx_reset(dctx);
|
||||
CHECK( ZSTD_DCtx_setFormat(dctx, ZSTD_f_zstd1_magicless) );
|
||||
ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
|
||||
CHECK( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) );
|
||||
{ ZSTD_frameHeader zfh;
|
||||
size_t const zfhrt = ZSTD_getFrameHeader_advanced(&zfh, compressedBuffer, cSize, ZSTD_f_zstd1_magicless);
|
||||
if (zfhrt != 0) goto _output_error;
|
||||
}
|
||||
/* one shot */
|
||||
{ size_t const result = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
|
||||
if (result != inputSize) goto _output_error;
|
||||
DISPLAYLEVEL(3, "one-shot OK, ");
|
||||
}
|
||||
/* streaming */
|
||||
{ ZSTD_inBuffer in = { compressedBuffer, cSize, 0 };
|
||||
ZSTD_outBuffer out = { decodedBuffer, CNBuffSize, 0 };
|
||||
size_t const result = ZSTD_decompress_generic(dctx, &out, &in);
|
||||
size_t const result = ZSTD_decompressStream(dctx, &out, &in);
|
||||
if (result != 0) goto _output_error;
|
||||
if (in.pos != in.size) goto _output_error;
|
||||
if (out.pos != inputSize) goto _output_error;
|
||||
DISPLAYLEVEL(3, "OK : regenerated %u bytes \n", (U32)out.pos);
|
||||
DISPLAYLEVEL(3, "streaming OK : regenerated %u bytes \n", (U32)out.pos);
|
||||
}
|
||||
|
||||
ZSTD_freeCCtx(cctx);
|
||||
ZSTD_freeDCtx(dctx);
|
||||
}
|
||||
|
||||
/* block API tests */
|
||||
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
|
||||
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
|
||||
static const size_t dictSize = 65 KB;
|
||||
static const size_t blockSize = 100 KB; /* won't cause pb with small dict size */
|
||||
size_t cSize2;
|
||||
assert(cctx != NULL); assert(dctx != NULL);
|
||||
|
||||
/* basic block compression */
|
||||
DISPLAYLEVEL(3, "test%3i : Block compression test : ", testNb++);
|
||||
@@ -1367,13 +1441,14 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
|
||||
/* very long stream of block compression */
|
||||
DISPLAYLEVEL(3, "test%3i : Huge block streaming compression test : ", testNb++);
|
||||
CHECK( ZSTD_compressBegin(cctx, -99) ); /* we just want to quickly overflow internal U32 index */
|
||||
CHECK( ZSTD_compressBegin(cctx, -199) ); /* we just want to quickly overflow internal U32 index */
|
||||
CHECK( ZSTD_getBlockSize(cctx) >= blockSize);
|
||||
{ U64 const toCompress = 5000000000ULL; /* > 4 GB */
|
||||
U64 compressed = 0;
|
||||
while (compressed < toCompress) {
|
||||
size_t const blockCSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize);
|
||||
if (ZSTD_isError(cSize)) goto _output_error;
|
||||
assert(blockCSize != 0);
|
||||
if (ZSTD_isError(blockCSize)) goto _output_error;
|
||||
compressed += blockCSize;
|
||||
}
|
||||
}
|
||||
@@ -1411,8 +1486,8 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
ZSTD_freeCCtx(cctx);
|
||||
ZSTD_freeDCtx(dctx);
|
||||
}
|
||||
ZSTD_freeDCtx(dctx);
|
||||
|
||||
/* long rle test */
|
||||
{ size_t sampleSize = 0;
|
||||
@@ -1927,7 +2002,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
|
||||
DISPLAYLEVEL(5, "fuzzer t%u: Bufferless streaming decompression test \n", testNb);
|
||||
/* ensure memory requirement is good enough (should always be true) */
|
||||
{ ZSTD_frameHeader zfh;
|
||||
CHECK( ZSTD_getFrameHeader(&zfh, cBuffer, ZSTD_frameHeaderSize_max),
|
||||
CHECK( ZSTD_getFrameHeader(&zfh, cBuffer, ZSTD_FRAMEHEADERSIZE_MAX),
|
||||
"ZSTD_getFrameHeader(): error retrieving frame information");
|
||||
{ size_t const roundBuffSize = ZSTD_decodingBufferSize_min(zfh.windowSize, zfh.frameContentSize);
|
||||
CHECK_Z(roundBuffSize);
|
||||
|
||||
+1
-1
@@ -50,7 +50,7 @@ int main(int argc, const char** argv)
|
||||
params.cParams.chainLog = 13;
|
||||
params.cParams.hashLog = 14;
|
||||
params.cParams.searchLog = 1;
|
||||
params.cParams.searchLength = 7;
|
||||
params.cParams.minMatch = 7;
|
||||
params.cParams.targetLength = 16;
|
||||
params.cParams.strategy = ZSTD_fast;
|
||||
windowLog = params.cParams.windowLog;
|
||||
|
||||
+24
-42
@@ -75,7 +75,7 @@ static const int g_maxNbVariations = 64;
|
||||
#define CLOG_RANGE (ZSTD_CHAINLOG_MAX - ZSTD_CHAINLOG_MIN + 1)
|
||||
#define HLOG_RANGE (ZSTD_HASHLOG_MAX - ZSTD_HASHLOG_MIN + 1)
|
||||
#define SLOG_RANGE (ZSTD_SEARCHLOG_MAX - ZSTD_SEARCHLOG_MIN + 1)
|
||||
#define SLEN_RANGE (ZSTD_SEARCHLENGTH_MAX - ZSTD_SEARCHLENGTH_MIN + 1)
|
||||
#define MML_RANGE (ZSTD_MINMATCH_MAX - ZSTD_MINMATCH_MIN + 1)
|
||||
#define TLEN_RANGE 17
|
||||
#define STRT_RANGE (ZSTD_btultra - ZSTD_fast + 1)
|
||||
#define FADT_RANGE 3
|
||||
@@ -103,7 +103,7 @@ typedef enum {
|
||||
clog_ind = 1,
|
||||
hlog_ind = 2,
|
||||
slog_ind = 3,
|
||||
slen_ind = 4,
|
||||
mml_ind = 4,
|
||||
tlen_ind = 5,
|
||||
strt_ind = 6,
|
||||
fadt_ind = 7, /* forceAttachDict */
|
||||
@@ -116,27 +116,27 @@ typedef struct {
|
||||
|
||||
/* maximum value of parameters */
|
||||
static const U32 mintable[NUM_PARAMS] =
|
||||
{ ZSTD_WINDOWLOG_MIN, ZSTD_CHAINLOG_MIN, ZSTD_HASHLOG_MIN, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLENGTH_MIN, ZSTD_TARGETLENGTH_MIN, ZSTD_fast, FADT_MIN };
|
||||
{ ZSTD_WINDOWLOG_MIN, ZSTD_CHAINLOG_MIN, ZSTD_HASHLOG_MIN, ZSTD_SEARCHLOG_MIN, ZSTD_MINMATCH_MIN, ZSTD_TARGETLENGTH_MIN, ZSTD_fast, FADT_MIN };
|
||||
|
||||
/* minimum value of parameters */
|
||||
static const U32 maxtable[NUM_PARAMS] =
|
||||
{ ZSTD_WINDOWLOG_MAX, ZSTD_CHAINLOG_MAX, ZSTD_HASHLOG_MAX, ZSTD_SEARCHLOG_MAX, ZSTD_SEARCHLENGTH_MAX, ZSTD_TARGETLENGTH_MAX, ZSTD_btultra, FADT_MAX };
|
||||
{ ZSTD_WINDOWLOG_MAX, ZSTD_CHAINLOG_MAX, ZSTD_HASHLOG_MAX, ZSTD_SEARCHLOG_MAX, ZSTD_MINMATCH_MAX, ZSTD_TARGETLENGTH_MAX, ZSTD_btultra, FADT_MAX };
|
||||
|
||||
/* # of values parameters can take on */
|
||||
static const U32 rangetable[NUM_PARAMS] =
|
||||
{ WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE, STRT_RANGE, FADT_RANGE };
|
||||
{ WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, MML_RANGE, TLEN_RANGE, STRT_RANGE, FADT_RANGE };
|
||||
|
||||
/* ZSTD_cctxSetParameter() index to set */
|
||||
static const ZSTD_cParameter cctxSetParamTable[NUM_PARAMS] =
|
||||
{ ZSTD_p_windowLog, ZSTD_p_chainLog, ZSTD_p_hashLog, ZSTD_p_searchLog, ZSTD_p_minMatch, ZSTD_p_targetLength, ZSTD_p_compressionStrategy, ZSTD_p_forceAttachDict };
|
||||
{ ZSTD_c_windowLog, ZSTD_c_chainLog, ZSTD_c_hashLog, ZSTD_c_searchLog, ZSTD_c_minMatch, ZSTD_c_targetLength, ZSTD_c_compressionStrategy, ZSTD_c_forceAttachDict };
|
||||
|
||||
/* names of parameters */
|
||||
static const char* g_paramNames[NUM_PARAMS] =
|
||||
{ "windowLog", "chainLog", "hashLog","searchLog", "searchLength", "targetLength", "strategy", "forceAttachDict" };
|
||||
{ "windowLog", "chainLog", "hashLog","searchLog", "minMatch", "targetLength", "strategy", "forceAttachDict" };
|
||||
|
||||
/* shortened names of parameters */
|
||||
static const char* g_shortParamNames[NUM_PARAMS] =
|
||||
{ "wlog", "clog", "hlog","slog", "slen", "tlen", "strt", "fadt" };
|
||||
{ "wlog", "clog", "hlog", "slog", "mml", "tlen", "strat", "fadt" };
|
||||
|
||||
/* maps value from { 0 to rangetable[param] - 1 } to valid paramvalues */
|
||||
static U32 rangeMap(varInds_t param, int ind) {
|
||||
@@ -150,7 +150,7 @@ static U32 rangeMap(varInds_t param, int ind) {
|
||||
case clog_ind:
|
||||
case hlog_ind:
|
||||
case slog_ind:
|
||||
case slen_ind:
|
||||
case mml_ind:
|
||||
case strt_ind:
|
||||
return mintable[param] + ind;
|
||||
case NUM_PARAMS:
|
||||
@@ -186,7 +186,7 @@ static int invRangeMap(varInds_t param, U32 value) {
|
||||
case clog_ind:
|
||||
case hlog_ind:
|
||||
case slog_ind:
|
||||
case slen_ind:
|
||||
case mml_ind:
|
||||
case strt_ind:
|
||||
return value - mintable[param];
|
||||
case NUM_PARAMS:
|
||||
@@ -205,7 +205,7 @@ static void displayParamVal(FILE* f, varInds_t param, U32 value, int width) {
|
||||
case clog_ind:
|
||||
case hlog_ind:
|
||||
case slog_ind:
|
||||
case slen_ind:
|
||||
case mml_ind:
|
||||
case tlen_ind: if(width) { fprintf(f, "%*u", width, value); } else { fprintf(f, "%u", value); } break;
|
||||
case NUM_PARAMS:
|
||||
DISPLAY("Error, not a valid param\n "); break;
|
||||
@@ -311,7 +311,7 @@ static ZSTD_compressionParameters pvalsToCParams(paramValues_t p) {
|
||||
c.chainLog = p.vals[clog_ind];
|
||||
c.hashLog = p.vals[hlog_ind];
|
||||
c.searchLog = p.vals[slog_ind];
|
||||
c.searchLength = p.vals[slen_ind];
|
||||
c.minMatch = p.vals[mml_ind];
|
||||
c.targetLength = p.vals[tlen_ind];
|
||||
c.strategy = p.vals[strt_ind];
|
||||
/* no forceAttachDict */
|
||||
@@ -325,7 +325,7 @@ static paramValues_t cParamsToPVals(ZSTD_compressionParameters c) {
|
||||
p.vals[clog_ind] = c.chainLog;
|
||||
p.vals[hlog_ind] = c.hashLog;
|
||||
p.vals[slog_ind] = c.searchLog;
|
||||
p.vals[slen_ind] = c.searchLength;
|
||||
p.vals[mml_ind] = c.minMatch;
|
||||
p.vals[tlen_ind] = c.targetLength;
|
||||
p.vals[strt_ind] = c.strategy;
|
||||
|
||||
@@ -893,9 +893,8 @@ typedef struct {
|
||||
static size_t local_initCCtx(void* payload) {
|
||||
const BMK_initCCtxArgs* ag = (const BMK_initCCtxArgs*)payload;
|
||||
varInds_t i;
|
||||
ZSTD_CCtx_reset(ag->cctx);
|
||||
ZSTD_CCtx_resetParameters(ag->cctx);
|
||||
ZSTD_CCtx_setParameter(ag->cctx, ZSTD_p_compressionLevel, ag->cLevel);
|
||||
ZSTD_CCtx_reset(ag->cctx, ZSTD_reset_session_and_parameters);
|
||||
ZSTD_CCtx_setParameter(ag->cctx, ZSTD_c_compressionLevel, ag->cLevel);
|
||||
|
||||
for(i = 0; i < NUM_PARAMS; i++) {
|
||||
if(ag->comprParams->vals[i] != PARAM_UNSET)
|
||||
@@ -914,37 +913,20 @@ typedef struct {
|
||||
|
||||
static size_t local_initDCtx(void* payload) {
|
||||
const BMK_initDCtxArgs* ag = (const BMK_initDCtxArgs*)payload;
|
||||
ZSTD_DCtx_reset(ag->dctx);
|
||||
ZSTD_DCtx_reset(ag->dctx, ZSTD_reset_session_and_parameters);
|
||||
ZSTD_DCtx_loadDictionary(ag->dctx, ag->dictBuffer, ag->dictBufferSize);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* additional argument is just the context */
|
||||
static size_t local_defaultCompress(
|
||||
const void* srcBuffer, size_t srcSize,
|
||||
void* dstBuffer, size_t dstSize,
|
||||
void* addArgs) {
|
||||
size_t moreToFlush = 1;
|
||||
ZSTD_CCtx* ctx = (ZSTD_CCtx*)addArgs;
|
||||
ZSTD_inBuffer in;
|
||||
ZSTD_outBuffer out;
|
||||
in.src = srcBuffer;
|
||||
in.size = srcSize;
|
||||
in.pos = 0;
|
||||
out.dst = dstBuffer;
|
||||
out.size = dstSize;
|
||||
out.pos = 0;
|
||||
const void* srcBuffer, size_t srcSize,
|
||||
void* dstBuffer, size_t dstSize,
|
||||
void* addArgs)
|
||||
{
|
||||
ZSTD_CCtx* cctx = (ZSTD_CCtx*)addArgs;
|
||||
assert(dstSize == ZSTD_compressBound(srcSize)); /* specific to this version, which is only used in paramgrill */
|
||||
while (moreToFlush) {
|
||||
if(out.pos == out.size) {
|
||||
return (size_t)-ZSTD_error_dstSize_tooSmall;
|
||||
}
|
||||
moreToFlush = ZSTD_compress_generic(ctx, &out, &in, ZSTD_e_end);
|
||||
if (ZSTD_isError(moreToFlush)) {
|
||||
return moreToFlush;
|
||||
}
|
||||
}
|
||||
return out.pos;
|
||||
return ZSTD_compress2(cctx, dstBuffer, dstSize, srcBuffer, srcSize);
|
||||
}
|
||||
|
||||
/* additional argument is just the context */
|
||||
@@ -966,7 +948,7 @@ static size_t local_defaultDecompress(
|
||||
if(out.pos == out.size) {
|
||||
return (size_t)-ZSTD_error_dstSize_tooSmall;
|
||||
}
|
||||
moreToFlush = ZSTD_decompress_generic(dctx,
|
||||
moreToFlush = ZSTD_decompressStream(dctx,
|
||||
&out, &in);
|
||||
if (ZSTD_isError(moreToFlush)) {
|
||||
return moreToFlush;
|
||||
@@ -2665,7 +2647,7 @@ int main(int argc, const char** argv)
|
||||
continue;
|
||||
case 'l': /* search length */
|
||||
argument++;
|
||||
g_params.vals[slen_ind] = readU32FromChar(&argument);
|
||||
g_params.vals[mml_ind] = readU32FromChar(&argument);
|
||||
continue;
|
||||
case 't': /* target length */
|
||||
argument++;
|
||||
|
||||
+4
-4
@@ -239,11 +239,11 @@ $ECHO "Hello world!" | $ZSTD --zstd=windowLo=21 - -o tmp.zst && die "wron
|
||||
$ECHO "Hello world!" | $ZSTD --zstd=windowLog=21,slog - -o tmp.zst && die "wrong parameters not detected!"
|
||||
test ! -f tmp.zst # tmp.zst should not be created
|
||||
roundTripTest -g512K
|
||||
roundTripTest -g512K " --zstd=slen=3,tlen=48,strat=6"
|
||||
roundTripTest -g512K " --zstd=mml=3,tlen=48,strat=6"
|
||||
roundTripTest -g512K " --zstd=strat=6,wlog=23,clog=23,hlog=22,slog=6"
|
||||
roundTripTest -g512K " --zstd=windowLog=23,chainLog=23,hashLog=22,searchLog=6,searchLength=3,targetLength=48,strategy=6"
|
||||
roundTripTest -g512K " --single-thread --long --zstd=ldmHashLog=20,ldmSearchLength=64,ldmBucketSizeLog=1,ldmHashEveryLog=7"
|
||||
roundTripTest -g512K " --single-thread --long --zstd=ldmhlog=20,ldmslen=64,ldmblog=1,ldmhevery=7"
|
||||
roundTripTest -g512K " --zstd=windowLog=23,chainLog=23,hashLog=22,searchLog=6,minMatch=3,targetLength=48,strategy=6"
|
||||
roundTripTest -g512K " --single-thread --long --zstd=ldmHashLog=20,ldmMinMatch=64,ldmBucketSizeLog=1,ldmHashRateLog=7"
|
||||
roundTripTest -g512K " --single-thread --long --zstd=lhlog=20,lmml=64,lblog=1,lhrlog=7"
|
||||
roundTripTest -g512K 19
|
||||
|
||||
|
||||
|
||||
@@ -85,7 +85,7 @@ static size_t cctxParamRoundTripTest(void* resultBuff, size_t resultBuffCapacity
|
||||
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
|
||||
ZSTD_CCtx_params* const cctxParams = ZSTD_createCCtxParams();
|
||||
ZSTD_inBuffer inBuffer = { srcBuff, srcBuffSize, 0 };
|
||||
ZSTD_outBuffer outBuffer = {compressedBuff, compressedBuffCapacity, 0 };
|
||||
ZSTD_outBuffer outBuffer = { compressedBuff, compressedBuffCapacity, 0 };
|
||||
|
||||
static const int maxClevel = 19;
|
||||
size_t const hashLength = MIN(128, srcBuffSize);
|
||||
@@ -93,15 +93,15 @@ static size_t cctxParamRoundTripTest(void* resultBuff, size_t resultBuffCapacity
|
||||
int const cLevel = h32 % maxClevel;
|
||||
|
||||
/* Set parameters */
|
||||
CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_compressionLevel, cLevel) );
|
||||
CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_nbWorkers, 2) );
|
||||
CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_overlapSizeLog, 5) );
|
||||
CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_c_compressionLevel, cLevel) );
|
||||
CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_c_nbWorkers, 2) );
|
||||
CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_c_overlapSizeLog, 5) );
|
||||
|
||||
|
||||
/* Apply parameters */
|
||||
CHECK_Z( ZSTD_CCtx_setParametersUsingCCtxParams(cctx, cctxParams) );
|
||||
|
||||
CHECK_Z (ZSTD_compress_generic(cctx, &outBuffer, &inBuffer, ZSTD_e_end) );
|
||||
CHECK_Z (ZSTD_compressStream2(cctx, &outBuffer, &inBuffer, ZSTD_e_end) );
|
||||
|
||||
ZSTD_freeCCtxParams(cctxParams);
|
||||
ZSTD_freeCCtx(cctx);
|
||||
|
||||
@@ -89,7 +89,6 @@ static const void *symbols[] = {
|
||||
&ZSTD_sizeof_CStream,
|
||||
&ZSTD_createDStream_advanced,
|
||||
&ZSTD_initDStream_usingDict,
|
||||
&ZSTD_setDStreamParameter,
|
||||
&ZSTD_initDStream_usingDDict,
|
||||
&ZSTD_resetDStream,
|
||||
&ZSTD_sizeof_DStream,
|
||||
|
||||
+64
-71
@@ -175,11 +175,11 @@ static size_t SEQ_roundTrip(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
|
||||
size_t cret;
|
||||
|
||||
do {
|
||||
ZSTD_outBuffer cout = {compressed, sizeof(compressed), 0};
|
||||
ZSTD_inBuffer din = {compressed, 0, 0};
|
||||
ZSTD_outBuffer dout = {uncompressed, 0, 0};
|
||||
ZSTD_outBuffer cout = { compressed, sizeof(compressed), 0 };
|
||||
ZSTD_inBuffer din = { compressed, 0, 0 };
|
||||
ZSTD_outBuffer dout = { uncompressed, 0, 0 };
|
||||
|
||||
cret = ZSTD_compress_generic(cctx, &cout, &cin, endOp);
|
||||
cret = ZSTD_compressStream2(cctx, &cout, &cin, endOp);
|
||||
if (ZSTD_isError(cret))
|
||||
return cret;
|
||||
|
||||
@@ -223,19 +223,19 @@ static size_t SEQ_generateRoundTrip(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
|
||||
|
||||
static size_t getCCtxParams(ZSTD_CCtx* zc, ZSTD_parameters* savedParams)
|
||||
{
|
||||
unsigned value;
|
||||
CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_p_windowLog, &savedParams->cParams.windowLog));
|
||||
CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_p_hashLog, &savedParams->cParams.hashLog));
|
||||
CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_p_chainLog, &savedParams->cParams.chainLog));
|
||||
CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_p_searchLog, &savedParams->cParams.searchLog));
|
||||
CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_p_minMatch, &savedParams->cParams.searchLength));
|
||||
CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_p_targetLength, &savedParams->cParams.targetLength));
|
||||
CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_p_compressionStrategy, &value));
|
||||
int value;
|
||||
CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_windowLog, (int*)&savedParams->cParams.windowLog));
|
||||
CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_hashLog, (int*)&savedParams->cParams.hashLog));
|
||||
CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_chainLog, (int*)&savedParams->cParams.chainLog));
|
||||
CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_searchLog, (int*)&savedParams->cParams.searchLog));
|
||||
CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_minMatch, (int*)&savedParams->cParams.minMatch));
|
||||
CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_targetLength, (int*)&savedParams->cParams.targetLength));
|
||||
CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_compressionStrategy, &value));
|
||||
savedParams->cParams.strategy = value;
|
||||
|
||||
CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_p_checksumFlag, &savedParams->fParams.checksumFlag));
|
||||
CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_p_contentSizeFlag, &savedParams->fParams.contentSizeFlag));
|
||||
CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_p_dictIDFlag, &value));
|
||||
CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_checksumFlag, &savedParams->fParams.checksumFlag));
|
||||
CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_contentSizeFlag, &savedParams->fParams.contentSizeFlag));
|
||||
CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_dictIDFlag, &value));
|
||||
savedParams->fParams.noDictIDFlag = !value;
|
||||
return 0;
|
||||
}
|
||||
@@ -248,7 +248,7 @@ static U32 badParameters(ZSTD_CCtx* zc, ZSTD_parameters const savedParams)
|
||||
CHECK_RET(2, params.cParams.hashLog != savedParams.cParams.hashLog, "hashLog");
|
||||
CHECK_RET(3, params.cParams.chainLog != savedParams.cParams.chainLog, "chainLog");
|
||||
CHECK_RET(4, params.cParams.searchLog != savedParams.cParams.searchLog, "searchLog");
|
||||
CHECK_RET(5, params.cParams.searchLength != savedParams.cParams.searchLength, "searchLength");
|
||||
CHECK_RET(5, params.cParams.minMatch != savedParams.cParams.minMatch, "minMatch");
|
||||
CHECK_RET(6, params.cParams.targetLength != savedParams.cParams.targetLength, "targetLength");
|
||||
|
||||
CHECK_RET(7, params.fParams.checksumFlag != savedParams.fParams.checksumFlag, "checksumFlag");
|
||||
@@ -353,7 +353,7 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
DISPLAYLEVEL(3, "test%3i : use bad compression parameters : ", testNb++);
|
||||
{ size_t r;
|
||||
ZSTD_parameters params = ZSTD_getParams(1, 0, 0);
|
||||
params.cParams.searchLength = 2;
|
||||
params.cParams.minMatch = 2;
|
||||
r = ZSTD_initCStream_advanced(zc, NULL, 0, params, 0);
|
||||
if (!ZSTD_isError(r)) goto _output_error;
|
||||
DISPLAYLEVEL(3, "init error : %s \n", ZSTD_getErrorName(r));
|
||||
@@ -379,7 +379,7 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
inBuff2 = inBuff;
|
||||
DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
|
||||
ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
|
||||
CHECK_Z( ZSTD_setDStreamParameter(zd, DStream_p_maxWindowSize, 1000000000) ); /* large limit */
|
||||
CHECK_Z( ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, ZSTD_WINDOWLOG_LIMIT_DEFAULT+1) ); /* large limit */
|
||||
{ size_t const remaining = ZSTD_decompressStream(zd, &outBuff, &inBuff);
|
||||
if (remaining != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */
|
||||
if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
|
||||
@@ -649,16 +649,10 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
}
|
||||
|
||||
/* test ZSTD_setDStreamParameter() resilience */
|
||||
DISPLAYLEVEL(3, "test%3i : wrong parameter for ZSTD_setDStreamParameter(): ", testNb++);
|
||||
{ size_t const r = ZSTD_setDStreamParameter(zd, (ZSTD_DStreamParameter_e)999, 1); /* large limit */
|
||||
if (!ZSTD_isError(r)) goto _output_error; }
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
/* Memory restriction */
|
||||
DISPLAYLEVEL(3, "test%3i : maxWindowSize < frame requirement : ", testNb++);
|
||||
ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
|
||||
CHECK_Z( ZSTD_setDStreamParameter(zd, DStream_p_maxWindowSize, 1000) ); /* too small limit */
|
||||
CHECK_Z( ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, 10) ); /* too small limit */
|
||||
outBuff.dst = decodedBuffer;
|
||||
outBuff.size = CNBufferSize;
|
||||
outBuff.pos = 0;
|
||||
@@ -668,7 +662,7 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
{ size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
|
||||
if (!ZSTD_isError(r)) goto _output_error; /* must fail : frame requires > 100 bytes */
|
||||
DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r)); }
|
||||
ZSTD_DCtx_reset(zd); /* leave zd in good shape for next tests */
|
||||
ZSTD_DCtx_reset(zd, ZSTD_reset_session_and_parameters); /* leave zd in good shape for next tests */
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : dictionary source size and level : ", testNb++);
|
||||
{ ZSTD_DCtx* const dctx = ZSTD_createDCtx();
|
||||
@@ -692,12 +686,12 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
inBuff.size = size;
|
||||
inBuff.pos = 0;
|
||||
CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
|
||||
CHECK_Z(ZSTD_compress_generic(cctx, &outBuff, &inBuff, ZSTD_e_end));
|
||||
CHECK_Z(ZSTD_compressStream2(cctx, &outBuff, &inBuff, ZSTD_e_end));
|
||||
CHECK(badParameters(cctx, savedParams), "Bad CCtx params");
|
||||
if (inBuff.pos != inBuff.size) goto _output_error;
|
||||
{ ZSTD_outBuffer decOut = {decodedBuffer, size, 0};
|
||||
ZSTD_inBuffer decIn = {outBuff.dst, outBuff.pos, 0};
|
||||
CHECK_Z( ZSTD_decompress_generic(dctx, &decOut, &decIn) );
|
||||
CHECK_Z( ZSTD_decompressStream(dctx, &decOut, &decIn) );
|
||||
if (decIn.pos != decIn.size) goto _output_error;
|
||||
if (decOut.pos != size) goto _output_error;
|
||||
{ U64 const crcDec = XXH64(decOut.dst, decOut.pos, 0);
|
||||
@@ -752,7 +746,7 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
inBuff.src = CNBuffer;
|
||||
inBuff.size = CNBufferSize;
|
||||
inBuff.pos = 0;
|
||||
CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end) );
|
||||
CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end) );
|
||||
if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
|
||||
cSize = outBuff.pos;
|
||||
DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBufferSize*100);
|
||||
@@ -765,7 +759,7 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
inBuff.src = compressedBuffer;
|
||||
inBuff.size = cSize;
|
||||
inBuff.pos = 0;
|
||||
CHECK_Z( ZSTD_decompress_generic(zd, &outBuff, &inBuff) );
|
||||
CHECK_Z( ZSTD_decompressStream(zd, &outBuff, &inBuff) );
|
||||
if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
|
||||
if (outBuff.pos != CNBufferSize) goto _output_error; /* must regenerate whole input */
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
@@ -776,14 +770,14 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r));
|
||||
}
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : compress again with ZSTD_compress_generic : ", testNb++);
|
||||
DISPLAYLEVEL(3, "test%3i : compress again with ZSTD_compressStream2 : ", testNb++);
|
||||
outBuff.dst = compressedBuffer;
|
||||
outBuff.size = compressedBufferSize;
|
||||
outBuff.pos = 0;
|
||||
inBuff.src = CNBuffer;
|
||||
inBuff.size = CNBufferSize;
|
||||
inBuff.pos = 0;
|
||||
CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end) );
|
||||
CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end) );
|
||||
if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
|
||||
cSize = outBuff.pos;
|
||||
DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBufferSize*100);
|
||||
@@ -874,9 +868,9 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
const BYTE* const srcToCopy = (const BYTE*)CNBuffer + start;
|
||||
BYTE* const dst = (BYTE*)CNBuffer + start - offset;
|
||||
DISPLAYLEVEL(3, "test%3i : compress %u bytes with multiple threads + dictionary : ", testNb++, (U32)srcSize);
|
||||
CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_compressionLevel, 3) );
|
||||
CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_nbWorkers, nbWorkers) );
|
||||
CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_jobSize, jobSize) );
|
||||
CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 3) );
|
||||
CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_nbWorkers, nbWorkers) );
|
||||
CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_jobSize, jobSize) );
|
||||
assert(start > offset);
|
||||
assert(start + segLength < COMPRESSIBLE_NOISE_LENGTH);
|
||||
memcpy(dst, srcToCopy, segLength); /* create a long repetition at long distance for job 2 */
|
||||
@@ -891,7 +885,7 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
|
||||
DISPLAYLEVEL(5, "cParams.windowLog = %u : ", cParams.windowLog);
|
||||
CHECK_Z( ZSTD_CCtx_refCDict(zc, cdict) );
|
||||
CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end) );
|
||||
CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end) );
|
||||
CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) ); /* do not keep a reference to cdict, as its lifetime ends */
|
||||
ZSTD_freeCDict(cdict);
|
||||
}
|
||||
@@ -936,11 +930,11 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
|
||||
if (!cdict || !ddict) goto _output_error;
|
||||
|
||||
ZSTD_CCtx_reset(zc);
|
||||
ZSTD_CCtx_reset(zc, ZSTD_reset_session_only);
|
||||
ZSTD_resetDStream(zd);
|
||||
CHECK_Z(ZSTD_CCtx_refCDict(zc, cdict));
|
||||
CHECK_Z(ZSTD_initDStream_usingDDict(zd, ddict));
|
||||
CHECK_Z(ZSTD_setDStreamParameter(zd, DStream_p_maxWindowSize, 1U << kMaxWindowLog));
|
||||
CHECK_Z(ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, kMaxWindowLog));
|
||||
/* Test all values < 300 */
|
||||
for (value = 0; value < 300; ++value) {
|
||||
for (type = (SEQ_gen_type)0; type < SEQ_gen_max; ++type) {
|
||||
@@ -968,12 +962,12 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_srcSize sets requestedParams : ", testNb++);
|
||||
{ unsigned level;
|
||||
{ int level;
|
||||
CHECK_Z(ZSTD_initCStream_srcSize(zc, 11, ZSTD_CONTENTSIZE_UNKNOWN));
|
||||
CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_p_compressionLevel, &level));
|
||||
CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_compressionLevel, &level));
|
||||
CHECK(level != 11, "Compression level does not match");
|
||||
ZSTD_resetCStream(zc, ZSTD_CONTENTSIZE_UNKNOWN);
|
||||
CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_p_compressionLevel, &level));
|
||||
CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_compressionLevel, &level));
|
||||
CHECK(level != 11, "Compression level does not match");
|
||||
}
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
@@ -1083,17 +1077,16 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
int remainingInput = 256 * 1024;
|
||||
int offset;
|
||||
|
||||
ZSTD_CCtx_reset(zc);
|
||||
CHECK_Z(ZSTD_CCtx_resetParameters(zc));
|
||||
CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
|
||||
CHECK_Z(ZSTD_CCtx_refCDict(zc, cdict));
|
||||
CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_p_checksumFlag, 1));
|
||||
CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_checksumFlag, 1));
|
||||
/* Write a bunch of 6 byte blocks */
|
||||
while (remainingInput > 0) {
|
||||
char testBuffer[6] = "\xAA\xAA\xAA\xAA\xAA\xAA";
|
||||
const size_t kSmallBlockSize = sizeof(testBuffer);
|
||||
ZSTD_inBuffer in = {testBuffer, kSmallBlockSize, 0};
|
||||
|
||||
CHECK_Z(ZSTD_compress_generic(zc, &out, &in, ZSTD_e_flush));
|
||||
CHECK_Z(ZSTD_compressStream2(zc, &out, &in, ZSTD_e_flush));
|
||||
CHECK(in.pos != in.size, "input not fully consumed");
|
||||
remainingInput -= kSmallBlockSize;
|
||||
}
|
||||
@@ -1101,7 +1094,7 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
for (offset = 1024; offset >= 0; offset -= 128) {
|
||||
ZSTD_inBuffer in = {dictionary.start + offset, 128, 0};
|
||||
ZSTD_EndDirective flush = offset > 0 ? ZSTD_e_continue : ZSTD_e_end;
|
||||
CHECK_Z(ZSTD_compress_generic(zc, &out, &in, flush));
|
||||
CHECK_Z(ZSTD_compressStream2(zc, &out, &in, flush));
|
||||
CHECK(in.pos != in.size, "input not fully consumed");
|
||||
}
|
||||
/* Ensure decompression works */
|
||||
@@ -1824,7 +1817,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest,
|
||||
if (maxTestSize >= srcBufferSize) maxTestSize = srcBufferSize-1;
|
||||
{ int const compressionLevel = (FUZ_rand(&lseed) % 5) + 1;
|
||||
DISPLAYLEVEL(5, "t%u : compression level : %i \n", testNb, compressionLevel);
|
||||
CHECK_Z (setCCtxParameter(zc, cctxParams, ZSTD_p_compressionLevel, compressionLevel, opaqueAPI) );
|
||||
CHECK_Z (setCCtxParameter(zc, cctxParams, ZSTD_c_compressionLevel, compressionLevel, opaqueAPI) );
|
||||
}
|
||||
} else {
|
||||
U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
|
||||
@@ -1858,45 +1851,45 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest,
|
||||
cParams.chainLog += (FUZ_rand(&lseed) & 3) - 1;
|
||||
cParams.searchLog += (FUZ_rand(&lseed) & 3) - 1;
|
||||
cParams.searchLog = MIN(searchLogMax, cParams.searchLog);
|
||||
cParams.searchLength += (FUZ_rand(&lseed) & 3) - 1;
|
||||
cParams.minMatch += (FUZ_rand(&lseed) & 3) - 1;
|
||||
cParams.targetLength = (U32)((cParams.targetLength + 1 ) * (0.5 + ((double)(FUZ_rand(&lseed) & 127) / 128)));
|
||||
cParams = ZSTD_adjustCParams(cParams, pledgedSrcSize, dictSize);
|
||||
|
||||
if (FUZ_rand(&lseed) & 1) {
|
||||
DISPLAYLEVEL(5, "t%u: windowLog : %u \n", testNb, cParams.windowLog);
|
||||
CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_windowLog, cParams.windowLog, opaqueAPI) );
|
||||
CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_windowLog, cParams.windowLog, opaqueAPI) );
|
||||
assert(cParams.windowLog >= ZSTD_WINDOWLOG_MIN); /* guaranteed by ZSTD_adjustCParams() */
|
||||
windowLogMalus = (cParams.windowLog - ZSTD_WINDOWLOG_MIN) / 5;
|
||||
}
|
||||
if (FUZ_rand(&lseed) & 1) {
|
||||
DISPLAYLEVEL(5, "t%u: hashLog : %u \n", testNb, cParams.hashLog);
|
||||
CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_hashLog, cParams.hashLog, opaqueAPI) );
|
||||
CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_hashLog, cParams.hashLog, opaqueAPI) );
|
||||
}
|
||||
if (FUZ_rand(&lseed) & 1) {
|
||||
DISPLAYLEVEL(5, "t%u: chainLog : %u \n", testNb, cParams.chainLog);
|
||||
CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_chainLog, cParams.chainLog, opaqueAPI) );
|
||||
CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_chainLog, cParams.chainLog, opaqueAPI) );
|
||||
}
|
||||
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_searchLog, cParams.searchLog, opaqueAPI) );
|
||||
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_minMatch, cParams.searchLength, opaqueAPI) );
|
||||
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_targetLength, cParams.targetLength, opaqueAPI) );
|
||||
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_searchLog, cParams.searchLog, opaqueAPI) );
|
||||
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_minMatch, cParams.minMatch, opaqueAPI) );
|
||||
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_targetLength, cParams.targetLength, opaqueAPI) );
|
||||
|
||||
/* mess with long distance matching parameters */
|
||||
if (bigTests) {
|
||||
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_enableLongDistanceMatching, FUZ_rand(&lseed) & 63, opaqueAPI) );
|
||||
if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmHashLog, FUZ_randomClampedLength(&lseed, ZSTD_HASHLOG_MIN, 23), opaqueAPI) );
|
||||
if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmMinMatch, FUZ_randomClampedLength(&lseed, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX), opaqueAPI) );
|
||||
if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmBucketSizeLog, FUZ_randomClampedLength(&lseed, 0, ZSTD_LDM_BUCKETSIZELOG_MAX), opaqueAPI) );
|
||||
if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmHashEveryLog, FUZ_randomClampedLength(&lseed, 0, ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN), opaqueAPI) );
|
||||
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_enableLongDistanceMatching, FUZ_rand(&lseed) & 63, opaqueAPI) );
|
||||
if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmHashLog, FUZ_randomClampedLength(&lseed, ZSTD_HASHLOG_MIN, 23), opaqueAPI) );
|
||||
if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmMinMatch, FUZ_randomClampedLength(&lseed, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX), opaqueAPI) );
|
||||
if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmBucketSizeLog, FUZ_randomClampedLength(&lseed, ZSTD_LDM_BUCKETSIZELOG_MIN, ZSTD_LDM_BUCKETSIZELOG_MAX), opaqueAPI) );
|
||||
if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmHashRateLog, FUZ_randomClampedLength(&lseed, ZSTD_LDM_HASHRATELOG_MIN, ZSTD_LDM_HASHRATELOG_MAX), opaqueAPI) );
|
||||
}
|
||||
|
||||
/* mess with frame parameters */
|
||||
if (FUZ_rand(&lseed) & 1) {
|
||||
U32 const checksumFlag = FUZ_rand(&lseed) & 1;
|
||||
DISPLAYLEVEL(5, "t%u: frame checksum : %u \n", testNb, checksumFlag);
|
||||
CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_checksumFlag, checksumFlag, opaqueAPI) );
|
||||
CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_checksumFlag, checksumFlag, opaqueAPI) );
|
||||
}
|
||||
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_dictIDFlag, FUZ_rand(&lseed) & 1, opaqueAPI) );
|
||||
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_contentSizeFlag, FUZ_rand(&lseed) & 1, opaqueAPI) );
|
||||
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_dictIDFlag, FUZ_rand(&lseed) & 1, opaqueAPI) );
|
||||
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_contentSizeFlag, FUZ_rand(&lseed) & 1, opaqueAPI) );
|
||||
if (FUZ_rand(&lseed) & 1) {
|
||||
DISPLAYLEVEL(5, "t%u: pledgedSrcSize : %u \n", testNb, (U32)pledgedSrcSize);
|
||||
CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
|
||||
@@ -1908,17 +1901,17 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest,
|
||||
U32 const nbThreadsAdjusted = (windowLogMalus < nbThreadsCandidate) ? nbThreadsCandidate - windowLogMalus : 1;
|
||||
U32 const nbThreads = MIN(nbThreadsAdjusted, nbThreadsMax);
|
||||
DISPLAYLEVEL(5, "t%u: nbThreads : %u \n", testNb, nbThreads);
|
||||
CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_nbWorkers, nbThreads, opaqueAPI) );
|
||||
CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_nbWorkers, nbThreads, opaqueAPI) );
|
||||
if (nbThreads > 1) {
|
||||
U32 const jobLog = FUZ_rand(&lseed) % (testLog+1);
|
||||
CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_overlapSizeLog, FUZ_rand(&lseed) % 10, opaqueAPI) );
|
||||
CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_jobSize, (U32)FUZ_rLogLength(&lseed, jobLog), opaqueAPI) );
|
||||
CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_overlapSizeLog, FUZ_rand(&lseed) % 10, opaqueAPI) );
|
||||
CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_jobSize, (U32)FUZ_rLogLength(&lseed, jobLog), opaqueAPI) );
|
||||
}
|
||||
}
|
||||
/* Enable rsyncable mode 1 in 4 times. */
|
||||
setCCtxParameter(zc, cctxParams, ZSTD_p_rsyncable, (FUZ_rand(&lseed) % 4 == 0), opaqueAPI);
|
||||
setCCtxParameter(zc, cctxParams, ZSTD_c_rsyncable, (FUZ_rand(&lseed) % 4 == 0), opaqueAPI);
|
||||
|
||||
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_forceMaxWindow, FUZ_rand(&lseed) & 1, opaqueAPI) );
|
||||
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_forceMaxWindow, FUZ_rand(&lseed) & 1, opaqueAPI) );
|
||||
|
||||
/* Apply parameters */
|
||||
if (opaqueAPI) {
|
||||
@@ -1938,7 +1931,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest,
|
||||
size_t const setError = ZSTD_CCtx_setParametersUsingCCtxParams(zc, cctxParams);
|
||||
CHECK(!ZSTD_isError(setError), "ZSTD_CCtx_setParametersUsingCCtxParams should have failed");
|
||||
} else {
|
||||
size_t const setError = ZSTD_CCtx_setParameter(zc, ZSTD_p_windowLog, cParams.windowLog-1);
|
||||
size_t const setError = ZSTD_CCtx_setParameter(zc, ZSTD_c_windowLog, cParams.windowLog-1);
|
||||
CHECK(!ZSTD_isError(setError), "ZSTD_CCtx_setParameter should have failed");
|
||||
}
|
||||
}
|
||||
@@ -1963,7 +1956,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest,
|
||||
ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 };
|
||||
outBuff.size = outBuff.pos + dstBuffSize;
|
||||
|
||||
CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, flush) );
|
||||
CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, flush) );
|
||||
DISPLAYLEVEL(6, "t%u: compress consumed %u bytes (total : %u) ; flush: %u (total : %u) \n",
|
||||
testNb, (U32)inBuff.pos, (U32)(totalTestSize + inBuff.pos), (U32)flush, (U32)outBuff.pos);
|
||||
|
||||
@@ -1980,10 +1973,10 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest,
|
||||
size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
|
||||
outBuff.size = outBuff.pos + adjustedDstSize;
|
||||
DISPLAYLEVEL(6, "t%u: End-flush into dst buffer of size %u \n", testNb, (U32)adjustedDstSize);
|
||||
remainingToFlush = ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end);
|
||||
remainingToFlush = ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end);
|
||||
DISPLAYLEVEL(6, "t%u: Total flushed so far : %u bytes \n", testNb, (U32)outBuff.pos);
|
||||
CHECK( ZSTD_isError(remainingToFlush),
|
||||
"ZSTD_compress_generic w/ ZSTD_e_end error : %s",
|
||||
"ZSTD_compressStream2 w/ ZSTD_e_end error : %s",
|
||||
ZSTD_getErrorName(remainingToFlush) );
|
||||
} }
|
||||
crcOrig = XXH64_digest(&xxhState);
|
||||
|
||||
Reference in New Issue
Block a user