Merge remote-tracking branch 'refs/remotes/Cyan4973/dev070' into dev070
This commit is contained in:
+1
-1
@@ -48,7 +48,7 @@ INCLUDEDIR=$(PREFIX)/include
|
|||||||
|
|
||||||
CPPFLAGS= -I./common -DXXH_NAMESPACE=ZSTD_
|
CPPFLAGS= -I./common -DXXH_NAMESPACE=ZSTD_
|
||||||
CFLAGS ?= -O3
|
CFLAGS ?= -O3
|
||||||
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wstrict-prototypes -Wundef
|
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef
|
||||||
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
|
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -32,9 +32,9 @@
|
|||||||
#ifndef ZSTD_BUFFERED_STATIC_H
|
#ifndef ZSTD_BUFFERED_STATIC_H
|
||||||
#define ZSTD_BUFFERED_STATIC_H
|
#define ZSTD_BUFFERED_STATIC_H
|
||||||
|
|
||||||
/* The objects defined into this file should be considered experimental.
|
/* The objects defined into this file must be considered experimental.
|
||||||
* They are not labelled stable, as their prototype may change in the future.
|
* Their prototype may change in future versions.
|
||||||
* You can use them for tests, provide feedback, or if you can endure risk of future changes.
|
* Never use them with a dynamic library.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
#if defined (__cplusplus)
|
#if defined (__cplusplus)
|
||||||
@@ -65,10 +65,13 @@ ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem);
|
|||||||
/* *************************************
|
/* *************************************
|
||||||
* Advanced Streaming functions
|
* Advanced Streaming functions
|
||||||
***************************************/
|
***************************************/
|
||||||
ZSTDLIB_API size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* cctx,
|
ZSTDLIB_API size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
|
||||||
const void* dict, size_t dictSize,
|
const void* dict, size_t dictSize,
|
||||||
ZSTD_parameters params, U64 pledgedSrcSize);
|
ZSTD_parameters params, U64 pledgedSrcSize);
|
||||||
|
|
||||||
|
|
||||||
|
/* internal util function */
|
||||||
|
|
||||||
MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||||
{
|
{
|
||||||
size_t length = ZBUFF_MIN(dstCapacity, srcSize);
|
size_t length = ZBUFF_MIN(dstCapacity, srcSize);
|
||||||
|
|||||||
@@ -128,7 +128,7 @@ ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
|
|||||||
/*! ZSTD_adjustParams() :
|
/*! ZSTD_adjustParams() :
|
||||||
* optimize params for a given `srcSize` and `dictSize`.
|
* optimize params for a given `srcSize` and `dictSize`.
|
||||||
* both values are optional, select `0` if unknown. */
|
* both values are optional, select `0` if unknown. */
|
||||||
ZSTDLIB_API void ZSTD_adjustCParams(ZSTD_compressionParameters* params, U64 srcSize, size_t dictSize);
|
ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, U64 srcSize, size_t dictSize);
|
||||||
|
|
||||||
/*! ZSTD_compress_advanced() :
|
/*! ZSTD_compress_advanced() :
|
||||||
* Same as ZSTD_compress_usingDict(), with fine-tune control of each compression parameter */
|
* Same as ZSTD_compress_usingDict(), with fine-tune control of each compression parameter */
|
||||||
|
|||||||
@@ -179,8 +179,6 @@ size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* zbc, const void* dict, size_t di
|
|||||||
ZSTD_parameters params;
|
ZSTD_parameters params;
|
||||||
memset(¶ms, 0, sizeof(params));
|
memset(¶ms, 0, sizeof(params));
|
||||||
params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize);
|
params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize);
|
||||||
params.fParams.contentSizeFlag = 0;
|
|
||||||
ZSTD_adjustCParams(¶ms.cParams, 0, dictSize);
|
|
||||||
return ZBUFF_compressInit_advanced(zbc, dict, dictSize, params, 0);
|
return ZBUFF_compressInit_advanced(zbc, dict, dictSize, params, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -197,30 +197,32 @@ size_t ZSTD_checkCParams_advanced(ZSTD_compressionParameters cParams, U64 srcSiz
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/** ZSTD_adjustParams() :
|
/** ZSTD_adjustCParams() :
|
||||||
optimize params for q given input (`srcSize` and `dictSize`).
|
optimize cPar for a given input (`srcSize` and `dictSize`).
|
||||||
mostly downsizing to reduce memory consumption and initialization.
|
mostly downsizing to reduce memory consumption and initialization.
|
||||||
Both `srcSize` and `dictSize` are optional (use 0 if unknown),
|
Both `srcSize` and `dictSize` are optional (use 0 if unknown),
|
||||||
but if both are 0, no optimization can be done.
|
but if both are 0, no optimization can be done.
|
||||||
Note : params is considered validated at this stage. Use ZSTD_checkParams() to ensure that. */
|
Note : cPar is considered validated at this stage. Use ZSTD_checkParams() to ensure that. */
|
||||||
void ZSTD_adjustCParams(ZSTD_compressionParameters* params, U64 srcSize, size_t dictSize)
|
ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, U64 srcSize, size_t dictSize)
|
||||||
{
|
{
|
||||||
if (srcSize+dictSize == 0) return; /* no size information available : no adjustment */
|
if (srcSize+dictSize == 0) return cPar; /* no size information available : no adjustment */
|
||||||
|
|
||||||
/* resize params, to use less memory when necessary */
|
/* resize params, to use less memory when necessary */
|
||||||
{ U32 const minSrcSize = (srcSize==0) ? 500 : 0;
|
{ U32 const minSrcSize = (srcSize==0) ? 500 : 0;
|
||||||
U64 const rSize = srcSize + dictSize + minSrcSize;
|
U64 const rSize = srcSize + dictSize + minSrcSize;
|
||||||
if (rSize < ((U64)1<<ZSTD_WINDOWLOG_MAX)) {
|
if (rSize < ((U64)1<<ZSTD_WINDOWLOG_MAX)) {
|
||||||
U32 const srcLog = ZSTD_highbit((U32)(rSize)-1) + 1;
|
U32 const srcLog = ZSTD_highbit((U32)(rSize)-1) + 1;
|
||||||
if (params->windowLog > srcLog) params->windowLog = srcLog;
|
if (cPar.windowLog > srcLog) cPar.windowLog = srcLog;
|
||||||
} }
|
} }
|
||||||
if (params->hashLog > params->windowLog) params->hashLog = params->windowLog;
|
if (cPar.hashLog > cPar.windowLog) cPar.hashLog = cPar.windowLog;
|
||||||
{ U32 const btPlus = (params->strategy == ZSTD_btlazy2) || (params->strategy == ZSTD_btopt);
|
{ U32 const btPlus = (cPar.strategy == ZSTD_btlazy2) || (cPar.strategy == ZSTD_btopt);
|
||||||
U32 const maxChainLog = params->windowLog+btPlus;
|
U32 const maxChainLog = cPar.windowLog+btPlus;
|
||||||
if (params->chainLog > maxChainLog) params->chainLog = maxChainLog; } /* <= ZSTD_CHAINLOG_MAX */
|
if (cPar.chainLog > maxChainLog) cPar.chainLog = maxChainLog; } /* <= ZSTD_CHAINLOG_MAX */
|
||||||
|
|
||||||
if (params->windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) params->windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */
|
if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */
|
||||||
if ((params->hashLog < ZSTD_HASHLOG_MIN) && ((U32)params->strategy >= (U32)ZSTD_btlazy2)) params->hashLog = ZSTD_HASHLOG_MIN; /* required to ensure collision resistance in bt */
|
if ((cPar.hashLog < ZSTD_HASHLOG_MIN) && ( (U32)cPar.strategy >= (U32)ZSTD_btlazy2)) cPar.hashLog = ZSTD_HASHLOG_MIN; /* required to ensure collision resistance in bt */
|
||||||
|
|
||||||
|
return cPar;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -2356,7 +2358,6 @@ size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* zc, const void* dict, size_t dict
|
|||||||
ZSTD_parameters params;
|
ZSTD_parameters params;
|
||||||
memset(¶ms, 0, sizeof(params));
|
memset(¶ms, 0, sizeof(params));
|
||||||
params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize);
|
params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize);
|
||||||
ZSTD_adjustCParams(¶ms.cParams, 0, dictSize);
|
|
||||||
ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin_usingDict compressionLevel=%d\n", zc->base, compressionLevel);
|
ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin_usingDict compressionLevel=%d\n", zc->base, compressionLevel);
|
||||||
return ZSTD_compressBegin_internal(zc, dict, dictSize, params, 0);
|
return ZSTD_compressBegin_internal(zc, dict, dictSize, params, 0);
|
||||||
}
|
}
|
||||||
@@ -2466,7 +2467,6 @@ size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, co
|
|||||||
ZSTD_LOG_BLOCK("%p: ZSTD_compress_usingDict srcSize=%d dictSize=%d compressionLevel=%d\n", ctx->base, (int)srcSize, (int)dictSize, compressionLevel);
|
ZSTD_LOG_BLOCK("%p: ZSTD_compress_usingDict srcSize=%d dictSize=%d compressionLevel=%d\n", ctx->base, (int)srcSize, (int)dictSize, compressionLevel);
|
||||||
params.cParams = ZSTD_getCParams(compressionLevel, srcSize, dictSize);
|
params.cParams = ZSTD_getCParams(compressionLevel, srcSize, dictSize);
|
||||||
params.fParams.contentSizeFlag = 1;
|
params.fParams.contentSizeFlag = 1;
|
||||||
ZSTD_adjustCParams(¶ms.cParams, srcSize, dictSize);
|
|
||||||
return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params);
|
return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2618,5 +2618,6 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, si
|
|||||||
if (cp.chainLog > ZSTD_CHAINLOG_MAX) cp.chainLog = ZSTD_CHAINLOG_MAX;
|
if (cp.chainLog > ZSTD_CHAINLOG_MAX) cp.chainLog = ZSTD_CHAINLOG_MAX;
|
||||||
if (cp.hashLog > ZSTD_HASHLOG_MAX) cp.hashLog = ZSTD_HASHLOG_MAX;
|
if (cp.hashLog > ZSTD_HASHLOG_MAX) cp.hashLog = ZSTD_HASHLOG_MAX;
|
||||||
}
|
}
|
||||||
|
cp = ZSTD_adjustCParams(cp, srcSize, dictSize);
|
||||||
return cp;
|
return cp;
|
||||||
}
|
}
|
||||||
|
|||||||
+1
-1
@@ -40,7 +40,7 @@ ZSTDDIR = ../lib
|
|||||||
|
|
||||||
CPPFLAGS= -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder -DXXH_NAMESPACE=ZSTD_
|
CPPFLAGS= -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder -DXXH_NAMESPACE=ZSTD_
|
||||||
CFLAGS ?= -O3 # -falign-loops=32 # not always beneficial
|
CFLAGS ?= -O3 # -falign-loops=32 # not always beneficial
|
||||||
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wstrict-prototypes -Wundef
|
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef
|
||||||
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
|
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -220,7 +220,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
|||||||
{ ZSTD_parameters params;
|
{ ZSTD_parameters params;
|
||||||
params.cParams = ZSTD_getCParams(cLevel, blockSize, dictBufferSize);
|
params.cParams = ZSTD_getCParams(cLevel, blockSize, dictBufferSize);
|
||||||
params.fParams.contentSizeFlag = 1;
|
params.fParams.contentSizeFlag = 1;
|
||||||
ZSTD_adjustCParams(¶ms.cParams, blockSize, dictBufferSize);
|
|
||||||
{ size_t const initResult = ZSTD_compressBegin_advanced(refCtx, dictBuffer, dictBufferSize, params, blockSize);
|
{ size_t const initResult = ZSTD_compressBegin_advanced(refCtx, dictBuffer, dictBufferSize, params, blockSize);
|
||||||
if (ZSTD_isError(initResult)) break;
|
if (ZSTD_isError(initResult)) break;
|
||||||
} }
|
} }
|
||||||
@@ -505,4 +504,3 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
|
|||||||
BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast);
|
BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast);
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -135,6 +135,8 @@ static U32 g_sparseFileSupport = 1; /* 0 : no sparse allowed; 1: auto (file ye
|
|||||||
void FIO_setSparseWrite(unsigned sparse) { g_sparseFileSupport=sparse; }
|
void FIO_setSparseWrite(unsigned sparse) { g_sparseFileSupport=sparse; }
|
||||||
static U32 g_dictIDFlag = 1;
|
static U32 g_dictIDFlag = 1;
|
||||||
void FIO_setDictIDFlag(unsigned dictIDFlag) { g_dictIDFlag = dictIDFlag; }
|
void FIO_setDictIDFlag(unsigned dictIDFlag) { g_dictIDFlag = dictIDFlag; }
|
||||||
|
static U32 g_checksumFlag = 0;
|
||||||
|
void FIO_setChecksumFlag(unsigned checksumFlag) { g_checksumFlag = checksumFlag; }
|
||||||
|
|
||||||
|
|
||||||
/*-*************************************
|
/*-*************************************
|
||||||
@@ -313,6 +315,7 @@ static int FIO_compressFilename_internal(cRess_t ress,
|
|||||||
memset(¶ms, 0, sizeof(params));
|
memset(¶ms, 0, sizeof(params));
|
||||||
params.cParams = ZSTD_getCParams(cLevel, fileSize, ress.dictBufferSize);
|
params.cParams = ZSTD_getCParams(cLevel, fileSize, ress.dictBufferSize);
|
||||||
params.fParams.contentSizeFlag = 1;
|
params.fParams.contentSizeFlag = 1;
|
||||||
|
params.fParams.checksumFlag = g_checksumFlag;
|
||||||
params.fParams.noDictIDFlag = !g_dictIDFlag;
|
params.fParams.noDictIDFlag = !g_dictIDFlag;
|
||||||
if (g_maxWLog) if (params.cParams.windowLog > g_maxWLog) params.cParams.windowLog = g_maxWLog;
|
if (g_maxWLog) if (params.cParams.windowLog > g_maxWLog) params.cParams.windowLog = g_maxWLog;
|
||||||
{ size_t const errorCode = ZBUFF_compressInit_advanced(ress.ctx, ress.dictBuffer, ress.dictBufferSize, params, fileSize);
|
{ size_t const errorCode = ZBUFF_compressInit_advanced(ress.ctx, ress.dictBuffer, ress.dictBufferSize, params, fileSize);
|
||||||
|
|||||||
@@ -49,6 +49,7 @@ void FIO_setNotificationLevel(unsigned level);
|
|||||||
void FIO_setMaxWLog(unsigned maxWLog); /**< if `maxWLog` == 0, no max enforced */
|
void FIO_setMaxWLog(unsigned maxWLog); /**< if `maxWLog` == 0, no max enforced */
|
||||||
void FIO_setSparseWrite(unsigned sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */
|
void FIO_setSparseWrite(unsigned sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */
|
||||||
void FIO_setDictIDFlag(unsigned dictIDFlag);
|
void FIO_setDictIDFlag(unsigned dictIDFlag);
|
||||||
|
void FIO_setChecksumFlag(unsigned checksumFlag);
|
||||||
|
|
||||||
|
|
||||||
/*-*************************************
|
/*-*************************************
|
||||||
|
|||||||
+15
-17
@@ -600,22 +600,22 @@ static void playAround(FILE* f, winnerInfo_t* winners,
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
static void potentialRandomParams(ZSTD_compressionParameters* p, U32 inverseChance)
|
static ZSTD_compressionParameters randomParams(void)
|
||||||
{
|
{
|
||||||
U32 chance = (FUZ_rand(&g_rand) % (inverseChance+1));
|
ZSTD_compressionParameters p;
|
||||||
U32 validated = 0;
|
U32 validated = 0;
|
||||||
if (!chance)
|
|
||||||
while (!validated) {
|
while (!validated) {
|
||||||
/* totally random entry */
|
/* totally random entry */
|
||||||
p->chainLog = FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN) + ZSTD_CHAINLOG_MIN;
|
p.chainLog = FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN) + ZSTD_CHAINLOG_MIN;
|
||||||
p->hashLog = FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN) + ZSTD_HASHLOG_MIN;
|
p.hashLog = FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN) + ZSTD_HASHLOG_MIN;
|
||||||
p->searchLog = FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN) + ZSTD_SEARCHLOG_MIN;
|
p.searchLog = FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN) + ZSTD_SEARCHLOG_MIN;
|
||||||
p->windowLog = FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN) + ZSTD_WINDOWLOG_MIN;
|
p.windowLog = FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN) + ZSTD_WINDOWLOG_MIN;
|
||||||
p->searchLength=FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN) + ZSTD_SEARCHLENGTH_MIN;
|
p.searchLength=FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN) + ZSTD_SEARCHLENGTH_MIN;
|
||||||
p->targetLength=FUZ_rand(&g_rand) % (ZSTD_TARGETLENGTH_MAX+1 - ZSTD_TARGETLENGTH_MIN) + ZSTD_TARGETLENGTH_MIN;
|
p.targetLength=FUZ_rand(&g_rand) % (ZSTD_TARGETLENGTH_MAX+1 - ZSTD_TARGETLENGTH_MIN) + ZSTD_TARGETLENGTH_MIN;
|
||||||
p->strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btopt +1));
|
p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btopt +1));
|
||||||
validated = !ZSTD_isError(ZSTD_checkCParams(*p));
|
validated = !ZSTD_isError(ZSTD_checkCParams(p));
|
||||||
}
|
}
|
||||||
|
return p;
|
||||||
}
|
}
|
||||||
|
|
||||||
static void BMK_selectRandomStart(
|
static void BMK_selectRandomStart(
|
||||||
@@ -623,12 +623,10 @@ static void BMK_selectRandomStart(
|
|||||||
const void* srcBuffer, size_t srcSize,
|
const void* srcBuffer, size_t srcSize,
|
||||||
ZSTD_CCtx* ctx)
|
ZSTD_CCtx* ctx)
|
||||||
{
|
{
|
||||||
U32 id = (FUZ_rand(&g_rand) % (ZSTD_maxCLevel()+1));
|
U32 const id = (FUZ_rand(&g_rand) % (ZSTD_maxCLevel()+1));
|
||||||
if ((id==0) || (winners[id].params.windowLog==0)) {
|
if ((id==0) || (winners[id].params.windowLog==0)) {
|
||||||
/* totally random entry */
|
/* totally random entry */
|
||||||
ZSTD_compressionParameters p;
|
ZSTD_compressionParameters const p = ZSTD_adjustCParams(randomParams(), srcSize, 0);
|
||||||
potentialRandomParams(&p, 1);
|
|
||||||
ZSTD_adjustCParams(&p, srcSize, 0);
|
|
||||||
playAround(f, winners, p, srcBuffer, srcSize, ctx);
|
playAround(f, winners, p, srcBuffer, srcSize, ctx);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
@@ -649,7 +647,7 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize)
|
|||||||
|
|
||||||
if (g_singleRun) {
|
if (g_singleRun) {
|
||||||
BMK_result_t testResult;
|
BMK_result_t testResult;
|
||||||
ZSTD_adjustCParams(&g_params, srcSize, 0);
|
g_params = ZSTD_adjustCParams(g_params, srcSize, 0);
|
||||||
BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, g_params);
|
BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, g_params);
|
||||||
DISPLAY("\n");
|
DISPLAY("\n");
|
||||||
return;
|
return;
|
||||||
@@ -861,7 +859,7 @@ int optimizeForSize(char* inFileName)
|
|||||||
do {
|
do {
|
||||||
params = winner.params;
|
params = winner.params;
|
||||||
paramVariation(¶ms);
|
paramVariation(¶ms);
|
||||||
potentialRandomParams(¶ms, 16);
|
if ((FUZ_rand(&g_rand) & 15) == 1) params = randomParams();
|
||||||
|
|
||||||
/* exclude faster if already played set of params */
|
/* exclude faster if already played set of params */
|
||||||
if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(params))-1)) continue;
|
if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(params))-1)) continue;
|
||||||
|
|||||||
+15
-9
@@ -348,8 +348,8 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
|
|||||||
FUZ_rand(&coreSeed);
|
FUZ_rand(&coreSeed);
|
||||||
lseed = coreSeed ^ prime1;
|
lseed = coreSeed ^ prime1;
|
||||||
|
|
||||||
/* state total reset */
|
/* states full reset (unsynchronized) */
|
||||||
/* some problems only happen when states are re-used in a specific order */
|
/* some issues only happen when reusing states in a specific sequence of parameters */
|
||||||
if ((FUZ_rand(&lseed) & 0xFF) == 131) { ZBUFF_freeCCtx(zc); zc = ZBUFF_createCCtx(); }
|
if ((FUZ_rand(&lseed) & 0xFF) == 131) { ZBUFF_freeCCtx(zc); zc = ZBUFF_createCCtx(); }
|
||||||
if ((FUZ_rand(&lseed) & 0xFF) == 132) { ZBUFF_freeDCtx(zd); zd = ZBUFF_createDCtx(); }
|
if ((FUZ_rand(&lseed) & 0xFF) == 132) { ZBUFF_freeDCtx(zd); zd = ZBUFF_createDCtx(); }
|
||||||
|
|
||||||
@@ -372,15 +372,21 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
|
|||||||
{ U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
|
{ U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
|
||||||
U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (testLog/3))) + 1;
|
U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (testLog/3))) + 1;
|
||||||
maxTestSize = FUZ_rLogLength(&lseed, testLog);
|
maxTestSize = FUZ_rLogLength(&lseed, testLog);
|
||||||
|
dictSize = (FUZ_rand(&lseed)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0;
|
||||||
/* random dictionary selection */
|
/* random dictionary selection */
|
||||||
{ size_t dictStart;
|
{ size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
|
||||||
dictSize = (FUZ_rand(&lseed)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0;
|
dict = srcBuffer + dictStart;
|
||||||
dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
|
|
||||||
dict = srcBuffer + dictStart;
|
|
||||||
}
|
}
|
||||||
{ size_t const initError = ZBUFF_compressInitDictionary(zc, dict, dictSize, cLevel);
|
{ ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, 0, dictSize);
|
||||||
CHECK (ZBUFF_isError(initError),"init error : %s", ZBUFF_getErrorName(initError));
|
U32 const checksum = FUZ_rand(&lseed) & 1;
|
||||||
} }
|
U32 const noDictIDFlag = FUZ_rand(&lseed) & 1;
|
||||||
|
ZSTD_frameParameters const fPar = { 0, checksum, noDictIDFlag };
|
||||||
|
ZSTD_parameters params;
|
||||||
|
params.cParams = cPar;
|
||||||
|
params.fParams = fPar;
|
||||||
|
{ size_t const initError = ZBUFF_compressInit_advanced(zc, dict, dictSize, params, 0);
|
||||||
|
CHECK (ZBUFF_isError(initError),"init error : %s", ZBUFF_getErrorName(initError));
|
||||||
|
} } }
|
||||||
|
|
||||||
/* multi-segments compression test */
|
/* multi-segments compression test */
|
||||||
XXH64_reset(&xxhState, 0);
|
XXH64_reset(&xxhState, 0);
|
||||||
|
|||||||
+8
-1
@@ -52,7 +52,7 @@ It also features a very fast decoder, with speed > 500 MB/s per core.
|
|||||||
use `file` as Dictionary to compress or decompress FILE(s)
|
use `file` as Dictionary to compress or decompress FILE(s)
|
||||||
.TP
|
.TP
|
||||||
.B \-o file
|
.B \-o file
|
||||||
save result into `file` (only possible with a single input FILE)
|
save result into `file` (only possible with a single INPUT-FILE)
|
||||||
.TP
|
.TP
|
||||||
.BR \-f ", " --force
|
.BR \-f ", " --force
|
||||||
overwrite output without prompting
|
overwrite output without prompting
|
||||||
@@ -71,6 +71,13 @@ It also features a very fast decoder, with speed > 500 MB/s per core.
|
|||||||
.TP
|
.TP
|
||||||
.BR \-c ", " --stdout
|
.BR \-c ", " --stdout
|
||||||
force write to standard output, even if it is the console
|
force write to standard output, even if it is the console
|
||||||
|
.TP
|
||||||
|
.BR \-C ", " --check
|
||||||
|
add integrity check computed from uncompressed data
|
||||||
|
.TP
|
||||||
|
.BR \-t ", " --test
|
||||||
|
Test the integrity of compressed files. This option is equivalent to \fB--decompress --stdout > /dev/null\fR.
|
||||||
|
No files are created or removed.
|
||||||
|
|
||||||
.SH DICTIONARY
|
.SH DICTIONARY
|
||||||
.PP
|
.PP
|
||||||
|
|||||||
+6
-1
@@ -134,9 +134,10 @@ static int usage_advanced(const char* programName)
|
|||||||
#ifndef ZSTD_NOCOMPRESS
|
#ifndef ZSTD_NOCOMPRESS
|
||||||
DISPLAY( "--ultra : enable ultra modes (requires more memory to decompress)\n");
|
DISPLAY( "--ultra : enable ultra modes (requires more memory to decompress)\n");
|
||||||
DISPLAY( "--no-dictID:don't write dictID into header (dictionary compression)\n");
|
DISPLAY( "--no-dictID:don't write dictID into header (dictionary compression)\n");
|
||||||
|
DISPLAY( "--check : enable integrity check\n");
|
||||||
#endif
|
#endif
|
||||||
#ifndef ZSTD_NODECOMPRESS
|
#ifndef ZSTD_NODECOMPRESS
|
||||||
DISPLAY( " -t : test compressed file integrity \n");
|
DISPLAY( "--test : test compressed file integrity \n");
|
||||||
DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n");
|
DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n");
|
||||||
#endif
|
#endif
|
||||||
#ifndef ZSTD_NODICT
|
#ifndef ZSTD_NODICT
|
||||||
@@ -240,6 +241,7 @@ int main(int argCount, const char** argv)
|
|||||||
if (!strcmp(argument, "--quiet")) { displayLevel--; continue; }
|
if (!strcmp(argument, "--quiet")) { displayLevel--; continue; }
|
||||||
if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; displayLevel=1; continue; }
|
if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; displayLevel=1; continue; }
|
||||||
if (!strcmp(argument, "--ultra")) { FIO_setMaxWLog(0); continue; }
|
if (!strcmp(argument, "--ultra")) { FIO_setMaxWLog(0); continue; }
|
||||||
|
if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(2); continue; }
|
||||||
if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; }
|
if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; }
|
||||||
if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; }
|
if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; }
|
||||||
if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; }
|
if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; }
|
||||||
@@ -303,6 +305,9 @@ int main(int argCount, const char** argv)
|
|||||||
/* keep source file (default anyway, so useless; for gzip/xz compatibility) */
|
/* keep source file (default anyway, so useless; for gzip/xz compatibility) */
|
||||||
case 'k': argument++; break;
|
case 'k': argument++; break;
|
||||||
|
|
||||||
|
/* Checksum */
|
||||||
|
case 'C': argument++; FIO_setChecksumFlag(2); break;
|
||||||
|
|
||||||
/* test compressed file */
|
/* test compressed file */
|
||||||
case 't': decode=1; outFileName=nulmark; argument++; break;
|
case 't': decode=1; outFileName=nulmark; argument++; break;
|
||||||
|
|
||||||
|
|||||||
@@ -1 +1,2 @@
|
|||||||
build
|
build
|
||||||
|
*Copy
|
||||||
|
|||||||
@@ -92,80 +92,6 @@
|
|||||||
Name="VCPostBuildEventTool"
|
Name="VCPostBuildEventTool"
|
||||||
/>
|
/>
|
||||||
</Configuration>
|
</Configuration>
|
||||||
<Configuration
|
|
||||||
Name="Debug|x64"
|
|
||||||
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
|
||||||
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
|
|
||||||
ConfigurationType="1"
|
|
||||||
CharacterSet="2"
|
|
||||||
>
|
|
||||||
<Tool
|
|
||||||
Name="VCPreBuildEventTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCCustomBuildTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCXMLDataGeneratorTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCWebServiceProxyGeneratorTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCMIDLTool"
|
|
||||||
TargetEnvironment="3"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCCLCompilerTool"
|
|
||||||
Optimization="0"
|
|
||||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
|
|
||||||
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
|
|
||||||
MinimalRebuild="true"
|
|
||||||
BasicRuntimeChecks="3"
|
|
||||||
RuntimeLibrary="3"
|
|
||||||
UsePrecompiledHeader="0"
|
|
||||||
WarningLevel="4"
|
|
||||||
WarnAsError="true"
|
|
||||||
DebugInformationFormat="3"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCManagedResourceCompilerTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCResourceCompilerTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCPreLinkEventTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCLinkerTool"
|
|
||||||
LinkIncremental="2"
|
|
||||||
GenerateDebugInformation="true"
|
|
||||||
SubSystem="1"
|
|
||||||
TargetMachine="17"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCALinkTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCManifestTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCXDCMakeTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCBscMakeTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCFxCopTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCAppVerifierTool"
|
|
||||||
/>
|
|
||||||
<Tool
|
|
||||||
Name="VCPostBuildEventTool"
|
|
||||||
/>
|
|
||||||
</Configuration>
|
|
||||||
<Configuration
|
<Configuration
|
||||||
Name="Release|Win32"
|
Name="Release|Win32"
|
||||||
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
||||||
@@ -242,6 +168,80 @@
|
|||||||
Name="VCPostBuildEventTool"
|
Name="VCPostBuildEventTool"
|
||||||
/>
|
/>
|
||||||
</Configuration>
|
</Configuration>
|
||||||
|
<Configuration
|
||||||
|
Name="Debug|x64"
|
||||||
|
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
||||||
|
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
|
||||||
|
ConfigurationType="1"
|
||||||
|
CharacterSet="2"
|
||||||
|
>
|
||||||
|
<Tool
|
||||||
|
Name="VCPreBuildEventTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCCustomBuildTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCXMLDataGeneratorTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCWebServiceProxyGeneratorTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCMIDLTool"
|
||||||
|
TargetEnvironment="3"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCCLCompilerTool"
|
||||||
|
Optimization="0"
|
||||||
|
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
|
||||||
|
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
|
||||||
|
MinimalRebuild="true"
|
||||||
|
BasicRuntimeChecks="3"
|
||||||
|
RuntimeLibrary="3"
|
||||||
|
UsePrecompiledHeader="0"
|
||||||
|
WarningLevel="4"
|
||||||
|
WarnAsError="true"
|
||||||
|
DebugInformationFormat="3"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCManagedResourceCompilerTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCResourceCompilerTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCPreLinkEventTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCLinkerTool"
|
||||||
|
LinkIncremental="2"
|
||||||
|
GenerateDebugInformation="true"
|
||||||
|
SubSystem="1"
|
||||||
|
TargetMachine="17"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCALinkTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCManifestTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCXDCMakeTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCBscMakeTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCFxCopTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCAppVerifierTool"
|
||||||
|
/>
|
||||||
|
<Tool
|
||||||
|
Name="VCPostBuildEventTool"
|
||||||
|
/>
|
||||||
|
</Configuration>
|
||||||
<Configuration
|
<Configuration
|
||||||
Name="Release|x64"
|
Name="Release|x64"
|
||||||
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
||||||
@@ -360,6 +360,10 @@
|
|||||||
RelativePath="..\..\..\lib\decompress\huf_decompress.c"
|
RelativePath="..\..\..\lib\decompress\huf_decompress.c"
|
||||||
>
|
>
|
||||||
</File>
|
</File>
|
||||||
|
<File
|
||||||
|
RelativePath="..\..\..\lib\common\xxhash.c"
|
||||||
|
>
|
||||||
|
</File>
|
||||||
<File
|
<File
|
||||||
RelativePath="..\..\..\lib\compress\zbuff_compress.c"
|
RelativePath="..\..\..\lib\compress\zbuff_compress.c"
|
||||||
>
|
>
|
||||||
@@ -422,6 +426,10 @@
|
|||||||
RelativePath="..\..\..\lib\common\mem.h"
|
RelativePath="..\..\..\lib\common\mem.h"
|
||||||
>
|
>
|
||||||
</File>
|
</File>
|
||||||
|
<File
|
||||||
|
RelativePath="..\..\..\lib\common\xxhash.h"
|
||||||
|
>
|
||||||
|
</File>
|
||||||
<File
|
<File
|
||||||
RelativePath="..\..\..\lib\common\zbuff_static.h"
|
RelativePath="..\..\..\lib\common\zbuff_static.h"
|
||||||
>
|
>
|
||||||
|
|||||||
@@ -157,6 +157,7 @@
|
|||||||
</ItemDefinitionGroup>
|
</ItemDefinitionGroup>
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<ClCompile Include="..\..\..\lib\common\entropy_common.c" />
|
<ClCompile Include="..\..\..\lib\common\entropy_common.c" />
|
||||||
|
<ClCompile Include="..\..\..\lib\common\xxhash.c" />
|
||||||
<ClCompile Include="..\..\..\lib\common\zstd_common.c" />
|
<ClCompile Include="..\..\..\lib\common\zstd_common.c" />
|
||||||
<ClCompile Include="..\..\..\lib\common\xxhash.c" />
|
<ClCompile Include="..\..\..\lib\common\xxhash.c" />
|
||||||
<ClCompile Include="..\..\..\lib\common\fse_decompress.c" />
|
<ClCompile Include="..\..\..\lib\common\fse_decompress.c" />
|
||||||
|
|||||||
Reference in New Issue
Block a user