From c30d8d50da018fadd1120c183505304470a8997a Mon Sep 17 00:00:00 2001 From: cyan Date: Wed, 1 Jun 2016 09:31:34 +0200 Subject: [PATCH 1/7] fixed fullbench visual project --- projects/.gitignore | 1 + projects/VS2008/fullbench/fullbench.vcproj | 156 ++++++++++---------- projects/VS2010/fullbench/fullbench.vcxproj | 2 + 3 files changed, 85 insertions(+), 74 deletions(-) diff --git a/projects/.gitignore b/projects/.gitignore index 378eac25d..dc2035158 100644 --- a/projects/.gitignore +++ b/projects/.gitignore @@ -1 +1,2 @@ build +*Copy diff --git a/projects/VS2008/fullbench/fullbench.vcproj b/projects/VS2008/fullbench/fullbench.vcproj index 6ab878cc9..f55602022 100644 --- a/projects/VS2008/fullbench/fullbench.vcproj +++ b/projects/VS2008/fullbench/fullbench.vcproj @@ -92,80 +92,6 @@ Name="VCPostBuildEventTool" /> - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + @@ -414,6 +418,10 @@ RelativePath="..\..\..\lib\common\mem.h" > + + diff --git a/projects/VS2010/fullbench/fullbench.vcxproj b/projects/VS2010/fullbench/fullbench.vcxproj index 2d9b78ce1..150faa8d6 100644 --- a/projects/VS2010/fullbench/fullbench.vcxproj +++ b/projects/VS2010/fullbench/fullbench.vcxproj @@ -157,6 +157,7 @@ + @@ -174,6 +175,7 @@ + From 83c3f4427c4bb42188c6abc29cd0dc2b415161b3 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 1 Jun 2016 17:44:53 +0200 Subject: [PATCH 2/7] upgraded zbufftest to also test advanced frame parameters no/checksum no/dictID --- lib/common/zbuff_static.h | 15 +++++++++------ programs/zbufftest.c | 27 +++++++++++++++++---------- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/lib/common/zbuff_static.h b/lib/common/zbuff_static.h index c50a3e166..44a0321d1 100644 --- a/lib/common/zbuff_static.h +++ b/lib/common/zbuff_static.h @@ -32,9 +32,9 @@ #ifndef ZSTD_BUFFERED_STATIC_H #define ZSTD_BUFFERED_STATIC_H -/* The objects defined into this file should be considered experimental. - * They are not labelled stable, as their prototype may change in the future. - * You can use them for tests, provide feedback, or if you can endure risk of future changes. +/* The objects defined into this file must be considered experimental. + * Their prototype may change in future versions. + * Never use them with a dynamic library. */ #if defined (__cplusplus) @@ -54,21 +54,24 @@ extern "C" { * Advanced functions ***************************************/ /*! ZBUFF_createCCtx_advanced() : - * Create a ZBUFF compression context using external alloc and free functions */ + * Create a ZBUFF compression context using external alloc and free functions */ ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem); /*! ZBUFF_createDCtx_advanced() : - * Create a ZBUFF decompression context using external alloc and free functions */ + * Create a ZBUFF decompression context using external alloc and free functions */ ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem); /* ************************************* * Advanced Streaming functions ***************************************/ -ZSTDLIB_API size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* cctx, +ZSTDLIB_API size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, ZSTD_parameters params, U64 pledgedSrcSize); + +/* internal util function */ + MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) { size_t length = ZBUFF_MIN(dstCapacity, srcSize); diff --git a/programs/zbufftest.c b/programs/zbufftest.c index 19a385b42..166922970 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -212,7 +212,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo /* Byte-by-byte decompression test */ DISPLAYLEVEL(4, "test%3i : decompress byte-by-byte : ", testNb++); { size_t r, pIn=0, pOut=0; - do + do { ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB); r = 1; while (r) { @@ -346,8 +346,8 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres FUZ_rand(&coreSeed); lseed = coreSeed ^ prime1; - /* state total reset */ - /* some problems only happen when states are re-used in a specific order */ + /* states full reset (unsynchronized) */ + /* some issues only happen when reusing states in a specific sequence of parameters */ if ((FUZ_rand(&lseed) & 0xFF) == 131) { ZBUFF_freeCCtx(zc); zc = ZBUFF_createCCtx(); } if ((FUZ_rand(&lseed) & 0xFF) == 132) { ZBUFF_freeDCtx(zd); zd = ZBUFF_createDCtx(); } @@ -370,15 +370,22 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres { U32 const testLog = FUZ_rand(&lseed) % maxSrcLog; U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (testLog/3))) + 1; maxTestSize = FUZ_rLogLength(&lseed, testLog); + dictSize = (FUZ_rand(&lseed)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0; /* random dictionary selection */ - { size_t dictStart; - dictSize = (FUZ_rand(&lseed)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0; - dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize); - dict = srcBuffer + dictStart; + { size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize); + dict = srcBuffer + dictStart; } - { size_t const initError = ZBUFF_compressInitDictionary(zc, dict, dictSize, cLevel); - CHECK (ZBUFF_isError(initError),"init error : %s", ZBUFF_getErrorName(initError)); - } } + { ZSTD_compressionParameters cPar = ZSTD_getCParams(cLevel, 0, dictSize); + U32 const checksum = FUZ_rand(&lseed) & 1; + U32 const noDictIDFlag = FUZ_rand(&lseed) & 1; + ZSTD_frameParameters const fPar = { 0, checksum, noDictIDFlag }; + ZSTD_parameters params; + ZSTD_adjustCParams(&cPar, 0, dictSize); + params.cParams = cPar; + params.fParams = fPar; + { size_t const initError = ZBUFF_compressInit_advanced(zc, dict, dictSize, params, 0); + CHECK (ZBUFF_isError(initError),"init error : %s", ZBUFF_getErrorName(initError)); + } } } /* multi-segments compression test */ XXH64_reset(&xxhState, 0); From 70d1301d6e84ce4285623b10e462e259b9bee312 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 1 Jun 2016 18:45:34 +0200 Subject: [PATCH 3/7] Changed `ZSTD_adjustCParams()` prototype `ZSTD_adjustCParams()` is now automatically invoked at the end of `ZSTD_getCParams()` --- lib/common/zstd_static.h | 2 +- lib/compress/zbuff_compress.c | 2 -- lib/compress/zstd_compress.c | 29 +++++++++++++++-------------- programs/bench.c | 2 -- programs/zbufftest.c | 3 +-- 5 files changed, 17 insertions(+), 21 deletions(-) diff --git a/lib/common/zstd_static.h b/lib/common/zstd_static.h index e4c992bee..01aaecf6d 100644 --- a/lib/common/zstd_static.h +++ b/lib/common/zstd_static.h @@ -127,7 +127,7 @@ ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params); /*! ZSTD_adjustParams() : * optimize params for a given `srcSize` and `dictSize`. * both values are optional, select `0` if unknown. */ -ZSTDLIB_API void ZSTD_adjustCParams(ZSTD_compressionParameters* params, U64 srcSize, size_t dictSize); +ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, U64 srcSize, size_t dictSize); /*! ZSTD_compress_advanced() : * Same as ZSTD_compress_usingDict(), with fine-tune control of each compression parameter */ diff --git a/lib/compress/zbuff_compress.c b/lib/compress/zbuff_compress.c index 66deb495f..ee33df541 100644 --- a/lib/compress/zbuff_compress.c +++ b/lib/compress/zbuff_compress.c @@ -182,8 +182,6 @@ size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* zbc, const void* dict, size_t di ZSTD_parameters params; memset(¶ms, 0, sizeof(params)); params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize); - params.fParams.contentSizeFlag = 0; - ZSTD_adjustCParams(¶ms.cParams, 0, dictSize); return ZBUFF_compressInit_advanced(zbc, dict, dictSize, params, 0); } diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index a9a2dc79d..7c7349b54 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -202,30 +202,32 @@ size_t ZSTD_checkCParams_advanced(ZSTD_compressionParameters cParams, U64 srcSiz } -/** ZSTD_adjustParams() : - optimize params for q given input (`srcSize` and `dictSize`). +/** ZSTD_adjustCParams() : + optimize cPar for a given input (`srcSize` and `dictSize`). mostly downsizing to reduce memory consumption and initialization. Both `srcSize` and `dictSize` are optional (use 0 if unknown), but if both are 0, no optimization can be done. - Note : params is considered validated at this stage. Use ZSTD_checkParams() to ensure that. */ -void ZSTD_adjustCParams(ZSTD_compressionParameters* params, U64 srcSize, size_t dictSize) + Note : cPar is considered validated at this stage. Use ZSTD_checkParams() to ensure that. */ +ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, U64 srcSize, size_t dictSize) { - if (srcSize+dictSize == 0) return; /* no size information available : no adjustment */ + if (srcSize+dictSize == 0) return cPar; /* no size information available : no adjustment */ /* resize params, to use less memory when necessary */ { U32 const minSrcSize = (srcSize==0) ? 500 : 0; U64 const rSize = srcSize + dictSize + minSrcSize; if (rSize < ((U64)1<windowLog > srcLog) params->windowLog = srcLog; + if (cPar.windowLog > srcLog) cPar.windowLog = srcLog; } } - if (params->hashLog > params->windowLog) params->hashLog = params->windowLog; - { U32 const btPlus = (params->strategy == ZSTD_btlazy2) || (params->strategy == ZSTD_btopt); - U32 const maxChainLog = params->windowLog+btPlus; - if (params->chainLog > maxChainLog) params->chainLog = maxChainLog; } /* <= ZSTD_CHAINLOG_MAX */ + if (cPar.hashLog > cPar.windowLog) cPar.hashLog = cPar.windowLog; + { U32 const btPlus = (cPar.strategy == ZSTD_btlazy2) || (cPar.strategy == ZSTD_btopt); + U32 const maxChainLog = cPar.windowLog+btPlus; + if (cPar.chainLog > maxChainLog) cPar.chainLog = maxChainLog; } /* <= ZSTD_CHAINLOG_MAX */ - if (params->windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) params->windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */ - if ((params->hashLog < ZSTD_HASHLOG_MIN) && ((U32)params->strategy >= (U32)ZSTD_btlazy2)) params->hashLog = ZSTD_HASHLOG_MIN; /* required to ensure collision resistance in bt */ + if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */ + if ((cPar.hashLog < ZSTD_HASHLOG_MIN) && ( (U32)cPar.strategy >= (U32)ZSTD_btlazy2)) cPar.hashLog = ZSTD_HASHLOG_MIN; /* required to ensure collision resistance in bt */ + + return cPar; } @@ -2362,7 +2364,6 @@ size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* zc, const void* dict, size_t dict ZSTD_parameters params; memset(¶ms, 0, sizeof(params)); params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize); - ZSTD_adjustCParams(¶ms.cParams, 0, dictSize); ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin_usingDict compressionLevel=%d\n", zc->base, compressionLevel); return ZSTD_compressBegin_internal(zc, dict, dictSize, params, 0); } @@ -2472,7 +2473,6 @@ size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, co ZSTD_LOG_BLOCK("%p: ZSTD_compress_usingDict srcSize=%d dictSize=%d compressionLevel=%d\n", ctx->base, (int)srcSize, (int)dictSize, compressionLevel); params.cParams = ZSTD_getCParams(compressionLevel, srcSize, dictSize); params.fParams.contentSizeFlag = 1; - ZSTD_adjustCParams(¶ms.cParams, srcSize, dictSize); return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params); } @@ -2625,5 +2625,6 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, si if (cp.chainLog > ZSTD_CHAINLOG_MAX) cp.chainLog = ZSTD_CHAINLOG_MAX; if (cp.hashLog > ZSTD_HASHLOG_MAX) cp.hashLog = ZSTD_HASHLOG_MAX; } + cp = ZSTD_adjustCParams(cp, srcSize, dictSize); return cp; } diff --git a/programs/bench.c b/programs/bench.c index 52c64d525..97f71162b 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -220,7 +220,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, { ZSTD_parameters params; params.cParams = ZSTD_getCParams(cLevel, blockSize, dictBufferSize); params.fParams.contentSizeFlag = 1; - ZSTD_adjustCParams(¶ms.cParams, blockSize, dictBufferSize); { size_t const initResult = ZSTD_compressBegin_advanced(refCtx, dictBuffer, dictBufferSize, params, blockSize); if (ZSTD_isError(initResult)) break; } } @@ -505,4 +504,3 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast); return 0; } - diff --git a/programs/zbufftest.c b/programs/zbufftest.c index 166922970..fd9269bd6 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -375,12 +375,11 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres { size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize); dict = srcBuffer + dictStart; } - { ZSTD_compressionParameters cPar = ZSTD_getCParams(cLevel, 0, dictSize); + { ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, 0, dictSize); U32 const checksum = FUZ_rand(&lseed) & 1; U32 const noDictIDFlag = FUZ_rand(&lseed) & 1; ZSTD_frameParameters const fPar = { 0, checksum, noDictIDFlag }; ZSTD_parameters params; - ZSTD_adjustCParams(&cPar, 0, dictSize); params.cParams = cPar; params.fParams = fPar; { size_t const initError = ZBUFF_compressInit_advanced(zc, dict, dictSize, params, 0); From eb70043e5937cf144ba9a85eeb5e5f21e231b612 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 1 Jun 2016 18:59:55 +0200 Subject: [PATCH 4/7] Fixed paramgrill --- programs/paramgrill.c | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/programs/paramgrill.c b/programs/paramgrill.c index 6dc90b219..cb9a2ea49 100644 --- a/programs/paramgrill.c +++ b/programs/paramgrill.c @@ -600,22 +600,22 @@ static void playAround(FILE* f, winnerInfo_t* winners, } -static void potentialRandomParams(ZSTD_compressionParameters* p, U32 inverseChance) +static ZSTD_compressionParameters randomParams(void) { - U32 chance = (FUZ_rand(&g_rand) % (inverseChance+1)); + ZSTD_compressionParameters p; U32 validated = 0; - if (!chance) while (!validated) { /* totally random entry */ - p->chainLog = FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN) + ZSTD_CHAINLOG_MIN; - p->hashLog = FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN) + ZSTD_HASHLOG_MIN; - p->searchLog = FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN) + ZSTD_SEARCHLOG_MIN; - p->windowLog = FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN) + ZSTD_WINDOWLOG_MIN; - p->searchLength=FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN) + ZSTD_SEARCHLENGTH_MIN; - p->targetLength=FUZ_rand(&g_rand) % (ZSTD_TARGETLENGTH_MAX+1 - ZSTD_TARGETLENGTH_MIN) + ZSTD_TARGETLENGTH_MIN; - p->strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btopt +1)); - validated = !ZSTD_isError(ZSTD_checkCParams(*p)); + p.chainLog = FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN) + ZSTD_CHAINLOG_MIN; + p.hashLog = FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN) + ZSTD_HASHLOG_MIN; + p.searchLog = FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN) + ZSTD_SEARCHLOG_MIN; + p.windowLog = FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN) + ZSTD_WINDOWLOG_MIN; + p.searchLength=FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN) + ZSTD_SEARCHLENGTH_MIN; + p.targetLength=FUZ_rand(&g_rand) % (ZSTD_TARGETLENGTH_MAX+1 - ZSTD_TARGETLENGTH_MIN) + ZSTD_TARGETLENGTH_MIN; + p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btopt +1)); + validated = !ZSTD_isError(ZSTD_checkCParams(p)); } + return p; } static void BMK_selectRandomStart( @@ -623,12 +623,10 @@ static void BMK_selectRandomStart( const void* srcBuffer, size_t srcSize, ZSTD_CCtx* ctx) { - U32 id = (FUZ_rand(&g_rand) % (ZSTD_maxCLevel()+1)); + U32 const id = (FUZ_rand(&g_rand) % (ZSTD_maxCLevel()+1)); if ((id==0) || (winners[id].params.windowLog==0)) { /* totally random entry */ - ZSTD_compressionParameters p; - potentialRandomParams(&p, 1); - ZSTD_adjustCParams(&p, srcSize, 0); + ZSTD_compressionParameters const p = ZSTD_adjustCParams(randomParams(), srcSize, 0); playAround(f, winners, p, srcBuffer, srcSize, ctx); } else @@ -649,7 +647,7 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize) if (g_singleRun) { BMK_result_t testResult; - ZSTD_adjustCParams(&g_params, srcSize, 0); + g_params = ZSTD_adjustCParams(g_params, srcSize, 0); BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, g_params); DISPLAY("\n"); return; @@ -861,7 +859,7 @@ int optimizeForSize(char* inFileName) do { params = winner.params; paramVariation(¶ms); - potentialRandomParams(¶ms, 16); + if ((FUZ_rand(&g_rand) & 15) == 1) params = randomParams(); /* exclude faster if already played set of params */ if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(params))-1)) continue; From 87cfbe3e5fede455209457120b6292d7e389f20e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 1 Jun 2016 19:22:15 +0200 Subject: [PATCH 5/7] checksum generation can be selected from command line (`--checksum`) --- programs/fileio.c | 3 +++ programs/fileio.h | 1 + programs/zstdcli.c | 2 ++ 3 files changed, 6 insertions(+) diff --git a/programs/fileio.c b/programs/fileio.c index 55e36cc55..48555424c 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -135,6 +135,8 @@ static U32 g_sparseFileSupport = 1; /* 0 : no sparse allowed; 1: auto (file ye void FIO_setSparseWrite(unsigned sparse) { g_sparseFileSupport=sparse; } static U32 g_dictIDFlag = 1; void FIO_setDictIDFlag(unsigned dictIDFlag) { g_dictIDFlag = dictIDFlag; } +static U32 g_checksumFlag = 0; +void FIO_setChecksumFlag(unsigned checksumFlag) { g_checksumFlag = checksumFlag; } /*-************************************* @@ -313,6 +315,7 @@ static int FIO_compressFilename_internal(cRess_t ress, memset(¶ms, 0, sizeof(params)); params.cParams = ZSTD_getCParams(cLevel, fileSize, ress.dictBufferSize); params.fParams.contentSizeFlag = 1; + params.fParams.checksumFlag = g_checksumFlag; params.fParams.noDictIDFlag = !g_dictIDFlag; if (g_maxWLog) if (params.cParams.windowLog > g_maxWLog) params.cParams.windowLog = g_maxWLog; { size_t const errorCode = ZBUFF_compressInit_advanced(ress.ctx, ress.dictBuffer, ress.dictBufferSize, params, fileSize); diff --git a/programs/fileio.h b/programs/fileio.h index 5a9cdc1ce..01e308340 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -49,6 +49,7 @@ void FIO_setNotificationLevel(unsigned level); void FIO_setMaxWLog(unsigned maxWLog); /**< if `maxWLog` == 0, no max enforced */ void FIO_setSparseWrite(unsigned sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */ void FIO_setDictIDFlag(unsigned dictIDFlag); +void FIO_setChecksumFlag(unsigned checksumFlag); /*-************************************* diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 24bba89db..f31b6e1f8 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -134,6 +134,7 @@ static int usage_advanced(const char* programName) #ifndef ZSTD_NOCOMPRESS DISPLAY( "--ultra : enable ultra modes (requires more memory to decompress)\n"); DISPLAY( "--no-dictID:don't write dictID into header (dictionary compression)\n"); + DISPLAY( "--checksum:compute checksum of content, for decompressor validation\n"); #endif #ifndef ZSTD_NODECOMPRESS DISPLAY( " -t : test compressed file integrity \n"); @@ -240,6 +241,7 @@ int main(int argCount, const char** argv) if (!strcmp(argument, "--quiet")) { displayLevel--; continue; } if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; displayLevel=1; continue; } if (!strcmp(argument, "--ultra")) { FIO_setMaxWLog(0); continue; } + if (!strcmp(argument, "--checksum")) { FIO_setChecksumFlag(2); continue; } if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; } if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; } if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; } From 2c4acdabdc85b86e957f753e10d12ddb9cc5dcde Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 2 Jun 2016 17:05:50 +0200 Subject: [PATCH 6/7] Changed command name to `--check` , `-C`, following `xz` convention --- programs/zstd.1 | 9 ++++++++- programs/zstdcli.c | 9 ++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/programs/zstd.1 b/programs/zstd.1 index 1bab57abb..cc62eb30f 100644 --- a/programs/zstd.1 +++ b/programs/zstd.1 @@ -52,7 +52,7 @@ It also features a very fast decoder, with speed > 500 MB/s per core. use `file` as Dictionary to compress or decompress FILE(s) .TP .B \-o file - save result into `file` (only possible with a single input FILE) + save result into `file` (only possible with a single INPUT-FILE) .TP .BR \-f ", " --force overwrite output without prompting @@ -71,6 +71,13 @@ It also features a very fast decoder, with speed > 500 MB/s per core. .TP .BR \-c ", " --stdout force write to standard output, even if it is the console +.TP +.BR \-C ", " --check + add integrity check computed from uncompressed data +.TP +.BR \-t ", " --test + Test the integrity of compressed files. This option is equivalent to \fB--decompress --stdout > /dev/null\fR. + No files are created or removed. .SH DICTIONARY .PP diff --git a/programs/zstdcli.c b/programs/zstdcli.c index f31b6e1f8..df7a33208 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -134,10 +134,10 @@ static int usage_advanced(const char* programName) #ifndef ZSTD_NOCOMPRESS DISPLAY( "--ultra : enable ultra modes (requires more memory to decompress)\n"); DISPLAY( "--no-dictID:don't write dictID into header (dictionary compression)\n"); - DISPLAY( "--checksum:compute checksum of content, for decompressor validation\n"); + DISPLAY( "--check : enable integrity check\n"); #endif #ifndef ZSTD_NODECOMPRESS - DISPLAY( " -t : test compressed file integrity \n"); + DISPLAY( "--test : test compressed file integrity \n"); DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n"); #endif #ifndef ZSTD_NODICT @@ -241,7 +241,7 @@ int main(int argCount, const char** argv) if (!strcmp(argument, "--quiet")) { displayLevel--; continue; } if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; displayLevel=1; continue; } if (!strcmp(argument, "--ultra")) { FIO_setMaxWLog(0); continue; } - if (!strcmp(argument, "--checksum")) { FIO_setChecksumFlag(2); continue; } + if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(2); continue; } if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; } if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; } if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; } @@ -305,6 +305,9 @@ int main(int argCount, const char** argv) /* keep source file (default anyway, so useless; for gzip/xz compatibility) */ case 'k': argument++; break; + /* Checksum */ + case 'C': argument++; FIO_setChecksumFlag(2); break; + /* test compressed file */ case 't': decode=1; outFileName=nulmark; argument++; break; From 923938edde18f6aeaf3dd705cf8ed73e4ad67a6a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 2 Jun 2016 17:56:00 +0200 Subject: [PATCH 7/7] Added `-Wdeclaration-after-statement` compilation flag --- lib/Makefile | 2 +- programs/Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/Makefile b/lib/Makefile index 75b09e5de..76731abc1 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -48,7 +48,7 @@ INCLUDEDIR=$(PREFIX)/include CPPFLAGS= -I./common -DXXH_NAMESPACE=ZSTD_ CFLAGS ?= -O3 -CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wstrict-prototypes -Wundef +CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS) diff --git a/programs/Makefile b/programs/Makefile index 1e8a7f268..52a7ca076 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -40,7 +40,7 @@ ZSTDDIR = ../lib CPPFLAGS= -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder -DXXH_NAMESPACE=ZSTD_ CFLAGS ?= -O3 # -falign-loops=32 # not always beneficial -CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wstrict-prototypes -Wundef +CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)