From 671f28d1e50865bac0662297eb0d2372a705385e Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 13 Dec 2016 12:18:07 +0100 Subject: [PATCH 01/10] added parseCompressionParameters --- programs/zstdcli.c | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 7ffb0bb62..9bc96e19c 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -204,6 +204,38 @@ static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) return result; } + +/** parseCompressionParameters() : + * reads compression parameters from *stringPtr (e.g. "--zstd=wlog=23,clog=23,hlog=22,slog=6,slen=3,tlen=48,strat=6") into *params + * @return 1 means that compression parameters were correct + * @return 0 in case of malformed parameters + */ +static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressionParameters* params) +{ + for ( ; ;) { + DISPLAY("arg=%s\n", stringPtr); + if (longCommandWArg(&stringPtr, "windowLog=") || longCommandWArg(&stringPtr, "wlog=")) { params->windowLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "chainLog=") || longCommandWArg(&stringPtr, "clog=")) { params->chainLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "hashLog=") || longCommandWArg(&stringPtr, "hlog=")) { params->hashLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "searchLog=") || longCommandWArg(&stringPtr, "slog=")) { params->searchLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "searchLength=") || longCommandWArg(&stringPtr, "slen=")) { params->searchLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "targetLength=") || longCommandWArg(&stringPtr, "tlen=")) { params->targetLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "strategy=") || longCommandWArg(&stringPtr, "strat=")) { params->strategy = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + return 0; + } + + if (stringPtr[0] != 0) return 0; /* check the end of string */ + DISPLAYLEVEL(4, "windowLog=%d\n", params->windowLog); + DISPLAYLEVEL(4, "chainLog=%d\n", params->chainLog); + DISPLAYLEVEL(4, "hashLog=%d\n", params->hashLog); + DISPLAYLEVEL(4, "searchLog=%d\n", params->searchLog); + DISPLAYLEVEL(4, "searchLength=%d\n", params->searchLength); + DISPLAYLEVEL(4, "targetLength=%d\n", params->targetLength); + DISPLAYLEVEL(4, "strategy=%d\n", params->strategy); + return 1; +} + + typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train } zstd_operation_mode; #define CLEAN_RETURN(i) { operationResult = (i); goto _end; } @@ -222,6 +254,7 @@ int main(int argCount, const char* argv[]) ultra=0, lastCommand = 0; zstd_operation_mode operation = zom_compress; + ZSTD_compressionParameters compressionParams; int cLevel = ZSTDCLI_CLEVEL_DEFAULT; int cLevelLast = 1; unsigned recursive = 0; @@ -258,6 +291,7 @@ int main(int argCount, const char* argv[]) /* preset behaviors */ if (!strcmp(programName, ZSTD_UNZSTD)) operation=zom_decompress; if (!strcmp(programName, ZSTD_CAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; displayLevel=1; } + memset(&compressionParams, 0, sizeof(compressionParams)); /* command switches */ for (argNb=1; argNbwindowLog) zparams.cParams.windowLog = comprParams->windowLog; + if (comprParams->chainLog) zparams.cParams.chainLog = comprParams->chainLog; + if (comprParams->hashLog) zparams.cParams.hashLog = comprParams->hashLog; + if (comprParams->searchLog) zparams.cParams.searchLog = comprParams->searchLog; + if (comprParams->searchLength) zparams.cParams.searchLength = comprParams->searchLength; + if (comprParams->targetLength) zparams.cParams.targetLength = comprParams->targetLength; + if (comprParams->strategy) zparams.cParams.strategy = comprParams->strategy; do { U32 blockNb; size_t rSize; @@ -247,9 +255,9 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize, cdict); } else { - rSize = ZSTD_compressCCtx (ctx, + rSize = ZSTD_compress_advanced (ctx, blockTable[blockNb].cPtr, blockTable[blockNb].cRoom, - blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize, cLevel); + blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize, NULL, 0, zparams); } if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_compress_usingCDict() failed : %s", ZSTD_getErrorName(rSize)); blockTable[blockNb].cSize = rSize; @@ -387,7 +395,8 @@ static size_t BMK_findMaxMem(U64 requiredMem) static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, const char* displayName, int cLevel, int cLevelLast, const size_t* fileSizes, unsigned nbFiles, - const void* dictBuffer, size_t dictBufferSize) + const void* dictBuffer, size_t dictBufferSize, + ZSTD_compressionParameters *compressionParams) { int l; @@ -406,7 +415,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, BMK_benchMem(srcBuffer, benchedSize, displayName, l, fileSizes, nbFiles, - dictBuffer, dictBufferSize); + dictBuffer, dictBufferSize, compressionParams); } } @@ -443,8 +452,8 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize, if (totalSize == 0) EXM_THROW(12, "no data to bench"); } -static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, - const char* dictFileName, int cLevel, int cLevelLast) +static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, + int cLevel, int cLevelLast, ZSTD_compressionParameters *compressionParams) { void* srcBuffer; size_t benchedSize; @@ -483,7 +492,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, BMK_benchCLevel(srcBuffer, benchedSize, displayName, cLevel, cLevelLast, fileSizes, nbFiles, - dictBuffer, dictBufferSize); + dictBuffer, dictBufferSize, compressionParams); } /* clean up */ @@ -493,7 +502,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, } -static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility) +static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility, ZSTD_compressionParameters* compressionParams) { char name[20] = {0}; size_t benchedSize = 10000000; @@ -507,15 +516,15 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility /* Bench */ snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100)); - BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0); + BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0, compressionParams); /* clean up */ free(srcBuffer); } -int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, - const char* dictFileName, int cLevel, int cLevelLast) +int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, + int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams) { double const compressibility = (double)g_compressibilityDefault / 100; @@ -525,8 +534,8 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast); if (nbFiles == 0) - BMK_syntheticTest(cLevel, cLevelLast, compressibility); + BMK_syntheticTest(cLevel, cLevelLast, compressibility, compressionParams); else - BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast); + BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams); return 0; } diff --git a/programs/bench.h b/programs/bench.h index 7009dc2f6..314f34655 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -12,9 +12,11 @@ #define BENCH_H_121279284357 #include /* size_t */ +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */ +#include "zstd.h" /* ZSTD_compressionParameters */ -int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, - const char* dictFileName, int cLevel, int cLevelLast); +int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,const char* dictFileName, + int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams); /* Set Parameters */ void BMK_SetNbSeconds(unsigned nbLoops); diff --git a/programs/zstdcli.c b/programs/zstdcli.c index e11f64842..fb873a867 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -214,7 +214,6 @@ static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressionParameters* params) { for ( ; ;) { - DISPLAY("arg=%s\n", stringPtr); if (longCommandWArg(&stringPtr, "windowLog=") || longCommandWArg(&stringPtr, "wlog=")) { params->windowLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } if (longCommandWArg(&stringPtr, "chainLog=") || longCommandWArg(&stringPtr, "clog=")) { params->chainLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } if (longCommandWArg(&stringPtr, "hashLog=") || longCommandWArg(&stringPtr, "hlog=")) { params->hashLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } @@ -226,13 +225,8 @@ static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressi } if (stringPtr[0] != 0) return 0; /* check the end of string */ - DISPLAYLEVEL(4, "windowLog=%d\n", params->windowLog); - DISPLAYLEVEL(4, "chainLog=%d\n", params->chainLog); - DISPLAYLEVEL(4, "hashLog=%d\n", params->hashLog); - DISPLAYLEVEL(4, "searchLog=%d\n", params->searchLog); - DISPLAYLEVEL(4, "searchLength=%d\n", params->searchLength); - DISPLAYLEVEL(4, "targetLength=%d\n", params->targetLength); - DISPLAYLEVEL(4, "strategy=%d\n", params->strategy); + DISPLAYLEVEL(4, "windowLog=%d\nchainLog=%d\nhashLog=%d\nsearchLog=%d\n", params->windowLog, params->chainLog, params->hashLog, params->searchLog); + DISPLAYLEVEL(4, "searchLength=%d\ntargetLength=%d\nstrategy=%d\n", params->searchLength, params->targetLength, params->strategy); return 1; } @@ -523,7 +517,7 @@ int main(int argCount, const char* argv[]) if (operation==zom_bench) { #ifndef ZSTD_NOBENCH BMK_setNotificationLevel(displayLevel); - BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast); + BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams); #endif goto _end; } From 8349d675e01faee2e1b33582175f4aaea71342bd Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 13 Dec 2016 13:24:59 +0100 Subject: [PATCH 03/10] fileio.c: support advanced compression parameters --- programs/fileio.c | 19 ++++++++++++++----- programs/fileio.h | 9 +++++++-- programs/zstdcli.c | 4 ++-- 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index a493e97c5..597422ae8 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -247,7 +247,8 @@ typedef struct { FILE* srcFile; } cRess_t; -static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, U64 srcSize) +static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, + U64 srcSize, ZSTD_compressionParameters* comprParams) { cRess_t ress; memset(&ress, 0, sizeof(ress)); @@ -268,6 +269,13 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, U64 sr params.fParams.contentSizeFlag = 1; params.fParams.checksumFlag = g_checksumFlag; params.fParams.noDictIDFlag = !g_dictIDFlag; + if (comprParams->windowLog) params.cParams.windowLog = comprParams->windowLog; + if (comprParams->chainLog) params.cParams.chainLog = comprParams->chainLog; + if (comprParams->hashLog) params.cParams.hashLog = comprParams->hashLog; + if (comprParams->searchLog) params.cParams.searchLog = comprParams->searchLog; + if (comprParams->searchLength) params.cParams.searchLength = comprParams->searchLength; + if (comprParams->targetLength) params.cParams.targetLength = comprParams->targetLength; + if (comprParams->strategy) params.cParams.strategy = comprParams->strategy; { size_t const errorCode = ZSTD_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize); if (ZSTD_isError(errorCode)) EXM_THROW(33, "Error initializing CStream : %s", ZSTD_getErrorName(errorCode)); } } @@ -401,12 +409,12 @@ static int FIO_compressFilename_dstFile(cRess_t ress, int FIO_compressFilename(const char* dstFileName, const char* srcFileName, - const char* dictFileName, int compressionLevel) + const char* dictFileName, int compressionLevel, ZSTD_compressionParameters* comprParams) { clock_t const start = clock(); U64 const srcSize = UTIL_getFileSize(srcFileName); - cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel, srcSize); + cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, comprParams); int const result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName); double const seconds = (double)(clock() - start) / CLOCKS_PER_SEC; @@ -419,14 +427,15 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName, int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFiles, const char* suffix, - const char* dictFileName, int compressionLevel) + const char* dictFileName, int compressionLevel, + ZSTD_compressionParameters* comprParams) { int missed_files = 0; size_t dfnSize = FNSPACE; char* dstFileName = (char*)malloc(FNSPACE); size_t const suffixSize = suffix ? strlen(suffix) : 0; U64 const srcSize = (nbFiles != 1) ? 0 : UTIL_getFileSize(inFileNamesTable[0]) ; - cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize); + cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, comprParams); /* init */ if (dstFileName==NULL) EXM_THROW(27, "FIO_compressMultipleFilenames : allocation error for dstFileName"); diff --git a/programs/fileio.h b/programs/fileio.h index 06e0be372..b71658334 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -11,6 +11,9 @@ #ifndef FILEIO_H_23981798732 #define FILEIO_H_23981798732 +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */ +#include "zstd.h" /* ZSTD_compressionParameters */ + #if defined (__cplusplus) extern "C" { #endif @@ -44,7 +47,8 @@ void FIO_setMemLimit(unsigned memLimit); ***************************************/ /** FIO_compressFilename() : @return : 0 == ok; 1 == pb with src file. */ -int FIO_compressFilename (const char* outfilename, const char* infilename, const char* dictFileName, int compressionLevel); +int FIO_compressFilename (const char* outfilename, const char* infilename, const char* dictFileName, + int compressionLevel, ZSTD_compressionParameters* comprParams); /** FIO_decompressFilename() : @return : 0 == ok; 1 == pb with src file. */ @@ -58,7 +62,8 @@ int FIO_decompressFilename (const char* outfilename, const char* infilename, con @return : nb of missing files */ int FIO_compressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles, const char* suffix, - const char* dictFileName, int compressionLevel); + const char* dictFileName, int compressionLevel, + ZSTD_compressionParameters* comprParams); /** FIO_decompressMultipleFilenames() : @return : nb of missing or skipped files */ diff --git a/programs/zstdcli.c b/programs/zstdcli.c index fb873a867..f2a6c71f7 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -569,9 +569,9 @@ int main(int argCount, const char* argv[]) if (operation==zom_compress) { #ifndef ZSTD_NOCOMPRESS if ((filenameIdx==1) && outFileName) - operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel); + operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel, &compressionParams); else - operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName ? outFileName : ZSTD_EXTENSION, dictFileName, cLevel); + operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName ? outFileName : ZSTD_EXTENSION, dictFileName, cLevel, &compressionParams); #else DISPLAY("Compression not supported\n"); #endif From 98ef0f98df98402f94a72c85b59559e2634f5336 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 13 Dec 2016 14:52:21 +0100 Subject: [PATCH 04/10] fixed conversion warning --- programs/zstdcli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index f2a6c71f7..68400f195 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -220,7 +220,7 @@ static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressi if (longCommandWArg(&stringPtr, "searchLog=") || longCommandWArg(&stringPtr, "slog=")) { params->searchLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } if (longCommandWArg(&stringPtr, "searchLength=") || longCommandWArg(&stringPtr, "slen=")) { params->searchLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } if (longCommandWArg(&stringPtr, "targetLength=") || longCommandWArg(&stringPtr, "tlen=")) { params->targetLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } - if (longCommandWArg(&stringPtr, "strategy=") || longCommandWArg(&stringPtr, "strat=")) { params->strategy = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "strategy=") || longCommandWArg(&stringPtr, "strat=")) { params->strategy = (ZSTD_strategy)readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } return 0; } From c71e552b2e91c1f95a998887edb9eb8fe5c3f4ce Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 13 Dec 2016 20:04:32 +0100 Subject: [PATCH 05/10] fixed "strategy" in advanced compression parameters --- programs/bench.c | 2 +- programs/fileio.c | 2 +- programs/zstdcli.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 6203af6bd..6c2383a87 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -244,7 +244,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, if (comprParams->searchLog) zparams.cParams.searchLog = comprParams->searchLog; if (comprParams->searchLength) zparams.cParams.searchLength = comprParams->searchLength; if (comprParams->targetLength) zparams.cParams.targetLength = comprParams->targetLength; - if (comprParams->strategy) zparams.cParams.strategy = comprParams->strategy; + if (comprParams->strategy) zparams.cParams.strategy = (ZSTD_strategy)(comprParams->strategy - 1); do { U32 blockNb; size_t rSize; diff --git a/programs/fileio.c b/programs/fileio.c index 597422ae8..555b56141 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -275,7 +275,7 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, if (comprParams->searchLog) params.cParams.searchLog = comprParams->searchLog; if (comprParams->searchLength) params.cParams.searchLength = comprParams->searchLength; if (comprParams->targetLength) params.cParams.targetLength = comprParams->targetLength; - if (comprParams->strategy) params.cParams.strategy = comprParams->strategy; + if (comprParams->strategy) params.cParams.strategy = (ZSTD_strategy)(comprParams->strategy - 1); { size_t const errorCode = ZSTD_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize); if (ZSTD_isError(errorCode)) EXM_THROW(33, "Error initializing CStream : %s", ZSTD_getErrorName(errorCode)); } } diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 68400f195..c1eb7b732 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -220,7 +220,7 @@ static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressi if (longCommandWArg(&stringPtr, "searchLog=") || longCommandWArg(&stringPtr, "slog=")) { params->searchLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } if (longCommandWArg(&stringPtr, "searchLength=") || longCommandWArg(&stringPtr, "slen=")) { params->searchLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } if (longCommandWArg(&stringPtr, "targetLength=") || longCommandWArg(&stringPtr, "tlen=")) { params->targetLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } - if (longCommandWArg(&stringPtr, "strategy=") || longCommandWArg(&stringPtr, "strat=")) { params->strategy = (ZSTD_strategy)readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "strategy=") || longCommandWArg(&stringPtr, "strat=")) { params->strategy = (ZSTD_strategy)(1 + readU32FromChar(&stringPtr)); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } return 0; } From 25314428c975b66891f3414836e117bd4093a4bc Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 14 Dec 2016 16:10:13 +0100 Subject: [PATCH 06/10] zstd.1: added advanced compression options --- programs/zstd.1 | 109 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/programs/zstd.1 b/programs/zstd.1 index 63b60d1e9..9b10b1873 100644 --- a/programs/zstd.1 +++ b/programs/zstd.1 @@ -264,6 +264,115 @@ and weight typically 100x the target dictionary size (for example, 10 MB for a 1 cut file into independent blocks of size # (default: no block) +.SH ADVANCED COMPRESSION OPTIONS +.TP +.B \--zstd[=\fIoptions\fR] +.PD +\fBzstd\fR provides 22 predefined compression levels. The selected or default predefined compression level can be changed with advanced compression options. +The \fIoptions\fR are provided as a comma-separated list. You may specify only the \fIoptions\fR you want to change and the rest will be taken from the selected or default compression level. +The list of available \fIoptions\fR: +.RS + +.TP +.BI strategy= strat +.PD 0 +.TP +.BI strat= strat +.PD +Specify a strategy used by a match finder. +.IP "" +There are 8 strategies numbered from 0 to 7, from faster to stronger: +0=ZSTD_fast, 1=ZSTD_dfast, 2=ZSTD_greedy, 3=ZSTD_lazy, 4=ZSTD_lazy2, 5=ZSTD_btlazy2, 6=ZSTD_btopt, 7=ZSTD_btopt2. +.IP "" + +.TP +.BI windowLog= wlog +.PD 0 +.TP +.BI wlog= wlog +.PD +Specify the maximum number of bits for a match distance. +.IP "" +The higher number of bits increases the chance to find a match what usually improves compression ratio. +It also increases memory requirements for compressor and decompressor. +.IP "" +The minimum \fIwlog\fR is 10 (1 KiB) and the maximum is 25 (32 MiB) for 32-bit compilation and 27 (128 MiB) for 64-bit compilation. +.IP "" + +.TP +.BI hashLog= hlog +.PD 0 +.TP +.BI hlog= hlog +.PD +Specify the maximum number of bits for a hash table. +.IP "" +The bigger hash table causes less collisions what usually make compression faster but requires more memory during compression. +.IP "" +The minimum \fIhlog\fR is 6 (64 B) and the maximum is 25 (32 MiB) for 32-bit compilation and 27 (128 MiB) for 64-bit compilation. + +.TP +.BI chainLog= clog +.PD 0 +.TP +.BI clog= clog +.PD +Specify the maximum number of bits for a hash chain or a binary tree. +.IP "" +The higher number of bits increases the chance to find a match what usually improves compression ratio. +It also slows down compression speed and increases memory requirements for compression. +This option is ignored for the ZSTD_fast strategy. +.IP "" +The minimum \fIclog\fR is 6 (64 B) and the maximum is 26 (64 MiB) for 32-bit compilation and 28 (256 MiB) for 64-bit compilation. +.IP "" + +.TP +.BI searchLog= slog +.PD 0 +.TP +.BI slog= slog +.PD +Specify the maximum number of searches in a hash chain or a binary tree using logarithmic scale. +.IP "" +The bigger number of searches increases the chance to find a match what usually improves compression ratio but decreases compression speed. +.IP "" +The minimum \fIslog\fR is 1 and the maximum is 24 for 32-bit compilation and 26 for 64-bit compilation. +.IP "" + +.TP +.BI searchLength= slen +.PD 0 +.TP +.BI slen= slen +.PD +Specify the minimum searched length of a match in a hash table. +.IP "" +The bigger search length usually decreases compression ratio but improves decompression speed. +.IP "" +The minimum \fIslen\fR is 3 and the maximum is 7. +.IP "" + +.TP +.BI targetLength= tlen +.PD 0 +.TP +.BI tlen= tlen +.PD +Specify the minimum match length that causes a match finder to interrupt searching of better matches. +.IP "" +The bigger minimum match length usually improves compression ratio but decreases compression speed. +This option is used only with ZSTD_btopt and ZSTD_btopt2 strategies. +.IP "" +The minimum \fItlen\fR is 4 and the maximum is 999. +.IP "" + +.PP +.B An example +.br +The following parameters sets advanced compression options to predefined level 19 for files bigger than 256 KB: +.IP "" +\fB--zstd=\fRwindowLog=23,chainLog=23,hashLog=22,searchLog=6,searchLength=3,targetLength=48,strategy=6 + .SH BUGS Report bugs at:- https://github.com/facebook/zstd/issues From 9b4fa0ddf722cc154e6f2274abaa346e62a4c2b4 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 14 Dec 2016 16:50:00 +0100 Subject: [PATCH 07/10] playTests.sh: added Advanced compression parameters --- tests/playTests.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/playTests.sh b/tests/playTests.sh index abde72c80..c5bc577dd 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -101,6 +101,15 @@ $ZSTD -f tmp && die "tmp not present : should have failed" ls tmp.zst && die "tmp.zst should not be created" +$ECHO "\n**** Advanced compression parameters **** " +$ZSTD --zstd=windowLog=21, && die "wrong parameters not detected!" +$ZSTD --zstd=windowLok=21 && die "wrong parameters not detected!" +$ZSTD --zstd=windowLog=21,slog= && die "wrong parameters not detected!" +roundTripTest -g512K +roundTripTest -g512K " --zstd=windowLog=23,chainLog=23,hashLog=22,searchLog=6,searchLength=3,targetLength=48,strategy=6" +roundTripTest -g512K 19 + + $ECHO "\n**** Pass-Through mode **** " $ECHO "Hello world 1!" | $ZSTD -df $ECHO "Hello world 2!" | $ZSTD -dcf From ab5ed6fa7fa7f1f917c19894e9c444dc8a1a2e48 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 14 Dec 2016 17:10:38 +0100 Subject: [PATCH 08/10] improved playTests.sh --- tests/playTests.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/playTests.sh b/tests/playTests.sh index c5bc577dd..6f7891fbc 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -106,6 +106,8 @@ $ZSTD --zstd=windowLog=21, && die "wrong parameters not detected!" $ZSTD --zstd=windowLok=21 && die "wrong parameters not detected!" $ZSTD --zstd=windowLog=21,slog= && die "wrong parameters not detected!" roundTripTest -g512K +roundTripTest -g512K " --zstd=slen=3,tlen=48,strat=6" +roundTripTest -g512K " --zstd=strat=6,wlog=23,clog=23,hlog=22,slog=6" roundTripTest -g512K " --zstd=windowLog=23,chainLog=23,hashLog=22,searchLog=6,searchLength=3,targetLength=48,strategy=6" roundTripTest -g512K 19 From 24a4236111d28b354b3712de942ae03b3ef8c161 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 14 Dec 2016 18:07:31 +0100 Subject: [PATCH 09/10] improved playTests.sh (2) --- tests/playTests.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/playTests.sh b/tests/playTests.sh index 6f7891fbc..b98acfb93 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -27,7 +27,6 @@ case "$OS" in Windows*) isWindows=true ECHO="echo -e" - INTOVOID="nul" ;; esac @@ -102,9 +101,10 @@ ls tmp.zst && die "tmp.zst should not be created" $ECHO "\n**** Advanced compression parameters **** " -$ZSTD --zstd=windowLog=21, && die "wrong parameters not detected!" -$ZSTD --zstd=windowLok=21 && die "wrong parameters not detected!" -$ZSTD --zstd=windowLog=21,slog= && die "wrong parameters not detected!" +$ECHO "Hello world!" | $ZSTD --zstd=windowLog=21, - -o tmp.zst && die "wrong parameters not detected!" +$ECHO "Hello world!" | $ZSTD --zstd=windowLo=21 - -o tmp.zst && die "wrong parameters not detected!" +$ECHO "Hello world!" | $ZSTD --zstd=windowLog=21,slog= - -o tmp.zst && die "wrong parameters not detected!" +ls tmp.zst && die "tmp.zst should not be created" roundTripTest -g512K roundTripTest -g512K " --zstd=slen=3,tlen=48,strat=6" roundTripTest -g512K " --zstd=strat=6,wlog=23,clog=23,hlog=22,slog=6" From f9a56668a697885d9b7b217c9ef377085f23a87b Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 14 Dec 2016 18:43:06 +0100 Subject: [PATCH 10/10] improved playTests.sh (3 --- tests/playTests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/playTests.sh b/tests/playTests.sh index b98acfb93..89ff45f3f 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -103,7 +103,7 @@ ls tmp.zst && die "tmp.zst should not be created" $ECHO "\n**** Advanced compression parameters **** " $ECHO "Hello world!" | $ZSTD --zstd=windowLog=21, - -o tmp.zst && die "wrong parameters not detected!" $ECHO "Hello world!" | $ZSTD --zstd=windowLo=21 - -o tmp.zst && die "wrong parameters not detected!" -$ECHO "Hello world!" | $ZSTD --zstd=windowLog=21,slog= - -o tmp.zst && die "wrong parameters not detected!" +$ECHO "Hello world!" | $ZSTD --zstd=windowLog=21,slog - -o tmp.zst && die "wrong parameters not detected!" ls tmp.zst && die "tmp.zst should not be created" roundTripTest -g512K roundTripTest -g512K " --zstd=slen=3,tlen=48,strat=6"