diff --git a/programs/bench.c b/programs/bench.c index 9104ea89c..6c2383a87 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -125,7 +125,8 @@ typedef struct { static int BMK_benchMem(const void* srcBuffer, size_t srcSize, const char* displayName, int cLevel, const size_t* fileSizes, U32 nbFiles, - const void* dictBuffer, size_t dictBufferSize) + const void* dictBuffer, size_t dictBufferSize, + ZSTD_compressionParameters *comprParams) { size_t const blockSize = ((g_blockSize>=32 && !g_decodeOnly) ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ; size_t const avgSize = MIN(g_blockSize, (srcSize / nbFiles)); @@ -231,12 +232,19 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, UTIL_getTime(&clockStart); if (!cCompleted) { /* still some time to do compression tests */ - ZSTD_parameters const zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize); + ZSTD_parameters zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize); ZSTD_customMem const cmem = { NULL, NULL, NULL }; U64 clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1; U32 nbLoops = 0; ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, zparams, cmem); if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure"); + if (comprParams->windowLog) zparams.cParams.windowLog = comprParams->windowLog; + if (comprParams->chainLog) zparams.cParams.chainLog = comprParams->chainLog; + if (comprParams->hashLog) zparams.cParams.hashLog = comprParams->hashLog; + if (comprParams->searchLog) zparams.cParams.searchLog = comprParams->searchLog; + if (comprParams->searchLength) zparams.cParams.searchLength = comprParams->searchLength; + if (comprParams->targetLength) zparams.cParams.targetLength = comprParams->targetLength; + if (comprParams->strategy) zparams.cParams.strategy = (ZSTD_strategy)(comprParams->strategy - 1); do { U32 blockNb; size_t rSize; @@ -247,9 +255,9 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize, cdict); } else { - rSize = ZSTD_compressCCtx (ctx, + rSize = ZSTD_compress_advanced (ctx, blockTable[blockNb].cPtr, blockTable[blockNb].cRoom, - blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize, cLevel); + blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize, NULL, 0, zparams); } if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_compress_usingCDict() failed : %s", ZSTD_getErrorName(rSize)); blockTable[blockNb].cSize = rSize; @@ -387,7 +395,8 @@ static size_t BMK_findMaxMem(U64 requiredMem) static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, const char* displayName, int cLevel, int cLevelLast, const size_t* fileSizes, unsigned nbFiles, - const void* dictBuffer, size_t dictBufferSize) + const void* dictBuffer, size_t dictBufferSize, + ZSTD_compressionParameters *compressionParams) { int l; @@ -406,7 +415,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, BMK_benchMem(srcBuffer, benchedSize, displayName, l, fileSizes, nbFiles, - dictBuffer, dictBufferSize); + dictBuffer, dictBufferSize, compressionParams); } } @@ -443,8 +452,8 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize, if (totalSize == 0) EXM_THROW(12, "no data to bench"); } -static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, - const char* dictFileName, int cLevel, int cLevelLast) +static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, + int cLevel, int cLevelLast, ZSTD_compressionParameters *compressionParams) { void* srcBuffer; size_t benchedSize; @@ -483,7 +492,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, BMK_benchCLevel(srcBuffer, benchedSize, displayName, cLevel, cLevelLast, fileSizes, nbFiles, - dictBuffer, dictBufferSize); + dictBuffer, dictBufferSize, compressionParams); } /* clean up */ @@ -493,7 +502,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, } -static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility) +static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility, ZSTD_compressionParameters* compressionParams) { char name[20] = {0}; size_t benchedSize = 10000000; @@ -507,15 +516,15 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility /* Bench */ snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100)); - BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0); + BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0, compressionParams); /* clean up */ free(srcBuffer); } -int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, - const char* dictFileName, int cLevel, int cLevelLast) +int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, + int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams) { double const compressibility = (double)g_compressibilityDefault / 100; @@ -525,8 +534,8 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast); if (nbFiles == 0) - BMK_syntheticTest(cLevel, cLevelLast, compressibility); + BMK_syntheticTest(cLevel, cLevelLast, compressibility, compressionParams); else - BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast); + BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams); return 0; } diff --git a/programs/bench.h b/programs/bench.h index 7009dc2f6..314f34655 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -12,9 +12,11 @@ #define BENCH_H_121279284357 #include /* size_t */ +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */ +#include "zstd.h" /* ZSTD_compressionParameters */ -int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, - const char* dictFileName, int cLevel, int cLevelLast); +int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,const char* dictFileName, + int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams); /* Set Parameters */ void BMK_SetNbSeconds(unsigned nbLoops); diff --git a/programs/fileio.c b/programs/fileio.c index a493e97c5..555b56141 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -247,7 +247,8 @@ typedef struct { FILE* srcFile; } cRess_t; -static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, U64 srcSize) +static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, + U64 srcSize, ZSTD_compressionParameters* comprParams) { cRess_t ress; memset(&ress, 0, sizeof(ress)); @@ -268,6 +269,13 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, U64 sr params.fParams.contentSizeFlag = 1; params.fParams.checksumFlag = g_checksumFlag; params.fParams.noDictIDFlag = !g_dictIDFlag; + if (comprParams->windowLog) params.cParams.windowLog = comprParams->windowLog; + if (comprParams->chainLog) params.cParams.chainLog = comprParams->chainLog; + if (comprParams->hashLog) params.cParams.hashLog = comprParams->hashLog; + if (comprParams->searchLog) params.cParams.searchLog = comprParams->searchLog; + if (comprParams->searchLength) params.cParams.searchLength = comprParams->searchLength; + if (comprParams->targetLength) params.cParams.targetLength = comprParams->targetLength; + if (comprParams->strategy) params.cParams.strategy = (ZSTD_strategy)(comprParams->strategy - 1); { size_t const errorCode = ZSTD_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize); if (ZSTD_isError(errorCode)) EXM_THROW(33, "Error initializing CStream : %s", ZSTD_getErrorName(errorCode)); } } @@ -401,12 +409,12 @@ static int FIO_compressFilename_dstFile(cRess_t ress, int FIO_compressFilename(const char* dstFileName, const char* srcFileName, - const char* dictFileName, int compressionLevel) + const char* dictFileName, int compressionLevel, ZSTD_compressionParameters* comprParams) { clock_t const start = clock(); U64 const srcSize = UTIL_getFileSize(srcFileName); - cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel, srcSize); + cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, comprParams); int const result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName); double const seconds = (double)(clock() - start) / CLOCKS_PER_SEC; @@ -419,14 +427,15 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName, int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFiles, const char* suffix, - const char* dictFileName, int compressionLevel) + const char* dictFileName, int compressionLevel, + ZSTD_compressionParameters* comprParams) { int missed_files = 0; size_t dfnSize = FNSPACE; char* dstFileName = (char*)malloc(FNSPACE); size_t const suffixSize = suffix ? strlen(suffix) : 0; U64 const srcSize = (nbFiles != 1) ? 0 : UTIL_getFileSize(inFileNamesTable[0]) ; - cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize); + cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, comprParams); /* init */ if (dstFileName==NULL) EXM_THROW(27, "FIO_compressMultipleFilenames : allocation error for dstFileName"); diff --git a/programs/fileio.h b/programs/fileio.h index 06e0be372..b71658334 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -11,6 +11,9 @@ #ifndef FILEIO_H_23981798732 #define FILEIO_H_23981798732 +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */ +#include "zstd.h" /* ZSTD_compressionParameters */ + #if defined (__cplusplus) extern "C" { #endif @@ -44,7 +47,8 @@ void FIO_setMemLimit(unsigned memLimit); ***************************************/ /** FIO_compressFilename() : @return : 0 == ok; 1 == pb with src file. */ -int FIO_compressFilename (const char* outfilename, const char* infilename, const char* dictFileName, int compressionLevel); +int FIO_compressFilename (const char* outfilename, const char* infilename, const char* dictFileName, + int compressionLevel, ZSTD_compressionParameters* comprParams); /** FIO_decompressFilename() : @return : 0 == ok; 1 == pb with src file. */ @@ -58,7 +62,8 @@ int FIO_decompressFilename (const char* outfilename, const char* infilename, con @return : nb of missing files */ int FIO_compressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles, const char* suffix, - const char* dictFileName, int compressionLevel); + const char* dictFileName, int compressionLevel, + ZSTD_compressionParameters* comprParams); /** FIO_decompressMultipleFilenames() : @return : nb of missing or skipped files */ diff --git a/programs/zstd.1 b/programs/zstd.1 index 63b60d1e9..9b10b1873 100644 --- a/programs/zstd.1 +++ b/programs/zstd.1 @@ -264,6 +264,115 @@ and weight typically 100x the target dictionary size (for example, 10 MB for a 1 cut file into independent blocks of size # (default: no block) +.SH ADVANCED COMPRESSION OPTIONS +.TP +.B \--zstd[=\fIoptions\fR] +.PD +\fBzstd\fR provides 22 predefined compression levels. The selected or default predefined compression level can be changed with advanced compression options. +The \fIoptions\fR are provided as a comma-separated list. You may specify only the \fIoptions\fR you want to change and the rest will be taken from the selected or default compression level. +The list of available \fIoptions\fR: +.RS + +.TP +.BI strategy= strat +.PD 0 +.TP +.BI strat= strat +.PD +Specify a strategy used by a match finder. +.IP "" +There are 8 strategies numbered from 0 to 7, from faster to stronger: +0=ZSTD_fast, 1=ZSTD_dfast, 2=ZSTD_greedy, 3=ZSTD_lazy, 4=ZSTD_lazy2, 5=ZSTD_btlazy2, 6=ZSTD_btopt, 7=ZSTD_btopt2. +.IP "" + +.TP +.BI windowLog= wlog +.PD 0 +.TP +.BI wlog= wlog +.PD +Specify the maximum number of bits for a match distance. +.IP "" +The higher number of bits increases the chance to find a match what usually improves compression ratio. +It also increases memory requirements for compressor and decompressor. +.IP "" +The minimum \fIwlog\fR is 10 (1 KiB) and the maximum is 25 (32 MiB) for 32-bit compilation and 27 (128 MiB) for 64-bit compilation. +.IP "" + +.TP +.BI hashLog= hlog +.PD 0 +.TP +.BI hlog= hlog +.PD +Specify the maximum number of bits for a hash table. +.IP "" +The bigger hash table causes less collisions what usually make compression faster but requires more memory during compression. +.IP "" +The minimum \fIhlog\fR is 6 (64 B) and the maximum is 25 (32 MiB) for 32-bit compilation and 27 (128 MiB) for 64-bit compilation. + +.TP +.BI chainLog= clog +.PD 0 +.TP +.BI clog= clog +.PD +Specify the maximum number of bits for a hash chain or a binary tree. +.IP "" +The higher number of bits increases the chance to find a match what usually improves compression ratio. +It also slows down compression speed and increases memory requirements for compression. +This option is ignored for the ZSTD_fast strategy. +.IP "" +The minimum \fIclog\fR is 6 (64 B) and the maximum is 26 (64 MiB) for 32-bit compilation and 28 (256 MiB) for 64-bit compilation. +.IP "" + +.TP +.BI searchLog= slog +.PD 0 +.TP +.BI slog= slog +.PD +Specify the maximum number of searches in a hash chain or a binary tree using logarithmic scale. +.IP "" +The bigger number of searches increases the chance to find a match what usually improves compression ratio but decreases compression speed. +.IP "" +The minimum \fIslog\fR is 1 and the maximum is 24 for 32-bit compilation and 26 for 64-bit compilation. +.IP "" + +.TP +.BI searchLength= slen +.PD 0 +.TP +.BI slen= slen +.PD +Specify the minimum searched length of a match in a hash table. +.IP "" +The bigger search length usually decreases compression ratio but improves decompression speed. +.IP "" +The minimum \fIslen\fR is 3 and the maximum is 7. +.IP "" + +.TP +.BI targetLength= tlen +.PD 0 +.TP +.BI tlen= tlen +.PD +Specify the minimum match length that causes a match finder to interrupt searching of better matches. +.IP "" +The bigger minimum match length usually improves compression ratio but decreases compression speed. +This option is used only with ZSTD_btopt and ZSTD_btopt2 strategies. +.IP "" +The minimum \fItlen\fR is 4 and the maximum is 999. +.IP "" + +.PP +.B An example +.br +The following parameters sets advanced compression options to predefined level 19 for files bigger than 256 KB: +.IP "" +\fB--zstd=\fRwindowLog=23,chainLog=23,hashLog=22,searchLog=6,searchLength=3,targetLength=48,strategy=6 + .SH BUGS Report bugs at:- https://github.com/facebook/zstd/issues diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 561730a5f..c1eb7b732 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -205,6 +205,32 @@ static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) return result; } + +/** parseCompressionParameters() : + * reads compression parameters from *stringPtr (e.g. "--zstd=wlog=23,clog=23,hlog=22,slog=6,slen=3,tlen=48,strat=6") into *params + * @return 1 means that compression parameters were correct + * @return 0 in case of malformed parameters + */ +static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressionParameters* params) +{ + for ( ; ;) { + if (longCommandWArg(&stringPtr, "windowLog=") || longCommandWArg(&stringPtr, "wlog=")) { params->windowLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "chainLog=") || longCommandWArg(&stringPtr, "clog=")) { params->chainLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "hashLog=") || longCommandWArg(&stringPtr, "hlog=")) { params->hashLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "searchLog=") || longCommandWArg(&stringPtr, "slog=")) { params->searchLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "searchLength=") || longCommandWArg(&stringPtr, "slen=")) { params->searchLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "targetLength=") || longCommandWArg(&stringPtr, "tlen=")) { params->targetLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "strategy=") || longCommandWArg(&stringPtr, "strat=")) { params->strategy = (ZSTD_strategy)(1 + readU32FromChar(&stringPtr)); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + return 0; + } + + if (stringPtr[0] != 0) return 0; /* check the end of string */ + DISPLAYLEVEL(4, "windowLog=%d\nchainLog=%d\nhashLog=%d\nsearchLog=%d\n", params->windowLog, params->chainLog, params->hashLog, params->searchLog); + DISPLAYLEVEL(4, "searchLength=%d\ntargetLength=%d\nstrategy=%d\n", params->searchLength, params->targetLength, params->strategy); + return 1; +} + + typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train } zstd_operation_mode; #define CLEAN_RETURN(i) { operationResult = (i); goto _end; } @@ -223,6 +249,7 @@ int main(int argCount, const char* argv[]) ultra=0, lastCommand = 0; zstd_operation_mode operation = zom_compress; + ZSTD_compressionParameters compressionParams; int cLevel = ZSTDCLI_CLEVEL_DEFAULT; int cLevelLast = 1; unsigned recursive = 0; @@ -259,6 +286,7 @@ int main(int argCount, const char* argv[]) /* preset behaviors */ if (!strcmp(programName, ZSTD_UNZSTD)) operation=zom_decompress; if (!strcmp(programName, ZSTD_CAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; displayLevel=1; } + memset(&compressionParams, 0, sizeof(compressionParams)); /* command switches */ for (argNb=1; argNb