Merge pull request #480 from inikep/dev11
advanced mode where each parameter can be selected on the command line
This commit is contained in:
+24
-15
@@ -125,7 +125,8 @@ typedef struct {
|
|||||||
static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||||
const char* displayName, int cLevel,
|
const char* displayName, int cLevel,
|
||||||
const size_t* fileSizes, U32 nbFiles,
|
const size_t* fileSizes, U32 nbFiles,
|
||||||
const void* dictBuffer, size_t dictBufferSize)
|
const void* dictBuffer, size_t dictBufferSize,
|
||||||
|
ZSTD_compressionParameters *comprParams)
|
||||||
{
|
{
|
||||||
size_t const blockSize = ((g_blockSize>=32 && !g_decodeOnly) ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ;
|
size_t const blockSize = ((g_blockSize>=32 && !g_decodeOnly) ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ;
|
||||||
size_t const avgSize = MIN(g_blockSize, (srcSize / nbFiles));
|
size_t const avgSize = MIN(g_blockSize, (srcSize / nbFiles));
|
||||||
@@ -231,12 +232,19 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
|||||||
UTIL_getTime(&clockStart);
|
UTIL_getTime(&clockStart);
|
||||||
|
|
||||||
if (!cCompleted) { /* still some time to do compression tests */
|
if (!cCompleted) { /* still some time to do compression tests */
|
||||||
ZSTD_parameters const zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize);
|
ZSTD_parameters zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize);
|
||||||
ZSTD_customMem const cmem = { NULL, NULL, NULL };
|
ZSTD_customMem const cmem = { NULL, NULL, NULL };
|
||||||
U64 clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1;
|
U64 clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1;
|
||||||
U32 nbLoops = 0;
|
U32 nbLoops = 0;
|
||||||
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, zparams, cmem);
|
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, zparams, cmem);
|
||||||
if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure");
|
if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure");
|
||||||
|
if (comprParams->windowLog) zparams.cParams.windowLog = comprParams->windowLog;
|
||||||
|
if (comprParams->chainLog) zparams.cParams.chainLog = comprParams->chainLog;
|
||||||
|
if (comprParams->hashLog) zparams.cParams.hashLog = comprParams->hashLog;
|
||||||
|
if (comprParams->searchLog) zparams.cParams.searchLog = comprParams->searchLog;
|
||||||
|
if (comprParams->searchLength) zparams.cParams.searchLength = comprParams->searchLength;
|
||||||
|
if (comprParams->targetLength) zparams.cParams.targetLength = comprParams->targetLength;
|
||||||
|
if (comprParams->strategy) zparams.cParams.strategy = (ZSTD_strategy)(comprParams->strategy - 1);
|
||||||
do {
|
do {
|
||||||
U32 blockNb;
|
U32 blockNb;
|
||||||
size_t rSize;
|
size_t rSize;
|
||||||
@@ -247,9 +255,9 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
|||||||
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
|
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
|
||||||
cdict);
|
cdict);
|
||||||
} else {
|
} else {
|
||||||
rSize = ZSTD_compressCCtx (ctx,
|
rSize = ZSTD_compress_advanced (ctx,
|
||||||
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
|
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
|
||||||
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize, cLevel);
|
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize, NULL, 0, zparams);
|
||||||
}
|
}
|
||||||
if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_compress_usingCDict() failed : %s", ZSTD_getErrorName(rSize));
|
if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_compress_usingCDict() failed : %s", ZSTD_getErrorName(rSize));
|
||||||
blockTable[blockNb].cSize = rSize;
|
blockTable[blockNb].cSize = rSize;
|
||||||
@@ -387,7 +395,8 @@ static size_t BMK_findMaxMem(U64 requiredMem)
|
|||||||
static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
|
static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
|
||||||
const char* displayName, int cLevel, int cLevelLast,
|
const char* displayName, int cLevel, int cLevelLast,
|
||||||
const size_t* fileSizes, unsigned nbFiles,
|
const size_t* fileSizes, unsigned nbFiles,
|
||||||
const void* dictBuffer, size_t dictBufferSize)
|
const void* dictBuffer, size_t dictBufferSize,
|
||||||
|
ZSTD_compressionParameters *compressionParams)
|
||||||
{
|
{
|
||||||
int l;
|
int l;
|
||||||
|
|
||||||
@@ -406,7 +415,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
|
|||||||
BMK_benchMem(srcBuffer, benchedSize,
|
BMK_benchMem(srcBuffer, benchedSize,
|
||||||
displayName, l,
|
displayName, l,
|
||||||
fileSizes, nbFiles,
|
fileSizes, nbFiles,
|
||||||
dictBuffer, dictBufferSize);
|
dictBuffer, dictBufferSize, compressionParams);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -443,8 +452,8 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize,
|
|||||||
if (totalSize == 0) EXM_THROW(12, "no data to bench");
|
if (totalSize == 0) EXM_THROW(12, "no data to bench");
|
||||||
}
|
}
|
||||||
|
|
||||||
static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
|
static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName,
|
||||||
const char* dictFileName, int cLevel, int cLevelLast)
|
int cLevel, int cLevelLast, ZSTD_compressionParameters *compressionParams)
|
||||||
{
|
{
|
||||||
void* srcBuffer;
|
void* srcBuffer;
|
||||||
size_t benchedSize;
|
size_t benchedSize;
|
||||||
@@ -483,7 +492,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
|
|||||||
BMK_benchCLevel(srcBuffer, benchedSize,
|
BMK_benchCLevel(srcBuffer, benchedSize,
|
||||||
displayName, cLevel, cLevelLast,
|
displayName, cLevel, cLevelLast,
|
||||||
fileSizes, nbFiles,
|
fileSizes, nbFiles,
|
||||||
dictBuffer, dictBufferSize);
|
dictBuffer, dictBufferSize, compressionParams);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* clean up */
|
/* clean up */
|
||||||
@@ -493,7 +502,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility)
|
static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility, ZSTD_compressionParameters* compressionParams)
|
||||||
{
|
{
|
||||||
char name[20] = {0};
|
char name[20] = {0};
|
||||||
size_t benchedSize = 10000000;
|
size_t benchedSize = 10000000;
|
||||||
@@ -507,15 +516,15 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility
|
|||||||
|
|
||||||
/* Bench */
|
/* Bench */
|
||||||
snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100));
|
snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100));
|
||||||
BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0);
|
BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0, compressionParams);
|
||||||
|
|
||||||
/* clean up */
|
/* clean up */
|
||||||
free(srcBuffer);
|
free(srcBuffer);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
|
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName,
|
||||||
const char* dictFileName, int cLevel, int cLevelLast)
|
int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams)
|
||||||
{
|
{
|
||||||
double const compressibility = (double)g_compressibilityDefault / 100;
|
double const compressibility = (double)g_compressibilityDefault / 100;
|
||||||
|
|
||||||
@@ -525,8 +534,8 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
|
|||||||
if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
|
if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
|
||||||
|
|
||||||
if (nbFiles == 0)
|
if (nbFiles == 0)
|
||||||
BMK_syntheticTest(cLevel, cLevelLast, compressibility);
|
BMK_syntheticTest(cLevel, cLevelLast, compressibility, compressionParams);
|
||||||
else
|
else
|
||||||
BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast);
|
BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams);
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|||||||
+4
-2
@@ -12,9 +12,11 @@
|
|||||||
#define BENCH_H_121279284357
|
#define BENCH_H_121279284357
|
||||||
|
|
||||||
#include <stddef.h> /* size_t */
|
#include <stddef.h> /* size_t */
|
||||||
|
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */
|
||||||
|
#include "zstd.h" /* ZSTD_compressionParameters */
|
||||||
|
|
||||||
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
|
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,const char* dictFileName,
|
||||||
const char* dictFileName, int cLevel, int cLevelLast);
|
int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams);
|
||||||
|
|
||||||
/* Set Parameters */
|
/* Set Parameters */
|
||||||
void BMK_SetNbSeconds(unsigned nbLoops);
|
void BMK_SetNbSeconds(unsigned nbLoops);
|
||||||
|
|||||||
+14
-5
@@ -247,7 +247,8 @@ typedef struct {
|
|||||||
FILE* srcFile;
|
FILE* srcFile;
|
||||||
} cRess_t;
|
} cRess_t;
|
||||||
|
|
||||||
static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, U64 srcSize)
|
static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
|
||||||
|
U64 srcSize, ZSTD_compressionParameters* comprParams)
|
||||||
{
|
{
|
||||||
cRess_t ress;
|
cRess_t ress;
|
||||||
memset(&ress, 0, sizeof(ress));
|
memset(&ress, 0, sizeof(ress));
|
||||||
@@ -268,6 +269,13 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, U64 sr
|
|||||||
params.fParams.contentSizeFlag = 1;
|
params.fParams.contentSizeFlag = 1;
|
||||||
params.fParams.checksumFlag = g_checksumFlag;
|
params.fParams.checksumFlag = g_checksumFlag;
|
||||||
params.fParams.noDictIDFlag = !g_dictIDFlag;
|
params.fParams.noDictIDFlag = !g_dictIDFlag;
|
||||||
|
if (comprParams->windowLog) params.cParams.windowLog = comprParams->windowLog;
|
||||||
|
if (comprParams->chainLog) params.cParams.chainLog = comprParams->chainLog;
|
||||||
|
if (comprParams->hashLog) params.cParams.hashLog = comprParams->hashLog;
|
||||||
|
if (comprParams->searchLog) params.cParams.searchLog = comprParams->searchLog;
|
||||||
|
if (comprParams->searchLength) params.cParams.searchLength = comprParams->searchLength;
|
||||||
|
if (comprParams->targetLength) params.cParams.targetLength = comprParams->targetLength;
|
||||||
|
if (comprParams->strategy) params.cParams.strategy = (ZSTD_strategy)(comprParams->strategy - 1);
|
||||||
{ size_t const errorCode = ZSTD_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize);
|
{ size_t const errorCode = ZSTD_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize);
|
||||||
if (ZSTD_isError(errorCode)) EXM_THROW(33, "Error initializing CStream : %s", ZSTD_getErrorName(errorCode));
|
if (ZSTD_isError(errorCode)) EXM_THROW(33, "Error initializing CStream : %s", ZSTD_getErrorName(errorCode));
|
||||||
} }
|
} }
|
||||||
@@ -401,12 +409,12 @@ static int FIO_compressFilename_dstFile(cRess_t ress,
|
|||||||
|
|
||||||
|
|
||||||
int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
|
int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
|
||||||
const char* dictFileName, int compressionLevel)
|
const char* dictFileName, int compressionLevel, ZSTD_compressionParameters* comprParams)
|
||||||
{
|
{
|
||||||
clock_t const start = clock();
|
clock_t const start = clock();
|
||||||
U64 const srcSize = UTIL_getFileSize(srcFileName);
|
U64 const srcSize = UTIL_getFileSize(srcFileName);
|
||||||
|
|
||||||
cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel, srcSize);
|
cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, comprParams);
|
||||||
int const result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName);
|
int const result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName);
|
||||||
|
|
||||||
double const seconds = (double)(clock() - start) / CLOCKS_PER_SEC;
|
double const seconds = (double)(clock() - start) / CLOCKS_PER_SEC;
|
||||||
@@ -419,14 +427,15 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
|
|||||||
|
|
||||||
int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFiles,
|
int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFiles,
|
||||||
const char* suffix,
|
const char* suffix,
|
||||||
const char* dictFileName, int compressionLevel)
|
const char* dictFileName, int compressionLevel,
|
||||||
|
ZSTD_compressionParameters* comprParams)
|
||||||
{
|
{
|
||||||
int missed_files = 0;
|
int missed_files = 0;
|
||||||
size_t dfnSize = FNSPACE;
|
size_t dfnSize = FNSPACE;
|
||||||
char* dstFileName = (char*)malloc(FNSPACE);
|
char* dstFileName = (char*)malloc(FNSPACE);
|
||||||
size_t const suffixSize = suffix ? strlen(suffix) : 0;
|
size_t const suffixSize = suffix ? strlen(suffix) : 0;
|
||||||
U64 const srcSize = (nbFiles != 1) ? 0 : UTIL_getFileSize(inFileNamesTable[0]) ;
|
U64 const srcSize = (nbFiles != 1) ? 0 : UTIL_getFileSize(inFileNamesTable[0]) ;
|
||||||
cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize);
|
cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, comprParams);
|
||||||
|
|
||||||
/* init */
|
/* init */
|
||||||
if (dstFileName==NULL) EXM_THROW(27, "FIO_compressMultipleFilenames : allocation error for dstFileName");
|
if (dstFileName==NULL) EXM_THROW(27, "FIO_compressMultipleFilenames : allocation error for dstFileName");
|
||||||
|
|||||||
+7
-2
@@ -11,6 +11,9 @@
|
|||||||
#ifndef FILEIO_H_23981798732
|
#ifndef FILEIO_H_23981798732
|
||||||
#define FILEIO_H_23981798732
|
#define FILEIO_H_23981798732
|
||||||
|
|
||||||
|
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */
|
||||||
|
#include "zstd.h" /* ZSTD_compressionParameters */
|
||||||
|
|
||||||
#if defined (__cplusplus)
|
#if defined (__cplusplus)
|
||||||
extern "C" {
|
extern "C" {
|
||||||
#endif
|
#endif
|
||||||
@@ -44,7 +47,8 @@ void FIO_setMemLimit(unsigned memLimit);
|
|||||||
***************************************/
|
***************************************/
|
||||||
/** FIO_compressFilename() :
|
/** FIO_compressFilename() :
|
||||||
@return : 0 == ok; 1 == pb with src file. */
|
@return : 0 == ok; 1 == pb with src file. */
|
||||||
int FIO_compressFilename (const char* outfilename, const char* infilename, const char* dictFileName, int compressionLevel);
|
int FIO_compressFilename (const char* outfilename, const char* infilename, const char* dictFileName,
|
||||||
|
int compressionLevel, ZSTD_compressionParameters* comprParams);
|
||||||
|
|
||||||
/** FIO_decompressFilename() :
|
/** FIO_decompressFilename() :
|
||||||
@return : 0 == ok; 1 == pb with src file. */
|
@return : 0 == ok; 1 == pb with src file. */
|
||||||
@@ -58,7 +62,8 @@ int FIO_decompressFilename (const char* outfilename, const char* infilename, con
|
|||||||
@return : nb of missing files */
|
@return : nb of missing files */
|
||||||
int FIO_compressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles,
|
int FIO_compressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles,
|
||||||
const char* suffix,
|
const char* suffix,
|
||||||
const char* dictFileName, int compressionLevel);
|
const char* dictFileName, int compressionLevel,
|
||||||
|
ZSTD_compressionParameters* comprParams);
|
||||||
|
|
||||||
/** FIO_decompressMultipleFilenames() :
|
/** FIO_decompressMultipleFilenames() :
|
||||||
@return : nb of missing or skipped files */
|
@return : nb of missing or skipped files */
|
||||||
|
|||||||
+109
@@ -264,6 +264,115 @@ and weight typically 100x the target dictionary size (for example, 10 MB for a 1
|
|||||||
cut file into independent blocks of size # (default: no block)
|
cut file into independent blocks of size # (default: no block)
|
||||||
|
|
||||||
|
|
||||||
|
.SH ADVANCED COMPRESSION OPTIONS
|
||||||
|
.TP
|
||||||
|
.B \--zstd[=\fIoptions\fR]
|
||||||
|
.PD
|
||||||
|
\fBzstd\fR provides 22 predefined compression levels. The selected or default predefined compression level can be changed with advanced compression options.
|
||||||
|
The \fIoptions\fR are provided as a comma-separated list. You may specify only the \fIoptions\fR you want to change and the rest will be taken from the selected or default compression level.
|
||||||
|
The list of available \fIoptions\fR:
|
||||||
|
.RS
|
||||||
|
|
||||||
|
.TP
|
||||||
|
.BI strategy= strat
|
||||||
|
.PD 0
|
||||||
|
.TP
|
||||||
|
.BI strat= strat
|
||||||
|
.PD
|
||||||
|
Specify a strategy used by a match finder.
|
||||||
|
.IP ""
|
||||||
|
There are 8 strategies numbered from 0 to 7, from faster to stronger:
|
||||||
|
0=ZSTD_fast, 1=ZSTD_dfast, 2=ZSTD_greedy, 3=ZSTD_lazy, 4=ZSTD_lazy2, 5=ZSTD_btlazy2, 6=ZSTD_btopt, 7=ZSTD_btopt2.
|
||||||
|
.IP ""
|
||||||
|
|
||||||
|
.TP
|
||||||
|
.BI windowLog= wlog
|
||||||
|
.PD 0
|
||||||
|
.TP
|
||||||
|
.BI wlog= wlog
|
||||||
|
.PD
|
||||||
|
Specify the maximum number of bits for a match distance.
|
||||||
|
.IP ""
|
||||||
|
The higher number of bits increases the chance to find a match what usually improves compression ratio.
|
||||||
|
It also increases memory requirements for compressor and decompressor.
|
||||||
|
.IP ""
|
||||||
|
The minimum \fIwlog\fR is 10 (1 KiB) and the maximum is 25 (32 MiB) for 32-bit compilation and 27 (128 MiB) for 64-bit compilation.
|
||||||
|
.IP ""
|
||||||
|
|
||||||
|
.TP
|
||||||
|
.BI hashLog= hlog
|
||||||
|
.PD 0
|
||||||
|
.TP
|
||||||
|
.BI hlog= hlog
|
||||||
|
.PD
|
||||||
|
Specify the maximum number of bits for a hash table.
|
||||||
|
.IP ""
|
||||||
|
The bigger hash table causes less collisions what usually make compression faster but requires more memory during compression.
|
||||||
|
.IP ""
|
||||||
|
The minimum \fIhlog\fR is 6 (64 B) and the maximum is 25 (32 MiB) for 32-bit compilation and 27 (128 MiB) for 64-bit compilation.
|
||||||
|
|
||||||
|
.TP
|
||||||
|
.BI chainLog= clog
|
||||||
|
.PD 0
|
||||||
|
.TP
|
||||||
|
.BI clog= clog
|
||||||
|
.PD
|
||||||
|
Specify the maximum number of bits for a hash chain or a binary tree.
|
||||||
|
.IP ""
|
||||||
|
The higher number of bits increases the chance to find a match what usually improves compression ratio.
|
||||||
|
It also slows down compression speed and increases memory requirements for compression.
|
||||||
|
This option is ignored for the ZSTD_fast strategy.
|
||||||
|
.IP ""
|
||||||
|
The minimum \fIclog\fR is 6 (64 B) and the maximum is 26 (64 MiB) for 32-bit compilation and 28 (256 MiB) for 64-bit compilation.
|
||||||
|
.IP ""
|
||||||
|
|
||||||
|
.TP
|
||||||
|
.BI searchLog= slog
|
||||||
|
.PD 0
|
||||||
|
.TP
|
||||||
|
.BI slog= slog
|
||||||
|
.PD
|
||||||
|
Specify the maximum number of searches in a hash chain or a binary tree using logarithmic scale.
|
||||||
|
.IP ""
|
||||||
|
The bigger number of searches increases the chance to find a match what usually improves compression ratio but decreases compression speed.
|
||||||
|
.IP ""
|
||||||
|
The minimum \fIslog\fR is 1 and the maximum is 24 for 32-bit compilation and 26 for 64-bit compilation.
|
||||||
|
.IP ""
|
||||||
|
|
||||||
|
.TP
|
||||||
|
.BI searchLength= slen
|
||||||
|
.PD 0
|
||||||
|
.TP
|
||||||
|
.BI slen= slen
|
||||||
|
.PD
|
||||||
|
Specify the minimum searched length of a match in a hash table.
|
||||||
|
.IP ""
|
||||||
|
The bigger search length usually decreases compression ratio but improves decompression speed.
|
||||||
|
.IP ""
|
||||||
|
The minimum \fIslen\fR is 3 and the maximum is 7.
|
||||||
|
.IP ""
|
||||||
|
|
||||||
|
.TP
|
||||||
|
.BI targetLength= tlen
|
||||||
|
.PD 0
|
||||||
|
.TP
|
||||||
|
.BI tlen= tlen
|
||||||
|
.PD
|
||||||
|
Specify the minimum match length that causes a match finder to interrupt searching of better matches.
|
||||||
|
.IP ""
|
||||||
|
The bigger minimum match length usually improves compression ratio but decreases compression speed.
|
||||||
|
This option is used only with ZSTD_btopt and ZSTD_btopt2 strategies.
|
||||||
|
.IP ""
|
||||||
|
The minimum \fItlen\fR is 4 and the maximum is 999.
|
||||||
|
.IP ""
|
||||||
|
|
||||||
|
.PP
|
||||||
|
.B An example
|
||||||
|
.br
|
||||||
|
The following parameters sets advanced compression options to predefined level 19 for files bigger than 256 KB:
|
||||||
|
.IP ""
|
||||||
|
\fB--zstd=\fRwindowLog=23,chainLog=23,hashLog=22,searchLog=6,searchLength=3,targetLength=48,strategy=6
|
||||||
|
|
||||||
.SH BUGS
|
.SH BUGS
|
||||||
Report bugs at:- https://github.com/facebook/zstd/issues
|
Report bugs at:- https://github.com/facebook/zstd/issues
|
||||||
|
|
||||||
|
|||||||
+32
-3
@@ -205,6 +205,32 @@ static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/** parseCompressionParameters() :
|
||||||
|
* reads compression parameters from *stringPtr (e.g. "--zstd=wlog=23,clog=23,hlog=22,slog=6,slen=3,tlen=48,strat=6") into *params
|
||||||
|
* @return 1 means that compression parameters were correct
|
||||||
|
* @return 0 in case of malformed parameters
|
||||||
|
*/
|
||||||
|
static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressionParameters* params)
|
||||||
|
{
|
||||||
|
for ( ; ;) {
|
||||||
|
if (longCommandWArg(&stringPtr, "windowLog=") || longCommandWArg(&stringPtr, "wlog=")) { params->windowLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||||
|
if (longCommandWArg(&stringPtr, "chainLog=") || longCommandWArg(&stringPtr, "clog=")) { params->chainLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||||
|
if (longCommandWArg(&stringPtr, "hashLog=") || longCommandWArg(&stringPtr, "hlog=")) { params->hashLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||||
|
if (longCommandWArg(&stringPtr, "searchLog=") || longCommandWArg(&stringPtr, "slog=")) { params->searchLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||||
|
if (longCommandWArg(&stringPtr, "searchLength=") || longCommandWArg(&stringPtr, "slen=")) { params->searchLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||||
|
if (longCommandWArg(&stringPtr, "targetLength=") || longCommandWArg(&stringPtr, "tlen=")) { params->targetLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||||
|
if (longCommandWArg(&stringPtr, "strategy=") || longCommandWArg(&stringPtr, "strat=")) { params->strategy = (ZSTD_strategy)(1 + readU32FromChar(&stringPtr)); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (stringPtr[0] != 0) return 0; /* check the end of string */
|
||||||
|
DISPLAYLEVEL(4, "windowLog=%d\nchainLog=%d\nhashLog=%d\nsearchLog=%d\n", params->windowLog, params->chainLog, params->hashLog, params->searchLog);
|
||||||
|
DISPLAYLEVEL(4, "searchLength=%d\ntargetLength=%d\nstrategy=%d\n", params->searchLength, params->targetLength, params->strategy);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train } zstd_operation_mode;
|
typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train } zstd_operation_mode;
|
||||||
|
|
||||||
#define CLEAN_RETURN(i) { operationResult = (i); goto _end; }
|
#define CLEAN_RETURN(i) { operationResult = (i); goto _end; }
|
||||||
@@ -223,6 +249,7 @@ int main(int argCount, const char* argv[])
|
|||||||
ultra=0,
|
ultra=0,
|
||||||
lastCommand = 0;
|
lastCommand = 0;
|
||||||
zstd_operation_mode operation = zom_compress;
|
zstd_operation_mode operation = zom_compress;
|
||||||
|
ZSTD_compressionParameters compressionParams;
|
||||||
int cLevel = ZSTDCLI_CLEVEL_DEFAULT;
|
int cLevel = ZSTDCLI_CLEVEL_DEFAULT;
|
||||||
int cLevelLast = 1;
|
int cLevelLast = 1;
|
||||||
unsigned recursive = 0;
|
unsigned recursive = 0;
|
||||||
@@ -259,6 +286,7 @@ int main(int argCount, const char* argv[])
|
|||||||
/* preset behaviors */
|
/* preset behaviors */
|
||||||
if (!strcmp(programName, ZSTD_UNZSTD)) operation=zom_decompress;
|
if (!strcmp(programName, ZSTD_UNZSTD)) operation=zom_decompress;
|
||||||
if (!strcmp(programName, ZSTD_CAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; displayLevel=1; }
|
if (!strcmp(programName, ZSTD_CAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; displayLevel=1; }
|
||||||
|
memset(&compressionParams, 0, sizeof(compressionParams));
|
||||||
|
|
||||||
/* command switches */
|
/* command switches */
|
||||||
for (argNb=1; argNb<argCount; argNb++) {
|
for (argNb=1; argNb<argCount; argNb++) {
|
||||||
@@ -307,6 +335,7 @@ int main(int argCount, const char* argv[])
|
|||||||
if (longCommandWArg(&argument, "--memlimit=")) { memLimit = readU32FromChar(&argument); continue; }
|
if (longCommandWArg(&argument, "--memlimit=")) { memLimit = readU32FromChar(&argument); continue; }
|
||||||
if (longCommandWArg(&argument, "--memory=")) { memLimit = readU32FromChar(&argument); continue; }
|
if (longCommandWArg(&argument, "--memory=")) { memLimit = readU32FromChar(&argument); continue; }
|
||||||
if (longCommandWArg(&argument, "--memlimit-decompress=")) { memLimit = readU32FromChar(&argument); continue; }
|
if (longCommandWArg(&argument, "--memlimit-decompress=")) { memLimit = readU32FromChar(&argument); continue; }
|
||||||
|
if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) CLEAN_RETURN(badusage(programName)); continue; }
|
||||||
/* fall-through, will trigger bad_usage() later on */
|
/* fall-through, will trigger bad_usage() later on */
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -488,7 +517,7 @@ int main(int argCount, const char* argv[])
|
|||||||
if (operation==zom_bench) {
|
if (operation==zom_bench) {
|
||||||
#ifndef ZSTD_NOBENCH
|
#ifndef ZSTD_NOBENCH
|
||||||
BMK_setNotificationLevel(displayLevel);
|
BMK_setNotificationLevel(displayLevel);
|
||||||
BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast);
|
BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams);
|
||||||
#endif
|
#endif
|
||||||
goto _end;
|
goto _end;
|
||||||
}
|
}
|
||||||
@@ -540,9 +569,9 @@ int main(int argCount, const char* argv[])
|
|||||||
if (operation==zom_compress) {
|
if (operation==zom_compress) {
|
||||||
#ifndef ZSTD_NOCOMPRESS
|
#ifndef ZSTD_NOCOMPRESS
|
||||||
if ((filenameIdx==1) && outFileName)
|
if ((filenameIdx==1) && outFileName)
|
||||||
operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel);
|
operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel, &compressionParams);
|
||||||
else
|
else
|
||||||
operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName ? outFileName : ZSTD_EXTENSION, dictFileName, cLevel);
|
operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName ? outFileName : ZSTD_EXTENSION, dictFileName, cLevel, &compressionParams);
|
||||||
#else
|
#else
|
||||||
DISPLAY("Compression not supported\n");
|
DISPLAY("Compression not supported\n");
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
+12
-1
@@ -27,7 +27,6 @@ case "$OS" in
|
|||||||
Windows*)
|
Windows*)
|
||||||
isWindows=true
|
isWindows=true
|
||||||
ECHO="echo -e"
|
ECHO="echo -e"
|
||||||
INTOVOID="nul"
|
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
|
|
||||||
@@ -101,6 +100,18 @@ $ZSTD -f tmp && die "tmp not present : should have failed"
|
|||||||
ls tmp.zst && die "tmp.zst should not be created"
|
ls tmp.zst && die "tmp.zst should not be created"
|
||||||
|
|
||||||
|
|
||||||
|
$ECHO "\n**** Advanced compression parameters **** "
|
||||||
|
$ECHO "Hello world!" | $ZSTD --zstd=windowLog=21, - -o tmp.zst && die "wrong parameters not detected!"
|
||||||
|
$ECHO "Hello world!" | $ZSTD --zstd=windowLo=21 - -o tmp.zst && die "wrong parameters not detected!"
|
||||||
|
$ECHO "Hello world!" | $ZSTD --zstd=windowLog=21,slog - -o tmp.zst && die "wrong parameters not detected!"
|
||||||
|
ls tmp.zst && die "tmp.zst should not be created"
|
||||||
|
roundTripTest -g512K
|
||||||
|
roundTripTest -g512K " --zstd=slen=3,tlen=48,strat=6"
|
||||||
|
roundTripTest -g512K " --zstd=strat=6,wlog=23,clog=23,hlog=22,slog=6"
|
||||||
|
roundTripTest -g512K " --zstd=windowLog=23,chainLog=23,hashLog=22,searchLog=6,searchLength=3,targetLength=48,strategy=6"
|
||||||
|
roundTripTest -g512K 19
|
||||||
|
|
||||||
|
|
||||||
$ECHO "\n**** Pass-Through mode **** "
|
$ECHO "\n**** Pass-Through mode **** "
|
||||||
$ECHO "Hello world 1!" | $ZSTD -df
|
$ECHO "Hello world 1!" | $ZSTD -df
|
||||||
$ECHO "Hello world 2!" | $ZSTD -dcf
|
$ECHO "Hello world 2!" | $ZSTD -dcf
|
||||||
|
|||||||
Reference in New Issue
Block a user