ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1041 from facebook/fasterFast

Browse Source 
Negative compression levels
This commit is contained in:
Yann Collet
2018-03-13 21:32:46 -07:00
committed by GitHub
parent bd7bb94361 50f763ec44
commit 4c5cbac179
12 changed files with 293 additions and 192 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/bench.c
+4 -5
View File
@@ -214,6 +214,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
/* init */
if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* display last 17 characters */
if (g_nbWorkers==1) g_nbWorkers=0; /* prefer synchronous mode */
if (g_decodeOnly) { /* benchmark only decompression : source must be already compressed */
const char* srcPtr = (const char*)srcBuffer;
@@ -533,9 +534,8 @@ static void BMK_benchCLevel(const void* srcBuffer, size_t benchedSize,
if (g_displayLevel == 1 && !g_additionalParam)
DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbSeconds, (U32)(g_blockSize>>10));
if (cLevelLast < cLevel) cLevelLast = cLevel;
for (l=cLevel; l <= cLevelLast; l++) {
if (l==0) continue; /* skip level 0 */
BMK_benchMem(srcBuffer, benchedSize,
displayName, l,
fileSizes, nbFiles,
@@ -545,8 +545,8 @@ static void BMK_benchCLevel(const void* srcBuffer, size_t benchedSize,
/*! BMK_loadFiles() :
Loads `buffer` with content of files listed within `fileNamesTable`.
At most, fills `buffer` entirely */
* Loads `buffer` with content of files listed within `fileNamesTable`.
* At most, fills `buffer` entirely. */
static void BMK_loadFiles(void* buffer, size_t bufferSize,
size_t* fileSizes,
const char* const * const fileNamesTable, unsigned nbFiles)
@@ -677,7 +677,6 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
{
double const compressibility = (double)g_compressibilityDefault / 100;
if (cLevel < 1) cLevel = 1; /* minimum compression level */
if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel();
if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel();
if (cLevelLast < cLevel) cLevelLast = cLevel;
programs/fileio.c
+3 -1
View File
@@ -420,6 +420,7 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
cRess_t ress;
memset(&ress, 0, sizeof(ress));
DISPLAYLEVEL(6, "FIO_createCResources \n");
ress.cctx = ZSTD_createCCtx();
if (ress.cctx == NULL)
EXM_THROW(30, "allocation error : can't create ZSTD_CCtx");
@@ -440,7 +441,7 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_dictIDFlag, g_dictIDFlag) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_checksumFlag, g_checksumFlag) );
/* compression level */
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, cLevel) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, (unsigned)cLevel) );
/* long distance matching */
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_enableLongDistanceMatching, g_ldmFlag) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmHashLog, g_ldmHashLog) );
@@ -908,6 +909,7 @@ static int FIO_compressFilename_dstFile(cRess_t ress,
stat_t statbuf;
int stat_result = 0;
DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: opening dst: %s", dstFileName);
ress.dstFile = FIO_openDstFile(dstFileName);
if (ress.dstFile==NULL) return 1; /* could not open dstFileName */
/* Must ony be added after FIO_openDstFile() succeeds.
programs/zstd.1.md
+21 -6
View File
@@ -115,6 +115,14 @@ the last one takes effect.
Note: If `windowLog` is set to larger than 27, `--long=windowLog` or
`--memory=windowSize` needs to be passed to the decompressor.
* `--fast[=#]`:
switch to ultra-fast compression levels.
If `=#` is not present, it defaults to `1`.
The higher the value, the faster the compression speed,
at the cost of some compression ratio.
This setting overwrites compression level if one was set previously.
Similarly, if a compression level is set after `--fast`, it overrides it.
* `-T#`, `--threads=#`:
Compress using `#` working threads (default: 1).
If `#` is 0, attempt to detect and use the number of physical CPU cores.
@@ -339,14 +347,21 @@ The list of available _options_:
The minimum _slen_ is 3 and the maximum is 7.
- `targetLen`=_tlen_, `tlen`=_tlen_:
Specify the minimum match length that causes a match finder to stop
searching for better matches.
The impact of this field vary depending on selected strategy.
A larger minimum match length usually improves compression ratio but
decreases compression speed.
This option is only used with strategies ZSTD_btopt and ZSTD_btultra.
For ZSTD\_btopt and ZSTD\_btultra, it specifies the minimum match length
that causes match finder to stop searching for better matches.
A larger `targetLen` usually improves compression ratio
but decreases compression speed.
The minimum _tlen_ is 4 and the maximum is 999.
For ZSTD\_fast, it specifies
the amount of data skipped between match sampling.
Impact is reversed : a larger `targetLen` increases compression speed
but decreases compression ratio.
For all other strategies, this field has no impact.
The minimum _tlen_ is 1 and the maximum is 999.
- `overlapLog`=_ovlog_, `ovlog`=_ovlog_:
Determine `overlapSize`, amount of data reloaded from previous job.
programs/zstdcli.c
+33 -11
View File
@@ -133,7 +133,8 @@ static int usage_advanced(const char* programName)
DISPLAY( " -l : print information about zstd compressed files \n");
#ifndef ZSTD_NOCOMPRESS
DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
DISPLAY( "--long[=#] : enable long distance matching with given window log (default: %u)\n", g_defaultMaxWindowLog);
DISPLAY( "--long[=#]: enable long distance matching with given window log (default: %u)\n", g_defaultMaxWindowLog);
DISPLAY( "--fast[=#]: switch to ultra fast compression level (default: %u)\n", 1);
#ifdef ZSTD_MULTITHREAD
DISPLAY( " -T# : spawns # compression threads (default: 1, 0==# cores) \n");
DISPLAY( " -B# : select size of each job (default: 0==automatic) \n");
@@ -219,10 +220,10 @@ static int exeNameMatch(const char* exeName, const char* test)
}
/*! readU32FromChar() :
@return : unsigned integer value read from input in `char` format
allows and interprets K, KB, KiB, M, MB and MiB suffix.
Will also modify `*stringPtr`, advancing it to position where it stopped reading.
Note : function result can overflow if digit string > MAX_UINT */
* @return : unsigned integer value read from input in `char` format.
* allows and interprets K, KB, KiB, M, MB and MiB suffix.
* Will also modify `*stringPtr`, advancing it to position where it stopped reading.
* Note : function result can overflow if digit string > MAX_UINT */
static unsigned readU32FromChar(const char** stringPtr)
{
unsigned result = 0;
@@ -241,7 +242,7 @@ static unsigned readU32FromChar(const char** stringPtr)
/** longCommandWArg() :
* check if *stringPtr is the same as longCommand.
* If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
* @return 0 and doesn't modify *stringPtr otherwise.
* @return 0 and doesn't modify *stringPtr otherwise.
*/
static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
{
@@ -387,7 +388,7 @@ int main(int argCount, const char* argv[])
zstd_operation_mode operation = zom_compress;
ZSTD_compressionParameters compressionParams;
int cLevel = ZSTDCLI_CLEVEL_DEFAULT;
int cLevelLast = 1;
int cLevelLast = -10000;
unsigned recursive = 0;
unsigned memLimit = 0;
const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */
@@ -543,6 +544,21 @@ int main(int argCount, const char* argv[])
compressionParams.windowLog = ldmWindowLog;
continue;
}
if (longCommandWArg(&argument, "--fast")) {
/* Parse optional window log */
if (*argument == '=') {
U32 fastLevel;
++argument;
fastLevel = readU32FromChar(&argument);
if (fastLevel) cLevel = - (int)fastLevel;
} else if (*argument != 0) {
/* Invalid character following --fast */
CLEAN_RETURN(badusage(programName));
} else {
cLevel = -1; /* default for --fast */
}
continue;
}
/* fall-through, will trigger bad_usage() later on */
}
@@ -813,16 +829,22 @@ int main(int argCount, const char* argv[])
}
#ifndef ZSTD_NODECOMPRESS
if (operation==zom_test) { outFileName=nulmark; FIO_setRemoveSrcFile(0); } /* test mode */
if (operation==zom_test) { outFileName=nulmark; FIO_setRemoveSrcFile(0); } /* test mode */
#endif
/* No input filename ==> use stdin and stdout */
filenameIdx += !filenameIdx; /* filenameTable[0] is stdin by default */
if (!strcmp(filenameTable[0], stdinmark) && !outFileName) outFileName = stdoutmark; /* when input is stdin, default output is stdout */
if (!strcmp(filenameTable[0], stdinmark) && !outFileName)
outFileName = stdoutmark; /* when input is stdin, default output is stdout */
/* Check if input/output defined as console; trigger an error in this case */
if (!strcmp(filenameTable[0], stdinmark) && IS_CONSOLE(stdin) ) CLEAN_RETURN(badusage(programName));
if (outFileName && !strcmp(outFileName, stdoutmark) && IS_CONSOLE(stdout) && !strcmp(filenameTable[0], stdinmark) && !forceStdout && operation!=zom_decompress)
if (!strcmp(filenameTable[0], stdinmark) && IS_CONSOLE(stdin) )
CLEAN_RETURN(badusage(programName));
if ( outFileName && !strcmp(outFileName, stdoutmark)
&& IS_CONSOLE(stdout)
&& !strcmp(filenameTable[0], stdinmark)
&& !forceStdout
&& operation!=zom_decompress )
CLEAN_RETURN(badusage(programName));
#ifndef ZSTD_NOCOMPRESS