ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Increase maximum window size

Browse Source 
* Maximum window size in 32-bit mode is 1GB, since allocations for 2GB fail
  on my Mac.
* Maximum window size in 64-bit mode is 2GB, since that is the largest
  power of 2 that works with the overflow prevention.
* Allow `--long=windowLog` to set the window log, along with
  `--zstd=wlog=#`. These options also set the window size during
  decompression, but don't override `--memory=#` if it is set.
* Present a helpful error message when the window size is too large during
  decompression.
* The long range matcher defaults to a hash log 7 less than the window log,
  which keeps it at 20 for window log 27.
* Keep the default long range matcher window size and the default maximum
  window size at 27 for the API and CLI.
* Add tests that use the maximum window size and hash size for compression
  and decompression.
This commit is contained in:
Nick Terrell
2017-09-26 14:00:01 -07:00
parent 8a4d0abc1c
commit c233bdbaee
12 changed files with 139 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/fileio.c
+33 -2
View File
@@ -37,6 +37,7 @@
# include <io.h>
#endif
#include "bitstream.h"
#include "mem.h"
#include "fileio.h"
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */
@@ -1167,6 +1168,35 @@ static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_
return 0;
}
static void FIO_zstdErrorHelp(dRess_t* ress, size_t ret, char const* srcFileName)
{
ZSTD_frameHeader header;
/* No special help for these errors */
if (ZSTD_getErrorCode(ret) != ZSTD_error_frameParameter_windowTooLarge)
return;
/* Try to decode the frame header */
ret = ZSTD_getFrameHeader(&header, ress->srcBuffer, ress->srcBufferLoaded);
if (ret == 0) {
U32 const windowSize = (U32)header.windowSize;
U32 const windowLog = BIT_highbit32(windowSize) + ((windowSize & (windowSize - 1)) != 0);
U32 const windowMB = (windowSize >> 20) + (windowSize & ((1 MB) - 1));
assert(header.windowSize <= (U64)((U32)-1));
assert(g_memLimit > 0);
DISPLAYLEVEL(1, "%s : Window size larger than maximum : %llu > %u\n",
srcFileName, header.windowSize, g_memLimit);
if (windowLog <= ZSTD_WINDOWLOG_MAX) {
DISPLAYLEVEL(1, "%s : Use --long=%u or --memory=%uMB\n",
srcFileName, windowLog, windowMB);
return;
}
} else if (ZSTD_getErrorCode(ret) != ZSTD_error_frameParameter_windowTooLarge) {
DISPLAYLEVEL(1, "%s : Error decoding frame header to read window size : %s\n",
srcFileName, ZSTD_getErrorName(ret));
return;
}
DISPLAYLEVEL(1, "%s : Window log larger than ZSTD_WINDOWLOG_MAX=%u not supported\n",
srcFileName, ZSTD_WINDOWLOG_MAX);
}
/** FIO_decompressFrame() :
* @return : size of decoded zstd frame, or an error code
@@ -1183,8 +1213,8 @@ unsigned long long FIO_decompressZstdFrame(dRess_t* ress,
ZSTD_resetDStream(ress->dctx);
if (strlen(srcFileName)>20) srcFileName += strlen(srcFileName)-20; /* display last 20 characters */
/* Header loading (optional, saves one loop) */
{ size_t const toRead = 9;
/* Header loading : ensures ZSTD_getFrameHeader() will succeed */
{ size_t const toRead = ZSTD_FRAMEHEADERSIZE_MAX;
if (ress->srcBufferLoaded < toRead)
ress->srcBufferLoaded += fread(((char*)ress->srcBuffer) + ress->srcBufferLoaded, 1, toRead - ress->srcBufferLoaded, finput);
}
@@ -1197,6 +1227,7 @@ unsigned long long FIO_decompressZstdFrame(dRess_t* ress,
if (ZSTD_isError(readSizeHint)) {
DISPLAYLEVEL(1, "%s : Decoding error (36) : %s \n",
srcFileName, ZSTD_getErrorName(readSizeHint));
FIO_zstdErrorHelp(ress, readSizeHint, srcFileName);
return FIO_ERROR_FRAME_DECODING;
}
programs/zstd.1
+14 -4
View File
@@ -104,8 +104,11 @@ Display information related to a zstd compressed file, such as size, ratio, and
unlocks high compression levels 20+ (maximum 22), using a lot more memory\. Note that decompression will also require more memory when using these levels\.
.
.TP
\fB\-\-long\fR
enables long distance matching\. This increases the window size (\fBwindowLog\fR) and memory usage for both the compressor and decompressor\. This setting is designed to improve the compression ratio for files with long matches at a large distance (up to the maximum window size, 128 MiB)\.
\fB\-\-long[=#]\fR
enables long distance matching with \fB#\fR \fBwindowLog\fR, if not \fB#\fR is not present it defaults to \fB27\fR\. This increases the window size (\fBwindowLog\fR) and memory usage for both the compressor and decompressor\. This setting is designed to improve the compression ratio for files with long matches at a large distance\.
.
.IP
Note: If \fBwindowLog\fR is set to larger than 27, \fB\-\-long=windowLog\fR or \fB\-\-memory=windowSize\fR needs to be passed to the decompressor\.
.
.TP
\fB\-T#\fR, \fB\-\-threads=#\fR
@@ -190,6 +193,10 @@ Dictionary saved into \fBfile\fR (default name: dictionary)\.
Limit dictionary to specified size (default: 112640)\.
.
.TP
\fB\-B#\fR
Split input files in blocks of size # (default: no split)
.
.TP
\fB\-\-dictID=#\fR
A dictionary ID is a locally unique ID that a decoder can use to verify it is using the right dictionary\. By default, zstd will create a 4\-bytes random number ID\. It\'s possible to give a precise number instead\. Short numbers have an advantage : an ID < 256 will only need 1 byte in the compressed frame header, and an ID < 65536 will only need 2 bytes\. This compares favorably to 4 bytes default\. However, it\'s up to the dictionary manager to not assign twice the same ID to 2 different dictionaries\.
.
@@ -267,7 +274,10 @@ There are 8 strategies numbered from 1 to 8, from faster to stronger: 1=ZSTD_fas
Specify the maximum number of bits for a match distance\.
.
.IP
The higher number of increases the chance to find a match which usually improves compression ratio\. It also increases memory requirements for the compressor and decompressor\. The minimum \fIwlog\fR is 10 (1 KiB) and the maximum is 27 (128 MiB)\.
The higher number of increases the chance to find a match which usually improves compression ratio\. It also increases memory requirements for the compressor and decompressor\. The minimum \fIwlog\fR is 10 (1 KiB) and the maximum is 30 (1 GiB) on 32\-bit platforms and 31 (2 GiB) on 64\-bit platforms\.
.
.IP
Note: If \fBwindowLog\fR is set to larger than 27, \fB\-\-long=windowLog\fR or \fB\-\-memory=windowSize\fR needs to be passed to the decompressor\.
.
.TP
\fBhashLog\fR=\fIhlog\fR, \fBhlog\fR=\fIhlog\fR
@@ -340,7 +350,7 @@ Bigger hash tables usually improve compression ratio at the expense of more memo
The minimum \fIldmhlog\fR is 6 and the maximum is 26 (default: 20)\.
.
.TP
\fBldmSearchLength\fR=\fIldmslen\fR, \fBldmSlen\fR=\fIldmslen\fR
\fBldmSearchLength\fR=\fIldmslen\fR, \fBldmslen\fR=\fIldmslen\fR
Specify the minimum searched length of a match for long distance matching\.
.
.IP
programs/zstd.1.md
+14 -6
View File
@@ -105,12 +105,16 @@ the last one takes effect.
* `--ultra`:
unlocks high compression levels 20+ (maximum 22), using a lot more memory.
Note that decompression will also require more memory when using these levels.
* `--long`:
enables long distance matching.
* `--long[=#]`:
enables long distance matching with `#` `windowLog`, if not `#` is not
present it defaults to `27`.
This increases the window size (`windowLog`) and memory usage for both the
compressor and decompressor. This setting is designed to improve the
compression ratio for files with long matches at a large distance
(up to the maximum window size, 128 MiB).
compressor and decompressor.
This setting is designed to improve the compression ratio for files with
long matches at a large distance.
Note: If `windowLog` is set to larger than 27, `--long=windowLog` or
`--memory=windowSize` needs to be passed to the decompressor.
* `-T#`, `--threads=#`:
Compress using `#` threads (default: 1).
If `#` is 0, attempt to detect and use the number of physical CPU cores.
@@ -275,7 +279,11 @@ The list of available _options_:
The higher number of increases the chance to find a match which usually
improves compression ratio.
It also increases memory requirements for the compressor and decompressor.
The minimum _wlog_ is 10 (1 KiB) and the maximum is 27 (128 MiB).
The minimum _wlog_ is 10 (1 KiB) and the maximum is 30 (1 GiB) on 32-bit
platforms and 31 (2 GiB) on 64-bit platforms.
Note: If `windowLog` is set to larger than 27, `--long=windowLog` or
`--memory=windowSize` needs to be passed to the decompressor.
- `hashLog`=_hlog_, `hlog`=_hlog_:
Specify the maximum number of bits for a hash table.
programs/zstdcli.c
+25 -2
View File
@@ -72,6 +72,7 @@ static const char* g_defaultDictName = "dictionary";
static const unsigned g_defaultMaxDictSize = 110 KB;
static const int g_defaultDictCLevel = 3;
static const unsigned g_defaultSelectivityLevel = 9;
static const unsigned g_defaultMaxWindowLog = 27;
#define OVERLAP_LOG_DEFAULT 9999
#define LDM_PARAM_DEFAULT 9999 /* Default for parameters where 0 is valid */
static U32 g_overlapLog = OVERLAP_LOG_DEFAULT;
@@ -129,7 +130,7 @@ static int usage_advanced(const char* programName)
DISPLAY( " -l : print information about zstd compressed files \n");
#ifndef ZSTD_NOCOMPRESS
DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
DISPLAY( "--long : enable long distance matching (requires more memory)\n");
DISPLAY( "--long[=#] : enable long distance matching with given window log (default : %u)\n", g_defaultMaxWindowLog);
#ifdef ZSTD_MULTITHREAD
DISPLAY( " -T# : use # threads for compression (default:1) \n");
DISPLAY( " -B# : select size of each job (default:0==automatic) \n");
@@ -459,7 +460,6 @@ int main(int argCount, const char* argv[])
if (!strcmp(argument, "--quiet")) { g_displayLevel--; continue; }
if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; g_displayLevel-=(g_displayLevel==2); continue; }
if (!strcmp(argument, "--ultra")) { ultra=1; continue; }
if (!strcmp(argument, "--long")) { ldmFlag = 1; continue; }
if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(2); continue; }
if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(0); continue; }
if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; }
@@ -514,6 +514,22 @@ int main(int argCount, const char* argv[])
if (longCommandWArg(&argument, "--maxdict=")) { maxDictSize = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "--dictID=")) { dictID = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) CLEAN_RETURN(badusage(programName)); continue; }
if (longCommandWArg(&argument, "--long")) {
unsigned ldmWindowLog = 0;
ldmFlag = 1;
/* Parse optional window log */
if (*argument == '=') {
++argument;
ldmWindowLog = readU32FromChar(&argument);
} else if (*argument != 0) {
/* Invalid character following --long */
CLEAN_RETURN(badusage(programName));
}
/* Only set windowLog if not already set by --zstd */
if (compressionParams.windowLog == 0)
compressionParams.windowLog = ldmWindowLog;
continue;
}
/* fall-through, will trigger bad_usage() later on */
}
@@ -830,6 +846,13 @@ int main(int argCount, const char* argv[])
#endif
} else { /* decompression or test */
#ifndef ZSTD_NODECOMPRESS
if (memLimit == 0) {
if (compressionParams.windowLog == 0)
memLimit = (U32)1 << g_defaultMaxWindowLog;
else {
memLimit = (U32)1 << (compressionParams.windowLog & 31);
}
}
FIO_setMemLimit(memLimit);
if (filenameIdx==1 && outFileName)
operationResult = FIO_decompressFilename(outFileName, filenameTable[0], dictFileName);