ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Increase maximum window size

Browse Source 
* Maximum window size in 32-bit mode is 1GB, since allocations for 2GB fail
  on my Mac.
* Maximum window size in 64-bit mode is 2GB, since that is the largest
  power of 2 that works with the overflow prevention.
* Allow `--long=windowLog` to set the window log, along with
  `--zstd=wlog=#`. These options also set the window size during
  decompression, but don't override `--memory=#` if it is set.
* Present a helpful error message when the window size is too large during
  decompression.
* The long range matcher defaults to a hash log 7 less than the window log,
  which keeps it at 20 for window log 27.
* Keep the default long range matcher window size and the default maximum
  window size at 27 for the API and CLI.
* Add tests that use the maximum window size and hash size for compression
  and decompression.
This commit is contained in:
Nick Terrell
2017-09-26 14:00:01 -07:00
parent 8a4d0abc1c
commit c233bdbaee
12 changed files with 139 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/playTests.sh
+27 -9
View File
@@ -13,11 +13,16 @@ roundTripTest() {
cLevel="$2"
proba=""
fi
if [ -n "$4" ]; then
dLevel="$4"
else
dLevel="$cLevel"
fi
rm -f tmp1 tmp2
$ECHO "roundTripTest: ./datagen $1 $proba | $ZSTD -v$cLevel | $ZSTD -d"
$ECHO "roundTripTest: ./datagen $1 $proba | $ZSTD -v$cLevel | $ZSTD -d$dLevel"
./datagen $1 $proba | $MD5SUM > tmp1
./datagen $1 $proba | $ZSTD --ultra -v$cLevel | $ZSTD -d | $MD5SUM > tmp2
./datagen $1 $proba | $ZSTD --ultra -v$cLevel | $ZSTD -d$dLevel | $MD5SUM > tmp2
$DIFF -q tmp1 tmp2
}
@@ -29,12 +34,17 @@ fileRoundTripTest() {
local_c="$2"
local_p=""
fi
if [ -n "$4" ]; then
local_d="$4"
else
local_d="$local_c"
fi
rm -f tmp.zstd tmp.md5.1 tmp.md5.2
$ECHO "fileRoundTripTest: ./datagen $1 $local_p > tmp && $ZSTD -v$local_c -c tmp | $ZSTD -d"
$ECHO "fileRoundTripTest: ./datagen $1 $local_p > tmp && $ZSTD -v$local_c -c tmp | $ZSTD -d$local_d"
./datagen $1 $local_p > tmp
cat tmp | $MD5SUM > tmp.md5.1
$ZSTD --ultra -v$local_c -c tmp | $ZSTD -d | $MD5SUM > tmp.md5.2
$ZSTD --ultra -v$local_c -c tmp | $ZSTD -d$local_d | $MD5SUM > tmp.md5.2
$DIFF -q tmp.md5.1 tmp.md5.2
}
@@ -669,16 +679,24 @@ roundTripTest -g140000000 -P60 "5 --long"
roundTripTest -g70000000 -P70 "8 --long"
roundTripTest -g18000001 -P80 "18 --long"
fileRoundTripTest -g4100M -P99 "1 --long"
# Test large window logs
roundTripTest -g4100M -P50 "1 --long=30"
roundTripTest -g4100M -P50 "1 --long --zstd=wlog=30,clog=30"
# Test parameter parsing
roundTripTest -g1M -P50 "1 --long=30" " --memory=1024MB"
roundTripTest -g1M -P50 "1 --long=30 --zstd=wlog=29" " --memory=512MB"
roundTripTest -g1M -P50 "1 --long=30" " --long=29 --memory=1024MB"
roundTripTest -g1M -P50 "1 --long=30" " --zstd=wlog=29 --memory=1024MB"
if [ -n "$hasMT" ]
then
$ECHO "\n**** zstdmt long round-trip tests **** "
roundTripTest -g99000000 -P99 "20 -T2"
roundTripTest -g6000000000 -P99 "1 -T2"
roundTripTest -g1500000000 -P97 "1 -T999"
fileRoundTripTest -g4195M -P98 " -T0"
roundTripTest -g1500000000 -P97 "1 --long -T999"
roundTripTest -g99000000 -P99 "20 -T2" " "
roundTripTest -g6000000000 -P99 "1 -T2" " "
roundTripTest -g1500000000 -P97 "1 -T999" " "
fileRoundTripTest -g4195M -P98 " -T0" " "
roundTripTest -g1500000000 -P97 "1 --long=23 -T2" " "
else
$ECHO "\n**** no multithreading, skipping zstdmt tests **** "
fi
tests/zstreamtest.c
+2
View File
@@ -50,6 +50,7 @@ static const U32 g_cLevelMax_smallTests = 10;
#define COMPRESSIBLE_NOISE_LENGTH (10 MB)
#define FUZ_COMPRESSIBILITY_DEFAULT 50
static const U32 prime32 = 2654435761U;
static const U32 windowLogMax = 27;
/*-************************************
@@ -1380,6 +1381,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
/* mess with compression parameters */
cParams.windowLog += (FUZ_rand(&lseed) & 3) - 1;
cParams.windowLog = MIN(windowLogMax, cParams.windowLog);
cParams.hashLog += (FUZ_rand(&lseed) & 3) - 1;
cParams.chainLog += (FUZ_rand(&lseed) & 3) - 1;
cParams.searchLog += (FUZ_rand(&lseed) & 3) - 1;