ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Allow user to specify memory limit for dictionary training

Browse Source
This commit is contained in:
Elliot Gorokhovsky
2021-12-14 14:29:01 -05:00
parent 57383d2317
commit 71c0c07c19
5 changed files with 23 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/playTests.sh
+7
View File
@@ -1029,6 +1029,13 @@ then
fi
rm -f tmp* dictionary
println "- Test --memory for dictionary compression"
datagen -g12M -P90 > tmpCorpusHighCompress
zstd --train -B2K tmpCorpusHighCompress -o tmpDictHighCompress --memory=10K && die "Dictionary training should fail : --memory too low (10K)"
zstd --train -B2K tmpCorpusHighCompress -o tmpDictHighCompress --memory=5MB 2> zstTrainWithMemLimitStdErr
cat zstTrainWithMemLimitStdErr | grep "setting manual memory limit for dictionary training data at 5 MB"
cat zstTrainWithMemLimitStdErr | grep "Training samples set too large (12 MB); training on 5 MB only..."
rm zstTrainWithMemLimitStdErr
println "\n===> fastCover dictionary builder : advanced options "
TESTFILE="$PRGDIR"/zstdcli.c