ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2925 from embg/dict_training_sample_limit_size

Browse Source 
Allow user to specify memory limit for dictionary training
This commit is contained in:
Elliot Gorokhovsky
2021-12-15 15:58:17 -05:00
committed by GitHub
parent 622fee6140 71c0c07c19
commit c5f1e826ca
5 changed files with 23 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/playTests.sh
+7
View File
@@ -1051,6 +1051,13 @@ then
fi
rm -f tmp* dictionary
println "- Test --memory for dictionary compression"
datagen -g12M -P90 > tmpCorpusHighCompress
zstd --train -B2K tmpCorpusHighCompress -o tmpDictHighCompress --memory=10K && die "Dictionary training should fail : --memory too low (10K)"
zstd --train -B2K tmpCorpusHighCompress -o tmpDictHighCompress --memory=5MB 2> zstTrainWithMemLimitStdErr
cat zstTrainWithMemLimitStdErr | grep "setting manual memory limit for dictionary training data at 5 MB"
cat zstTrainWithMemLimitStdErr | grep "Training samples set too large (12 MB); training on 5 MB only..."
rm zstTrainWithMemLimitStdErr
println "\n===> fastCover dictionary builder : advanced options "
TESTFILE="$PRGDIR"/zstdcli.c