ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add a dictionary training large corpus test

Browse Source
This commit is contained in:
Sen Huang
2021-09-13 12:29:17 -04:00
committed by senhuang42
parent 1daf3c8dbc
commit 4a498fb9c3
1 changed files with 7 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/playTests.sh
+7 -2
View File
@@ -936,8 +936,13 @@ cat tmp | zstd -14 -f --size-hint=5500 | zstd -t # considerably too low
println "\n===> dictionary tests " println "\n===> dictionary tests "
println "- Test high/low compressibility corpus training"
println "- test with raw dict (content only) " datagen -g12M -P90 > tmpCorpusHighCompress
datagen -g12M -P5 > tmpCorpusLowCompress
zstd --train -B2K tmpCorpusHighCompress -o tmpDictHighCompress
zstd --train -B2K tmpCorpusLowCompress -o tmpDictLowCompress
rm -f tmpCorpusHighCompress tmpCorpusLowCompress tmpDictHighCompress tmpDictLowCompress
println "- Test with raw dict (content only) "
datagen > tmpDict datagen > tmpDict
datagen -g1M | $MD5SUM > tmp1 datagen -g1M | $MD5SUM > tmp1
datagen -g1M | zstd -D tmpDict | zstd -D tmpDict -dvq | $MD5SUM > tmp2 datagen -g1M | zstd -D tmpDict | zstd -D tmpDict -dvq | $MD5SUM > tmp2