Async write for decompression (#2975)

* Async IO decompression:
- Added --[no-]asyncio flag for CLI decompression.
- Replaced dstBuffer in decompression with a pool of write jobs.
- Added an ability to execute write jobs in a separate thread.
- Added an ability to wait (join) on all jobs in a thread pool (queued and running).
This commit is contained in:
Yonatan Komornik
2022-01-21 13:55:41 -08:00
committed by GitHub
parent 2f03c1996f
commit 1598e6c634
7 changed files with 394 additions and 103 deletions
+38
View File
@@ -1575,6 +1575,44 @@ elif [ "$longCSize19wlog23" -gt "$optCSize19wlog23" ]; then
exit 1
fi
println "\n===> zstd asyncio decompression tests "
addFrame() {
datagen -g2M -s$2 >> tmp_uncompressed
datagen -g2M -s$2 | zstd --format=$1 >> tmp_compressed.zst
}
addTwoFrames() {
addFrame $1 1
addFrame $1 2
}
testAsyncIO() {
roundTripTest -g2M "3 --asyncio --format=$1"
roundTripTest -g2M "3 --no-asyncio --format=$1"
}
rm -f tmp_compressed tmp_uncompressed
testAsyncIO zstd
addTwoFrames zstd
if [ $GZIPMODE -eq 1 ]; then
testAsyncIO gzip
addTwoFrames gzip
fi
if [ $LZMAMODE -eq 1 ]; then
testAsyncIO lzma
addTwoFrames lzma
fi
if [ $LZ4MODE -eq 1 ]; then
testAsyncIO lz4
addTwoFrames lz4
fi
cat tmp_uncompressed | $MD5SUM > tmp2
zstd -d tmp_compressed.zst --asyncio -c | $MD5SUM > tmp1
$DIFF -q tmp1 tmp2
rm tmp1
zstd -d tmp_compressed.zst --no-asyncio -c | $MD5SUM > tmp1
$DIFF -q tmp1 tmp2
if [ "$1" != "--test-large-data" ]; then
println "Skipping large data tests"