Async write for decompression (#2975)
* Async IO decompression: - Added --[no-]asyncio flag for CLI decompression. - Replaced dstBuffer in decompression with a pool of write jobs. - Added an ability to execute write jobs in a separate thread. - Added an ability to wait (join) on all jobs in a thread pool (queued and running).
This commit is contained in:
@@ -1575,6 +1575,44 @@ elif [ "$longCSize19wlog23" -gt "$optCSize19wlog23" ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
println "\n===> zstd asyncio decompression tests "
|
||||
|
||||
addFrame() {
|
||||
datagen -g2M -s$2 >> tmp_uncompressed
|
||||
datagen -g2M -s$2 | zstd --format=$1 >> tmp_compressed.zst
|
||||
}
|
||||
|
||||
addTwoFrames() {
|
||||
addFrame $1 1
|
||||
addFrame $1 2
|
||||
}
|
||||
|
||||
testAsyncIO() {
|
||||
roundTripTest -g2M "3 --asyncio --format=$1"
|
||||
roundTripTest -g2M "3 --no-asyncio --format=$1"
|
||||
}
|
||||
|
||||
rm -f tmp_compressed tmp_uncompressed
|
||||
testAsyncIO zstd
|
||||
addTwoFrames zstd
|
||||
if [ $GZIPMODE -eq 1 ]; then
|
||||
testAsyncIO gzip
|
||||
addTwoFrames gzip
|
||||
fi
|
||||
if [ $LZMAMODE -eq 1 ]; then
|
||||
testAsyncIO lzma
|
||||
addTwoFrames lzma
|
||||
fi
|
||||
if [ $LZ4MODE -eq 1 ]; then
|
||||
testAsyncIO lz4
|
||||
addTwoFrames lz4
|
||||
fi
|
||||
cat tmp_uncompressed | $MD5SUM > tmp2
|
||||
zstd -d tmp_compressed.zst --asyncio -c | $MD5SUM > tmp1
|
||||
$DIFF -q tmp1 tmp2
|
||||
rm tmp1
|
||||
zstd -d tmp_compressed.zst --no-asyncio -c | $MD5SUM > tmp1
|
||||
$DIFF -q tmp1 tmp2
|
||||
|
||||
if [ "$1" != "--test-large-data" ]; then
|
||||
println "Skipping large data tests"
|
||||
|
||||
Reference in New Issue
Block a user