ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fixed decompression bug (buffered mode)

Browse Source
This commit is contained in:
Yann Collet
2015-11-26 10:32:17 +01:00
parent 91a97960f7
commit 5f2ec63852
3 changed files with 37 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/zstd_buffered.c
+2 -2
View File
@@ -479,10 +479,10 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePt
case ZBUFFds_load:
{
size_t neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc);
size_t toLoad = neededInSize - zbc->inPos;
size_t toLoad = neededInSize - zbc->inPos; /* should always be <= remaining space within inBuff */
size_t loadedSize;
if (toLoad > zbc->inBuffSize - zbc->inPos) return ERROR(corruption_detected); /* should never happen */
loadedSize = ZBUFF_limitCopy(zbc->inBuff + zbc->inPos, zbc->inBuffSize - zbc->inPos, ip, iend-ip);
loadedSize = ZBUFF_limitCopy(zbc->inBuff + zbc->inPos, toLoad, ip, iend-ip);
ip += loadedSize;
zbc->inPos += loadedSize;
if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */
programs/Makefile
+16 -4
View File
@@ -53,10 +53,11 @@ EXT =
VOID = /dev/null
endif
.PHONY: default all clean install uninstall test test32 test-all
default: zstd
all: zstd zstd32 fullbench fullbench32 fuzzer fuzzer32 paramgrill datagen
all: zstd zstd32 fullbench fullbench32 fuzzer fuzzer32 zbufftest zbufftest32 paramgrill datagen
zstd: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/zstd_buffered.c \
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
@@ -93,6 +94,11 @@ zbufftest : $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/
datagen.c xxhash.c zbufftest.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
zbufftest32: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/zstd_buffered.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
datagen.c xxhash.c zbufftest.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
paramgrill : $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
datagen.c xxhash.c paramgrill.c
@@ -105,7 +111,7 @@ clean:
@rm -f core *.o tmp* \
zstd$(EXT) zstd32$(EXT) \
fullbench$(EXT) fullbench32$(EXT) \
fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) \
fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \
datagen$(EXT) paramgrill$(EXT)
@echo Cleaning completed
@@ -135,9 +141,9 @@ uninstall:
[ -f $(DESTDIR)$(MANDIR)/zstd.1 ] && rm -f $(DESTDIR)$(MANDIR)/zstd.1
@echo zstd programs successfully uninstalled
test: test-zstd test-fullbench test-fuzzer
test: test-zstd test-fullbench test-fuzzer test-zbuff
test32: test-zstd32 test-fullbench32 test-fuzzer32
test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32
test-all: test test32 valgrindTest
@@ -233,6 +239,12 @@ test-fuzzer: fuzzer
test-fuzzer32: fuzzer32
./fuzzer32
test-zbuff: zbufftest
./zbufftest
test-zbuff32: zbufftest32
./zbufftest32
valgrindTest: zstd datagen fuzzer fullbench
@echo "\n ---- valgrind tests : memory analyzer ----"
valgrind --leak-check=yes --error-exitcode=1 ./datagen -g50M > $(VOID)
programs/zbufftest.c
+19 -9
View File
@@ -64,7 +64,7 @@
#define MB *(1U<<20)
#define GB *(1U<<30)
static const U32 nbTestsDefault = 30000;
static const U32 nbTestsDefault = 10000;
#define COMPRESSIBLE_NOISE_LENGTH (10 MB)
#define FUZ_COMPRESSIBILITY_DEFAULT 50
static const U32 prime1 = 2654435761U;
@@ -280,7 +280,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
{
size_t sampleSize, sampleStart;
size_t cSize;
size_t maxTestSize, totalTestSize, readSize, genSize;
size_t maxTestSize, totalTestSize, readSize, totalCSize, genSize, totalGenSize;
size_t errorCode;
U32 sampleSizeLog, buffNb, n, nbChunks;
U64 crcOrig, crcDest;
@@ -330,13 +330,12 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
genSize = cBufferSize - cSize;
errorCode = ZBUFF_compressContinue(zc, cBuffer+cSize, &genSize, srcBuffer+sampleStart, &readSize);
CHECK (ZBUFF_isError(errorCode), "compression error : %s", ZBUFF_getErrorName(errorCode));
CHECK (readSize != sampleSize, "test condition not respected : input should be fully consumed")
CHECK (readSize != sampleSize, "compression test condition not respected : input should be fully consumed")
cSize += genSize;
totalTestSize += sampleSize;
if (totalTestSize > maxTestSize) break;
}
genSize = cBufferSize - cSize;
errorCode = ZBUFF_compressEnd(zc, cBuffer+cSize, &genSize);
CHECK (ZBUFF_isError(errorCode), "compression error : %s", ZBUFF_getErrorName(errorCode));
@@ -346,12 +345,23 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
/* multi - fragments decompression test */
ZBUFF_decompressInit(zd);
genSize = dstBufferSize;
readSize = cBufferSize;
errorCode = ZBUFF_decompressContinue(zd, dstBuffer, &genSize, cBuffer, &readSize);
CHECK (ZBUFF_isError(errorCode), "decompression error : %s", ZBUFF_getErrorName(errorCode));
totalCSize = 0;
totalGenSize = 0;
while (totalCSize < cSize)
{
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
readSize = sampleSize;
genSize = dstBufferSize - totalGenSize;
errorCode = ZBUFF_decompressContinue(zd, dstBuffer+totalGenSize, &genSize, cBuffer+totalCSize, &readSize);
CHECK (ZBUFF_isError(errorCode), "decompression error : %s", ZBUFF_getErrorName(errorCode));
totalGenSize += genSize;
totalCSize += readSize;
}
CHECK (errorCode != 0, "frame not fully decoded");
CHECK (genSize != totalTestSize, "decompressed data : wrong size")
CHECK (totalGenSize != totalTestSize, "decompressed data : wrong size")
CHECK (totalCSize != cSize, "compressed data should be fully read")
crcDest = XXH64(dstBuffer, totalTestSize, 0);
if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize);
CHECK (crcDest!=crcOrig, "decompressed data corrupted");