ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fixed decompression bug (buffered mode)

Browse Source
This commit is contained in:
Yann Collet
2015-11-26 10:32:17 +01:00
parent 91a97960f7
commit 5f2ec63852
3 changed files with 37 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/zstd_buffered.c
+2 -2
View File
@@ -479,10 +479,10 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePt
case ZBUFFds_load: case ZBUFFds_load:
{ {
size_t neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc); size_t neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc);
size_t toLoad = neededInSize - zbc->inPos; size_t toLoad = neededInSize - zbc->inPos; /* should always be <= remaining space within inBuff */
size_t loadedSize; size_t loadedSize;
if (toLoad > zbc->inBuffSize - zbc->inPos) return ERROR(corruption_detected); /* should never happen */ if (toLoad > zbc->inBuffSize - zbc->inPos) return ERROR(corruption_detected); /* should never happen */
loadedSize = ZBUFF_limitCopy(zbc->inBuff + zbc->inPos, zbc->inBuffSize - zbc->inPos, ip, iend-ip); loadedSize = ZBUFF_limitCopy(zbc->inBuff + zbc->inPos, toLoad, ip, iend-ip);
ip += loadedSize; ip += loadedSize;
zbc->inPos += loadedSize; zbc->inPos += loadedSize;
if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */ if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */
programs/Makefile
+16 -4
View File
@@ -53,10 +53,11 @@ EXT =
VOID = /dev/null VOID = /dev/null
endif endif
.PHONY: default all clean install uninstall test test32 test-all
default: zstd default: zstd
all: zstd zstd32 fullbench fullbench32 fuzzer fuzzer32 paramgrill datagen all: zstd zstd32 fullbench fullbench32 fuzzer fuzzer32 zbufftest zbufftest32 paramgrill datagen
zstd: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/zstd_buffered.c \ zstd: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/zstd_buffered.c \
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \ $(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
@@ -93,6 +94,11 @@ zbufftest : $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/
datagen.c xxhash.c zbufftest.c datagen.c xxhash.c zbufftest.c
$(CC) $(FLAGS) $^ -o $@$(EXT) $(CC) $(FLAGS) $^ -o $@$(EXT)
zbufftest32: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/zstd_buffered.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
datagen.c xxhash.c zbufftest.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
paramgrill : $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \ paramgrill : $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \ $(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
datagen.c xxhash.c paramgrill.c datagen.c xxhash.c paramgrill.c
@@ -105,7 +111,7 @@ clean:
@rm -f core *.o tmp* \ @rm -f core *.o tmp* \
zstd$(EXT) zstd32$(EXT) \ zstd$(EXT) zstd32$(EXT) \
fullbench$(EXT) fullbench32$(EXT) \ fullbench$(EXT) fullbench32$(EXT) \
fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) \ fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \
datagen$(EXT) paramgrill$(EXT) datagen$(EXT) paramgrill$(EXT)
@echo Cleaning completed @echo Cleaning completed
@@ -135,9 +141,9 @@ uninstall:
[ -f $(DESTDIR)$(MANDIR)/zstd.1 ] && rm -f $(DESTDIR)$(MANDIR)/zstd.1 [ -f $(DESTDIR)$(MANDIR)/zstd.1 ] && rm -f $(DESTDIR)$(MANDIR)/zstd.1
@echo zstd programs successfully uninstalled @echo zstd programs successfully uninstalled
test: test-zstd test-fullbench test-fuzzer test: test-zstd test-fullbench test-fuzzer test-zbuff
test32: test-zstd32 test-fullbench32 test-fuzzer32 test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32
test-all: test test32 valgrindTest test-all: test test32 valgrindTest
@@ -233,6 +239,12 @@ test-fuzzer: fuzzer
test-fuzzer32: fuzzer32 test-fuzzer32: fuzzer32
./fuzzer32 ./fuzzer32
test-zbuff: zbufftest
./zbufftest
test-zbuff32: zbufftest32
./zbufftest32
valgrindTest: zstd datagen fuzzer fullbench valgrindTest: zstd datagen fuzzer fullbench
@echo "\n ---- valgrind tests : memory analyzer ----" @echo "\n ---- valgrind tests : memory analyzer ----"
valgrind --leak-check=yes --error-exitcode=1 ./datagen -g50M > $(VOID) valgrind --leak-check=yes --error-exitcode=1 ./datagen -g50M > $(VOID)
programs/zbufftest.c
+19 -9
View File
@@ -64,7 +64,7 @@
#define MB *(1U<<20) #define MB *(1U<<20)
#define GB *(1U<<30) #define GB *(1U<<30)
static const U32 nbTestsDefault = 30000; static const U32 nbTestsDefault = 10000;
#define COMPRESSIBLE_NOISE_LENGTH (10 MB) #define COMPRESSIBLE_NOISE_LENGTH (10 MB)
#define FUZ_COMPRESSIBILITY_DEFAULT 50 #define FUZ_COMPRESSIBILITY_DEFAULT 50
static const U32 prime1 = 2654435761U; static const U32 prime1 = 2654435761U;
@@ -280,7 +280,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
{ {
size_t sampleSize, sampleStart; size_t sampleSize, sampleStart;
size_t cSize; size_t cSize;
size_t maxTestSize, totalTestSize, readSize, genSize; size_t maxTestSize, totalTestSize, readSize, totalCSize, genSize, totalGenSize;
size_t errorCode; size_t errorCode;
U32 sampleSizeLog, buffNb, n, nbChunks; U32 sampleSizeLog, buffNb, n, nbChunks;
U64 crcOrig, crcDest; U64 crcOrig, crcDest;
@@ -330,13 +330,12 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
genSize = cBufferSize - cSize; genSize = cBufferSize - cSize;
errorCode = ZBUFF_compressContinue(zc, cBuffer+cSize, &genSize, srcBuffer+sampleStart, &readSize); errorCode = ZBUFF_compressContinue(zc, cBuffer+cSize, &genSize, srcBuffer+sampleStart, &readSize);
CHECK (ZBUFF_isError(errorCode), "compression error : %s", ZBUFF_getErrorName(errorCode)); CHECK (ZBUFF_isError(errorCode), "compression error : %s", ZBUFF_getErrorName(errorCode));
CHECK (readSize != sampleSize, "test condition not respected : input should be fully consumed") CHECK (readSize != sampleSize, "compression test condition not respected : input should be fully consumed")
cSize += genSize; cSize += genSize;
totalTestSize += sampleSize; totalTestSize += sampleSize;
if (totalTestSize > maxTestSize) break; if (totalTestSize > maxTestSize) break;
} }
genSize = cBufferSize - cSize; genSize = cBufferSize - cSize;
errorCode = ZBUFF_compressEnd(zc, cBuffer+cSize, &genSize); errorCode = ZBUFF_compressEnd(zc, cBuffer+cSize, &genSize);
CHECK (ZBUFF_isError(errorCode), "compression error : %s", ZBUFF_getErrorName(errorCode)); CHECK (ZBUFF_isError(errorCode), "compression error : %s", ZBUFF_getErrorName(errorCode));
@@ -346,12 +345,23 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
/* multi - fragments decompression test */ /* multi - fragments decompression test */
ZBUFF_decompressInit(zd); ZBUFF_decompressInit(zd);
genSize = dstBufferSize; totalCSize = 0;
readSize = cBufferSize; totalGenSize = 0;
errorCode = ZBUFF_decompressContinue(zd, dstBuffer, &genSize, cBuffer, &readSize); while (totalCSize < cSize)
CHECK (ZBUFF_isError(errorCode), "decompression error : %s", ZBUFF_getErrorName(errorCode)); {
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
readSize = sampleSize;
genSize = dstBufferSize - totalGenSize;
errorCode = ZBUFF_decompressContinue(zd, dstBuffer+totalGenSize, &genSize, cBuffer+totalCSize, &readSize);
CHECK (ZBUFF_isError(errorCode), "decompression error : %s", ZBUFF_getErrorName(errorCode));
totalGenSize += genSize;
totalCSize += readSize;
}
CHECK (errorCode != 0, "frame not fully decoded"); CHECK (errorCode != 0, "frame not fully decoded");
CHECK (genSize != totalTestSize, "decompressed data : wrong size") CHECK (totalGenSize != totalTestSize, "decompressed data : wrong size")
CHECK (totalCSize != cSize, "compressed data should be fully read")
crcDest = XXH64(dstBuffer, totalTestSize, 0); crcDest = XXH64(dstBuffer, totalTestSize, 0);
if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize); if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize);
CHECK (crcDest!=crcOrig, "decompressed data corrupted"); CHECK (crcDest!=crcOrig, "decompressed data corrupted");