From dc904ad17bf608db483f6187bdbcd5ed43fea88d Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Fri, 4 Nov 2016 16:18:59 -0700 Subject: [PATCH 1/6] Fix bug in zstd v0.{5, 6} dictionary decompression Introduced by bb68062c590dbd46905907dd2a63a658040a79d4. --- lib/legacy/zstd_v05.c | 2 +- lib/legacy/zstd_v06.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index 201bf3c6b..f63a97fd7 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -2996,7 +2996,7 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx, lhSize=3; litSize = ((istart[0] & 15) << 6) + (istart[1] >> 2); litCSize = ((istart[1] & 3) << 8) + istart[2]; - if (litCSize + litSize > srcSize) return ERROR(corruption_detected); + if (litCSize + lhSize > srcSize) return ERROR(corruption_detected); errorCode = HUFv05_decompress1X4_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTableX4); if (HUFv05_isError(errorCode)) return ERROR(corruption_detected); diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c index b6fde3aa6..88be49438 100644 --- a/lib/legacy/zstd_v06.c +++ b/lib/legacy/zstd_v06.c @@ -3186,7 +3186,7 @@ size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx, lhSize=3; litSize = ((istart[0] & 15) << 6) + (istart[1] >> 2); litCSize = ((istart[1] & 3) << 8) + istart[2]; - if (litCSize + litSize > srcSize) return ERROR(corruption_detected); + if (litCSize + lhSize > srcSize) return ERROR(corruption_detected); { size_t const errorCode = HUFv06_decompress1X4_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTableX4); if (HUFv06_isError(errorCode)) return ERROR(corruption_detected); From fd3be6bc97e33d0db6cbe93984ac71dd3de3327c Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 7 Nov 2016 14:35:41 -0800 Subject: [PATCH 2/6] bump version number to 1.1.2 --- lib/zstd.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/zstd.h b/lib/zstd.h index 6d3cf56c6..eb2451cd3 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -55,7 +55,7 @@ ZSTDLIB_API unsigned ZSTD_versionNumber (void); /**< returns version number of #define ZSTD_VERSION_MAJOR 1 #define ZSTD_VERSION_MINOR 1 -#define ZSTD_VERSION_RELEASE 1 +#define ZSTD_VERSION_RELEASE 2 #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE #define ZSTD_QUOTE(str) #str From 0018ca28dcd1091d6f2903395b2f5e19d8ef6572 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 7 Nov 2016 14:41:21 -0800 Subject: [PATCH 3/6] zstd cli : displays total decoded size, even when a stream consists of multiple frames --- NEWS | 3 +++ programs/fileio.c | 9 +++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/NEWS b/NEWS index 7710a07aa..df23a2914 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,6 @@ +v1.1.2 +New : cli : status displays total amount decoded when stream/file consists of multiple appended frames (like pzstd) + v1.1.1 New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption New : doc/zstd_manual.html, by Przemyslaw Skibinski diff --git a/programs/fileio.c b/programs/fileio.c index a17a78f30..16a74a0f6 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -580,7 +580,8 @@ static void FIO_fwriteSparseEnd(FILE* file, unsigned storedSkips) @return : size of decoded frame */ unsigned long long FIO_decompressFrame(dRess_t ress, - FILE* foutput, FILE* finput, size_t alreadyLoaded) + FILE* foutput, FILE* finput, size_t alreadyLoaded, + U64 alreadyDecoded) { U64 frameSize = 0; size_t readSize; @@ -604,7 +605,7 @@ unsigned long long FIO_decompressFrame(dRess_t ress, /* Write block */ storedSkips = FIO_fwriteSparse(foutput, ress.dstBuffer, outBuff.pos, storedSkips); frameSize += outBuff.pos; - DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(frameSize>>20) ); + DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)((alreadyDecoded+frameSize)>>20) ); if (readSizeHint == 0) break; /* end of frame */ if (inBuff.size != inBuff.pos) EXM_THROW(37, "Decoding error : should consume entire input"); @@ -683,7 +684,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) fclose(srcFile); return 1; } } - filesize += FIO_decompressFrame(ress, dstFile, srcFile, toRead); + filesize += FIO_decompressFrame(ress, dstFile, srcFile, toRead, filesize); } /* Final Status */ @@ -715,7 +716,7 @@ static int FIO_decompressDstFile(dRess_t ress, if (strcmp (srcFileName, stdinmark) && UTIL_getFileStat(srcFileName, &statbuf)) stat_result = 1; result = FIO_decompressSrcFile(ress, srcFileName); - if (fclose(ress.dstFile)) EXM_THROW(38, "Write error : cannot properly close %s", dstFileName); + if (fclose(ress.dstFile)) EXM_THROW(38, "Write error : cannot properly close %s", dstFileName); if ( (result != 0) && strcmp(dstFileName, nulmark) /* special case : don't remove() /dev/null (#316) */ From d41380ea5d12be87e3397577ebd13d0a4063d29d Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 7 Nov 2016 14:55:12 -0800 Subject: [PATCH 4/6] make zstd a phony target at root --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a57587341..856b1a07b 100644 --- a/Makefile +++ b/Makefile @@ -35,8 +35,9 @@ all: lib: @$(MAKE) -C $(ZSTDDIR) +.PHONY: zstd zstd: - @$(MAKE) -C $(PRGDIR) + @$(MAKE) -C $(PRGDIR) $@ cp $(PRGDIR)/zstd$(EXT) . .PHONY: zlibwrapper From 8e4901eccd14d70a7e2bd31054a2c12c234430cf Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 8 Nov 2016 15:45:39 -0800 Subject: [PATCH 5/6] removed zbuff.h from include installation --- lib/Makefile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/Makefile b/lib/Makefile index a928c60c0..02eefc3f7 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -75,8 +75,8 @@ $(LIBZSTD): $(ZSTD_FILES) @$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@ ifeq (,$(filter Windows%,$(OS))) @echo creating versioned links - @ln -sf $(LIBZSTD) libzstd.$(SHARED_EXT_MAJOR) - @ln -sf $(LIBZSTD) libzstd.$(SHARED_EXT) + @ln -sf $@ libzstd.$(SHARED_EXT_MAJOR) + @ln -sf $@ libzstd.$(SHARED_EXT) endif libzstd : $(LIBZSTD) @@ -110,7 +110,6 @@ install: libzstd.a libzstd libzstd.pc @install -m 644 libzstd.a $(DESTDIR)$(LIBDIR)/libzstd.a @install -m 644 zstd.h $(DESTDIR)$(INCLUDEDIR)/zstd.h @install -m 644 common/zstd_errors.h $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h - @install -m 644 common/zbuff.h $(DESTDIR)$(INCLUDEDIR)/zbuff.h # Deprecated streaming functions @install -m 644 dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR)/zdict.h @echo zstd static and shared library installed From cdff19c4b33186315902c4e2991c182a0a21f579 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 11 Nov 2016 17:26:54 -0800 Subject: [PATCH 6/6] minor comment change --- programs/fileio.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/programs/fileio.c b/programs/fileio.c index 16a74a0f6..71593ac90 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -124,7 +124,7 @@ void FIO_setMemLimit(unsigned memLimit) { g_memLimit = memLimit; } DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ DISPLAYLEVEL(1, "Error %i : ", error); \ DISPLAYLEVEL(1, __VA_ARGS__); \ - DISPLAYLEVEL(1, "\n"); \ + DISPLAYLEVEL(1, " \n"); \ exit(error); \ } @@ -132,6 +132,9 @@ void FIO_setMemLimit(unsigned memLimit) { g_memLimit = memLimit; } /*-************************************* * Functions ***************************************/ +/** FIO_openSrcFile() : + * condition : `dstFileName` must be non-NULL. + * @result : FILE* to `dstFileName`, or NULL if it fails */ static FILE* FIO_openSrcFile(const char* srcFileName) { FILE* f;