Merge pull request #217 from Cyan4973/dev

v0.7.1 tag

.travis.yml

@@ -6,85 +6,82 @@ matrix:
     # Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes)
     - os: linux
       sudo: false
-      env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=travis-install
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make travis-install"
     - os: linux
       sudo: false
-      env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=cmaketest
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make cmaketest"
     - os: linux
       sudo: false
-      env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=test
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make test"
     - os: linux
       sudo: false
-      env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM="-C programs test-zstd_nolegacy"
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make -C programs test-zstd_nolegacy"
     - os: linux
       sudo: false
-      env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=usan
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make usan"
     - os: linux
       sudo: false
-      env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=asan
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make asan"
     - os: linux
       sudo: false
-      env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=zlibwrapper
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make zlibwrapper"
     # Standard Ubuntu 12.04 LTS Server Edition 64 bit
     - os: linux
       sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=clangtest
+      env: PLATFORM="Ubuntu 12.04" CMD="make libc6install clangtest"
     - os: linux
       sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gpptest
+      env: PLATFORM="Ubuntu 12.04" CMD="make gppinstall gpptest"
     - os: linux
       sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gnu90test
+      env: PLATFORM="Ubuntu 12.04" CMD="make libc6install gnu90test"
     - os: linux
       sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=c99test
+      env: PLATFORM="Ubuntu 12.04" CMD="make libc6install c99test"
     - os: linux
       sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gnu99test
+      env: PLATFORM="Ubuntu 12.04" CMD="make libc6install gnu99test"
     - os: linux
       sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=armtest-w-install
+      env: PLATFORM="Ubuntu 12.04" CMD="make arminstall armtest"
     - os: linux
       sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C programs test32"
+      env: PLATFORM="Ubuntu 12.04" CMD="make libc6install && make -C programs test32"
     - os: linux
       sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C tests versionsTest"
+      env: PLATFORM="Ubuntu 12.04" CMD="make -C tests versionsTest"
     - os: linux
       sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=asan32
+      env: PLATFORM="Ubuntu 12.04" CMD="make libc6install asan32"
     - os: linux
       sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C programs valgrindTest"
+      env: PLATFORM="Ubuntu 12.04" CMD="make valgrindinstall && make -C programs valgrindTest"
     # Ubuntu 14.04 LTS Server Edition 64 bit
     - os: linux
       dist: trusty
       sudo: required
-      env: PLATFORM="Ubuntu 14.04" MAKE_PARAM=ppctest-w-install
+      env: PLATFORM="Ubuntu 14.04" CMD="make gcc5install gcc5test"
     - os: linux
       dist: trusty
       sudo: required
-      env: PLATFORM="Ubuntu 14.04" MAKE_PARAM=zlibwrapper
+      env: PLATFORM="Ubuntu 14.04" CMD="make gcc6install gcc6test"
+    - os: linux
+      dist: trusty
+      sudo: required
+      env: PLATFORM="Ubuntu 14.04" CMD="make ppcinstall ppctest"
+    - os: linux
+      dist: trusty
+      sudo: required
+      env: PLATFORM="Ubuntu 14.04" CMD="make zlibwrapper"
     # OS X Mavericks
     - os: osx
-      env: PLATFORM="OS X Mavericks" MAKE_PARAM=travis-install
+      env: PLATFORM="OS X Mavericks" CMD="make travis-install"
     - os: osx
-      env: PLATFORM="OS X Mavericks" MAKE_PARAM=gnu90test
+      env: PLATFORM="OS X Mavericks" CMD="make gnu90test"
     - os: osx
-      env: PLATFORM="OS X Mavericks" MAKE_PARAM=test
+      env: PLATFORM="OS X Mavericks" CMD="make test"
   exclude:
     - compiler: gcc
 
-before_install:
-  - set -e
-  - |
-    if [ "$TRAVIS_OS_NAME" == "linux" ]; then
-      CAN_I_RUN_SUDO=$(sudo -n uptime 2>&1|grep "load"|wc -l)
-      echo "CAN_I_RUN_SUDO=$CAN_I_RUN_SUDO\n"
-      if [ ${CAN_I_RUN_SUDO} -gt 0 ]; then
-        sudo apt-get install -y -qq clang g++-multilib gcc-multilib valgrind
-      fi
-    fi
-
 script:
-  - make $MAKE_PARAM
+  - sh -c "$CMD"

Makefile

@@ -51,7 +51,7 @@ all:
 
 zstdprogram:
 	$(MAKE) -C $(PRGDIR)
-	mv $(PRGDIR)/zstd .
+	cp $(PRGDIR)/zstd .
 
 zlibwrapper:
 	$(MAKE) -C $(ZSTDDIR) all
@@ -87,6 +87,14 @@ travis-install:
 gpptest: clean
 	$(MAKE) all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
 
+gcc5test: clean
+	gcc-5 -v
+	$(MAKE) all CC=gcc-5 MOREFLAGS="-Werror"
+
+gcc6test: clean
+	gcc-6 -v
+	$(MAKE) all CC=gcc-6 MOREFLAGS="-Werror"
+
 clangtest: clean
 	clang -v
 	$(MAKE) all CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion"
@@ -95,35 +103,13 @@ armtest: clean
 	$(MAKE) -C $(PRGDIR) datagen   # use native, faster
 	$(MAKE) -C $(PRGDIR) test CC=arm-linux-gnueabi-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static"
 
-# for Travis CI
-arminstall: clean
-	sudo apt-get install -y -q qemu binfmt-support qemu-user-static gcc-arm-linux-gnueabi
-
-# for Travis CI
-armtest-w-install: clean arminstall armtest
-
 ppctest: clean
 	$(MAKE) -C $(PRGDIR) datagen   # use native, faster
 	$(MAKE) -C $(PRGDIR) test CC=powerpc-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static"
 
-# for Travis CI
-ppcinstall: clean
-	# sudo apt-get update  -y -q
-	sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu  # doesn't work with Ubuntu 12.04
-
-# for Travis CI
-ppctest-w-install: clean ppcinstall ppctest
-
 ppc64test: clean
 	$(MAKE) -C $(PRGDIR) datagen   # use native, faster
-	$(MAKE) -C $(PRGDIR) test CC=powerpc64le-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static"
+	$(MAKE) -C $(PRGDIR) test CC=powerpc-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-m64 -Werror -static"
-
-ppc64install: clean
-	sudo apt-get update  -y -q
-	sudo apt-get install -y -q qemu-ppc64le binfmt-support qemu-user-static gcc-powerpc64le-linux-gnu
-	update-binfmts --displ
-
-ppc64test-w-install: clean ppc64install ppc64test
 
 usan: clean
 	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=undefined"
@@ -183,3 +169,37 @@ bmix32test: clean
 bmi32test: clean
 	CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(PRGDIR) test
 endif
+
+
+#------------------------------------------------------------------------
+# for Travis CI
+#------------------------------------------------------------------------
+libc6install:
+	sudo apt-get install -y -qq libc6-dev-i386
+
+gppinstall:
+	sudo apt-get install -y -qq g++-multilib
+
+gcc5install:
+	sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
+	sudo apt-get update -y -qq
+	sudo apt-get install -y -qq gcc-5-multilib
+
+gcc6install:
+	sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
+	sudo apt-get update -y -qq 
+	sudo apt-get install -y -qq gcc-6-multilib
+
+arminstall: clean
+	sudo apt-get install -y -q qemu binfmt-support qemu-user-static gcc-arm-linux-gnueabi
+
+ppcinstall: clean
+	sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu  # doesn't work with Ubuntu 12.04
+
+ppc64install: clean   # compilation works but cannot be executed
+	sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu
+	update-binfmts --displ
+
+valgrindinstall:
+	echo sudo apt-get install -y -qq valgrind
+	sudo apt-get install -y -qq valgrind

NEWS

@@ -1,4 +1,6 @@
 v0.7.1
+fixed : ZBUFF_compressEnd() called multiple times with too small `dst` buffer, reported by Christophe Chevalier
+fixed : dictBuilder fails if first sample is too small, reported by Руслан Ковалёв
 fixed : corruption issue, reported by cj
 modified : checksum enabled by default in command line mode
 

lib/common/huf.h

@@ -136,7 +136,7 @@ typedef U32 HUF_DTable;
 size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< single-symbol decoder */
 size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< double-symbols decoder */
 
-size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
+size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< decodes RLE and uncompressed */
 size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */
 size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< single-symbol decoder */
 size_t HUF_decompress4X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< double-symbols decoder */

lib/common/zstd.h

@@ -61,7 +61,7 @@ extern "C" {
 ***************************************/
 #define ZSTD_VERSION_MAJOR    0
 #define ZSTD_VERSION_MINOR    7
-#define ZSTD_VERSION_RELEASE  0
+#define ZSTD_VERSION_RELEASE  1
 
 #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
 #define ZSTD_QUOTE(str) #str
@@ -132,7 +132,7 @@ ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapa
 /*! ZSTD_compress_usingDict() :
 *   Compression using a pre-defined Dictionary content (see dictBuilder).
 *   Note 1 : This function load the dictionary, resulting in a significant startup time.
-*   Note 2 : `dict` must remain valid and unmodified during compression operation.
+*   Note 2 : `dict` must remain accessible and unmodified during compression operation.
 *   Note 3 : `dict` can be `NULL`, in which case, it's equivalent to ZSTD_compressCCtx() */
 ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
                                            void* dst, size_t dstCapacity,
@@ -144,7 +144,7 @@ ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
 *   Decompression using a pre-defined Dictionary content (see dictBuilder).
 *   Dictionary must be identical to the one used during compression.
 *   Note 1 : This function load the dictionary, resulting in a significant startup time
-*   Note 2 : `dict` must remain valid and unmodified during compression operation.
+*   Note 2 : `dict` must remain accessible and unmodified during compression operation.
 *   Note 3 : `dict` can be `NULL`, in which case, it's equivalent to ZSTD_decompressDCtx() */
 ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
                                              void* dst, size_t dstCapacity,
@@ -192,7 +192,7 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
 
 /* ====================================================================================
  * The definitions in this section are considered experimental.
- * They should never be used in association with a dynamic library, as they may change in the future.
+ * They should never be used with a dynamic library, as they may change in the future.
  * They are provided for advanced usages.
  * Use them only in association with static linking.
  * ==================================================================================== */
@@ -322,15 +322,22 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
   It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx()
 
   Then, consume your input using ZSTD_compressContinue().
-  ZSTD_compressContinue() presumes prior data is still accessible and unmodified (up to maximum distance size, see WindowLog).
+  There are some important considerations to keep in mind when using this advanced function :
-  The interface is synchronous, so input will be entirely consumed and produce associated compressed output.
+  - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffer only.
-  You must ensure there is enough space in destination buffer to store compressed data under worst case scenario.
+  - Interface is synchronous : input will be entirely consumed and produce 1+ compressed blocks.
+  - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario.
     Worst case evaluation is provided by ZSTD_compressBound().
+    ZSTD_compressContinue() doesn't guarantee recover after a failed compression.
+  - ZSTD_compressContinue() presumes prior input ***is still accessible and unmodified*** (up to maximum distance size, see WindowLog).
+    It remembers all previous contiguous blocks, plus one separated memory segment (which can itself consists of multiple contiguous blocks)
+  - ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps.
+    In which case, it will "discard" the relevant memory section from its history.
+
 
   Finish a frame with ZSTD_compressEnd(), which will write the epilogue.
   Without epilogue, frames will be considered unfinished (broken) by decoders.
 
-  You can then reuse ZSTD_CCtx to compress some new frame.
+  You can then reuse `ZSTD_CCtx` (ZSTD_compressBegin()) to compress some new frame.
 */
 
 typedef struct {
@@ -357,7 +364,7 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds
   A ZSTD_DCtx object can be re-used multiple times.
 
   First optional operation is to retrieve frame parameters, using ZSTD_getFrameParams(), which doesn't consume the input.
-  It can provide the minimum size of rolling buffer required to properly decompress data,
+  It can provide the minimum size of rolling buffer required to properly decompress data (`windowSize`),
   and optionally the final size of uncompressed content.
   (Note : content size is an optional info that may not be present. 0 means : content size unknown)
   Frame parameters are extracted from the beginning of compressed frame.
@@ -373,7 +380,7 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds
   Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively.
   ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue().
   ZSTD_decompressContinue() requires this exact amount of bytes, or it will fail.
-  ZSTD_decompressContinue() needs previous data blocks during decompression, up to (1 << windowlog).
+  ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize`.
   They should preferably be located contiguously, prior to current block. Alternatively, a round buffer is also possible.
 
   @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).

lib/compress/zbuff_compress.c

@@ -79,7 +79,7 @@ static size_t const ZBUFF_endFrameSize = ZSTD_BLOCKHEADERSIZE;
 *  output : ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + ZBUFF_endFrameSize : ensures it's always possible to write/flush/end a full block at best speed.
 * **************************************************/
 
-typedef enum { ZBUFFcs_init, ZBUFFcs_load, ZBUFFcs_flush } ZBUFF_cStage;
+typedef enum { ZBUFFcs_init, ZBUFFcs_load, ZBUFFcs_flush, ZBUFFcs_final } ZBUFF_cStage;
 
 /* *** Resources *** */
 struct ZBUFF_CCtx_s {
@@ -162,7 +162,7 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
     zbc->inToCompress = 0;
     zbc->inBuffPos = 0;
     zbc->inBuffTarget = zbc->blockSize;
-    zbc->outBuffFlushedSize = 0;
+    zbc->outBuffContentSize = zbc->outBuffFlushedSize = 0;
     zbc->stage = ZBUFFcs_load;
     return 0;   /* ready to go */
 }
@@ -196,7 +196,7 @@ MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src
 static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc,
                               void* dst, size_t* dstCapacityPtr,
                         const void* src, size_t* srcSizePtr,
-                              int flush)   /* aggregate : wait for full block before compressing */
+                              int flush)
 {
     U32 notDone = 1;
     const char* const istart = (const char*)src;
@@ -243,17 +243,20 @@ static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc,
             }
 
         case ZBUFFcs_flush:
-            /* flush into dst */
             {   size_t const toFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize;
                 size_t const flushed = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outBuffFlushedSize, toFlush);
                 op += flushed;
                 zbc->outBuffFlushedSize += flushed;
-                if (toFlush!=flushed) { notDone = 0; break; } /* not enough space within dst to store compressed block : stop there */
+                if (toFlush!=flushed) { notDone = 0; break; } /* dst too small to store flushed data : stop there */
-                zbc->outBuffContentSize = 0;
+                zbc->outBuffContentSize = zbc->outBuffFlushedSize = 0;
-                zbc->outBuffFlushedSize = 0;
                 zbc->stage = ZBUFFcs_load;
                 break;
             }
+
+        case ZBUFFcs_final:
+            notDone = 0;   /* do nothing */
+            break;
+
         default:
             return ERROR(GENERIC);   /* impossible */
         }
@@ -291,19 +294,30 @@ size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr)
     BYTE* const ostart = (BYTE*)dst;
     BYTE* const oend = ostart + *dstCapacityPtr;
     BYTE* op = ostart;
+
+    if (zbc->stage != ZBUFFcs_final) {
+        /* flush whatever remains */
         size_t outSize = *dstCapacityPtr;
-    size_t epilogueSize, remaining;
+        size_t const remainingToFlush = ZBUFF_compressFlush(zbc, dst, &outSize);
-    ZBUFF_compressFlush(zbc, dst, &outSize);     /* flush any remaining inBuff */
         op += outSize;
-    epilogueSize = ZSTD_compressEnd(zbc->zc, zbc->outBuff + zbc->outBuffContentSize, zbc->outBuffSize - zbc->outBuffContentSize);   /* epilogue into outBuff */
+        if (remainingToFlush) {
-    zbc->outBuffContentSize += epilogueSize;
+            *dstCapacityPtr = op-ostart;
-    outSize = oend-op;
+            return remainingToFlush + ZBUFF_endFrameSize;
-    zbc->stage = ZBUFFcs_flush;
+        }
-    remaining = ZBUFF_compressFlush(zbc, op, &outSize);   /* attempt to flush epilogue into dst */
+        /* create epilogue */
-    op += outSize;
+        zbc->stage = ZBUFFcs_final;
-    if (!remaining) zbc->stage = ZBUFFcs_init;   /* close only if nothing left to flush */
+        zbc->outBuffContentSize = ZSTD_compressEnd(zbc->zc, zbc->outBuff, zbc->outBuffSize); /* epilogue into outBuff */
-    *dstCapacityPtr = op-ostart;                 /* tells how many bytes were written */
+    }
-    return remaining;
+
+    /* flush epilogue */
+    {   size_t const toFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize;
+        size_t const flushed = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outBuffFlushedSize, toFlush);
+        op += flushed;
+        zbc->outBuffFlushedSize += flushed;
+        *dstCapacityPtr = op-ostart;
+        if (toFlush==flushed) zbc->stage = ZBUFFcs_init;  /* end reached */
+        return toFlush - flushed;
+    }
 }
 
 

lib/compress/zstd_compress.c

@@ -295,7 +295,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
     zc->seqStore.buffer = zc->hashTable3 + h3Size;
     zc->hufTable = (HUF_CElt*)zc->seqStore.buffer;
     zc->flagStaticTables = 0;
-    zc->seqStore.buffer = ((U32*)(zc->seqStore.buffer)) + 256;
+    zc->seqStore.buffer = ((U32*)(zc->seqStore.buffer)) + 256;  /* note : HUF_CElt* is incomplete type, size is simulated using U32 */
 
     zc->nextToUpdate = 1;
     zc->nextSrc = NULL;
@@ -313,14 +313,17 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
         zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1<<Litbits);
         zc->seqStore.matchLengthFreq = zc->seqStore.litLengthFreq + (MaxLL+1);
         zc->seqStore.offCodeFreq = zc->seqStore.matchLengthFreq + (MaxML+1);
-        zc->seqStore.matchTable = (ZSTD_match_t*)((void*)(zc->seqStore.offCodeFreq + (MaxOff+1)));
+        zc->seqStore.buffer = zc->seqStore.offCodeFreq + (MaxOff+1);
-        zc->seqStore.priceTable = (ZSTD_optimal_t*)((void*)(zc->seqStore.matchTable + ZSTD_OPT_NUM+1));
+        zc->seqStore.matchTable = (ZSTD_match_t*)zc->seqStore.buffer;
+        zc->seqStore.buffer = zc->seqStore.matchTable + ZSTD_OPT_NUM+1;
+        zc->seqStore.priceTable = (ZSTD_optimal_t*)zc->seqStore.buffer;
         zc->seqStore.buffer = zc->seqStore.priceTable + ZSTD_OPT_NUM+1;
         zc->seqStore.litLengthSum = 0;
     }
     zc->seqStore.offsetStart = (U32*)(zc->seqStore.buffer);
-    zc->seqStore.litLengthStart = (U16*) (void*)(zc->seqStore.offsetStart + maxNbSeq);
+    zc->seqStore.buffer = zc->seqStore.offsetStart + maxNbSeq;
-    zc->seqStore.matchLengthStart = (U16*) (void*)(zc->seqStore.litLengthStart + maxNbSeq);
+    zc->seqStore.litLengthStart = (U16*)zc->seqStore.buffer;
+    zc->seqStore.matchLengthStart = zc->seqStore.litLengthStart + maxNbSeq;
     zc->seqStore.llCodeStart = (BYTE*) (zc->seqStore.matchLengthStart + maxNbSeq);
     zc->seqStore.mlCodeStart = zc->seqStore.llCodeStart + maxNbSeq;
     zc->seqStore.offCodeStart = zc->seqStore.mlCodeStart + maxNbSeq;
@@ -1330,8 +1333,6 @@ static void ZSTD_compressBlock_fast_extDict(ZSTD_CCtx* ctx,
 }
 
 
-
-
 /*-*************************************
 *  Binary Tree search
 ***************************************/
@@ -1614,7 +1615,7 @@ U32 ZSTD_insertAndFindFirstIndex (ZSTD_CCtx* zc, const BYTE* ip, U32 mls)
     const U32 target = (U32)(ip - base);
     U32 idx = zc->nextToUpdate;
 
-    while(idx < target) {
+    while(idx < target) { /* catch up */
         size_t const h = ZSTD_hashPtr(base+idx, hashLog, mls);
         NEXT_IN_CHAIN(idx, chainMask) = hashTable[h];
         hashTable[h] = idx;
@@ -1651,7 +1652,7 @@ size_t ZSTD_HcFindBestMatch_generic (
     /* HC4 match finder */
     U32 matchIndex = ZSTD_insertAndFindFirstIndex (zc, ip, mls);
 
-    for ( ; (matchIndex>lowLimit) && (nbAttempts) ; nbAttempts--) {
+    for ( ; (matchIndex>lowLimit) & (nbAttempts>0) ; nbAttempts--) {
         const BYTE* match;
         size_t currentMl=0;
         if ((!extDict) || matchIndex >= dictLimit) {
@@ -1665,7 +1666,7 @@ size_t ZSTD_HcFindBestMatch_generic (
         }
 
         /* save best solution */
-        if (currentMl > ml) { ml = currentMl; *offsetPtr = ZSTD_REP_MOVE + current - matchIndex; if (ip+currentMl == iLimit) break; /* best possible, and avoid read overflow*/ }
+        if (currentMl > ml) { ml = currentMl; *offsetPtr = current - matchIndex + ZSTD_REP_MOVE; if (ip+currentMl == iLimit) break; /* best possible, and avoid read overflow*/ }
 
         if (matchIndex <= minChain) break;
         matchIndex = NEXT_IN_CHAIN(matchIndex, chainMask);

lib/dictBuilder/zdict.c

@@ -584,12 +584,15 @@ static void ZDICT_countEStats(EStats_ress_t esr,
                             U32* countLit, U32* offsetcodeCount, U32* matchlengthCount, U32* litlengthCount, U32* repOffsets,
                             const void* src, size_t srcSize)
 {
-    const seqStore_t* seqStorePtr;
+    size_t cSize;
 
     if (srcSize > ZSTD_BLOCKSIZE_MAX) srcSize = ZSTD_BLOCKSIZE_MAX;   /* protection vs large samples */
     ZSTD_copyCCtx(esr.zc, esr.ref);
-    ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_MAX, src, srcSize);
+    cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_MAX, src, srcSize);
-    seqStorePtr = ZSTD_getSeqStore(esr.zc);
+    if (ZSTD_isError(cSize)) { DISPLAYLEVEL(1, "warning : could not compress sample size %u \n", (U32)srcSize); return; }
+
+    if (cSize) {  /* if == 0; block is not compressible */
+        const seqStore_t* seqStorePtr = ZSTD_getSeqStore(esr.zc);
 
         /* literals stats */
         {   const BYTE* bytePtr;
@@ -625,7 +628,7 @@ static void ZDICT_countEStats(EStats_ress_t esr,
             repOffsets[offset1] += 3;
             repOffsets[offset2] += 1;
         }
-
+    }
 }
 
 /*

programs/bench.c

@@ -42,10 +42,6 @@
 /* *************************************
 *  Constants
 ***************************************/
-#ifndef ZSTD_VERSION
-#  define ZSTD_VERSION ""
-#endif
-
 #define NBLOOPS               3
 #define TIMELOOP_MICROSEC     1*1000000ULL /* 1 second */
 #define ACTIVEPERIOD_MICROSEC 70*1000000ULL /* 70 seconds */
@@ -363,7 +359,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
     memset(&total, 0, sizeof(total));
 
     if (g_displayLevel == 1 && !g_additionalParam)
-        DISPLAY("bench %s: input %u bytes, %i iterations, %u KB blocks\n", ZSTD_VERSION, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10));
+        DISPLAY("bench %s: input %u bytes, %i iterations, %u KB blocks\n", ZSTD_VERSION_STRING, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10));
 
     if (cLevelLast < cLevel) cLevelLast = cLevel;
 

programs/fileio.c

@@ -413,7 +413,7 @@ static int FIO_compressFilename_dstFile(cRess_t ress,
     int result;
 
     ress.dstFile = FIO_openDstFile(dstFileName);
-    if (ress.dstFile==0) { fclose(ress.srcFile); return 1; }
+    if (ress.dstFile==0) return 1;
 
     result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName, cLevel);
 

programs/playTests.sh

@@ -22,10 +22,12 @@ roundTripTest() {
 
 isWindows=false
 ECHO="echo"
+INTOVOID="/dev/null"
 case "$OS" in
   Windows*)
     isWindows=true
     ECHO="echo -e"
+    INTOVOID="nul"
     ;;
 esac
 
@@ -53,19 +55,21 @@ $ECHO "test : null-length file roundtrip"
 $ECHO -n '' | $ZSTD - --stdout | $ZSTD -d --stdout
 $ECHO "test : decompress file with wrong suffix (must fail)"
 $ZSTD -d tmpCompressed && die "wrong suffix error not detected!"
+$ZSTD -df tmp && die "should have refused : wrong extension"
+$ECHO "test : decompress into stdout"
 $ZSTD -d tmpCompressed -c > tmpResult    # decompression using stdout
 $ZSTD --decompress tmpCompressed -c > tmpResult
 $ZSTD --decompress tmpCompressed --stdout > tmpResult
-if [ "$isWindows" = false ] ; then
+$ECHO "test : decompress from stdin into stdout"
-    $ZSTD -d    < tmp.zst > /dev/null    # combine decompression, stdin & stdout
+$ZSTD -dc   < tmp.zst > $INTOVOID   # combine decompression, stdin & stdout
-    $ZSTD -d  - < tmp.zst > /dev/null
+$ZSTD -dc - < tmp.zst > $INTOVOID
-fi
+$ZSTD -d    < tmp.zst > $INTOVOID   # implicit stdout when stdin is used
-$ZSTD -dc   < tmp.zst > /dev/null
+$ZSTD -d  - < tmp.zst > $INTOVOID
-$ZSTD -dc - < tmp.zst > /dev/null
+$ECHO "test : overwrite protection"
 $ZSTD -q tmp && die "overwrite check failed!"
+$ECHO "test : force overwrite"
 $ZSTD -q -f tmp
 $ZSTD -q --force tmp
-$ZSTD -df tmp && die "should have refused : wrong extension"
 $ECHO "test : file removal"
 $ZSTD -f --rm tmp
 ls tmp && die "tmp should no longer be present"
@@ -135,9 +139,9 @@ rm tmpSparse*
 
 $ECHO "\n**** multiple files tests **** "
 
-./datagen -s1        > tmp1 2> /dev/null
+./datagen -s1        > tmp1 2> $INTOVOID
-./datagen -s2 -g100K > tmp2 2> /dev/null
+./datagen -s2 -g100K > tmp2 2> $INTOVOID
-./datagen -s3 -g1M   > tmp3 2> /dev/null
+./datagen -s3 -g1M   > tmp3 2> $INTOVOID
 $ZSTD -f tmp*
 $ECHO "compress tmp* : "
 ls -ls tmp*

programs/zbufftest.c

@@ -421,11 +421,17 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
         }   }
 
         /* final frame epilogue */
-        {   size_t dstBuffSize = cBufferSize - cSize;
+        {   size_t remainingToFlush = (size_t)(-1);
-            size_t const flushError = ZBUFF_compressEnd(zc, cBuffer+cSize, &dstBuffSize);
+            while (remainingToFlush) {
-            CHECK (ZBUFF_isError(flushError), "flush error : %s", ZBUFF_getErrorName(flushError));
+                size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
+                size_t dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
+                U32 const enoughDstSize = dstBuffSize >= remainingToFlush;
+                remainingToFlush = ZBUFF_compressEnd(zc, cBuffer+cSize, &dstBuffSize);
+                CHECK (ZBUFF_isError(remainingToFlush), "flush error : %s", ZBUFF_getErrorName(remainingToFlush));
+                //DISPLAY("flush %u bytes : still within context : %i \n", (U32)dstBuffSize, (int)remainingToFlush);
+                CHECK (enoughDstSize && remainingToFlush, "ZBUFF_compressEnd() not fully flushed, but enough space available");
                 cSize += dstBuffSize;
-        }
+        }   }
         crcOrig = XXH64_digest(&xxhState);
 
         /* multi - fragments decompression test */

tests/test-zstd-speed.py

@@ -8,9 +8,9 @@ import time
 import traceback
 from subprocess import Popen, PIPE
 
-repo_url = 'https://github.com/Cyan4973/zstd.git'
+default_repo_url = 'https://github.com/Cyan4973/zstd.git'
 test_dir_name = 'speedTest'
-
+email_header = '[ZSTD_speedTest]'
 
 def log(text):
     print time.strftime("%Y/%m/%d %H:%M:%S") + ' - ' + text
@@ -126,14 +126,23 @@ def benchmark_and_compare(branch, commit, resultsFileName, lastCLevel, testFileP
         return text
 
 
+def send_simple_email(emails, email_topic, have_mutt, have_mail):
+    if have_mutt:
+        execute('mutt -s "' + email_header + ' ' + email_topic + '" ' + emails + ' </dev/null')
+    elif have_mail:
+        execute('mail -s "' + email_header + ' ' + email_topic + '" ' + emails + ' </dev/null')
+    else:
+        log("e-mail cannot be sent (mail and mutt not found)")
+
+
 def send_email(branch, commit, last_commit, emails, text, results_files, logFileName, lower_limit, have_mutt, have_mail):
     with open(logFileName, "w") as myfile:
         myfile.writelines(text)
         myfile.close()
         if have_mutt:
-            execute("mutt -s \"[ZSTD_speedTest] Warning for branch=" + branch + " commit=" + commit + " last_commit=" + last_commit + " speed<" + str(lower_limit) + "\" " + emails + " -a " + results_files + " < " + logFileName)
+            execute('mutt -s "' + email_header + ' Warning for branch=' + branch + ' commit=' + commit + ' last_commit=' + last_commit + ' speed<' + str(lower_limit) + '" ' + emails + ' -a ' + results_files + ' < ' + logFileName)
         elif have_mail:
-            execute("mail -s \"[ZSTD_speedTest] Warning for branch=" + branch + " commit=" + commit + " last_commit=" + last_commit + " speed<" + str(lower_limit) + "\" " + emails + " < " + logFileName)
+            execute('mail -s "' + email_header + ' Warning for branch=' + branch + ' commit=' + commit + ' last_commit=' + last_commit + ' speed<' + str(lower_limit) + '" ' + emails + ' < ' + logFileName)
         else:
             log("e-mail cannot be sent (mail and mutt not found)")
 
@@ -183,6 +192,7 @@ if __name__ == '__main__':
     parser.add_argument('testFileNames', help='file names list for speed benchmark')
     parser.add_argument('emails', help='list of e-mail addresses to send warnings')
     parser.add_argument('--message', help='attach an additional message to e-mail', default="")
+    parser.add_argument('--repoURL', help='changes default repository URL', default=default_repo_url)
     parser.add_argument('--lowerLimit', type=float, help='send email if speed is lower than given limit', default=0.98)
     parser.add_argument('--maxLoadAvg', type=float, help='maximum load average to start testing', default=0.75)
     parser.add_argument('--lastCLevel', type=int, help='last compression level for testing', default=5)
@@ -197,7 +207,8 @@ if __name__ == '__main__':
         if os.path.isfile(fileName):
             testFilePaths.append(os.path.abspath(fileName))
         else:
-            raise RuntimeError("File not found: " + fileName)
+            log("ERROR: File not found: " + fileName)
+            exit(1)
 
     test_path = os.getcwd() + '/' + test_dir_name     # /path/to/zstd/tests/speedTest 
     clone_path = test_path + '/' + 'zstd'             # /path/to/zstd/tests/speedTest/zstd 
@@ -206,19 +217,11 @@ if __name__ == '__main__':
     have_mutt = does_command_exist("mutt --help");
     have_mail = does_command_exist("mail -V");
     if not have_mutt and not have_mail:
-        log("WARNING: e-mail senders mail and mutt not found")
+        log("ERROR: e-mail senders 'mail' or 'mutt' not found")
+        exit(1)
 
-    # clone ZSTD repo if needed
+    print "PARAMETERS:\nrepoURL=%s" % args.repoURL
-    if not os.path.isdir(test_path):
+    print "test_path=%s" % test_path
-        os.mkdir(test_path)
-    if not os.path.isdir(clone_path):
-        execute.cwd = test_path
-        execute('git clone ' + repo_url)
-    if not os.path.isdir(clone_path):
-        raise RuntimeError("ZSTD clone not found: " + clone_path)
-    execute.cwd = clone_path
-
-    print "PARAMETERS:\ntest_path=%s" % test_path
     print "clone_path=%s" % clone_path
     print "testFilePath(%s)=%s" % (len(testFilePaths), testFilePaths)
     print "message=%s" % args.message
@@ -230,12 +233,26 @@ if __name__ == '__main__':
     print "dry_run=%s" % args.dry_run
     print "have_mutt=%s have_mail=%s" % (have_mutt, have_mail)
 
-    while True:
+    # clone ZSTD repo if needed
+    if not os.path.isdir(test_path):
+        os.mkdir(test_path)
+    if not os.path.isdir(clone_path):
+        execute.cwd = test_path
+        execute('git clone ' + args.repoURL)
+    if not os.path.isdir(clone_path):
+        log("ERROR: ZSTD clone not found: " + clone_path)
+        exit(1)
+    execute.cwd = clone_path
+
+    # check if speedTest.pid already exists
     pid = str(os.getpid())
     pidfile = "./speedTest.pid"
     if os.path.isfile(pidfile):
-            log("%s already exists, exiting" % pidfile)
+        log("ERROR: %s already exists, exiting" % pidfile)
-        else:
+        exit(1)
+
+    send_simple_email(args.emails, "test-zstd-speed.py(%s) has been started" % pid, have_mutt, have_mail)
+    while True:
         file(pidfile, 'w').write(pid)
         try:
             loadavg = os.getloadavg()[0]