Merge pull request #222 from Cyan4973/dev

Improved speed test

.gitignore

@@ -13,6 +13,7 @@
 *.dylib
 
 # Executables
+zstd
 *.exe
 *.out
 *.app

.travis.yml

@@ -6,85 +6,82 @@ matrix:
     # Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes)
     - os: linux
       sudo: false
-      env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=travis-install
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make travis-install"
     - os: linux
       sudo: false
-      env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=cmaketest
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make cmaketest"
     - os: linux
       sudo: false
-      env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=test
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make test"
     - os: linux
       sudo: false
-      env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM="-C programs test-zstd_nolegacy"
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make -C programs test-zstd_nolegacy"
     - os: linux
       sudo: false
-      env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=usan
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make usan"
     - os: linux
       sudo: false
-      env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=asan
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make asan"
     - os: linux
       sudo: false
-      env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=zlibwrapper
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make zlibwrapper"
     # Standard Ubuntu 12.04 LTS Server Edition 64 bit
     - os: linux
       sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=clangtest
+      env: PLATFORM="Ubuntu 12.04" CMD="make libc6install clangtest"
     - os: linux
       sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gpptest
+      env: PLATFORM="Ubuntu 12.04" CMD="make gppinstall gpptest"
     - os: linux
       sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gnu90test
+      env: PLATFORM="Ubuntu 12.04" CMD="make libc6install gnu90test"
     - os: linux
       sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=c99test
+      env: PLATFORM="Ubuntu 12.04" CMD="make libc6install c99test"
     - os: linux
       sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gnu99test
+      env: PLATFORM="Ubuntu 12.04" CMD="make libc6install gnu99test"
     - os: linux
       sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=armtest-w-install
+      env: PLATFORM="Ubuntu 12.04" CMD="make arminstall armtest"
     - os: linux
       sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C programs test32"
+      env: PLATFORM="Ubuntu 12.04" CMD="make libc6install && make -C programs test32"
     - os: linux
       sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C tests versionsTest"
+      env: PLATFORM="Ubuntu 12.04" CMD="make -C tests versionsTest"
     - os: linux
       sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=asan32
+      env: PLATFORM="Ubuntu 12.04" CMD="make libc6install asan32"
     - os: linux
       sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C programs valgrindTest"
+      env: PLATFORM="Ubuntu 12.04" CMD="make valgrindinstall && make -C programs valgrindTest"
     # Ubuntu 14.04 LTS Server Edition 64 bit
     - os: linux
       dist: trusty
       sudo: required
-      env: PLATFORM="Ubuntu 14.04" MAKE_PARAM=ppctest-w-install
+      env: PLATFORM="Ubuntu 14.04" CMD="make gcc5install gcc5test"
     - os: linux
       dist: trusty
       sudo: required
-      env: PLATFORM="Ubuntu 14.04" MAKE_PARAM=zlibwrapper
+      env: PLATFORM="Ubuntu 14.04" CMD="make gcc6install gcc6test"
+    - os: linux
+      dist: trusty
+      sudo: required
+      env: PLATFORM="Ubuntu 14.04" CMD="make ppcinstall ppctest"
+    - os: linux
+      dist: trusty
+      sudo: required
+      env: PLATFORM="Ubuntu 14.04" CMD="make zlibwrapper"
     # OS X Mavericks
     - os: osx
-      env: PLATFORM="OS X Mavericks" MAKE_PARAM=travis-install
+      env: PLATFORM="OS X Mavericks" CMD="make travis-install"
     - os: osx
-      env: PLATFORM="OS X Mavericks" MAKE_PARAM=gnu90test
+      env: PLATFORM="OS X Mavericks" CMD="make gnu90test"
     - os: osx
-      env: PLATFORM="OS X Mavericks" MAKE_PARAM=test
+      env: PLATFORM="OS X Mavericks" CMD="make test"
   exclude:
     - compiler: gcc
 
-before_install:
-  - set -e
-  - |
-    if [ "$TRAVIS_OS_NAME" == "linux" ]; then
-      CAN_I_RUN_SUDO=$(sudo -n uptime 2>&1|grep "load"|wc -l)
-      echo "CAN_I_RUN_SUDO=$CAN_I_RUN_SUDO\n"
-      if [ ${CAN_I_RUN_SUDO} -gt 0 ]; then
-        sudo apt-get install -y -qq clang g++-multilib gcc-multilib valgrind
-      fi
-    fi
-
 script:
-  - make $MAKE_PARAM
+  - sh -c "$CMD"

Makefile

@@ -51,6 +51,7 @@ all:
 
 zstdprogram:
 	$(MAKE) -C $(PRGDIR)
+	cp $(PRGDIR)/zstd .
 
 zlibwrapper:
 	$(MAKE) -C $(ZSTDDIR) all
@@ -63,6 +64,7 @@ clean:
 	@$(MAKE) -C $(ZSTDDIR) $@ > $(VOID)
 	@$(MAKE) -C $(PRGDIR) $@ > $(VOID)
 	@$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID)
+	@rm -f zstd
 	@echo Cleaning completed
 
 
@@ -85,6 +87,14 @@ travis-install:
 gpptest: clean
 	$(MAKE) all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
 
+gcc5test: clean
+	gcc-5 -v
+	$(MAKE) all CC=gcc-5 MOREFLAGS="-Werror"
+
+gcc6test: clean
+	gcc-6 -v
+	$(MAKE) all CC=gcc-6 MOREFLAGS="-Werror"
+
 clangtest: clean
 	clang -v
 	$(MAKE) all CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion"
@@ -93,35 +103,13 @@ armtest: clean
 	$(MAKE) -C $(PRGDIR) datagen   # use native, faster
 	$(MAKE) -C $(PRGDIR) test CC=arm-linux-gnueabi-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static"
 
-# for Travis CI
-arminstall: clean
-	sudo apt-get install -y -q qemu binfmt-support qemu-user-static gcc-arm-linux-gnueabi
-
-# for Travis CI
-armtest-w-install: clean arminstall armtest
-
 ppctest: clean
 	$(MAKE) -C $(PRGDIR) datagen   # use native, faster
 	$(MAKE) -C $(PRGDIR) test CC=powerpc-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static"
 
-# for Travis CI
-ppcinstall: clean
-	# sudo apt-get update  -y -q
-	sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu  # doesn't work with Ubuntu 12.04
-
-# for Travis CI
-ppctest-w-install: clean ppcinstall ppctest
-
 ppc64test: clean
 	$(MAKE) -C $(PRGDIR) datagen   # use native, faster
-	$(MAKE) -C $(PRGDIR) test CC=powerpc64le-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static"
+	$(MAKE) -C $(PRGDIR) test CC=powerpc-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-m64 -Werror -static"
-
-ppc64install: clean
-	sudo apt-get update  -y -q
-	sudo apt-get install -y -q qemu-ppc64le binfmt-support qemu-user-static gcc-powerpc64le-linux-gnu
-	update-binfmts --displ
-
-ppc64test-w-install: clean ppc64install ppc64test
 
 usan: clean
 	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=undefined"
@@ -181,3 +169,37 @@ bmix32test: clean
 bmi32test: clean
 	CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(PRGDIR) test
 endif
+
+
+#------------------------------------------------------------------------
+# for Travis CI
+#------------------------------------------------------------------------
+libc6install:
+	sudo apt-get install -y -qq libc6-dev-i386
+
+gppinstall:
+	sudo apt-get install -y -qq g++-multilib
+
+gcc5install:
+	sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
+	sudo apt-get update -y -qq
+	sudo apt-get install -y -qq gcc-5-multilib
+
+gcc6install:
+	sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
+	sudo apt-get update -y -qq 
+	sudo apt-get install -y -qq gcc-6-multilib
+
+arminstall: clean
+	sudo apt-get install -y -q qemu binfmt-support qemu-user-static gcc-arm-linux-gnueabi
+
+ppcinstall: clean
+	sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu  # doesn't work with Ubuntu 12.04
+
+ppc64install: clean   # compilation works but cannot be executed
+	sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu
+	update-binfmts --displ
+
+valgrindinstall:
+	echo sudo apt-get install -y -qq valgrind
+	sudo apt-get install -y -qq valgrind

NEWS

@@ -1,3 +1,9 @@
+v0.7.1
+fixed : ZBUFF_compressEnd() called multiple times with too small `dst` buffer, reported by Christophe Chevalier
+fixed : dictBuilder fails if first sample is too small, reported by Руслан Ковалёв
+fixed : corruption issue, reported by cj
+modified : checksum enabled by default in command line mode
+
 v0.7.0
 New : Support for directory compression, using `-r`, thanks to Przemyslaw Skibinski
 New : Command `--rm`, to remove source file after successful de/compression

lib/common/huf.h

@@ -136,7 +136,7 @@ typedef U32 HUF_DTable;
 size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< single-symbol decoder */
 size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< double-symbols decoder */
 
-size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
+size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< decodes RLE and uncompressed */
 size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */
 size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< single-symbol decoder */
 size_t HUF_decompress4X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< double-symbols decoder */

lib/common/zstd.h

@@ -61,7 +61,7 @@ extern "C" {
 ***************************************/
 #define ZSTD_VERSION_MAJOR    0
 #define ZSTD_VERSION_MINOR    7
-#define ZSTD_VERSION_RELEASE  0
+#define ZSTD_VERSION_RELEASE  1
 
 #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
 #define ZSTD_QUOTE(str) #str
@@ -132,7 +132,7 @@ ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapa
 /*! ZSTD_compress_usingDict() :
 *   Compression using a pre-defined Dictionary content (see dictBuilder).
 *   Note 1 : This function load the dictionary, resulting in a significant startup time.
-*   Note 2 : `dict` must remain valid and unmodified during compression operation.
+*   Note 2 : `dict` must remain accessible and unmodified during compression operation.
 *   Note 3 : `dict` can be `NULL`, in which case, it's equivalent to ZSTD_compressCCtx() */
 ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
                                            void* dst, size_t dstCapacity,
@@ -144,7 +144,7 @@ ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
 *   Decompression using a pre-defined Dictionary content (see dictBuilder).
 *   Dictionary must be identical to the one used during compression.
 *   Note 1 : This function load the dictionary, resulting in a significant startup time
-*   Note 2 : `dict` must remain valid and unmodified during compression operation.
+*   Note 2 : `dict` must remain accessible and unmodified during compression operation.
 *   Note 3 : `dict` can be `NULL`, in which case, it's equivalent to ZSTD_decompressDCtx() */
 ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
                                              void* dst, size_t dstCapacity,
@@ -192,7 +192,7 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
 
 /* ====================================================================================
  * The definitions in this section are considered experimental.
- * They should never be used in association with a dynamic library, as they may change in the future.
+ * They should never be used with a dynamic library, as they may change in the future.
  * They are provided for advanced usages.
  * Use them only in association with static linking.
  * ==================================================================================== */
@@ -322,15 +322,22 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
   It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx()
 
   Then, consume your input using ZSTD_compressContinue().
-  ZSTD_compressContinue() presumes prior data is still accessible and unmodified (up to maximum distance size, see WindowLog).
+  There are some important considerations to keep in mind when using this advanced function :
-  The interface is synchronous, so input will be entirely consumed and produce associated compressed output.
+  - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffer only.
-  You must ensure there is enough space in destination buffer to store compressed data under worst case scenario.
+  - Interface is synchronous : input will be entirely consumed and produce 1+ compressed blocks.
-  Worst case evaluation is provided by ZSTD_compressBound().
+  - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario.
+    Worst case evaluation is provided by ZSTD_compressBound().
+    ZSTD_compressContinue() doesn't guarantee recover after a failed compression.
+  - ZSTD_compressContinue() presumes prior input ***is still accessible and unmodified*** (up to maximum distance size, see WindowLog).
+    It remembers all previous contiguous blocks, plus one separated memory segment (which can itself consists of multiple contiguous blocks)
+  - ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps.
+    In which case, it will "discard" the relevant memory section from its history.
+
 
   Finish a frame with ZSTD_compressEnd(), which will write the epilogue.
   Without epilogue, frames will be considered unfinished (broken) by decoders.
 
-  You can then reuse ZSTD_CCtx to compress some new frame.
+  You can then reuse `ZSTD_CCtx` (ZSTD_compressBegin()) to compress some new frame.
 */
 
 typedef struct {
@@ -357,7 +364,7 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds
   A ZSTD_DCtx object can be re-used multiple times.
 
   First optional operation is to retrieve frame parameters, using ZSTD_getFrameParams(), which doesn't consume the input.
-  It can provide the minimum size of rolling buffer required to properly decompress data,
+  It can provide the minimum size of rolling buffer required to properly decompress data (`windowSize`),
   and optionally the final size of uncompressed content.
   (Note : content size is an optional info that may not be present. 0 means : content size unknown)
   Frame parameters are extracted from the beginning of compressed frame.
@@ -373,7 +380,7 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds
   Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively.
   ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue().
   ZSTD_decompressContinue() requires this exact amount of bytes, or it will fail.
-  ZSTD_decompressContinue() needs previous data blocks during decompression, up to (1 << windowlog).
+  ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize`.
   They should preferably be located contiguously, prior to current block. Alternatively, a round buffer is also possible.
 
   @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).

lib/compress/zbuff_compress.c

@@ -79,7 +79,7 @@ static size_t const ZBUFF_endFrameSize = ZSTD_BLOCKHEADERSIZE;
 *  output : ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + ZBUFF_endFrameSize : ensures it's always possible to write/flush/end a full block at best speed.
 * **************************************************/
 
-typedef enum { ZBUFFcs_init, ZBUFFcs_load, ZBUFFcs_flush } ZBUFF_cStage;
+typedef enum { ZBUFFcs_init, ZBUFFcs_load, ZBUFFcs_flush, ZBUFFcs_final } ZBUFF_cStage;
 
 /* *** Resources *** */
 struct ZBUFF_CCtx_s {
@@ -162,7 +162,7 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
     zbc->inToCompress = 0;
     zbc->inBuffPos = 0;
     zbc->inBuffTarget = zbc->blockSize;
-    zbc->outBuffFlushedSize = 0;
+    zbc->outBuffContentSize = zbc->outBuffFlushedSize = 0;
     zbc->stage = ZBUFFcs_load;
     return 0;   /* ready to go */
 }
@@ -196,7 +196,7 @@ MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src
 static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc,
                               void* dst, size_t* dstCapacityPtr,
                         const void* src, size_t* srcSizePtr,
-                              int flush)   /* aggregate : wait for full block before compressing */
+                              int flush)
 {
     U32 notDone = 1;
     const char* const istart = (const char*)src;
@@ -243,17 +243,20 @@ static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc,
             }
 
         case ZBUFFcs_flush:
-            /* flush into dst */
             {   size_t const toFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize;
                 size_t const flushed = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outBuffFlushedSize, toFlush);
                 op += flushed;
                 zbc->outBuffFlushedSize += flushed;
-                if (toFlush!=flushed) { notDone = 0; break; } /* not enough space within dst to store compressed block : stop there */
+                if (toFlush!=flushed) { notDone = 0; break; } /* dst too small to store flushed data : stop there */
-                zbc->outBuffContentSize = 0;
+                zbc->outBuffContentSize = zbc->outBuffFlushedSize = 0;
-                zbc->outBuffFlushedSize = 0;
                 zbc->stage = ZBUFFcs_load;
                 break;
             }
+
+        case ZBUFFcs_final:
+            notDone = 0;   /* do nothing */
+            break;
+
         default:
             return ERROR(GENERIC);   /* impossible */
         }
@@ -291,19 +294,30 @@ size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr)
     BYTE* const ostart = (BYTE*)dst;
     BYTE* const oend = ostart + *dstCapacityPtr;
     BYTE* op = ostart;
-    size_t outSize = *dstCapacityPtr;
+
-    size_t epilogueSize, remaining;
+    if (zbc->stage != ZBUFFcs_final) {
-    ZBUFF_compressFlush(zbc, dst, &outSize);     /* flush any remaining inBuff */
+        /* flush whatever remains */
-    op += outSize;
+        size_t outSize = *dstCapacityPtr;
-    epilogueSize = ZSTD_compressEnd(zbc->zc, zbc->outBuff + zbc->outBuffContentSize, zbc->outBuffSize - zbc->outBuffContentSize);   /* epilogue into outBuff */
+        size_t const remainingToFlush = ZBUFF_compressFlush(zbc, dst, &outSize);
-    zbc->outBuffContentSize += epilogueSize;
+        op += outSize;
-    outSize = oend-op;
+        if (remainingToFlush) {
-    zbc->stage = ZBUFFcs_flush;
+            *dstCapacityPtr = op-ostart;
-    remaining = ZBUFF_compressFlush(zbc, op, &outSize);   /* attempt to flush epilogue into dst */
+            return remainingToFlush + ZBUFF_endFrameSize;
-    op += outSize;
+        }
-    if (!remaining) zbc->stage = ZBUFFcs_init;   /* close only if nothing left to flush */
+        /* create epilogue */
-    *dstCapacityPtr = op-ostart;                 /* tells how many bytes were written */
+        zbc->stage = ZBUFFcs_final;
-    return remaining;
+        zbc->outBuffContentSize = ZSTD_compressEnd(zbc->zc, zbc->outBuff, zbc->outBuffSize); /* epilogue into outBuff */
+    }
+
+    /* flush epilogue */
+    {   size_t const toFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize;
+        size_t const flushed = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outBuffFlushedSize, toFlush);
+        op += flushed;
+        zbc->outBuffFlushedSize += flushed;
+        *dstCapacityPtr = op-ostart;
+        if (toFlush==flushed) zbc->stage = ZBUFFcs_init;  /* end reached */
+        return toFlush - flushed;
+    }
 }
 
 

lib/compress/zstd_compress.c

@@ -246,7 +246,7 @@ ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, U
 
 size_t ZSTD_sizeofCCtx(ZSTD_compressionParameters cParams)   /* hidden interface, for paramagrill */
 {
-    ZSTD_CCtx* zc = ZSTD_createCCtx();
+    ZSTD_CCtx* const zc = ZSTD_createCCtx();
     ZSTD_parameters params;
     memset(&params, 0, sizeof(params));
     params.cParams = cParams;
@@ -295,7 +295,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
     zc->seqStore.buffer = zc->hashTable3 + h3Size;
     zc->hufTable = (HUF_CElt*)zc->seqStore.buffer;
     zc->flagStaticTables = 0;
-    zc->seqStore.buffer = ((U32*)(zc->seqStore.buffer)) + 256;
+    zc->seqStore.buffer = ((U32*)(zc->seqStore.buffer)) + 256;  /* note : HUF_CElt* is incomplete type, size is simulated using U32 */
 
     zc->nextToUpdate = 1;
     zc->nextSrc = NULL;
@@ -313,14 +313,17 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
         zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1<<Litbits);
         zc->seqStore.matchLengthFreq = zc->seqStore.litLengthFreq + (MaxLL+1);
         zc->seqStore.offCodeFreq = zc->seqStore.matchLengthFreq + (MaxML+1);
-        zc->seqStore.matchTable = (ZSTD_match_t*)((void*)(zc->seqStore.offCodeFreq + (MaxOff+1)));
+        zc->seqStore.buffer = zc->seqStore.offCodeFreq + (MaxOff+1);
-        zc->seqStore.priceTable = (ZSTD_optimal_t*)((void*)(zc->seqStore.matchTable + ZSTD_OPT_NUM+1));
+        zc->seqStore.matchTable = (ZSTD_match_t*)zc->seqStore.buffer;
+        zc->seqStore.buffer = zc->seqStore.matchTable + ZSTD_OPT_NUM+1;
+        zc->seqStore.priceTable = (ZSTD_optimal_t*)zc->seqStore.buffer;
         zc->seqStore.buffer = zc->seqStore.priceTable + ZSTD_OPT_NUM+1;
         zc->seqStore.litLengthSum = 0;
     }
-    zc->seqStore.offsetStart = (U32*) (zc->seqStore.buffer);
+    zc->seqStore.offsetStart = (U32*)(zc->seqStore.buffer);
-    zc->seqStore.litLengthStart = (U16*) (void*)(zc->seqStore.offsetStart + maxNbSeq);
+    zc->seqStore.buffer = zc->seqStore.offsetStart + maxNbSeq;
-    zc->seqStore.matchLengthStart = (U16*) (void*)(zc->seqStore.litLengthStart + maxNbSeq);
+    zc->seqStore.litLengthStart = (U16*)zc->seqStore.buffer;
+    zc->seqStore.matchLengthStart = zc->seqStore.litLengthStart + maxNbSeq;
     zc->seqStore.llCodeStart = (BYTE*) (zc->seqStore.matchLengthStart + maxNbSeq);
     zc->seqStore.mlCodeStart = zc->seqStore.llCodeStart + maxNbSeq;
     zc->seqStore.offCodeStart = zc->seqStore.mlCodeStart + maxNbSeq;
@@ -1024,11 +1027,12 @@ static unsigned ZSTD_NbCommonBytes (register size_t val)
 }
 
 
-static size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit)
+static size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* const pInLimit)
 {
     const BYTE* const pStart = pIn;
+    const BYTE* const pInLoopLimit = pInLimit - (sizeof(size_t)-1);
 
-    while ((pIn<pInLimit-(sizeof(size_t)-1))) {
+    while (pIn < pInLoopLimit) {
         size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
         if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; }
         pIn += ZSTD_NbCommonBytes(diff);
@@ -1128,7 +1132,6 @@ void ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx,
     size_t offset_1=cctx->rep[0], offset_2=cctx->rep[1];
 
     /* init */
-    ZSTD_resetSeqStore(seqStorePtr);
     ip += (ip==lowest);
     {   U32 const maxRep = (U32)(ip-lowest);
         if (offset_1 > maxRep) offset_1 = 0;
@@ -1137,35 +1140,34 @@ void ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx,
 
     /* Main Search Loop */
     while (ip < ilimit) {   /* < instead of <=, because repcode check at (ip+1) */
-        size_t mlCode;
+        size_t mLength;
-        size_t offset;
         size_t const h = ZSTD_hashPtr(ip, hBits, mls);
         U32 const current = (U32)(ip-base);
         U32 const matchIndex = hashTable[h];
         const BYTE* match = base + matchIndex;
         hashTable[h] = current;   /* update hash table */
 
-        if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) {   /* note : by construction, offset_1 <= current */
+        if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) { /* note : by construction, offset_1 <= current */
-            mlCode = ZSTD_count(ip+1+EQUAL_READ32, ip+1+EQUAL_READ32-offset_1, iend) + EQUAL_READ32;
+            mLength = ZSTD_count(ip+1+EQUAL_READ32, ip+1+EQUAL_READ32-offset_1, iend) + EQUAL_READ32;
             ip++;
-            ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mlCode-MINMATCH);
+            ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mLength-MINMATCH);
-       } else {
+        } else {
-            if ( (matchIndex <= lowestIndex) ||
+            size_t offset;
-                 (MEM_read32(match) != MEM_read32(ip)) ) {
+            if ( (matchIndex <= lowestIndex) || (MEM_read32(match) != MEM_read32(ip)) ) {
                 ip += ((ip-anchor) >> g_searchStrength) + 1;
                 continue;
             }
-            mlCode = ZSTD_count(ip+EQUAL_READ32, match+EQUAL_READ32, iend) + EQUAL_READ32;
+            mLength = ZSTD_count(ip+EQUAL_READ32, match+EQUAL_READ32, iend) + EQUAL_READ32;
             offset = ip-match;
-            while ((ip>anchor) && (match>lowest) && (ip[-1] == match[-1])) { ip--; match--; mlCode++; }  /* catch up */
+            while (((ip>anchor) & (match>lowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */
             offset_2 = offset_1;
             offset_1 = offset;
 
-            ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mlCode-MINMATCH);
+            ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
         }
 
         /* match found */
-        ip += mlCode;
+        ip += mLength;
         anchor = ip;
 
         if (ip <= ilimit) {
@@ -1177,18 +1179,18 @@ void ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx,
                  && ( (offset_2>0)
                  & (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) {
                 /* store sequence */
-                size_t const rlCode = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-offset_2, iend) + EQUAL_READ32;
+                size_t const rLength = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-offset_2, iend) + EQUAL_READ32;
                 { size_t const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; } /* swap offset_2 <=> offset_1 */
                 hashTable[ZSTD_hashPtr(ip, hBits, mls)] = (U32)(ip-base);
-                ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rlCode-MINMATCH);
+                ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rLength-MINMATCH);
-                ip += rlCode;
+                ip += rLength;
                 anchor = ip;
                 continue;   /* faster when present ... (?) */
     }   }   }
 
     /* save reps for next block */
-    cctx->savedRep[0] = offset_1 ? (U32)offset_1 : (U32)(iend-base);
+    cctx->savedRep[0] = offset_1 ? (U32)offset_1 : (U32)(iend - base) + 1;
-    cctx->savedRep[1] = offset_2 ? (U32)offset_2 : (U32)(iend-base);
+    cctx->savedRep[1] = offset_2 ? (U32)offset_2 : (U32)(iend - base) + 1;
 
     /* Last Literals */
     {   size_t const lastLLSize = iend - anchor;
@@ -1238,32 +1240,25 @@ static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx,
     const BYTE* const ilimit = iend - 8;
     U32 offset_1=ctx->rep[0], offset_2=ctx->rep[1];
 
-    /* init */
+    /* Search Loop */
-    ZSTD_resetSeqStore(seqStorePtr);
-    /* skip first position to avoid read overflow during repcode match check */
-    hashTable[ZSTD_hashPtr(ip, hBits, mls)] = (U32)(ip-base);
-    ip++;
-
-    /* Main Search Loop */
     while (ip < ilimit) {  /* < instead of <=, because (ip+1) */
         const size_t h = ZSTD_hashPtr(ip, hBits, mls);
         const U32 matchIndex = hashTable[h];
         const BYTE* matchBase = matchIndex < dictLimit ? dictBase : base;
         const BYTE* match = matchBase + matchIndex;
         const U32 current = (U32)(ip-base);
-        const U32 repIndex = current + 1 - offset_1;
+        const U32 repIndex = current + 1 - offset_1;   /* offset_1 expected <= current +1 */
         const BYTE* repBase = repIndex < dictLimit ? dictBase : base;
         const BYTE* repMatch = repBase + repIndex;
-        size_t mlCode;
+        size_t mLength;
-        U32 offset;
         hashTable[h] = current;   /* update hash table */
 
-        if ( (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > lowestIndex))   /* intentional overflow */
+        if ( (((U32)((dictLimit-1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > lowestIndex))
            && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
             const BYTE* repMatchEnd = repIndex < dictLimit ? dictEnd : iend;
-            mlCode = ZSTD_count_2segments(ip+1+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repMatchEnd, lowPrefixPtr) + EQUAL_READ32;
+            mLength = ZSTD_count_2segments(ip+1+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repMatchEnd, lowPrefixPtr) + EQUAL_READ32;
             ip++;
-            ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mlCode-MINMATCH);
+            ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mLength-MINMATCH);
         } else {
             if ( (matchIndex < lowestIndex) ||
                  (MEM_read32(match) != MEM_read32(ip)) ) {
@@ -1272,16 +1267,17 @@ static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx,
             }
             {   const BYTE* matchEnd = matchIndex < dictLimit ? dictEnd : iend;
                 const BYTE* lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr;
-                mlCode = ZSTD_count_2segments(ip+EQUAL_READ32, match+EQUAL_READ32, iend, matchEnd, lowPrefixPtr) + EQUAL_READ32;
+                U32 offset;
-                while ((ip>anchor) && (match>lowMatchPtr) && (ip[-1] == match[-1])) { ip--; match--; mlCode++; }   /* catch up */
+                mLength = ZSTD_count_2segments(ip+EQUAL_READ32, match+EQUAL_READ32, iend, matchEnd, lowPrefixPtr) + EQUAL_READ32;
+                while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; }   /* catch up */
                 offset = current - matchIndex;
                 offset_2 = offset_1;
                 offset_1 = offset;
-                ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mlCode-MINMATCH);
+                ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
         }   }
 
         /* found a match : store it */
-        ip += mlCode;
+        ip += mLength;
         anchor = ip;
 
         if (ip <= ilimit) {
@@ -1337,8 +1333,6 @@ static void ZSTD_compressBlock_fast_extDict(ZSTD_CCtx* ctx,
 }
 
 
-
-
 /*-*************************************
 *  Binary Tree search
 ***************************************/
@@ -1437,7 +1431,7 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co
     }   }
 
     *smallerPtr = *largerPtr = 0;
-    if (bestLength > 384) return MIN(192, (U32)(bestLength - 384));
+    if (bestLength > 384) return MIN(192, (U32)(bestLength - 384));   /* speed optimization */
     if (matchEndIdx > current + 8) return matchEndIdx - current - 8;
     return 1;
 }
@@ -1571,7 +1565,6 @@ static void ZSTD_updateTree_extDict(ZSTD_CCtx* zc, const BYTE* const ip, const B
 }
 
 
-
 /** Tree updater, providing best match */
 static size_t ZSTD_BtFindBestMatch_extDict (
                         ZSTD_CCtx* zc,
@@ -1622,7 +1615,7 @@ U32 ZSTD_insertAndFindFirstIndex (ZSTD_CCtx* zc, const BYTE* ip, U32 mls)
     const U32 target = (U32)(ip - base);
     U32 idx = zc->nextToUpdate;
 
-    while(idx < target) {
+    while(idx < target) { /* catch up */
         size_t const h = ZSTD_hashPtr(base+idx, hashLog, mls);
         NEXT_IN_CHAIN(idx, chainMask) = hashTable[h];
         hashTable[h] = idx;
@@ -1659,7 +1652,7 @@ size_t ZSTD_HcFindBestMatch_generic (
     /* HC4 match finder */
     U32 matchIndex = ZSTD_insertAndFindFirstIndex (zc, ip, mls);
 
-    for ( ; (matchIndex>lowLimit) && (nbAttempts) ; nbAttempts--) {
+    for ( ; (matchIndex>lowLimit) & (nbAttempts>0) ; nbAttempts--) {
         const BYTE* match;
         size_t currentMl=0;
         if ((!extDict) || matchIndex >= dictLimit) {
@@ -1673,7 +1666,7 @@ size_t ZSTD_HcFindBestMatch_generic (
         }
 
         /* save best solution */
-        if (currentMl > ml) { ml = currentMl; *offsetPtr = ZSTD_REP_MOVE + current - matchIndex; if (ip+currentMl == iLimit) break; /* best possible, and avoid read overflow*/ }
+        if (currentMl > ml) { ml = currentMl; *offsetPtr = current - matchIndex + ZSTD_REP_MOVE; if (ip+currentMl == iLimit) break; /* best possible, and avoid read overflow*/ }
 
         if (matchIndex <= minChain) break;
         matchIndex = NEXT_IN_CHAIN(matchIndex, chainMask);
@@ -1743,7 +1736,6 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
     /* init */
     ip += (ip==base);
     ctx->nextToUpdate3 = ctx->nextToUpdate;
-    ZSTD_resetSeqStore(seqStorePtr);
     {   U32 i;
         U32 const maxRep = (U32)(ip-base);
         for (i=0; i<ZSTD_REP_INIT; i++) {
@@ -1847,7 +1839,7 @@ _storeSequence:
     /* Save reps for next block */
     {   int i;
         for (i=0; i<ZSTD_REP_NUM; i++) {
-            if (!rep[i]) rep[i] = (U32)(iend - ctx->base);   /* in case some zero are left */
+            if (!rep[i]) rep[i] = (U32)(iend - ctx->base) + 1;   /* in case some zero are left */
             ctx->savedRep[i] = rep[i];
     }   }
 
@@ -1913,7 +1905,6 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
     { U32 i; for (i=0; i<ZSTD_REP_INIT; i++) rep[i]=ctx->rep[i]; }
 
     ctx->nextToUpdate3 = ctx->nextToUpdate;
-    ZSTD_resetSeqStore(seqStorePtr);
     ip += (ip == prefixStart);
 
     /* Match Loop */
@@ -2097,11 +2088,7 @@ typedef void (*ZSTD_blockCompressor) (ZSTD_CCtx* ctx, const void* src, size_t sr
 static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict)
 {
     static const ZSTD_blockCompressor blockCompressor[2][6] = {
-#if 1
         { ZSTD_compressBlock_fast, ZSTD_compressBlock_greedy, ZSTD_compressBlock_lazy, ZSTD_compressBlock_lazy2, ZSTD_compressBlock_btlazy2, ZSTD_compressBlock_btopt },
-#else
-        { ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_greedy_extDict, ZSTD_compressBlock_lazy_extDict,ZSTD_compressBlock_lazy2_extDict, ZSTD_compressBlock_btlazy2_extDict, ZSTD_compressBlock_btopt_extDict },
-#endif
         { ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_greedy_extDict, ZSTD_compressBlock_lazy_extDict,ZSTD_compressBlock_lazy2_extDict, ZSTD_compressBlock_btlazy2_extDict, ZSTD_compressBlock_btopt_extDict }
     };
 
@@ -2111,8 +2098,9 @@ static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int
 
 static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
 {
-    ZSTD_blockCompressor blockCompressor = ZSTD_selectBlockCompressor(zc->params.cParams.strategy, zc->lowLimit < zc->dictLimit);
+    ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->params.cParams.strategy, zc->lowLimit < zc->dictLimit);
     if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) return 0;   /* don't even attempt compression below a certain srcSize */
+    ZSTD_resetSeqStore(&(zc->seqStore));
     blockCompressor(zc, src, srcSize);
     return ZSTD_compressSequences(zc, dst, dstCapacity, srcSize);
 }
@@ -2245,7 +2233,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* zc,
 
     /* preemptive overflow correction */
     if (zc->lowLimit > (1<<30)) {
-        U32 const btplus = (zc->params.cParams.strategy == ZSTD_btlazy2) || (zc->params.cParams.strategy == ZSTD_btopt);
+        U32 const btplus = (zc->params.cParams.strategy == ZSTD_btlazy2) | (zc->params.cParams.strategy == ZSTD_btopt);
         U32 const chainMask = (1 << (zc->params.cParams.chainLog - btplus)) - 1;
         U32 const newLowLimit = zc->lowLimit & chainMask;   /* preserve position % chainSize */
         U32 const correction = zc->lowLimit - newLowLimit;

lib/compress/zstd_opt.h

@@ -465,7 +465,6 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
 
     /* init */
     ctx->nextToUpdate3 = ctx->nextToUpdate;
-    ZSTD_resetSeqStore(seqStorePtr);
     ZSTD_rescaleFreqs(seqStorePtr);
     ip += (ip==prefixStart);
     { U32 i; for (i=0; i<ZSTD_REP_INIT; i++) rep[i]=ctx->rep[i]; }
@@ -757,7 +756,6 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
     { U32 i; for (i=0; i<ZSTD_REP_INIT; i++) rep[i]=ctx->rep[i]; }
 
     ctx->nextToUpdate3 = ctx->nextToUpdate;
-    ZSTD_resetSeqStore(seqStorePtr);
     ZSTD_rescaleFreqs(seqStorePtr);
     ip += (ip==prefixStart);
 

lib/decompress/zstd_decompress.c

@@ -749,7 +749,7 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState)
     if (MEM_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&(seqState->DStream));
 
     seq.litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBits(&(seqState->DStream), llBits) : 0);   /* <=  16 bits */
-    if (MEM_32bits() ||
+    if (MEM_32bits() |
        (totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) ) BIT_reloadDStream(&(seqState->DStream));
 
     /* ANS state update */
@@ -765,23 +765,22 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState)
 FORCE_INLINE
 size_t ZSTD_execSequence(BYTE* op,
                                 BYTE* const oend, seq_t sequence,
-                                const BYTE** litPtr, const BYTE* const litLimit_8,
+                                const BYTE** litPtr, const BYTE* const litLimit_w,
                                 const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
 {
     BYTE* const oLitEnd = op + sequence.litLength;
     size_t const sequenceLength = sequence.litLength + sequence.matchLength;
     BYTE* const oMatchEnd = op + sequenceLength;   /* risk : address space overflow (32-bits) */
-    BYTE* const oend_8 = oend-8;
+    BYTE* const oend_w = oend-WILDCOPY_OVERLENGTH;
     const BYTE* const iLitEnd = *litPtr + sequence.litLength;
     const BYTE* match = oLitEnd - sequence.offset;
 
     /* check */
-    if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall);   /* last match must start at a minimum distance of 8 from oend */
+    if ((oLitEnd>oend_w) | (oMatchEnd>oend)) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
-    if (oMatchEnd > oend) return ERROR(dstSize_tooSmall);   /* overwrite beyond dst buffer */
+    if (iLitEnd > litLimit_w) return ERROR(corruption_detected);   /* over-read beyond lit buffer */
-    if (iLitEnd > litLimit_8) return ERROR(corruption_detected);   /* over-read beyond lit buffer */
 
     /* copy Literals */
-    ZSTD_wildcopy(op, *litPtr, sequence.litLength);   /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
+    ZSTD_wildcopy(op, *litPtr, sequence.litLength);   /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */
     op = oLitEnd;
     *litPtr = iLitEnd;   /* update for next sequence */
 
@@ -821,10 +820,10 @@ size_t ZSTD_execSequence(BYTE* op,
     op += 8; match += 8;
 
     if (oMatchEnd > oend-(16-MINMATCH)) {
-        if (op < oend_8) {
+        if (op < oend_w) {
-            ZSTD_wildcopy(op, match, oend_8 - op);
+            ZSTD_wildcopy(op, match, oend_w - op);
-            match += oend_8 - op;
+            match += oend_w - op;
-            op = oend_8;
+            op = oend_w;
         }
         while (op < oMatchEnd) *op++ = *match++;
     } else {
@@ -845,7 +844,7 @@ static size_t ZSTD_decompressSequences(
     BYTE* const oend = ostart + maxDstSize;
     BYTE* op = ostart;
     const BYTE* litPtr = dctx->litPtr;
-    const BYTE* const litLimit_8 = litPtr + dctx->litBufSize - 8;
+    const BYTE* const litLimit_w = litPtr + dctx->litBufSize - WILDCOPY_OVERLENGTH;
     const BYTE* const litEnd = litPtr + dctx->litSize;
     FSE_DTable* DTableLL = dctx->LLTable;
     FSE_DTable* DTableML = dctx->MLTable;
@@ -875,7 +874,7 @@ static size_t ZSTD_decompressSequences(
         for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq ; ) {
             nbSeq--;
             {   seq_t const sequence = ZSTD_decodeSequence(&seqState);
-                size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litLimit_8, base, vBase, dictEnd);
+                size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litLimit_w, base, vBase, dictEnd);
                 if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
                 op += oneSeqSize;
         }   }
@@ -888,8 +887,8 @@ static size_t ZSTD_decompressSequences(
 
     /* last literal segment */
     {   size_t const lastLLSize = litEnd - litPtr;
-        if (litPtr > litEnd) return ERROR(corruption_detected);   /* too many literals already used */
+        //if (litPtr > litEnd) return ERROR(corruption_detected);   /* too many literals already used */
-        if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
+        if (lastLLSize > (size_t)(oend-op)) return ERROR(dstSize_tooSmall);
         memcpy(op, litPtr, lastLLSize);
         op += lastLLSize;
     }
@@ -1180,12 +1179,13 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
 }
 
 
-static void ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
+static size_t ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
 {
     dctx->dictEnd = dctx->previousDstEnd;
     dctx->vBase = (const char*)dict - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->base));
     dctx->base = dict;
     dctx->previousDstEnd = (const char*)dict + dictSize;
+    return 0;
 }
 
 static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t const dictSize)
@@ -1237,29 +1237,24 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t c
 
 static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
 {
-    if (dictSize < 8) return ERROR(dictionary_corrupted);
+    if (dictSize < 8) return ZSTD_refDictContent(dctx, dict, dictSize);
     {   U32 const magic = MEM_readLE32(dict);
         if (magic != ZSTD_DICT_MAGIC) {
-            /* pure content mode */
+            return ZSTD_refDictContent(dctx, dict, dictSize);   /* pure content mode */
-            ZSTD_refDictContent(dctx, dict, dictSize);
+    }   }
-            return 0;
+    dctx->dictID = MEM_readLE32((const char*)dict + 4);
-        }
-        dctx->dictID = MEM_readLE32((const char*)dict + 4);
 
-        /* load entropy tables */
+    /* load entropy tables */
-        dict = (const char*)dict + 8;
+    dict = (const char*)dict + 8;
-        dictSize -= 8;
+    dictSize -= 8;
-        {   size_t const eSize = ZSTD_loadEntropy(dctx, dict, dictSize);
+    {   size_t const eSize = ZSTD_loadEntropy(dctx, dict, dictSize);
-            if (ZSTD_isError(eSize)) return ERROR(dictionary_corrupted);
+        if (ZSTD_isError(eSize)) return ERROR(dictionary_corrupted);
-            dict = (const char*)dict + eSize;
+        dict = (const char*)dict + eSize;
-            dictSize -= eSize;
+        dictSize -= eSize;
-        }
-
-        /* reference dictionary content */
-        ZSTD_refDictContent(dctx, dict, dictSize);
-
-        return 0;
     }
+
+    /* reference dictionary content */
+    return ZSTD_refDictContent(dctx, dict, dictSize);
 }
 
 
@@ -1319,7 +1314,7 @@ ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_cu
 }
 
 /*! ZSTD_createDDict() :
-*   Create a digested dictionary, ready to start decompression operation without startup delay.
+*   Create a digested dictionary, ready to start decompression without startup delay.
 *   `dict` can be released after `ZSTD_DDict` creation */
 ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize)
 {

lib/dictBuilder/zdict.c

@@ -584,48 +584,51 @@ static void ZDICT_countEStats(EStats_ress_t esr,
                             U32* countLit, U32* offsetcodeCount, U32* matchlengthCount, U32* litlengthCount, U32* repOffsets,
                             const void* src, size_t srcSize)
 {
-    const seqStore_t* seqStorePtr;
+    size_t cSize;
 
     if (srcSize > ZSTD_BLOCKSIZE_MAX) srcSize = ZSTD_BLOCKSIZE_MAX;   /* protection vs large samples */
     ZSTD_copyCCtx(esr.zc, esr.ref);
-    ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_MAX, src, srcSize);
+    cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_MAX, src, srcSize);
-    seqStorePtr = ZSTD_getSeqStore(esr.zc);
+    if (ZSTD_isError(cSize)) { DISPLAYLEVEL(1, "warning : could not compress sample size %u \n", (U32)srcSize); return; }
 
-    /* literals stats */
+    if (cSize) {  /* if == 0; block is not compressible */
-    {   const BYTE* bytePtr;
+        const seqStore_t* seqStorePtr = ZSTD_getSeqStore(esr.zc);
-        for(bytePtr = seqStorePtr->litStart; bytePtr < seqStorePtr->lit; bytePtr++)
-            countLit[*bytePtr]++;
-    }
 
-    /* seqStats */
+        /* literals stats */
-    {   size_t const nbSeq = (size_t)(seqStorePtr->offset - seqStorePtr->offsetStart);
+        {   const BYTE* bytePtr;
-        ZSTD_seqToCodes(seqStorePtr, nbSeq);
+            for(bytePtr = seqStorePtr->litStart; bytePtr < seqStorePtr->lit; bytePtr++)
-
+                countLit[*bytePtr]++;
-        {   const BYTE* codePtr = seqStorePtr->offCodeStart;
-            size_t u;
-            for (u=0; u<nbSeq; u++) offsetcodeCount[codePtr[u]]++;
         }
 
-        {   const BYTE* codePtr = seqStorePtr->mlCodeStart;
+        /* seqStats */
-            size_t u;
+        {   size_t const nbSeq = (size_t)(seqStorePtr->offset - seqStorePtr->offsetStart);
-            for (u=0; u<nbSeq; u++) matchlengthCount[codePtr[u]]++;
+            ZSTD_seqToCodes(seqStorePtr, nbSeq);
+
+            {   const BYTE* codePtr = seqStorePtr->offCodeStart;
+                size_t u;
+                for (u=0; u<nbSeq; u++) offsetcodeCount[codePtr[u]]++;
+            }
+
+            {   const BYTE* codePtr = seqStorePtr->mlCodeStart;
+                size_t u;
+                for (u=0; u<nbSeq; u++) matchlengthCount[codePtr[u]]++;
+            }
+
+            {   const BYTE* codePtr = seqStorePtr->llCodeStart;
+                size_t u;
+                for (u=0; u<nbSeq; u++) litlengthCount[codePtr[u]]++;
+        }   }
+
+        /* rep offsets */
+        {   const U32* const offsetPtr = seqStorePtr->offsetStart;
+            U32 offset1 = offsetPtr[0] - 3;
+            U32 offset2 = offsetPtr[1] - 3;
+            if (offset1 >= MAXREPOFFSET) offset1 = 0;
+            if (offset2 >= MAXREPOFFSET) offset2 = 0;
+            repOffsets[offset1] += 3;
+            repOffsets[offset2] += 1;
         }
-
-        {   const BYTE* codePtr = seqStorePtr->llCodeStart;
-            size_t u;
-            for (u=0; u<nbSeq; u++) litlengthCount[codePtr[u]]++;
-    }   }
-
-    /* rep offsets */
-    {   const U32* const offsetPtr = seqStorePtr->offsetStart;
-        U32 offset1 = offsetPtr[0] - 3;
-        U32 offset2 = offsetPtr[1] - 3;
-        if (offset1 >= MAXREPOFFSET) offset1 = 0;
-        if (offset2 >= MAXREPOFFSET) offset2 = 0;
-        repOffsets[offset1] += 3;
-        repOffsets[offset2] += 1;
     }
-
 }
 
 /*

programs/.gitignore

@@ -50,3 +50,4 @@ afl
 # Misc files
 *.bat
 fileTests.sh
+dirTest*

programs/bench.c

@@ -42,8 +42,10 @@
 /* *************************************
 *  Constants
 ***************************************/
-#ifndef ZSTD_VERSION
+#ifndef ZSTD_GIT_COMMIT
-#  define ZSTD_VERSION ""
+#  define ZSTD_GIT_COMMIT_STRING ""
+#else
+#  define ZSTD_GIT_COMMIT_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_GIT_COMMIT)
 #endif
 
 #define NBLOOPS               3
@@ -363,7 +365,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
     memset(&total, 0, sizeof(total));
 
     if (g_displayLevel == 1 && !g_additionalParam)
-        DISPLAY("bench %s: input %u bytes, %i iterations, %u KB blocks\n", ZSTD_VERSION, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10));
+        DISPLAY("bench %s %s: input %u bytes, %i iterations, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10));
 
     if (cLevelLast < cLevel) cLevelLast = cLevel;
 

programs/fileio.c

@@ -137,7 +137,7 @@ static U32 g_sparseFileSupport = 1;   /* 0 : no sparse allowed; 1: auto (file ye
 void FIO_setSparseWrite(unsigned sparse) { g_sparseFileSupport=sparse; }
 static U32 g_dictIDFlag = 1;
 void FIO_setDictIDFlag(unsigned dictIDFlag) { g_dictIDFlag = dictIDFlag; }
-static U32 g_checksumFlag = 0;
+static U32 g_checksumFlag = 1;
 void FIO_setChecksumFlag(unsigned checksumFlag) { g_checksumFlag = checksumFlag; }
 static U32 g_removeSrcFile = 0;
 void FIO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); }
@@ -413,7 +413,7 @@ static int FIO_compressFilename_dstFile(cRess_t ress,
     int result;
 
     ress.dstFile = FIO_openDstFile(dstFileName);
-    if (ress.dstFile==0) { fclose(ress.srcFile); return 1; }
+    if (ress.dstFile==0) return 1;
 
     result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName, cLevel);
 

programs/playTests.sh

@@ -22,10 +22,12 @@ roundTripTest() {
 
 isWindows=false
 ECHO="echo"
+INTOVOID="/dev/null"
 case "$OS" in
   Windows*)
     isWindows=true
     ECHO="echo -e"
+    INTOVOID="nul"
     ;;
 esac
 
@@ -53,19 +55,21 @@ $ECHO "test : null-length file roundtrip"
 $ECHO -n '' | $ZSTD - --stdout | $ZSTD -d --stdout
 $ECHO "test : decompress file with wrong suffix (must fail)"
 $ZSTD -d tmpCompressed && die "wrong suffix error not detected!"
+$ZSTD -df tmp && die "should have refused : wrong extension"
+$ECHO "test : decompress into stdout"
 $ZSTD -d tmpCompressed -c > tmpResult    # decompression using stdout
 $ZSTD --decompress tmpCompressed -c > tmpResult
 $ZSTD --decompress tmpCompressed --stdout > tmpResult
-if [ "$isWindows" = false ] ; then
+$ECHO "test : decompress from stdin into stdout"
-    $ZSTD -d    < tmp.zst > /dev/null    # combine decompression, stdin & stdout
+$ZSTD -dc   < tmp.zst > $INTOVOID   # combine decompression, stdin & stdout
-    $ZSTD -d  - < tmp.zst > /dev/null
+$ZSTD -dc - < tmp.zst > $INTOVOID
-fi
+$ZSTD -d    < tmp.zst > $INTOVOID   # implicit stdout when stdin is used
-$ZSTD -dc   < tmp.zst > /dev/null
+$ZSTD -d  - < tmp.zst > $INTOVOID
-$ZSTD -dc - < tmp.zst > /dev/null
+$ECHO "test : overwrite protection"
 $ZSTD -q tmp && die "overwrite check failed!"
+$ECHO "test : force overwrite"
 $ZSTD -q -f tmp
 $ZSTD -q --force tmp
-$ZSTD -df tmp && die "should have refused : wrong extension"
 $ECHO "test : file removal"
 $ZSTD -f --rm tmp
 ls tmp && die "tmp should no longer be present"
@@ -135,9 +139,9 @@ rm tmpSparse*
 
 $ECHO "\n**** multiple files tests **** "
 
-./datagen -s1        > tmp1 2> /dev/null
+./datagen -s1        > tmp1 2> $INTOVOID
-./datagen -s2 -g100K > tmp2 2> /dev/null
+./datagen -s2 -g100K > tmp2 2> $INTOVOID
-./datagen -s3 -g1M   > tmp3 2> /dev/null
+./datagen -s3 -g1M   > tmp3 2> $INTOVOID
 $ZSTD -f tmp*
 $ECHO "compress tmp* : "
 ls -ls tmp*

programs/roundTripCrash.c

@@ -36,8 +36,14 @@
 #include <stdio.h>      /* fprintf */
 #include <sys/types.h>  /* stat */
 #include <sys/stat.h>   /* stat */
+#include "xxhash.h"
 #include "zstd.h"
 
+/*===========================================
+*   Macros
+*==========================================*/
+#define MIN(a,b)  ( (a) < (b) ? (a) : (b) )
+
 /** roundTripTest() :
 *   Compresses `srcBuff` into `compressedBuff`,
 *   then decompresses `compressedBuff` into `resultBuff`.
@@ -51,7 +57,9 @@ static size_t roundTripTest(void* resultBuff, size_t resultBuffCapacity,
                       const void* srcBuff, size_t srcBuffSize)
 {
     static const int maxClevel = 19;
-    int const cLevel = (!srcBuffSize) ? 1 : (*(const unsigned char*)srcBuff) % maxClevel;
+    size_t const hashLength = MIN(128, srcBuffSize);
+    unsigned const h32 = XXH32(srcBuff, hashLength, 0);
+    int const cLevel = h32 % maxClevel;
     size_t const cSize = ZSTD_compress(compressedBuff, compressedBuffCapacity, srcBuff, srcBuffSize, cLevel);
     if (ZSTD_isError(cSize)) {
         fprintf(stderr, "Compression error : %s \n", ZSTD_getErrorName(cSize));

programs/zbufftest.c

@@ -421,11 +421,17 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
         }   }
 
         /* final frame epilogue */
-        {   size_t dstBuffSize = cBufferSize - cSize;
+        {   size_t remainingToFlush = (size_t)(-1);
-            size_t const flushError = ZBUFF_compressEnd(zc, cBuffer+cSize, &dstBuffSize);
+            while (remainingToFlush) {
-            CHECK (ZBUFF_isError(flushError), "flush error : %s", ZBUFF_getErrorName(flushError));
+                size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
-            cSize += dstBuffSize;
+                size_t dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
-        }
+                U32 const enoughDstSize = dstBuffSize >= remainingToFlush;
+                remainingToFlush = ZBUFF_compressEnd(zc, cBuffer+cSize, &dstBuffSize);
+                CHECK (ZBUFF_isError(remainingToFlush), "flush error : %s", ZBUFF_getErrorName(remainingToFlush));
+                //DISPLAY("flush %u bytes : still within context : %i \n", (U32)dstBuffSize, (int)remainingToFlush);
+                CHECK (enoughDstSize && remainingToFlush, "ZBUFF_compressEnd() not fully flushed, but enough space available");
+                cSize += dstBuffSize;
+        }   }
         crcOrig = XXH64_digest(&xxhState);
 
         /* multi - fragments decompression test */

programs/zstdcli.c

@@ -136,7 +136,7 @@ static int usage_advanced(const char* programName)
 #ifndef ZSTD_NOCOMPRESS
     DISPLAY( "--ultra : enable ultra modes (requires more memory to decompress)\n");
     DISPLAY( "--no-dictID : don't write dictID into header (dictionary compression)\n");
-    DISPLAY( "--check : enable integrity check\n");
+    DISPLAY( "--[no-]check : integrity check (default:enabled)\n");
 #endif
 #ifndef ZSTD_NODECOMPRESS
     DISPLAY( "--test  : test compressed file integrity \n");
@@ -257,6 +257,7 @@ int main(int argCount, const char** argv)
         if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; displayLevel=1; continue; }
         if (!strcmp(argument, "--ultra")) { FIO_setMaxWLog(0); continue; }
         if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(2); continue; }
+        if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(0); continue; }
         if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; }
         if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; }
         if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; }

tests/test-zstd-speed.py

@@ -1,37 +1,38 @@
 #! /usr/bin/env python
-# execute(), fetch(), notify() are based on https://github.com/getlantern/build-automation/blob/master/build.py
 
 import argparse
 import os
 import string
 import time
 import traceback
-from subprocess import Popen, PIPE
+import subprocess
+import signal
  
-repo_url = 'https://github.com/Cyan4973/zstd.git'
+
-test_dir_name = 'speedTest'
+default_repo_url = 'https://github.com/Cyan4973/zstd.git'
+working_dir_name = 'speedTest'
+working_path = os.getcwd() + '/' + working_dir_name     # /path/to/zstd/tests/speedTest 
+clone_path = working_path + '/' + 'zstd'                # /path/to/zstd/tests/speedTest/zstd 
+email_header = '[ZSTD_speedTest]'
+pid = str(os.getpid())
 
 
 def log(text):
-    print time.strftime("%Y/%m/%d %H:%M:%S") + ' - ' + text
+    print(time.strftime("%Y/%m/%d %H:%M:%S") + ' - ' + text)
 
-def execute(command, print_output=False, print_error=True):
+
+def execute(command, print_output=False, print_error=True, param_shell=True):
     log("> " + command)
-    popen = Popen(command, stdout=PIPE, stderr=PIPE, shell=True, cwd=execute.cwd)
+    popen = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=param_shell, cwd=execute.cwd)
-    itout = iter(popen.stdout.readline, b"")
+    stdout = popen.communicate()[0]
-    iterr = iter(popen.stderr.readline, b"")
+    stdout_lines = stdout.splitlines()
-    stdout_lines = list(itout)
     if print_output:
-        print ''.join(stdout_lines)
+        print('\n'.join(stdout_lines))
-    stderr_lines = list(iterr)
-    if print_output:
-        print ''.join(stderr_lines)
-    popen.communicate()
     if popen.returncode is not None and popen.returncode != 0:
         if not print_output and print_error:
-            print ''.join(stderr_lines)    
+            print('\n'.join(stdout_lines))
-        raise RuntimeError(''.join(stderr_lines))
+        raise RuntimeError('\n'.join(stdout_lines))
-    return stdout_lines + stderr_lines
+    return stdout_lines
 execute.cwd = None
 
 
@@ -43,39 +44,51 @@ def does_command_exist(command):
     return True
 
 
-def fetch():
+def send_email(emails, topic, text, have_mutt, have_mail):
+    logFileName = working_path + '/' + 'tmpEmailContent'
+    with open(logFileName, "w") as myfile:
+        myfile.writelines(text)
+        myfile.close()
+        if have_mutt:
+            execute('mutt -s "' + topic + '" ' + emails + ' < ' + logFileName)
+        elif have_mail:
+            execute('mail -s "' + topic + '" ' + emails + ' < ' + logFileName)
+        else:
+            log("e-mail cannot be sent (mail or mutt not found)")
+
+
+def send_email_with_attachments(branch, commit, last_commit, emails, text, results_files, logFileName, lower_limit, have_mutt, have_mail):
+    with open(logFileName, "w") as myfile:
+        myfile.writelines(text)
+        myfile.close()
+        email_topic = '%s:%s Warning for %s:%s last_commit=%s speed<%s' % (email_header, pid, branch, commit, last_commit, lower_limit)
+        if have_mutt:
+            execute('mutt -s "' + email_topic + '" ' + emails + ' -a ' + results_files + ' < ' + logFileName)
+        elif have_mail:
+            execute('mail -s "' + email_topic + '" ' + emails + ' < ' + logFileName)
+        else:
+            log("e-mail cannot be sent (mail or mutt not found)")
+
+
+def git_get_branches():
     execute('git fetch -p')
     output = execute('git branch -rl')
     for line in output:
         if "HEAD" in line: 
-            output.remove(line) # remove "origin/HEAD -> origin/dev"
+            output.remove(line)  # remove "origin/HEAD -> origin/dev"
-    branches = map(lambda l: l.strip(), output)
+    return map(lambda l: l.strip(), output)
-    return map(lambda b: (b, execute('git show -s --format=%h ' + b)[0].strip()), branches)
 
 
-def notify(branch, commit, last_commit):
+def git_get_changes(branch, commit, last_commit):
-    text_tmpl = string.Template('Changes since $last_commit:\r\n$commits')
-    branch = branch.split('/')[1]
     fmt = '--format="%h: (%an) %s, %ar"'
     if last_commit is None:
         commits = execute('git log -n 10 %s %s' % (fmt, commit))
     else:
         commits = execute('git --no-pager log %s %s..%s' % (fmt, last_commit, commit))
-
+    return str('Changes in %s since %s:\n' % (branch, last_commit)) + '\n'.join(commits)
-    text = text_tmpl.substitute({'last_commit': last_commit, 'commits': ''.join(commits)})
-    print str("commits for %s: %s" % (commit, text))
 
 
-def compile(branch, commit, dry_run):
+def get_last_results(resultsFileName):
-    local_branch = string.split(branch, '/')[1]
-    version = local_branch.rpartition('-')[2]
-    version = version + '_' + commit
-    execute('git checkout -- . && git checkout ' + branch)
-    if not dry_run:
-        execute('VERSION=' + version + '; make clean zstdprogram')
-
-
-def get_last_commit(resultsFileName):
     if not os.path.isfile(resultsFileName):
         return None, None, None
     commit = None
@@ -84,15 +97,13 @@ def get_last_commit(resultsFileName):
     with open(resultsFileName,'r') as f:
         for line in f:
             words = line.split()
-            if len(words) == 2: # branch + commit
+            if len(words) == 2:   # branch + commit
                 commit = words[1];
                 cspeed = []
                 dspeed = []
-            if (len(words) == 8): 
+            if (len(words) == 8):  # results
                 cspeed.append(float(words[3]))
                 dspeed.append(float(words[5]))
-        #if commit != None:
-        #    print "commit=%s cspeed=%s dspeed=%s" % (commit, cspeed, dspeed)
     return commit, cspeed, dspeed
 
 
@@ -102,80 +113,62 @@ def benchmark_and_compare(branch, commit, resultsFileName, lastCLevel, testFileP
         log("WARNING: bench loadavg=%.2f is higher than %s, sleeping for %s seconds" % (os.getloadavg()[0], maxLoadAvg, sleepTime))
         time.sleep(sleepTime)
     start_load = str(os.getloadavg())
-    result = execute('programs/zstd -qi5b1e' + str(lastCLevel) + ' ' + testFilePath)
+    result = execute('programs/zstd -qi5b1e%s %s' % (lastCLevel, testFilePath), print_output=True)
     end_load = str(os.getloadavg())
     linesExpected = lastCLevel + 2;
     if len(result) != linesExpected:
-        log("ERROR: number of result lines=%d is different that expected %d" % (len(result), linesExpected))
+        raise RuntimeError("ERROR: number of result lines=%d is different that expected %d\n%s" % (len(result), linesExpected, '\n'.join(result)))
-        return ""
     with open(resultsFileName, "a") as myfile:
         myfile.write(branch + " " + commit + "\n")
-        myfile.writelines(result)
+        myfile.write('\n'.join(result) + '\n')
         myfile.close()
         if (last_cspeed == None):
+            log("WARNING: No data for comparison for branch=%s file=%s " % (branch, fileName))
             return ""
-        commit, cspeed, dspeed = get_last_commit(resultsFileName)
+        commit, cspeed, dspeed = get_last_results(resultsFileName)
         text = ""
         for i in range(0, min(len(cspeed), len(last_cspeed))):
+            print("%s:%s -%d cspeed=%6.2f clast=%6.2f cdiff=%1.4f dspeed=%6.2f dlast=%6.2f ddiff=%1.4f %s" % (branch, commit, i+1, cspeed[i], last_cspeed[i], cspeed[i]/last_cspeed[i], dspeed[i], last_dspeed[i], dspeed[i]/last_dspeed[i], fileName))
             if (cspeed[i]/last_cspeed[i] < lower_limit):
-                text += "WARNING: File=%s level=%d cspeed=%s last=%s diff=%s\n" % (fileName, i+1, cspeed[i], last_cspeed[i], cspeed[i]/last_cspeed[i])
+                text += "WARNING: -%d cspeed=%.2f clast=%.2f cdiff=%.4f %s\n" % (i+1, cspeed[i], last_cspeed[i], cspeed[i]/last_cspeed[i], fileName)
             if (dspeed[i]/last_dspeed[i] < lower_limit):
-                text += "WARNING: File=%s level=%d dspeed=%s last=%s diff=%s\n" % (fileName, i+1, dspeed[i], last_dspeed[i], dspeed[i]/last_dspeed[i])
+                text += "WARNING: -%d dspeed=%.2f dlast=%.2f ddiff=%.4f %s\n" % (i+1, dspeed[i], last_dspeed[i], dspeed[i]/last_dspeed[i], fileName)
         if text:
             text = message + ("\nmaxLoadAvg=%s  load average at start=%s end=%s\n" % (maxLoadAvg, start_load, end_load)) + text
         return text
 
 
-def send_email(branch, commit, last_commit, emails, text, results_files, logFileName, lower_limit, have_mutt, have_mail):
+def update_config_file(branch, commit):
-    with open(logFileName, "w") as myfile:
+    last_commit = None
-        myfile.writelines(text)
+    commitFileName = working_path + "/commit_" + branch.replace("/", "_") + ".txt"
-        myfile.close()
+    if os.path.isfile(commitFileName):
-        if have_mutt:
+        last_commit = file(commitFileName, 'r').read()
-            execute("mutt -s \"[ZSTD_speedTest] Warning for branch=" + branch + " commit=" + commit + " last_commit=" + last_commit + " speed<" + str(lower_limit) + "\" " + emails + " -a " + results_files + " < " + logFileName)
+    file(commitFileName, 'w').write(commit)
-        elif have_mail:
+    return last_commit
-            execute("mail -s \"[ZSTD_speedTest] Warning for branch=" + branch + " commit=" + commit + " last_commit=" + last_commit + " speed<" + str(lower_limit) + "\" " + emails + " < " + logFileName)
-        else:
-            log("e-mail cannot be sent (mail and mutt not found)")
 
 
-def check_branches(args, test_path, testFilePaths, have_mutt, have_mail):
+def test_commit(branch, commit, last_commit, args, testFilePaths, have_mutt, have_mail):
-    for branch, commit in fetch():
+    local_branch = string.split(branch, '/')[1]
-        try:
+    version = local_branch.rpartition('-')[2] + '_' + commit
-            commitFileName = test_path + "/commit_" + branch.replace("/", "_")
+    if not args.dry_run:
-            if os.path.isfile(commitFileName):
+        execute('make clean zstdprogram MOREFLAGS="-DZSTD_GIT_COMMIT=%s"' % version)
-                last_commit = file(commitFileName, 'r').read()
+    logFileName = working_path + "/log_" + branch.replace("/", "_") + ".txt"
-            else:
+    text_to_send = []
-                last_commit = None
+    results_files = ""
-            file(commitFileName, 'w').write(commit)
+    for filePath in testFilePaths:
-
+        fileName = filePath.rpartition('/')[2]
-            if commit == last_commit:
+        resultsFileName = working_path + "/results_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt"
-                log("skipping branch %s: head %s already processed" % (branch, commit))
+        last_commit, cspeed, dspeed = get_last_results(resultsFileName)
-            else:
+        if not args.dry_run:
-                log("build branch %s: head %s is different from prev %s" % (branch, commit, last_commit))
+            text = benchmark_and_compare(branch, commit, resultsFileName, args.lastCLevel, filePath, fileName, cspeed, dspeed, args.lowerLimit, args.maxLoadAvg, args.message)
-                compile(branch, commit, args.dry_run)
+            if text:
-
+                log("WARNING: redoing tests for branch %s: commit %s" % (branch, commit))
-                logFileName = test_path + "/log_" + branch.replace("/", "_")
+                text = benchmark_and_compare(branch, commit, resultsFileName, args.lastCLevel, filePath, fileName, cspeed, dspeed, args.lowerLimit, args.maxLoadAvg, args.message)
-                text_to_send = []
+                if text:
-                results_files = ""
+                    text_to_send.append(text)
-                for filePath in testFilePaths:
+                    results_files += resultsFileName + " "
-                    fileName = filePath.rpartition('/')[2]
+    if text_to_send:
-                    resultsFileName = test_path + "/results_" + branch.replace("/", "_") + "_" + fileName
+        send_email_with_attachments(branch, commit, last_commit, args.emails, text_to_send, results_files, logFileName, args.lowerLimit, have_mutt, have_mail)
-                    last_commit, cspeed, dspeed = get_last_commit(resultsFileName)
-
-                    if not args.dry_run:
-                        text = benchmark_and_compare(branch, commit, resultsFileName, args.lastCLevel, filePath, fileName, cspeed, dspeed, args.lowerLimit, args.maxLoadAvg, args.message)
-                        if text:
-                            text = benchmark_and_compare(branch, commit, resultsFileName, args.lastCLevel, filePath, fileName, cspeed, dspeed, args.lowerLimit, args.maxLoadAvg, args.message)
-                            if text:
-                                text_to_send.append(text)
-                                results_files += resultsFileName + " "
-                if text_to_send:
-                    send_email(branch, commit, last_commit, args.emails, text_to_send, results_files, logFileName, args.lowerLimit, have_mutt, have_mail)
-                notify(branch, commit, last_commit)
-        except Exception as e:
-            stack = traceback.format_exc()
-            log("ERROR: build %s, error %s" % (branch, str(e)) )
-            print stack
 
 
 if __name__ == '__main__':
@@ -183,6 +176,7 @@ if __name__ == '__main__':
     parser.add_argument('testFileNames', help='file names list for speed benchmark')
     parser.add_argument('emails', help='list of e-mail addresses to send warnings')
     parser.add_argument('--message', help='attach an additional message to e-mail', default="")
+    parser.add_argument('--repoURL', help='changes default repository URL', default=default_repo_url)
     parser.add_argument('--lowerLimit', type=float, help='send email if speed is lower than given limit', default=0.98)
     parser.add_argument('--maxLoadAvg', type=float, help='maximum load average to start testing', default=0.75)
     parser.add_argument('--lastCLevel', type=int, help='last compression level for testing', default=5)
@@ -194,56 +188,78 @@ if __name__ == '__main__':
     testFileNames = args.testFileNames.split()
     testFilePaths = []
     for fileName in testFileNames:
+        fileName = os.path.expanduser(fileName)
         if os.path.isfile(fileName):
             testFilePaths.append(os.path.abspath(fileName))
         else:
-            raise RuntimeError("File not found: " + fileName)
+            log("ERROR: File not found: " + fileName)
-
+            exit(1)
-    test_path = os.getcwd() + '/' + test_dir_name     # /path/to/zstd/tests/speedTest 
-    clone_path = test_path + '/' + 'zstd'             # /path/to/zstd/tests/speedTest/zstd 
 
     # check availability of e-mail senders
-    have_mutt = does_command_exist("mutt --help");
+    have_mutt = does_command_exist("mutt -h");
     have_mail = does_command_exist("mail -V");
     if not have_mutt and not have_mail:
-        log("WARNING: e-mail senders mail and mutt not found")
+        log("ERROR: e-mail senders 'mail' or 'mutt' not found")
+        exit(1)
+
+    print("PARAMETERS:\nrepoURL=%s" % args.repoURL)
+    print("working_path=%s" % working_path)
+    print("clone_path=%s" % clone_path)
+    print("testFilePath(%s)=%s" % (len(testFilePaths), testFilePaths))
+    print("message=%s" % args.message)
+    print("emails=%s" % args.emails)
+    print("maxLoadAvg=%s" % args.maxLoadAvg)
+    print("lowerLimit=%s" % args.lowerLimit)
+    print("lastCLevel=%s" % args.lastCLevel)
+    print("sleepTime=%s" % args.sleepTime)
+    print("dry_run=%s" % args.dry_run)
+    print("have_mutt=%s have_mail=%s" % (have_mutt, have_mail))
 
     # clone ZSTD repo if needed
-    if not os.path.isdir(test_path):
+    if not os.path.isdir(working_path):
-        os.mkdir(test_path)
+        os.mkdir(working_path)
     if not os.path.isdir(clone_path):
-        execute.cwd = test_path
+        execute.cwd = working_path
-        execute('git clone ' + repo_url)
+        execute('git clone ' + args.repoURL)
     if not os.path.isdir(clone_path):
-        raise RuntimeError("ZSTD clone not found: " + clone_path)
+        log("ERROR: ZSTD clone not found: " + clone_path)
+        exit(1)
     execute.cwd = clone_path
 
-    print "PARAMETERS:\ntest_path=%s" % test_path
+    # check if speedTest.pid already exists
-    print "clone_path=%s" % clone_path
+    pidfile = "./speedTest.pid"
-    print "testFilePath(%s)=%s" % (len(testFilePaths), testFilePaths)
+    if os.path.isfile(pidfile):
-    print "message=%s" % args.message
+        log("ERROR: %s already exists, exiting" % pidfile)
-    print "emails=%s" % args.emails
+        exit(1)
-    print "maxLoadAvg=%s" % args.maxLoadAvg
+
-    print "lowerLimit=%s" % args.lowerLimit
+    send_email(args.emails, email_header + ':%s test-zstd-speed.py has been started' % pid, args.message, have_mutt, have_mail)
-    print "lastCLevel=%s" % args.lastCLevel
+    file(pidfile, 'w').write(pid)
-    print "sleepTime=%s" % args.sleepTime
-    print "dry_run=%s" % args.dry_run
-    print "have_mutt=%s have_mail=%s" % (have_mutt, have_mail)
 
     while True:
-        pid = str(os.getpid())
+        try:
-        pidfile = "./speedTest.pid"
+            loadavg = os.getloadavg()[0]
-        if os.path.isfile(pidfile):
+            if (loadavg <= args.maxLoadAvg):
-            log("%s already exists, exiting" % pidfile)
+                branches = git_get_branches()
-        else:
+                for branch in branches:
-            file(pidfile, 'w').write(pid)
+                    commit = execute('git show -s --format=%h ' + branch)[0]
-            try:
+                    last_commit = update_config_file(branch, commit)
-                loadavg = os.getloadavg()[0]
+                    if commit == last_commit:
-                if (loadavg <= args.maxLoadAvg):
+                        log("skipping branch %s: head %s already processed" % (branch, commit))
-                    check_branches(args, test_path, testFilePaths, have_mutt, have_mail)
+                    else:
-                else:
+                        log("build branch %s: head %s is different from prev %s" % (branch, commit, last_commit))
-                    log("WARNING: main loadavg=%.2f is higher than %s" % (loadavg, args.maxLoadAvg))
+                        execute('git checkout -- . && git checkout ' + branch)
-            finally:
+                        print(git_get_changes(branch, commit, last_commit))
-                os.unlink(pidfile)
+                        test_commit(branch, commit, last_commit, args, testFilePaths, have_mutt, have_mail)
-        log("sleep for %s seconds" % args.sleepTime)
+            else:
-        time.sleep(args.sleepTime)
+                log("WARNING: main loadavg=%.2f is higher than %s" % (loadavg, args.maxLoadAvg))
+            log("sleep for %s seconds" % args.sleepTime)
+            time.sleep(args.sleepTime)
+        except Exception as e:
+            stack = traceback.format_exc()
+            email_topic = '%s:%s ERROR in %s:%s' % (email_header, pid, branch, commit)
+            send_email(args.emails, email_topic, stack, have_mutt, have_mail)
+            print(stack)
+        except KeyboardInterrupt:
+            os.unlink(pidfile)
+            send_email(args.emails, email_header + ':%s test-zstd-speed.py has been stopped' % pid, args.message, have_mutt, have_mail)
+            exit(0)

tests/test-zstd-versions.py

@@ -7,8 +7,9 @@ import glob
 import hashlib
 import os
 import shutil
-import subprocess
 import sys
+import subprocess
+from subprocess import Popen, PIPE
 
 repo_url = 'https://github.com/Cyan4973/zstd.git'
 tmp_dir_name = 'tests/versionsTest'
@@ -17,17 +18,32 @@ git_cmd = 'git'
 test_dat_src = 'README.md'
 test_dat = 'test_dat'
 head = 'vdevel'
+dict_source = 'dict_source'
+dict_files = './zstd/programs/*.c ./zstd/lib/common/*.c ./zstd/lib/compress/*.c ./zstd/lib/decompress/*.c ./zstd/lib/dictBuilder/*.c ./zstd/lib/legacy/*.c '
+dict_files += './zstd/programs/*.h ./zstd/lib/common/*.h ./zstd/lib/compress/*.h ./zstd/lib/dictBuilder/*.h ./zstd/lib/legacy/*.h'
+
+
+def execute(command, print_output=False, print_error=True, param_shell=False):
+    popen = Popen(command, stdout=PIPE, stderr=PIPE, shell=param_shell)
+    stdout_lines, stderr_lines = popen.communicate()
+    stderr_lines = stderr_lines.decode("utf-8")
+    stdout_lines = stdout_lines.decode("utf-8")
+    if print_output:
+        print(stdout_lines)
+        print(stderr_lines)
+    if popen.returncode is not None and popen.returncode != 0:
+        if not print_output and print_error:
+            print(stderr_lines)
+    return popen.returncode
 
 
 def proc(cmd_args, pipe=True, dummy=False):
     if dummy:
         return
     if pipe:
-        subproc = subprocess.Popen(cmd_args,
+        subproc = Popen(cmd_args, stdout=PIPE, stderr=PIPE)
-                                   stdout=subprocess.PIPE,
-                                   stderr=subprocess.PIPE)
     else:
-        subproc = subprocess.Popen(cmd_args)
+        subproc = Popen(cmd_args)
     return subproc.communicate()
 
 
@@ -45,21 +61,53 @@ def get_git_tags():
     return tags
 
 
+def create_dict(tag, dict_source_path):
+    dict_name = 'dict.' + tag
+    if not os.path.isfile(dict_name):
+        cFiles = glob.glob(dict_source_path + "/*.c")
+        hFiles = glob.glob(dict_source_path + "/*.h")
+        if tag == 'v0.5.0':
+            result = execute('./dictBuilder.' + tag + ' ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name, print_output=False, param_shell=True)
+        else:
+            result = execute('./zstd.' + tag + ' -f --train ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name, print_output=False, param_shell=True)
+        if result == 0:
+            print(dict_name + ' created')
+        else:
+            print('ERROR: creating of ' + dict_name + ' failed')
+    else:
+        print(dict_name + ' already exists')
+
+
+def dict_compress_sample(tag, sample):
+    dict_name = 'dict.' + tag
+    DEVNULL = open(os.devnull, 'wb')
+    if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-f',   sample], stderr=DEVNULL) == 0:
+        os.rename(sample + '.zst', sample + '_01_64_' + tag + '_dictio.zst')
+    if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-5f',  sample], stderr=DEVNULL) == 0:
+        os.rename(sample + '.zst', sample + '_05_64_' + tag + '_dictio.zst')
+    if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-9f',  sample], stderr=DEVNULL) == 0:
+        os.rename(sample + '.zst', sample + '_09_64_' + tag + '_dictio.zst')
+    if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-15f', sample], stderr=DEVNULL) == 0:
+        os.rename(sample + '.zst', sample + '_15_64_' + tag + '_dictio.zst')
+    if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-18f', sample], stderr=DEVNULL) == 0:
+        os.rename(sample + '.zst', sample + '_18_64_' + tag + '_dictio.zst')
+    # zstdFiles = glob.glob("*.zst*")
+    # print(zstdFiles)
+    print(tag + " : dict compression completed")
+
+
 def compress_sample(tag, sample):
-    try:
+    DEVNULL = open(os.devnull, 'wb')
-        from subprocess import DEVNULL  # py3k
-    except ImportError:
-        DEVNULL = open(os.devnull, 'wb')
     if subprocess.call(['./zstd.' + tag, '-f',   sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_01_64_' + tag + '.zst')
+        os.rename(sample + '.zst', sample + '_01_64_' + tag + '_nodict.zst')
     if subprocess.call(['./zstd.' + tag, '-5f',  sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_05_64_' + tag + '.zst')
+        os.rename(sample + '.zst', sample + '_05_64_' + tag + '_nodict.zst')
     if subprocess.call(['./zstd.' + tag, '-9f',  sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_09_64_' + tag + '.zst')
+        os.rename(sample + '.zst', sample + '_09_64_' + tag + '_nodict.zst')
     if subprocess.call(['./zstd.' + tag, '-15f', sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_15_64_' + tag + '.zst')
+        os.rename(sample + '.zst', sample + '_15_64_' + tag + '_nodict.zst')
     if subprocess.call(['./zstd.' + tag, '-18f', sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_18_64_' + tag + '.zst')
+        os.rename(sample + '.zst', sample + '_18_64_' + tag + '_nodict.zst')
     # zstdFiles = glob.glob("*.zst*")
     # print(zstdFiles)
     print(tag + " : compression completed")
@@ -87,16 +135,16 @@ def remove_duplicates():
 
 def decompress_zst(tag):
     dec_error = 0
-    list_zst = sorted(glob.glob('*.zst'))
+    list_zst = sorted(glob.glob('*_nodict.zst'))
-    try:
-        from subprocess import DEVNULL  # py3k
-    except ImportError:
-        DEVNULL = open(os.devnull, 'wb')
     for file_zst in list_zst:
         print(file_zst, end=' ')
         print(tag, end=' ')
         file_dec = file_zst + '_d64_' + tag + '.dec'
-        if subprocess.call(['./zstd.' + tag, '-df', file_zst, '-o', file_dec], stderr=DEVNULL) == 0:
+        if tag <= 'v0.5.0':
+            params = ['./zstd.' + tag, '-df', file_zst, file_dec]
+        else:
+            params = ['./zstd.' + tag, '-df', file_zst, '-o', file_dec]
+        if execute(params) == 0:
             if not filecmp.cmp(file_dec, test_dat):
                 print('ERR !! ')
                 dec_error = 1
@@ -107,12 +155,43 @@ def decompress_zst(tag):
     return dec_error
 
 
+def decompress_dict(tag):
+    dec_error = 0
+    list_zst = sorted(glob.glob('*_dictio.zst'))
+    for file_zst in list_zst:
+        dict_tag = file_zst[0:len(file_zst)-11]  # remove "_dictio.zst"
+        if head in dict_tag: # find vdevel
+            dict_tag = head
+        else:
+            dict_tag = dict_tag[dict_tag.rfind('v'):]
+        if tag == 'v0.6.0' and dict_tag < 'v0.6.0':
+            continue
+        dict_name = 'dict.' + dict_tag
+        print(file_zst + ' ' + tag + ' dict=' + dict_tag, end=' ')
+        file_dec = file_zst + '_d64_' + tag + '.dec'
+        if tag <= 'v0.5.0':
+            params = ['./zstd.' + tag, '-D', dict_name, '-df', file_zst, file_dec]
+        else:
+            params = ['./zstd.' + tag, '-D', dict_name, '-df', file_zst, '-o', file_dec]
+        if execute(params) == 0:
+            if not filecmp.cmp(file_dec, test_dat):
+                print('ERR !! ')
+                dec_error = 1
+            else:
+                print('OK     ')
+        else:
+            print('command does not work')
+            dec_error = 1
+    return dec_error
+
+
 if __name__ == '__main__':
     error_code = 0
-    base_dir = os.getcwd() + '/..'           # /path/to/zstd
+    base_dir = os.getcwd() + '/..'                  # /path/to/zstd
-    tmp_dir = base_dir + '/' + tmp_dir_name  # /path/to/zstd/tests/versionsTest
+    tmp_dir = base_dir + '/' + tmp_dir_name         # /path/to/zstd/tests/versionsTest
-    clone_dir = tmp_dir + '/' + 'zstd'       # /path/to/zstd/tests/versionsTest/zstd
+    clone_dir = tmp_dir + '/' + 'zstd'              # /path/to/zstd/tests/versionsTest/zstd
-    programs_dir = base_dir + '/programs'    # /path/to/zstd/programs
+    dict_source_path = tmp_dir + '/' + dict_source  # /path/to/zstd/tests/versionsTest/dict_source
+    programs_dir = base_dir + '/programs'           # /path/to/zstd/programs
     os.makedirs(tmp_dir, exist_ok=True)
 
     # since Travis clones limited depth, we should clone full repository
@@ -137,6 +216,10 @@ if __name__ == '__main__':
                 os.makedirs(r_dir, exist_ok=True)
                 os.chdir(clone_dir)
                 git(['--work-tree=' + r_dir, 'checkout', tag, '--', '.'], False)
+                if tag == 'v0.5.0':
+                    os.chdir(r_dir + '/dictBuilder')  # /path/to/zstd/tests/versionsTest/v0.5.0/dictBuilder
+                    make(['clean', 'dictBuilder'], False)
+                    shutil.copy2('dictBuilder', '{}/dictBuilder.{}'.format(tmp_dir, tag))
                 os.chdir(r_dir + '/programs')  # /path/to/zstd/tests/versionsTest/<TAG>/programs
                 make(['clean', 'zstd'], False)
             else:
@@ -151,15 +234,25 @@ if __name__ == '__main__':
     for dec in glob.glob("*.dec"):
         os.remove(dec)
 
+    # copy *.c and *.h to a temporary directory ("dict_source")
+    if not os.path.isdir(dict_source_path):
+        os.mkdir(dict_source_path)
+        print('cp ' + dict_files + ' ' + dict_source_path)
+        execute('cp ' + dict_files + ' ' + dict_source_path, param_shell=True)
+
     print('Compress test.dat by all released zstd')
 
     error_code = 0
     for tag in tags:
         print(tag)
+        if tag >= 'v0.5.0':
+            create_dict(tag, dict_source_path)
+            dict_compress_sample(tag, test_dat)
+            remove_duplicates()
+            error_code += decompress_dict(tag)
         compress_sample(tag, test_dat)
         remove_duplicates()
-        if tag >= 'v0.5.1':
+        error_code += decompress_zst(tag)
-            error_code += decompress_zst(tag)
 
     print('')
     print('Enumerate different compressed files')