Merge pull request #222 from Cyan4973/dev

Improved speed test
Merge pull request #221 from inikep/dev
2016-06-23 01:32:56 +02:00 · 2016-06-22 23:34:19 +02:00 · 2016-06-22 20:06:42 +02:00 · 2016-06-22 18:12:57 +02:00 · 2016-06-22 17:11:01 +02:00 · 2016-06-22 16:30:26 +02:00
20 changed files with 565 additions and 403 deletions
@@ -13,6 +13,7 @@
 *.dylib

 # Executables
+zstd
 *.exe
 *.out
 *.app
@@ -6,85 +6,82 @@ matrix:
    # Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes)
    - os: linux
      sudo: false
-      env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=travis-install
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make travis-install"
    - os: linux
      sudo: false
-      env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=cmaketest
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make cmaketest"
    - os: linux
      sudo: false
-      env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=test
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make test"
    - os: linux
      sudo: false
-      env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM="-C programs test-zstd_nolegacy"
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make -C programs test-zstd_nolegacy"
    - os: linux
      sudo: false
-      env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=usan
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make usan"
    - os: linux
      sudo: false
-      env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=asan
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make asan"
    - os: linux
      sudo: false
-      env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=zlibwrapper
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make zlibwrapper"
    # Standard Ubuntu 12.04 LTS Server Edition 64 bit
    - os: linux
      sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=clangtest
+      env: PLATFORM="Ubuntu 12.04" CMD="make libc6install clangtest"
    - os: linux
      sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gpptest
+      env: PLATFORM="Ubuntu 12.04" CMD="make gppinstall gpptest"
    - os: linux
      sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gnu90test
+      env: PLATFORM="Ubuntu 12.04" CMD="make libc6install gnu90test"
    - os: linux
      sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=c99test
+      env: PLATFORM="Ubuntu 12.04" CMD="make libc6install c99test"
    - os: linux
      sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gnu99test
+      env: PLATFORM="Ubuntu 12.04" CMD="make libc6install gnu99test"
    - os: linux
      sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=armtest-w-install
+      env: PLATFORM="Ubuntu 12.04" CMD="make arminstall armtest"
    - os: linux
      sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C programs test32"
+      env: PLATFORM="Ubuntu 12.04" CMD="make libc6install && make -C programs test32"
    - os: linux
      sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C tests versionsTest"
+      env: PLATFORM="Ubuntu 12.04" CMD="make -C tests versionsTest"
    - os: linux
      sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=asan32
+      env: PLATFORM="Ubuntu 12.04" CMD="make libc6install asan32"
    - os: linux
      sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C programs valgrindTest"
+      env: PLATFORM="Ubuntu 12.04" CMD="make valgrindinstall && make -C programs valgrindTest"
    # Ubuntu 14.04 LTS Server Edition 64 bit
    - os: linux
      dist: trusty
      sudo: required
-      env: PLATFORM="Ubuntu 14.04" MAKE_PARAM=ppctest-w-install
+      env: PLATFORM="Ubuntu 14.04" CMD="make gcc5install gcc5test"
    - os: linux
      dist: trusty
      sudo: required
-      env: PLATFORM="Ubuntu 14.04" MAKE_PARAM=zlibwrapper
+      env: PLATFORM="Ubuntu 14.04" CMD="make gcc6install gcc6test"
+    - os: linux
+      dist: trusty
+      sudo: required
+      env: PLATFORM="Ubuntu 14.04" CMD="make ppcinstall ppctest"
+    - os: linux
+      dist: trusty
+      sudo: required
+      env: PLATFORM="Ubuntu 14.04" CMD="make zlibwrapper"
    # OS X Mavericks
    - os: osx
-      env: PLATFORM="OS X Mavericks" MAKE_PARAM=travis-install
+      env: PLATFORM="OS X Mavericks" CMD="make travis-install"
    - os: osx
-      env: PLATFORM="OS X Mavericks" MAKE_PARAM=gnu90test
+      env: PLATFORM="OS X Mavericks" CMD="make gnu90test"
    - os: osx
-      env: PLATFORM="OS X Mavericks" MAKE_PARAM=test
+      env: PLATFORM="OS X Mavericks" CMD="make test"
  exclude:
    - compiler: gcc

-before_install:
-  - set -e
-  - |
-    if [ "$TRAVIS_OS_NAME" == "linux" ]; then
-      CAN_I_RUN_SUDO=$(sudo -n uptime 2>&1|grep "load"|wc -l)
-      echo "CAN_I_RUN_SUDO=$CAN_I_RUN_SUDO\n"
-      if [ ${CAN_I_RUN_SUDO} -gt 0 ]; then
-        sudo apt-get install -y -qq clang g++-multilib gcc-multilib valgrind
-      fi
-    fi
-
 script:
-  - make $MAKE_PARAM
+  - sh -c "$CMD"
@@ -51,6 +51,7 @@ all:

 zstdprogram:
 	$(MAKE) -C $(PRGDIR)
+	cp $(PRGDIR)/zstd .

 zlibwrapper:
 	$(MAKE) -C $(ZSTDDIR) all
@@ -63,6 +64,7 @@ clean:
 	@$(MAKE) -C $(ZSTDDIR) $@ > $(VOID)
 	@$(MAKE) -C $(PRGDIR) $@ > $(VOID)
 	@$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID)
+	@rm -f zstd
 	@echo Cleaning completed


@@ -85,6 +87,14 @@ travis-install:
 gpptest: clean
 	$(MAKE) all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"

+gcc5test: clean
+	gcc-5 -v
+	$(MAKE) all CC=gcc-5 MOREFLAGS="-Werror"
+
+gcc6test: clean
+	gcc-6 -v
+	$(MAKE) all CC=gcc-6 MOREFLAGS="-Werror"
+
 clangtest: clean
 	clang -v
 	$(MAKE) all CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion"
@@ -93,35 +103,13 @@ armtest: clean
 	$(MAKE) -C $(PRGDIR) datagen   # use native, faster
 	$(MAKE) -C $(PRGDIR) test CC=arm-linux-gnueabi-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static"

-# for Travis CI
-arminstall: clean
-	sudo apt-get install -y -q qemu binfmt-support qemu-user-static gcc-arm-linux-gnueabi
-
-# for Travis CI
-armtest-w-install: clean arminstall armtest
-
 ppctest: clean
 	$(MAKE) -C $(PRGDIR) datagen   # use native, faster
 	$(MAKE) -C $(PRGDIR) test CC=powerpc-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static"

-# for Travis CI
-ppcinstall: clean
-	# sudo apt-get update  -y -q
-	sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu  # doesn't work with Ubuntu 12.04
-
-# for Travis CI
-ppctest-w-install: clean ppcinstall ppctest
-
 ppc64test: clean
 	$(MAKE) -C $(PRGDIR) datagen   # use native, faster
-	$(MAKE) -C $(PRGDIR) test CC=powerpc64le-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static"
-
-ppc64install: clean
-	sudo apt-get update  -y -q
-	sudo apt-get install -y -q qemu-ppc64le binfmt-support qemu-user-static gcc-powerpc64le-linux-gnu
-	update-binfmts --displ
-
-ppc64test-w-install: clean ppc64install ppc64test
+	$(MAKE) -C $(PRGDIR) test CC=powerpc-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-m64 -Werror -static"

 usan: clean
 	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=undefined"
@@ -181,3 +169,37 @@ bmix32test: clean
 bmi32test: clean
 	CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(PRGDIR) test
 endif
+
+
+#------------------------------------------------------------------------
+# for Travis CI
+#------------------------------------------------------------------------
+libc6install:
+	sudo apt-get install -y -qq libc6-dev-i386
+
+gppinstall:
+	sudo apt-get install -y -qq g++-multilib
+
+gcc5install:
+	sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
+	sudo apt-get update -y -qq
+	sudo apt-get install -y -qq gcc-5-multilib
+
+gcc6install:
+	sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
+	sudo apt-get update -y -qq 
+	sudo apt-get install -y -qq gcc-6-multilib
+
+arminstall: clean
+	sudo apt-get install -y -q qemu binfmt-support qemu-user-static gcc-arm-linux-gnueabi
+
+ppcinstall: clean
+	sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu  # doesn't work with Ubuntu 12.04
+
+ppc64install: clean   # compilation works but cannot be executed
+	sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu
+	update-binfmts --displ
+
+valgrindinstall:
+	echo sudo apt-get install -y -qq valgrind
+	sudo apt-get install -y -qq valgrind
@@ -1,3 +1,9 @@
+v0.7.1
+fixed : ZBUFF_compressEnd() called multiple times with too small `dst` buffer, reported by Christophe Chevalier
+fixed : dictBuilder fails if first sample is too small, reported by Руслан Ковалёв
+fixed : corruption issue, reported by cj
+modified : checksum enabled by default in command line mode
+
 v0.7.0
 New : Support for directory compression, using `-r`, thanks to Przemyslaw Skibinski
 New : Command `--rm`, to remove source file after successful de/compression
@@ -136,7 +136,7 @@ typedef U32 HUF_DTable;
 size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< single-symbol decoder */
 size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< double-symbols decoder */

-size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
+size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< decodes RLE and uncompressed */
 size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */
 size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< single-symbol decoder */
 size_t HUF_decompress4X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< double-symbols decoder */
@@ -61,7 +61,7 @@ extern "C" {
 ***************************************/
 #define ZSTD_VERSION_MAJOR    0
 #define ZSTD_VERSION_MINOR    7
-#define ZSTD_VERSION_RELEASE  0
+#define ZSTD_VERSION_RELEASE  1

 #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
 #define ZSTD_QUOTE(str) #str
@@ -132,7 +132,7 @@ ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapa
 /*! ZSTD_compress_usingDict() :
 *   Compression using a pre-defined Dictionary content (see dictBuilder).
 *   Note 1 : This function load the dictionary, resulting in a significant startup time.
-*   Note 2 : `dict` must remain valid and unmodified during compression operation.
+*   Note 2 : `dict` must remain accessible and unmodified during compression operation.
 *   Note 3 : `dict` can be `NULL`, in which case, it's equivalent to ZSTD_compressCCtx() */
 ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
                                           void* dst, size_t dstCapacity,
@@ -144,7 +144,7 @@ ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
 *   Decompression using a pre-defined Dictionary content (see dictBuilder).
 *   Dictionary must be identical to the one used during compression.
 *   Note 1 : This function load the dictionary, resulting in a significant startup time
-*   Note 2 : `dict` must remain valid and unmodified during compression operation.
+*   Note 2 : `dict` must remain accessible and unmodified during compression operation.
 *   Note 3 : `dict` can be `NULL`, in which case, it's equivalent to ZSTD_decompressDCtx() */
 ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
                                             void* dst, size_t dstCapacity,
@@ -192,7 +192,7 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,

 /* ====================================================================================
 * The definitions in this section are considered experimental.
- * They should never be used in association with a dynamic library, as they may change in the future.
+ * They should never be used with a dynamic library, as they may change in the future.
 * They are provided for advanced usages.
 * Use them only in association with static linking.
 * ==================================================================================== */
@@ -322,15 +322,22 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
  It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx()

  Then, consume your input using ZSTD_compressContinue().
-  ZSTD_compressContinue() presumes prior data is still accessible and unmodified (up to maximum distance size, see WindowLog).
-  The interface is synchronous, so input will be entirely consumed and produce associated compressed output.
-  You must ensure there is enough space in destination buffer to store compressed data under worst case scenario.
-  Worst case evaluation is provided by ZSTD_compressBound().
+  There are some important considerations to keep in mind when using this advanced function :
+  - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffer only.
+  - Interface is synchronous : input will be entirely consumed and produce 1+ compressed blocks.
+  - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario.
+    Worst case evaluation is provided by ZSTD_compressBound().
+    ZSTD_compressContinue() doesn't guarantee recover after a failed compression.
+  - ZSTD_compressContinue() presumes prior input ***is still accessible and unmodified*** (up to maximum distance size, see WindowLog).
+    It remembers all previous contiguous blocks, plus one separated memory segment (which can itself consists of multiple contiguous blocks)
+  - ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps.
+    In which case, it will "discard" the relevant memory section from its history.
+

  Finish a frame with ZSTD_compressEnd(), which will write the epilogue.
  Without epilogue, frames will be considered unfinished (broken) by decoders.

-  You can then reuse ZSTD_CCtx to compress some new frame.
+  You can then reuse `ZSTD_CCtx` (ZSTD_compressBegin()) to compress some new frame.
 */

 typedef struct {
@@ -357,7 +364,7 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds
  A ZSTD_DCtx object can be re-used multiple times.

  First optional operation is to retrieve frame parameters, using ZSTD_getFrameParams(), which doesn't consume the input.
-  It can provide the minimum size of rolling buffer required to properly decompress data,
+  It can provide the minimum size of rolling buffer required to properly decompress data (`windowSize`),
  and optionally the final size of uncompressed content.
  (Note : content size is an optional info that may not be present. 0 means : content size unknown)
  Frame parameters are extracted from the beginning of compressed frame.
@@ -373,7 +380,7 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds
  Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively.
  ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue().
  ZSTD_decompressContinue() requires this exact amount of bytes, or it will fail.
-  ZSTD_decompressContinue() needs previous data blocks during decompression, up to (1 << windowlog).
+  ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize`.
  They should preferably be located contiguously, prior to current block. Alternatively, a round buffer is also possible.

  @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).
@@ -79,7 +79,7 @@ static size_t const ZBUFF_endFrameSize = ZSTD_BLOCKHEADERSIZE;
 *  output : ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + ZBUFF_endFrameSize : ensures it's always possible to write/flush/end a full block at best speed.
 * **************************************************/

-typedef enum { ZBUFFcs_init, ZBUFFcs_load, ZBUFFcs_flush } ZBUFF_cStage;
+typedef enum { ZBUFFcs_init, ZBUFFcs_load, ZBUFFcs_flush, ZBUFFcs_final } ZBUFF_cStage;

 /* *** Resources *** */
 struct ZBUFF_CCtx_s {
@@ -162,7 +162,7 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
    zbc->inToCompress = 0;
    zbc->inBuffPos = 0;
    zbc->inBuffTarget = zbc->blockSize;
-    zbc->outBuffFlushedSize = 0;
+    zbc->outBuffContentSize = zbc->outBuffFlushedSize = 0;
    zbc->stage = ZBUFFcs_load;
    return 0;   /* ready to go */
 }
@@ -196,7 +196,7 @@ MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src
 static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc,
                              void* dst, size_t* dstCapacityPtr,
                        const void* src, size_t* srcSizePtr,
-                              int flush)   /* aggregate : wait for full block before compressing */
+                              int flush)
 {
    U32 notDone = 1;
    const char* const istart = (const char*)src;
@@ -243,17 +243,20 @@ static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc,
            }

        case ZBUFFcs_flush:
-            /* flush into dst */
            {   size_t const toFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize;
                size_t const flushed = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outBuffFlushedSize, toFlush);
                op += flushed;
                zbc->outBuffFlushedSize += flushed;
-                if (toFlush!=flushed) { notDone = 0; break; } /* not enough space within dst to store compressed block : stop there */
-                zbc->outBuffContentSize = 0;
-                zbc->outBuffFlushedSize = 0;
+                if (toFlush!=flushed) { notDone = 0; break; } /* dst too small to store flushed data : stop there */
+                zbc->outBuffContentSize = zbc->outBuffFlushedSize = 0;
                zbc->stage = ZBUFFcs_load;
                break;
            }
+
+        case ZBUFFcs_final:
+            notDone = 0;   /* do nothing */
+            break;
+
        default:
            return ERROR(GENERIC);   /* impossible */
        }
@@ -291,19 +294,30 @@ size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr)
    BYTE* const ostart = (BYTE*)dst;
    BYTE* const oend = ostart + *dstCapacityPtr;
    BYTE* op = ostart;
-    size_t outSize = *dstCapacityPtr;
-    size_t epilogueSize, remaining;
-    ZBUFF_compressFlush(zbc, dst, &outSize);     /* flush any remaining inBuff */
-    op += outSize;
-    epilogueSize = ZSTD_compressEnd(zbc->zc, zbc->outBuff + zbc->outBuffContentSize, zbc->outBuffSize - zbc->outBuffContentSize);   /* epilogue into outBuff */
-    zbc->outBuffContentSize += epilogueSize;
-    outSize = oend-op;
-    zbc->stage = ZBUFFcs_flush;
-    remaining = ZBUFF_compressFlush(zbc, op, &outSize);   /* attempt to flush epilogue into dst */
-    op += outSize;
-    if (!remaining) zbc->stage = ZBUFFcs_init;   /* close only if nothing left to flush */
-    *dstCapacityPtr = op-ostart;                 /* tells how many bytes were written */
-    return remaining;
+
+    if (zbc->stage != ZBUFFcs_final) {
+        /* flush whatever remains */
+        size_t outSize = *dstCapacityPtr;
+        size_t const remainingToFlush = ZBUFF_compressFlush(zbc, dst, &outSize);
+        op += outSize;
+        if (remainingToFlush) {
+            *dstCapacityPtr = op-ostart;
+            return remainingToFlush + ZBUFF_endFrameSize;
+        }
+        /* create epilogue */
+        zbc->stage = ZBUFFcs_final;
+        zbc->outBuffContentSize = ZSTD_compressEnd(zbc->zc, zbc->outBuff, zbc->outBuffSize); /* epilogue into outBuff */
+    }
+
+    /* flush epilogue */
+    {   size_t const toFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize;
+        size_t const flushed = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outBuffFlushedSize, toFlush);
+        op += flushed;
+        zbc->outBuffFlushedSize += flushed;
+        *dstCapacityPtr = op-ostart;
+        if (toFlush==flushed) zbc->stage = ZBUFFcs_init;  /* end reached */
+        return toFlush - flushed;
+    }
 }


@@ -246,7 +246,7 @@ ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, U

 size_t ZSTD_sizeofCCtx(ZSTD_compressionParameters cParams)   /* hidden interface, for paramagrill */
 {
-    ZSTD_CCtx* zc = ZSTD_createCCtx();
+    ZSTD_CCtx* const zc = ZSTD_createCCtx();
    ZSTD_parameters params;
    memset(&params, 0, sizeof(params));
    params.cParams = cParams;
@@ -295,7 +295,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
    zc->seqStore.buffer = zc->hashTable3 + h3Size;
    zc->hufTable = (HUF_CElt*)zc->seqStore.buffer;
    zc->flagStaticTables = 0;
-    zc->seqStore.buffer = ((U32*)(zc->seqStore.buffer)) + 256;
+    zc->seqStore.buffer = ((U32*)(zc->seqStore.buffer)) + 256;  /* note : HUF_CElt* is incomplete type, size is simulated using U32 */

    zc->nextToUpdate = 1;
    zc->nextSrc = NULL;
@@ -313,14 +313,17 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
        zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1<<Litbits);
        zc->seqStore.matchLengthFreq = zc->seqStore.litLengthFreq + (MaxLL+1);
        zc->seqStore.offCodeFreq = zc->seqStore.matchLengthFreq + (MaxML+1);
-        zc->seqStore.matchTable = (ZSTD_match_t*)((void*)(zc->seqStore.offCodeFreq + (MaxOff+1)));
-        zc->seqStore.priceTable = (ZSTD_optimal_t*)((void*)(zc->seqStore.matchTable + ZSTD_OPT_NUM+1));
+        zc->seqStore.buffer = zc->seqStore.offCodeFreq + (MaxOff+1);
+        zc->seqStore.matchTable = (ZSTD_match_t*)zc->seqStore.buffer;
+        zc->seqStore.buffer = zc->seqStore.matchTable + ZSTD_OPT_NUM+1;
+        zc->seqStore.priceTable = (ZSTD_optimal_t*)zc->seqStore.buffer;
        zc->seqStore.buffer = zc->seqStore.priceTable + ZSTD_OPT_NUM+1;
        zc->seqStore.litLengthSum = 0;
    }
-    zc->seqStore.offsetStart = (U32*) (zc->seqStore.buffer);
-    zc->seqStore.litLengthStart = (U16*) (void*)(zc->seqStore.offsetStart + maxNbSeq);
-    zc->seqStore.matchLengthStart = (U16*) (void*)(zc->seqStore.litLengthStart + maxNbSeq);
+    zc->seqStore.offsetStart = (U32*)(zc->seqStore.buffer);
+    zc->seqStore.buffer = zc->seqStore.offsetStart + maxNbSeq;
+    zc->seqStore.litLengthStart = (U16*)zc->seqStore.buffer;
+    zc->seqStore.matchLengthStart = zc->seqStore.litLengthStart + maxNbSeq;
    zc->seqStore.llCodeStart = (BYTE*) (zc->seqStore.matchLengthStart + maxNbSeq);
    zc->seqStore.mlCodeStart = zc->seqStore.llCodeStart + maxNbSeq;
    zc->seqStore.offCodeStart = zc->seqStore.mlCodeStart + maxNbSeq;
@@ -1024,11 +1027,12 @@ static unsigned ZSTD_NbCommonBytes (register size_t val)
 }


-static size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit)
+static size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* const pInLimit)
 {
    const BYTE* const pStart = pIn;
+    const BYTE* const pInLoopLimit = pInLimit - (sizeof(size_t)-1);

-    while ((pIn<pInLimit-(sizeof(size_t)-1))) {
+    while (pIn < pInLoopLimit) {
        size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
        if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; }
        pIn += ZSTD_NbCommonBytes(diff);
@@ -1128,7 +1132,6 @@ void ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx,
    size_t offset_1=cctx->rep[0], offset_2=cctx->rep[1];

    /* init */
-    ZSTD_resetSeqStore(seqStorePtr);
    ip += (ip==lowest);
    {   U32 const maxRep = (U32)(ip-lowest);
        if (offset_1 > maxRep) offset_1 = 0;
@@ -1137,35 +1140,34 @@ void ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx,

    /* Main Search Loop */
    while (ip < ilimit) {   /* < instead of <=, because repcode check at (ip+1) */
-        size_t mlCode;
-        size_t offset;
+        size_t mLength;
        size_t const h = ZSTD_hashPtr(ip, hBits, mls);
        U32 const current = (U32)(ip-base);
        U32 const matchIndex = hashTable[h];
        const BYTE* match = base + matchIndex;
        hashTable[h] = current;   /* update hash table */

-        if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) {   /* note : by construction, offset_1 <= current */
-            mlCode = ZSTD_count(ip+1+EQUAL_READ32, ip+1+EQUAL_READ32-offset_1, iend) + EQUAL_READ32;
+        if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) { /* note : by construction, offset_1 <= current */
+            mLength = ZSTD_count(ip+1+EQUAL_READ32, ip+1+EQUAL_READ32-offset_1, iend) + EQUAL_READ32;
            ip++;
-            ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mlCode-MINMATCH);
-       } else {
-            if ( (matchIndex <= lowestIndex) ||
-                 (MEM_read32(match) != MEM_read32(ip)) ) {
+            ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mLength-MINMATCH);
+        } else {
+            size_t offset;
+            if ( (matchIndex <= lowestIndex) || (MEM_read32(match) != MEM_read32(ip)) ) {
                ip += ((ip-anchor) >> g_searchStrength) + 1;
                continue;
            }
-            mlCode = ZSTD_count(ip+EQUAL_READ32, match+EQUAL_READ32, iend) + EQUAL_READ32;
+            mLength = ZSTD_count(ip+EQUAL_READ32, match+EQUAL_READ32, iend) + EQUAL_READ32;
            offset = ip-match;
-            while ((ip>anchor) && (match>lowest) && (ip[-1] == match[-1])) { ip--; match--; mlCode++; }  /* catch up */
+            while (((ip>anchor) & (match>lowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */
            offset_2 = offset_1;
            offset_1 = offset;

-            ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mlCode-MINMATCH);
+            ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
        }

        /* match found */
-        ip += mlCode;
+        ip += mLength;
        anchor = ip;

        if (ip <= ilimit) {
@@ -1177,18 +1179,18 @@ void ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx,
                 && ( (offset_2>0)
                 & (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) {
                /* store sequence */
-                size_t const rlCode = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-offset_2, iend) + EQUAL_READ32;
+                size_t const rLength = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-offset_2, iend) + EQUAL_READ32;
                { size_t const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; } /* swap offset_2 <=> offset_1 */
                hashTable[ZSTD_hashPtr(ip, hBits, mls)] = (U32)(ip-base);
-                ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rlCode-MINMATCH);
-                ip += rlCode;
+                ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rLength-MINMATCH);
+                ip += rLength;
                anchor = ip;
                continue;   /* faster when present ... (?) */
    }   }   }

    /* save reps for next block */
-    cctx->savedRep[0] = offset_1 ? (U32)offset_1 : (U32)(iend-base);
-    cctx->savedRep[1] = offset_2 ? (U32)offset_2 : (U32)(iend-base);
+    cctx->savedRep[0] = offset_1 ? (U32)offset_1 : (U32)(iend - base) + 1;
+    cctx->savedRep[1] = offset_2 ? (U32)offset_2 : (U32)(iend - base) + 1;

    /* Last Literals */
    {   size_t const lastLLSize = iend - anchor;
@@ -1238,32 +1240,25 @@ static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx,
    const BYTE* const ilimit = iend - 8;
    U32 offset_1=ctx->rep[0], offset_2=ctx->rep[1];

-    /* init */
-    ZSTD_resetSeqStore(seqStorePtr);
-    /* skip first position to avoid read overflow during repcode match check */
-    hashTable[ZSTD_hashPtr(ip, hBits, mls)] = (U32)(ip-base);
-    ip++;
-
-    /* Main Search Loop */
+    /* Search Loop */
    while (ip < ilimit) {  /* < instead of <=, because (ip+1) */
        const size_t h = ZSTD_hashPtr(ip, hBits, mls);
        const U32 matchIndex = hashTable[h];
        const BYTE* matchBase = matchIndex < dictLimit ? dictBase : base;
        const BYTE* match = matchBase + matchIndex;
        const U32 current = (U32)(ip-base);
-        const U32 repIndex = current + 1 - offset_1;
+        const U32 repIndex = current + 1 - offset_1;   /* offset_1 expected <= current +1 */
        const BYTE* repBase = repIndex < dictLimit ? dictBase : base;
        const BYTE* repMatch = repBase + repIndex;
-        size_t mlCode;
-        U32 offset;
+        size_t mLength;
        hashTable[h] = current;   /* update hash table */

-        if ( (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > lowestIndex))   /* intentional overflow */
+        if ( (((U32)((dictLimit-1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > lowestIndex))
           && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
            const BYTE* repMatchEnd = repIndex < dictLimit ? dictEnd : iend;
-            mlCode = ZSTD_count_2segments(ip+1+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repMatchEnd, lowPrefixPtr) + EQUAL_READ32;
+            mLength = ZSTD_count_2segments(ip+1+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repMatchEnd, lowPrefixPtr) + EQUAL_READ32;
            ip++;
-            ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mlCode-MINMATCH);
+            ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mLength-MINMATCH);
        } else {
            if ( (matchIndex < lowestIndex) ||
                 (MEM_read32(match) != MEM_read32(ip)) ) {
@@ -1272,16 +1267,17 @@ static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx,
            }
            {   const BYTE* matchEnd = matchIndex < dictLimit ? dictEnd : iend;
                const BYTE* lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr;
-                mlCode = ZSTD_count_2segments(ip+EQUAL_READ32, match+EQUAL_READ32, iend, matchEnd, lowPrefixPtr) + EQUAL_READ32;
-                while ((ip>anchor) && (match>lowMatchPtr) && (ip[-1] == match[-1])) { ip--; match--; mlCode++; }   /* catch up */
+                U32 offset;
+                mLength = ZSTD_count_2segments(ip+EQUAL_READ32, match+EQUAL_READ32, iend, matchEnd, lowPrefixPtr) + EQUAL_READ32;
+                while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; }   /* catch up */
                offset = current - matchIndex;
                offset_2 = offset_1;
                offset_1 = offset;
-                ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mlCode-MINMATCH);
+                ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
        }   }

        /* found a match : store it */
-        ip += mlCode;
+        ip += mLength;
        anchor = ip;

        if (ip <= ilimit) {
@@ -1337,8 +1333,6 @@ static void ZSTD_compressBlock_fast_extDict(ZSTD_CCtx* ctx,
 }


-
-
 /*-*************************************
 *  Binary Tree search
 ***************************************/
@@ -1437,7 +1431,7 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co
    }   }

    *smallerPtr = *largerPtr = 0;
-    if (bestLength > 384) return MIN(192, (U32)(bestLength - 384));
+    if (bestLength > 384) return MIN(192, (U32)(bestLength - 384));   /* speed optimization */
    if (matchEndIdx > current + 8) return matchEndIdx - current - 8;
    return 1;
 }
@@ -1571,7 +1565,6 @@ static void ZSTD_updateTree_extDict(ZSTD_CCtx* zc, const BYTE* const ip, const B
 }


-
 /** Tree updater, providing best match */
 static size_t ZSTD_BtFindBestMatch_extDict (
                        ZSTD_CCtx* zc,
@@ -1622,7 +1615,7 @@ U32 ZSTD_insertAndFindFirstIndex (ZSTD_CCtx* zc, const BYTE* ip, U32 mls)
    const U32 target = (U32)(ip - base);
    U32 idx = zc->nextToUpdate;

-    while(idx < target) {
+    while(idx < target) { /* catch up */
        size_t const h = ZSTD_hashPtr(base+idx, hashLog, mls);
        NEXT_IN_CHAIN(idx, chainMask) = hashTable[h];
        hashTable[h] = idx;
@@ -1659,7 +1652,7 @@ size_t ZSTD_HcFindBestMatch_generic (
    /* HC4 match finder */
    U32 matchIndex = ZSTD_insertAndFindFirstIndex (zc, ip, mls);

-    for ( ; (matchIndex>lowLimit) && (nbAttempts) ; nbAttempts--) {
+    for ( ; (matchIndex>lowLimit) & (nbAttempts>0) ; nbAttempts--) {
        const BYTE* match;
        size_t currentMl=0;
        if ((!extDict) || matchIndex >= dictLimit) {
@@ -1673,7 +1666,7 @@ size_t ZSTD_HcFindBestMatch_generic (
        }

        /* save best solution */
-        if (currentMl > ml) { ml = currentMl; *offsetPtr = ZSTD_REP_MOVE + current - matchIndex; if (ip+currentMl == iLimit) break; /* best possible, and avoid read overflow*/ }
+        if (currentMl > ml) { ml = currentMl; *offsetPtr = current - matchIndex + ZSTD_REP_MOVE; if (ip+currentMl == iLimit) break; /* best possible, and avoid read overflow*/ }

        if (matchIndex <= minChain) break;
        matchIndex = NEXT_IN_CHAIN(matchIndex, chainMask);
@@ -1743,7 +1736,6 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
    /* init */
    ip += (ip==base);
    ctx->nextToUpdate3 = ctx->nextToUpdate;
-    ZSTD_resetSeqStore(seqStorePtr);
    {   U32 i;
        U32 const maxRep = (U32)(ip-base);
        for (i=0; i<ZSTD_REP_INIT; i++) {
@@ -1847,7 +1839,7 @@ _storeSequence:
    /* Save reps for next block */
    {   int i;
        for (i=0; i<ZSTD_REP_NUM; i++) {
-            if (!rep[i]) rep[i] = (U32)(iend - ctx->base);   /* in case some zero are left */
+            if (!rep[i]) rep[i] = (U32)(iend - ctx->base) + 1;   /* in case some zero are left */
            ctx->savedRep[i] = rep[i];
    }   }

@@ -1913,7 +1905,6 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
    { U32 i; for (i=0; i<ZSTD_REP_INIT; i++) rep[i]=ctx->rep[i]; }

    ctx->nextToUpdate3 = ctx->nextToUpdate;
-    ZSTD_resetSeqStore(seqStorePtr);
    ip += (ip == prefixStart);

    /* Match Loop */
@@ -2097,11 +2088,7 @@ typedef void (*ZSTD_blockCompressor) (ZSTD_CCtx* ctx, const void* src, size_t sr
 static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict)
 {
    static const ZSTD_blockCompressor blockCompressor[2][6] = {
-#if 1
        { ZSTD_compressBlock_fast, ZSTD_compressBlock_greedy, ZSTD_compressBlock_lazy, ZSTD_compressBlock_lazy2, ZSTD_compressBlock_btlazy2, ZSTD_compressBlock_btopt },
-#else
-        { ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_greedy_extDict, ZSTD_compressBlock_lazy_extDict,ZSTD_compressBlock_lazy2_extDict, ZSTD_compressBlock_btlazy2_extDict, ZSTD_compressBlock_btopt_extDict },
-#endif
        { ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_greedy_extDict, ZSTD_compressBlock_lazy_extDict,ZSTD_compressBlock_lazy2_extDict, ZSTD_compressBlock_btlazy2_extDict, ZSTD_compressBlock_btopt_extDict }
    };

@@ -2111,8 +2098,9 @@ static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int

 static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
 {
-    ZSTD_blockCompressor blockCompressor = ZSTD_selectBlockCompressor(zc->params.cParams.strategy, zc->lowLimit < zc->dictLimit);
+    ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->params.cParams.strategy, zc->lowLimit < zc->dictLimit);
    if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) return 0;   /* don't even attempt compression below a certain srcSize */
+    ZSTD_resetSeqStore(&(zc->seqStore));
    blockCompressor(zc, src, srcSize);
    return ZSTD_compressSequences(zc, dst, dstCapacity, srcSize);
 }
@@ -2245,7 +2233,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* zc,

    /* preemptive overflow correction */
    if (zc->lowLimit > (1<<30)) {
-        U32 const btplus = (zc->params.cParams.strategy == ZSTD_btlazy2) || (zc->params.cParams.strategy == ZSTD_btopt);
+        U32 const btplus = (zc->params.cParams.strategy == ZSTD_btlazy2) | (zc->params.cParams.strategy == ZSTD_btopt);
        U32 const chainMask = (1 << (zc->params.cParams.chainLog - btplus)) - 1;
        U32 const newLowLimit = zc->lowLimit & chainMask;   /* preserve position % chainSize */
        U32 const correction = zc->lowLimit - newLowLimit;
@@ -465,7 +465,6 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,

    /* init */
    ctx->nextToUpdate3 = ctx->nextToUpdate;
-    ZSTD_resetSeqStore(seqStorePtr);
    ZSTD_rescaleFreqs(seqStorePtr);
    ip += (ip==prefixStart);
    { U32 i; for (i=0; i<ZSTD_REP_INIT; i++) rep[i]=ctx->rep[i]; }
@@ -757,7 +756,6 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
    { U32 i; for (i=0; i<ZSTD_REP_INIT; i++) rep[i]=ctx->rep[i]; }

    ctx->nextToUpdate3 = ctx->nextToUpdate;
-    ZSTD_resetSeqStore(seqStorePtr);
    ZSTD_rescaleFreqs(seqStorePtr);
    ip += (ip==prefixStart);

@@ -749,7 +749,7 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState)
    if (MEM_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&(seqState->DStream));

    seq.litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBits(&(seqState->DStream), llBits) : 0);   /* <=  16 bits */
-    if (MEM_32bits() ||
+    if (MEM_32bits() |
       (totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) ) BIT_reloadDStream(&(seqState->DStream));

    /* ANS state update */
@@ -765,23 +765,22 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState)
 FORCE_INLINE
 size_t ZSTD_execSequence(BYTE* op,
                                BYTE* const oend, seq_t sequence,
-                                const BYTE** litPtr, const BYTE* const litLimit_8,
+                                const BYTE** litPtr, const BYTE* const litLimit_w,
                                const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
 {
    BYTE* const oLitEnd = op + sequence.litLength;
    size_t const sequenceLength = sequence.litLength + sequence.matchLength;
    BYTE* const oMatchEnd = op + sequenceLength;   /* risk : address space overflow (32-bits) */
-    BYTE* const oend_8 = oend-8;
+    BYTE* const oend_w = oend-WILDCOPY_OVERLENGTH;
    const BYTE* const iLitEnd = *litPtr + sequence.litLength;
    const BYTE* match = oLitEnd - sequence.offset;

    /* check */
-    if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall);   /* last match must start at a minimum distance of 8 from oend */
-    if (oMatchEnd > oend) return ERROR(dstSize_tooSmall);   /* overwrite beyond dst buffer */
-    if (iLitEnd > litLimit_8) return ERROR(corruption_detected);   /* over-read beyond lit buffer */
+    if ((oLitEnd>oend_w) | (oMatchEnd>oend)) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
+    if (iLitEnd > litLimit_w) return ERROR(corruption_detected);   /* over-read beyond lit buffer */

    /* copy Literals */
-    ZSTD_wildcopy(op, *litPtr, sequence.litLength);   /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
+    ZSTD_wildcopy(op, *litPtr, sequence.litLength);   /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */
    op = oLitEnd;
    *litPtr = iLitEnd;   /* update for next sequence */

@@ -821,10 +820,10 @@ size_t ZSTD_execSequence(BYTE* op,
    op += 8; match += 8;

    if (oMatchEnd > oend-(16-MINMATCH)) {
-        if (op < oend_8) {
-            ZSTD_wildcopy(op, match, oend_8 - op);
-            match += oend_8 - op;
-            op = oend_8;
+        if (op < oend_w) {
+            ZSTD_wildcopy(op, match, oend_w - op);
+            match += oend_w - op;
+            op = oend_w;
        }
        while (op < oMatchEnd) *op++ = *match++;
    } else {
@@ -845,7 +844,7 @@ static size_t ZSTD_decompressSequences(
    BYTE* const oend = ostart + maxDstSize;
    BYTE* op = ostart;
    const BYTE* litPtr = dctx->litPtr;
-    const BYTE* const litLimit_8 = litPtr + dctx->litBufSize - 8;
+    const BYTE* const litLimit_w = litPtr + dctx->litBufSize - WILDCOPY_OVERLENGTH;
    const BYTE* const litEnd = litPtr + dctx->litSize;
    FSE_DTable* DTableLL = dctx->LLTable;
    FSE_DTable* DTableML = dctx->MLTable;
@@ -875,7 +874,7 @@ static size_t ZSTD_decompressSequences(
        for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq ; ) {
            nbSeq--;
            {   seq_t const sequence = ZSTD_decodeSequence(&seqState);
-                size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litLimit_8, base, vBase, dictEnd);
+                size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litLimit_w, base, vBase, dictEnd);
                if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
                op += oneSeqSize;
        }   }
@@ -888,8 +887,8 @@ static size_t ZSTD_decompressSequences(

    /* last literal segment */
    {   size_t const lastLLSize = litEnd - litPtr;
-        if (litPtr > litEnd) return ERROR(corruption_detected);   /* too many literals already used */
-        if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
+        //if (litPtr > litEnd) return ERROR(corruption_detected);   /* too many literals already used */
+        if (lastLLSize > (size_t)(oend-op)) return ERROR(dstSize_tooSmall);
        memcpy(op, litPtr, lastLLSize);
        op += lastLLSize;
    }
@@ -1180,12 +1179,13 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
 }


-static void ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
+static size_t ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
 {
    dctx->dictEnd = dctx->previousDstEnd;
    dctx->vBase = (const char*)dict - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->base));
    dctx->base = dict;
    dctx->previousDstEnd = (const char*)dict + dictSize;
+    return 0;
 }

 static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t const dictSize)
@@ -1237,29 +1237,24 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t c

 static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
 {
-    if (dictSize < 8) return ERROR(dictionary_corrupted);
+    if (dictSize < 8) return ZSTD_refDictContent(dctx, dict, dictSize);
    {   U32 const magic = MEM_readLE32(dict);
        if (magic != ZSTD_DICT_MAGIC) {
-            /* pure content mode */
-            ZSTD_refDictContent(dctx, dict, dictSize);
-            return 0;
-        }
-        dctx->dictID = MEM_readLE32((const char*)dict + 4);
+            return ZSTD_refDictContent(dctx, dict, dictSize);   /* pure content mode */
+    }   }
+    dctx->dictID = MEM_readLE32((const char*)dict + 4);

-        /* load entropy tables */
-        dict = (const char*)dict + 8;
-        dictSize -= 8;
-        {   size_t const eSize = ZSTD_loadEntropy(dctx, dict, dictSize);
-            if (ZSTD_isError(eSize)) return ERROR(dictionary_corrupted);
-            dict = (const char*)dict + eSize;
-            dictSize -= eSize;
-        }
-
-        /* reference dictionary content */
-        ZSTD_refDictContent(dctx, dict, dictSize);
-
-        return 0;
+    /* load entropy tables */
+    dict = (const char*)dict + 8;
+    dictSize -= 8;
+    {   size_t const eSize = ZSTD_loadEntropy(dctx, dict, dictSize);
+        if (ZSTD_isError(eSize)) return ERROR(dictionary_corrupted);
+        dict = (const char*)dict + eSize;
+        dictSize -= eSize;
    }
+
+    /* reference dictionary content */
+    return ZSTD_refDictContent(dctx, dict, dictSize);
 }


@@ -1319,7 +1314,7 @@ ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_cu
 }

 /*! ZSTD_createDDict() :
-*   Create a digested dictionary, ready to start decompression operation without startup delay.
+*   Create a digested dictionary, ready to start decompression without startup delay.
 *   `dict` can be released after `ZSTD_DDict` creation */
 ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize)
 {
@@ -584,48 +584,51 @@ static void ZDICT_countEStats(EStats_ress_t esr,
                            U32* countLit, U32* offsetcodeCount, U32* matchlengthCount, U32* litlengthCount, U32* repOffsets,
                            const void* src, size_t srcSize)
 {
-    const seqStore_t* seqStorePtr;
+    size_t cSize;

    if (srcSize > ZSTD_BLOCKSIZE_MAX) srcSize = ZSTD_BLOCKSIZE_MAX;   /* protection vs large samples */
    ZSTD_copyCCtx(esr.zc, esr.ref);
-    ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_MAX, src, srcSize);
-    seqStorePtr = ZSTD_getSeqStore(esr.zc);
+    cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_MAX, src, srcSize);
+    if (ZSTD_isError(cSize)) { DISPLAYLEVEL(1, "warning : could not compress sample size %u \n", (U32)srcSize); return; }

-    /* literals stats */
-    {   const BYTE* bytePtr;
-        for(bytePtr = seqStorePtr->litStart; bytePtr < seqStorePtr->lit; bytePtr++)
-            countLit[*bytePtr]++;
-    }
+    if (cSize) {  /* if == 0; block is not compressible */
+        const seqStore_t* seqStorePtr = ZSTD_getSeqStore(esr.zc);

-    /* seqStats */
-    {   size_t const nbSeq = (size_t)(seqStorePtr->offset - seqStorePtr->offsetStart);
-        ZSTD_seqToCodes(seqStorePtr, nbSeq);
-
-        {   const BYTE* codePtr = seqStorePtr->offCodeStart;
-            size_t u;
-            for (u=0; u<nbSeq; u++) offsetcodeCount[codePtr[u]]++;
+        /* literals stats */
+        {   const BYTE* bytePtr;
+            for(bytePtr = seqStorePtr->litStart; bytePtr < seqStorePtr->lit; bytePtr++)
+                countLit[*bytePtr]++;
        }

-        {   const BYTE* codePtr = seqStorePtr->mlCodeStart;
-            size_t u;
-            for (u=0; u<nbSeq; u++) matchlengthCount[codePtr[u]]++;
+        /* seqStats */
+        {   size_t const nbSeq = (size_t)(seqStorePtr->offset - seqStorePtr->offsetStart);
+            ZSTD_seqToCodes(seqStorePtr, nbSeq);
+
+            {   const BYTE* codePtr = seqStorePtr->offCodeStart;
+                size_t u;
+                for (u=0; u<nbSeq; u++) offsetcodeCount[codePtr[u]]++;
+            }
+
+            {   const BYTE* codePtr = seqStorePtr->mlCodeStart;
+                size_t u;
+                for (u=0; u<nbSeq; u++) matchlengthCount[codePtr[u]]++;
+            }
+
+            {   const BYTE* codePtr = seqStorePtr->llCodeStart;
+                size_t u;
+                for (u=0; u<nbSeq; u++) litlengthCount[codePtr[u]]++;
+        }   }
+
+        /* rep offsets */
+        {   const U32* const offsetPtr = seqStorePtr->offsetStart;
+            U32 offset1 = offsetPtr[0] - 3;
+            U32 offset2 = offsetPtr[1] - 3;
+            if (offset1 >= MAXREPOFFSET) offset1 = 0;
+            if (offset2 >= MAXREPOFFSET) offset2 = 0;
+            repOffsets[offset1] += 3;
+            repOffsets[offset2] += 1;
        }
-
-        {   const BYTE* codePtr = seqStorePtr->llCodeStart;
-            size_t u;
-            for (u=0; u<nbSeq; u++) litlengthCount[codePtr[u]]++;
-    }   }
-
-    /* rep offsets */
-    {   const U32* const offsetPtr = seqStorePtr->offsetStart;
-        U32 offset1 = offsetPtr[0] - 3;
-        U32 offset2 = offsetPtr[1] - 3;
-        if (offset1 >= MAXREPOFFSET) offset1 = 0;
-        if (offset2 >= MAXREPOFFSET) offset2 = 0;
-        repOffsets[offset1] += 3;
-        repOffsets[offset2] += 1;
    }
-
 }

 /*
@@ -50,3 +50,4 @@ afl
 # Misc files
 *.bat
 fileTests.sh
+dirTest*
@@ -42,8 +42,10 @@
 /* *************************************
 *  Constants
 ***************************************/
-#ifndef ZSTD_VERSION
-#  define ZSTD_VERSION ""
+#ifndef ZSTD_GIT_COMMIT
+#  define ZSTD_GIT_COMMIT_STRING ""
+#else
+#  define ZSTD_GIT_COMMIT_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_GIT_COMMIT)
 #endif

 #define NBLOOPS               3
@@ -363,7 +365,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
    memset(&total, 0, sizeof(total));

    if (g_displayLevel == 1 && !g_additionalParam)
-        DISPLAY("bench %s: input %u bytes, %i iterations, %u KB blocks\n", ZSTD_VERSION, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10));
+        DISPLAY("bench %s %s: input %u bytes, %i iterations, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10));

    if (cLevelLast < cLevel) cLevelLast = cLevel;

@@ -137,7 +137,7 @@ static U32 g_sparseFileSupport = 1;   /* 0 : no sparse allowed; 1: auto (file ye
 void FIO_setSparseWrite(unsigned sparse) { g_sparseFileSupport=sparse; }
 static U32 g_dictIDFlag = 1;
 void FIO_setDictIDFlag(unsigned dictIDFlag) { g_dictIDFlag = dictIDFlag; }
-static U32 g_checksumFlag = 0;
+static U32 g_checksumFlag = 1;
 void FIO_setChecksumFlag(unsigned checksumFlag) { g_checksumFlag = checksumFlag; }
 static U32 g_removeSrcFile = 0;
 void FIO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); }
@@ -413,7 +413,7 @@ static int FIO_compressFilename_dstFile(cRess_t ress,
    int result;

    ress.dstFile = FIO_openDstFile(dstFileName);
-    if (ress.dstFile==0) { fclose(ress.srcFile); return 1; }
+    if (ress.dstFile==0) return 1;

    result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName, cLevel);

@@ -22,10 +22,12 @@ roundTripTest() {

 isWindows=false
 ECHO="echo"
+INTOVOID="/dev/null"
 case "$OS" in
  Windows*)
    isWindows=true
    ECHO="echo -e"
+    INTOVOID="nul"
    ;;
 esac

@@ -53,19 +55,21 @@ $ECHO "test : null-length file roundtrip"
 $ECHO -n '' | $ZSTD - --stdout | $ZSTD -d --stdout
 $ECHO "test : decompress file with wrong suffix (must fail)"
 $ZSTD -d tmpCompressed && die "wrong suffix error not detected!"
+$ZSTD -df tmp && die "should have refused : wrong extension"
+$ECHO "test : decompress into stdout"
 $ZSTD -d tmpCompressed -c > tmpResult    # decompression using stdout
 $ZSTD --decompress tmpCompressed -c > tmpResult
 $ZSTD --decompress tmpCompressed --stdout > tmpResult
-if [ "$isWindows" = false ] ; then
-    $ZSTD -d    < tmp.zst > /dev/null    # combine decompression, stdin & stdout
-    $ZSTD -d  - < tmp.zst > /dev/null
-fi
-$ZSTD -dc   < tmp.zst > /dev/null
-$ZSTD -dc - < tmp.zst > /dev/null
+$ECHO "test : decompress from stdin into stdout"
+$ZSTD -dc   < tmp.zst > $INTOVOID   # combine decompression, stdin & stdout
+$ZSTD -dc - < tmp.zst > $INTOVOID
+$ZSTD -d    < tmp.zst > $INTOVOID   # implicit stdout when stdin is used
+$ZSTD -d  - < tmp.zst > $INTOVOID
+$ECHO "test : overwrite protection"
 $ZSTD -q tmp && die "overwrite check failed!"
+$ECHO "test : force overwrite"
 $ZSTD -q -f tmp
 $ZSTD -q --force tmp
-$ZSTD -df tmp && die "should have refused : wrong extension"
 $ECHO "test : file removal"
 $ZSTD -f --rm tmp
 ls tmp && die "tmp should no longer be present"
@@ -135,9 +139,9 @@ rm tmpSparse*

 $ECHO "\n**** multiple files tests **** "

-./datagen -s1        > tmp1 2> /dev/null
-./datagen -s2 -g100K > tmp2 2> /dev/null
-./datagen -s3 -g1M   > tmp3 2> /dev/null
+./datagen -s1        > tmp1 2> $INTOVOID
+./datagen -s2 -g100K > tmp2 2> $INTOVOID
+./datagen -s3 -g1M   > tmp3 2> $INTOVOID
 $ZSTD -f tmp*
 $ECHO "compress tmp* : "
 ls -ls tmp*
@@ -36,8 +36,14 @@
 #include <stdio.h>      /* fprintf */
 #include <sys/types.h>  /* stat */
 #include <sys/stat.h>   /* stat */
+#include "xxhash.h"
 #include "zstd.h"

+/*===========================================
+*   Macros
+*==========================================*/
+#define MIN(a,b)  ( (a) < (b) ? (a) : (b) )
+
 /** roundTripTest() :
 *   Compresses `srcBuff` into `compressedBuff`,
 *   then decompresses `compressedBuff` into `resultBuff`.
@@ -51,7 +57,9 @@ static size_t roundTripTest(void* resultBuff, size_t resultBuffCapacity,
                      const void* srcBuff, size_t srcBuffSize)
 {
    static const int maxClevel = 19;
-    int const cLevel = (!srcBuffSize) ? 1 : (*(const unsigned char*)srcBuff) % maxClevel;
+    size_t const hashLength = MIN(128, srcBuffSize);
+    unsigned const h32 = XXH32(srcBuff, hashLength, 0);
+    int const cLevel = h32 % maxClevel;
    size_t const cSize = ZSTD_compress(compressedBuff, compressedBuffCapacity, srcBuff, srcBuffSize, cLevel);
    if (ZSTD_isError(cSize)) {
        fprintf(stderr, "Compression error : %s \n", ZSTD_getErrorName(cSize));
@@ -421,11 +421,17 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
        }   }

        /* final frame epilogue */
-        {   size_t dstBuffSize = cBufferSize - cSize;
-            size_t const flushError = ZBUFF_compressEnd(zc, cBuffer+cSize, &dstBuffSize);
-            CHECK (ZBUFF_isError(flushError), "flush error : %s", ZBUFF_getErrorName(flushError));
-            cSize += dstBuffSize;
-        }
+        {   size_t remainingToFlush = (size_t)(-1);
+            while (remainingToFlush) {
+                size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
+                size_t dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
+                U32 const enoughDstSize = dstBuffSize >= remainingToFlush;
+                remainingToFlush = ZBUFF_compressEnd(zc, cBuffer+cSize, &dstBuffSize);
+                CHECK (ZBUFF_isError(remainingToFlush), "flush error : %s", ZBUFF_getErrorName(remainingToFlush));
+                //DISPLAY("flush %u bytes : still within context : %i \n", (U32)dstBuffSize, (int)remainingToFlush);
+                CHECK (enoughDstSize && remainingToFlush, "ZBUFF_compressEnd() not fully flushed, but enough space available");
+                cSize += dstBuffSize;
+        }   }
        crcOrig = XXH64_digest(&xxhState);

        /* multi - fragments decompression test */
@@ -136,7 +136,7 @@ static int usage_advanced(const char* programName)
 #ifndef ZSTD_NOCOMPRESS
    DISPLAY( "--ultra : enable ultra modes (requires more memory to decompress)\n");
    DISPLAY( "--no-dictID : don't write dictID into header (dictionary compression)\n");
-    DISPLAY( "--check : enable integrity check\n");
+    DISPLAY( "--[no-]check : integrity check (default:enabled)\n");
 #endif
 #ifndef ZSTD_NODECOMPRESS
    DISPLAY( "--test  : test compressed file integrity \n");
@@ -257,6 +257,7 @@ int main(int argCount, const char** argv)
        if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; displayLevel=1; continue; }
        if (!strcmp(argument, "--ultra")) { FIO_setMaxWLog(0); continue; }
        if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(2); continue; }
+        if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(0); continue; }
        if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; }
        if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; }
        if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; }
@@ -1,37 +1,38 @@
 #! /usr/bin/env python
-# execute(), fetch(), notify() are based on https://github.com/getlantern/build-automation/blob/master/build.py

 import argparse
 import os
 import string
 import time
 import traceback
-from subprocess import Popen, PIPE
+import subprocess
+import signal
+ 

-repo_url = 'https://github.com/Cyan4973/zstd.git'
-test_dir_name = 'speedTest'
+default_repo_url = 'https://github.com/Cyan4973/zstd.git'
+working_dir_name = 'speedTest'
+working_path = os.getcwd() + '/' + working_dir_name     # /path/to/zstd/tests/speedTest 
+clone_path = working_path + '/' + 'zstd'                # /path/to/zstd/tests/speedTest/zstd 
+email_header = '[ZSTD_speedTest]'
+pid = str(os.getpid())


 def log(text):
-    print time.strftime("%Y/%m/%d %H:%M:%S") + ' - ' + text
+    print(time.strftime("%Y/%m/%d %H:%M:%S") + ' - ' + text)

-def execute(command, print_output=False, print_error=True):
+
+def execute(command, print_output=False, print_error=True, param_shell=True):
    log("> " + command)
-    popen = Popen(command, stdout=PIPE, stderr=PIPE, shell=True, cwd=execute.cwd)
-    itout = iter(popen.stdout.readline, b"")
-    iterr = iter(popen.stderr.readline, b"")
-    stdout_lines = list(itout)
+    popen = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=param_shell, cwd=execute.cwd)
+    stdout = popen.communicate()[0]
+    stdout_lines = stdout.splitlines()
    if print_output:
-        print ''.join(stdout_lines)
-    stderr_lines = list(iterr)
-    if print_output:
-        print ''.join(stderr_lines)
-    popen.communicate()
+        print('\n'.join(stdout_lines))
    if popen.returncode is not None and popen.returncode != 0:
        if not print_output and print_error:
-            print ''.join(stderr_lines)    
-        raise RuntimeError(''.join(stderr_lines))
-    return stdout_lines + stderr_lines
+            print('\n'.join(stdout_lines))
+        raise RuntimeError('\n'.join(stdout_lines))
+    return stdout_lines
 execute.cwd = None


@@ -43,39 +44,51 @@ def does_command_exist(command):
    return True


-def fetch():
+def send_email(emails, topic, text, have_mutt, have_mail):
+    logFileName = working_path + '/' + 'tmpEmailContent'
+    with open(logFileName, "w") as myfile:
+        myfile.writelines(text)
+        myfile.close()
+        if have_mutt:
+            execute('mutt -s "' + topic + '" ' + emails + ' < ' + logFileName)
+        elif have_mail:
+            execute('mail -s "' + topic + '" ' + emails + ' < ' + logFileName)
+        else:
+            log("e-mail cannot be sent (mail or mutt not found)")
+
+
+def send_email_with_attachments(branch, commit, last_commit, emails, text, results_files, logFileName, lower_limit, have_mutt, have_mail):
+    with open(logFileName, "w") as myfile:
+        myfile.writelines(text)
+        myfile.close()
+        email_topic = '%s:%s Warning for %s:%s last_commit=%s speed<%s' % (email_header, pid, branch, commit, last_commit, lower_limit)
+        if have_mutt:
+            execute('mutt -s "' + email_topic + '" ' + emails + ' -a ' + results_files + ' < ' + logFileName)
+        elif have_mail:
+            execute('mail -s "' + email_topic + '" ' + emails + ' < ' + logFileName)
+        else:
+            log("e-mail cannot be sent (mail or mutt not found)")
+
+
+def git_get_branches():
    execute('git fetch -p')
    output = execute('git branch -rl')
    for line in output:
        if "HEAD" in line: 
-            output.remove(line) # remove "origin/HEAD -> origin/dev"
-    branches = map(lambda l: l.strip(), output)
-    return map(lambda b: (b, execute('git show -s --format=%h ' + b)[0].strip()), branches)
+            output.remove(line)  # remove "origin/HEAD -> origin/dev"
+    return map(lambda l: l.strip(), output)


-def notify(branch, commit, last_commit):
-    text_tmpl = string.Template('Changes since $last_commit:\r\n$commits')
-    branch = branch.split('/')[1]
+def git_get_changes(branch, commit, last_commit):
    fmt = '--format="%h: (%an) %s, %ar"'
    if last_commit is None:
        commits = execute('git log -n 10 %s %s' % (fmt, commit))
    else:
        commits = execute('git --no-pager log %s %s..%s' % (fmt, last_commit, commit))
-
-    text = text_tmpl.substitute({'last_commit': last_commit, 'commits': ''.join(commits)})
-    print str("commits for %s: %s" % (commit, text))
+    return str('Changes in %s since %s:\n' % (branch, last_commit)) + '\n'.join(commits)


-def compile(branch, commit, dry_run):
-    local_branch = string.split(branch, '/')[1]
-    version = local_branch.rpartition('-')[2]
-    version = version + '_' + commit
-    execute('git checkout -- . && git checkout ' + branch)
-    if not dry_run:
-        execute('VERSION=' + version + '; make clean zstdprogram')
-
-
-def get_last_commit(resultsFileName):
+def get_last_results(resultsFileName):
    if not os.path.isfile(resultsFileName):
        return None, None, None
    commit = None
@@ -84,15 +97,13 @@ def get_last_commit(resultsFileName):
    with open(resultsFileName,'r') as f:
        for line in f:
            words = line.split()
-            if len(words) == 2: # branch + commit
+            if len(words) == 2:   # branch + commit
                commit = words[1];
                cspeed = []
                dspeed = []
-            if (len(words) == 8): 
+            if (len(words) == 8):  # results
                cspeed.append(float(words[3]))
                dspeed.append(float(words[5]))
-        #if commit != None:
-        #    print "commit=%s cspeed=%s dspeed=%s" % (commit, cspeed, dspeed)
    return commit, cspeed, dspeed


@@ -102,80 +113,62 @@ def benchmark_and_compare(branch, commit, resultsFileName, lastCLevel, testFileP
        log("WARNING: bench loadavg=%.2f is higher than %s, sleeping for %s seconds" % (os.getloadavg()[0], maxLoadAvg, sleepTime))
        time.sleep(sleepTime)
    start_load = str(os.getloadavg())
-    result = execute('programs/zstd -qi5b1e' + str(lastCLevel) + ' ' + testFilePath)
+    result = execute('programs/zstd -qi5b1e%s %s' % (lastCLevel, testFilePath), print_output=True)
    end_load = str(os.getloadavg())
    linesExpected = lastCLevel + 2;
    if len(result) != linesExpected:
-        log("ERROR: number of result lines=%d is different that expected %d" % (len(result), linesExpected))
-        return ""
+        raise RuntimeError("ERROR: number of result lines=%d is different that expected %d\n%s" % (len(result), linesExpected, '\n'.join(result)))
    with open(resultsFileName, "a") as myfile:
        myfile.write(branch + " " + commit + "\n")
-        myfile.writelines(result)
+        myfile.write('\n'.join(result) + '\n')
        myfile.close()
        if (last_cspeed == None):
+            log("WARNING: No data for comparison for branch=%s file=%s " % (branch, fileName))
            return ""
-        commit, cspeed, dspeed = get_last_commit(resultsFileName)
+        commit, cspeed, dspeed = get_last_results(resultsFileName)
        text = ""
        for i in range(0, min(len(cspeed), len(last_cspeed))):
+            print("%s:%s -%d cspeed=%6.2f clast=%6.2f cdiff=%1.4f dspeed=%6.2f dlast=%6.2f ddiff=%1.4f %s" % (branch, commit, i+1, cspeed[i], last_cspeed[i], cspeed[i]/last_cspeed[i], dspeed[i], last_dspeed[i], dspeed[i]/last_dspeed[i], fileName))
            if (cspeed[i]/last_cspeed[i] < lower_limit):
-                text += "WARNING: File=%s level=%d cspeed=%s last=%s diff=%s\n" % (fileName, i+1, cspeed[i], last_cspeed[i], cspeed[i]/last_cspeed[i])
+                text += "WARNING: -%d cspeed=%.2f clast=%.2f cdiff=%.4f %s\n" % (i+1, cspeed[i], last_cspeed[i], cspeed[i]/last_cspeed[i], fileName)
            if (dspeed[i]/last_dspeed[i] < lower_limit):
-                text += "WARNING: File=%s level=%d dspeed=%s last=%s diff=%s\n" % (fileName, i+1, dspeed[i], last_dspeed[i], dspeed[i]/last_dspeed[i])
+                text += "WARNING: -%d dspeed=%.2f dlast=%.2f ddiff=%.4f %s\n" % (i+1, dspeed[i], last_dspeed[i], dspeed[i]/last_dspeed[i], fileName)
        if text:
            text = message + ("\nmaxLoadAvg=%s  load average at start=%s end=%s\n" % (maxLoadAvg, start_load, end_load)) + text
        return text


-def send_email(branch, commit, last_commit, emails, text, results_files, logFileName, lower_limit, have_mutt, have_mail):
-    with open(logFileName, "w") as myfile:
-        myfile.writelines(text)
-        myfile.close()
-        if have_mutt:
-            execute("mutt -s \"[ZSTD_speedTest] Warning for branch=" + branch + " commit=" + commit + " last_commit=" + last_commit + " speed<" + str(lower_limit) + "\" " + emails + " -a " + results_files + " < " + logFileName)
-        elif have_mail:
-            execute("mail -s \"[ZSTD_speedTest] Warning for branch=" + branch + " commit=" + commit + " last_commit=" + last_commit + " speed<" + str(lower_limit) + "\" " + emails + " < " + logFileName)
-        else:
-            log("e-mail cannot be sent (mail and mutt not found)")
+def update_config_file(branch, commit):
+    last_commit = None
+    commitFileName = working_path + "/commit_" + branch.replace("/", "_") + ".txt"
+    if os.path.isfile(commitFileName):
+        last_commit = file(commitFileName, 'r').read()
+    file(commitFileName, 'w').write(commit)
+    return last_commit


-def check_branches(args, test_path, testFilePaths, have_mutt, have_mail):
-    for branch, commit in fetch():
-        try:
-            commitFileName = test_path + "/commit_" + branch.replace("/", "_")
-            if os.path.isfile(commitFileName):
-                last_commit = file(commitFileName, 'r').read()
-            else:
-                last_commit = None
-            file(commitFileName, 'w').write(commit)
-
-            if commit == last_commit:
-                log("skipping branch %s: head %s already processed" % (branch, commit))
-            else:
-                log("build branch %s: head %s is different from prev %s" % (branch, commit, last_commit))
-                compile(branch, commit, args.dry_run)
-
-                logFileName = test_path + "/log_" + branch.replace("/", "_")
-                text_to_send = []
-                results_files = ""
-                for filePath in testFilePaths:
-                    fileName = filePath.rpartition('/')[2]
-                    resultsFileName = test_path + "/results_" + branch.replace("/", "_") + "_" + fileName
-                    last_commit, cspeed, dspeed = get_last_commit(resultsFileName)
-
-                    if not args.dry_run:
-                        text = benchmark_and_compare(branch, commit, resultsFileName, args.lastCLevel, filePath, fileName, cspeed, dspeed, args.lowerLimit, args.maxLoadAvg, args.message)
-                        if text:
-                            text = benchmark_and_compare(branch, commit, resultsFileName, args.lastCLevel, filePath, fileName, cspeed, dspeed, args.lowerLimit, args.maxLoadAvg, args.message)
-                            if text:
-                                text_to_send.append(text)
-                                results_files += resultsFileName + " "
-                if text_to_send:
-                    send_email(branch, commit, last_commit, args.emails, text_to_send, results_files, logFileName, args.lowerLimit, have_mutt, have_mail)
-                notify(branch, commit, last_commit)
-        except Exception as e:
-            stack = traceback.format_exc()
-            log("ERROR: build %s, error %s" % (branch, str(e)) )
-            print stack
+def test_commit(branch, commit, last_commit, args, testFilePaths, have_mutt, have_mail):
+    local_branch = string.split(branch, '/')[1]
+    version = local_branch.rpartition('-')[2] + '_' + commit
+    if not args.dry_run:
+        execute('make clean zstdprogram MOREFLAGS="-DZSTD_GIT_COMMIT=%s"' % version)
+    logFileName = working_path + "/log_" + branch.replace("/", "_") + ".txt"
+    text_to_send = []
+    results_files = ""
+    for filePath in testFilePaths:
+        fileName = filePath.rpartition('/')[2]
+        resultsFileName = working_path + "/results_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt"
+        last_commit, cspeed, dspeed = get_last_results(resultsFileName)
+        if not args.dry_run:
+            text = benchmark_and_compare(branch, commit, resultsFileName, args.lastCLevel, filePath, fileName, cspeed, dspeed, args.lowerLimit, args.maxLoadAvg, args.message)
+            if text:
+                log("WARNING: redoing tests for branch %s: commit %s" % (branch, commit))
+                text = benchmark_and_compare(branch, commit, resultsFileName, args.lastCLevel, filePath, fileName, cspeed, dspeed, args.lowerLimit, args.maxLoadAvg, args.message)
+                if text:
+                    text_to_send.append(text)
+                    results_files += resultsFileName + " "
+    if text_to_send:
+        send_email_with_attachments(branch, commit, last_commit, args.emails, text_to_send, results_files, logFileName, args.lowerLimit, have_mutt, have_mail)


 if __name__ == '__main__':
@@ -183,6 +176,7 @@ if __name__ == '__main__':
    parser.add_argument('testFileNames', help='file names list for speed benchmark')
    parser.add_argument('emails', help='list of e-mail addresses to send warnings')
    parser.add_argument('--message', help='attach an additional message to e-mail', default="")
+    parser.add_argument('--repoURL', help='changes default repository URL', default=default_repo_url)
    parser.add_argument('--lowerLimit', type=float, help='send email if speed is lower than given limit', default=0.98)
    parser.add_argument('--maxLoadAvg', type=float, help='maximum load average to start testing', default=0.75)
    parser.add_argument('--lastCLevel', type=int, help='last compression level for testing', default=5)
@@ -194,56 +188,78 @@ if __name__ == '__main__':
    testFileNames = args.testFileNames.split()
    testFilePaths = []
    for fileName in testFileNames:
+        fileName = os.path.expanduser(fileName)
        if os.path.isfile(fileName):
            testFilePaths.append(os.path.abspath(fileName))
        else:
-            raise RuntimeError("File not found: " + fileName)
-
-    test_path = os.getcwd() + '/' + test_dir_name     # /path/to/zstd/tests/speedTest 
-    clone_path = test_path + '/' + 'zstd'             # /path/to/zstd/tests/speedTest/zstd 
+            log("ERROR: File not found: " + fileName)
+            exit(1)

    # check availability of e-mail senders
-    have_mutt = does_command_exist("mutt --help");
+    have_mutt = does_command_exist("mutt -h");
    have_mail = does_command_exist("mail -V");
    if not have_mutt and not have_mail:
-        log("WARNING: e-mail senders mail and mutt not found")
+        log("ERROR: e-mail senders 'mail' or 'mutt' not found")
+        exit(1)
+
+    print("PARAMETERS:\nrepoURL=%s" % args.repoURL)
+    print("working_path=%s" % working_path)
+    print("clone_path=%s" % clone_path)
+    print("testFilePath(%s)=%s" % (len(testFilePaths), testFilePaths))
+    print("message=%s" % args.message)
+    print("emails=%s" % args.emails)
+    print("maxLoadAvg=%s" % args.maxLoadAvg)
+    print("lowerLimit=%s" % args.lowerLimit)
+    print("lastCLevel=%s" % args.lastCLevel)
+    print("sleepTime=%s" % args.sleepTime)
+    print("dry_run=%s" % args.dry_run)
+    print("have_mutt=%s have_mail=%s" % (have_mutt, have_mail))

    # clone ZSTD repo if needed
-    if not os.path.isdir(test_path):
-        os.mkdir(test_path)
+    if not os.path.isdir(working_path):
+        os.mkdir(working_path)
    if not os.path.isdir(clone_path):
-        execute.cwd = test_path
-        execute('git clone ' + repo_url)
+        execute.cwd = working_path
+        execute('git clone ' + args.repoURL)
    if not os.path.isdir(clone_path):
-        raise RuntimeError("ZSTD clone not found: " + clone_path)
+        log("ERROR: ZSTD clone not found: " + clone_path)
+        exit(1)
    execute.cwd = clone_path

-    print "PARAMETERS:\ntest_path=%s" % test_path
-    print "clone_path=%s" % clone_path
-    print "testFilePath(%s)=%s" % (len(testFilePaths), testFilePaths)
-    print "message=%s" % args.message
-    print "emails=%s" % args.emails
-    print "maxLoadAvg=%s" % args.maxLoadAvg
-    print "lowerLimit=%s" % args.lowerLimit
-    print "lastCLevel=%s" % args.lastCLevel
-    print "sleepTime=%s" % args.sleepTime
-    print "dry_run=%s" % args.dry_run
-    print "have_mutt=%s have_mail=%s" % (have_mutt, have_mail)
+    # check if speedTest.pid already exists
+    pidfile = "./speedTest.pid"
+    if os.path.isfile(pidfile):
+        log("ERROR: %s already exists, exiting" % pidfile)
+        exit(1)
+
+    send_email(args.emails, email_header + ':%s test-zstd-speed.py has been started' % pid, args.message, have_mutt, have_mail)
+    file(pidfile, 'w').write(pid)

    while True:
-        pid = str(os.getpid())
-        pidfile = "./speedTest.pid"
-        if os.path.isfile(pidfile):
-            log("%s already exists, exiting" % pidfile)
-        else:
-            file(pidfile, 'w').write(pid)
-            try:
-                loadavg = os.getloadavg()[0]
-                if (loadavg <= args.maxLoadAvg):
-                    check_branches(args, test_path, testFilePaths, have_mutt, have_mail)
-                else:
-                    log("WARNING: main loadavg=%.2f is higher than %s" % (loadavg, args.maxLoadAvg))
-            finally:
-                os.unlink(pidfile)
-        log("sleep for %s seconds" % args.sleepTime)
-        time.sleep(args.sleepTime)
+        try:
+            loadavg = os.getloadavg()[0]
+            if (loadavg <= args.maxLoadAvg):
+                branches = git_get_branches()
+                for branch in branches:
+                    commit = execute('git show -s --format=%h ' + branch)[0]
+                    last_commit = update_config_file(branch, commit)
+                    if commit == last_commit:
+                        log("skipping branch %s: head %s already processed" % (branch, commit))
+                    else:
+                        log("build branch %s: head %s is different from prev %s" % (branch, commit, last_commit))
+                        execute('git checkout -- . && git checkout ' + branch)
+                        print(git_get_changes(branch, commit, last_commit))
+                        test_commit(branch, commit, last_commit, args, testFilePaths, have_mutt, have_mail)
+            else:
+                log("WARNING: main loadavg=%.2f is higher than %s" % (loadavg, args.maxLoadAvg))
+            log("sleep for %s seconds" % args.sleepTime)
+            time.sleep(args.sleepTime)
+        except Exception as e:
+            stack = traceback.format_exc()
+            email_topic = '%s:%s ERROR in %s:%s' % (email_header, pid, branch, commit)
+            send_email(args.emails, email_topic, stack, have_mutt, have_mail)
+            print(stack)
+        except KeyboardInterrupt:
+            os.unlink(pidfile)
+            send_email(args.emails, email_header + ':%s test-zstd-speed.py has been stopped' % pid, args.message, have_mutt, have_mail)
+            exit(0)
@@ -7,8 +7,9 @@ import glob
 import hashlib
 import os
 import shutil
-import subprocess
 import sys
+import subprocess
+from subprocess import Popen, PIPE

 repo_url = 'https://github.com/Cyan4973/zstd.git'
 tmp_dir_name = 'tests/versionsTest'
@@ -17,17 +18,32 @@ git_cmd = 'git'
 test_dat_src = 'README.md'
 test_dat = 'test_dat'
 head = 'vdevel'
+dict_source = 'dict_source'
+dict_files = './zstd/programs/*.c ./zstd/lib/common/*.c ./zstd/lib/compress/*.c ./zstd/lib/decompress/*.c ./zstd/lib/dictBuilder/*.c ./zstd/lib/legacy/*.c '
+dict_files += './zstd/programs/*.h ./zstd/lib/common/*.h ./zstd/lib/compress/*.h ./zstd/lib/dictBuilder/*.h ./zstd/lib/legacy/*.h'
+
+
+def execute(command, print_output=False, print_error=True, param_shell=False):
+    popen = Popen(command, stdout=PIPE, stderr=PIPE, shell=param_shell)
+    stdout_lines, stderr_lines = popen.communicate()
+    stderr_lines = stderr_lines.decode("utf-8")
+    stdout_lines = stdout_lines.decode("utf-8")
+    if print_output:
+        print(stdout_lines)
+        print(stderr_lines)
+    if popen.returncode is not None and popen.returncode != 0:
+        if not print_output and print_error:
+            print(stderr_lines)
+    return popen.returncode


 def proc(cmd_args, pipe=True, dummy=False):
    if dummy:
        return
    if pipe:
-        subproc = subprocess.Popen(cmd_args,
-                                   stdout=subprocess.PIPE,
-                                   stderr=subprocess.PIPE)
+        subproc = Popen(cmd_args, stdout=PIPE, stderr=PIPE)
    else:
-        subproc = subprocess.Popen(cmd_args)
+        subproc = Popen(cmd_args)
    return subproc.communicate()


@@ -45,21 +61,53 @@ def get_git_tags():
    return tags


+def create_dict(tag, dict_source_path):
+    dict_name = 'dict.' + tag
+    if not os.path.isfile(dict_name):
+        cFiles = glob.glob(dict_source_path + "/*.c")
+        hFiles = glob.glob(dict_source_path + "/*.h")
+        if tag == 'v0.5.0':
+            result = execute('./dictBuilder.' + tag + ' ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name, print_output=False, param_shell=True)
+        else:
+            result = execute('./zstd.' + tag + ' -f --train ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name, print_output=False, param_shell=True)
+        if result == 0:
+            print(dict_name + ' created')
+        else:
+            print('ERROR: creating of ' + dict_name + ' failed')
+    else:
+        print(dict_name + ' already exists')
+
+
+def dict_compress_sample(tag, sample):
+    dict_name = 'dict.' + tag
+    DEVNULL = open(os.devnull, 'wb')
+    if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-f',   sample], stderr=DEVNULL) == 0:
+        os.rename(sample + '.zst', sample + '_01_64_' + tag + '_dictio.zst')
+    if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-5f',  sample], stderr=DEVNULL) == 0:
+        os.rename(sample + '.zst', sample + '_05_64_' + tag + '_dictio.zst')
+    if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-9f',  sample], stderr=DEVNULL) == 0:
+        os.rename(sample + '.zst', sample + '_09_64_' + tag + '_dictio.zst')
+    if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-15f', sample], stderr=DEVNULL) == 0:
+        os.rename(sample + '.zst', sample + '_15_64_' + tag + '_dictio.zst')
+    if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-18f', sample], stderr=DEVNULL) == 0:
+        os.rename(sample + '.zst', sample + '_18_64_' + tag + '_dictio.zst')
+    # zstdFiles = glob.glob("*.zst*")
+    # print(zstdFiles)
+    print(tag + " : dict compression completed")
+
+
 def compress_sample(tag, sample):
-    try:
-        from subprocess import DEVNULL  # py3k
-    except ImportError:
-        DEVNULL = open(os.devnull, 'wb')
+    DEVNULL = open(os.devnull, 'wb')
    if subprocess.call(['./zstd.' + tag, '-f',   sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_01_64_' + tag + '.zst')
+        os.rename(sample + '.zst', sample + '_01_64_' + tag + '_nodict.zst')
    if subprocess.call(['./zstd.' + tag, '-5f',  sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_05_64_' + tag + '.zst')
+        os.rename(sample + '.zst', sample + '_05_64_' + tag + '_nodict.zst')
    if subprocess.call(['./zstd.' + tag, '-9f',  sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_09_64_' + tag + '.zst')
+        os.rename(sample + '.zst', sample + '_09_64_' + tag + '_nodict.zst')
    if subprocess.call(['./zstd.' + tag, '-15f', sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_15_64_' + tag + '.zst')
+        os.rename(sample + '.zst', sample + '_15_64_' + tag + '_nodict.zst')
    if subprocess.call(['./zstd.' + tag, '-18f', sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_18_64_' + tag + '.zst')
+        os.rename(sample + '.zst', sample + '_18_64_' + tag + '_nodict.zst')
    # zstdFiles = glob.glob("*.zst*")
    # print(zstdFiles)
    print(tag + " : compression completed")
@@ -87,16 +135,16 @@ def remove_duplicates():

 def decompress_zst(tag):
    dec_error = 0
-    list_zst = sorted(glob.glob('*.zst'))
-    try:
-        from subprocess import DEVNULL  # py3k
-    except ImportError:
-        DEVNULL = open(os.devnull, 'wb')
+    list_zst = sorted(glob.glob('*_nodict.zst'))
    for file_zst in list_zst:
        print(file_zst, end=' ')
        print(tag, end=' ')
        file_dec = file_zst + '_d64_' + tag + '.dec'
-        if subprocess.call(['./zstd.' + tag, '-df', file_zst, '-o', file_dec], stderr=DEVNULL) == 0:
+        if tag <= 'v0.5.0':
+            params = ['./zstd.' + tag, '-df', file_zst, file_dec]
+        else:
+            params = ['./zstd.' + tag, '-df', file_zst, '-o', file_dec]
+        if execute(params) == 0:
            if not filecmp.cmp(file_dec, test_dat):
                print('ERR !! ')
                dec_error = 1
@@ -107,12 +155,43 @@ def decompress_zst(tag):
    return dec_error


+def decompress_dict(tag):
+    dec_error = 0
+    list_zst = sorted(glob.glob('*_dictio.zst'))
+    for file_zst in list_zst:
+        dict_tag = file_zst[0:len(file_zst)-11]  # remove "_dictio.zst"
+        if head in dict_tag: # find vdevel
+            dict_tag = head
+        else:
+            dict_tag = dict_tag[dict_tag.rfind('v'):]
+        if tag == 'v0.6.0' and dict_tag < 'v0.6.0':
+            continue
+        dict_name = 'dict.' + dict_tag
+        print(file_zst + ' ' + tag + ' dict=' + dict_tag, end=' ')
+        file_dec = file_zst + '_d64_' + tag + '.dec'
+        if tag <= 'v0.5.0':
+            params = ['./zstd.' + tag, '-D', dict_name, '-df', file_zst, file_dec]
+        else:
+            params = ['./zstd.' + tag, '-D', dict_name, '-df', file_zst, '-o', file_dec]
+        if execute(params) == 0:
+            if not filecmp.cmp(file_dec, test_dat):
+                print('ERR !! ')
+                dec_error = 1
+            else:
+                print('OK     ')
+        else:
+            print('command does not work')
+            dec_error = 1
+    return dec_error
+
+
 if __name__ == '__main__':
    error_code = 0
-    base_dir = os.getcwd() + '/..'           # /path/to/zstd
-    tmp_dir = base_dir + '/' + tmp_dir_name  # /path/to/zstd/tests/versionsTest
-    clone_dir = tmp_dir + '/' + 'zstd'       # /path/to/zstd/tests/versionsTest/zstd
-    programs_dir = base_dir + '/programs'    # /path/to/zstd/programs
+    base_dir = os.getcwd() + '/..'                  # /path/to/zstd
+    tmp_dir = base_dir + '/' + tmp_dir_name         # /path/to/zstd/tests/versionsTest
+    clone_dir = tmp_dir + '/' + 'zstd'              # /path/to/zstd/tests/versionsTest/zstd
+    dict_source_path = tmp_dir + '/' + dict_source  # /path/to/zstd/tests/versionsTest/dict_source
+    programs_dir = base_dir + '/programs'           # /path/to/zstd/programs
    os.makedirs(tmp_dir, exist_ok=True)

    # since Travis clones limited depth, we should clone full repository
@@ -137,6 +216,10 @@ if __name__ == '__main__':
                os.makedirs(r_dir, exist_ok=True)
                os.chdir(clone_dir)
                git(['--work-tree=' + r_dir, 'checkout', tag, '--', '.'], False)
+                if tag == 'v0.5.0':
+                    os.chdir(r_dir + '/dictBuilder')  # /path/to/zstd/tests/versionsTest/v0.5.0/dictBuilder
+                    make(['clean', 'dictBuilder'], False)
+                    shutil.copy2('dictBuilder', '{}/dictBuilder.{}'.format(tmp_dir, tag))
                os.chdir(r_dir + '/programs')  # /path/to/zstd/tests/versionsTest/<TAG>/programs
                make(['clean', 'zstd'], False)
            else:
@@ -151,15 +234,25 @@ if __name__ == '__main__':
    for dec in glob.glob("*.dec"):
        os.remove(dec)

+    # copy *.c and *.h to a temporary directory ("dict_source")
+    if not os.path.isdir(dict_source_path):
+        os.mkdir(dict_source_path)
+        print('cp ' + dict_files + ' ' + dict_source_path)
+        execute('cp ' + dict_files + ' ' + dict_source_path, param_shell=True)
+
    print('Compress test.dat by all released zstd')

    error_code = 0
    for tag in tags:
        print(tag)
+        if tag >= 'v0.5.0':
+            create_dict(tag, dict_source_path)
+            dict_compress_sample(tag, test_dat)
+            remove_duplicates()
+            error_code += decompress_dict(tag)
        compress_sample(tag, test_dat)
        remove_duplicates()
-        if tag >= 'v0.5.1':
-            error_code += decompress_zst(tag)
+        error_code += decompress_zst(tag)

    print('')
    print('Enumerate different compressed files')
Author	SHA1	Message	Date
Yann ColletandGitHub	77054b452d	Merge pull request #222 from Cyan4973/dev Improved speed test	2016-06-23 01:32:56 +02:00
Yann ColletandGitHub	91d67f0226	Merge pull request #221 from inikep/dev test-zstd-speed.py: added --message in start and finish e-mail	2016-06-22 23:34:19 +02:00
inikep	82babfc0c6	test-zstd-speed.py: reordering of functions	2016-06-22 20:06:42 +02:00
inikep	116128ce41	test-zstd-speed.py: improved exception handling	2016-06-22 18:12:57 +02:00
inikep	47020670b6	test-zstd-speed.py: added --message in start and finish e-mail - support for ~ in filenames - result files ending with .txt	2016-06-22 17:11:01 +02:00
Yann ColletandGitHub	d4f5a176d4	Merge pull request #220 from inikep/dev improved test-zstd-speed.py	2016-06-22 16:30:26 +02:00
inikep	9649126f70	Merge remote-tracking branch 'refs/remotes/origin/speedTest' into dev	2016-06-22 16:22:29 +02:00
inikep	da1685e98d	Merge remote-tracking branch 'refs/remotes/Cyan4973/dev' into dev	2016-06-22 16:21:34 +02:00
inikep	d7d251ccb5	bench.c: added support for ZSTD_GIT_COMMIT	2016-06-22 16:13:25 +02:00
inikep	f2f59d758e	test-zstd-speed.py: added ZSTD_GIT_COMMIT	2016-06-22 15:42:26 +02:00
Yann ColletandGitHub	378822521a	Merge pull request #217 from Cyan4973/dev v0.7.1 tag	2016-06-22 14:07:04 +02:00
inikep	c364ee7867	test-zstd-speed.py: sent e-mail after KeyboardInterrupt	2016-06-22 14:01:53 +02:00
Yann Collet	3755eb8fea	fixed strict-aliasing warning on gcc6	2016-06-22 13:15:53 +02:00
inikep	bcb9aad727	test-zstd-speed.py: improved log and e-mail formatting	2016-06-22 13:07:58 +02:00
inikep	95da743a56	test-zstd-speed.py: send e-mail in case of error	2016-06-22 12:12:35 +02:00
Yann Collet	23042929da	Fixed : dictBuilder fails if first sample is too small	2016-06-22 11:05:34 +02:00
Yann ColletandGitHub	8d30cd562a	Merge pull request #219 from inikep/dev added gcc5test and gcc6test for Travis	2016-06-21 21:51:52 +02:00
inikep	2d9272f1dd	test-zstd-speed.py: improved formatting	2016-06-21 19:28:51 +02:00
inikep	c9bb102401	Makefile: added gcc5test and gcc6test improved Travis tests	2016-06-21 18:04:13 +02:00
Yann Collet	391a128794	fix : segfault in command line during automatic overwrite protection mode	2016-06-21 17:06:25 +02:00
Yann Collet	bda68c253b	refactored ZBUFF_compressEnd() for better maintainability	2016-06-21 15:18:11 +02:00
Yann Collet	446d442989	zbufftest now ensures ZBUFF_compressEnd() is fully flushed in a single pass if dst buffer is large enough	2016-06-21 14:14:02 +02:00
Yann Collet	aa29226b7c	fix : ZBUFF_compressEnd() gives right amount remaining to flush, including future epilogue	2016-06-21 14:04:57 +02:00
Yann Collet	f15c1cb00c	Fixed : ZBUFF_compressEnd() called multiple times with too small dst buffer (#206 )	2016-06-21 13:11:48 +02:00
Yann Collet	2565971ea4	stronger fuzzer tests for ZBUFF_compressEnd()	2016-06-21 12:17:26 +02:00
Yann Collet	a49e066b26	clarified comments on `ZSTD_compressContinue()`	2016-06-21 11:54:03 +02:00
inikep	d731de84ab	test-zstd-speed.py: added send_simple_email	2016-06-21 11:26:17 +02:00
inikep	9bf5357101	bench.c: use ZSTD_VERSION_STRING	2016-06-21 11:01:29 +02:00
Yann Collet	d4f38d0dcd	updated library to v0.7.1	2016-06-21 10:15:43 +02:00
Yann ColletandGitHub	d000042108	Merge pull request #216 from Cyan4973/dev v0.7.1	2016-06-21 08:43:04 +02:00
Yann Collet	22d76322ce	minor refactor	2016-06-21 08:01:51 +02:00
Yann Collet	a436a529bc	minor : fast_extDict does no longer skip first byte	2016-06-20 23:34:04 +02:00
Yann Collet	4623d11571	new correction, less extreme replacement value	2016-06-20 19:15:37 +02:00
Yann Collet	5477cc25f7	fixed corruption error related to inter-blocks rep-offset	2016-06-20 18:31:25 +02:00
Yann Collet	c98f8e7b57	Modified : zstd CLI enables checksum by default	2016-06-20 16:31:24 +02:00
Yann Collet	cbf8c52954	roundTripCrash automatic compression level now depends on first 128 bytes	2016-06-20 16:18:19 +02:00
Yann Collet	e4811ba761	Modified : ZSTD_createDDict() accepts dictionary < 8 bytes in pure content mode (reported by @chipturner)	2016-06-19 23:06:54 +02:00
Yann Collet	a050204342	Merge branch 'dev' of github.com:Cyan4973/zstd into dev	2016-06-19 14:27:54 +02:00
Yann Collet	06d9a73b48	minor refactor, using `WILDCOPY_OVERLENGTH` macro instead of hard-coded 8	2016-06-19 14:27:21 +02:00
Yann ColletandGitHub	06a40ac1a7	Merge pull request #213 from inikep/dev test-zstd-versions.py: test dictionary compression of v0.5.1 and newer	2016-06-17 16:15:34 +02:00
inikep	dba8b44370	Merge remote-tracking branch 'refs/remotes/Cyan4973/dev' into dev	2016-06-17 15:17:37 +02:00
inikep	e16f65675b	test-zstd-versions.py: fixed creation of dictionaries for v0.5.1+	2016-06-17 15:17:35 +02:00
inikep	7e3597bf38	test-zstd-versions.py: create and test dictionaries for v0.5.0	2016-06-17 14:43:24 +02:00
inikep	2ef16501ca	test-zstd-versions.py: improved error handling	2016-06-17 14:07:42 +02:00
inikep	4545671b44	test-zstd-versions.py: fixed DEVNULL	2016-06-17 13:39:43 +02:00
Yann Collet	19cab46f2f	Joined `seqStore` initialization at dispatch point	2016-06-17 12:54:52 +02:00
inikep	d1af4e66b6	test-zstd-versions.py: dictionary compression with levels 1, 5, 9, 15, 18	2016-06-16 20:23:11 +02:00
inikep	150152fb8a	test-zstd-versions.py: test dictiony compression for v0.5.1 and newer	2016-06-16 19:29:09 +02:00
inikep	24aa7b4238	test-zstd-versions.py: create dictionaries for v0.5.1 and newer	2016-06-16 14:15:32 +02:00