Merge pull request #236 from Cyan4973/dev

v0.7.2
fixed minor visual warning
2016-07-03 21:06:42 +02:00 · 2016-07-03 19:03:13 +02:00 · 2016-07-03 18:49:35 +02:00 · 2016-07-03 01:48:26 +02:00 · 2016-07-03 01:36:57 +02:00 · 2016-07-03 01:28:16 +02:00
28 changed files with 1397 additions and 622 deletions
@@ -13,6 +13,7 @@
 *.dylib

 # Executables
+zstd
 *.exe
 *.out
 *.app
@@ -6,85 +6,82 @@ matrix:
    # Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes)
    - os: linux
      sudo: false
-      env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=travis-install
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make travis-install"
    - os: linux
      sudo: false
-      env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=cmaketest
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make cmaketest"
    - os: linux
      sudo: false
-      env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=test
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make test"
    - os: linux
      sudo: false
-      env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM="-C programs test-zstd_nolegacy"
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make -C programs test-zstd_nolegacy"
    - os: linux
      sudo: false
-      env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=usan
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make usan"
    - os: linux
      sudo: false
-      env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=asan
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make asan"
    - os: linux
      sudo: false
-      env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=zlibwrapper
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make zlibwrapper"
    # Standard Ubuntu 12.04 LTS Server Edition 64 bit
    - os: linux
      sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=clangtest
+      env: PLATFORM="Ubuntu 12.04" CMD="make libc6install clangtest"
    - os: linux
      sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gpptest
+      env: PLATFORM="Ubuntu 12.04" CMD="make gppinstall gpptest"
    - os: linux
      sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gnu90test
+      env: PLATFORM="Ubuntu 12.04" CMD="make libc6install gnu90test"
    - os: linux
      sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=c99test
+      env: PLATFORM="Ubuntu 12.04" CMD="make libc6install c99test"
    - os: linux
      sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gnu99test
+      env: PLATFORM="Ubuntu 12.04" CMD="make libc6install gnu99test"
    - os: linux
      sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=armtest-w-install
+      env: PLATFORM="Ubuntu 12.04" CMD="make arminstall armtest"
    - os: linux
      sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C programs test32"
+      env: PLATFORM="Ubuntu 12.04" CMD="make libc6install && make -C programs test32"
    - os: linux
      sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C tests versionsTest"
+      env: PLATFORM="Ubuntu 12.04" CMD="make -C tests versionsTest"
    - os: linux
      sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=asan32
+      env: PLATFORM="Ubuntu 12.04" CMD="make libc6install asan32"
    - os: linux
      sudo: required
-      env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C programs valgrindTest"
+      env: PLATFORM="Ubuntu 12.04" CMD="make valgrindinstall && make -C programs valgrindTest"
    # Ubuntu 14.04 LTS Server Edition 64 bit
    - os: linux
      dist: trusty
      sudo: required
-      env: PLATFORM="Ubuntu 14.04" MAKE_PARAM=ppctest-w-install
+      env: PLATFORM="Ubuntu 14.04" CMD="make gcc5install gcc5test"
    - os: linux
      dist: trusty
      sudo: required
-      env: PLATFORM="Ubuntu 14.04" MAKE_PARAM=zlibwrapper
+      env: PLATFORM="Ubuntu 14.04" CMD="make gcc6install gcc6test"
+    - os: linux
+      dist: trusty
+      sudo: required
+      env: PLATFORM="Ubuntu 14.04" CMD="make ppcinstall ppctest"
+    - os: linux
+      dist: trusty
+      sudo: required
+      env: PLATFORM="Ubuntu 14.04" CMD="make zlibwrapper"
    # OS X Mavericks
    - os: osx
-      env: PLATFORM="OS X Mavericks" MAKE_PARAM=travis-install
+      env: PLATFORM="OS X Mavericks" CMD="make travis-install"
    - os: osx
-      env: PLATFORM="OS X Mavericks" MAKE_PARAM=gnu90test
+      env: PLATFORM="OS X Mavericks" CMD="make gnu90test"
    - os: osx
-      env: PLATFORM="OS X Mavericks" MAKE_PARAM=test
+      env: PLATFORM="OS X Mavericks" CMD="make test"
  exclude:
    - compiler: gcc

-before_install:
-  - set -e
-  - |
-    if [ "$TRAVIS_OS_NAME" == "linux" ]; then
-      CAN_I_RUN_SUDO=$(sudo -n uptime 2>&1|grep "load"|wc -l)
-      echo "CAN_I_RUN_SUDO=$CAN_I_RUN_SUDO\n"
-      if [ ${CAN_I_RUN_SUDO} -gt 0 ]; then
-        sudo apt-get install -y -qq clang g++-multilib gcc-multilib valgrind
-      fi
-    fi
-
 script:
-  - make $MAKE_PARAM
+  - sh -c "$CMD"
@@ -51,6 +51,7 @@ all:

 zstdprogram:
 	$(MAKE) -C $(PRGDIR)
+	cp $(PRGDIR)/zstd .

 zlibwrapper:
 	$(MAKE) -C $(ZSTDDIR) all
@@ -63,6 +64,7 @@ clean:
 	@$(MAKE) -C $(ZSTDDIR) $@ > $(VOID)
 	@$(MAKE) -C $(PRGDIR) $@ > $(VOID)
 	@$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID)
+	@rm -f zstd
 	@echo Cleaning completed


@@ -85,6 +87,14 @@ travis-install:
 gpptest: clean
 	$(MAKE) all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"

+gcc5test: clean
+	gcc-5 -v
+	$(MAKE) all CC=gcc-5 MOREFLAGS="-Werror"
+
+gcc6test: clean
+	gcc-6 -v
+	$(MAKE) all CC=gcc-6 MOREFLAGS="-Werror"
+
 clangtest: clean
 	clang -v
 	$(MAKE) all CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion"
@@ -93,35 +103,13 @@ armtest: clean
 	$(MAKE) -C $(PRGDIR) datagen   # use native, faster
 	$(MAKE) -C $(PRGDIR) test CC=arm-linux-gnueabi-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static"

-# for Travis CI
-arminstall: clean
-	sudo apt-get install -y -q qemu binfmt-support qemu-user-static gcc-arm-linux-gnueabi
-
-# for Travis CI
-armtest-w-install: clean arminstall armtest
-
 ppctest: clean
 	$(MAKE) -C $(PRGDIR) datagen   # use native, faster
 	$(MAKE) -C $(PRGDIR) test CC=powerpc-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static"

-# for Travis CI
-ppcinstall: clean
-	# sudo apt-get update  -y -q
-	sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu  # doesn't work with Ubuntu 12.04
-
-# for Travis CI
-ppctest-w-install: clean ppcinstall ppctest
-
 ppc64test: clean
 	$(MAKE) -C $(PRGDIR) datagen   # use native, faster
-	$(MAKE) -C $(PRGDIR) test CC=powerpc64le-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static"
-
-ppc64install: clean
-	sudo apt-get update  -y -q
-	sudo apt-get install -y -q qemu-ppc64le binfmt-support qemu-user-static gcc-powerpc64le-linux-gnu
-	update-binfmts --displ
-
-ppc64test-w-install: clean ppc64install ppc64test
+	$(MAKE) -C $(PRGDIR) test CC=powerpc-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-m64 -Werror -static"

 usan: clean
 	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=undefined"
@@ -180,4 +168,41 @@ bmix32test: clean

 bmi32test: clean
 	CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(PRGDIR) test
+
+staticAnalyze: clean
+	CPPFLAGS=-g scan-build --status-bugs -v $(MAKE) all	
 endif
+
+
+#------------------------------------------------------------------------
+# for Travis CI
+#------------------------------------------------------------------------
+libc6install:
+	sudo apt-get install -y -qq libc6-dev-i386
+
+gppinstall:
+	sudo apt-get install -y -qq g++-multilib
+
+gcc5install:
+	sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
+	sudo apt-get update -y -qq
+	sudo apt-get install -y -qq gcc-5-multilib
+
+gcc6install:
+	sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
+	sudo apt-get update -y -qq
+	sudo apt-get install -y -qq gcc-6-multilib
+
+arminstall: clean
+	sudo apt-get install -y -q qemu binfmt-support qemu-user-static gcc-arm-linux-gnueabi
+
+ppcinstall: clean
+	sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu  # doesn't work with Ubuntu 12.04
+
+ppc64install: clean   # compilation works but cannot be executed
+	sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu
+	update-binfmts --displ
+
+valgrindinstall:
+	echo sudo apt-get install -y -qq valgrind
+	sudo apt-get install -y -qq valgrind
@@ -1,3 +1,14 @@
+v0.7.2
+fixed : ZSTD_decompressBlock() using multiple consecutive blocks. Reported by Greg Slazinski
+fixed : potential segfault on very large files (many gigabytes). Reported by Chip Turner.
+fixed : CLI displays system error message when destination file cannot be created (#231). Reported by Chip Turner.
+
+v0.7.1
+fixed : ZBUFF_compressEnd() called multiple times with too small `dst` buffer, reported by Christophe Chevalier
+fixed : dictBuilder fails if first sample is too small, reported by Руслан Ковалёв
+fixed : corruption issue, reported by cj
+modified : checksum enabled by default in command line mode
+
 v0.7.0
 New : Support for directory compression, using `-r`, thanks to Przemyslaw Skibinski
 New : Command `--rm`, to remove source file after successful de/compression
@@ -136,7 +136,7 @@ typedef U32 HUF_DTable;
 size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< single-symbol decoder */
 size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< double-symbols decoder */

-size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
+size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< decodes RLE and uncompressed */
 size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */
 size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< single-symbol decoder */
 size_t HUF_decompress4X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< double-symbols decoder */
@@ -44,10 +44,8 @@ extern "C" {
 /* ***************************************************************
 *  Compiler specifics
 *****************************************************************/
-/*!
-*  ZSTD_DLL_EXPORT :
-*  Enable exporting of functions when building a Windows DLL
-*/
+/* ZSTD_DLL_EXPORT :
+*  Enable exporting of functions when building a Windows DLL */
 #if defined(_WIN32) && defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
 #  define ZSTDLIB_API __declspec(dllexport)
 #else
@@ -103,8 +101,8 @@ ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCap
 *  @return : nb of bytes still present into internal buffer (0 if it's empty)
 *            or an error code, which can be tested using ZBUFF_isError().
 *
-*  Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedCInSize / ZBUFF_recommendedCOutSize
-*  input : ZBUFF_recommendedCInSize==128 KB block size is the internal unit, it improves latency to use this value (skipped buffering).
+*  Hint : _recommended buffer_ sizes (not compulsory) : ZBUFF_recommendedCInSize() / ZBUFF_recommendedCOutSize()
+*  input : ZBUFF_recommendedCInSize==128 KB block size is the internal unit, use this value to reduce intermediate stages (better latency)
 *  output : ZBUFF_recommendedCOutSize==ZSTD_compressBound(128 KB) + 3 + 3 : ensures it's always possible to write/flush/end a full block. Skip some buffering.
 *  By using both, it ensures that input will be entirely consumed, and output will always contain the result, reducing intermediate buffering.
 * **************************************************/
@@ -61,7 +61,7 @@ extern "C" {
 ***************************************/
 #define ZSTD_VERSION_MAJOR    0
 #define ZSTD_VERSION_MINOR    7
-#define ZSTD_VERSION_RELEASE  0
+#define ZSTD_VERSION_RELEASE  2

 #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
 #define ZSTD_QUOTE(str) #str
@@ -132,7 +132,7 @@ ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapa
 /*! ZSTD_compress_usingDict() :
 *   Compression using a pre-defined Dictionary content (see dictBuilder).
 *   Note 1 : This function load the dictionary, resulting in a significant startup time.
-*   Note 2 : `dict` must remain valid and unmodified during compression operation.
+*   Note 2 : `dict` must remain accessible and unmodified during compression operation.
 *   Note 3 : `dict` can be `NULL`, in which case, it's equivalent to ZSTD_compressCCtx() */
 ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
                                           void* dst, size_t dstCapacity,
@@ -144,7 +144,7 @@ ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
 *   Decompression using a pre-defined Dictionary content (see dictBuilder).
 *   Dictionary must be identical to the one used during compression.
 *   Note 1 : This function load the dictionary, resulting in a significant startup time
-*   Note 2 : `dict` must remain valid and unmodified during compression operation.
+*   Note 2 : `dict` must remain accessible and unmodified during compression operation.
 *   Note 3 : `dict` can be `NULL`, in which case, it's equivalent to ZSTD_decompressDCtx() */
 ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
                                             void* dst, size_t dstCapacity,
@@ -192,7 +192,7 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,

 /* ====================================================================================
 * The definitions in this section are considered experimental.
- * They should never be used in association with a dynamic library, as they may change in the future.
+ * They should never be used with a dynamic library, as they may change in the future.
 * They are provided for advanced usages.
 * Use them only in association with static linking.
 * ==================================================================================== */
@@ -200,7 +200,6 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
 /*--- Dependency ---*/
 #include "mem.h"   /* U32 */

-
 /*--- Constants ---*/
 #define ZSTD_MAGICNUMBER            0xFD2FB527   /* v0.7 */
 #define ZSTD_MAGIC_SKIPPABLE_START  0x184D2A50U
@@ -230,19 +229,19 @@ static const size_t ZSTD_skippableHeaderSize = 8;  /* magic number + skippable f
 typedef enum { ZSTD_fast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt } ZSTD_strategy;   /*< from faster to stronger */

 typedef struct {
-    U32 windowLog;     /*< largest match distance : larger == more compression, more memory needed during decompression */
-    U32 chainLog;      /*< fully searched segment : larger == more compression, slower, more memory (useless for fast) */
-    U32 hashLog;       /*< dispatch table : larger == faster, more memory */
-    U32 searchLog;     /*< nb of searches : larger == more compression, slower */
-    U32 searchLength;  /*< match length searched : larger == faster decompression, sometimes less compression */
-    U32 targetLength;  /*< acceptable match size for optimal parser (only) : larger == more compression, slower */
+    U32 windowLog;      /*< largest match distance : larger == more compression, more memory needed during decompression */
+    U32 chainLog;       /*< fully searched segment : larger == more compression, slower, more memory (useless for fast) */
+    U32 hashLog;        /*< dispatch table : larger == faster, more memory */
+    U32 searchLog;      /*< nb of searches : larger == more compression, slower */
+    U32 searchLength;   /*< match length searched : larger == faster decompression, sometimes less compression */
+    U32 targetLength;   /*< acceptable match size for optimal parser (only) : larger == more compression, slower */
    ZSTD_strategy strategy;
 } ZSTD_compressionParameters;

 typedef struct {
-    U32 contentSizeFlag;  /*< 1: content size will be in frame header (if known). */
-    U32 checksumFlag;     /*< 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */
-    U32 noDictIDFlag;     /*< 1: no dict ID will be saved into frame header (if dictionary compression) */
+    U32 contentSizeFlag; /*< 1: content size will be in frame header (if known). */
+    U32 checksumFlag;    /*< 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */
+    U32 noDictIDFlag;    /*< 1: no dict ID will be saved into frame header (if dictionary compression) */
 } ZSTD_frameParameters;

 typedef struct {
@@ -270,16 +269,21 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictS

 ZSTDLIB_API unsigned ZSTD_maxCLevel (void);

+/*! ZSTD_getParams() :
+*   same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of a `ZSTD_compressionParameters`.
+*   All fields of `ZSTD_frameParameters` are set to default (0) */
+ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSize, size_t dictSize);
+
 /*! ZSTD_getCParams() :
 *   @return ZSTD_compressionParameters structure for a selected compression level and srcSize.
 *   `srcSize` value is optional, select 0 if not known */
 ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, size_t dictSize);

-/*! ZSTD_checkParams() :
+/*! ZSTD_checkCParams() :
 *   Ensure param values remain within authorized range */
 ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);

-/*! ZSTD_adjustParams() :
+/*! ZSTD_adjustCParams() :
 *   optimize params for a given `srcSize` and `dictSize`.
 *   both values are optional, select `0` if unknown. */
 ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, U64 srcSize, size_t dictSize);
@@ -322,15 +326,22 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
  It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx()

  Then, consume your input using ZSTD_compressContinue().
-  ZSTD_compressContinue() presumes prior data is still accessible and unmodified (up to maximum distance size, see WindowLog).
-  The interface is synchronous, so input will be entirely consumed and produce associated compressed output.
-  You must ensure there is enough space in destination buffer to store compressed data under worst case scenario.
-  Worst case evaluation is provided by ZSTD_compressBound().
+  There are some important considerations to keep in mind when using this advanced function :
+  - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffer only.
+  - Interface is synchronous : input will be entirely consumed and produce 1+ compressed blocks.
+  - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario.
+    Worst case evaluation is provided by ZSTD_compressBound().
+    ZSTD_compressContinue() doesn't guarantee recover after a failed compression.
+  - ZSTD_compressContinue() presumes prior input ***is still accessible and unmodified*** (up to maximum distance size, see WindowLog).
+    It remembers all previous contiguous blocks, plus one separated memory segment (which can itself consists of multiple contiguous blocks)
+  - ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps.
+    In which case, it will "discard" the relevant memory section from its history.
+

  Finish a frame with ZSTD_compressEnd(), which will write the epilogue.
  Without epilogue, frames will be considered unfinished (broken) by decoders.

-  You can then reuse ZSTD_CCtx to compress some new frame.
+  You can then reuse `ZSTD_CCtx` (ZSTD_compressBegin()) to compress some new frame.
 */

 typedef struct {
@@ -357,7 +368,7 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds
  A ZSTD_DCtx object can be re-used multiple times.

  First optional operation is to retrieve frame parameters, using ZSTD_getFrameParams(), which doesn't consume the input.
-  It can provide the minimum size of rolling buffer required to properly decompress data,
+  It can provide the minimum size of rolling buffer required to properly decompress data (`windowSize`),
  and optionally the final size of uncompressed content.
  (Note : content size is an optional info that may not be present. 0 means : content size unknown)
  Frame parameters are extracted from the beginning of compressed frame.
@@ -373,7 +384,7 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds
  Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively.
  ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue().
  ZSTD_decompressContinue() requires this exact amount of bytes, or it will fail.
-  ZSTD_decompressContinue() needs previous data blocks during decompression, up to (1 << windowlog).
+  ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize`.
  They should preferably be located contiguously, prior to current block. Alternatively, a round buffer is also possible.

  @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).
@@ -401,6 +412,7 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds

    A few rules to respect :
    - Uncompressed block size must be <= ZSTD_BLOCKSIZE_MAX (128 KB)
+      + If you need to compress more, it's recommended to use ZSTD_compress() instead, since frame metadata costs become negligible.
    - Compressing or decompressing requires a context structure
      + Use ZSTD_createCCtx() and ZSTD_createDCtx()
    - It is necessary to init context before starting
@@ -51,7 +51,7 @@
 /*-*************************************
 *  Common constants
 ***************************************/
-#define ZSTD_OPT_DEBUG 0     // 3 = compression stats;  5 = check encoded sequences;  9 = full logs
+#define ZSTD_OPT_DEBUG 0     /* 3 = compression stats;  5 = check encoded sequences;  9 = full logs */
 #include <stdio.h>
 #if defined(ZSTD_OPT_DEBUG) && ZSTD_OPT_DEBUG>=9
    #define ZSTD_LOG_PARSER(...) printf(__VA_ARGS__)
@@ -233,6 +233,6 @@ int ZSTD_isSkipFrame(ZSTD_DCtx* dctx);
 /* custom memory allocation functions */
 void* ZSTD_defaultAllocFunction(void* opaque, size_t size);
 void ZSTD_defaultFreeFunction(void* opaque, void* address);
-static ZSTD_customMem const defaultCustomMem = { ZSTD_defaultAllocFunction, ZSTD_defaultFreeFunction, NULL };
+static const ZSTD_customMem defaultCustomMem = { ZSTD_defaultAllocFunction, ZSTD_defaultFreeFunction, NULL };

 #endif   /* ZSTD_CCOMMON_H_MODULE */
@@ -79,7 +79,7 @@ static size_t const ZBUFF_endFrameSize = ZSTD_BLOCKHEADERSIZE;
 *  output : ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + ZBUFF_endFrameSize : ensures it's always possible to write/flush/end a full block at best speed.
 * **************************************************/

-typedef enum { ZBUFFcs_init, ZBUFFcs_load, ZBUFFcs_flush } ZBUFF_cStage;
+typedef enum { ZBUFFcs_init, ZBUFFcs_load, ZBUFFcs_flush, ZBUFFcs_final } ZBUFF_cStage;

 /* *** Resources *** */
 struct ZBUFF_CCtx_s {
@@ -162,7 +162,7 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
    zbc->inToCompress = 0;
    zbc->inBuffPos = 0;
    zbc->inBuffTarget = zbc->blockSize;
-    zbc->outBuffFlushedSize = 0;
+    zbc->outBuffContentSize = zbc->outBuffFlushedSize = 0;
    zbc->stage = ZBUFFcs_load;
    return 0;   /* ready to go */
 }
@@ -170,9 +170,7 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,

 size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, int compressionLevel)
 {
-    ZSTD_parameters params;
-    memset(&params, 0, sizeof(params));
-    params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize);
+    ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize);
    return ZBUFF_compressInit_advanced(zbc, dict, dictSize, params, 0);
 }

@@ -196,7 +194,7 @@ MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src
 static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc,
                              void* dst, size_t* dstCapacityPtr,
                        const void* src, size_t* srcSizePtr,
-                              int flush)   /* aggregate : wait for full block before compressing */
+                              int flush)
 {
    U32 notDone = 1;
    const char* const istart = (const char*)src;
@@ -243,17 +241,20 @@ static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc,
            }

        case ZBUFFcs_flush:
-            /* flush into dst */
            {   size_t const toFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize;
                size_t const flushed = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outBuffFlushedSize, toFlush);
                op += flushed;
                zbc->outBuffFlushedSize += flushed;
-                if (toFlush!=flushed) { notDone = 0; break; } /* not enough space within dst to store compressed block : stop there */
-                zbc->outBuffContentSize = 0;
-                zbc->outBuffFlushedSize = 0;
+                if (toFlush!=flushed) { notDone = 0; break; } /* dst too small to store flushed data : stop there */
+                zbc->outBuffContentSize = zbc->outBuffFlushedSize = 0;
                zbc->stage = ZBUFFcs_load;
                break;
            }
+
+        case ZBUFFcs_final:
+            notDone = 0;   /* do nothing */
+            break;
+
        default:
            return ERROR(GENERIC);   /* impossible */
        }
@@ -291,19 +292,30 @@ size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr)
    BYTE* const ostart = (BYTE*)dst;
    BYTE* const oend = ostart + *dstCapacityPtr;
    BYTE* op = ostart;
-    size_t outSize = *dstCapacityPtr;
-    size_t epilogueSize, remaining;
-    ZBUFF_compressFlush(zbc, dst, &outSize);     /* flush any remaining inBuff */
-    op += outSize;
-    epilogueSize = ZSTD_compressEnd(zbc->zc, zbc->outBuff + zbc->outBuffContentSize, zbc->outBuffSize - zbc->outBuffContentSize);   /* epilogue into outBuff */
-    zbc->outBuffContentSize += epilogueSize;
-    outSize = oend-op;
-    zbc->stage = ZBUFFcs_flush;
-    remaining = ZBUFF_compressFlush(zbc, op, &outSize);   /* attempt to flush epilogue into dst */
-    op += outSize;
-    if (!remaining) zbc->stage = ZBUFFcs_init;   /* close only if nothing left to flush */
-    *dstCapacityPtr = op-ostart;                 /* tells how many bytes were written */
-    return remaining;
+
+    if (zbc->stage != ZBUFFcs_final) {
+        /* flush whatever remains */
+        size_t outSize = *dstCapacityPtr;
+        size_t const remainingToFlush = ZBUFF_compressFlush(zbc, dst, &outSize);
+        op += outSize;
+        if (remainingToFlush) {
+            *dstCapacityPtr = op-ostart;
+            return remainingToFlush + ZBUFF_endFrameSize;
+        }
+        /* create epilogue */
+        zbc->stage = ZBUFFcs_final;
+        zbc->outBuffContentSize = ZSTD_compressEnd(zbc->zc, zbc->outBuff, zbc->outBuffSize); /* epilogue into outBuff */
+    }
+
+    /* flush epilogue */
+    {   size_t const toFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize;
+        size_t const flushed = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outBuffFlushedSize, toFlush);
+        op += flushed;
+        zbc->outBuffFlushedSize += flushed;
+        *dstCapacityPtr = op-ostart;
+        if (toFlush==flushed) zbc->stage = ZBUFFcs_init;  /* end reached */
+        return toFlush - flushed;
+    }
 }


@@ -246,7 +246,7 @@ ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, U

 size_t ZSTD_sizeofCCtx(ZSTD_compressionParameters cParams)   /* hidden interface, for paramagrill */
 {
-    ZSTD_CCtx* zc = ZSTD_createCCtx();
+    ZSTD_CCtx* const zc = ZSTD_createCCtx();
    ZSTD_parameters params;
    memset(&params, 0, sizeof(params));
    params.cParams = cParams;
@@ -295,7 +295,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
    zc->seqStore.buffer = zc->hashTable3 + h3Size;
    zc->hufTable = (HUF_CElt*)zc->seqStore.buffer;
    zc->flagStaticTables = 0;
-    zc->seqStore.buffer = ((U32*)(zc->seqStore.buffer)) + 256;
+    zc->seqStore.buffer = ((U32*)(zc->seqStore.buffer)) + 256;  /* note : HUF_CElt* is incomplete type, size is simulated using U32 */

    zc->nextToUpdate = 1;
    zc->nextSrc = NULL;
@@ -313,14 +313,17 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
        zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1<<Litbits);
        zc->seqStore.matchLengthFreq = zc->seqStore.litLengthFreq + (MaxLL+1);
        zc->seqStore.offCodeFreq = zc->seqStore.matchLengthFreq + (MaxML+1);
-        zc->seqStore.matchTable = (ZSTD_match_t*)((void*)(zc->seqStore.offCodeFreq + (MaxOff+1)));
-        zc->seqStore.priceTable = (ZSTD_optimal_t*)((void*)(zc->seqStore.matchTable + ZSTD_OPT_NUM+1));
+        zc->seqStore.buffer = zc->seqStore.offCodeFreq + (MaxOff+1);
+        zc->seqStore.matchTable = (ZSTD_match_t*)zc->seqStore.buffer;
+        zc->seqStore.buffer = zc->seqStore.matchTable + ZSTD_OPT_NUM+1;
+        zc->seqStore.priceTable = (ZSTD_optimal_t*)zc->seqStore.buffer;
        zc->seqStore.buffer = zc->seqStore.priceTable + ZSTD_OPT_NUM+1;
        zc->seqStore.litLengthSum = 0;
    }
-    zc->seqStore.offsetStart = (U32*) (zc->seqStore.buffer);
-    zc->seqStore.litLengthStart = (U16*) (void*)(zc->seqStore.offsetStart + maxNbSeq);
-    zc->seqStore.matchLengthStart = (U16*) (void*)(zc->seqStore.litLengthStart + maxNbSeq);
+    zc->seqStore.offsetStart = (U32*)(zc->seqStore.buffer);
+    zc->seqStore.buffer = zc->seqStore.offsetStart + maxNbSeq;
+    zc->seqStore.litLengthStart = (U16*)zc->seqStore.buffer;
+    zc->seqStore.matchLengthStart = zc->seqStore.litLengthStart + maxNbSeq;
    zc->seqStore.llCodeStart = (BYTE*) (zc->seqStore.matchLengthStart + maxNbSeq);
    zc->seqStore.mlCodeStart = zc->seqStore.llCodeStart + maxNbSeq;
    zc->seqStore.offCodeStart = zc->seqStore.mlCodeStart + maxNbSeq;
@@ -424,21 +427,8 @@ static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue)
 */


-/* Frame descriptor
+/* Frame header :

-    // old
-   1 byte - Alloc :
-   bit 0-3 : windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN   (see zstd_internal.h)
-   bit 4   : reserved for windowLog (must be zero)
-   bit 5   : reserved (must be zero)
-   bit 6-7 : Frame content size : unknown, 1 byte, 2 bytes, 8 bytes
-
-   1 byte - checker :
-   bit 0-1 : dictID (0, 1, 2 or 4 bytes)
-   bit 2-7 : reserved (must be zero)
-
-
-    // new
   1 byte - FrameHeaderDescription :
   bit 0-1 : dictID (0, 1, 2 or 4 bytes)
   bit 2-4 : reserved (must be zero)
@@ -450,24 +440,24 @@ static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue)
   bit 0-2 : octal Fractional (1/8th)
   bit 3-7 : Power of 2, with 0 = 1 KB (up to 2 TB)

+   Optional : content size (0, 1, 2, 4 or 8 bytes)
+   0 : unknown
+   1 : 0-255 bytes
+   2 : 256 - 65535+256
+   8 : up to 16 exa
+
   Optional : dictID (0, 1, 2 or 4 bytes)
   Automatic adaptation
   0 : no dictID
   1 : 1 - 255
   2 : 256 - 65535
   4 : all other values
-
-   Optional : content size (0, 1, 2, 4 or 8 bytes)
-   0 : unknown
-   1 : 0-255 bytes
-   2 : 256 - 65535+256
-   8 : up to 16 exa
 */


 /* Block format description

-   Block = Literal Section - Sequences Section
+   Block = Literals Section - Sequences Section
   Prerequisite : size of (compressed) block, maximum size of regenerated data

   1) Literal Section
@@ -475,7 +465,7 @@ static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue)
   1.1) Header : 1-5 bytes
        flags: 2 bits
            00 compressed by Huff0
-            01 unused
+            01 repeat
            10 is Raw (uncompressed)
            11 is Rle
            Note : using 01 => Huff0 with precomputed table ?
@@ -511,7 +501,7 @@ static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue)
            else           => 5 bytes (2-2-18-18)
            big endian convention

-        1- CTable available (stored into workspace ?)
+        1- CTable available (stored into workspace)
        2- Small input (fast heuristic ? Full comparison ? depend on clevel ?)


@@ -933,7 +923,7 @@ _check_compressibility:
    `offsetCode` : distance to match, or 0 == repCode.
    `matchCode` : matchLength - MINMATCH
 */
-MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const void* literals, size_t offsetCode, size_t matchCode)
+MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const void* literals, U32 offsetCode, size_t matchCode)
 {
 #if 0  /* for debug */
    static const BYTE* g_start = NULL;
@@ -954,7 +944,7 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const v
    *seqStorePtr->litLength++ = (U16)litLength;

    /* match offset */
-    *(seqStorePtr->offset++) = (U32)offsetCode + 1;
+    *(seqStorePtr->offset++) = offsetCode + 1;

    /* match Length */
    if (matchCode>0xFFFF) { seqStorePtr->longLengthID = 2; seqStorePtr->longLengthPos = (U32)(seqStorePtr->matchLength - seqStorePtr->matchLengthStart); }
@@ -1024,11 +1014,12 @@ static unsigned ZSTD_NbCommonBytes (register size_t val)
 }


-static size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit)
+static size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* const pInLimit)
 {
    const BYTE* const pStart = pIn;
+    const BYTE* const pInLoopLimit = pInLimit - (sizeof(size_t)-1);

-    while ((pIn<pInLimit-(sizeof(size_t)-1))) {
+    while (pIn < pInLoopLimit) {
        size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
        if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; }
        pIn += ZSTD_NbCommonBytes(diff);
@@ -1059,7 +1050,7 @@ static size_t ZSTD_count_2segments(const BYTE* ip, const BYTE* match, const BYTE
 ***************************************/
 static const U32 prime3bytes = 506832829U;
 static U32    ZSTD_hash3(U32 u, U32 h) { return ((u << (32-24)) * prime3bytes)  >> (32-h) ; }
-static size_t ZSTD_hash3Ptr(const void* ptr, U32 h) { return ZSTD_hash3(MEM_readLE32(ptr), h); }
+MEM_STATIC size_t ZSTD_hash3Ptr(const void* ptr, U32 h) { return ZSTD_hash3(MEM_readLE32(ptr), h); }   /* only in zstd_opt.h */

 static const U32 prime4bytes = 2654435761U;
 static U32    ZSTD_hash4(U32 u, U32 h) { return (u * prime4bytes) >> (32-h) ; }
@@ -1125,47 +1116,46 @@ void ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx,
    const BYTE* const lowest = base + lowestIndex;
    const BYTE* const iend = istart + srcSize;
    const BYTE* const ilimit = iend - 8;
-    size_t offset_1=cctx->rep[0], offset_2=cctx->rep[1];
+    U32 offset_1=cctx->rep[0], offset_2=cctx->rep[1];
+    U32 offsetSaved = 0;

    /* init */
-    ZSTD_resetSeqStore(seqStorePtr);
    ip += (ip==lowest);
    {   U32 const maxRep = (U32)(ip-lowest);
-        if (offset_1 > maxRep) offset_1 = 0;
-        if (offset_2 > maxRep) offset_2 = 0;
+        if (offset_2 > maxRep) offsetSaved = offset_2, offset_2 = 0;
+        if (offset_1 > maxRep) offsetSaved = offset_1, offset_1 = 0;
    }

    /* Main Search Loop */
    while (ip < ilimit) {   /* < instead of <=, because repcode check at (ip+1) */
-        size_t mlCode;
-        size_t offset;
+        size_t mLength;
        size_t const h = ZSTD_hashPtr(ip, hBits, mls);
        U32 const current = (U32)(ip-base);
        U32 const matchIndex = hashTable[h];
        const BYTE* match = base + matchIndex;
        hashTable[h] = current;   /* update hash table */

-        if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) {   /* note : by construction, offset_1 <= current */
-            mlCode = ZSTD_count(ip+1+EQUAL_READ32, ip+1+EQUAL_READ32-offset_1, iend) + EQUAL_READ32;
+        if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) { /* note : by construction, offset_1 <= current */
+            mLength = ZSTD_count(ip+1+EQUAL_READ32, ip+1+EQUAL_READ32-offset_1, iend) + EQUAL_READ32;
            ip++;
-            ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mlCode-MINMATCH);
-       } else {
-            if ( (matchIndex <= lowestIndex) ||
-                 (MEM_read32(match) != MEM_read32(ip)) ) {
+            ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mLength-MINMATCH);
+        } else {
+            U32 offset;
+            if ( (matchIndex <= lowestIndex) || (MEM_read32(match) != MEM_read32(ip)) ) {
                ip += ((ip-anchor) >> g_searchStrength) + 1;
                continue;
            }
-            mlCode = ZSTD_count(ip+EQUAL_READ32, match+EQUAL_READ32, iend) + EQUAL_READ32;
-            offset = ip-match;
-            while ((ip>anchor) && (match>lowest) && (ip[-1] == match[-1])) { ip--; match--; mlCode++; }  /* catch up */
+            mLength = ZSTD_count(ip+EQUAL_READ32, match+EQUAL_READ32, iend) + EQUAL_READ32;
+            offset = (U32)(ip-match);
+            while (((ip>anchor) & (match>lowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */
            offset_2 = offset_1;
            offset_1 = offset;

-            ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mlCode-MINMATCH);
+            ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
        }

        /* match found */
-        ip += mlCode;
+        ip += mLength;
        anchor = ip;

        if (ip <= ilimit) {
@@ -1177,18 +1167,18 @@ void ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx,
                 && ( (offset_2>0)
                 & (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) {
                /* store sequence */
-                size_t const rlCode = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-offset_2, iend) + EQUAL_READ32;
-                { size_t const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; } /* swap offset_2 <=> offset_1 */
+                size_t const rLength = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-offset_2, iend) + EQUAL_READ32;
+                { U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; }  /* swap offset_2 <=> offset_1 */
                hashTable[ZSTD_hashPtr(ip, hBits, mls)] = (U32)(ip-base);
-                ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rlCode-MINMATCH);
-                ip += rlCode;
+                ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rLength-MINMATCH);
+                ip += rLength;
                anchor = ip;
                continue;   /* faster when present ... (?) */
    }   }   }

    /* save reps for next block */
-    cctx->savedRep[0] = offset_1 ? (U32)offset_1 : (U32)(iend-base);
-    cctx->savedRep[1] = offset_2 ? (U32)offset_2 : (U32)(iend-base);
+    cctx->savedRep[0] = offset_1 ? offset_1 : offsetSaved;
+    cctx->savedRep[1] = offset_2 ? offset_2 : offsetSaved;

    /* Last Literals */
    {   size_t const lastLLSize = iend - anchor;
@@ -1238,32 +1228,25 @@ static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx,
    const BYTE* const ilimit = iend - 8;
    U32 offset_1=ctx->rep[0], offset_2=ctx->rep[1];

-    /* init */
-    ZSTD_resetSeqStore(seqStorePtr);
-    /* skip first position to avoid read overflow during repcode match check */
-    hashTable[ZSTD_hashPtr(ip, hBits, mls)] = (U32)(ip-base);
-    ip++;
-
-    /* Main Search Loop */
+    /* Search Loop */
    while (ip < ilimit) {  /* < instead of <=, because (ip+1) */
        const size_t h = ZSTD_hashPtr(ip, hBits, mls);
        const U32 matchIndex = hashTable[h];
        const BYTE* matchBase = matchIndex < dictLimit ? dictBase : base;
        const BYTE* match = matchBase + matchIndex;
        const U32 current = (U32)(ip-base);
-        const U32 repIndex = current + 1 - offset_1;
+        const U32 repIndex = current + 1 - offset_1;   /* offset_1 expected <= current +1 */
        const BYTE* repBase = repIndex < dictLimit ? dictBase : base;
        const BYTE* repMatch = repBase + repIndex;
-        size_t mlCode;
-        U32 offset;
+        size_t mLength;
        hashTable[h] = current;   /* update hash table */

-        if ( (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > lowestIndex))   /* intentional overflow */
+        if ( (((U32)((dictLimit-1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > lowestIndex))
           && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
            const BYTE* repMatchEnd = repIndex < dictLimit ? dictEnd : iend;
-            mlCode = ZSTD_count_2segments(ip+1+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repMatchEnd, lowPrefixPtr) + EQUAL_READ32;
+            mLength = ZSTD_count_2segments(ip+1+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repMatchEnd, lowPrefixPtr) + EQUAL_READ32;
            ip++;
-            ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mlCode-MINMATCH);
+            ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mLength-MINMATCH);
        } else {
            if ( (matchIndex < lowestIndex) ||
                 (MEM_read32(match) != MEM_read32(ip)) ) {
@@ -1272,16 +1255,17 @@ static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx,
            }
            {   const BYTE* matchEnd = matchIndex < dictLimit ? dictEnd : iend;
                const BYTE* lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr;
-                mlCode = ZSTD_count_2segments(ip+EQUAL_READ32, match+EQUAL_READ32, iend, matchEnd, lowPrefixPtr) + EQUAL_READ32;
-                while ((ip>anchor) && (match>lowMatchPtr) && (ip[-1] == match[-1])) { ip--; match--; mlCode++; }   /* catch up */
+                U32 offset;
+                mLength = ZSTD_count_2segments(ip+EQUAL_READ32, match+EQUAL_READ32, iend, matchEnd, lowPrefixPtr) + EQUAL_READ32;
+                while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; }   /* catch up */
                offset = current - matchIndex;
                offset_2 = offset_1;
                offset_1 = offset;
-                ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mlCode-MINMATCH);
+                ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
        }   }

        /* found a match : store it */
-        ip += mlCode;
+        ip += mLength;
        anchor = ip;

        if (ip <= ilimit) {
@@ -1337,8 +1321,6 @@ static void ZSTD_compressBlock_fast_extDict(ZSTD_CCtx* ctx,
 }


-
-
 /*-*************************************
 *  Binary Tree search
 ***************************************/
@@ -1370,17 +1352,19 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co
    const U32 windowLow = zc->lowLimit;
    U32 matchEndIdx = current+8;
    size_t bestLength = 8;
+#ifdef ZSTD_C_PREDICT
    U32 predictedSmall = *(bt + 2*((current-1)&btMask) + 0);
    U32 predictedLarge = *(bt + 2*((current-1)&btMask) + 1);
    predictedSmall += (predictedSmall>0);
    predictedLarge += (predictedLarge>0);
+#endif /* ZSTD_C_PREDICT */

    hashTable[h] = current;   /* Update Hash Table */

    while (nbCompares-- && (matchIndex > windowLow)) {
        U32* nextPtr = bt + 2*(matchIndex & btMask);
        size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger);   /* guaranteed minimum nb of common bytes */
-#if 0   /* note : can create issues when hlog small <= 11 */
+#ifdef ZSTD_C_PREDICT   /* note : can create issues when hlog small <= 11 */
        const U32* predictPtr = bt + 2*((matchIndex-1) & btMask);   /* written this way, as bt is a roll buffer */
        if (matchIndex == predictedSmall) {
            /* no need to check length, result known */
@@ -1437,7 +1421,7 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co
    }   }

    *smallerPtr = *largerPtr = 0;
-    if (bestLength > 384) return MIN(192, (U32)(bestLength - 384));
+    if (bestLength > 384) return MIN(192, (U32)(bestLength - 384));   /* speed optimization */
    if (matchEndIdx > current + 8) return matchEndIdx - current - 8;
    return 1;
 }
@@ -1571,7 +1555,6 @@ static void ZSTD_updateTree_extDict(ZSTD_CCtx* zc, const BYTE* const ip, const B
 }


-
 /** Tree updater, providing best match */
 static size_t ZSTD_BtFindBestMatch_extDict (
                        ZSTD_CCtx* zc,
@@ -1622,7 +1605,7 @@ U32 ZSTD_insertAndFindFirstIndex (ZSTD_CCtx* zc, const BYTE* ip, U32 mls)
    const U32 target = (U32)(ip - base);
    U32 idx = zc->nextToUpdate;

-    while(idx < target) {
+    while(idx < target) { /* catch up */
        size_t const h = ZSTD_hashPtr(base+idx, hashLog, mls);
        NEXT_IN_CHAIN(idx, chainMask) = hashTable[h];
        hashTable[h] = idx;
@@ -1659,7 +1642,7 @@ size_t ZSTD_HcFindBestMatch_generic (
    /* HC4 match finder */
    U32 matchIndex = ZSTD_insertAndFindFirstIndex (zc, ip, mls);

-    for ( ; (matchIndex>lowLimit) && (nbAttempts) ; nbAttempts--) {
+    for ( ; (matchIndex>lowLimit) & (nbAttempts>0) ; nbAttempts--) {
        const BYTE* match;
        size_t currentMl=0;
        if ((!extDict) || matchIndex >= dictLimit) {
@@ -1673,7 +1656,7 @@ size_t ZSTD_HcFindBestMatch_generic (
        }

        /* save best solution */
-        if (currentMl > ml) { ml = currentMl; *offsetPtr = ZSTD_REP_MOVE + current - matchIndex; if (ip+currentMl == iLimit) break; /* best possible, and avoid read overflow*/ }
+        if (currentMl > ml) { ml = currentMl; *offsetPtr = current - matchIndex + ZSTD_REP_MOVE; if (ip+currentMl == iLimit) break; /* best possible, and avoid read overflow*/ }

        if (matchIndex <= minChain) break;
        matchIndex = NEXT_IN_CHAIN(matchIndex, chainMask);
@@ -1738,18 +1721,15 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
                        size_t* offsetPtr,
                        U32 maxNbAttempts, U32 matchLengthSearch);
    searchMax_f const searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS : ZSTD_HcFindBestMatch_selectMLS;
-    U32 rep[ZSTD_REP_INIT];
+    U32 offset_1 = ctx->rep[0], offset_2 = ctx->rep[1], savedOffset=0;

    /* init */
    ip += (ip==base);
    ctx->nextToUpdate3 = ctx->nextToUpdate;
-    ZSTD_resetSeqStore(seqStorePtr);
-    {   U32 i;
-        U32 const maxRep = (U32)(ip-base);
-        for (i=0; i<ZSTD_REP_INIT; i++) {
-            rep[i]=ctx->rep[i];
-            if (rep[i]>maxRep) rep[i]=0;
-    }   }
+    {   U32 const maxRep = (U32)(ip-base);
+        if (offset_2 > maxRep) savedOffset = offset_2, offset_2 = 0;
+        if (offset_1 > maxRep) savedOffset = offset_1, offset_1 = 0;
+    }

    /* Match Loop */
    while (ip < ilimit) {
@@ -1758,9 +1738,9 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
        const BYTE* start=ip+1;

        /* check repCode */
-        if ((rep[0]>0) & (MEM_read32(ip+1) == MEM_read32(ip+1 - rep[0]))) {
+        if ((offset_1>0) & (MEM_read32(ip+1) == MEM_read32(ip+1 - offset_1))) {
            /* repcode : we take it */
-            matchLength = ZSTD_count(ip+1+EQUAL_READ32, ip+1+EQUAL_READ32-rep[0], iend) + EQUAL_READ32;
+            matchLength = ZSTD_count(ip+1+EQUAL_READ32, ip+1+EQUAL_READ32-offset_1, iend) + EQUAL_READ32;
            if (depth==0) goto _storeSequence;
        }

@@ -1780,8 +1760,8 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
        if (depth>=1)
        while (ip<ilimit) {
            ip ++;
-            if ((offset) && ((rep[0]>0) & (MEM_read32(ip) == MEM_read32(ip - rep[0])))) {
-                size_t const mlRep = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-rep[0], iend) + EQUAL_READ32;
+            if ((offset) && ((offset_1>0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) {
+                size_t const mlRep = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-offset_1, iend) + EQUAL_READ32;
                int const gain2 = (int)(mlRep * 3);
                int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offset+1) + 1);
                if ((mlRep >= EQUAL_READ32) && (gain2 > gain1))
@@ -1799,8 +1779,8 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
            /* let's find an even better one */
            if ((depth==2) && (ip<ilimit)) {
                ip ++;
-                if ((offset) && ((rep[0]>0) & (MEM_read32(ip) == MEM_read32(ip - rep[0])))) {
-                    size_t const ml2 = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-rep[0], iend) + EQUAL_READ32;
+                if ((offset) && ((offset_1>0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) {
+                    size_t const ml2 = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-offset_1, iend) + EQUAL_READ32;
                    int const gain2 = (int)(ml2 * 4);
                    int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 1);
                    if ((ml2 >= EQUAL_READ32) && (gain2 > gain1))
@@ -1821,23 +1801,23 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
        if (offset) {
            while ((start>anchor) && (start>base+offset-ZSTD_REP_MOVE) && (start[-1] == start[-1-offset+ZSTD_REP_MOVE]))   /* only search for offset within prefix */
                { start--; matchLength++; }
-            rep[1] = rep[0]; rep[0] = (U32)(offset - ZSTD_REP_MOVE);
+            offset_2 = offset_1; offset_1 = (U32)(offset - ZSTD_REP_MOVE);
        }

        /* store sequence */
 _storeSequence:
        {   size_t const litLength = start - anchor;
-            ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, matchLength-MINMATCH);
+            ZSTD_storeSeq(seqStorePtr, litLength, anchor, (U32)offset, matchLength-MINMATCH);
            anchor = ip = start + matchLength;
        }

        /* check immediate repcode */
        while ( (ip <= ilimit)
-             && ((rep[1]>0)
-             & (MEM_read32(ip) == MEM_read32(ip - rep[1])) )) {
+             && ((offset_2>0)
+             & (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) {
            /* store sequence */
-            matchLength = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-rep[1], iend) + EQUAL_READ32;
-            offset = rep[1]; rep[1] = rep[0]; rep[0] = (U32)offset; /* swap repcodes */
+            matchLength = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-offset_2, iend) + EQUAL_READ32;
+            offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap repcodes */
            ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, matchLength-MINMATCH);
            ip += matchLength;
            anchor = ip;
@@ -1845,11 +1825,8 @@ _storeSequence:
    }   }

    /* Save reps for next block */
-    {   int i;
-        for (i=0; i<ZSTD_REP_NUM; i++) {
-            if (!rep[i]) rep[i] = (U32)(iend - ctx->base);   /* in case some zero are left */
-            ctx->savedRep[i] = rep[i];
-    }   }
+    ctx->savedRep[0] = offset_1 ? offset_1 : savedOffset;
+    ctx->savedRep[1] = offset_2 ? offset_2 : savedOffset;

    /* Last Literals */
    {   size_t const lastLLSize = iend - anchor;
@@ -1908,12 +1885,10 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
                        U32 maxNbAttempts, U32 matchLengthSearch);
    searchMax_f searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS_extDict : ZSTD_HcFindBestMatch_extDict_selectMLS;

-    /* init */
-    U32 rep[ZSTD_REP_INIT];
-    { U32 i; for (i=0; i<ZSTD_REP_INIT; i++) rep[i]=ctx->rep[i]; }
+    U32 offset_1 = ctx->rep[0], offset_2 = ctx->rep[1];

+    /* init */
    ctx->nextToUpdate3 = ctx->nextToUpdate;
-    ZSTD_resetSeqStore(seqStorePtr);
    ip += (ip == prefixStart);

    /* Match Loop */
@@ -1924,7 +1899,7 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
        U32 current = (U32)(ip-base);

        /* check repCode */
-        {   const U32 repIndex = (U32)(current+1 - rep[0]);
+        {   const U32 repIndex = (U32)(current+1 - offset_1);
            const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
            const BYTE* const repMatch = repBase + repIndex;
            if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > lowestIndex))   /* intentional overflow */
@@ -1954,7 +1929,7 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
            current++;
            /* check repCode */
            if (offset) {
-                const U32 repIndex = (U32)(current - rep[0]);
+                const U32 repIndex = (U32)(current - offset_1);
                const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
                const BYTE* const repMatch = repBase + repIndex;
                if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > lowestIndex))  /* intentional overflow */
@@ -1984,7 +1959,7 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
                current++;
                /* check repCode */
                if (offset) {
-                    const U32 repIndex = (U32)(current - rep[0]);
+                    const U32 repIndex = (U32)(current - offset_1);
                    const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
                    const BYTE* const repMatch = repBase + repIndex;
                    if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > lowestIndex))  /* intentional overflow */
@@ -2016,19 +1991,19 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
            const BYTE* match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex;
            const BYTE* const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart;
            while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; }  /* catch up */
-            rep[1] = rep[0]; rep[0] = (U32)(offset - ZSTD_REP_MOVE);
+            offset_2 = offset_1; offset_1 = (U32)(offset - ZSTD_REP_MOVE);
        }

        /* store sequence */
 _storeSequence:
        {   size_t const litLength = start - anchor;
-            ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, matchLength-MINMATCH);
+            ZSTD_storeSeq(seqStorePtr, litLength, anchor, (U32)offset, matchLength-MINMATCH);
            anchor = ip = start + matchLength;
        }

        /* check immediate repcode */
        while (ip <= ilimit) {
-            const U32 repIndex = (U32)((ip-base) - rep[1]);
+            const U32 repIndex = (U32)((ip-base) - offset_2);
            const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
            const BYTE* const repMatch = repBase + repIndex;
            if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > lowestIndex))  /* intentional overflow */
@@ -2036,7 +2011,7 @@ _storeSequence:
                /* repcode detected we should take it */
                const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
                matchLength = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32;
-                offset = rep[1]; rep[1] = rep[0]; rep[0] = (U32)offset;   /* swap offset history */
+                offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset;   /* swap offset history */
                ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, matchLength-MINMATCH);
                ip += matchLength;
                anchor = ip;
@@ -2046,7 +2021,7 @@ _storeSequence:
    }   }

    /* Save reps for next block */
-    ctx->savedRep[0] = rep[0]; ctx->savedRep[1] = rep[1]; ctx->savedRep[2] = rep[2];
+    ctx->savedRep[0] = offset_1; ctx->savedRep[1] = offset_2;

    /* Last Literals */
    {   size_t const lastLLSize = iend - anchor;
@@ -2077,18 +2052,27 @@ static void ZSTD_compressBlock_btlazy2_extDict(ZSTD_CCtx* ctx, const void* src,
 }


-
 /* The optimal parser */
 #include "zstd_opt.h"

 static void ZSTD_compressBlock_btopt(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
 {
+#ifdef ZSTD_OPT_H_91842398743
    ZSTD_compressBlock_opt_generic(ctx, src, srcSize);
+#else
+    (void)ctx; (void)src; (void)srcSize;
+    return;
+#endif
 }

 static void ZSTD_compressBlock_btopt_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
 {
+#ifdef ZSTD_OPT_H_91842398743
    ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize);
+#else
+    (void)ctx; (void)src; (void)srcSize;
+    return;
+#endif
 }


@@ -2097,11 +2081,7 @@ typedef void (*ZSTD_blockCompressor) (ZSTD_CCtx* ctx, const void* src, size_t sr
 static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict)
 {
    static const ZSTD_blockCompressor blockCompressor[2][6] = {
-#if 1
        { ZSTD_compressBlock_fast, ZSTD_compressBlock_greedy, ZSTD_compressBlock_lazy, ZSTD_compressBlock_lazy2, ZSTD_compressBlock_btlazy2, ZSTD_compressBlock_btopt },
-#else
-        { ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_greedy_extDict, ZSTD_compressBlock_lazy_extDict,ZSTD_compressBlock_lazy2_extDict, ZSTD_compressBlock_btlazy2_extDict, ZSTD_compressBlock_btopt_extDict },
-#endif
        { ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_greedy_extDict, ZSTD_compressBlock_lazy_extDict,ZSTD_compressBlock_lazy2_extDict, ZSTD_compressBlock_btlazy2_extDict, ZSTD_compressBlock_btopt_extDict }
    };

@@ -2111,8 +2091,9 @@ static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int

 static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
 {
-    ZSTD_blockCompressor blockCompressor = ZSTD_selectBlockCompressor(zc->params.cParams.strategy, zc->lowLimit < zc->dictLimit);
+    ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->params.cParams.strategy, zc->lowLimit < zc->dictLimit);
    if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) return 0;   /* don't even attempt compression below a certain srcSize */
+    ZSTD_resetSeqStore(&(zc->seqStore));
    blockCompressor(zc, src, srcSize);
    return ZSTD_compressSequences(zc, dst, dstCapacity, srcSize);
 }
@@ -2245,7 +2226,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* zc,

    /* preemptive overflow correction */
    if (zc->lowLimit > (1<<30)) {
-        U32 const btplus = (zc->params.cParams.strategy == ZSTD_btlazy2) || (zc->params.cParams.strategy == ZSTD_btopt);
+        U32 const btplus = (zc->params.cParams.strategy == ZSTD_btlazy2) | (zc->params.cParams.strategy == ZSTD_btopt);
        U32 const chainMask = (1 << (zc->params.cParams.chainLog - btplus)) - 1;
        U32 const newLowLimit = zc->lowLimit & chainMask;   /* preserve position % chainSize */
        U32 const correction = zc->lowLimit - newLowLimit;
@@ -2438,9 +2419,7 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx,

 size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel)
 {
-    ZSTD_parameters params;
-    memset(&params, 0, sizeof(params));
-    params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize);
+    ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize);
    ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin_usingDict compressionLevel=%d\n", cctx->base, compressionLevel);
    return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, 0);
 }
@@ -2550,11 +2529,9 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,

 size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict, size_t dictSize, int compressionLevel)
 {
-    ZSTD_parameters params;
-    memset(&params, 0, sizeof(params));
-    ZSTD_LOG_BLOCK("%p: ZSTD_compress_usingDict srcSize=%d dictSize=%d compressionLevel=%d\n", ctx->base, (int)srcSize, (int)dictSize, compressionLevel);
-    params.cParams =  ZSTD_getCParams(compressionLevel, srcSize, dictSize);
+    ZSTD_parameters params = ZSTD_getParams(compressionLevel, srcSize, dictSize);
    params.fParams.contentSizeFlag = 1;
+    ZSTD_LOG_BLOCK("%p: ZSTD_compress_usingDict srcSize=%d dictSize=%d compressionLevel=%d\n", ctx->base, (int)srcSize, (int)dictSize, compressionLevel);
    return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params);
 }

@@ -2589,7 +2566,7 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTD_pa
    if (!customMem.customAlloc && !customMem.customFree)
        customMem = defaultCustomMem;

-    if (!customMem.customAlloc || !customMem.customFree)
+    if (!customMem.customAlloc || !customMem.customFree)  /* can't have 1/2 custom alloc/free as NULL */
        return NULL;

    {   ZSTD_CDict* const cdict = (ZSTD_CDict*) customMem.customAlloc(customMem.opaque, sizeof(*cdict));
@@ -2784,3 +2761,14 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, si
    cp = ZSTD_adjustCParams(cp, srcSize, dictSize);
    return cp;
 }
+
+/*! ZSTD_getParams() :
+*   same as ZSTD_getCParams(), but @return a `ZSTD_parameters` object instead of a `ZSTD_compressionParameters`.
+*   All fields of `ZSTD_frameParameters` are set to default (0) */
+ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSize, size_t dictSize) {
+    ZSTD_parameters params;
+    ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, srcSize, dictSize);
+    memset(&params, 0, sizeof(params));
+    params.cParams = cParams;
+    return params;
+}
@@ -34,6 +34,10 @@
 /* Note : this file is intended to be included within zstd_compress.c */


+#ifndef ZSTD_OPT_H_91842398743
+#define ZSTD_OPT_H_91842398743
+
+
 #define ZSTD_FREQ_DIV   5

 /*-*************************************
@@ -110,7 +114,7 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BY

    /* literals */
    if (ssPtr->cachedLiterals == literals) {
-        U32 additional = litLength - ssPtr->cachedLitLength;
+        U32 const additional = litLength - ssPtr->cachedLitLength;
        const BYTE* literals2 = ssPtr->cachedLiterals + ssPtr->cachedLitLength;
        price = ssPtr->cachedPrice + additional * ssPtr->log2litSum;
        for (u=0; u < additional; u++)
@@ -150,7 +154,7 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BY
 FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength)
 {
    /* offset */
-    BYTE offCode = (BYTE)ZSTD_highbit32(offset+1);
+    BYTE const offCode = (BYTE)ZSTD_highbit32(offset+1);
    U32 price = offCode + seqStorePtr->log2offCodeSum - ZSTD_highbit32(seqStorePtr->offCodeFreq[offCode]+1);

    /* match Length */
@@ -196,7 +200,7 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B
    }

    /* match offset */
-	{   BYTE offCode = (BYTE)ZSTD_highbit32(offset+1);
+	{   BYTE const offCode = (BYTE)ZSTD_highbit32(offset+1);
 		seqStorePtr->offCodeSum++;
 		seqStorePtr->offCodeFreq[offCode]++;
 	}
@@ -232,7 +236,6 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B



-
 /* Update hashTable3 up to ip (excluded)
   Assumption : always within prefix (ie. not within extDict) */
 FORCE_INLINE
@@ -465,7 +468,6 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,

    /* init */
    ctx->nextToUpdate3 = ctx->nextToUpdate;
-    ZSTD_resetSeqStore(seqStorePtr);
    ZSTD_rescaleFreqs(seqStorePtr);
    ip += (ip==prefixStart);
    { U32 i; for (i=0; i<ZSTD_REP_INIT; i++) rep[i]=ctx->rep[i]; }
@@ -757,7 +759,6 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
    { U32 i; for (i=0; i<ZSTD_REP_INIT; i++) rep[i]=ctx->rep[i]; }

    ctx->nextToUpdate3 = ctx->nextToUpdate;
-    ZSTD_resetSeqStore(seqStorePtr);
    ZSTD_rescaleFreqs(seqStorePtr);
    ip += (ip==prefixStart);

@@ -1041,3 +1042,5 @@ _storeSequence:   /* cur, last_pos, best_mlen, best_off have to be set */
        seqStorePtr->lit += lastLLSize;
    }
 }
+
+#endif  /* ZSTD_OPT_H_91842398743 */
@@ -173,7 +173,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
                    if (ZSTD_isError(hSize)) return hSize;
                    if (toLoad > (size_t)(iend-ip)) {   /* not enough input to load full header */
                        memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip);
-                        zbd->lhSize += iend-ip; ip = iend; notDone = 0;
+                        zbd->lhSize += iend-ip;
                        *dstCapacityPtr = 0;
                        return (hSize - zbd->lhSize) + ZSTD_blockHeaderSize;   /* remaining header bytes + next block header */
                    }
@@ -207,20 +207,8 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
 */


-/* Frame descriptor
+/* Frame Header :

-    // old
-   1 byte - Alloc :
-   bit 0-3 : windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN   (see zstd_internal.h)
-   bit 4   : reserved for windowLog (must be zero)
-   bit 5   : reserved (must be zero)
-   bit 6-7 : Frame content size : unknown, 1 byte, 2 bytes, 8 bytes
-
-   1 byte - checker :
-   bit 0-1 : dictID (0, 1, 2 or 4 bytes)
-   bit 2-7 : reserved (must be zero)
-
-    // new
   1 byte - FrameHeaderDescription :
   bit 0-1 : dictID (0, 1, 2 or 4 bytes)
   bit 2   : checksumFlag
@@ -454,16 +442,14 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
                          const void* src, size_t srcSize)   /* note : srcSize < BLOCKSIZE */
 {
    const BYTE* const istart = (const BYTE*) src;
-    litBlockType_t lbt;

    if (srcSize < MIN_CBLOCK_SIZE) return ERROR(corruption_detected);
-    lbt = (litBlockType_t)(istart[0]>> 6);

-    switch(lbt)
+    switch((litBlockType_t)(istart[0]>> 6))
    {
    case lbt_huffman:
        {   size_t litSize, litCSize, singleStream=0;
-            U32 lhSize = ((istart[0]) >> 4) & 3;
+            U32 lhSize = (istart[0] >> 4) & 3;
            if (srcSize < 5) return ERROR(corruption_detected);   /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for lhSize, + cSize (+nbSeq) */
            switch(lhSize)
            {
@@ -749,7 +735,7 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState)
    if (MEM_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&(seqState->DStream));

    seq.litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBits(&(seqState->DStream), llBits) : 0);   /* <=  16 bits */
-    if (MEM_32bits() ||
+    if (MEM_32bits() |
       (totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) ) BIT_reloadDStream(&(seqState->DStream));

    /* ANS state update */
@@ -765,23 +751,22 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState)
 FORCE_INLINE
 size_t ZSTD_execSequence(BYTE* op,
                                BYTE* const oend, seq_t sequence,
-                                const BYTE** litPtr, const BYTE* const litLimit_8,
+                                const BYTE** litPtr, const BYTE* const litLimit_w,
                                const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
 {
    BYTE* const oLitEnd = op + sequence.litLength;
    size_t const sequenceLength = sequence.litLength + sequence.matchLength;
    BYTE* const oMatchEnd = op + sequenceLength;   /* risk : address space overflow (32-bits) */
-    BYTE* const oend_8 = oend-8;
+    BYTE* const oend_w = oend-WILDCOPY_OVERLENGTH;
    const BYTE* const iLitEnd = *litPtr + sequence.litLength;
    const BYTE* match = oLitEnd - sequence.offset;

    /* check */
-    if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall);   /* last match must start at a minimum distance of 8 from oend */
-    if (oMatchEnd > oend) return ERROR(dstSize_tooSmall);   /* overwrite beyond dst buffer */
-    if (iLitEnd > litLimit_8) return ERROR(corruption_detected);   /* over-read beyond lit buffer */
+    if ((oLitEnd>oend_w) | (oMatchEnd>oend)) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
+    if (iLitEnd > litLimit_w) return ERROR(corruption_detected);   /* over-read beyond lit buffer */

    /* copy Literals */
-    ZSTD_wildcopy(op, *litPtr, sequence.litLength);   /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
+    ZSTD_wildcopy(op, *litPtr, sequence.litLength);   /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */
    op = oLitEnd;
    *litPtr = iLitEnd;   /* update for next sequence */

@@ -821,10 +806,10 @@ size_t ZSTD_execSequence(BYTE* op,
    op += 8; match += 8;

    if (oMatchEnd > oend-(16-MINMATCH)) {
-        if (op < oend_8) {
-            ZSTD_wildcopy(op, match, oend_8 - op);
-            match += oend_8 - op;
-            op = oend_8;
+        if (op < oend_w) {
+            ZSTD_wildcopy(op, match, oend_w - op);
+            match += oend_w - op;
+            op = oend_w;
        }
        while (op < oMatchEnd) *op++ = *match++;
    } else {
@@ -845,7 +830,7 @@ static size_t ZSTD_decompressSequences(
    BYTE* const oend = ostart + maxDstSize;
    BYTE* op = ostart;
    const BYTE* litPtr = dctx->litPtr;
-    const BYTE* const litLimit_8 = litPtr + dctx->litBufSize - 8;
+    const BYTE* const litLimit_w = litPtr + dctx->litBufSize - WILDCOPY_OVERLENGTH;
    const BYTE* const litEnd = litPtr + dctx->litSize;
    FSE_DTable* DTableLL = dctx->LLTable;
    FSE_DTable* DTableML = dctx->MLTable;
@@ -875,7 +860,7 @@ static size_t ZSTD_decompressSequences(
        for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq ; ) {
            nbSeq--;
            {   seq_t const sequence = ZSTD_decodeSequence(&seqState);
-                size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litLimit_8, base, vBase, dictEnd);
+                size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litLimit_w, base, vBase, dictEnd);
                if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
                op += oneSeqSize;
        }   }
@@ -888,8 +873,8 @@ static size_t ZSTD_decompressSequences(

    /* last literal segment */
    {   size_t const lastLLSize = litEnd - litPtr;
-        if (litPtr > litEnd) return ERROR(corruption_detected);   /* too many literals already used */
-        if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
+        //if (litPtr > litEnd) return ERROR(corruption_detected);   /* too many literals already used */
+        if (lastLLSize > (size_t)(oend-op)) return ERROR(dstSize_tooSmall);
        memcpy(op, litPtr, lastLLSize);
        op += lastLLSize;
    }
@@ -931,8 +916,11 @@ size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx,
                            void* dst, size_t dstCapacity,
                      const void* src, size_t srcSize)
 {
+    size_t dSize;
    ZSTD_checkContinuity(dctx, dst);
-    return ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize);
+    dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize);
+    dctx->previousDstEnd = (char*)dst + dSize;
+    return dSize;
 }


@@ -1180,12 +1168,13 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
 }


-static void ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
+static size_t ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
 {
    dctx->dictEnd = dctx->previousDstEnd;
    dctx->vBase = (const char*)dict - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->base));
    dctx->base = dict;
    dctx->previousDstEnd = (const char*)dict + dictSize;
+    return 0;
 }

 static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t const dictSize)
@@ -1237,29 +1226,24 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t c

 static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
 {
-    if (dictSize < 8) return ERROR(dictionary_corrupted);
+    if (dictSize < 8) return ZSTD_refDictContent(dctx, dict, dictSize);
    {   U32 const magic = MEM_readLE32(dict);
        if (magic != ZSTD_DICT_MAGIC) {
-            /* pure content mode */
-            ZSTD_refDictContent(dctx, dict, dictSize);
-            return 0;
-        }
-        dctx->dictID = MEM_readLE32((const char*)dict + 4);
+            return ZSTD_refDictContent(dctx, dict, dictSize);   /* pure content mode */
+    }   }
+    dctx->dictID = MEM_readLE32((const char*)dict + 4);

-        /* load entropy tables */
-        dict = (const char*)dict + 8;
-        dictSize -= 8;
-        {   size_t const eSize = ZSTD_loadEntropy(dctx, dict, dictSize);
-            if (ZSTD_isError(eSize)) return ERROR(dictionary_corrupted);
-            dict = (const char*)dict + eSize;
-            dictSize -= eSize;
-        }
-
-        /* reference dictionary content */
-        ZSTD_refDictContent(dctx, dict, dictSize);
-
-        return 0;
+    /* load entropy tables */
+    dict = (const char*)dict + 8;
+    dictSize -= 8;
+    {   size_t const eSize = ZSTD_loadEntropy(dctx, dict, dictSize);
+        if (ZSTD_isError(eSize)) return ERROR(dictionary_corrupted);
+        dict = (const char*)dict + eSize;
+        dictSize -= eSize;
    }
+
+    /* reference dictionary content */
+    return ZSTD_refDictContent(dctx, dict, dictSize);
 }


@@ -1319,7 +1303,7 @@ ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_cu
 }

 /*! ZSTD_createDDict() :
-*   Create a digested dictionary, ready to start decompression operation without startup delay.
+*   Create a digested dictionary, ready to start decompression without startup delay.
 *   `dict` can be released after `ZSTD_DDict` creation */
 ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize)
 {
@@ -584,48 +584,51 @@ static void ZDICT_countEStats(EStats_ress_t esr,
                            U32* countLit, U32* offsetcodeCount, U32* matchlengthCount, U32* litlengthCount, U32* repOffsets,
                            const void* src, size_t srcSize)
 {
-    const seqStore_t* seqStorePtr;
+    size_t cSize;

    if (srcSize > ZSTD_BLOCKSIZE_MAX) srcSize = ZSTD_BLOCKSIZE_MAX;   /* protection vs large samples */
    ZSTD_copyCCtx(esr.zc, esr.ref);
-    ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_MAX, src, srcSize);
-    seqStorePtr = ZSTD_getSeqStore(esr.zc);
+    cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_MAX, src, srcSize);
+    if (ZSTD_isError(cSize)) { DISPLAYLEVEL(1, "warning : could not compress sample size %u \n", (U32)srcSize); return; }

-    /* literals stats */
-    {   const BYTE* bytePtr;
-        for(bytePtr = seqStorePtr->litStart; bytePtr < seqStorePtr->lit; bytePtr++)
-            countLit[*bytePtr]++;
-    }
+    if (cSize) {  /* if == 0; block is not compressible */
+        const seqStore_t* seqStorePtr = ZSTD_getSeqStore(esr.zc);

-    /* seqStats */
-    {   size_t const nbSeq = (size_t)(seqStorePtr->offset - seqStorePtr->offsetStart);
-        ZSTD_seqToCodes(seqStorePtr, nbSeq);
-
-        {   const BYTE* codePtr = seqStorePtr->offCodeStart;
-            size_t u;
-            for (u=0; u<nbSeq; u++) offsetcodeCount[codePtr[u]]++;
+        /* literals stats */
+        {   const BYTE* bytePtr;
+            for(bytePtr = seqStorePtr->litStart; bytePtr < seqStorePtr->lit; bytePtr++)
+                countLit[*bytePtr]++;
        }

-        {   const BYTE* codePtr = seqStorePtr->mlCodeStart;
-            size_t u;
-            for (u=0; u<nbSeq; u++) matchlengthCount[codePtr[u]]++;
+        /* seqStats */
+        {   size_t const nbSeq = (size_t)(seqStorePtr->offset - seqStorePtr->offsetStart);
+            ZSTD_seqToCodes(seqStorePtr, nbSeq);
+
+            {   const BYTE* codePtr = seqStorePtr->offCodeStart;
+                size_t u;
+                for (u=0; u<nbSeq; u++) offsetcodeCount[codePtr[u]]++;
+            }
+
+            {   const BYTE* codePtr = seqStorePtr->mlCodeStart;
+                size_t u;
+                for (u=0; u<nbSeq; u++) matchlengthCount[codePtr[u]]++;
+            }
+
+            {   const BYTE* codePtr = seqStorePtr->llCodeStart;
+                size_t u;
+                for (u=0; u<nbSeq; u++) litlengthCount[codePtr[u]]++;
+        }   }
+
+        /* rep offsets */
+        {   const U32* const offsetPtr = seqStorePtr->offsetStart;
+            U32 offset1 = offsetPtr[0] - 3;
+            U32 offset2 = offsetPtr[1] - 3;
+            if (offset1 >= MAXREPOFFSET) offset1 = 0;
+            if (offset2 >= MAXREPOFFSET) offset2 = 0;
+            repOffsets[offset1] += 3;
+            repOffsets[offset2] += 1;
        }
-
-        {   const BYTE* codePtr = seqStorePtr->llCodeStart;
-            size_t u;
-            for (u=0; u<nbSeq; u++) litlengthCount[codePtr[u]]++;
-    }   }
-
-    /* rep offsets */
-    {   const U32* const offsetPtr = seqStorePtr->offsetStart;
-        U32 offset1 = offsetPtr[0] - 3;
-        U32 offset2 = offsetPtr[1] - 3;
-        if (offset1 >= MAXREPOFFSET) offset1 = 0;
-        if (offset2 >= MAXREPOFFSET) offset2 = 0;
-        repOffsets[offset1] += 3;
-        repOffsets[offset2] += 1;
    }
-
 }

 /*
@@ -823,7 +826,6 @@ static size_t ZDICT_analyzeEntropy(void*  dstBuffer, size_t maxDstSize,
    MEM_writeLE32(dstPtr+4, repStartValue[1]);
    MEM_writeLE32(dstPtr+8, repStartValue[2]);
 #endif
-    dstPtr += 12;
    eSize += 12;

 _cleanup:
@@ -903,6 +905,7 @@ size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictCo
 }

 #define DIB_MINSAMPLESSIZE (DIB_FASTSEGMENTSIZE*3)
+#define EXIT(e) { dictSize = ERROR(e); goto _cleanup; }
 /*! ZDICT_trainFromBuffer_unsafe() :
 *   `samplesBuffer` must be followed by noisy guard band.
 *   @return : size of dictionary.
@@ -920,12 +923,12 @@ size_t ZDICT_trainFromBuffer_unsafe(
    size_t dictSize = 0;

    /* checks */
-    if (maxDictSize <= g_provision_entropySize + g_min_fast_dictContent) return ERROR(dstSize_tooSmall);
+    if (maxDictSize <= g_provision_entropySize + g_min_fast_dictContent) EXIT(dstSize_tooSmall);
    if (!dictList) return ERROR(memory_allocation);

    /* init */
    { unsigned u; for (u=0, sBuffSize=0; u<nbSamples; u++) sBuffSize += samplesSizes[u]; }
-    if (sBuffSize < DIB_MINSAMPLESSIZE) return 0;   /* not enough source to create dictionary */
+    if (sBuffSize < DIB_MINSAMPLESSIZE) EXIT(no_error);   /* not enough source to create dictionary */
    ZDICT_initDictItem(dictList);
    g_displayLevel = params.notificationLevel;
    if (selectivity==0) selectivity = g_selectivity_default;
@@ -945,9 +948,9 @@ size_t ZDICT_trainFromBuffer_unsafe(
            DISPLAYLEVEL(3, "\n %u segments found, of total size %u \n", dictList[0].pos, dictContentSize);
            DISPLAYLEVEL(3, "list %u best segments \n", nb);
            for (u=1; u<=nb; u++) {
-                U32 p = dictList[u].pos;
-                U32 l = dictList[u].length;
-                U32 d = MIN(40, l);
+                U32 const p = dictList[u].pos;
+                U32 const l = dictList[u].length;
+                U32 const d = MIN(40, l);
                DISPLAYLEVEL(3, "%3u:%3u bytes at pos %8u, savings %7u bytes |",
                             u, l, p, dictList[u].savings);
                ZDICT_printHex(3, (const char*)samplesBuffer+p, d);
@@ -963,7 +966,7 @@ size_t ZDICT_trainFromBuffer_unsafe(
            for (u=1; u<dictList->pos; u++) {
                U32 l = dictList[u].length;
                ptr -= l;
-                if (ptr<(BYTE*)dictBuffer) return ERROR(GENERIC);   /* should not happen */
+                if (ptr<(BYTE*)dictBuffer) EXIT(GENERIC);   /* should not happen */
                memcpy(ptr, (const char*)samplesBuffer+dictList[u].pos, l);
        }   }

@@ -980,7 +983,7 @@ size_t ZDICT_trainFromBuffer_unsafe(
                                                             params);
    }

-    /* clean up */
+_cleanup :
    free(dictList);
    return dictSize;
 }
@@ -43,6 +43,7 @@ _*
 tmp*
 *.zst
 result
+out

 # fuzzer
 afl
@@ -50,3 +51,4 @@ afl
 # Misc files
 *.bat
 fileTests.sh
+dirTest*
@@ -42,8 +42,10 @@
 /* *************************************
 *  Constants
 ***************************************/
-#ifndef ZSTD_VERSION
-#  define ZSTD_VERSION ""
+#ifndef ZSTD_GIT_COMMIT
+#  define ZSTD_GIT_COMMIT_STRING ""
+#else
+#  define ZSTD_GIT_COMMIT_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_GIT_COMMIT)
 #endif

 #define NBLOOPS               3
@@ -146,16 +148,14 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
    size_t const maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024);   /* add some room for safety */
    void* const compressedBuffer = malloc(maxCompressedSize);
    void* const resultBuffer = malloc(srcSize);
-    ZSTD_CCtx* refCtx = ZSTD_createCCtx();
    ZSTD_CCtx* ctx = ZSTD_createCCtx();
-    ZSTD_DCtx* refDCtx = ZSTD_createDCtx();
    ZSTD_DCtx* dctx = ZSTD_createDCtx();
    U32 nbBlocks;
    UTIL_time_t ticksPerSecond;

    /* checks */
-    if (!compressedBuffer || !resultBuffer || !blockTable || !refCtx || !ctx || !refDCtx || !dctx)
-        EXM_THROW(31, "not enough memory");
+    if (!compressedBuffer || !resultBuffer || !blockTable || !ctx || !dctx)
+        EXM_THROW(31, "allocation error : not enough memory");

    /* init */
    if (strlen(displayName)>17) displayName += strlen(displayName)-17;   /* can only display 17 characters */
@@ -211,12 +211,15 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
            DISPLAYLEVEL(2, "%2i-%-17.17s :%10u ->\r", testNb, displayName, (U32)srcSize);
            memset(compressedBuffer, 0xE5, maxCompressedSize);  /* warm up and erase result buffer */

-            UTIL_sleepMilli(1); /* give processor time to other processes */
+            UTIL_sleepMilli(1);  /* give processor time to other processes */
            UTIL_waitForNextTick(ticksPerSecond);
            UTIL_getTime(&clockStart);

-            {   U32 nbLoops = 0;
-                ZSTD_CDict* cdict = ZSTD_createCDict(dictBuffer, dictBufferSize, cLevel);
+            {   size_t const refSrcSize = (nbBlocks == 1) ? srcSize : 0;
+                ZSTD_parameters const zparams = ZSTD_getParams(cLevel, refSrcSize, dictBufferSize);
+                ZSTD_customMem const cmem = { NULL, NULL, NULL };
+                U32 nbLoops = 0;
+                ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, zparams, cmem);
                if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict() allocation failure");
                do {
                    U32 blockNb;
@@ -225,7 +228,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                                            blockTable[blockNb].cPtr,  blockTable[blockNb].cRoom,
                                            blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
                                            cdict);
-                        if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_compress_usingPreparedCCtx() failed : %s", ZSTD_getErrorName(rSize));
+                        if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_compress_usingCDict() failed : %s", ZSTD_getErrorName(rSize));
                        blockTable[blockNb].cSize = rSize;
                    }
                    nbLoops++;
@@ -262,7 +265,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                            blockTable[blockNb].cPtr, blockTable[blockNb].cSize,
                            ddict);
                        if (ZSTD_isError(regenSize)) {
-                            DISPLAY("ZSTD_decompress_usingPreparedDCtx() failed on block %u : %s  \n",
+                            DISPLAY("ZSTD_decompress_usingDDict() failed on block %u : %s  \n",
                                      blockNb, ZSTD_getErrorName(regenSize));
                            clockLoop = 0;   /* force immediate test end */
                            break;
@@ -319,9 +322,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
    free(blockTable);
    free(compressedBuffer);
    free(resultBuffer);
-    ZSTD_freeCCtx(refCtx);
    ZSTD_freeCCtx(ctx);
-    ZSTD_freeDCtx(refDCtx);
    ZSTD_freeDCtx(dctx);
    return 0;
 }
@@ -363,7 +364,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
    memset(&total, 0, sizeof(total));

    if (g_displayLevel == 1 && !g_additionalParam)
-        DISPLAY("bench %s: input %u bytes, %i iterations, %u KB blocks\n", ZSTD_VERSION, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10));
+        DISPLAY("bench %s %s: input %u bytes, %i iterations, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10));

    if (cLevelLast < cLevel) cLevelLast = cLevel;

@@ -23,12 +23,19 @@
    - source repository : https://github.com/Cyan4973/zstd
 */

+/* *************************************
+*  Compiler Options
+***************************************/
+#define _CRT_SECURE_NO_WARNINGS  /* removes Visual warning on strerror() */
+
+
 /*-************************************
 *  Includes
 **************************************/
 #include <stdlib.h>    /* malloc */
 #include <stdio.h>     /* FILE, fwrite, fprintf */
 #include <string.h>    /* memcpy */
+#include <errno.h>     /* errno */
 #include "mem.h"       /* U32 */


@@ -104,7 +111,7 @@ static BYTE RDG_genChar(U32* seed, const BYTE* ldt)
    U32 const id = RDG_rand(seed) & LTMASK;
    //TRACE(" %u : \n", id);
    //TRACE(" %4u [%4u] ; val : %4u \n", id, id&255, ldt[id]);
-    return (ldt[id]);  /* memory-sanitizer fails here, stating "uninitialized value" when table initialized with 0.0. Checked : table is fully initialized */
+    return ldt[id];  /* memory-sanitizer fails here, stating "uninitialized value" when table initialized with P==0.0. Checked : table is fully initialized */
 }


@@ -115,8 +122,7 @@ static U32 RDG_rand15Bits (unsigned* seedPtr)

 static U32 RDG_randLength(unsigned* seedPtr)
 {
-    if (RDG_rand(seedPtr) & 7)
-        return (RDG_rand(seedPtr) & 0xF);
+    if (RDG_rand(seedPtr) & 7) return (RDG_rand(seedPtr) & 0xF);   /* small length */
    return (RDG_rand(seedPtr) & 0x1FF) + 0xF;
 }

@@ -185,10 +191,10 @@ void RDG_genStdout(unsigned long long size, double matchProba, double litProba,
    size_t const stdDictSize = 32 KB;
    BYTE* const buff = (BYTE*)malloc(stdDictSize + stdBlockSize);
    U64 total = 0;
-    BYTE ldt[LTSIZE];
+    BYTE ldt[LTSIZE];   /* literals distribution table */

    /* init */
-    if (buff==NULL) { fprintf(stdout, "not enough memory\n"); exit(1); }
+    if (buff==NULL) { fprintf(stderr, "datagen: error: %s \n", strerror(errno)); exit(1); }
    if (litProba<=0.0) litProba = matchProba / 4.5;
    memset(ldt, '0', sizeof(ldt));
    RDG_fillLiteralDistrib(ldt, litProba);
@@ -41,13 +41,14 @@
 /* *************************************
 *  Compiler Options
 ***************************************/
-#define _POSIX_SOURCE 1        /* enable %llu on Windows */
+#define _POSIX_SOURCE 1          /* enable %llu on Windows */
+#define _CRT_SECURE_NO_WARNINGS  /* removes Visual warning on strerror() */


 /*-*************************************
 *  Includes
 ***************************************/
-#include "util.h"       /* Compiler options, UTIL_GetFileSize */
+#include "util.h"       /* Compiler options, UTIL_GetFileSize, _LARGEFILE64_SOURCE */
 #include <stdio.h>      /* fprintf, fopen, fread, _fileno, stdin, stdout */
 #include <stdlib.h>     /* malloc, free */
 #include <string.h>     /* strcmp, strlen */
@@ -58,7 +59,6 @@
 #include "fileio.h"
 #define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */
 #include "zstd.h"
-#include "zstd_internal.h" /* MIN, KB, MB */
 #define ZBUFF_STATIC_LINKING_ONLY
 #include "zbuff.h"

@@ -84,6 +84,10 @@
 /*-*************************************
 *  Constants
 ***************************************/
+#define KB *(1<<10)
+#define MB *(1<<20)
+#define GB *(1U<<30)
+
 #define _1BIT  0x01
 #define _2BITS 0x03
 #define _3BITS 0x07
@@ -113,21 +117,17 @@ static U32 g_displayLevel = 2;   /* 0 : no display;   1: errors;   2 : + result
 void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; }

 #define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
-            if ((FIO_GetMilliSpan(g_time) > refreshRate) || (g_displayLevel>=4)) \
+            if ((clock() - g_time > refreshRate) || (g_displayLevel>=4)) \
            { g_time = clock(); DISPLAY(__VA_ARGS__); \
            if (g_displayLevel>=4) fflush(stdout); } }
-static const unsigned refreshRate = 150;
+static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
 static clock_t g_time = 0;

-static unsigned FIO_GetMilliSpan(clock_t nPrevious)
-{
-    clock_t const nCurrent = clock();
-    return (unsigned)(((nCurrent - nPrevious) * 1000) / CLOCKS_PER_SEC);
-}
+#define MIN(a,b)    ((a) < (b) ? (a) : (b))


 /*-*************************************
-*  Local Parameters
+*  Local Parameters - Not thread safe
 ***************************************/
 static U32 g_overwrite = 0;
 void FIO_overwriteMode(void) { g_overwrite=1; }
@@ -137,7 +137,7 @@ static U32 g_sparseFileSupport = 1;   /* 0 : no sparse allowed; 1: auto (file ye
 void FIO_setSparseWrite(unsigned sparse) { g_sparseFileSupport=sparse; }
 static U32 g_dictIDFlag = 1;
 void FIO_setDictIDFlag(unsigned dictIDFlag) { g_dictIDFlag = dictIDFlag; }
-static U32 g_checksumFlag = 0;
+static U32 g_checksumFlag = 1;
 void FIO_setChecksumFlag(unsigned checksumFlag) { g_checksumFlag = checksumFlag; }
 static U32 g_removeSrcFile = 0;
 void FIO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); }
@@ -175,7 +175,7 @@ static FILE* FIO_openSrcFile(const char* srcFileName)
        f = fopen(srcFileName, "rb");
    }

-    if ( f==NULL ) DISPLAYLEVEL(1, "zstd: %s: No such file\n", srcFileName);
+    if ( f==NULL ) DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno));

    return f;
 }
@@ -201,18 +201,20 @@ static FILE* FIO_openDstFile(const char* dstFileName)
                if (g_displayLevel <= 1) {
                    /* No interaction possible */
                    DISPLAY("zstd: %s already exists; not overwritten  \n", dstFileName);
-                    return 0;
+                    return NULL;
                }
                DISPLAY("zstd: %s already exists; do you wish to overwrite (y/N) ? ", dstFileName);
                {   int ch = getchar();
                    if ((ch!='Y') && (ch!='y')) {
                        DISPLAY("    not overwritten  \n");
-                        return 0;
+                        return NULL;
                    }
                    while ((ch!=EOF) && (ch!='\n')) ch = getchar();  /* flush rest of input line */
        }   }   }
        f = fopen( dstFileName, "wb" );
    }
+
+    if (f==NULL) DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno));
    return f;
 }

@@ -233,18 +235,18 @@ static size_t FIO_loadFile(void** bufferPtr, const char* fileName)

    DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName);
    fileHandle = fopen(fileName, "rb");
-    if (fileHandle==0) EXM_THROW(31, "Error opening file %s", fileName);
+    if (fileHandle==0) EXM_THROW(31, "zstd: %s: %s", fileName, strerror(errno));
    fileSize = UTIL_getFileSize(fileName);
    if (fileSize > MAX_DICT_SIZE) {
        int seekResult;
        if (fileSize > 1 GB) EXM_THROW(32, "Dictionary file %s is too large", fileName);   /* avoid extreme cases */
        DISPLAYLEVEL(2,"Dictionary %s is too large : using last %u bytes only \n", fileName, MAX_DICT_SIZE);
        seekResult = fseek(fileHandle, (long int)(fileSize-MAX_DICT_SIZE), SEEK_SET);   /* use end of file */
-        if (seekResult != 0) EXM_THROW(33, "Error seeking into file %s", fileName);
+        if (seekResult != 0) EXM_THROW(33, "zstd: %s: %s", fileName, strerror(errno));
        fileSize = MAX_DICT_SIZE;
    }
-    *bufferPtr = (BYTE*)malloc((size_t)fileSize);
-    if (*bufferPtr==NULL) EXM_THROW(34, "Allocation error : not enough memory for dictBuffer");
+    *bufferPtr = malloc((size_t)fileSize);
+    if (*bufferPtr==NULL) EXM_THROW(34, "zstd: %s", strerror(errno));
    { size_t const readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle);
      if (readSize!=fileSize) EXM_THROW(35, "Error reading dictionary file %s", fileName); }
    fclose(fileHandle);
@@ -273,14 +275,14 @@ static cRess_t FIO_createCResources(const char* dictFileName)
    cRess_t ress;

    ress.ctx = ZBUFF_createCCtx();
-    if (ress.ctx == NULL) EXM_THROW(30, "Allocation error : can't create ZBUFF context");
+    if (ress.ctx == NULL) EXM_THROW(30, "zstd: allocation error : can't create ZBUFF context");

    /* Allocate Memory */
    ress.srcBufferSize = ZBUFF_recommendedCInSize();
    ress.srcBuffer = malloc(ress.srcBufferSize);
    ress.dstBufferSize = ZBUFF_recommendedCOutSize();
    ress.dstBuffer = malloc(ress.dstBufferSize);
-    if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(31, "Allocation error : not enough memory");
+    if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(31, "zstd: allocation error : not enough memory");

    /* dictionary */
    ress.dictBufferSize = FIO_loadFile(&(ress.dictBuffer), dictFileName);
@@ -295,7 +297,7 @@ static void FIO_freeCResources(cRess_t ress)
    free(ress.dstBuffer);
    free(ress.dictBuffer);
    errorCode = ZBUFF_freeCCtx(ress.ctx);
-    if (ZBUFF_isError(errorCode)) EXM_THROW(38, "Error : can't release ZBUFF context resource : %s", ZBUFF_getErrorName(errorCode));
+    if (ZBUFF_isError(errorCode)) EXM_THROW(38, "zstd: error : can't release ZBUFF context resource : %s", ZBUFF_getErrorName(errorCode));
 }


@@ -315,9 +317,7 @@ static int FIO_compressFilename_internal(cRess_t ress,
    U64 const fileSize = UTIL_getFileSize(srcFileName);

    /* init */
-    {   ZSTD_parameters params;
-        memset(&params, 0, sizeof(params));
-        params.cParams = ZSTD_getCParams(cLevel, fileSize, ress.dictBufferSize);
+    {   ZSTD_parameters params = ZSTD_getParams(cLevel, fileSize, ress.dictBufferSize);
        params.fParams.contentSizeFlag = 1;
        params.fParams.checksumFlag = g_checksumFlag;
        params.fParams.noDictIDFlag = !g_dictIDFlag;
@@ -375,8 +375,8 @@ static int FIO_compressFilename_internal(cRess_t ress,
 }


-/*! FIO_compressFilename_internal() :
- *  same as FIO_compressFilename_extRess(), with ress.destFile already opened (typically stdout)
+/*! FIO_compressFilename_srcFile() :
+ *  note : ress.destFile already opened
 *  @return : 0 : compression completed correctly,
 *            1 : missing or pb opening srcFileName
 */
@@ -413,11 +413,11 @@ static int FIO_compressFilename_dstFile(cRess_t ress,
    int result;

    ress.dstFile = FIO_openDstFile(dstFileName);
-    if (ress.dstFile==0) { fclose(ress.srcFile); return 1; }
+    if (ress.dstFile==0) return 1;

    result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName, cLevel);

-    if (fclose(ress.dstFile)) EXM_THROW(28, "Write error : cannot properly close %s", dstFileName);
+    if (fclose(ress.dstFile)) { DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); result=1; }
    if (result!=0) remove(dstFileName);   /* remove operation artefact */
    return result;
 }
@@ -429,13 +429,13 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
    clock_t const start = clock();

    cRess_t const ress = FIO_createCResources(dictFileName);
-    int const issueWithSrcFile = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName, compressionLevel);
-    FIO_freeCResources(ress);
+    int const result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName, compressionLevel);

-    {   double const seconds = (double)(clock() - start) / CLOCKS_PER_SEC;
-        DISPLAYLEVEL(4, "Completed in %.2f sec \n", seconds);
-    }
-    return issueWithSrcFile;
+    double const seconds = (double)(clock() - start) / CLOCKS_PER_SEC;
+    DISPLAYLEVEL(4, "Completed in %.2f sec \n", seconds);
+
+    FIO_freeCResources(ress);
+    return result;
 }


@@ -40,7 +40,7 @@
 #include <sys/timeb.h>   /* timeb */
 #include <string.h>      /* strcmp */
 #include <time.h>        /* clock_t */
-#define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_compressContinue */
+#define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_compressContinue, ZSTD_compressBlock */
 #include "zstd.h"        /* ZSTD_VERSION_STRING, ZSTD_getErrorCode */
 #include "zdict.h"       /* ZDICT_trainFromBuffer */
 #include "datagen.h"     /* RDG_genBuffer */
@@ -109,9 +109,9 @@ static unsigned FUZ_highbit32(U32 v32)
 }


-#define CHECKTEST(var, fn)  size_t const var = fn; if (ZSTD_isError(var)) goto _output_error
-#define CHECK(fn)  { CHECKTEST(err, fn); }
-#define CHECKPLUS(var, fn, more)  { CHECKTEST(var, fn); more; }
+#define CHECK_V(var, fn)  size_t const var = fn; if (ZSTD_isError(var)) goto _output_error
+#define CHECK(fn)  { CHECK_V(err, fn); }
+#define CHECKPLUS(var, fn, more)  { CHECK_V(var, fn); more; }
 static int basicUnitTests(U32 seed, double compressibility)
 {
    size_t const CNBuffSize = 5 MB;
@@ -216,10 +216,8 @@ static int basicUnitTests(U32 seed, double compressibility)

        DISPLAYLEVEL(4, "test%3i : check content size on duplicated context : ", testNb++);
        {   size_t const testSize = CNBuffSize / 3;
-            {   ZSTD_compressionParameters const cPar = ZSTD_getCParams(2, testSize, dictSize);
-                ZSTD_frameParameters const fPar = { 1 , 0 , 0 };
-                ZSTD_parameters p;
-                p.cParams = cPar; p.fParams = fPar;
+            {   ZSTD_parameters p = ZSTD_getParams(2, testSize, dictSize);
+                p.fParams.contentSizeFlag = 1;
                CHECK( ZSTD_compressBegin_advanced(ctxOrig, CNBuffer, dictSize, p, testSize-1) );
            }
            CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig) );
@@ -277,10 +275,8 @@ static int basicUnitTests(U32 seed, double compressibility)
        DISPLAYLEVEL(4, "OK \n");

        DISPLAYLEVEL(4, "test%3i : compress without dictID : ", testNb++);
-        {   ZSTD_frameParameters const fParams = { 0 /*contentSize*/, 0 /*checksum*/, 1 /*NoDictID*/ };
-            ZSTD_compressionParameters const cParams = ZSTD_getCParams(3, CNBuffSize, dictSize);
-            ZSTD_parameters p;
-            p.cParams = cParams; p.fParams = fParams;
+        {   ZSTD_parameters p = ZSTD_getParams(3, CNBuffSize, dictSize);
+            p.fParams.noDictIDFlag = 1;
            cSize = ZSTD_compress_advanced(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize),
                                           CNBuffer, CNBuffSize,
                                           dictBuffer, dictSize, p);
@@ -320,6 +316,7 @@ static int basicUnitTests(U32 seed, double compressibility)
        ZSTD_DCtx* const dctx = ZSTD_createDCtx();
        static const size_t blockSize = 100 KB;
        static const size_t dictSize = 16 KB;
+        size_t cSize2;

        /* basic block compression */
        DISPLAYLEVEL(4, "test%3i : Block compression test : ", testNb++);
@@ -330,7 +327,7 @@ static int basicUnitTests(U32 seed, double compressibility)

        DISPLAYLEVEL(4, "test%3i : Block decompression test : ", testNb++);
        CHECK( ZSTD_decompressBegin(dctx) );
-        { CHECKTEST(r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
+        { CHECK_V(r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
          if (r != blockSize) goto _output_error; }
        DISPLAYLEVEL(4, "OK \n");

@@ -339,11 +336,15 @@ static int basicUnitTests(U32 seed, double compressibility)
        CHECK( ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5) );
        cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize);
        if (ZSTD_isError(cSize)) goto _output_error;
+        cSize2 = ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize+blockSize, blockSize);
+        if (ZSTD_isError(cSize2)) goto _output_error;
        DISPLAYLEVEL(4, "OK \n");

        DISPLAYLEVEL(4, "test%3i : Dictionary Block decompression test : ", testNb++);
        CHECK( ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize) );
-        { CHECKTEST( r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
+        { CHECK_V( r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
+          if (r != blockSize) goto _output_error; }
+        { CHECK_V( r, ZSTD_decompressBlock(dctx, (char*)decodedBuffer+blockSize, CNBuffSize, (char*)compressedBuffer+cSize, cSize2) );
          if (r != blockSize) goto _output_error; }
        DISPLAYLEVEL(4, "OK \n");

@@ -361,7 +362,7 @@ static int basicUnitTests(U32 seed, double compressibility)
        sampleSize += 96 KB;
        cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1);
        if (ZSTD_isError(cSize)) goto _output_error;
-        { CHECKTEST(regenSize, ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize));
+        { CHECK_V(regenSize, ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize));
          if (regenSize!=sampleSize) goto _output_error; }
        DISPLAYLEVEL(4, "OK \n");
    }
@@ -370,12 +371,12 @@ static int basicUnitTests(U32 seed, double compressibility)
    #define ZEROESLENGTH 100
    DISPLAYLEVEL(4, "test%3i : compress %u zeroes : ", testNb++, ZEROESLENGTH);
    memset(CNBuffer, 0, ZEROESLENGTH);
-    { CHECKTEST(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(ZEROESLENGTH), CNBuffer, ZEROESLENGTH, 1) );
+    { CHECK_V(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(ZEROESLENGTH), CNBuffer, ZEROESLENGTH, 1) );
      cSize = r; }
    DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/ZEROESLENGTH*100);

    DISPLAYLEVEL(4, "test%3i : decompress %u zeroes : ", testNb++, ZEROESLENGTH);
-    { CHECKTEST(r, ZSTD_decompress(decodedBuffer, ZEROESLENGTH, compressedBuffer, cSize) );
+    { CHECK_V(r, ZSTD_decompress(decodedBuffer, ZEROESLENGTH, compressedBuffer, cSize) );
      if (r != ZEROESLENGTH) goto _output_error; }
    DISPLAYLEVEL(4, "OK \n");

@@ -403,13 +404,13 @@ static int basicUnitTests(U32 seed, double compressibility)
            ((BYTE*)CNBuffer)[i+2] = _3BytesSeqs[id][2];
    }   }}
    DISPLAYLEVEL(4, "test%3i : compress lots 3-bytes sequences : ", testNb++);
-    { CHECKTEST(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(_3BYTESTESTLENGTH),
+    { CHECK_V(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(_3BYTESTESTLENGTH),
                                 CNBuffer, _3BYTESTESTLENGTH, 19) );
      cSize = r; }
    DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/_3BYTESTESTLENGTH*100);

    DISPLAYLEVEL(4, "test%3i : decompress lots 3-bytes sequence : ", testNb++);
-    { CHECKTEST(r, ZSTD_decompress(decodedBuffer, _3BYTESTESTLENGTH, compressedBuffer, cSize) );
+    { CHECK_V(r, ZSTD_decompress(decodedBuffer, _3BYTESTESTLENGTH, compressedBuffer, cSize) );
      if (r != _3BYTESTESTLENGTH) goto _output_error; }
    DISPLAYLEVEL(4, "OK \n");

@@ -22,10 +22,12 @@ roundTripTest() {

 isWindows=false
 ECHO="echo"
+INTOVOID="/dev/null"
 case "$OS" in
  Windows*)
    isWindows=true
    ECHO="echo -e"
+    INTOVOID="nul"
    ;;
 esac

@@ -53,19 +55,21 @@ $ECHO "test : null-length file roundtrip"
 $ECHO -n '' | $ZSTD - --stdout | $ZSTD -d --stdout
 $ECHO "test : decompress file with wrong suffix (must fail)"
 $ZSTD -d tmpCompressed && die "wrong suffix error not detected!"
+$ZSTD -df tmp && die "should have refused : wrong extension"
+$ECHO "test : decompress into stdout"
 $ZSTD -d tmpCompressed -c > tmpResult    # decompression using stdout
 $ZSTD --decompress tmpCompressed -c > tmpResult
 $ZSTD --decompress tmpCompressed --stdout > tmpResult
-if [ "$isWindows" = false ] ; then
-    $ZSTD -d    < tmp.zst > /dev/null    # combine decompression, stdin & stdout
-    $ZSTD -d  - < tmp.zst > /dev/null
-fi
-$ZSTD -dc   < tmp.zst > /dev/null
-$ZSTD -dc - < tmp.zst > /dev/null
+$ECHO "test : decompress from stdin into stdout"
+$ZSTD -dc   < tmp.zst > $INTOVOID   # combine decompression, stdin & stdout
+$ZSTD -dc - < tmp.zst > $INTOVOID
+$ZSTD -d    < tmp.zst > $INTOVOID   # implicit stdout when stdin is used
+$ZSTD -d  - < tmp.zst > $INTOVOID
+$ECHO "test : overwrite protection"
 $ZSTD -q tmp && die "overwrite check failed!"
+$ECHO "test : force overwrite"
 $ZSTD -q -f tmp
 $ZSTD -q --force tmp
-$ZSTD -df tmp && die "should have refused : wrong extension"
 $ECHO "test : file removal"
 $ZSTD -f --rm tmp
 ls tmp && die "tmp should no longer be present"
@@ -135,9 +139,9 @@ rm tmpSparse*

 $ECHO "\n**** multiple files tests **** "

-./datagen -s1        > tmp1 2> /dev/null
-./datagen -s2 -g100K > tmp2 2> /dev/null
-./datagen -s3 -g1M   > tmp3 2> /dev/null
+./datagen -s1        > tmp1 2> $INTOVOID
+./datagen -s2 -g100K > tmp2 2> $INTOVOID
+./datagen -s3 -g1M   > tmp3 2> $INTOVOID
 $ZSTD -f tmp*
 $ECHO "compress tmp* : "
 ls -ls tmp*
@@ -36,8 +36,14 @@
 #include <stdio.h>      /* fprintf */
 #include <sys/types.h>  /* stat */
 #include <sys/stat.h>   /* stat */
+#include "xxhash.h"
 #include "zstd.h"

+/*===========================================
+*   Macros
+*==========================================*/
+#define MIN(a,b)  ( (a) < (b) ? (a) : (b) )
+
 /** roundTripTest() :
 *   Compresses `srcBuff` into `compressedBuff`,
 *   then decompresses `compressedBuff` into `resultBuff`.
@@ -51,7 +57,9 @@ static size_t roundTripTest(void* resultBuff, size_t resultBuffCapacity,
                      const void* srcBuff, size_t srcBuffSize)
 {
    static const int maxClevel = 19;
-    int const cLevel = (!srcBuffSize) ? 1 : (*(const unsigned char*)srcBuff) % maxClevel;
+    size_t const hashLength = MIN(128, srcBuffSize);
+    unsigned const h32 = XXH32(srcBuff, hashLength, 0);
+    int const cLevel = h32 % maxClevel;
    size_t const cSize = ZSTD_compress(compressedBuff, compressedBuffCapacity, srcBuff, srcBuffSize, cLevel);
    if (ZSTD_isError(cSize)) {
        fprintf(stderr, "Compression error : %s \n", ZSTD_getErrorName(cSize));
@@ -381,13 +381,9 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
            {   size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
                dict = srcBuffer + dictStart;
            }
-            {   ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, 0, dictSize);
-                U32 const checksum = FUZ_rand(&lseed) & 1;
-                U32 const noDictIDFlag = FUZ_rand(&lseed) & 1;
-                ZSTD_frameParameters const fPar = { 0, checksum, noDictIDFlag };
-                ZSTD_parameters params;
-                params.cParams = cPar;
-                params.fParams = fPar;
+            {   ZSTD_parameters params = ZSTD_getParams(cLevel, 0, dictSize);
+                params.fParams.checksumFlag = FUZ_rand(&lseed) & 1;
+                params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1;
                {   size_t const initError = ZBUFF_compressInit_advanced(zc, dict, dictSize, params, 0);
                    CHECK (ZBUFF_isError(initError),"init error : %s", ZBUFF_getErrorName(initError));
        }   }   }
@@ -421,11 +417,17 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
        }   }

        /* final frame epilogue */
-        {   size_t dstBuffSize = cBufferSize - cSize;
-            size_t const flushError = ZBUFF_compressEnd(zc, cBuffer+cSize, &dstBuffSize);
-            CHECK (ZBUFF_isError(flushError), "flush error : %s", ZBUFF_getErrorName(flushError));
-            cSize += dstBuffSize;
-        }
+        {   size_t remainingToFlush = (size_t)(-1);
+            while (remainingToFlush) {
+                size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
+                size_t dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
+                U32 const enoughDstSize = dstBuffSize >= remainingToFlush;
+                remainingToFlush = ZBUFF_compressEnd(zc, cBuffer+cSize, &dstBuffSize);
+                CHECK (ZBUFF_isError(remainingToFlush), "flush error : %s", ZBUFF_getErrorName(remainingToFlush));
+                //DISPLAY("flush %u bytes : still within context : %i \n", (U32)dstBuffSize, (int)remainingToFlush);
+                CHECK (enoughDstSize && remainingToFlush, "ZBUFF_compressEnd() not fully flushed, but enough space available");
+                cSize += dstBuffSize;
+        }   }
        crcOrig = XXH64_digest(&xxhState);

        /* multi - fragments decompression test */
@@ -33,7 +33,8 @@ It is based on the \fBLZ77\fR family, with further FSE & huff0 entropy stages.
 It also features a very fast decoder, with speed > 500 MB/s per core.

 \fBzstd\fR command line is generally similar to gzip, but features the following differences :
- - Original files are preserved
+ - Source files are preserved by default
+   It's possible to remove them automatically by using \fB--rm\fR command
 - By default, when compressing a single file, \fBzstd\fR displays progress notifications and result summary.
     Use \fB-q\fR to turn them off

@@ -57,6 +58,19 @@ It also features a very fast decoder, with speed > 500 MB/s per core.
 .BR \-f ", " --force
 overwrite output without prompting
 .TP
+.BR \-c ", " --stdout
+ force write to standard output, even if it is the console
+.TP
+.BR \--rm
+ remove source file(s) after successful compression or decompression
+.TP
+.BR \-k ", " --keep
+ keep source file(s) after successful compression or decompression.
+ This is the default behavior.
+.TP
+.BR \-r
+ operate recursively on directories
+.TP
 .BR \-h/\-H ", " --help
 display help/long help and exit
 .TP
@@ -69,14 +83,11 @@ It also features a very fast decoder, with speed > 500 MB/s per core.
 .BR \-q ", " --quiet
 suppress warnings and notifications; specify twice to suppress errors too
 .TP
-.BR \-c ", " --stdout
- force write to standard output, even if it is the console
-.TP
 .BR \-C ", " --check
 add integrity check computed from uncompressed data
 .TP
 .BR \-t ", " --test
- Test the integrity of compressed files.  This option is equivalent to \fB--decompress --stdout > /dev/null\fR.
+ Test the integrity of compressed files. This option is equivalent to \fB--decompress --stdout > /dev/null\fR.
 No files are created or removed.

 .SH DICTIONARY
@@ -115,6 +115,7 @@ static int usage(const char* programName)
    DISPLAY( " -D file: use `file` as Dictionary \n");
    DISPLAY( " -o file: result stored into `file` (only if 1 input file) \n");
    DISPLAY( " -f     : overwrite output without prompting \n");
+    DISPLAY( "--rm    : remove source file(s) after successful de/compression \n");
    DISPLAY( " -h/-H  : display help/long help and exit\n");
    return 0;
 }
@@ -132,11 +133,10 @@ static int usage_advanced(const char* programName)
 #ifdef UTIL_HAS_CREATEFILELIST
    DISPLAY( " -r     : operate recursively on directories\n");
 #endif
-    DISPLAY( "--rm    : remove source files after successful de/compression \n");
 #ifndef ZSTD_NOCOMPRESS
    DISPLAY( "--ultra : enable ultra modes (requires more memory to decompress)\n");
    DISPLAY( "--no-dictID : don't write dictID into header (dictionary compression)\n");
-    DISPLAY( "--check : enable integrity check\n");
+    DISPLAY( "--[no-]check : integrity check (default:enabled)\n");
 #endif
 #ifndef ZSTD_NODECOMPRESS
    DISPLAY( "--test  : test compressed file integrity \n");
@@ -181,7 +181,7 @@ static void waitEnter(void)
 /*! readU32FromChar() :
    @return : unsigned integer value reach from input in `char` format
    Will also modify `*stringPtr`, advancing it to position where it stopped reading.
-    Note : this function can overflow if result > MAX_UNIT */
+    Note : this function can overflow if result > MAX_UINT */
 static unsigned readU32FromChar(const char** stringPtr)
 {
    unsigned result = 0;
@@ -254,9 +254,10 @@ int main(int argCount, const char** argv)
        if (!strcmp(argument, "--help")) { displayOut=stdout; CLEAN_RETURN(usage_advanced(programName)); }
        if (!strcmp(argument, "--verbose")) { displayLevel=4; continue; }
        if (!strcmp(argument, "--quiet")) { displayLevel--; continue; }
-        if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; displayLevel=1; continue; }
+        if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; displayLevel-=(displayLevel==2); continue; }
        if (!strcmp(argument, "--ultra")) { FIO_setMaxWLog(0); continue; }
        if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(2); continue; }
+        if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(0); continue; }
        if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; }
        if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; }
        if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; }
@@ -264,13 +265,17 @@ int main(int argCount, const char** argv)
        if (!strcmp(argument, "--train")) { dictBuild=1; outFileName=g_defaultDictName; continue; }
        if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; continue; }
        if (!strcmp(argument, "--dictID")) { nextArgumentIsDictID=1; continue; }
-        if (!strcmp(argument, "--keep")) { continue; }   /* does nothing, since preserving input is default; for gzip/xz compatibility */
+        if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; }
        if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; }

        /* '-' means stdin/stdout */
        if (!strcmp(argument, "-")){
-            if (!filenameIdx) { filenameIdx=1, filenameTable[0]=stdinmark; outFileName=stdoutmark; continue; }
-        }
+            if (!filenameIdx) {
+                filenameIdx=1, filenameTable[0]=stdinmark;
+                outFileName=stdoutmark;
+                displayLevel-=(displayLevel==2);
+                continue;
+        }   }

        /* Decode commands (note : aggregated commands are allowed) */
        if (argument[0]=='-') {
@@ -299,7 +304,7 @@ int main(int argCount, const char** argv)
                case 'd': decode=1; argument++; break;

                    /* Force stdout, even if stdout==console */
-                case 'c': forceStdout=1; outFileName=stdoutmark; displayLevel=1; argument++; break;
+                case 'c': forceStdout=1; outFileName=stdoutmark; displayLevel-=(displayLevel==2); argument++; break;

                    /* Use file content as dictionary */
                case 'D': nextEntryIsDictionary = 1; argument++; break;
@@ -313,8 +318,8 @@ int main(int argCount, const char** argv)
                    /* Quiet mode */
                case 'q': displayLevel--; argument++; break;

-                    /* keep source file (default anyway, so useless; for gzip/xz compatibility) */
-                case 'k': argument++; break;
+                    /* keep source file (default); for gzip/xz compatibility */
+                case 'k': FIO_setRemoveSrcFile(0); argument++; break;

                    /* Checksum */
                case 'C': argument++; FIO_setChecksumFlag(2); break;
@@ -1,4 +1,4 @@
-projects for various integrated development environments (IDE) 
+projects for various integrated development environments (IDE)
 ================================

 #### Included projects
@@ -7,3 +7,4 @@ The following projects are included with the zstd distribution:
 - cmake - CMake project contributed by Artyom Dymchenko
 - VS2008 - Visual Studio 2008 project
 - VS2010 - Visual Studio 2010 project (which also works well with Visual Studio 2012, 2013, 2015)
+- build - command line scripts prepared for Visual Studio compilation without IDE
@@ -1,37 +1,38 @@
 #! /usr/bin/env python
-# execute(), fetch(), notify() are based on https://github.com/getlantern/build-automation/blob/master/build.py

 import argparse
 import os
 import string
 import time
 import traceback
-from subprocess import Popen, PIPE
+import subprocess
+import signal
+ 

-repo_url = 'https://github.com/Cyan4973/zstd.git'
-test_dir_name = 'speedTest'
+default_repo_url = 'https://github.com/Cyan4973/zstd.git'
+working_dir_name = 'speedTest'
+working_path = os.getcwd() + '/' + working_dir_name     # /path/to/zstd/tests/speedTest 
+clone_path = working_path + '/' + 'zstd'                # /path/to/zstd/tests/speedTest/zstd 
+email_header = '[ZSTD_speedTest]'
+pid = str(os.getpid())


 def log(text):
-    print time.strftime("%Y/%m/%d %H:%M:%S") + ' - ' + text
+    print(time.strftime("%Y/%m/%d %H:%M:%S") + ' - ' + text)

-def execute(command, print_output=False, print_error=True):
+
+def execute(command, print_output=False, print_error=True, param_shell=True):
    log("> " + command)
-    popen = Popen(command, stdout=PIPE, stderr=PIPE, shell=True, cwd=execute.cwd)
-    itout = iter(popen.stdout.readline, b"")
-    iterr = iter(popen.stderr.readline, b"")
-    stdout_lines = list(itout)
+    popen = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=param_shell, cwd=execute.cwd)
+    stdout = popen.communicate()[0]
+    stdout_lines = stdout.splitlines()
    if print_output:
-        print ''.join(stdout_lines)
-    stderr_lines = list(iterr)
-    if print_output:
-        print ''.join(stderr_lines)
-    popen.communicate()
+        print('\n'.join(stdout_lines))
    if popen.returncode is not None and popen.returncode != 0:
        if not print_output and print_error:
-            print ''.join(stderr_lines)    
-        raise RuntimeError(''.join(stderr_lines))
-    return stdout_lines + stderr_lines
+            print('\n'.join(stdout_lines))
+        raise RuntimeError('\n'.join(stdout_lines))
+    return stdout_lines
 execute.cwd = None


@@ -43,39 +44,51 @@ def does_command_exist(command):
    return True


-def fetch():
+def send_email(emails, topic, text, have_mutt, have_mail):
+    logFileName = working_path + '/' + 'tmpEmailContent'
+    with open(logFileName, "w") as myfile:
+        myfile.writelines(text)
+        myfile.close()
+        if have_mutt:
+            execute('mutt -s "' + topic + '" ' + emails + ' < ' + logFileName)
+        elif have_mail:
+            execute('mail -s "' + topic + '" ' + emails + ' < ' + logFileName)
+        else:
+            log("e-mail cannot be sent (mail or mutt not found)")
+
+
+def send_email_with_attachments(branch, commit, last_commit, emails, text, results_files, logFileName, lower_limit, have_mutt, have_mail):
+    with open(logFileName, "w") as myfile:
+        myfile.writelines(text)
+        myfile.close()
+        email_topic = '%s:%s Warning for %s:%s last_commit=%s speed<%s' % (email_header, pid, branch, commit, last_commit, lower_limit)
+        if have_mutt:
+            execute('mutt -s "' + email_topic + '" ' + emails + ' -a ' + results_files + ' < ' + logFileName)
+        elif have_mail:
+            execute('mail -s "' + email_topic + '" ' + emails + ' < ' + logFileName)
+        else:
+            log("e-mail cannot be sent (mail or mutt not found)")
+
+
+def git_get_branches():
    execute('git fetch -p')
    output = execute('git branch -rl')
    for line in output:
        if "HEAD" in line: 
-            output.remove(line) # remove "origin/HEAD -> origin/dev"
-    branches = map(lambda l: l.strip(), output)
-    return map(lambda b: (b, execute('git show -s --format=%h ' + b)[0].strip()), branches)
+            output.remove(line)  # remove "origin/HEAD -> origin/dev"
+    return map(lambda l: l.strip(), output)


-def notify(branch, commit, last_commit):
-    text_tmpl = string.Template('Changes since $last_commit:\r\n$commits')
-    branch = branch.split('/')[1]
+def git_get_changes(branch, commit, last_commit):
    fmt = '--format="%h: (%an) %s, %ar"'
    if last_commit is None:
        commits = execute('git log -n 10 %s %s' % (fmt, commit))
    else:
        commits = execute('git --no-pager log %s %s..%s' % (fmt, last_commit, commit))
-
-    text = text_tmpl.substitute({'last_commit': last_commit, 'commits': ''.join(commits)})
-    print str("commits for %s: %s" % (commit, text))
+    return str('Changes in %s since %s:\n' % (branch, last_commit)) + '\n'.join(commits)


-def compile(branch, commit, dry_run):
-    local_branch = string.split(branch, '/')[1]
-    version = local_branch.rpartition('-')[2]
-    version = version + '_' + commit
-    execute('git checkout -- . && git checkout ' + branch)
-    if not dry_run:
-        execute('VERSION=' + version + '; make clean zstdprogram')
-
-
-def get_last_commit(resultsFileName):
+def get_last_results(resultsFileName):
    if not os.path.isfile(resultsFileName):
        return None, None, None
    commit = None
@@ -84,15 +97,13 @@ def get_last_commit(resultsFileName):
    with open(resultsFileName,'r') as f:
        for line in f:
            words = line.split()
-            if len(words) == 2: # branch + commit
+            if len(words) == 2:   # branch + commit
                commit = words[1];
                cspeed = []
                dspeed = []
-            if (len(words) == 8): 
+            if (len(words) == 8):  # results
                cspeed.append(float(words[3]))
                dspeed.append(float(words[5]))
-        #if commit != None:
-        #    print "commit=%s cspeed=%s dspeed=%s" % (commit, cspeed, dspeed)
    return commit, cspeed, dspeed


@@ -102,80 +113,62 @@ def benchmark_and_compare(branch, commit, resultsFileName, lastCLevel, testFileP
        log("WARNING: bench loadavg=%.2f is higher than %s, sleeping for %s seconds" % (os.getloadavg()[0], maxLoadAvg, sleepTime))
        time.sleep(sleepTime)
    start_load = str(os.getloadavg())
-    result = execute('programs/zstd -qi5b1e' + str(lastCLevel) + ' ' + testFilePath)
+    result = execute('programs/zstd -qi5b1e%s %s' % (lastCLevel, testFilePath), print_output=True)
    end_load = str(os.getloadavg())
    linesExpected = lastCLevel + 2;
    if len(result) != linesExpected:
-        log("ERROR: number of result lines=%d is different that expected %d" % (len(result), linesExpected))
-        return ""
+        raise RuntimeError("ERROR: number of result lines=%d is different that expected %d\n%s" % (len(result), linesExpected, '\n'.join(result)))
    with open(resultsFileName, "a") as myfile:
        myfile.write(branch + " " + commit + "\n")
-        myfile.writelines(result)
+        myfile.write('\n'.join(result) + '\n')
        myfile.close()
        if (last_cspeed == None):
+            log("WARNING: No data for comparison for branch=%s file=%s " % (branch, fileName))
            return ""
-        commit, cspeed, dspeed = get_last_commit(resultsFileName)
+        commit, cspeed, dspeed = get_last_results(resultsFileName)
        text = ""
        for i in range(0, min(len(cspeed), len(last_cspeed))):
+            print("%s:%s -%d cspeed=%6.2f clast=%6.2f cdiff=%1.4f dspeed=%6.2f dlast=%6.2f ddiff=%1.4f %s" % (branch, commit, i+1, cspeed[i], last_cspeed[i], cspeed[i]/last_cspeed[i], dspeed[i], last_dspeed[i], dspeed[i]/last_dspeed[i], fileName))
            if (cspeed[i]/last_cspeed[i] < lower_limit):
-                text += "WARNING: File=%s level=%d cspeed=%s last=%s diff=%s\n" % (fileName, i+1, cspeed[i], last_cspeed[i], cspeed[i]/last_cspeed[i])
+                text += "WARNING: -%d cspeed=%.2f clast=%.2f cdiff=%.4f %s\n" % (i+1, cspeed[i], last_cspeed[i], cspeed[i]/last_cspeed[i], fileName)
            if (dspeed[i]/last_dspeed[i] < lower_limit):
-                text += "WARNING: File=%s level=%d dspeed=%s last=%s diff=%s\n" % (fileName, i+1, dspeed[i], last_dspeed[i], dspeed[i]/last_dspeed[i])
+                text += "WARNING: -%d dspeed=%.2f dlast=%.2f ddiff=%.4f %s\n" % (i+1, dspeed[i], last_dspeed[i], dspeed[i]/last_dspeed[i], fileName)
        if text:
            text = message + ("\nmaxLoadAvg=%s  load average at start=%s end=%s\n" % (maxLoadAvg, start_load, end_load)) + text
        return text


-def send_email(branch, commit, last_commit, emails, text, results_files, logFileName, lower_limit, have_mutt, have_mail):
-    with open(logFileName, "w") as myfile:
-        myfile.writelines(text)
-        myfile.close()
-        if have_mutt:
-            execute("mutt -s \"[ZSTD_speedTest] Warning for branch=" + branch + " commit=" + commit + " last_commit=" + last_commit + " speed<" + str(lower_limit) + "\" " + emails + " -a " + results_files + " < " + logFileName)
-        elif have_mail:
-            execute("mail -s \"[ZSTD_speedTest] Warning for branch=" + branch + " commit=" + commit + " last_commit=" + last_commit + " speed<" + str(lower_limit) + "\" " + emails + " < " + logFileName)
-        else:
-            log("e-mail cannot be sent (mail and mutt not found)")
+def update_config_file(branch, commit):
+    last_commit = None
+    commitFileName = working_path + "/commit_" + branch.replace("/", "_") + ".txt"
+    if os.path.isfile(commitFileName):
+        last_commit = file(commitFileName, 'r').read()
+    file(commitFileName, 'w').write(commit)
+    return last_commit


-def check_branches(args, test_path, testFilePaths, have_mutt, have_mail):
-    for branch, commit in fetch():
-        try:
-            commitFileName = test_path + "/commit_" + branch.replace("/", "_")
-            if os.path.isfile(commitFileName):
-                last_commit = file(commitFileName, 'r').read()
-            else:
-                last_commit = None
-            file(commitFileName, 'w').write(commit)
-
-            if commit == last_commit:
-                log("skipping branch %s: head %s already processed" % (branch, commit))
-            else:
-                log("build branch %s: head %s is different from prev %s" % (branch, commit, last_commit))
-                compile(branch, commit, args.dry_run)
-
-                logFileName = test_path + "/log_" + branch.replace("/", "_")
-                text_to_send = []
-                results_files = ""
-                for filePath in testFilePaths:
-                    fileName = filePath.rpartition('/')[2]
-                    resultsFileName = test_path + "/results_" + branch.replace("/", "_") + "_" + fileName
-                    last_commit, cspeed, dspeed = get_last_commit(resultsFileName)
-
-                    if not args.dry_run:
-                        text = benchmark_and_compare(branch, commit, resultsFileName, args.lastCLevel, filePath, fileName, cspeed, dspeed, args.lowerLimit, args.maxLoadAvg, args.message)
-                        if text:
-                            text = benchmark_and_compare(branch, commit, resultsFileName, args.lastCLevel, filePath, fileName, cspeed, dspeed, args.lowerLimit, args.maxLoadAvg, args.message)
-                            if text:
-                                text_to_send.append(text)
-                                results_files += resultsFileName + " "
-                if text_to_send:
-                    send_email(branch, commit, last_commit, args.emails, text_to_send, results_files, logFileName, args.lowerLimit, have_mutt, have_mail)
-                notify(branch, commit, last_commit)
-        except Exception as e:
-            stack = traceback.format_exc()
-            log("ERROR: build %s, error %s" % (branch, str(e)) )
-            print stack
+def test_commit(branch, commit, last_commit, args, testFilePaths, have_mutt, have_mail):
+    local_branch = string.split(branch, '/')[1]
+    version = local_branch.rpartition('-')[2] + '_' + commit
+    if not args.dry_run:
+        execute('make clean zstdprogram MOREFLAGS="-DZSTD_GIT_COMMIT=%s"' % version)
+    logFileName = working_path + "/log_" + branch.replace("/", "_") + ".txt"
+    text_to_send = []
+    results_files = ""
+    for filePath in testFilePaths:
+        fileName = filePath.rpartition('/')[2]
+        resultsFileName = working_path + "/results_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt"
+        last_commit, cspeed, dspeed = get_last_results(resultsFileName)
+        if not args.dry_run:
+            text = benchmark_and_compare(branch, commit, resultsFileName, args.lastCLevel, filePath, fileName, cspeed, dspeed, args.lowerLimit, args.maxLoadAvg, args.message)
+            if text:
+                log("WARNING: redoing tests for branch %s: commit %s" % (branch, commit))
+                text = benchmark_and_compare(branch, commit, resultsFileName, args.lastCLevel, filePath, fileName, cspeed, dspeed, args.lowerLimit, args.maxLoadAvg, args.message)
+                if text:
+                    text_to_send.append(text)
+                    results_files += resultsFileName + " "
+    if text_to_send:
+        send_email_with_attachments(branch, commit, last_commit, args.emails, text_to_send, results_files, logFileName, args.lowerLimit, have_mutt, have_mail)


 if __name__ == '__main__':
@@ -183,6 +176,7 @@ if __name__ == '__main__':
    parser.add_argument('testFileNames', help='file names list for speed benchmark')
    parser.add_argument('emails', help='list of e-mail addresses to send warnings')
    parser.add_argument('--message', help='attach an additional message to e-mail', default="")
+    parser.add_argument('--repoURL', help='changes default repository URL', default=default_repo_url)
    parser.add_argument('--lowerLimit', type=float, help='send email if speed is lower than given limit', default=0.98)
    parser.add_argument('--maxLoadAvg', type=float, help='maximum load average to start testing', default=0.75)
    parser.add_argument('--lastCLevel', type=int, help='last compression level for testing', default=5)
@@ -194,56 +188,78 @@ if __name__ == '__main__':
    testFileNames = args.testFileNames.split()
    testFilePaths = []
    for fileName in testFileNames:
+        fileName = os.path.expanduser(fileName)
        if os.path.isfile(fileName):
            testFilePaths.append(os.path.abspath(fileName))
        else:
-            raise RuntimeError("File not found: " + fileName)
-
-    test_path = os.getcwd() + '/' + test_dir_name     # /path/to/zstd/tests/speedTest 
-    clone_path = test_path + '/' + 'zstd'             # /path/to/zstd/tests/speedTest/zstd 
+            log("ERROR: File not found: " + fileName)
+            exit(1)

    # check availability of e-mail senders
-    have_mutt = does_command_exist("mutt --help");
+    have_mutt = does_command_exist("mutt -h");
    have_mail = does_command_exist("mail -V");
    if not have_mutt and not have_mail:
-        log("WARNING: e-mail senders mail and mutt not found")
+        log("ERROR: e-mail senders 'mail' or 'mutt' not found")
+        exit(1)
+
+    print("PARAMETERS:\nrepoURL=%s" % args.repoURL)
+    print("working_path=%s" % working_path)
+    print("clone_path=%s" % clone_path)
+    print("testFilePath(%s)=%s" % (len(testFilePaths), testFilePaths))
+    print("message=%s" % args.message)
+    print("emails=%s" % args.emails)
+    print("maxLoadAvg=%s" % args.maxLoadAvg)
+    print("lowerLimit=%s" % args.lowerLimit)
+    print("lastCLevel=%s" % args.lastCLevel)
+    print("sleepTime=%s" % args.sleepTime)
+    print("dry_run=%s" % args.dry_run)
+    print("have_mutt=%s have_mail=%s" % (have_mutt, have_mail))

    # clone ZSTD repo if needed
-    if not os.path.isdir(test_path):
-        os.mkdir(test_path)
+    if not os.path.isdir(working_path):
+        os.mkdir(working_path)
    if not os.path.isdir(clone_path):
-        execute.cwd = test_path
-        execute('git clone ' + repo_url)
+        execute.cwd = working_path
+        execute('git clone ' + args.repoURL)
    if not os.path.isdir(clone_path):
-        raise RuntimeError("ZSTD clone not found: " + clone_path)
+        log("ERROR: ZSTD clone not found: " + clone_path)
+        exit(1)
    execute.cwd = clone_path

-    print "PARAMETERS:\ntest_path=%s" % test_path
-    print "clone_path=%s" % clone_path
-    print "testFilePath(%s)=%s" % (len(testFilePaths), testFilePaths)
-    print "message=%s" % args.message
-    print "emails=%s" % args.emails
-    print "maxLoadAvg=%s" % args.maxLoadAvg
-    print "lowerLimit=%s" % args.lowerLimit
-    print "lastCLevel=%s" % args.lastCLevel
-    print "sleepTime=%s" % args.sleepTime
-    print "dry_run=%s" % args.dry_run
-    print "have_mutt=%s have_mail=%s" % (have_mutt, have_mail)
+    # check if speedTest.pid already exists
+    pidfile = "./speedTest.pid"
+    if os.path.isfile(pidfile):
+        log("ERROR: %s already exists, exiting" % pidfile)
+        exit(1)
+
+    send_email(args.emails, email_header + ':%s test-zstd-speed.py has been started' % pid, args.message, have_mutt, have_mail)
+    file(pidfile, 'w').write(pid)

    while True:
-        pid = str(os.getpid())
-        pidfile = "./speedTest.pid"
-        if os.path.isfile(pidfile):
-            log("%s already exists, exiting" % pidfile)
-        else:
-            file(pidfile, 'w').write(pid)
-            try:
-                loadavg = os.getloadavg()[0]
-                if (loadavg <= args.maxLoadAvg):
-                    check_branches(args, test_path, testFilePaths, have_mutt, have_mail)
-                else:
-                    log("WARNING: main loadavg=%.2f is higher than %s" % (loadavg, args.maxLoadAvg))
-            finally:
-                os.unlink(pidfile)
-        log("sleep for %s seconds" % args.sleepTime)
-        time.sleep(args.sleepTime)
+        try:
+            loadavg = os.getloadavg()[0]
+            if (loadavg <= args.maxLoadAvg):
+                branches = git_get_branches()
+                for branch in branches:
+                    commit = execute('git show -s --format=%h ' + branch)[0]
+                    last_commit = update_config_file(branch, commit)
+                    if commit == last_commit:
+                        log("skipping branch %s: head %s already processed" % (branch, commit))
+                    else:
+                        log("build branch %s: head %s is different from prev %s" % (branch, commit, last_commit))
+                        execute('git checkout -- . && git checkout ' + branch)
+                        print(git_get_changes(branch, commit, last_commit))
+                        test_commit(branch, commit, last_commit, args, testFilePaths, have_mutt, have_mail)
+            else:
+                log("WARNING: main loadavg=%.2f is higher than %s" % (loadavg, args.maxLoadAvg))
+            log("sleep for %s seconds" % args.sleepTime)
+            time.sleep(args.sleepTime)
+        except Exception as e:
+            stack = traceback.format_exc()
+            email_topic = '%s:%s ERROR in %s:%s' % (email_header, pid, branch, commit)
+            send_email(args.emails, email_topic, stack, have_mutt, have_mail)
+            print(stack)
+        except KeyboardInterrupt:
+            os.unlink(pidfile)
+            send_email(args.emails, email_header + ':%s test-zstd-speed.py has been stopped' % pid, args.message, have_mutt, have_mail)
+            exit(0)
@@ -7,8 +7,9 @@ import glob
 import hashlib
 import os
 import shutil
-import subprocess
 import sys
+import subprocess
+from subprocess import Popen, PIPE

 repo_url = 'https://github.com/Cyan4973/zstd.git'
 tmp_dir_name = 'tests/versionsTest'
@@ -17,17 +18,32 @@ git_cmd = 'git'
 test_dat_src = 'README.md'
 test_dat = 'test_dat'
 head = 'vdevel'
+dict_source = 'dict_source'
+dict_files = './zstd/programs/*.c ./zstd/lib/common/*.c ./zstd/lib/compress/*.c ./zstd/lib/decompress/*.c ./zstd/lib/dictBuilder/*.c ./zstd/lib/legacy/*.c '
+dict_files += './zstd/programs/*.h ./zstd/lib/common/*.h ./zstd/lib/compress/*.h ./zstd/lib/dictBuilder/*.h ./zstd/lib/legacy/*.h'
+
+
+def execute(command, print_output=False, print_error=True, param_shell=False):
+    popen = Popen(command, stdout=PIPE, stderr=PIPE, shell=param_shell)
+    stdout_lines, stderr_lines = popen.communicate()
+    stderr_lines = stderr_lines.decode("utf-8")
+    stdout_lines = stdout_lines.decode("utf-8")
+    if print_output:
+        print(stdout_lines)
+        print(stderr_lines)
+    if popen.returncode is not None and popen.returncode != 0:
+        if not print_output and print_error:
+            print(stderr_lines)
+    return popen.returncode


 def proc(cmd_args, pipe=True, dummy=False):
    if dummy:
        return
    if pipe:
-        subproc = subprocess.Popen(cmd_args,
-                                   stdout=subprocess.PIPE,
-                                   stderr=subprocess.PIPE)
+        subproc = Popen(cmd_args, stdout=PIPE, stderr=PIPE)
    else:
-        subproc = subprocess.Popen(cmd_args)
+        subproc = Popen(cmd_args)
    return subproc.communicate()


@@ -45,21 +61,53 @@ def get_git_tags():
    return tags


+def create_dict(tag, dict_source_path):
+    dict_name = 'dict.' + tag
+    if not os.path.isfile(dict_name):
+        cFiles = glob.glob(dict_source_path + "/*.c")
+        hFiles = glob.glob(dict_source_path + "/*.h")
+        if tag == 'v0.5.0':
+            result = execute('./dictBuilder.' + tag + ' ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name, print_output=False, param_shell=True)
+        else:
+            result = execute('./zstd.' + tag + ' -f --train ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name, print_output=False, param_shell=True)
+        if result == 0:
+            print(dict_name + ' created')
+        else:
+            print('ERROR: creating of ' + dict_name + ' failed')
+    else:
+        print(dict_name + ' already exists')
+
+
+def dict_compress_sample(tag, sample):
+    dict_name = 'dict.' + tag
+    DEVNULL = open(os.devnull, 'wb')
+    if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-f',   sample], stderr=DEVNULL) == 0:
+        os.rename(sample + '.zst', sample + '_01_64_' + tag + '_dictio.zst')
+    if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-5f',  sample], stderr=DEVNULL) == 0:
+        os.rename(sample + '.zst', sample + '_05_64_' + tag + '_dictio.zst')
+    if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-9f',  sample], stderr=DEVNULL) == 0:
+        os.rename(sample + '.zst', sample + '_09_64_' + tag + '_dictio.zst')
+    if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-15f', sample], stderr=DEVNULL) == 0:
+        os.rename(sample + '.zst', sample + '_15_64_' + tag + '_dictio.zst')
+    if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-18f', sample], stderr=DEVNULL) == 0:
+        os.rename(sample + '.zst', sample + '_18_64_' + tag + '_dictio.zst')
+    # zstdFiles = glob.glob("*.zst*")
+    # print(zstdFiles)
+    print(tag + " : dict compression completed")
+
+
 def compress_sample(tag, sample):
-    try:
-        from subprocess import DEVNULL  # py3k
-    except ImportError:
-        DEVNULL = open(os.devnull, 'wb')
+    DEVNULL = open(os.devnull, 'wb')
    if subprocess.call(['./zstd.' + tag, '-f',   sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_01_64_' + tag + '.zst')
+        os.rename(sample + '.zst', sample + '_01_64_' + tag + '_nodict.zst')
    if subprocess.call(['./zstd.' + tag, '-5f',  sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_05_64_' + tag + '.zst')
+        os.rename(sample + '.zst', sample + '_05_64_' + tag + '_nodict.zst')
    if subprocess.call(['./zstd.' + tag, '-9f',  sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_09_64_' + tag + '.zst')
+        os.rename(sample + '.zst', sample + '_09_64_' + tag + '_nodict.zst')
    if subprocess.call(['./zstd.' + tag, '-15f', sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_15_64_' + tag + '.zst')
+        os.rename(sample + '.zst', sample + '_15_64_' + tag + '_nodict.zst')
    if subprocess.call(['./zstd.' + tag, '-18f', sample], stderr=DEVNULL) == 0:
-        os.rename(sample + '.zst', sample + '_18_64_' + tag + '.zst')
+        os.rename(sample + '.zst', sample + '_18_64_' + tag + '_nodict.zst')
    # zstdFiles = glob.glob("*.zst*")
    # print(zstdFiles)
    print(tag + " : compression completed")
@@ -87,16 +135,16 @@ def remove_duplicates():

 def decompress_zst(tag):
    dec_error = 0
-    list_zst = sorted(glob.glob('*.zst'))
-    try:
-        from subprocess import DEVNULL  # py3k
-    except ImportError:
-        DEVNULL = open(os.devnull, 'wb')
+    list_zst = sorted(glob.glob('*_nodict.zst'))
    for file_zst in list_zst:
        print(file_zst, end=' ')
        print(tag, end=' ')
        file_dec = file_zst + '_d64_' + tag + '.dec'
-        if subprocess.call(['./zstd.' + tag, '-df', file_zst, '-o', file_dec], stderr=DEVNULL) == 0:
+        if tag <= 'v0.5.0':
+            params = ['./zstd.' + tag, '-df', file_zst, file_dec]
+        else:
+            params = ['./zstd.' + tag, '-df', file_zst, '-o', file_dec]
+        if execute(params) == 0:
            if not filecmp.cmp(file_dec, test_dat):
                print('ERR !! ')
                dec_error = 1
@@ -107,12 +155,43 @@ def decompress_zst(tag):
    return dec_error


+def decompress_dict(tag):
+    dec_error = 0
+    list_zst = sorted(glob.glob('*_dictio.zst'))
+    for file_zst in list_zst:
+        dict_tag = file_zst[0:len(file_zst)-11]  # remove "_dictio.zst"
+        if head in dict_tag: # find vdevel
+            dict_tag = head
+        else:
+            dict_tag = dict_tag[dict_tag.rfind('v'):]
+        if tag == 'v0.6.0' and dict_tag < 'v0.6.0':
+            continue
+        dict_name = 'dict.' + dict_tag
+        print(file_zst + ' ' + tag + ' dict=' + dict_tag, end=' ')
+        file_dec = file_zst + '_d64_' + tag + '.dec'
+        if tag <= 'v0.5.0':
+            params = ['./zstd.' + tag, '-D', dict_name, '-df', file_zst, file_dec]
+        else:
+            params = ['./zstd.' + tag, '-D', dict_name, '-df', file_zst, '-o', file_dec]
+        if execute(params) == 0:
+            if not filecmp.cmp(file_dec, test_dat):
+                print('ERR !! ')
+                dec_error = 1
+            else:
+                print('OK     ')
+        else:
+            print('command does not work')
+            dec_error = 1
+    return dec_error
+
+
 if __name__ == '__main__':
    error_code = 0
-    base_dir = os.getcwd() + '/..'           # /path/to/zstd
-    tmp_dir = base_dir + '/' + tmp_dir_name  # /path/to/zstd/tests/versionsTest
-    clone_dir = tmp_dir + '/' + 'zstd'       # /path/to/zstd/tests/versionsTest/zstd
-    programs_dir = base_dir + '/programs'    # /path/to/zstd/programs
+    base_dir = os.getcwd() + '/..'                  # /path/to/zstd
+    tmp_dir = base_dir + '/' + tmp_dir_name         # /path/to/zstd/tests/versionsTest
+    clone_dir = tmp_dir + '/' + 'zstd'              # /path/to/zstd/tests/versionsTest/zstd
+    dict_source_path = tmp_dir + '/' + dict_source  # /path/to/zstd/tests/versionsTest/dict_source
+    programs_dir = base_dir + '/programs'           # /path/to/zstd/programs
    os.makedirs(tmp_dir, exist_ok=True)

    # since Travis clones limited depth, we should clone full repository
@@ -137,6 +216,10 @@ if __name__ == '__main__':
                os.makedirs(r_dir, exist_ok=True)
                os.chdir(clone_dir)
                git(['--work-tree=' + r_dir, 'checkout', tag, '--', '.'], False)
+                if tag == 'v0.5.0':
+                    os.chdir(r_dir + '/dictBuilder')  # /path/to/zstd/tests/versionsTest/v0.5.0/dictBuilder
+                    make(['clean', 'dictBuilder'], False)
+                    shutil.copy2('dictBuilder', '{}/dictBuilder.{}'.format(tmp_dir, tag))
                os.chdir(r_dir + '/programs')  # /path/to/zstd/tests/versionsTest/<TAG>/programs
                make(['clean', 'zstd'], False)
            else:
@@ -151,15 +234,25 @@ if __name__ == '__main__':
    for dec in glob.glob("*.dec"):
        os.remove(dec)

+    # copy *.c and *.h to a temporary directory ("dict_source")
+    if not os.path.isdir(dict_source_path):
+        os.mkdir(dict_source_path)
+        print('cp ' + dict_files + ' ' + dict_source_path)
+        execute('cp ' + dict_files + ' ' + dict_source_path, param_shell=True)
+
    print('Compress test.dat by all released zstd')

    error_code = 0
    for tag in tags:
        print(tag)
+        if tag >= 'v0.5.0':
+            create_dict(tag, dict_source_path)
+            dict_compress_sample(tag, test_dat)
+            remove_duplicates()
+            error_code += decompress_dict(tag)
        compress_sample(tag, test_dat)
        remove_duplicates()
-        if tag >= 'v0.5.1':
-            error_code += decompress_zst(tag)
+        error_code += decompress_zst(tag)

    print('')
    print('Enumerate different compressed files')
@@ -0,0 +1,591 @@
+Zstandard Compression Format
+============================
+
+### Notices
+
+Copyright (c) 2016 Yann Collet
+
+Permission is granted to copy and distribute this document
+for any  purpose and without charge,
+including translations into other  languages
+and incorporation into compilations,
+provided that the copyright notice and this notice are preserved,
+and that any substantive changes or deletions from the original
+are clearly marked.
+Distribution of this document is unlimited.
+
+### Version
+
+0.0.1 (30/06/2016 - Work in progress - unfinished)
+
+
+Introduction
+------------
+
+The purpose of this document is to define a lossless compressed data format,
+that is independent of CPU type, operating system,
+file system and character set, suitable for
+File compression, Pipe and streaming compression
+using the [Zstandard algorithm](http://www.zstandard.org).
+
+The data can be produced or consumed,
+even for an arbitrarily long sequentially presented input data stream,
+using only an a priori bounded amount of intermediate storage,
+and hence can be used in data communications.
+The format uses the Zstandard compression method,
+and optional [xxHash-64 checksum method](http://www.xxhash.org),
+for detection of data corruption.
+
+The data format defined by this specification
+does not attempt to allow random access to compressed data.
+
+This specification is intended for use by implementers of software
+to compress data into Zstandard format and/or decompress data from Zstandard format.
+The text of the specification assumes a basic background in programming
+at the level of bits and other primitive data representations.
+
+Unless otherwise indicated below,
+a compliant compressor must produce data sets
+that conform to the specifications presented here.
+It doesn’t need to support all options though.
+
+A compliant decompressor must be able to decompress
+at least one working set of parameters
+that conforms to the specifications presented here.
+It may also ignore informative fields, such as checksum.
+Whenever it does not support a parameter defined in the compressed stream,
+it must produce a non-ambiguous error code and associated error message
+explaining which parameter is unsupported.
+
+
+Definitions
+-----------
+A content compressed by Zstandard is transformed into a Zstandard __frame__.
+Multiple frames can be appended into a single file or stream.
+A frame is totally independent, has a defined beginning and end,
+and a set of parameters which tells the decoder how to decompress it.
+
+A frame encapsulates one or multiple __blocks__.
+Each block can be compressed or not,
+and has a guaranteed maximum content size, which depends on frame parameters.
+Unlike frames, each block depends on previous blocks for proper decoding.
+However, each block can be decompressed without waiting for its successor,
+allowing streaming operations.
+
+
+General Structure of Zstandard Frame format
+-------------------------------------------
+
+| MagicNb |  F. Header | Block | (More blocks) | EndMark |
+|:-------:|:----------:| ----- | ------------- | ------- |
+| 4 bytes | 2-14 bytes |       |               | 3 bytes |
+
+__Magic Number__
+
+4 Bytes, Little endian format.
+Value : 0xFD2FB527
+
+__Frame Header__
+
+2 to 14 Bytes, to be detailed in the next part.
+
+__Data Blocks__
+
+To be detailed later on.
+That’s where compressed data is stored.
+
+__EndMark__
+
+The flow of blocks ends when the last block header brings an _end signal_ .
+This last block header may optionally host a __Content Checksum__ .
+
+__Content Checksum__
+
+Content Checksum verify that frame content has been regenrated correctly.
+The content checksum is the result
+of [xxh64() hash function](https://www.xxHash.com)
+digesting the original (decoded) data as input, and a seed of zero.
+Bits from 11 to 32 (included) are extracted to form a 22 bits checksum
+stored into the last block header.
+```
+contentChecksum = (XXH64(content, size, 0) >> 11) & (1<<22)-1);
+```
+Content checksum is only present when its associated flag
+is set in the frame descriptor.
+Its usage is optional.
+
+__Frame Concatenation__
+
+In some circumstances, it may be required to append multiple frames,
+for example in order to add new data to an existing compressed file
+without re-framing it.
+
+In such case, each frame brings its own set of descriptor flags.
+Each frame is considered independent.
+The only relation between frames is their sequential order.
+
+The ability to decode multiple concatenated frames
+within a single stream or file is left outside of this specification.
+As an example, the reference `zstd` command line utility is able
+to decode all concatenated frames in their sequential order,
+delivering the final decompressed result as if it was a single content.
+
+
+Frame Header
+-------------
+
+| FHD     | (WD)      | (Content Size) | (dictID)  |
+| ------- | --------- |:--------------:| --------- |
+| 1 byte  | 0-1 byte  |  0 - 8 bytes   | 0-4 bytes |
+
+Frame header has a variable size, which uses a minimum of 2 bytes,
+and up to 14 bytes depending on optional parameters.
+
+__FHD byte__ (Frame Header Descriptor)
+
+The first Header's byte is called the Frame Header Descriptor.
+It tells which other fields are present.
+Decoding this byte is enough to get the full size of the Frame Header.
+
+|  BitNb  |   7-6  |    5    |   4    |    3     |    2     |    1-0   |
+| ------- | ------ | ------- | ------ | -------- | -------- | -------- |
+|FieldName| FCSize | Segment | Unused | Reserved | Checksum |  dictID  |
+
+In this table, bit 7 is highest bit, while bit 0 is lowest.
+
+__Frame Content Size flag__
+
+This is a 2-bits flag (`= FHD >> 6`),
+specifying if decompressed data size is provided within the header.
+
+|  Value  |  0  |  1  |  2  |  3  |
+| ------- | --- | --- | --- | --- |
+|FieldSize| 0-1 |  2  |  4  |  8  |
+
+Value 0 has a double meaning :
+it either means `0` (size not provided) _if_ the `WD` byte is present,
+or it means `1` byte (size <= 255 bytes).
+
+__Single Segment__
+
+If this flag is set,
+data shall be regenerated within a single continuous memory segment.
+In which case, `WD` byte __is not present__,
+but `Frame Content Size` field necessarily is.
+
+As a consequence, the decoder must allocate a memory segment
+of size `>= Frame Content Size`.
+
+In order to preserve the decoder from unreasonable memory requirement,
+a decoder can refuse a compressed frame
+which requests a memory size beyond decoder's authorized range.
+
+For broader compatibility, decoders are recommended to support
+memory sizes of 8 MB at least.
+However, this is merely a recommendation,
+and each decoder is free to support higher or lower limits,
+depending on local limitations.
+
+__Unused bit__
+
+The value of this bit is unimportant
+and not interpreted by a decoder compliant with this specification version.
+It may be used in a future revision,
+to signal a property which is not required to properly decode the frame.
+
+__Reserved bit__
+
+This bit is reserved for some future feature.
+Its value _must be zero_.
+A decoder compliant with this specification version must ensure it is not set.
+This bit may be used in a future revision,
+to signal a feature that must be interpreted in order to decode the frame.
+
+__Content checksum flag__
+
+If this flag is set, a content checksum will be present into the EndMark.
+The checksum is a 22 bits value extracted from the XXH64() of data.
+See __Content Checksum__ .
+
+__Dictionary ID flag__
+
+This is a 2-bits flag (`= FHD & 3`),
+telling if a dictionary ID is provided within the header
+
+|  Value  |  0  |  1  |  2  |  3  |
+| ------- | --- | --- | --- | --- |
+|FieldSize|  0  |  1  |  2  |  4  |
+
+__WD byte__ (Window Descriptor)
+
+Provides guarantees on maximum back-reference distance
+that will be present within compressed data.
+This information is useful for decoders to allocate enough memory.
+
+|   BitNb   |    7-3   |    0-2   |
+| --------- | -------- | -------- |
+| FieldName | Exponent | Mantissa |
+
+Maximum distance is given by the following formulae :
+```
+windowLog = 10 + Exponent;
+windowBase = 1 << windowLog;
+windowAdd = (windowBase / 8) * Mantissa;
+windowSize = windowBase + windowAdd;
+```
+The minimum window size is 1 KB.
+The maximum size is (15*(2^38))-1 bytes, which is almost 1.875 TB.
+
+To properly decode compressed data,
+a decoder will need to allocate a buffer of at least `windowSize` bytes.
+
+Note that `WD` byte is optional. It's not present in `single segment` mode.
+In which case, the maximum back-reference distance is the content size itself,
+which can be any value from 1 to 2^64-1 bytes (16 EB).
+
+In order to preserve decoder from unreasonable memory requirements,
+a decoder can refuse a compressed frame
+which requests a memory size beyond decoder's authorized range.
+
+For better interoperability,
+decoders are recommended to be compatible with window sizes of 8 MB.
+Encoders are recommended to not request more than 8 MB.
+It's merely a recommendation though,
+decoders are free to support larger or lower limits,
+depending on local limitations.
+
+__Frame Content Size__
+
+This is the original (uncompressed) size.
+This information is optional, and only present if associated flag is set.
+Content size is provided using 1, 2, 4 or 8 Bytes.
+Format is Little endian.
+
+| Field Size |    Range   |
+| ---------- | ---------- |
+|     0      |      0     |
+|     1      |   0 - 255  |
+|     2      | 256 - 65791|
+|     4      | 0 - 2^32-1 |
+|     8      | 0 - 2^64-1 |
+
+When field size is 1, 4 or 8 bytes, the value is read directly.
+When field size is 2, _an offset of 256 is added_.
+It's allowed to represent a small size (ex: `18`) using the 8-bytes variant.
+A size of `0` means `content size is unknown`.
+In which case, the `WD` byte will necessarily be present,
+and becomes the only hint to determine memory allocation.
+
+In order to preserve decoder from unreasonable memory requirement,
+a decoder can refuse a compressed frame
+which requests a memory size beyond decoder's authorized range.
+
+__Dictionary ID__
+
+This is a variable size field, which contains a single ID.
+It checks if the correct dictionary is used for decoding.
+Note that this field is optional. If it's not present,
+it's up to the caller to make sure it uses the correct dictionary.
+
+Field size depends on __Dictionary ID flag__.
+1 byte can represent an ID 0-255.
+2 bytes can represent an ID 0-65535.
+4 bytes can represent an ID 0-(2^32-1).
+
+It's allowed to represent a small ID (for example `13`)
+with a large 4-bytes dictionary ID, losing some efficiency in the process.
+
+
+Data Blocks
+-----------
+
+| B. Header |  data  |
+|:---------:| ------ |
+|  3 bytes  |        |
+
+
+__Block Header__
+
+This field uses 3-bytes, format is __big-endian__.
+
+The 2 highest bits represent the `block type`,
+while the remaining 22 bits represent the (compressed) block size.
+
+There are 4 block types :
+
+|    Value   |      0     |  1  |  2  |    3    |
+| ---------- | ---------- | --- | --- | ------- |
+| Block Type | Compressed | Raw | RLE | EndMark |
+
+- Compressed : this is a Zstandard compressed block,
+  detailed in a later part of this specification.
+  "block size" is the compressed size.
+  Decompressed size is unknown,
+  but its maximum possible value is guaranteed (see below)
+- Raw : this is an uncompressed block.
+  "block size" is the number of bytes to read and copy.
+- RLE : this is a single byte, repeated N times.
+  In which case, "block size" is the size to regenerate,
+  while the "compressed" block is just 1 byte (the byte to repeat).
+- EndMark : this is not a block. Signal the end of the frame.
+  The rest of the field may be optionally filled by a checksum
+  (see frame checksum).
+
+Block sizes must respect a few rules :
+- In compressed mode, compressed size if always strictly `< contentSize`.
+- Block decompressed size is necessarily <= maximum back-reference distance .
+- Block decompressed size is necessarily <= 128 KB
+
+
+__Data__
+
+Where the actual data to decode stands.
+It might be compressed or not, depending on previous field indications.
+A data block is not necessarily "full" :
+since an arbitrary “flush” may happen anytime,
+block content can be any size, up to Block Maximum Size.
+Block Maximum Size is the smallest of :
+- Max back-reference distance
+- 128 KB
+
+
+Skippable Frames
+----------------
+
+| Magic Number | Frame Size | User Data |
+|:------------:|:----------:| --------- |
+|   4 bytes    |  4 bytes   |           |
+
+Skippable frames allow the insertion of user-defined data
+into a flow of concatenated frames.
+Its design is pretty straightforward,
+with the sole objective to allow the decoder to quickly skip
+over user-defined data and continue decoding.
+
+Skippable frames defined in this specification are compatible with LZ4 ones.
+
+
+__Magic Number__ :
+
+4 Bytes, Little endian format.
+Value : 0x184D2A5X, which means any value from 0x184D2A50 to 0x184D2A5F.
+All 16 values are valid to identify a skippable frame.
+
+__Frame Size__ :
+
+This is the size, in bytes, of the following User Data
+(without including the magic number nor the size field itself).
+4 Bytes, Little endian format, unsigned 32-bits.
+This means User Data can’t be bigger than (2^32-1) Bytes.
+
+__User Data__ :
+
+User Data can be anything. Data will just be skipped by the decoder.
+
+
+Compressed block format
+-----------------------
+This specification details the content of a _compressed block_.
+A compressed block has a size, which must be known in order to decode it.
+It also has a guaranteed maximum regenerated size,
+in order to properly allocate destination buffer.
+See "Frame format" for more details.
+
+A compressed block consists of 2 sections :
+- Literals section
+- Sequences section
+
+### Prerequisite
+For proper decoding, a compressed block requires access to following elements :
+- Previous decoded blocks, up to a distance of `windowSize`,
+  or all previous blocks in the same frame "single segment" mode.
+- List of "recent offsets" from previous compressed block.
+
+
+### Compressed Literals
+
+Literals are compressed using order-0 huffman compression.
+During sequence phase, literals will be entangled with match copy operations.
+All literals are regrouped in the first part of the block.
+They can be decoded first, and then copied during sequence operations,
+or they can be decoded on the flow, as needed by sequences.
+
+| Header | (Tree Description) | Stream1 | (Stream2) | (Stream3) | (Stream4) |
+| ------ | ------------------ | ------- | --------- | --------- | --------- |
+
+Literals can be compressed, or uncompressed.
+When compressed, an optional tree description can be present,
+followed by 1 or 4 streams.
+
+#### Block Literal Header
+
+Header is in charge of describing precisely how literals are packed.
+It's a byte-aligned variable-size bitfield, ranging from 1 to 5 bytes,
+using big-endian convention.
+
+| BlockType | sizes format | (compressed size) | regenerated size |
+| --------- | ------------ | ----------------- | ---------------- |
+|   2 bits  |  1 - 2 bits  |    0 - 18 bits    |    5 - 20 bits   |
+
+__Block Type__ :
+
+This is a 2-bits field, describing 4 different block types :
+
+|    Value   |      0     |    1   |  2  |    3    |
+| ---------- | ---------- | ------ | --- | ------- |
+| Block Type | Compressed | Repeat | Raw |   RLE   |
+
+- Compressed : This is a standard huffman-compressed block,
+               starting with a huffman tree description.
+               See details below.
+- Repeat Stats : This is a huffman-compressed block,
+               using huffman tree from previous huffman-compressed block.
+               Huffman tree description will be skipped.
+               Compressed stream is equivalent to "compressed" block type.
+- Raw : Literals are stored uncompressed.
+- RLE : Literals consist of a single byte value repeated N times.
+
+__Sizes format__ :
+
+Sizes format are divided into 2 families :
+
+- For compressed block, it requires to decode both the compressed size
+  and the decompressed size. It will also decode the number of streams.
+- For Raw or RLE blocks, it's enough to decode the size to regenerate.
+
+For values spanning several bytes, convention is Big-endian.
+
+__Sizes format for Raw or RLE block__ :
+
+- Value : 0x : Regenerated size uses 5 bits (0-31).
+               Total literal header size is 1 byte.
+               `size = h[0] & 31;`
+- Value : 10 : Regenerated size uses 12 bits (0-4095).
+               Total literal header size is 2 bytes.
+               `size = ((h[0] & 15) << 8) + h[1];`
+- Value : 11 : Regenerated size uses 20 bits (0-1048575).
+               Total literal header size is 2 bytes.
+               `size = ((h[0] & 15) << 16) + (h[1]<<8) + h[2];`
+
+Note : it's allowed to represent a short value (ex : `13`)
+using a long format, accepting the reduced compacity.
+
+__Sizes format for Compressed Block__ :
+
+Note : also applicable to "repeat-stats" blocks.
+- Value : 00 : 4 streams
+               Compressed and regenerated sizes use 10 bits (0-1023)
+               Total literal header size is 3 bytes
+- Value : 01 : _Single stream_
+               Compressed and regenerated sizes use 10 bits (0-1023)
+               Total literal header size is 3 bytes
+- Value : 10 : 4 streams
+               Compressed and regenerated sizes use 14 bits (0-16383)
+               Total literal header size is 4 bytes
+- Value : 10 : 4 streams
+               Compressed and regenerated sizes use 18 bits (0-262143)
+               Total literal header size is 5 bytes
+
+Compressed and regenerated size fields follow big endian convention.
+
+#### Huffman Tree description
+
+This section is only present when block type is _compressed_ (`0`).
+It describes the different leaf nodes of the huffman tree,
+and their relative weights.
+
+##### Representation
+
+All byte values from zero (included) to last present one (excluded)
+are represented by `weight` values, from 0 to `maxBits`.
+Transformation from `weight` to `nbBits` follows this formulae :
+`nbBits = weight ? maxBits + 1 - weight : 0;` .
+The last symbol's weight is deduced from previously decoded ones,
+by completing to the nearest power of 2.
+This power of 2 gives `maxBits`, the depth of the current tree.
+
+__Example__ :
+Let's presume the following huffman tree must be described :
+
+|  Value | 0 | 1 | 2 | 3 | 4 | 5 |
+| ------ | - | - | - | - | - | - |
+| nbBits | 1 | 2 | 3 | 0 | 4 | 4 |
+
+The tree depth is 4, since its smallest element uses 4 bits.
+Value `5` will not be listed, nor will values above `5`.
+Values from `0` to `4` will be listed using `weight` instead of `nbBits`.
+Weight formula is : `weight = nbBits ? maxBits + 1 - nbBits : 0;`
+It gives the following serie of weights :
+
+| weight | 4 | 3 | 2 | 0 | 1 |
+| ------ | - | - | - | - | - |
+|  Value | 0 | 1 | 2 | 3 | 4 |
+
+The decoder will do the inverse operation :
+having collected weights of symbols from `0` to `4`,
+it knows the last symbol, `5`, is present with a non-zero weight.
+The weight of `5` can be deduced by joining to the nearest power of 2.
+Sum of 2^(weight-1) (excluding 0) is :
+8 + 4 + 2 + 0 + 1 = 15
+Nearest power of 2 is 16.
+Therefore, `maxBits = 4` and `weight[5] = 1`.
+It can then proceed to transform back weights into nbBits :
+`weight = nbBits ? maxBits + 1 - nbBits : 0;` .
+
+##### Huffman Tree header
+
+This is a single byte value (0-255), which tells how to decode the tree.
+
+- if headerByte >= 242 : this is one of 14 pre-defined weight distributions :
+  + 242 :  1x1 (+ 1x1)
+  + 243 :  2x1 (+ 1x2)
+  + 244 :  3x1 (+ 1x1)
+  + 245 :  4x1 (+ 1x4)
+  + 246 :  7x1 (+ 1x1)
+  + 247 :  8x1 (+ 1x8)
+  + 248 : 15x1 (+ 1x1)
+  + 249 : 16x1 (+ 1x16)
+  + 250 : 31x1 (+ 1x1)
+  + 251 : 32x1 (+ 1x32)
+  + 252 : 63x1 (+ 1x1)
+  + 253 : 64x1 (+ 1x64)
+  + 254 :127x1 (+ 1x1)
+  + 255 :128x1 (+ 1x128)
+
+- if headerByte >= 128 : this is a direct representation,
+  where each weight is written directly as a 4 bits field (0-15).
+  The full representation occupies (nbSymbols+1/2) bytes,
+  meaning it uses a last full byte even if nbSymbols is odd.
+  `nbSymbols = headerByte - 127;`
+
+- if headerByte < 128 :
+  the serie of weights is compressed by FSE.
+  The length of the compressed serie is `headerByte` (0-127).
+
+##### FSE (Finite State Entropy) compression of huffman weights
+
+The serie of weights is compressed using standard FSE compression.
+It's a single bitstream with 2 interleaved states,
+using a single distribution table.
+
+To decode an FSE bitstream, it is necessary to know its compressed size.
+Compressed size is provided by `headerByte`.
+It's also necessary to know its maximum decompressed size.
+In this case, it's `255`, since literal values range from `0` to `255`,
+and the last symbol value is not represented.
+
+An FSE bitstream starts by a header, describing probabilities distribution.
+Result will create a Decoding Table.
+It is necessary to know the maximum accuracy of distribution
+to properly allocate space for the Table.
+For a list of huffman weights, this maximum is 8 bits.
+
+FSE header and bitstreams are described in a separated chapter.
+
+##### Conversion from weights to huffman prefix codes
+
+
+
+
+Version changes
+---------------
Author	SHA1	Message	Date
Yann ColletandGitHub	2b61f74b1e	Merge pull request #236 from Cyan4973/dev v0.7.2	2016-07-03 21:06:42 +02:00
Yann Collet	5cc1882d45	fixed minor visual warning	2016-07-03 19:03:13 +02:00
Yann Collet	698cb63305	Updated specifications	2016-07-03 18:49:35 +02:00
Yann Collet	d57dffbe76	ZSTD_storeSeq takes an U32 as offset type	2016-07-03 01:48:26 +02:00
Yann Collet	c62cda9def	fileio.c no longer depends on zstd_internal.h	2016-07-03 01:36:57 +02:00
Yann Collet	302ff036f6	simplified repcodes for lazy_extDict	2016-07-03 01:28:16 +02:00
Yann Collet	9634f67107	fix lazy parser	2016-07-03 01:23:58 +02:00
Yann Collet	92d75667e4	fix for fast mode	2016-07-03 01:10:53 +02:00
Yann Collet	b2b5309c6f	removes Visual warning on strerror()	2016-07-03 00:20:03 +02:00
Yann Collet	5e734ad09b	revert fix	2016-07-02 23:55:34 +02:00
Yann Collet	f4b7b747c7	`-c` does no longer modify displayLevel when `-v` set.	2016-07-02 22:57:28 +02:00
Yann Collet	d96e8f20b9	updated NEWS	2016-07-02 22:03:08 +02:00
Yann Collet	0d5bf8f06f	fixed risk of segfault on very large files (multiple GB)	2016-07-02 21:39:47 +02:00
Yann Collet	ed7fb8413c	extended use of strerror(errno)	2016-07-02 21:01:54 +02:00
Yann Collet	b71adf45c1	extended use of strerror(errno)	2016-07-02 01:51:12 +02:00
Yann Collet	b7f197633a	use strerror(errno) to return system error message	2016-07-02 00:59:25 +02:00
Yann Collet	a7a5d7736c	Updated doc	2016-07-02 00:37:32 +02:00
Yann Collet	cebab1d322	Explicit error message when dst file cannot be created / opened	2016-07-02 00:12:14 +02:00
Yann Collet	2fa9904844	update specification and comments	2016-07-01 20:55:28 +02:00
Yann Collet	406dcca98a	updated frame specification with recommended max window size	2016-06-30 17:05:42 +02:00
Yann Collet	c35cd5b1db	New : Zstandard Frame format specification	2016-06-30 15:40:28 +02:00
Yann Collet	c093208ab8	fix : potential leak (#229 )	2016-06-30 14:07:30 +02:00
Yann Collet	6c6e1751f6	use ZSTD_getParams() to simplify code	2016-06-27 15:28:45 +02:00
Yann Collet	3d2cd7f816	Introduced ZSTD_getParams() bench now uses ZSTD_createCDict_advanced()	2016-06-27 15:12:26 +02:00
Yann Collet	ec224d256d	removed useless context	2016-06-27 13:39:30 +02:00
Yann Collet	4c56f4a3cf	fixed error messages	2016-06-27 13:36:54 +02:00
Yann Collet	529d9c7dee	updated version to v0.7.2	2016-06-27 10:03:10 +02:00
Yann Collet	d4f4e58ee1	fixed ZSTD_decompressBlock() using multiple blocks	2016-06-27 01:31:35 +02:00
Yann Collet	63b5e7a2ea	Improved comments	2016-06-26 17:42:15 +02:00
Yann ColletandGitHub	77054b452d	Merge pull request #222 from Cyan4973/dev Improved speed test	2016-06-23 01:32:56 +02:00
Yann ColletandGitHub	91d67f0226	Merge pull request #221 from inikep/dev test-zstd-speed.py: added --message in start and finish e-mail	2016-06-22 23:34:19 +02:00
inikep	82babfc0c6	test-zstd-speed.py: reordering of functions	2016-06-22 20:06:42 +02:00
inikep	116128ce41	test-zstd-speed.py: improved exception handling	2016-06-22 18:12:57 +02:00
inikep	47020670b6	test-zstd-speed.py: added --message in start and finish e-mail - support for ~ in filenames - result files ending with .txt	2016-06-22 17:11:01 +02:00
Yann ColletandGitHub	d4f5a176d4	Merge pull request #220 from inikep/dev improved test-zstd-speed.py	2016-06-22 16:30:26 +02:00
inikep	9649126f70	Merge remote-tracking branch 'refs/remotes/origin/speedTest' into dev	2016-06-22 16:22:29 +02:00
inikep	da1685e98d	Merge remote-tracking branch 'refs/remotes/Cyan4973/dev' into dev	2016-06-22 16:21:34 +02:00
inikep	d7d251ccb5	bench.c: added support for ZSTD_GIT_COMMIT	2016-06-22 16:13:25 +02:00
inikep	f2f59d758e	test-zstd-speed.py: added ZSTD_GIT_COMMIT	2016-06-22 15:42:26 +02:00
Yann ColletandGitHub	378822521a	Merge pull request #217 from Cyan4973/dev v0.7.1 tag	2016-06-22 14:07:04 +02:00
inikep	c364ee7867	test-zstd-speed.py: sent e-mail after KeyboardInterrupt	2016-06-22 14:01:53 +02:00
Yann Collet	3755eb8fea	fixed strict-aliasing warning on gcc6	2016-06-22 13:15:53 +02:00
inikep	bcb9aad727	test-zstd-speed.py: improved log and e-mail formatting	2016-06-22 13:07:58 +02:00
inikep	95da743a56	test-zstd-speed.py: send e-mail in case of error	2016-06-22 12:12:35 +02:00
Yann Collet	23042929da	Fixed : dictBuilder fails if first sample is too small	2016-06-22 11:05:34 +02:00
Yann ColletandGitHub	8d30cd562a	Merge pull request #219 from inikep/dev added gcc5test and gcc6test for Travis	2016-06-21 21:51:52 +02:00
inikep	2d9272f1dd	test-zstd-speed.py: improved formatting	2016-06-21 19:28:51 +02:00
inikep	c9bb102401	Makefile: added gcc5test and gcc6test improved Travis tests	2016-06-21 18:04:13 +02:00
Yann Collet	391a128794	fix : segfault in command line during automatic overwrite protection mode	2016-06-21 17:06:25 +02:00
Yann Collet	bda68c253b	refactored ZBUFF_compressEnd() for better maintainability	2016-06-21 15:18:11 +02:00
Yann Collet	446d442989	zbufftest now ensures ZBUFF_compressEnd() is fully flushed in a single pass if dst buffer is large enough	2016-06-21 14:14:02 +02:00
Yann Collet	aa29226b7c	fix : ZBUFF_compressEnd() gives right amount remaining to flush, including future epilogue	2016-06-21 14:04:57 +02:00
Yann Collet	f15c1cb00c	Fixed : ZBUFF_compressEnd() called multiple times with too small dst buffer (#206 )	2016-06-21 13:11:48 +02:00
Yann Collet	2565971ea4	stronger fuzzer tests for ZBUFF_compressEnd()	2016-06-21 12:17:26 +02:00
Yann Collet	a49e066b26	clarified comments on `ZSTD_compressContinue()`	2016-06-21 11:54:03 +02:00
inikep	d731de84ab	test-zstd-speed.py: added send_simple_email	2016-06-21 11:26:17 +02:00
inikep	9bf5357101	bench.c: use ZSTD_VERSION_STRING	2016-06-21 11:01:29 +02:00
Yann Collet	d4f38d0dcd	updated library to v0.7.1	2016-06-21 10:15:43 +02:00
Yann ColletandGitHub	d000042108	Merge pull request #216 from Cyan4973/dev v0.7.1	2016-06-21 08:43:04 +02:00
Yann Collet	22d76322ce	minor refactor	2016-06-21 08:01:51 +02:00
Yann Collet	a436a529bc	minor : fast_extDict does no longer skip first byte	2016-06-20 23:34:04 +02:00
Yann Collet	4623d11571	new correction, less extreme replacement value	2016-06-20 19:15:37 +02:00
Yann Collet	5477cc25f7	fixed corruption error related to inter-blocks rep-offset	2016-06-20 18:31:25 +02:00
Yann Collet	c98f8e7b57	Modified : zstd CLI enables checksum by default	2016-06-20 16:31:24 +02:00
Yann Collet	cbf8c52954	roundTripCrash automatic compression level now depends on first 128 bytes	2016-06-20 16:18:19 +02:00
Yann Collet	e4811ba761	Modified : ZSTD_createDDict() accepts dictionary < 8 bytes in pure content mode (reported by @chipturner)	2016-06-19 23:06:54 +02:00
Yann Collet	a050204342	Merge branch 'dev' of github.com:Cyan4973/zstd into dev	2016-06-19 14:27:54 +02:00
Yann Collet	06d9a73b48	minor refactor, using `WILDCOPY_OVERLENGTH` macro instead of hard-coded 8	2016-06-19 14:27:21 +02:00
Yann ColletandGitHub	06a40ac1a7	Merge pull request #213 from inikep/dev test-zstd-versions.py: test dictionary compression of v0.5.1 and newer	2016-06-17 16:15:34 +02:00
inikep	dba8b44370	Merge remote-tracking branch 'refs/remotes/Cyan4973/dev' into dev	2016-06-17 15:17:37 +02:00
inikep	e16f65675b	test-zstd-versions.py: fixed creation of dictionaries for v0.5.1+	2016-06-17 15:17:35 +02:00
inikep	7e3597bf38	test-zstd-versions.py: create and test dictionaries for v0.5.0	2016-06-17 14:43:24 +02:00
inikep	2ef16501ca	test-zstd-versions.py: improved error handling	2016-06-17 14:07:42 +02:00
inikep	4545671b44	test-zstd-versions.py: fixed DEVNULL	2016-06-17 13:39:43 +02:00
Yann Collet	19cab46f2f	Joined `seqStore` initialization at dispatch point	2016-06-17 12:54:52 +02:00
inikep	d1af4e66b6	test-zstd-versions.py: dictionary compression with levels 1, 5, 9, 15, 18	2016-06-16 20:23:11 +02:00
inikep	150152fb8a	test-zstd-versions.py: test dictiony compression for v0.5.1 and newer	2016-06-16 19:29:09 +02:00
inikep	24aa7b4238	test-zstd-versions.py: create dictionaries for v0.5.1 and newer	2016-06-16 14:15:32 +02:00