From 4b987ad8ce9fe0deb90853eace96a9fe9ef4622e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 10 Apr 2017 17:50:44 -0700 Subject: [PATCH 01/53] Introduce ZSTD_initCStream_internal() This is now the regroup point for ZSTD_initCStream*() functions ZSTD_initCStream_advanced() now properly checks for parameters validity. Also : added usage inside zstd_compress.c Needs ZSTD_DEBUG=1 macro to be triggered. Will be triggered by default from `tests` directory --- doc/zstd_manual.html | 14 +++++++++++--- lib/compress/zstd_compress.c | 30 ++++++++++++++++++++++++------ tests/Makefile | 4 +++- tests/zstreamtest.c | 10 ++++++++++ 4 files changed, 48 insertions(+), 10 deletions(-) diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html index a66f0a49c..a5b66f7d3 100644 --- a/doc/zstd_manual.html +++ b/doc/zstd_manual.html @@ -497,14 +497,22 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v

Advanced streaming functions


 
 
Advanced Streaming compression functions
ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
+size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);   /**< size of CStream is variable, depending primarily on compression level */
 size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);   /**< pledgedSrcSize must be correct, a size of 0 means unknown.  for a frame size of 0 use initCStream_advanced */
 size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */
 size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
                                              ZSTD_parameters params, unsigned long long pledgedSrcSize);  /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */
 size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);  /**< note : cdict will just be referenced, and must outlive compression session */
-size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);  /**< re-use compression parameters from previous init; skip dictionary loading stage; zcs must be init at least once before. note: pledgedSrcSize must be correct, a size of 0 means unknown.  for a frame size of 0 use initCStream_advanced */
-size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
 


+
size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
+
  start a new compression job, using same parameters from previous job.
+  This is typically useful to skip dictionary loading stage, since it will re-use it in-place..
+  Note that zcs must be init at least once before using ZSTD_resetCStream().
+  pledgedSrcSize==0 means "srcSize unknown".
+  If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end.
+  @return : 0, or an error code (which can be tested using ZSTD_isError()) 
+


+
 
Advanced Streaming decompression functions
typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e;
 ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
 size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */
@@ -553,7 +561,7 @@ size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
 size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
 size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */
 size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**<  note: if pledgedSrcSize can be 0, indicating unknown size.  if it is non-zero, it must be accurate.  for 0 size frames, use compressBegin_advanced */
-size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize can be 0, indicating unknown size.  if it is non-zero, it must be accurate.  for 0 size frames, use compressBegin_advanced */
+size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize); /**< note: fail if cdict==NULL. pledgedSrcSize can be 0, indicating unknown size.  For 0 size frames, use compressBegin_advanced */
 size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 


diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 4ad978a4b..c0ccf83cd 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -31,7 +31,14 @@ typedef enum { ZSTDcs_created=0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZS
 /*-*************************************
 *  Helper functions
 ***************************************/
+#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG==1)
+#  include 
+#else
+#  define assert(condition) ((void)0)
+#endif
+
 #define ZSTD_STATIC_ASSERT(c) { enum { ZSTD_static_assert = 1/(int)(!!(c)) }; }
+
 size_t ZSTD_compressBound(size_t srcSize) {
     size_t const lowLimit = 256 KB;
     size_t const margin = (srcSize < lowLimit) ? (lowLimit-srcSize) >> 12 : 0;  /* from 64 to 0 */
@@ -3031,10 +3038,13 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
     return ZSTD_resetCStream_internal(zcs, pledgedSrcSize);
 }
 
-size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
-                                 const void* dict, size_t dictSize,
-                                 ZSTD_parameters params, unsigned long long pledgedSrcSize)
+/* ZSTD_initCStream_internal() :
+ * params are supposed validated at this stage */
+static size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
+                                const void* dict, size_t dictSize,
+                                ZSTD_parameters params, unsigned long long pledgedSrcSize)
 {
+    assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
     /* allocate buffers */
     {   size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog;
         if (zcs->inBuffSize < neededInBuffSize) {
@@ -3068,11 +3078,19 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
     return ZSTD_resetCStream_internal(zcs, pledgedSrcSize);
 }
 
+size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
+                                 const void* dict, size_t dictSize,
+                                 ZSTD_parameters params, unsigned long long pledgedSrcSize)
+{
+    CHECK_F( ZSTD_checkCParams(params.cParams) );
+    return ZSTD_initCStream_internal(zcs, dict, dictSize, params, pledgedSrcSize);
+}
+
 /* note : cdict must outlive compression session */
 size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict)
 {
     ZSTD_parameters const params = ZSTD_getParamsFromCDict(cdict);
-    size_t const initError =  ZSTD_initCStream_advanced(zcs, NULL, 0, params, 0);
+    size_t const initError =  ZSTD_initCStream_internal(zcs, NULL, 0, params, 0);
     zcs->cdict = cdict;
     if (ZSTD_isError(initError)) return initError;
     return ZSTD_resetCStream_internal(zcs, 0);
@@ -3081,14 +3099,14 @@ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict)
 size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel)
 {
     ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize);
-    return ZSTD_initCStream_advanced(zcs, dict, dictSize, params, 0);
+    return ZSTD_initCStream_internal(zcs, dict, dictSize, params, 0);
 }
 
 size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize)
 {
     ZSTD_parameters params = ZSTD_getParams(compressionLevel, pledgedSrcSize, 0);
     if (pledgedSrcSize) params.fParams.contentSizeFlag = 1;
-    return ZSTD_initCStream_advanced(zcs, NULL, 0, params, pledgedSrcSize);
+    return ZSTD_initCStream_internal(zcs, NULL, 0, params, pledgedSrcSize);
 }
 
 size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel)
diff --git a/tests/Makefile b/tests/Makefile
index 809c90df5..70ef51878 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -26,7 +26,9 @@ PYTHON ?= python3
 TESTARTEFACT := versionsTest namespaceTest
 
 
-CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR)
+CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
+           -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) \
+           -DZSTD_DEBUG=1
 CFLAGS  ?= -O3
 CFLAGS  += -g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
            -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 270c0221b..6f26ea555 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -207,6 +207,16 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
       DISPLAYLEVEL(3, "OK (%u bytes) \n", (U32)s);
     }
 
+    /* Attempt bad compression parameters */
+    DISPLAYLEVEL(3, "test%3i : use bad compression parameters : ", testNb++);
+    {   size_t r;
+        ZSTD_parameters params = ZSTD_getParams(1, 0, 0);
+        params.cParams.searchLength = 2;
+        r = ZSTD_initCStream_advanced(zc, NULL, 0, params, 0);
+        if (!ZSTD_isError(r)) goto _output_error;
+        DISPLAYLEVEL(3, "init error : %s \n", ZSTD_getErrorName(r));
+    }
+
     /* skippable frame test */
     DISPLAYLEVEL(3, "test%3i : decompress skippable frame : ", testNb++);
     ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);

From e88034fe2639f2d83470df861e1ce95676113236 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 10 Apr 2017 22:24:02 -0700
Subject: [PATCH 02/53] simplified ZSTD_initCStream*() flow

all variants converge towards ZSTD_initCStream_stage2()
---
 lib/compress/zstd_compress.c | 60 +++++++++++++++++++++---------------
 1 file changed, 36 insertions(+), 24 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index c0ccf83cd..a4a3ee084 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -3034,17 +3034,18 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
 {
 
     zcs->params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
-
     return ZSTD_resetCStream_internal(zcs, pledgedSrcSize);
 }
 
 /* ZSTD_initCStream_internal() :
- * params are supposed validated at this stage */
-static size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
-                                const void* dict, size_t dictSize,
-                                ZSTD_parameters params, unsigned long long pledgedSrcSize)
+ * params are supposed validated at this stage
+ * and zcs->cdict is supposed to be correct */
+static size_t ZSTD_initCStream_stage2(ZSTD_CStream* zcs,
+                                const ZSTD_parameters params,
+                                unsigned long long pledgedSrcSize)
 {
     assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
+
     /* allocate buffers */
     {   size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog;
         if (zcs->inBuffSize < neededInBuffSize) {
@@ -3065,19 +3066,39 @@ static size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
         zcs->outBuffSize = outBuffSize;
     }
 
-    if (dict && dictSize >= 8) {
-        ZSTD_freeCDict(zcs->cdictLocal);
-        zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, 0, params, zcs->customMem);
-        if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
-        zcs->cdict = zcs->cdictLocal;
-    } else zcs->cdict = NULL;
-
     zcs->checksum = params.fParams.checksumFlag > 0;
     zcs->params = params;
 
     return ZSTD_resetCStream_internal(zcs, pledgedSrcSize);
 }
 
+/* note : cdict must outlive compression session */
+size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict)
+{
+    if (!cdict) return ERROR(GENERIC);   /* cannot handle NULL cdict (does not know what to do) */
+    {   ZSTD_parameters const params = ZSTD_getParamsFromCDict(cdict);
+        zcs->cdict = cdict;
+        return ZSTD_initCStream_stage2(zcs, params, 0);
+    }
+}
+
+static size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
+                                const void* dict, size_t dictSize,
+                                ZSTD_parameters params, unsigned long long pledgedSrcSize)
+{
+    assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
+    zcs->cdict = NULL;
+
+    if (dict && dictSize >= 8) {
+        ZSTD_freeCDict(zcs->cdictLocal);
+        zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, 0 /* copy */, params, zcs->customMem);
+        if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
+        zcs->cdict = zcs->cdictLocal;
+    }
+
+    return ZSTD_initCStream_stage2(zcs, params, pledgedSrcSize);
+}
+
 size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
                                  const void* dict, size_t dictSize,
                                  ZSTD_parameters params, unsigned long long pledgedSrcSize)
@@ -3086,16 +3107,6 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
     return ZSTD_initCStream_internal(zcs, dict, dictSize, params, pledgedSrcSize);
 }
 
-/* note : cdict must outlive compression session */
-size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict)
-{
-    ZSTD_parameters const params = ZSTD_getParamsFromCDict(cdict);
-    size_t const initError =  ZSTD_initCStream_internal(zcs, NULL, 0, params, 0);
-    zcs->cdict = cdict;
-    if (ZSTD_isError(initError)) return initError;
-    return ZSTD_resetCStream_internal(zcs, 0);
-}
-
 size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel)
 {
     ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize);
@@ -3105,13 +3116,14 @@ size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t di
 size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize)
 {
     ZSTD_parameters params = ZSTD_getParams(compressionLevel, pledgedSrcSize, 0);
-    if (pledgedSrcSize) params.fParams.contentSizeFlag = 1;
+    params.fParams.contentSizeFlag = (pledgedSrcSize>0);
     return ZSTD_initCStream_internal(zcs, NULL, 0, params, pledgedSrcSize);
 }
 
 size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel)
 {
-    return ZSTD_initCStream_usingDict(zcs, NULL, 0, compressionLevel);
+    ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, 0);
+    return ZSTD_initCStream_internal(zcs, NULL, 0, params, 0);
 }
 
 size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs)

From ab9162ebb49ff00e2df0431e3b2a9de8510604b6 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 11 Apr 2017 10:46:20 -0700
Subject: [PATCH 03/53] simplified call graph

by calling ZSTD_compressBegin_internal() instead of ZSTD_compressBegin_advanced()
---
 lib/compress/zstd_compress.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index a4a3ee084..62c1c6cdd 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2690,6 +2690,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
                                    ZSTD_parameters params, U64 pledgedSrcSize)
 {
     ZSTD_compResetPolicy_e const crp = dictSize ? ZSTDcrp_fullReset : ZSTDcrp_continue;
+    assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
     CHECK_F(ZSTD_resetCCtx_advanced(cctx, params, pledgedSrcSize, crp));
     return ZSTD_compress_insertDictionary(cctx, dict, dictSize);
 }
@@ -2911,7 +2912,7 @@ size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, u
     else {
         ZSTD_parameters params = cdict->refContext->params;
         params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
-        CHECK_F(ZSTD_compressBegin_advanced(cctx, NULL, 0, params, pledgedSrcSize));
+        CHECK_F(ZSTD_compressBegin_internal(cctx, NULL, 0, params, pledgedSrcSize));
     }
     return 0;
 }
@@ -3017,7 +3018,7 @@ static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs, unsigned long long p
     if (zcs->inBuffSize==0) return ERROR(stage_wrong);   /* zcs has not been init at least once => can't reset */
 
     if (zcs->cdict) CHECK_F(ZSTD_compressBegin_usingCDict(zcs->cctx, zcs->cdict, pledgedSrcSize))
-    else CHECK_F(ZSTD_compressBegin_advanced(zcs->cctx, NULL, 0, zcs->params, pledgedSrcSize));
+    else CHECK_F(ZSTD_compressBegin_internal(zcs->cctx, NULL, 0, zcs->params, pledgedSrcSize));
 
     zcs->inToCompress = 0;
     zcs->inBuffPos = 0;

From 4ee6b15dac055e7e653b1fbfd7f65755f42597c8 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 11 Apr 2017 11:59:44 -0700
Subject: [PATCH 04/53] force contentSizeFlag=0 when using
 ZSTD_initCStream_usingCDict()

because by definition srcSize is not known when using this prototype.
added relevant test

Note : this use was already working, because at a later stage
(both ZSTD_compressBegin_usingCDict() and ZSTD_copyCCtx())
pledgedSrcSize=0 is translated into "unknown", no matter the frame parameter.
This is not correct, but of little importance,
as the medium term plan is to no longer set fParams within CDict
---
 lib/compress/zstd_compress.c |  3 ++-
 tests/zstreamtest.c          | 14 +++++++++-----
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 62c1c6cdd..0ce2054fc 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -3077,7 +3077,8 @@ static size_t ZSTD_initCStream_stage2(ZSTD_CStream* zcs,
 size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict)
 {
     if (!cdict) return ERROR(GENERIC);   /* cannot handle NULL cdict (does not know what to do) */
-    {   ZSTD_parameters const params = ZSTD_getParamsFromCDict(cdict);
+    {   ZSTD_parameters params = ZSTD_getParamsFromCDict(cdict);
+        params.fParams.contentSizeFlag = 0;
         zcs->cdict = cdict;
         return ZSTD_initCStream_stage2(zcs, params, 0);
     }
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 6f26ea555..ac200a7a6 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -448,10 +448,11 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
       if (!ZSTD_isError(r)) goto _output_error;  /* must fail : frame requires > 100 bytes */
       DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r)); }
 
-    DISPLAYLEVEL(3, "test%3i : check dictionary used : ", testNb++);
+    DISPLAYLEVEL(3, "test%3i : dictionary compression with masked dictID : ", testNb++);
     {   ZSTD_parameters params = ZSTD_getParams(1, CNBufferSize, dictionary.filled);
         ZSTD_CDict* cdict;
         params.fParams.noDictIDFlag = 1;
+        params.fParams.contentSizeFlag = 1;  /* test contentSize, should be disabled with initCStream_usingCDict */
         cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, 1, params, customMem);
         { size_t const initError = ZSTD_initCStream_usingCDict(zc, cdict);
           if (ZSTD_isError(initError)) goto _output_error; }
@@ -470,17 +471,20 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
         cSize = outBuff.pos;
         ZSTD_freeCDict(cdict);
         DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBufferSize*100);
+    }
 
-        { size_t const r = ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize);
-          if (!ZSTD_isError(r)) goto _output_error; /* must fail : dictionary not used */
-          DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r)); }
+    DISPLAYLEVEL(3, "test%3i : decompress without dictionary : ", testNb++);
+    {   size_t const r = ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize);
+        if (!ZSTD_isError(r)) goto _output_error;  /* must fail : dictionary not used */
+        DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r));
     }
 
     /* Unknown srcSize */
     DISPLAYLEVEL(3, "test%3i : pledgedSrcSize == 0 behaves properly : ", testNb++);
     {   ZSTD_parameters params = ZSTD_getParams(5, 0, 0);
         params.fParams.contentSizeFlag = 1;
-        ZSTD_initCStream_advanced(zc, NULL, 0, params, 0); } /* cstream advanced should write the 0 size field */
+        ZSTD_initCStream_advanced(zc, NULL, 0, params, 0);
+    } /* cstream advanced shall write content size = 0 */
     inBuff.src = CNBuffer;
     inBuff.size = 0;
     inBuff.pos = 0;

From 0e30059ba1cba30730174bc44c7a69e5bd7c3c5d Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 11 Apr 2017 14:41:02 -0700
Subject: [PATCH 05/53] cli : FIO_createDictBuffer() replaces FIO_loadFile()

makes it more explicit that it allocates a buffer
and that it's meant to be used for dictionary.

Also : simplified function a bit,
now only works for dictionaries up to DICTSIZE_MAX
---
 programs/fileio.c | 29 +++++++++++------------------
 1 file changed, 11 insertions(+), 18 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index b384c3c2b..a3bc7e913 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -76,7 +76,7 @@
 
 #define CACHELINE 64
 
-#define MAX_DICT_SIZE (8 MB)   /* protection against large input (attack scenario) */
+#define DICTSIZE_MAX (32 MB)   /* protection against large input (attack scenario) */
 
 #define FNSPACE 30
 
@@ -278,13 +278,13 @@ static FILE* FIO_openDstFile(const char* dstFileName)
 }
 
 
-/*! FIO_loadFile() :
-*   creates a buffer, pointed by `*bufferPtr`,
-*   loads `filename` content into it,
-*   up to MAX_DICT_SIZE bytes.
-*   @return : loaded size
-*/
-static size_t FIO_loadFile(void** bufferPtr, const char* fileName)
+/*! FIO_createDictBuffer() :
+ *  creates a buffer, pointed by `*bufferPtr`,
+ *  loads `filename` content into it, up to DICTSIZE_MAX bytes.
+ *  @return : loaded size
+ *  if fileName==NULL, returns 0 and a NULL pointer
+ */
+static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName)
 {
     FILE* fileHandle;
     U64 fileSize;
@@ -296,14 +296,7 @@ static size_t FIO_loadFile(void** bufferPtr, const char* fileName)
     fileHandle = fopen(fileName, "rb");
     if (fileHandle==0) EXM_THROW(31, "zstd: %s: %s", fileName, strerror(errno));
     fileSize = UTIL_getFileSize(fileName);
-    if (fileSize > MAX_DICT_SIZE) {
-        int seekResult;
-        if (fileSize > 1 GB) EXM_THROW(32, "Dictionary file %s is too large", fileName);   /* avoid extreme cases */
-        DISPLAYLEVEL(2,"Dictionary %s is too large : using last %u bytes only \n", fileName, (U32)MAX_DICT_SIZE);
-        seekResult = fseek(fileHandle, (long int)(fileSize-MAX_DICT_SIZE), SEEK_SET);   /* use end of file */
-        if (seekResult != 0) EXM_THROW(33, "zstd: %s: %s", fileName, strerror(errno));
-        fileSize = MAX_DICT_SIZE;
-    }
+    if (fileSize > DICTSIZE_MAX) EXM_THROW(32, "Dictionary file %s is too large (> %u MB)", fileName, DICTSIZE_MAX >> 20);   /* avoid extreme cases */
     *bufferPtr = malloc((size_t)fileSize);
     if (*bufferPtr==NULL) EXM_THROW(34, "zstd: %s", strerror(errno));
     { size_t const readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle);
@@ -356,7 +349,7 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
 
     /* dictionary */
     {   void* dictBuffer;
-        size_t const dictBuffSize = FIO_loadFile(&dictBuffer, dictFileName);
+        size_t const dictBuffSize = FIO_createDictBuffer(&dictBuffer, dictFileName);
         if (dictFileName && (dictBuffer==NULL)) EXM_THROW(32, "zstd: allocation error : can't create dictBuffer");
         {   ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize);
             params.fParams.contentSizeFlag = srcRegFile;
@@ -776,7 +769,7 @@ static dRess_t FIO_createDResources(const char* dictFileName)
 
     /* dictionary */
     {   void* dictBuffer;
-        size_t const dictBufferSize = FIO_loadFile(&dictBuffer, dictFileName);
+        size_t const dictBufferSize = FIO_createDictBuffer(&dictBuffer, dictFileName);
         size_t const initError = ZSTD_initDStream_usingDict(ress.dctx, dictBuffer, dictBufferSize);
         if (ZSTD_isError(initError)) EXM_THROW(61, "ZSTD_initDStream_usingDict error : %s", ZSTD_getErrorName(initError));
         free(dictBuffer);

From 5c42d0edc88e75fdc6ee9cf3e400e3dedd670162 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 11 Apr 2017 16:57:32 -0700
Subject: [PATCH 06/53] cli : better status display for zstdmt in 1-thread mode

---
 programs/fileio.c | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index a3bc7e913..f1f5a5577 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -349,7 +349,7 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
 
     /* dictionary */
     {   void* dictBuffer;
-        size_t const dictBuffSize = FIO_createDictBuffer(&dictBuffer, dictFileName);
+        size_t const dictBuffSize = FIO_createDictBuffer(&dictBuffer, dictFileName);   /* works with dictFileName==NULL */
         if (dictFileName && (dictBuffer==NULL)) EXM_THROW(32, "zstd: allocation error : can't create dictBuffer");
         {   ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize);
             params.fParams.contentSizeFlag = srcRegFile;
@@ -361,7 +361,7 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
             if (comprParams->searchLog) params.cParams.searchLog = comprParams->searchLog;
             if (comprParams->searchLength) params.cParams.searchLength = comprParams->searchLength;
             if (comprParams->targetLength) params.cParams.targetLength = comprParams->targetLength;
-            if (comprParams->strategy) params.cParams.strategy = (ZSTD_strategy)(comprParams->strategy - 1);
+            if (comprParams->strategy) params.cParams.strategy = (ZSTD_strategy)(comprParams->strategy - 1);   /* 0 means : do not change */
 #ifdef ZSTD_MULTITHREAD
             {   size_t const errorCode = ZSTDMT_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize);
                 if (ZSTD_isError(errorCode)) EXM_THROW(33, "Error initializing CStream : %s", ZSTD_getErrorName(errorCode));
@@ -567,8 +567,8 @@ static int FIO_compressFilename_internal(cRess_t ress,
         readsize += inSize;
 
         {   ZSTD_inBuffer  inBuff = { ress.srcBuffer, inSize, 0 };
-            while (inBuff.pos != inBuff.size) {   /* note : is there any possibility of endless loop ? for example, if outBuff is not large enough ? */
-                ZSTD_outBuffer outBuff= { ress.dstBuffer, ress.dstBufferSize, 0 };
+            while (inBuff.pos != inBuff.size) {
+                ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 };
 #ifdef ZSTD_MULTITHREAD
                 size_t const result = ZSTDMT_compressStream(ress.cctx, &outBuff, &inBuff);
 #else
@@ -582,13 +582,13 @@ static int FIO_compressFilename_internal(cRess_t ress,
                     if (sizeCheck!=outBuff.pos) EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName);
                     compressedfilesize += outBuff.pos;
         }   }   }
-#ifdef ZSTD_MULTITHREAD
-        if (!fileSize) DISPLAYUPDATE(2, "\rRead : %u MB", (U32)(readsize>>20))
-        else DISPLAYUPDATE(2, "\rRead : %u / %u MB", (U32)(readsize>>20), (U32)(fileSize>>20));
-#else
-        if (!fileSize) DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%", (U32)(readsize>>20), (double)compressedfilesize/readsize*100)
-        else DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%", (U32)(readsize>>20), (U32)(fileSize>>20), (double)compressedfilesize/readsize*100);
-#endif
+        if (g_nbThreads > 1) {
+            if (!fileSize) DISPLAYUPDATE(2, "\rRead : %u MB", (U32)(readsize>>20))
+            else DISPLAYUPDATE(2, "\rRead : %u / %u MB", (U32)(readsize>>20), (U32)(fileSize>>20));
+        } else {
+            if (!fileSize) DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%", (U32)(readsize>>20), (double)compressedfilesize/readsize*100)
+            else DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%", (U32)(readsize>>20), (U32)(fileSize>>20), (double)compressedfilesize/readsize*100);
+        }
     }
 
     /* End of Frame */

From eb70d219fdbd0834f83bf2c66551a4b3fa145de0 Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Tue, 11 Apr 2017 17:15:13 -0700
Subject: [PATCH 07/53] Add test of file > 4GB to playTests

---
 tests/playTests.sh | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/tests/playTests.sh b/tests/playTests.sh
index 266bbd91b..6ef5914a9 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -11,6 +11,7 @@ roundTripTest() {
         local_p="$2"
     else
         local_c="$2"
+        local_p=""
     fi
 
     rm -f tmp1 tmp2
@@ -20,6 +21,23 @@ roundTripTest() {
     $DIFF -q tmp1 tmp2
 }
 
+fileRoundTripTest() {
+    if [ -n "$3" ]; then
+        local_c="$3"
+        local_p="$2"
+    else
+        local_c="$2"
+        local_p=""
+    fi
+
+    rm -f tmp.zstd tmp.md5.1 tmp.md5.2
+    $ECHO "fileRoundTripTest: ./datagen $1 $local_p > tmp1 && $ZSTD -v$local_c -c | $ZSTD -d | $MD5SUM > tmp.md5.2"
+    ./datagen $1 $local_p > tmp
+    cat tmp | $MD5SUM > tmp.md5.1
+    $ZSTD --ultra -v$local_c -c tmp | $ZSTD -d | $MD5SUM > tmp.md5.2
+    $DIFF -q tmp.md5.1 tmp.md5.2
+}
+
 isTerminal=false
 if [ -t 0 ] && [ -t 1 ]
 then
@@ -441,6 +459,8 @@ roundTripTest -g519K 6    # greedy, hash chain
 roundTripTest -g517K 16   # btlazy2
 roundTripTest -g516K 19   # btopt
 
+fileRoundTripTest -g500K
+
 rm tmp*
 
 if [ "$1" != "--test-large-data" ]; then
@@ -476,4 +496,6 @@ roundTripTest -g50000000 -P94 19
 roundTripTest -g99000000 -P99 20
 roundTripTest -g6000000000 -P99 1
 
+fileRoundTripTest -g4193M -P99 1
+
 rm tmp*

From d37e1df2ab26b9cbb22e6b602ea5da9b3662af89 Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Tue, 11 Apr 2017 17:33:26 -0700
Subject: [PATCH 08/53] Fix message

---
 tests/playTests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/playTests.sh b/tests/playTests.sh
index 6ef5914a9..3675bb168 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -31,7 +31,7 @@ fileRoundTripTest() {
     fi
 
     rm -f tmp.zstd tmp.md5.1 tmp.md5.2
-    $ECHO "fileRoundTripTest: ./datagen $1 $local_p > tmp1 && $ZSTD -v$local_c -c | $ZSTD -d | $MD5SUM > tmp.md5.2"
+    $ECHO "fileRoundTripTest: ./datagen $1 $local_p > tmp && $ZSTD -v$local_c -c | $ZSTD -d"
     ./datagen $1 $local_p > tmp
     cat tmp | $MD5SUM > tmp.md5.1
     $ZSTD --ultra -v$local_c -c tmp | $ZSTD -d | $MD5SUM > tmp.md5.2

From 20d5e03893004cf236297808a0ee51a2a94d506b Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 11 Apr 2017 18:34:02 -0700
Subject: [PATCH 09/53] content size is controlled at bufferless level

so it's active for all entry points

Also : added relevant test (wrong content size) in fuzzer
---
 lib/common/error_private.c     |  2 +-
 lib/compress/zstd_compress.c   | 13 ++++++++++---
 lib/compress/zstdmt_compress.c | 11 ++++++-----
 tests/fuzzer.c                 | 30 ++++++++++++++++++++++--------
 4 files changed, 39 insertions(+), 17 deletions(-)

diff --git a/lib/common/error_private.c b/lib/common/error_private.c
index 44ae20104..b3287245f 100644
--- a/lib/common/error_private.c
+++ b/lib/common/error_private.c
@@ -29,7 +29,7 @@ const char* ERR_getErrorString(ERR_enum code)
     case PREFIX(memory_allocation): return "Allocation error : not enough memory";
     case PREFIX(stage_wrong): return "Operation not authorized at current processing stage";
     case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
-    case PREFIX(srcSize_wrong): return "Src size incorrect";
+    case PREFIX(srcSize_wrong): return "Src size is incorrect";
     case PREFIX(corruption_detected): return "Corrupted block detected";
     case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
     case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 0ce2054fc..110698c2f 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -81,6 +81,7 @@ struct ZSTD_CCtx_s {
     size_t workSpaceSize;
     size_t blockSize;
     U64 frameContentSize;
+    U64 consumedSrcSize;
     XXH64_state_t xxhState;
     ZSTD_customMem customMem;
 
@@ -241,6 +242,7 @@ static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_parameters params, U64 fra
     U32 const end = (U32)(cctx->nextSrc - cctx->base);
     cctx->params = params;
     cctx->frameContentSize = frameContentSize;
+    cctx->consumedSrcSize = 0;
     cctx->lowLimit = end;
     cctx->dictLimit = end;
     cctx->nextToUpdate = end+1;
@@ -313,6 +315,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
         zc->params = params;
         zc->blockSize = blockSize;
         zc->frameContentSize = frameContentSize;
+        zc->consumedSrcSize = 0;
         { int i; for (i=0; irep[i] = repStartValue[i]; }
 
         if ((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btopt2)) {
@@ -2493,6 +2496,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx,
                              ZSTD_compress_generic (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) :
                              ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize);
         if (ZSTD_isError(cSize)) return cSize;
+        cctx->consumedSrcSize += srcSize;
         return cSize + fhSize;
     } else
         return fhSize;
@@ -2503,7 +2507,7 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* cctx,
                               void* dst, size_t dstCapacity,
                         const void* src, size_t srcSize)
 {
-    return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1, 0);
+    return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1 /* frame mode */, 0 /* last chunk */);
 }
 
 
@@ -2516,7 +2520,7 @@ size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const
 {
     size_t const blockSizeMax = ZSTD_getBlockSizeMax(cctx);
     if (srcSize > blockSizeMax) return ERROR(srcSize_wrong);
-    return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0, 0);
+    return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0 /* frame mode */, 0 /* last chunk */);
 }
 
 /*! ZSTD_loadDictionaryContent() :
@@ -2767,10 +2771,13 @@ size_t ZSTD_compressEnd (ZSTD_CCtx* cctx,
                    const void* src, size_t srcSize)
 {
     size_t endResult;
-    size_t const cSize = ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1, 1);
+    size_t const cSize = ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1 /* frame mode */, 1 /* last chunk */);
     if (ZSTD_isError(cSize)) return cSize;
     endResult = ZSTD_writeEpilogue(cctx, (char*)dst + cSize, dstCapacity-cSize);
     if (ZSTD_isError(endResult)) return endResult;
+    if (cctx->params.fParams.contentSizeFlag) {  /* control src size */
+        if (cctx->frameContentSize != cctx->consumedSrcSize) return ERROR(srcSize_wrong);
+    }
     return cSize + endResult;
 }
 
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index c9c3e8533..50c98ae93 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -235,11 +235,12 @@ void ZSTDMT_compressChunk(void* jobDescription)
         if (job->cdict) DEBUGLOG(3, "using CDict ");
         if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; }
     } else {  /* srcStart points at reloaded section */
-        size_t const dictModeError = ZSTD_setCCtxParameter(job->cctx, ZSTD_p_forceRawDict, 1);  /* Force loading dictionary in "content-only" mode (no header analysis) */
-        size_t const initError = ZSTD_compressBegin_advanced(job->cctx, job->srcStart, job->dictSize, job->params, job->fullFrameSize);
-        if (ZSTD_isError(initError) || ZSTD_isError(dictModeError)) { job->cSize = initError; goto _endJob; }
-        ZSTD_setCCtxParameter(job->cctx, ZSTD_p_forceWindow, 1);
-    }
+        if (!job->firstChunk) job->params.fParams.contentSizeFlag = 0;  /* ensure no srcSize control */
+        {   size_t const dictModeError = ZSTD_setCCtxParameter(job->cctx, ZSTD_p_forceRawDict, 1);  /* Force loading dictionary in "content-only" mode (no header analysis) */
+            size_t const initError = ZSTD_compressBegin_advanced(job->cctx, job->srcStart, job->dictSize, job->params, job->fullFrameSize);
+            if (ZSTD_isError(initError) || ZSTD_isError(dictModeError)) { job->cSize = initError; goto _endJob; }
+            ZSTD_setCCtxParameter(job->cctx, ZSTD_p_forceWindow, 1);
+    }   }
     if (!job->firstChunk) {  /* flush and overwrite frame header when it's not first segment */
         size_t const hSize = ZSTD_compressContinue(job->cctx, dstBuff.start, dstBuff.size, src, 0);
         if (ZSTD_isError(hSize)) { job->cSize = hSize; goto _endJob; }
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 342c953e9..cdbbe8a22 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -524,9 +524,9 @@ static int basicUnitTests(U32 seed, double compressibility)
 
     /* Decompression defense tests */
     DISPLAYLEVEL(4, "test%3i : Check input length for magic number : ", testNb++);
-    { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 3);
+    { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 3);   /* too small input */
       if (!ZSTD_isError(r)) goto _output_error;
-      if (r != (size_t)-ZSTD_error_srcSize_wrong) goto _output_error; }
+      if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
     DISPLAYLEVEL(4, "OK \n");
 
     DISPLAYLEVEL(4, "test%3i : Check magic Number : ", testNb++);
@@ -535,6 +535,22 @@ static int basicUnitTests(U32 seed, double compressibility)
       if (!ZSTD_isError(r)) goto _output_error; }
     DISPLAYLEVEL(4, "OK \n");
 
+    /* content size verification test */
+    DISPLAYLEVEL(4, "test%3i : Content size verification : ", testNb++);
+    {   ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+        size_t const srcSize = 5000;
+        size_t const wrongSrcSize = (srcSize + 1000);
+        ZSTD_parameters params = ZSTD_getParams(1, wrongSrcSize, 0);
+        params.fParams.contentSizeFlag = 1;
+        {   size_t const result = ZSTD_compressBegin_advanced(cctx, NULL, 0, params, wrongSrcSize);
+            if (ZSTD_isError(result)) goto _output_error;
+        }
+        {   size_t const result = ZSTD_compressEnd(cctx, decodedBuffer, CNBuffSize, CNBuffer, srcSize);
+            if (!ZSTD_isError(result)) goto _output_error;
+            if (ZSTD_getErrorCode(result) != ZSTD_error_srcSize_wrong) goto _output_error;
+            DISPLAYLEVEL(4, "OK : %s \n", ZSTD_getErrorName(result));
+    }   }
+
     /* block API tests */
     {   ZSTD_CCtx* const cctx = ZSTD_createCCtx();
         static const size_t dictSize = 65 KB;
@@ -788,12 +804,10 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
         crcOrig = XXH64(sampleBuffer, sampleSize, 0);
 
         /* compression tests */
-        {
-            unsigned const cLevel =
-                    (FUZ_rand(&lseed) %
-                     (ZSTD_maxCLevel() -
-                      (FUZ_highbit32((U32)sampleSize) / cLevelLimiter))) +
-                    1;
+        {   unsigned const cLevel =
+                    ( FUZ_rand(&lseed) %
+                     (ZSTD_maxCLevel() - (FUZ_highbit32((U32)sampleSize) / cLevelLimiter)) )
+                     + 1;
             cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel);
             CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed : %s", ZSTD_getErrorName(cSize));
 

From 88009a8ba27820f9a261d1f48cc8f307eb546b35 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 12 Apr 2017 00:51:24 -0700
Subject: [PATCH 10/53] removed srcSize control from CStream

since it's already done from lower bufferless API level
---
 lib/compress/zstd_compress.c | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 110698c2f..d6774372b 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2971,7 +2971,6 @@ struct ZSTD_CStream_s {
     U32    checksum;
     U32    frameEnded;
     U64    pledgedSrcSize;
-    U64    inputProcessed;
     ZSTD_parameters params;
     ZSTD_customMem customMem;
 };   /* typedef'd to ZSTD_CStream within "zstd.h" */
@@ -3034,7 +3033,6 @@ static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs, unsigned long long p
     zcs->stage = zcss_load;
     zcs->frameEnded = 0;
     zcs->pledgedSrcSize = pledgedSrcSize;
-    zcs->inputProcessed = 0;
     return 0;   /* ready to go */
 }
 
@@ -3226,7 +3224,6 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
 
     *srcSizePtr = ip - istart;
     *dstCapacityPtr = op - ostart;
-    zcs->inputProcessed += *srcSizePtr;
     if (zcs->frameEnded) return 0;
     {   size_t hintInSize = zcs->inBuffTarget - zcs->inBuffPos;
         if (hintInSize==0) hintInSize = zcs->blockSize;
@@ -3271,9 +3268,6 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
     BYTE* const oend = (BYTE*)(output->dst) + output->size;
     BYTE* op = ostart;
 
-    if ((zcs->pledgedSrcSize) && (zcs->inputProcessed != zcs->pledgedSrcSize))
-        return ERROR(srcSize_wrong);   /* pledgedSrcSize not respected */
-
     if (zcs->stage != zcss_final) {
         /* flush whatever remains */
         size_t srcSize = 0;
@@ -3289,6 +3283,7 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
         zcs->stage = zcss_final;
         zcs->outBuffContentSize = !notEnded ? 0 :
             ZSTD_compressEnd(zcs->cctx, zcs->outBuff, zcs->outBuffSize, NULL, 0);  /* write epilogue, including final empty block, into outBuff */
+        if (ZSTD_isError(zcs->outBuffContentSize)) return zcs->outBuffContentSize;
     }
 
     /* flush epilogue */

From afa48518e2fbd5dfce3982e4ed003f4c707d1801 Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Thu, 13 Apr 2017 12:28:28 -0700
Subject: [PATCH 11/53] -T0 detects number of physical cores

---
 programs/util.h    | 172 +++++++++++++++++++++++++++++++++++++++++++++
 programs/zstdcli.c |   5 ++
 2 files changed, 177 insertions(+)

diff --git a/programs/util.h b/programs/util.h
index 9992d79da..0b90cda19 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -25,6 +25,7 @@ extern "C" {
 #include        /* malloc */
 #include        /* size_t, ptrdiff_t */
 #include         /* fprintf */
+#include        /* strncmp */
 #include     /* stat, utime */
 #include      /* stat */
 #if defined(_MSC_VER)
@@ -488,6 +489,177 @@ UTIL_STATIC void UTIL_freeFileList(const char** filenameTable, char* allocatedBu
     if (filenameTable) free((void*)filenameTable);
 }
 
+/* count the number of physical cores */
+#if defined(_WIN32) || defined(WIN32)
+
+#include 
+
+typedef BOOL(WINAPI* LPFN_GLPI)(PSYSTEM_LOGICAL_PROCESSOR_INFORMATION, PDWORD);
+
+UTIL_STATIC int UTIL_countPhysicalCores(void)
+{
+    static int numPhysicalCores;
+    if (numPhysicalCores != 0) return numPhysicalCores;
+
+    {   LPFN_GLPI glpi;
+        BOOL done = FALSE;
+        PSYSTEM_LOGICAL_PROCESSOR_INFORMATION buffer = NULL;
+        PSYSTEM_LOGICAL_PROCESSOR_INFORMATION ptr = NULL;
+        DWORD returnLength = 0;
+        size_t byteOffset = 0;
+
+        glpi = (LPFN_GLPI)GetProcAddress(GetModuleHandle(TEXT("kernel32")),
+                                         "GetLogicalProcessorInformation");
+
+        if (glpi == NULL) {
+            goto failed;
+        }
+
+        while(!done) {
+            DWORD rc = glpi(buffer, &returnLength);
+            if (FALSE == rc) {
+                if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) {
+                    if (buffer)
+                        free(buffer);
+                    buffer = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION)malloc(returnLength);
+
+                    if (buffer == NULL) {
+                        perror("zstd");
+                        exit(1);
+                    }
+                } else {
+                    /* some other error */
+                    goto failed;
+                }
+            } else {
+                done = TRUE;
+            }
+        }
+
+        while (byteOffset + sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION) <= returnLength) {
+            ptr = buffer;
+
+            if (ptr->RelationShip == RelationProcessorCore) {
+                numPhysicalCores++;
+            }
+        }
+
+        free(buffer);
+
+        return numPhysicalCores;
+    }
+
+failed:
+    /* try to fall back on GetSystemInfo */
+    SYSTEM_INFO sysinfo;
+    GetSystemInfo(&sysinfo);
+    numPhysicalCores = sysinfo.dwNumberOfProcessors;
+    if (numPhysicalCores == 0) numPhysicalCores = 1; /* just in case */
+    return numPhysicalCores;
+}
+
+#elif defined(__APPLE__)
+
+#include 
+
+/* Use apple-provided syscall
+ * see: man 3 sysctl */
+UTIL_STATIC int UTIL_countPhysicalCores(void)
+{
+    static S32 numPhysicalCores; /* apple specifies int32_t */
+    if (numPhysicalCores != 0) return numPhysicalCores;
+
+    {   int const ret = sysctlbyname("hw.physicalcpu", &numPhysicalCores, sizeof(int), NULL, 0);
+        if (ret != 0) {
+            if (errno == ENOENT) {
+                /* entry not present, fall back on 1 */
+                numPhysicalCores = 1;
+            } else {
+                perror("zstd: can't get number of physical cpus");
+                exit(1);
+            }
+        }
+
+        return numPhysicalCores;
+    }
+}
+
+#elif defined(__linux__)
+
+/* parse /proc/cpuinfo
+ * siblings / cpu cores should give hyperthreading ratio
+ * otherwise fall back on sysconf */
+UTIL_STATIC int UTIL_countPhysicalCores(void)
+{
+    static int numPhysicalCores;
+
+    if (numPhysicalCores != 0) return numPhysicalCores;
+
+    numPhysicalCores = sysconf(_SC_NPROCESSORS_ONLN);
+    if (numPhysicalCores == -1) {
+        /* value not queryable, fall back on 1 */
+        return numPhysicalCores = 1;
+    }
+
+    /* try to determine if there's hyperthreading */
+    {   FILE* const cpuinfo = fopen("/proc/cpuinfo", "r");
+        size_t const BUF_SIZE = 80;
+        char buff[BUF_SIZE];
+
+        int siblings = 0;
+        int cpu_cores = 0;
+        int ratio = 1;
+
+        if (cpuinfo == NULL) {
+            /* fall back on the sysconf value */
+            return numPhysicalCores;
+        }
+
+        /* assume the cpu cores/siblings values will be constant across all
+         * present processors */
+        while (!feof(cpuinfo)) {
+            if (fgets(buff, BUF_SIZE, cpuinfo) != NULL) {
+                if (strncmp(buff, "siblings", 8) == 0) {
+                    const char* const sep = strchr(buff, ':');
+                    if (*sep == '\0') {
+                        /* formatting was broken? */
+                        goto failed;
+                    }
+
+                    siblings = atoi(sep + 1);
+                }
+                if (strncmp(buff, "cpu cores", 9) == 0) {
+                    const char* const sep = strchr(buff, ':');
+                    if (*sep == '\0') {
+                        /* formatting was broken? */
+                        goto failed;
+                    }
+
+                    cpu_cores = atoi(sep + 1);
+                }
+            } else if (ferror(cpuinfo)) {
+                /* fall back on the sysconf value */
+                goto failed;
+            }
+        }
+        if (siblings && cpu_cores) {
+            ratio = siblings / cpu_cores;
+        }
+failed:
+        fclose(cpuinfo);
+        return numPhysicalCores = numPhysicalCores / ratio;
+    }
+}
+
+#else
+
+UTIL_STATIC int UTIL_countPhysicalCores(void)
+{
+    /* assume 1 */
+    return 1;
+}
+
+#endif
 
 #if defined (__cplusplus)
 }
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 18e259c74..73b4eab9d 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -602,6 +602,11 @@ int main(int argCount, const char* argv[])
     DISPLAYLEVEL(4, "PLATFORM_POSIX_VERSION defined: %ldL\n", (long) PLATFORM_POSIX_VERSION);
 #endif
 
+    if (nbThreads == 0) {
+        /* try to guess */
+        nbThreads = UTIL_countPhysicalCores();
+        DISPLAYLEVEL(3, "Note: %d physical core(s) detected\n", nbThreads);
+    }
 
     g_utilDisplayLevel = g_displayLevel;
     if (!followLinks) {

From f876f1200cab5d616bdecfb30144a23ea76bf9fd Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Thu, 13 Apr 2017 12:33:45 -0700
Subject: [PATCH 12/53] Fix compilation on macOS

---
 programs/bench.c  | 9 +++++++--
 programs/dibio.c  | 4 +++-
 programs/fileio.c | 4 +++-
 programs/util.h   | 3 ++-
 4 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/programs/bench.c b/programs/bench.c
index d25ff8f04..8d8ed4ffa 100644
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -146,8 +146,13 @@ typedef struct {
 } blockParam_t;
 
 
-#define MIN(a,b) ((a)<(b) ? (a) : (b))
-#define MAX(a,b) ((a)>(b) ? (a) : (b))
+
+#ifndef MIN
+#   define MIN(a,b)    ((a) < (b) ? (a) : (b))
+#endif
+#ifndef MAX
+#   define MAX(a,b)    ((a) > (b) ? (a) : (b))
+#endif
 
 static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                         const char* displayName, int cLevel,
diff --git a/programs/dibio.c b/programs/dibio.c
index b7ed280ea..5e685a325 100644
--- a/programs/dibio.c
+++ b/programs/dibio.c
@@ -89,7 +89,9 @@ unsigned DiB_isError(size_t errorCode) { return ERR_isError(errorCode); }
 
 const char* DiB_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
 
-#define MIN(a,b)   ( (a) < (b) ? (a) : (b) )
+#ifndef MIN
+#   define MIN(a,b)    ((a) < (b) ? (a) : (b))
+#endif
 
 
 /* ********************************************************
diff --git a/programs/fileio.c b/programs/fileio.c
index f1f5a5577..1dc222b65 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -96,7 +96,9 @@ void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; }
 static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
 static clock_t g_time = 0;
 
-#define MIN(a,b)    ((a) < (b) ? (a) : (b))
+#ifndef MIN
+#   define MIN(a,b)    ((a) < (b) ? (a) : (b))
+#endif
 
 /* ************************************************************
 * Avoid fseek()'s 2GiB barrier with MSVC, MacOS, *BSD, MinGW
diff --git a/programs/util.h b/programs/util.h
index 0b90cda19..104f8dad2 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -569,7 +569,8 @@ UTIL_STATIC int UTIL_countPhysicalCores(void)
     static S32 numPhysicalCores; /* apple specifies int32_t */
     if (numPhysicalCores != 0) return numPhysicalCores;
 
-    {   int const ret = sysctlbyname("hw.physicalcpu", &numPhysicalCores, sizeof(int), NULL, 0);
+    {   size_t size = sizeof(S32);
+        int const ret = sysctlbyname("hw.physicalcpu", &numPhysicalCores, &size, NULL, 0);
         if (ret != 0) {
             if (errno == ENOENT) {
                 /* entry not present, fall back on 1 */

From 3b6207d4bd9143204745f90a38278891dbaa4474 Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Thu, 13 Apr 2017 14:03:56 -0700
Subject: [PATCH 13/53] Fix compilation on windows

---
 programs/util.h | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/programs/util.h b/programs/util.h
index 104f8dad2..63cfa9647 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -536,12 +536,16 @@ UTIL_STATIC int UTIL_countPhysicalCores(void)
             }
         }
 
-        while (byteOffset + sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION) <= returnLength) {
-            ptr = buffer;
+        ptr = buffer;
 
-            if (ptr->RelationShip == RelationProcessorCore) {
+        while (byteOffset + sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION) <= returnLength) {
+
+            if (ptr->Relationship == RelationProcessorCore) {
                 numPhysicalCores++;
             }
+
+            ptr++;
+            byteOffset += sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION);
         }
 
         free(buffer);
@@ -551,10 +555,11 @@ UTIL_STATIC int UTIL_countPhysicalCores(void)
 
 failed:
     /* try to fall back on GetSystemInfo */
-    SYSTEM_INFO sysinfo;
-    GetSystemInfo(&sysinfo);
-    numPhysicalCores = sysinfo.dwNumberOfProcessors;
-    if (numPhysicalCores == 0) numPhysicalCores = 1; /* just in case */
+    {   SYSTEM_INFO sysinfo;
+        GetSystemInfo(&sysinfo);
+        numPhysicalCores = sysinfo.dwNumberOfProcessors;
+        if (numPhysicalCores == 0) numPhysicalCores = 1; /* just in case */
+    }
     return numPhysicalCores;
 }
 

From 9227aae0013b44fc7628911d735731e01d1f9480 Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Thu, 13 Apr 2017 14:06:40 -0700
Subject: [PATCH 14/53] Fix clang linux compilation

---
 programs/util.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/programs/util.h b/programs/util.h
index 63cfa9647..a50b5e55b 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -601,7 +601,7 @@ UTIL_STATIC int UTIL_countPhysicalCores(void)
 
     if (numPhysicalCores != 0) return numPhysicalCores;
 
-    numPhysicalCores = sysconf(_SC_NPROCESSORS_ONLN);
+    numPhysicalCores = (int)sysconf(_SC_NPROCESSORS_ONLN);
     if (numPhysicalCores == -1) {
         /* value not queryable, fall back on 1 */
         return numPhysicalCores = 1;

From ad8da8855bd3a14c84789eda116d7f33920f454b Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Thu, 13 Apr 2017 14:40:06 -0700
Subject: [PATCH 15/53] Make appveyor small tests use new mingw as well

---
 appveyor.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/appveyor.yml b/appveyor.yml
index 27994853a..5887d52e6 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -218,10 +218,10 @@
   - ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION%
   - SET PATH_ORIGINAL=%PATH%
   - if [%HOST%]==[mingw] (
-      SET "PATH_MINGW32=C:\MinGW\bin;C:\MinGW\usr\bin" &&
-      SET "PATH_MINGW64=C:\msys64\mingw64\bin;C:\msys64\usr\bin" &&
-      COPY C:\msys64\usr\bin\make.exe C:\MinGW\bin\make.exe &&
-      COPY C:\MinGW\bin\gcc.exe C:\MinGW\bin\cc.exe
+      SET "PATH_MINGW32=C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin" &&
+      SET "PATH_MINGW64=C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin" &&
+      COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin\make.exe &&
+      COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin\make.exe
     )
   - IF [%HOST%]==[visual] IF [%PLATFORM%]==[x64] (
       SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;"

From 42bac7fa84e5ce26c582ea517e4c51befd19cd88 Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Thu, 13 Apr 2017 15:35:05 -0700
Subject: [PATCH 16/53] Change ifndef's to undef's

---
 programs/bench.c  | 10 ++++------
 programs/dibio.c  |  5 ++---
 programs/fileio.c |  5 ++---
 3 files changed, 8 insertions(+), 12 deletions(-)

diff --git a/programs/bench.c b/programs/bench.c
index 8d8ed4ffa..c3681eb05 100644
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -147,12 +147,10 @@ typedef struct {
 
 
 
-#ifndef MIN
-#   define MIN(a,b)    ((a) < (b) ? (a) : (b))
-#endif
-#ifndef MAX
-#   define MAX(a,b)    ((a) > (b) ? (a) : (b))
-#endif
+#undef MIN
+#undef MAX
+#define MIN(a,b)    ((a) < (b) ? (a) : (b))
+#define MAX(a,b)    ((a) > (b) ? (a) : (b))
 
 static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                         const char* displayName, int cLevel,
diff --git a/programs/dibio.c b/programs/dibio.c
index 5e685a325..aac36425c 100644
--- a/programs/dibio.c
+++ b/programs/dibio.c
@@ -89,9 +89,8 @@ unsigned DiB_isError(size_t errorCode) { return ERR_isError(errorCode); }
 
 const char* DiB_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
 
-#ifndef MIN
-#   define MIN(a,b)    ((a) < (b) ? (a) : (b))
-#endif
+#undef MIN
+#define MIN(a,b)    ((a) < (b) ? (a) : (b))
 
 
 /* ********************************************************
diff --git a/programs/fileio.c b/programs/fileio.c
index 1dc222b65..9bca20664 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -96,9 +96,8 @@ void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; }
 static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
 static clock_t g_time = 0;
 
-#ifndef MIN
-#   define MIN(a,b)    ((a) < (b) ? (a) : (b))
-#endif
+#undef MIN
+#define MIN(a,b)    ((a) < (b) ? (a) : (b))
 
 /* ************************************************************
 * Avoid fseek()'s 2GiB barrier with MSVC, MacOS, *BSD, MinGW

From e4f3235c85303cee20a2f93732b93a346f273c48 Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Thu, 13 Apr 2017 16:34:28 -0700
Subject: [PATCH 17/53] Add 0 initializers to static variables

---
 programs/util.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/programs/util.h b/programs/util.h
index a50b5e55b..b989e8232 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -498,7 +498,7 @@ typedef BOOL(WINAPI* LPFN_GLPI)(PSYSTEM_LOGICAL_PROCESSOR_INFORMATION, PDWORD);
 
 UTIL_STATIC int UTIL_countPhysicalCores(void)
 {
-    static int numPhysicalCores;
+    static int numPhysicalCores = 0;
     if (numPhysicalCores != 0) return numPhysicalCores;
 
     {   LPFN_GLPI glpi;
@@ -571,7 +571,7 @@ failed:
  * see: man 3 sysctl */
 UTIL_STATIC int UTIL_countPhysicalCores(void)
 {
-    static S32 numPhysicalCores; /* apple specifies int32_t */
+    static S32 numPhysicalCores = 0; /* apple specifies int32_t */
     if (numPhysicalCores != 0) return numPhysicalCores;
 
     {   size_t size = sizeof(S32);
@@ -597,7 +597,7 @@ UTIL_STATIC int UTIL_countPhysicalCores(void)
  * otherwise fall back on sysconf */
 UTIL_STATIC int UTIL_countPhysicalCores(void)
 {
-    static int numPhysicalCores;
+    static int numPhysicalCores = 0;
 
     if (numPhysicalCores != 0) return numPhysicalCores;
 

From f913cbed33e4707fab0fdf61bacf1b5076454a14 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 13 Apr 2017 22:46:41 -0700
Subject: [PATCH 18/53] fixed : memory leak in fuzzer test

---
 tests/fuzzer.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index cdbbe8a22..ee051be3e 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -549,7 +549,9 @@ static int basicUnitTests(U32 seed, double compressibility)
             if (!ZSTD_isError(result)) goto _output_error;
             if (ZSTD_getErrorCode(result) != ZSTD_error_srcSize_wrong) goto _output_error;
             DISPLAYLEVEL(4, "OK : %s \n", ZSTD_getErrorName(result));
-    }   }
+        }
+        ZSTD_freeCCtx(cctx);
+    }
 
     /* block API tests */
     {   ZSTD_CCtx* const cctx = ZSTD_createCCtx();

From 7dd14d03b069965c9577869841ce5646da7b7081 Mon Sep 17 00:00:00 2001
From: Baptiste Daroussin 
Date: Sat, 15 Apr 2017 16:25:08 +0200
Subject: [PATCH 19/53] Enable multithreading on BSD

---
 programs/util.h | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/programs/util.h b/programs/util.h
index b989e8232..5f437b2b2 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -657,6 +657,24 @@ failed:
     }
 }
 
+#elif defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__DragonFly__)
+
+/* Use apple-provided syscall
+ * see: man 3 sysctl */
+UTIL_STATIC int UTIL_countPhysicalCores(void)
+{
+    static int numPhysicalCores = 0;
+
+    if (numPhysicalCores != 0) return numPhysicalCores;
+
+    numPhysicalCores = (int)sysconf(_SC_NPROCESSORS_ONLN);
+    if (numPhysicalCores == -1) {
+        /* value not queryable, fall back on 1 */
+        return numPhysicalCores = 1;
+    }
+    return numPhysicalCores;
+}
+
 #else
 
 UTIL_STATIC int UTIL_countPhysicalCores(void)

From c424ec2eae3cbb5451ab82f7390ceb3a28b98878 Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Mon, 17 Apr 2017 11:38:53 -0700
Subject: [PATCH 20/53] Add multithreading tests to playTests.sh

---
 tests/playTests.sh | 33 +++++++++++++++++++++++++++++----
 1 file changed, 29 insertions(+), 4 deletions(-)

diff --git a/tests/playTests.sh b/tests/playTests.sh
index 3675bb168..ce1e2582e 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -31,7 +31,7 @@ fileRoundTripTest() {
     fi
 
     rm -f tmp.zstd tmp.md5.1 tmp.md5.2
-    $ECHO "fileRoundTripTest: ./datagen $1 $local_p > tmp && $ZSTD -v$local_c -c | $ZSTD -d"
+    $ECHO "fileRoundTripTest: ./datagen $1 $local_p > tmp && $ZSTD -v$local_c -c tmp | $ZSTD -d"
     ./datagen $1 $local_p > tmp
     cat tmp | $MD5SUM > tmp.md5.1
     $ZSTD --ultra -v$local_c -c tmp | $ZSTD -d | $MD5SUM > tmp.md5.2
@@ -45,12 +45,11 @@ then
 fi
 
 isWindows=false
-ECHO="echo"
+ECHO="echo -e"
 INTOVOID="/dev/null"
 case "$OS" in
   Windows*)
     isWindows=true
-    ECHO="echo -e"
     INTOVOID="NUL"
     ;;
 esac
@@ -67,11 +66,17 @@ case "$UNAME" in
   SunOS) DIFF="gdiff" ;;
 esac
 
-
 $ECHO "\nStarting playTests.sh isWindows=$isWindows ZSTD='$ZSTD'"
 
 [ -n "$ZSTD" ] || die "ZSTD variable must be defined!"
 
+if [ -n "$(echo hello | $ZSTD -v -T2 2>&1 > $INTOVOID | grep 'multi-threading is disabled')" ]
+then
+    hasMT=""
+else
+    hasMT="true"
+fi
+
 $ECHO "\n**** simple tests **** "
 
 ./datagen > tmp
@@ -461,6 +466,16 @@ roundTripTest -g516K 19   # btopt
 
 fileRoundTripTest -g500K
 
+if [ -n "$hasMT" ]
+then
+    $ECHO "\n**** zstdmt round-trip tests **** "
+    roundTripTest -g516K "16 -T0"
+    roundTripTest -g516K "19 -T2"
+    fileRoundTripTest -g500K " -T2"
+else
+    $ECHO "\n**** no multithreading, skipping zstdmt tests **** "
+fi
+
 rm tmp*
 
 if [ "$1" != "--test-large-data" ]; then
@@ -498,4 +513,14 @@ roundTripTest -g6000000000 -P99 1
 
 fileRoundTripTest -g4193M -P99 1
 
+if [ -n "$hasMT" ]
+then
+    $ECHO "\n**** zstdmt long round-trip tests **** "
+    roundTripTest -g99000000 -P99 "20 -T2"
+    roundTripTest -g6000000000 -P99 "1 -T2"
+    fileRoundTripTest -g4193M -P98 " -T0"
+else
+    $ECHO "\n**** no multithreading, skipping zstdmt tests **** "
+fi
+
 rm tmp*

From 5a61f36474b7384dc16bdc77f884ef9b0ce41c9f Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Fri, 14 Apr 2017 11:33:04 -0700
Subject: [PATCH 21/53] Make zstd compile with mt by default

---
 appveyor.yml                                 |  7 +++--
 build/VS2005/zstd/zstd.vcproj                |  8 ++---
 build/VS2005/zstdlib/zstdlib.vcproj          |  8 ++---
 build/VS2008/zstd/zstd.vcproj                |  8 ++---
 build/VS2008/zstdlib/zstdlib.vcproj          |  8 ++---
 build/VS2010/libzstd-dll/libzstd-dll.vcxproj |  8 ++---
 build/VS2010/libzstd/libzstd.vcxproj         |  8 ++---
 build/VS2010/zstd/zstd.vcxproj               |  8 ++---
 build/cmake/programs/CMakeLists.txt          |  9 +++---
 programs/Makefile                            | 32 +++++++++++++++-----
 programs/zstdcli.c                           |  2 ++
 11 files changed, 63 insertions(+), 43 deletions(-)

diff --git a/appveyor.yml b/appveyor.yml
index 5887d52e6..4aad71469 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -15,7 +15,7 @@
     - COMPILER: "gcc"
       HOST:     "mingw"
       PLATFORM: "x86"
-      SCRIPT:   "make lib && make zstd && make -C tests allnothread"
+      SCRIPT:   "make allarch"
       ARTIFACT: "true"
       BUILD:    "true"
     - COMPILER: "clang"
@@ -191,7 +191,7 @@
     - COMPILER: "gcc"
       HOST:     "mingw"
       PLATFORM: "x86"
-      SCRIPT:   "make lib && make zstd && make -C tests allnothread"
+      SCRIPT:   "make allarch"
     - COMPILER: "clang"
       HOST:     "mingw"
       PLATFORM: "x64"
@@ -237,7 +237,8 @@
       ) ) &&
       make -v &&
       sh -c "%COMPILER% -v" &&
-      sh -c "CC=%COMPILER% %SCRIPT%"
+      set "CC=%COMPILER%" &&
+      sh -c "%SCRIPT%"
     )
   - if [%HOST%]==[visual] (
       ECHO *** &&
diff --git a/build/VS2005/zstd/zstd.vcproj b/build/VS2005/zstd/zstd.vcproj
index 1f4febead..46cabbf6e 100644
--- a/build/VS2005/zstd/zstd.vcproj
+++ b/build/VS2005/zstd/zstd.vcproj
@@ -44,7 +44,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
-				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@@ -121,7 +121,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
-				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
@@ -196,7 +196,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
-				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@@ -274,7 +274,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
-				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
diff --git a/build/VS2005/zstdlib/zstdlib.vcproj b/build/VS2005/zstdlib/zstdlib.vcproj
index 8da313673..f77df786f 100644
--- a/build/VS2005/zstdlib/zstdlib.vcproj
+++ b/build/VS2005/zstdlib/zstdlib.vcproj
@@ -44,7 +44,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@@ -120,7 +120,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
@@ -194,7 +194,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@@ -271,7 +271,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
diff --git a/build/VS2008/zstd/zstd.vcproj b/build/VS2008/zstd/zstd.vcproj
index 468d25672..2e2923d55 100644
--- a/build/VS2008/zstd/zstd.vcproj
+++ b/build/VS2008/zstd/zstd.vcproj
@@ -45,7 +45,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
-				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@@ -122,7 +122,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
-				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
@@ -197,7 +197,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
-				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@@ -275,7 +275,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
-				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
diff --git a/build/VS2008/zstdlib/zstdlib.vcproj b/build/VS2008/zstdlib/zstdlib.vcproj
index 857e1463e..ac85f7aeb 100644
--- a/build/VS2008/zstdlib/zstdlib.vcproj
+++ b/build/VS2008/zstdlib/zstdlib.vcproj
@@ -45,7 +45,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@@ -121,7 +121,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
@@ -195,7 +195,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@@ -272,7 +272,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
diff --git a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj
index 866d04a04..364b3bea5 100644
--- a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj
+++ b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj
@@ -149,7 +149,7 @@
       
       Level4
       Disabled
-      ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
+      ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
       true
       EnableFastChecks
       MultiThreadedDebugDLL
@@ -169,7 +169,7 @@
       
       Level4
       Disabled
-      ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
+      ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
       true
       EnableFastChecks
       MultiThreadedDebugDLL
@@ -189,7 +189,7 @@
       MaxSpeed
       true
       true
-      ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
+      ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
       false
       MultiThreaded
       ProgramDatabase
@@ -211,7 +211,7 @@
       MaxSpeed
       true
       true
-      ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
+      ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
       false
       false
       MultiThreaded
diff --git a/build/VS2010/libzstd/libzstd.vcxproj b/build/VS2010/libzstd/libzstd.vcxproj
index 186b4c4da..6087d737c 100644
--- a/build/VS2010/libzstd/libzstd.vcxproj
+++ b/build/VS2010/libzstd/libzstd.vcxproj
@@ -146,7 +146,7 @@
       
       Level4
       Disabled
-      ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
+      ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
       true
       EnableFastChecks
       MultiThreadedDebugDLL
@@ -166,7 +166,7 @@
       
       Level4
       Disabled
-      ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
+      ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
       true
       EnableFastChecks
       MultiThreadedDebugDLL
@@ -186,7 +186,7 @@
       MaxSpeed
       true
       true
-      ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
+      ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
       false
       MultiThreaded
       ProgramDatabase
@@ -208,7 +208,7 @@
       MaxSpeed
       true
       true
-      ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
+      ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
       false
       false
       MultiThreaded
diff --git a/build/VS2010/zstd/zstd.vcxproj b/build/VS2010/zstd/zstd.vcxproj
index 7568e4902..438dc6173 100644
--- a/build/VS2010/zstd/zstd.vcxproj
+++ b/build/VS2010/zstd/zstd.vcxproj
@@ -155,7 +155,7 @@
       
       Level4
       Disabled
-      ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)
+      ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)
       true
       false
     
@@ -171,7 +171,7 @@
       
       Level4
       Disabled
-      ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)
+      ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)
       true
       false
     
@@ -189,7 +189,7 @@
       MaxSpeed
       true
       true
-      ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)
+      ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)
       false
       false
       MultiThreaded
@@ -210,7 +210,7 @@
       MaxSpeed
       true
       true
-      ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)
+      ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)
       false
       false
       MultiThreaded
diff --git a/build/cmake/programs/CMakeLists.txt b/build/cmake/programs/CMakeLists.txt
index a481a8e1d..588fea41e 100644
--- a/build/cmake/programs/CMakeLists.txt
+++ b/build/cmake/programs/CMakeLists.txt
@@ -51,17 +51,16 @@ IF (UNIX)
 ENDIF (UNIX)
 
 IF (ZSTD_MULTITHREAD_SUPPORT)
-    ADD_EXECUTABLE(zstdmt ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PlatformDependResources})
-    SET_TARGET_PROPERTIES(zstdmt PROPERTIES COMPILE_DEFINITIONS "ZSTD_MULTITHREAD")
-    TARGET_LINK_LIBRARIES(zstdmt libzstd_shared)
+    SET_TARGET_PROPERTIES(zstd PROPERTIES COMPILE_DEFINITIONS "ZSTD_MULTITHREAD")
 
     SET(THREADS_PREFER_PTHREAD_FLAG ON)
     FIND_PACKAGE(Threads REQUIRED)
     IF (CMAKE_USE_PTHREADS_INIT)
-        TARGET_LINK_LIBRARIES(zstdmt ${CMAKE_THREAD_LIBS_INIT})
+        TARGET_LINK_LIBRARIES(zstd ${CMAKE_THREAD_LIBS_INIT})
     ELSE()
         MESSAGE(SEND_ERROR "ZSTD currently does not support thread libraries other than pthreads")
     ENDIF()
 
-    INSTALL(TARGETS zstdmt RUNTIME DESTINATION "bin")
+    ADD_CUSTOM_COMMAND(TARGET zstd POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink zstd zstdmt COMMENT "Creating zstdmt symlink")
+    INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdmt DESTINATION "bin")
 ENDIF (ZSTD_MULTITHREAD_SUPPORT)
diff --git a/programs/Makefile b/programs/Makefile
index b8e976b21..3f71a5334 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -82,9 +82,22 @@ else
 EXT =
 endif
 
+VOID = /dev/null
+
+# thread detection
+NO_THREAD_MSG := ==> no threads, building without multithreading support
+HAVE_PTHREAD := $(shell printf '\#include \nint main(void) { return 0; }' | $(CC) $(FLAGS) -o have_pthread$(EXT) -x c - -pthread 2> $(VOID) && rm have_pthread$(EXT) && echo 1 || echo 0)
+HAVE_THREAD := $(shell [ "$(HAVE_PTHREAD)" -eq "1" -o -n "$(filter Windows%,$(OS))" ] && echo 1 || echo 0)
+ifeq ($(HAVE_THREAD), 1)
+THREAD_MSG := ==> building with threading support
+THREAD_CPP := -DZSTD_MULTITHREAD
+THREAD_LD := -pthread
+else
+THREAD_MSG := NO_THREAD_MSG
+endif
+
 # zlib detection
 NO_ZLIB_MSG := ==> no zlib, building zstd without .gz support
-VOID = /dev/null
 HAVE_ZLIB := $(shell printf '\#include \nint main(void) { return 0; }' | $(CC) $(FLAGS) -o have_zlib$(EXT) -x c - -lz 2> $(VOID) && rm have_zlib$(EXT) && echo 1 || echo 0)
 ifeq ($(HAVE_ZLIB), 1)
 ZLIB_MSG := ==> building zstd with .gz compression support
@@ -115,8 +128,8 @@ all: zstd
 
 $(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP)
 
-zstd : CPPFLAGS += $(ZLIBCPP)
-zstd : LDFLAGS += $(ZLIBLD)
+zstd : CPPFLAGS += $(ZLIBCPP) $(THREAD_CPP)
+zstd : LDFLAGS += $(ZLIBLD) $(THREAD_LD)
 zstd : LZMA_MSG := $(NO_LZMA_MSG)
 zstd-nogz : ZLIB_MSG := $(NO_ZLIB_MSG)
 zstd-nogz : LZMA_MSG := $(NO_LZMA_MSG)
@@ -124,6 +137,7 @@ xzstd : CPPFLAGS += $(ZLIBCPP) $(LZMACPP)
 xzstd : LDFLAGS += $(ZLIBLD) $(LZMALD)
 zstd zstd-nogz xzstd : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
 zstd zstd-nogz xzstd : $(ZSTDLIB_OBJ) zstdcli.o fileio.o bench.o datagen.o dibio.o
+	@echo "$(THREAD_MSG)"
 	@echo "$(ZLIB_MSG)"
 	@echo "$(LZMA_MSG)"
 ifneq (,$(filter Windows%,$(OS)))
@@ -145,6 +159,11 @@ endif
 zstd-nolegacy : clean_decomp_o
 	$(MAKE) zstd ZSTD_LEGACY_SUPPORT=0
 
+zstd-nomt : THREAD_CPP :=
+zstd-nomt : THREAD_LD :=
+zstd-nomt : THREAD_MSG := NO_THREAD_MSG
+zstd-nomt : zstd
+
 zstd-pgo : MOREFLAGS = -fprofile-generate
 zstd-pgo : clean zstd
 	./zstd -b19i1 $(PROFILE_WITH)
@@ -169,10 +188,6 @@ zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) zstdcli.c fileio.c
 zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) zstdcli.c fileio.c
 	$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS $^ -o $@$(EXT)
 
-zstdmt: CPPFLAGS += -DZSTD_MULTITHREAD
-ifeq (,$(filter Windows%,$(OS)))
-zstdmt: LDFLAGS += -lpthread
-endif
 zstdmt: zstd
 
 generate_res:
@@ -231,6 +246,9 @@ install: zstd
 	@$(INSTALL_PROGRAM) zstd $(DESTDIR)$(BINDIR)/zstd
 	@ln -sf zstd $(DESTDIR)$(BINDIR)/zstdcat
 	@ln -sf zstd $(DESTDIR)$(BINDIR)/unzstd
+ifeq ($(HAVE_THREAD),1)
+	@ln -sf zstd $(DESTDIR)$(BINDIR)/zstdmt
+endif
 	@$(INSTALL_SCRIPT) zstdless $(DESTDIR)$(BINDIR)/zstdless
 	@$(INSTALL_SCRIPT) zstdgrep $(DESTDIR)$(BINDIR)/zstdgrep
 	@echo Installing man pages
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 73b4eab9d..6cbe7383f 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -49,6 +49,7 @@
 #define AUTHOR "Yann Collet"
 #define WELCOME_MESSAGE "*** %s %i-bits %s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(size_t)*8), ZSTD_VERSION, AUTHOR
 
+#define ZSTD_ZSTDMT "zstdmt"
 #define ZSTD_UNZSTD "unzstd"
 #define ZSTD_CAT "zstdcat"
 #define ZSTD_GZ "gzip"
@@ -343,6 +344,7 @@ int main(int argCount, const char* argv[])
     programName = lastNameFromPath(programName);
 
     /* preset behaviors */
+    if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbThreads=0;
     if (exeNameMatch(programName, ZSTD_UNZSTD)) operation=zom_decompress;
     if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; }
     if (exeNameMatch(programName, ZSTD_GZ)) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); FIO_setRemoveSrcFile(1); }    /* behave like gzip */

From d845dab69c15896ea2d857266ef889ef52e9f7aa Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Mon, 17 Apr 2017 12:10:58 -0700
Subject: [PATCH 22/53] Fix input size too small to trigger zstdmt

---
 tests/playTests.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/playTests.sh b/tests/playTests.sh
index ce1e2582e..deeebee2e 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -469,9 +469,9 @@ fileRoundTripTest -g500K
 if [ -n "$hasMT" ]
 then
     $ECHO "\n**** zstdmt round-trip tests **** "
-    roundTripTest -g516K "16 -T0"
-    roundTripTest -g516K "19 -T2"
-    fileRoundTripTest -g500K " -T2"
+    roundTripTest -g4M "1 -T0"
+    roundTripTest -g8M "3 -T2"
+    fileRoundTripTest -g4M "19 -T2 -B1M"
 else
     $ECHO "\n**** no multithreading, skipping zstdmt tests **** "
 fi

From f6ef4db20e3025146a338fc399b48b4f1705fa4d Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Mon, 17 Apr 2017 12:21:11 -0700
Subject: [PATCH 23/53] Install zstdmt even without threading support

---
 programs/Makefile | 2 --
 1 file changed, 2 deletions(-)

diff --git a/programs/Makefile b/programs/Makefile
index 3f71a5334..aca442644 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -246,9 +246,7 @@ install: zstd
 	@$(INSTALL_PROGRAM) zstd $(DESTDIR)$(BINDIR)/zstd
 	@ln -sf zstd $(DESTDIR)$(BINDIR)/zstdcat
 	@ln -sf zstd $(DESTDIR)$(BINDIR)/unzstd
-ifeq ($(HAVE_THREAD),1)
 	@ln -sf zstd $(DESTDIR)$(BINDIR)/zstdmt
-endif
 	@$(INSTALL_SCRIPT) zstdless $(DESTDIR)$(BINDIR)/zstdless
 	@$(INSTALL_SCRIPT) zstdgrep $(DESTDIR)$(BINDIR)/zstdgrep
 	@echo Installing man pages

From 5935c990a0653abd71727d2cb5ebc5775a792864 Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Mon, 17 Apr 2017 16:05:20 -0700
Subject: [PATCH 24/53] Add zstdmt and -T0 to man page

---
 programs/zstd.1    | 7 +++++--
 programs/zstd.1.md | 8 ++++++--
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/programs/zstd.1 b/programs/zstd.1
index c6b2c2742..b8389042e 100644
--- a/programs/zstd.1
+++ b/programs/zstd.1
@@ -2,12 +2,15 @@
 .TH "ZSTD" "1" "April 2017" "zstd 1.1.5" "User Commands"
 .
 .SH "NAME"
-\fBzstd\fR \- zstd, unzstd, zstdcat \- Compress or decompress \.zst files
+\fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files
 .
 .SH "SYNOPSIS"
 \fBzstd\fR [\fIOPTIONS\fR] [\-|] [\-o ]
 .
 .P
+\fBzstdmt\fR is equivalent to \fBzstd \-T0\fR
+.
+.P
 \fBunzstd\fR is equivalent to \fBzstd \-d\fR
 .
 .P
@@ -98,7 +101,7 @@ unlocks high compression levels 20+ (maximum 22), using a lot more memory\. Note
 .
 .TP
 \fB\-T#\fR
-Compress using # threads (default: 1)\. This modifier is only available if \fBzstd\fR was compiled with multithreading support\.
+Compress using # threads (default: 1)\. If \fB#\fR is 0, attempt to detect the number of physical CPU cores and compress with that many threads\. This modifier is only available if \fBzstd\fR was compiled with multithreading support\.
 .
 .TP
 \fB\-D file\fR
diff --git a/programs/zstd.1.md b/programs/zstd.1.md
index 07c36f683..9044eb34e 100644
--- a/programs/zstd.1.md
+++ b/programs/zstd.1.md
@@ -1,11 +1,13 @@
-zstd(1) -- zstd, unzstd, zstdcat - Compress or decompress .zst files
-====================================================================
+zstd(1) -- zstd, zstdmt, unzstd, zstdcat - Compress or decompress .zst files
+============================================================================
 
 SYNOPSIS
 --------
 
 `zstd` [*OPTIONS*] [-|<INPUT-FILE>] [-o <OUTPUT-FILE>]
 
+`zstdmt` is equivalent to `zstd -T0`
+
 `unzstd` is equivalent to `zstd -d`
 
 `zstdcat` is equivalent to `zstd -dcf`
@@ -101,6 +103,8 @@ the last one takes effect.
     Note that decompression will also require more memory when using these levels.
 * `-T#`:
     Compress using # threads (default: 1).
+    If `#` is 0, attempt to detect the number of physical CPU cores and compress with
+    that many threads.
     This modifier is only available if `zstd` was compiled with multithreading support.
 * `-D file`:
     use `file` as Dictionary to compress or decompress FILE(s)

From c47c68f6ca6f782cd9d24c4b8a11c45553cd1700 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 17 Apr 2017 16:14:21 -0700
Subject: [PATCH 25/53] proper evaluation of Huffman CTable size

---
 lib/common/huf.h             | 4 +++-
 lib/compress/zstd_compress.c | 5 ++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/lib/common/huf.h b/lib/common/huf.h
index e5572760a..32313cf74 100644
--- a/lib/common/huf.h
+++ b/lib/common/huf.h
@@ -121,8 +121,10 @@ size_t HUF_compress4X_wksp (void* dst, size_t dstSize, const void* src, size_t s
 #define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size))   /* Macro version, useful for static allocation */
 
 /* static allocation of HUF's Compression Table */
+#define HUF_CTABLE_SIZE_U32(maxSymbolValue)   ((maxSymbolValue)+1)   /* Use tables of U32, for proper alignment */
+#define HUF_CTABLE_SIZE(maxSymbolValue)       (HUF_CTABLE_SIZE_U32(maxSymbolValue) * sizeof(U32))
 #define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \
-    U32 name##hb[maxSymbolValue+1]; \
+    U32 name##hb[HUF_CTABLE_SIZE_U32(maxSymbolValue)]; \
     void* name##hv = &(name##hb); \
     HUF_CElt* name = (HUF_CElt*)(name##hv)   /* no final ; */
 
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index d6774372b..04791ba95 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -304,7 +304,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
         zc->hufTable = (HUF_CElt*)ptr;
         zc->flagStaticTables = 0;
         zc->flagStaticHufTable = HUF_repeat_none;
-        ptr = ((U32*)ptr) + 256;  /* note : HUF_CElt* is incomplete type, size is simulated using U32 */
+        ptr = ((U32*)ptr) + HUF_CTABLE_SIZE_U32(255);  /* note : HUF_CElt* is incomplete type, size is simulated using U32 */
 
         zc->nextToUpdate = 1;
         zc->nextSrc = NULL;
@@ -362,7 +362,6 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long
 {
     if (srcCCtx->stage!=ZSTDcs_init) return ERROR(stage_wrong);
 
-
     memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem));
     {   ZSTD_parameters params = srcCCtx->params;
         params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
@@ -397,7 +396,7 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long
         memcpy(dstCCtx->offcodeCTable, srcCCtx->offcodeCTable, sizeof(dstCCtx->offcodeCTable));
     }
     if (srcCCtx->flagStaticHufTable) {
-        memcpy(dstCCtx->hufTable, srcCCtx->hufTable, 256*4);
+        memcpy(dstCCtx->hufTable, srcCCtx->hufTable, HUF_CTABLE_SIZE(255));
     }
 
     return 0;

From e6c504dbe604ffab1e6f9b68a62face1c4c23758 Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Mon, 17 Apr 2017 17:11:33 -0700
Subject: [PATCH 26/53] Update -T0 comment in man page

---
 programs/zstd.1    | 2 +-
 programs/zstd.1.md | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/programs/zstd.1 b/programs/zstd.1
index b8389042e..b84ab3324 100644
--- a/programs/zstd.1
+++ b/programs/zstd.1
@@ -101,7 +101,7 @@ unlocks high compression levels 20+ (maximum 22), using a lot more memory\. Note
 .
 .TP
 \fB\-T#\fR
-Compress using # threads (default: 1)\. If \fB#\fR is 0, attempt to detect the number of physical CPU cores and compress with that many threads\. This modifier is only available if \fBzstd\fR was compiled with multithreading support\.
+Compress using # threads (default: 1)\. If \fB#\fR is 0, attempt to detect the number of physical CPU cores and compress with that many threads\. This modified does nothing if \fBzstd\fR was compiled without multithread support\.
 .
 .TP
 \fB\-D file\fR
diff --git a/programs/zstd.1.md b/programs/zstd.1.md
index 9044eb34e..4b918d8fd 100644
--- a/programs/zstd.1.md
+++ b/programs/zstd.1.md
@@ -105,7 +105,7 @@ the last one takes effect.
     Compress using # threads (default: 1).
     If `#` is 0, attempt to detect the number of physical CPU cores and compress with
     that many threads.
-    This modifier is only available if `zstd` was compiled with multithreading support.
+    This modified does nothing if `zstd` was compiled without multithread support.
 * `-D file`:
     use `file` as Dictionary to compress or decompress FILE(s)
 * `--nodictID`:

From c8b2df7d6220f62f4f930e62dc361098b5a002c7 Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Mon, 17 Apr 2017 17:13:44 -0700
Subject: [PATCH 27/53] Compile CLI using files instead of objs

This avoids conflicts between how the library was
configured and how the CLI was configured.
---
 programs/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/programs/Makefile b/programs/Makefile
index aca442644..f3140a19c 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -136,7 +136,7 @@ zstd-nogz : LZMA_MSG := $(NO_LZMA_MSG)
 xzstd : CPPFLAGS += $(ZLIBCPP) $(LZMACPP)
 xzstd : LDFLAGS += $(ZLIBLD) $(LZMALD)
 zstd zstd-nogz xzstd : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
-zstd zstd-nogz xzstd : $(ZSTDLIB_OBJ) zstdcli.o fileio.o bench.o datagen.o dibio.o
+zstd zstd-nogz xzstd : $(ZSTDLIB_FILES) zstdcli.o fileio.o bench.o datagen.o dibio.o
 	@echo "$(THREAD_MSG)"
 	@echo "$(ZLIB_MSG)"
 	@echo "$(LZMA_MSG)"

From 4f818182b80946c21b4196755228bb7136923176 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 17 Apr 2017 17:57:35 -0700
Subject: [PATCH 28/53] clarified frame parameters for
 ZSTD_compress*_usingCDict()

created ZSTD_compressBegin_usingCDict_internal(),
which gives direct control to frame Parameters.
ZSTD_resetCStream_internal() now points into it.
---
 lib/common/zstd_internal.h        |  2 ++
 lib/compress/zstd_compress.c      | 41 +++++++++++++++++++------------
 lib/zstd.h                        |  7 +++---
 tests/fuzzer.c                    |  7 ------
 tests/paramgrill.c                |  9 ++++---
 zlibWrapper/examples/zwrapbench.c |  8 +++---
 6 files changed, 41 insertions(+), 33 deletions(-)

diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index 5c5b28732..1e2cbd4d7 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -58,6 +58,8 @@
 /*-*************************************
 *  shared macros
 ***************************************/
+#undef MIN
+#undef MAX
 #define MIN(a,b) ((a)<(b) ? (a) : (b))
 #define MAX(a,b) ((a)>(b) ? (a) : (b))
 #define CHECK_F(f) { size_t const errcod = f; if (ERR_isError(errcod)) return errcod; }  /* check and Forward error code */
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 04791ba95..851158f61 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2911,36 +2911,45 @@ static ZSTD_parameters ZSTD_getParamsFromCDict(const ZSTD_CDict* cdict) {
     return ZSTD_getParamsFromCCtx(cdict->refContext);
 }
 
-size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize)
+/* ZSTD_compressBegin_usingCDict_internal() :
+ * cdict must be != NULL */
+static size_t ZSTD_compressBegin_usingCDict_internal(
+    ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict,
+    ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize)
 {
     if (cdict==NULL) return ERROR(GENERIC);  /* does not support NULL cdict */
-    if (cdict->dictContentSize) CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext, pledgedSrcSize))
+    if (cdict->dictContentSize)
+        CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext, pledgedSrcSize))  /* to be changed, to consider fParams */
     else {
         ZSTD_parameters params = cdict->refContext->params;
-        params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
+        params.fParams = fParams;
         CHECK_F(ZSTD_compressBegin_internal(cctx, NULL, 0, params, pledgedSrcSize));
     }
     return 0;
 }
 
+/* ZSTD_compressBegin_usingCDict() :
+ * pledgedSrcSize=0 means "unknown"
+ * if pledgedSrcSize>0, it will enable contentSizeFlag */
+size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize)
+{
+    ZSTD_frameParameters fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ };
+    fParams.contentSizeFlag = (pledgedSrcSize > 0);
+    return ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, pledgedSrcSize);
+}
+
 /*! ZSTD_compress_usingCDict() :
-*   Compression using a digested Dictionary.
-*   Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times.
-*   Note that compression level is decided during dictionary creation */
+ *  Compression using a digested Dictionary.
+ *  Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times.
+ *  Note that compression parameters are decided at CDict creation time
+ *  while frame parameters are hardcoded */
 size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
                                 void* dst, size_t dstCapacity,
                                 const void* src, size_t srcSize,
                                 const ZSTD_CDict* cdict)
 {
-    CHECK_F(ZSTD_compressBegin_usingCDict(cctx, cdict, srcSize));  /* will check if cdict != NULL */
-
-    if (cdict->refContext->params.fParams.contentSizeFlag == 1) {
-        cctx->params.fParams.contentSizeFlag = 1;
-        cctx->frameContentSize = srcSize;
-    } else {
-        cctx->params.fParams.contentSizeFlag = 0;
-    }
-
+    ZSTD_frameParameters const fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ };
+    CHECK_F (ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, srcSize));   /* will check if cdict != NULL */
     return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
 }
 
@@ -3022,7 +3031,7 @@ static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs, unsigned long long p
 {
     if (zcs->inBuffSize==0) return ERROR(stage_wrong);   /* zcs has not been init at least once => can't reset */
 
-    if (zcs->cdict) CHECK_F(ZSTD_compressBegin_usingCDict(zcs->cctx, zcs->cdict, pledgedSrcSize))
+    if (zcs->cdict) CHECK_F(ZSTD_compressBegin_usingCDict_internal(zcs->cctx, zcs->cdict, zcs->params.fParams, pledgedSrcSize))
     else CHECK_F(ZSTD_compressBegin_internal(zcs->cctx, NULL, 0, zcs->params, pledgedSrcSize));
 
     zcs->inToCompress = 0;
diff --git a/lib/zstd.h b/lib/zstd.h
index 6066db45e..18ec1cd7f 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -194,9 +194,10 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize
 ZSTDLIB_API size_t      ZSTD_freeCDict(ZSTD_CDict* CDict);
 
 /*! ZSTD_compress_usingCDict() :
-*   Compression using a digested Dictionary.
-*   Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times.
-*   Note that compression level is decided during dictionary creation. */
+ *  Compression using a digested Dictionary.
+ *  Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times.
+ *  Note that compression level is decided during dictionary creation.
+ *  Frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) */
 ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
                                             void* dst, size_t dstCapacity,
                                       const void* src, size_t srcSize,
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index ee051be3e..3123d1825 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -406,7 +406,6 @@ static int basicUnitTests(U32 seed, double compressibility)
 
         DISPLAYLEVEL(4, "test%3i : compress with preprocessed dictionary : ", testNb++);
         {   ZSTD_parameters params = ZSTD_getParams(1, CNBuffSize, dictSize);
-            params.fParams.contentSizeFlag = 0;
             {   ZSTD_customMem customMem = { NULL, NULL, NULL };
                 ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, 1, params, customMem);
                 cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize),
@@ -423,12 +422,6 @@ static int basicUnitTests(U32 seed, double compressibility)
         }
         DISPLAYLEVEL(4, "OK \n");
 
-        DISPLAYLEVEL(4, "test%3i : frame should not have content size : ", testNb++);
-        {   unsigned long long const contentSize = ZSTD_findDecompressedSize(compressedBuffer, cSize);
-            if (contentSize != ZSTD_CONTENTSIZE_UNKNOWN)  goto _output_error;  /* cdict contentSizeFlag not used */
-        }
-        DISPLAYLEVEL(4, "OK \n");
-
         DISPLAYLEVEL(4, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
         CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
                                        decodedBuffer, CNBuffSize,
diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index 3c1f0150b..1913b54d0 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -58,6 +58,11 @@ static const int g_maxNbVariations = 64;
 **************************************/
 #define DISPLAY(...)  fprintf(stderr, __VA_ARGS__)
 
+#undef MIN
+#undef MAX
+#define MIN(a,b)   ( (a) < (b) ? (a) : (b) )
+#define MAX(a,b)   ( (a) > (b) ? (a) : (b) )
+
 
 /*-************************************
 *  Benchmark Parameters
@@ -145,8 +150,6 @@ typedef struct
 } blockParam_t;
 
 
-#define MIN(a,b)  ( (a) < (b) ? (a) : (b) )
-
 static size_t BMK_benchParam(BMK_result_t* resultPtr,
                              const void* srcBuffer, size_t srcSize,
                              ZSTD_CCtx* ctx,
@@ -513,8 +516,6 @@ static BYTE g_alreadyTested[PARAMTABLESIZE] = {0};   /* init to zero */
     g_alreadyTested[(XXH64(sanitizeParams(p), sizeof(p), 0) >> 3) & PARAMTABLEMASK]
 
 
-#define MAX(a,b)   ( (a) > (b) ? (a) : (b) )
-
 static void playAround(FILE* f, winnerInfo_t* winners,
                        ZSTD_compressionParameters params,
                        const void* srcBuffer, size_t srcSize,
diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c
index 1c3391e90..401daa1b5 100644
--- a/zlibWrapper/examples/zwrapbench.c
+++ b/zlibWrapper/examples/zwrapbench.c
@@ -128,6 +128,11 @@ void BMK_SetBlockSize(size_t blockSize)
 /* ********************************************************
 *  Bench functions
 **********************************************************/
+#undef MIN
+#undef MAX
+#define MIN(a,b) ((a)<(b) ? (a) : (b))
+#define MAX(a,b) ((a)>(b) ? (a) : (b))
+
 typedef struct
 {
     z_const char* srcPtr;
@@ -142,9 +147,6 @@ typedef struct
 typedef enum { BMK_ZSTD, BMK_ZSTD_STREAM, BMK_ZLIB, BMK_ZWRAP_ZLIB, BMK_ZWRAP_ZSTD, BMK_ZLIB_REUSE, BMK_ZWRAP_ZLIB_REUSE, BMK_ZWRAP_ZSTD_REUSE } BMK_compressor;
 
 
-#define MIN(a,b) ((a)<(b) ? (a) : (b))
-#define MAX(a,b) ((a)>(b) ? (a) : (b))
-
 static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
                         const char* displayName, int cLevel,
                         const size_t* fileSizes, U32 nbFiles,

From af4f45b68243748b7c16c09a7a1cb4529363b054 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 18 Apr 2017 03:17:44 -0700
Subject: [PATCH 29/53] Improved code comments for block functions

---
 lib/zstd.h | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/lib/zstd.h b/lib/zstd.h
index 18ec1cd7f..2f195636a 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -754,19 +754,20 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
     - Compressing and decompressing require a context structure
       + Use ZSTD_createCCtx() and ZSTD_createDCtx()
     - It is necessary to init context before starting
-      + compression : ZSTD_compressBegin()
-      + decompression : ZSTD_decompressBegin()
-      + variants _usingDict() are also allowed
-      + copyCCtx() and copyDCtx() work too
-    - Block size is limited, it must be <= ZSTD_getBlockSizeMax()
-      + If you need to compress more, cut data into multiple blocks
-      + Consider using the regular ZSTD_compress() instead, as frame metadata costs become negligible when source size is large.
+      + compression : any ZSTD_compressBegin*() variant, including with dictionary
+      + decompression : any ZSTD_decompressBegin*() variant, including with dictionary
+      + copyCCtx() and copyDCtx() can be used too
+    - Block size is limited, it must be <= ZSTD_getBlockSizeMax() <= ZSTD_BLOCKSIZE_ABSOLUTEMAX
+      + If input is larger than a block size, it's necessary to split input data into multiple blocks
+      + For inputs larger than a single block size, consider using the regular ZSTD_compress() instead.
+        Frame metadata is not that costly, and quickly becomes negligible as source size grows larger.
     - When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero.
       In which case, nothing is produced into `dst`.
       + User must test for such outcome and deal directly with uncompressed data
       + ZSTD_decompressBlock() doesn't accept uncompressed data as input !!!
-      + In case of multiple successive blocks, decoder must be informed of uncompressed block existence to follow proper history.
-        Use ZSTD_insertBlock() in such a case.
+      + In case of multiple successive blocks, should some of them be uncompressed,
+        decoder must be informed of their existence in order to follow proper history.
+        Use ZSTD_insertBlock() for such a case.
 */
 
 #define ZSTD_BLOCKSIZE_ABSOLUTEMAX (128 * 1024)   /* define, for static allocation */

From 9606256a8d4c24ecf93740d37d55e14676122626 Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Tue, 18 Apr 2017 13:52:00 -0700
Subject: [PATCH 30/53] Fix no thread message

---
 programs/Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/programs/Makefile b/programs/Makefile
index f3140a19c..4ab4dfd62 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -93,7 +93,7 @@ THREAD_MSG := ==> building with threading support
 THREAD_CPP := -DZSTD_MULTITHREAD
 THREAD_LD := -pthread
 else
-THREAD_MSG := NO_THREAD_MSG
+THREAD_MSG := $(NO_THREAD_MSG)
 endif
 
 # zlib detection
@@ -161,7 +161,7 @@ zstd-nolegacy : clean_decomp_o
 
 zstd-nomt : THREAD_CPP :=
 zstd-nomt : THREAD_LD :=
-zstd-nomt : THREAD_MSG := NO_THREAD_MSG
+zstd-nomt : THREAD_MSG := $(NO_THREAD_MSG)
 zstd-nomt : zstd
 
 zstd-pgo : MOREFLAGS = -fprofile-generate

From 715b9aa113e4c9d7953b0cbeb1756d0ea3572f79 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 18 Apr 2017 13:55:53 -0700
Subject: [PATCH 31/53] created ZSTD_compressBegin_usingCDict_advanced()

---
 doc/zstd_manual.html         | 31 ++++++++++++++++---------------
 lib/compress/zstd_compress.c | 10 +++++-----
 lib/zstd.h                   |  4 +++-
 3 files changed, 24 insertions(+), 21 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index a5b66f7d3..d267cfc3c 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -169,9 +169,10 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
                                 void* dst, size_t dstCapacity,
                           const void* src, size_t srcSize,
                           const ZSTD_CDict* cdict);
-
   Compression using a digested Dictionary.
-   Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times.
-   Note that compression level is decided during dictionary creation. 
+
  Compression using a digested Dictionary.
+  Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times.
+  Note that compression level is decided during dictionary creation.
+  Frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) 
 


 
 
ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize);
@@ -560,10 +561,9 @@ size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
 
Buffer-less streaming compression functions
size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
 size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
 size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */
-size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**<  note: if pledgedSrcSize can be 0, indicating unknown size.  if it is non-zero, it must be accurate.  for 0 size frames, use compressBegin_advanced */
 size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize); /**< note: fail if cdict==NULL. pledgedSrcSize can be 0, indicating unknown size.  For 0 size frames, use compressBegin_advanced */
-size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
-size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize);   /* compression parameters are already set within cdict. pledgedSrcSize=0 means null-size */
+size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**<  note: if pledgedSrcSize can be 0, indicating unknown size.  if it is non-zero, it must be accurate.  for 0 size frames, use compressBegin_advanced */
 


 
Buffer-less streaming decompression (synchronous mode)
   A ZSTD_DCtx object is required to track streaming operations.
@@ -648,19 +648,20 @@ ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
     - Compressing and decompressing require a context structure
       + Use ZSTD_createCCtx() and ZSTD_createDCtx()
     - It is necessary to init context before starting
-      + compression : ZSTD_compressBegin()
-      + decompression : ZSTD_decompressBegin()
-      + variants _usingDict() are also allowed
-      + copyCCtx() and copyDCtx() work too
-    - Block size is limited, it must be <= ZSTD_getBlockSizeMax()
-      + If you need to compress more, cut data into multiple blocks
-      + Consider using the regular ZSTD_compress() instead, as frame metadata costs become negligible when source size is large.
+      + compression : any ZSTD_compressBegin*() variant, including with dictionary
+      + decompression : any ZSTD_decompressBegin*() variant, including with dictionary
+      + copyCCtx() and copyDCtx() can be used too
+    - Block size is limited, it must be <= ZSTD_getBlockSizeMax() <= ZSTD_BLOCKSIZE_ABSOLUTEMAX
+      + If input is larger than a block size, it's necessary to split input data into multiple blocks
+      + For inputs larger than a single block size, consider using the regular ZSTD_compress() instead.
+        Frame metadata is not that costly, and quickly becomes negligible as source size grows larger.
     - When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero.
       In which case, nothing is produced into `dst`.
       + User must test for such outcome and deal directly with uncompressed data
       + ZSTD_decompressBlock() doesn't accept uncompressed data as input !!!
-      + In case of multiple successive blocks, decoder must be informed of uncompressed block existence to follow proper history.
-        Use ZSTD_insertBlock() in such a case.
+      + In case of multiple successive blocks, should some of them be uncompressed,
+        decoder must be informed of their existence in order to follow proper history.
+        Use ZSTD_insertBlock() for such a case.
 

 
 
Raw zstd block functions
size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx);
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 851158f61..33d4ab005 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2911,9 +2911,9 @@ static ZSTD_parameters ZSTD_getParamsFromCDict(const ZSTD_CDict* cdict) {
     return ZSTD_getParamsFromCCtx(cdict->refContext);
 }
 
-/* ZSTD_compressBegin_usingCDict_internal() :
+/* ZSTD_compressBegin_usingCDict_advanced() :
  * cdict must be != NULL */
-static size_t ZSTD_compressBegin_usingCDict_internal(
+size_t ZSTD_compressBegin_usingCDict_advanced(
     ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict,
     ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize)
 {
@@ -2935,7 +2935,7 @@ size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, u
 {
     ZSTD_frameParameters fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ };
     fParams.contentSizeFlag = (pledgedSrcSize > 0);
-    return ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, pledgedSrcSize);
+    return ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, pledgedSrcSize);
 }
 
 /*! ZSTD_compress_usingCDict() :
@@ -2949,7 +2949,7 @@ size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
                                 const ZSTD_CDict* cdict)
 {
     ZSTD_frameParameters const fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ };
-    CHECK_F (ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, srcSize));   /* will check if cdict != NULL */
+    CHECK_F (ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, srcSize));   /* will check if cdict != NULL */
     return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
 }
 
@@ -3031,7 +3031,7 @@ static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs, unsigned long long p
 {
     if (zcs->inBuffSize==0) return ERROR(stage_wrong);   /* zcs has not been init at least once => can't reset */
 
-    if (zcs->cdict) CHECK_F(ZSTD_compressBegin_usingCDict_internal(zcs->cctx, zcs->cdict, zcs->params.fParams, pledgedSrcSize))
+    if (zcs->cdict) CHECK_F(ZSTD_compressBegin_usingCDict_advanced(zcs->cctx, zcs->cdict, zcs->params.fParams, pledgedSrcSize))
     else CHECK_F(ZSTD_compressBegin_internal(zcs->cctx, NULL, 0, zcs->params, pledgedSrcSize));
 
     zcs->inToCompress = 0;
diff --git a/lib/zstd.h b/lib/zstd.h
index 2f195636a..5101fd26c 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -659,8 +659,10 @@ ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
 ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
 ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
 ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */
-ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**<  note: if pledgedSrcSize can be 0, indicating unknown size.  if it is non-zero, it must be accurate.  for 0 size frames, use compressBegin_advanced */
 ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize); /**< note: fail if cdict==NULL. pledgedSrcSize can be 0, indicating unknown size.  For 0 size frames, use compressBegin_advanced */
+ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize);   /* compression parameters are already set within cdict. pledgedSrcSize=0 means null-size */
+ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**<  note: if pledgedSrcSize can be 0, indicating unknown size.  if it is non-zero, it must be accurate.  for 0 size frames, use compressBegin_advanced */
+
 ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 

From 30fb499208e650ba3e5f28bafde788fda474dfff Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 18 Apr 2017 14:08:50 -0700
Subject: [PATCH 32/53] Changed ZSTD_resetCCtx_advanced() into
 ZSTD_resetCCtx_internal()

for naming consistency :
_advanced() can be invoked
while _internal() are strictly static
---
 lib/compress/zstd_compress.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 33d4ab005..9328b7c40 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -257,9 +257,9 @@ static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_parameters params, U64 fra
 
 typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset, ZSTDcrp_fullReset } ZSTD_compResetPolicy_e;
 
-/*! ZSTD_resetCCtx_advanced() :
+/*! ZSTD_resetCCtx_internal() :
     note : `params` must be validated */
-static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
+static size_t ZSTD_resetCCtx_internal (ZSTD_CCtx* zc,
                                        ZSTD_parameters params, U64 frameContentSize,
                                        ZSTD_compResetPolicy_e const crp)
 {
@@ -365,7 +365,7 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long
     memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem));
     {   ZSTD_parameters params = srcCCtx->params;
         params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
-        ZSTD_resetCCtx_advanced(dstCCtx, params, pledgedSrcSize, ZSTDcrp_noMemset);
+        ZSTD_resetCCtx_internal(dstCCtx, params, pledgedSrcSize, ZSTDcrp_noMemset);
     }
 
     /* copy tables */
@@ -2694,7 +2694,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
 {
     ZSTD_compResetPolicy_e const crp = dictSize ? ZSTDcrp_fullReset : ZSTDcrp_continue;
     assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
-    CHECK_F(ZSTD_resetCCtx_advanced(cctx, params, pledgedSrcSize, crp));
+    CHECK_F(ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, crp));
     return ZSTD_compress_insertDictionary(cctx, dict, dictSize);
 }
 

From ca6fae78080d0a532ce90be830584212c7350fc2 Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Tue, 18 Apr 2017 14:13:01 -0700
Subject: [PATCH 33/53] Add MT enabled targets for libzstd

---
 lib/Makefile | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/lib/Makefile b/lib/Makefile
index 197fdeeea..d8d8e179d 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -71,6 +71,9 @@ libzstd.a: $(ZSTD_OBJ)
 	@echo compiling static library
 	@$(AR) $(ARFLAGS) $@ $^
 
+libzstd.a-mt: CPPFLAGS += -DZSTD_MULTHREAD
+libzstd.a-mt: libzstd.a
+
 $(LIBZSTD): LDFLAGS += -shared -fPIC -fvisibility=hidden
 $(LIBZSTD): $(ZSTD_FILES)
 	@echo compiling dynamic library $(LIBVER)
@@ -86,10 +89,17 @@ endif
 
 libzstd : $(LIBZSTD)
 
+libzstd-mt : CPPFLAGS += -DZSTD_MULTITHREAD
+libzstd-mt : libzstd
+
 lib: libzstd.a libzstd
 
-lib-release: DEBUGFLAGS :=
+lib-mt: CPPFLAGS += -DZSTD_MULTITHREAD
+lib-mt: lib
+
+lib-release lib-release-mt: DEBUGFLAGS :=
 lib-release: lib
+lib-release-mt: lib-mt
 
 clean:
 	@$(RM) -r *.dSYM   # Mac OS-X specific

From b402f1ef37b76a44803da8ed6104d8e01ce8f223 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 18 Apr 2017 14:34:24 -0700
Subject: [PATCH 34/53] added make list

---
 Makefile | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 7e57e1680..dc2ac4ce0 100644
--- a/Makefile
+++ b/Makefile
@@ -109,10 +109,15 @@ clean:
 # make install is validated only for Linux, OSX, Hurd and some BSD targets
 #------------------------------------------------------------------------------
 ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD))
+
 HOST_OS = POSIX
 CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON
-.PHONY: install uninstall travis-install clangtest gpptest armtest usan asan uasan
 
+.PHONY: list
+list:
+	@$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs
+
+.PHONY: install uninstall travis-install clangtest gpptest armtest usan asan uasan
 install:
 	@$(MAKE) -C $(ZSTDDIR) $@
 	@$(MAKE) -C $(PRGDIR) $@

From a4cab801835561ef63a8005482b8090f81c14aca Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 18 Apr 2017 14:54:54 -0700
Subject: [PATCH 35/53] added ZSTD_copyCCtx_internal()

which respects provided fParams.
---
 lib/compress/zstd_compress.c | 32 ++++++++++++++++++++++++--------
 1 file changed, 24 insertions(+), 8 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 9328b7c40..9d9cb394f 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -354,17 +354,20 @@ void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) {
     for (i=0; irep[i] = 0;
 }
 
-/*! ZSTD_copyCCtx() :
-*   Duplicate an existing context `srcCCtx` into another one `dstCCtx`.
-*   Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()).
-*   @return : 0, or an error code */
-size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long long pledgedSrcSize)
+
+/*! ZSTD_copyCCtx_internal() :
+ *  Duplicate an existing context `srcCCtx` into another one `dstCCtx`.
+ *  Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()).
+ *  pledgedSrcSize=0 means "empty" if fParams.contentSizeFlag=1
+ *  @return : 0, or an error code */
+size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx,
+                              ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize)
 {
     if (srcCCtx->stage!=ZSTDcs_init) return ERROR(stage_wrong);
 
     memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem));
     {   ZSTD_parameters params = srcCCtx->params;
-        params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
+        params.fParams = fParams;
         ZSTD_resetCCtx_internal(dstCCtx, params, pledgedSrcSize, ZSTDcrp_noMemset);
     }
 
@@ -402,9 +405,22 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long
     return 0;
 }
 
+/*! ZSTD_copyCCtx() :
+ *  Duplicate an existing context `srcCCtx` into another one `dstCCtx`.
+ *  Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()).
+ *  pledgedSrcSize==0 means "unknown".
+*   @return : 0, or an error code */
+size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long long pledgedSrcSize)
+{
+    ZSTD_frameParameters fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ };
+    fParams.contentSizeFlag = pledgedSrcSize>0;
+
+    return ZSTD_copyCCtx_internal(dstCCtx, srcCCtx, fParams, pledgedSrcSize);
+}
+
 
 /*! ZSTD_reduceTable() :
-*   reduce table indexes by `reducerValue` */
+ *  reduce table indexes by `reducerValue` */
 static void ZSTD_reduceTable (U32* const table, U32 const size, U32 const reducerValue)
 {
     U32 u;
@@ -2919,7 +2935,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced(
 {
     if (cdict==NULL) return ERROR(GENERIC);  /* does not support NULL cdict */
     if (cdict->dictContentSize)
-        CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext, pledgedSrcSize))  /* to be changed, to consider fParams */
+        CHECK_F( ZSTD_copyCCtx_internal(cctx, cdict->refContext, fParams, pledgedSrcSize) )
     else {
         ZSTD_parameters params = cdict->refContext->params;
         params.fParams = fParams;

From 0bb381dad88105946728d04c3ea45c19204c2636 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 18 Apr 2017 15:08:52 -0700
Subject: [PATCH 36/53] added test for ZSTD_initCStream_advanced()

when params.fParams.contentSizeFlags = 1
and pledgedSrcSize = 0
then srcSize should be considered 0 (empty), and not "unknown"
---
 tests/zstreamtest.c | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index ac200a7a6..6673208b0 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -479,7 +479,24 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
         DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r));
     }
 
-    /* Unknown srcSize */
+    /* Empty srcSize */
+    DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_advanced with pledgedSrcSize=0 and dict : ", testNb++);
+    {   ZSTD_parameters params = ZSTD_getParams(5, 0, 0);
+        params.fParams.contentSizeFlag = 1;
+        ZSTD_initCStream_advanced(zc, dictionary.start, dictionary.filled, params, 0);
+    } /* cstream advanced shall write content size = 0 */
+    inBuff.src = CNBuffer;
+    inBuff.size = 0;
+    inBuff.pos = 0;
+    outBuff.dst = compressedBuffer;
+    outBuff.size = compressedBufferSize;
+    outBuff.pos = 0;
+    if (ZSTD_isError(ZSTD_compressStream(zc, &outBuff, &inBuff))) goto _output_error;
+    if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
+    cSize = outBuff.pos;
+    if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != 0) goto _output_error;
+    DISPLAYLEVEL(3, "OK \n");
+
     DISPLAYLEVEL(3, "test%3i : pledgedSrcSize == 0 behaves properly : ", testNb++);
     {   ZSTD_parameters params = ZSTD_getParams(5, 0, 0);
         params.fParams.contentSizeFlag = 1;

From 98cf7fcb2ab311dc40a616cd15774813e1e8517d Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Tue, 18 Apr 2017 17:03:37 -0700
Subject: [PATCH 37/53] Update README

---
 lib/README.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/lib/README.md b/lib/README.md
index 3357e3d87..8fa60a20f 100644
--- a/lib/README.md
+++ b/lib/README.md
@@ -22,6 +22,14 @@ Some additional API may be useful if you're looking into advanced features :
                           They are not "stable", their definition may change in the future.
                           Only static linking is allowed.
 
+#### ZSTDMT API
+
+To enable building the multithreaded compression API, use the `make lib-mt` target.
+The header file zstdmt_compress.h (in compress/) provides the prototypes for the API.
+If linking a program that uses the ZSTDMT API against libzstd.a on a POSIX system,
+the -pthread flag must be provided to the compiler at link time.
+Note that these prototypes may still be used in a version built without multithread support,
+but they will be single threaded (i.e. no benefits will be given over the standard ZSTD API).
 
 #### Modular build
 

From 2c5514c759a747ea902a346a9a9911ce726272e7 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 18 Apr 2017 22:52:41 -0700
Subject: [PATCH 38/53] fixed ZSTDMT_initCStream_advanced()

Must use the new ZSTD_compressBegin_usingCDict_advanced()
to enforce correct frame parameters
---
 lib/compress/zstd_compress.c   | 2 +-
 lib/compress/zstdmt_compress.c | 2 +-
 tests/zstreamtest.c            | 6 ++++--
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 9d9cb394f..29dc0b773 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -392,12 +392,12 @@ size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx,
 
     /* copy entropy tables */
     dstCCtx->flagStaticTables = srcCCtx->flagStaticTables;
-    dstCCtx->flagStaticHufTable = srcCCtx->flagStaticHufTable;
     if (srcCCtx->flagStaticTables) {
         memcpy(dstCCtx->litlengthCTable, srcCCtx->litlengthCTable, sizeof(dstCCtx->litlengthCTable));
         memcpy(dstCCtx->matchlengthCTable, srcCCtx->matchlengthCTable, sizeof(dstCCtx->matchlengthCTable));
         memcpy(dstCCtx->offcodeCTable, srcCCtx->offcodeCTable, sizeof(dstCCtx->offcodeCTable));
     }
+    dstCCtx->flagStaticHufTable = srcCCtx->flagStaticHufTable;
     if (srcCCtx->flagStaticHufTable) {
         memcpy(dstCCtx->hufTable, srcCCtx->hufTable, HUF_CTABLE_SIZE(255));
     }
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 50c98ae93..04a448981 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -231,7 +231,7 @@ void ZSTDMT_compressChunk(void* jobDescription)
     DEBUGLOG(3, "job (first:%u) (last:%u) : dictSize %u, srcSize %u",
                  job->firstChunk, job->lastChunk, (U32)job->dictSize, (U32)job->srcSize);
     if (job->cdict) {  /* should only happen for first segment */
-        size_t const initError = ZSTD_compressBegin_usingCDict(job->cctx, job->cdict, job->fullFrameSize);
+        size_t const initError = ZSTD_compressBegin_usingCDict_advanced(job->cctx, job->cdict, job->params.fParams, job->fullFrameSize);
         if (job->cdict) DEBUGLOG(3, "using CDict ");
         if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; }
     } else {  /* srcStart points at reloaded section */
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 6673208b0..4048a7df0 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -131,7 +131,7 @@ static buffer_t FUZ_createDictionary(const void* src, size_t srcSize, size_t blo
     }
     {   size_t const dictSize = ZDICT_trainFromBuffer(dict.start, requestedDictSize, src, blockSizes, (unsigned)nbBlocks);
         free(blockSizes);
-        if (ZDICT_isError(dictSize)) { free(dict.start); return (buffer_t){ NULL, 0, 0 }; }
+        if (ZDICT_isError(dictSize)) { free(dict.start); return g_nullBuffer; }
         dict.size = requestedDictSize;
         dict.filled = dictSize;
         return dict;   /* how to return dictSize ? */
@@ -972,6 +972,7 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
                 params.fParams.checksumFlag = FUZ_rand(&lseed) & 1;
                 params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1;
                 params.fParams.contentSizeFlag = pledgedSrcSize>0;
+                DISPLAYLEVEL(5, "checksumFlag : %u \n", params.fParams.checksumFlag);
                 { size_t const initError = ZSTDMT_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize);
                   CHECK (ZSTD_isError(initError),"ZSTDMT_initCStream_advanced error : %s", ZSTD_getErrorName(initError)); }
                 ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_overlapSectionLog, FUZ_rand(&lseed) % 12);
@@ -1023,9 +1024,9 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
                     CHECK (ZSTD_isError(remainingToFlush), "ZSTDMT_endStream error : %s", ZSTD_getErrorName(remainingToFlush));
                     DISPLAYLEVEL(5, "endStream : remainingToFlush : %u \n", (U32)remainingToFlush);
             }   }
-            DISPLAYLEVEL(5, "Frame completed \n");
             crcOrig = XXH64_digest(&xxhState);
             cSize = outBuff.pos;
+            DISPLAYLEVEL(5, "Frame completed : %u bytes \n", (U32)cSize);
         }
 
         /* multi - fragments decompression test */
@@ -1046,6 +1047,7 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
                 DISPLAYLEVEL(5, "ZSTD_decompressStream input %u bytes \n", (U32)readCSrcSize);
                 decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff);
                 CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult));
+                DISPLAYLEVEL(5, "inBuff.pos = %u \n", (U32)readCSrcSize);
             }
             CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)", (U32)outBuff.pos, (U32)totalTestSize);
             CHECK (inBuff.pos != cSize, "compressed data should be fully read (%u != %u)", (U32)inBuff.pos, (U32)cSize);

From e847730452ba155888c193acf63f1038af0bfa81 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 18 Apr 2017 23:15:28 -0700
Subject: [PATCH 39/53] slightly refined README comments on lib-mt

---
 lib/README.md | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/lib/README.md b/lib/README.md
index 8fa60a20f..79b6fd500 100644
--- a/lib/README.md
+++ b/lib/README.md
@@ -24,12 +24,12 @@ Some additional API may be useful if you're looking into advanced features :
 
 #### ZSTDMT API
 
-To enable building the multithreaded compression API, use the `make lib-mt` target.
-The header file zstdmt_compress.h (in compress/) provides the prototypes for the API.
-If linking a program that uses the ZSTDMT API against libzstd.a on a POSIX system,
-the -pthread flag must be provided to the compiler at link time.
-Note that these prototypes may still be used in a version built without multithread support,
-but they will be single threaded (i.e. no benefits will be given over the standard ZSTD API).
+To enable multithreaded compression within the library, invoke `make lib-mt` target.
+Prototypes are defined in header file `compress/zstdmt_compress.h`.
+When linking a program that uses ZSTDMT API against libzstd.a on a POSIX system,
+`-pthread` flag must be provided to the compiler and linker.
+Note : ZSTDMT prototypes can still be used with a library built without multithread support,
+but in this case, they will be single threaded only.
 
 #### Modular build
 

From 522df42e10710a543414f84d620a8b858645d0e2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Milan=20=C5=A0ev=C4=8D=C3=ADk?= 
Date: Wed, 19 Apr 2017 19:20:27 +0200
Subject: [PATCH 40/53] Add lzma and zlib support to cmake build system

cmake 2.8.9 needed for FindLibLZMA
---
 build/cmake/CMakeLists.txt          |  2 +-
 build/cmake/programs/CMakeLists.txt | 27 +++++++++++++++++++++++++++
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/build/cmake/CMakeLists.txt b/build/cmake/CMakeLists.txt
index 9ce52ed21..bf6890818 100644
--- a/build/cmake/CMakeLists.txt
+++ b/build/cmake/CMakeLists.txt
@@ -8,7 +8,7 @@
 # ################################################################
 
 PROJECT(zstd)
-CMAKE_MINIMUM_REQUIRED(VERSION 2.8.7)
+CMAKE_MINIMUM_REQUIRED(VERSION 2.8.9)
 SET(ZSTD_SOURCE_DIR "${CMAKE_SOURCE_DIR}/../..")
 
 #-----------------------------------------------------------------------------
diff --git a/build/cmake/programs/CMakeLists.txt b/build/cmake/programs/CMakeLists.txt
index 588fea41e..b6f79aba3 100644
--- a/build/cmake/programs/CMakeLists.txt
+++ b/build/cmake/programs/CMakeLists.txt
@@ -64,3 +64,30 @@ IF (ZSTD_MULTITHREAD_SUPPORT)
     ADD_CUSTOM_COMMAND(TARGET zstd POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink zstd zstdmt COMMENT "Creating zstdmt symlink")
     INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdmt DESTINATION "bin")
 ENDIF (ZSTD_MULTITHREAD_SUPPORT)
+
+OPTION(ZSTD_ZLIB_SUPPORT "ZLIB SUPPORT" OFF)
+OPTION(ZSTD_LZMA_SUPPORT "LZMA SUPPORT" OFF)
+
+IF (ZSTD_ZLIB_SUPPORT)
+    FIND_PACKAGE(ZLIB REQUIRED)
+
+    IF (ZLIB_FOUND)
+        INCLUDE_DIRECTORIES(${ZLIB_INCLUDE_DIRS})
+	TARGET_LINK_LIBRARIES(zstd ${ZLIB_LIBRARIES})
+        SET_TARGET_PROPERTIES(zstd PROPERTIES COMPILE_DEFINITIONS "ZSTD_GZCOMPRESS;ZSTD_GZDECOMPRESS")
+    ELSE ()
+        MESSAGE(SEND_ERROR "zlib library is missing")
+    ENDIF ()
+ENDIF ()
+
+IF (ZSTD_LZMA_SUPPORT)
+    FIND_PACKAGE(LibLZMA REQUIRED)
+
+    IF (LIBLZMA_FOUND)
+        INCLUDE_DIRECTORIES(${LIBLZMA_INCLUDE_DIRS})
+	TARGET_LINK_LIBRARIES(zstd ${LIBLZMA_LIBRARIES})
+	SET_TARGET_PROPERTIES(zstd PROPERTIES COMPILE_DEFINITIONS "ZSTD_LZMACOMPRESS;ZSTD_LZMADECOMPRESS")
+    ELSE ()
+        MESSAGE(SEND_ERROR "lzma library is missing")
+    ENDIF ()
+ENDIF ()

From fce21777bdce8695ab75f0ae781bfbca39b22d53 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Milan=20=C5=A0ev=C4=8D=C3=ADk?= 
Date: Wed, 19 Apr 2017 19:22:17 +0200
Subject: [PATCH 41/53] Copy files during build phase, custom targets instead
 of commands

Previously some files were copied only during configure phase.
Custom targets seem nicer.
---
 build/cmake/programs/CMakeLists.txt | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/build/cmake/programs/CMakeLists.txt b/build/cmake/programs/CMakeLists.txt
index b6f79aba3..c3e68e01c 100644
--- a/build/cmake/programs/CMakeLists.txt
+++ b/build/cmake/programs/CMakeLists.txt
@@ -31,16 +31,18 @@ ENDIF (MSVC)
 
 ADD_EXECUTABLE(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PlatformDependResources})
 TARGET_LINK_LIBRARIES(zstd libzstd_shared)
-ADD_CUSTOM_COMMAND(TARGET zstd POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink zstd zstdcat COMMENT "Creating zstdcat symlink")
-ADD_CUSTOM_COMMAND(TARGET zstd POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink zstd unzstd COMMENT "Creating unzstd symlink")
+ADD_CUSTOM_TARGET(zstdcat ALL ${CMAKE_COMMAND} -E create_symlink zstd zstdcat DEPENDS zstd COMMENT "Creating zstdcat symlink")
+ADD_CUSTOM_TARGET(unzstd ALL ${CMAKE_COMMAND} -E create_symlink zstd unzstd DEPENDS zstd COMMENT "Creating unzstd symlink")
 INSTALL(TARGETS zstd RUNTIME DESTINATION "bin")
 INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdcat DESTINATION "bin")
 INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/unzstd DESTINATION "bin")
 
 IF (UNIX)
-    FILE(COPY ${PROGRAMS_DIR}/zstd.1 DESTINATION .)
-    ADD_CUSTOM_COMMAND(TARGET zstd POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink zstd.1 zstdcat.1 COMMENT "Creating zstdcat.1 symlink")
-    ADD_CUSTOM_COMMAND(TARGET zstd POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink zstd.1 unzstd.1 COMMENT "Creating unzstd.1 symlink")
+    ADD_CUSTOM_TARGET(zstd.1 ALL
+        ${CMAKE_COMMAND} -E copy ${PROGRAMS_DIR}/zstd.1 .
+        COMMENT "Copying manpage zstd.1")
+    ADD_CUSTOM_TARGET(zstdcat.1 ALL ${CMAKE_COMMAND} -E create_symlink zstd.1 zstdcat.1 DEPENDS zstd.1 COMMENT "Creating zstdcat.1 symlink")
+    ADD_CUSTOM_TARGET(unzstd.1 ALL ${CMAKE_COMMAND} -E create_symlink zstd.1 unzstd.1 DEPENDS zstd.1 COMMENT "Creating unzstd.1 symlink")
     INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstd.1 DESTINATION "share/man/man1")
     INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdcat.1 DESTINATION "share/man/man1")
     INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/unzstd.1 DESTINATION "share/man/man1")
@@ -61,7 +63,7 @@ IF (ZSTD_MULTITHREAD_SUPPORT)
         MESSAGE(SEND_ERROR "ZSTD currently does not support thread libraries other than pthreads")
     ENDIF()
 
-    ADD_CUSTOM_COMMAND(TARGET zstd POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink zstd zstdmt COMMENT "Creating zstdmt symlink")
+    ADD_CUSTOM_TARGET(zstdmt ALL ${CMAKE_COMMAND} -E create_symlink zstd zstdmt DEPENDS zstd COMMENT "Creating zstdmt symlink")
     INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdmt DESTINATION "bin")
 ENDIF (ZSTD_MULTITHREAD_SUPPORT)
 

From cba4e79a9314b4bc216d25bf80b59f1d221c141f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Milan=20=C5=A0ev=C4=8D=C3=ADk?= 
Date: Wed, 19 Apr 2017 19:25:29 +0200
Subject: [PATCH 42/53] Create and install pkg-config file with cmake

---
 build/cmake/lib/CMakeLists.txt  | 12 ++++++++++++
 build/cmake/lib/pkgconfig.cmake |  1 +
 2 files changed, 13 insertions(+)
 create mode 100644 build/cmake/lib/pkgconfig.cmake

diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt
index 7a345bf56..50618f4a8 100644
--- a/build/cmake/lib/CMakeLists.txt
+++ b/build/cmake/lib/CMakeLists.txt
@@ -130,8 +130,20 @@ IF (ZSTD_BUILD_STATIC)
 ENDIF (ZSTD_BUILD_STATIC)
 
 IF (UNIX)
+    # pkg-config
+    SET(PREFIX "${CMAKE_INSTALL_PREFIX}")
+    SET(LIBDIR "${CMAKE_INSTALL_PREFIX}/lib")
+    SET(INCLUDEDIR "${CMAKE_INSTALL_PREFIX}/include")
+    SET(VERSION "${LIBVER_MAJOR}.${LIBVER_MINOR}.${LIBVER_RELEASE}")
+    ADD_CUSTOM_TARGET(libzstd.pc ALL
+            ${CMAKE_COMMAND} -DIN="${LIBRARY_DIR}/libzstd.pc.in" -DOUT="libzstd.pc"
+            -DPREFIX="${PREFIX}" -DLIBDIR="${LIBDIR}" -DINCLUDEDIR="${INCLUDEDIR}" -DVERSION="${VERSION}"
+            -P "${CMAKE_SOURCE_DIR}/lib/pkgconfig.cmake"
+            COMMENT "Creating pkg-config file")
+
     # install target
     INSTALL(FILES ${LIBRARY_DIR}/zstd.h ${LIBRARY_DIR}/deprecated/zbuff.h ${LIBRARY_DIR}/dictBuilder/zdict.h DESTINATION "include")
+    INSTALL(FILES "${CMAKE_CURRENT_BINARY_DIR}/libzstd.pc" DESTINATION "share/pkgconfig")
     INSTALL(TARGETS libzstd_shared LIBRARY DESTINATION "lib")
     IF (ZSTD_BUILD_STATIC)
         INSTALL(TARGETS libzstd_static ARCHIVE DESTINATION "lib")
diff --git a/build/cmake/lib/pkgconfig.cmake b/build/cmake/lib/pkgconfig.cmake
new file mode 100644
index 000000000..5434ff75c
--- /dev/null
+++ b/build/cmake/lib/pkgconfig.cmake
@@ -0,0 +1 @@
+CONFIGURE_FILE("${IN}" "${OUT}" @ONLY)

From f49f760b41caea2ead424c8e8f2f8affef2a594b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Milan=20=C5=A0ev=C4=8D=C3=ADk?= 
Date: Wed, 19 Apr 2017 19:25:53 +0200
Subject: [PATCH 43/53] Test new cmake branches with Circle CI

---
 Makefile   | 2 +-
 circle.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 7e57e1680..ae2f68867 100644
--- a/Makefile
+++ b/Makefile
@@ -110,7 +110,7 @@ clean:
 #------------------------------------------------------------------------------
 ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD))
 HOST_OS = POSIX
-CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON
+CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON
 .PHONY: install uninstall travis-install clangtest gpptest armtest usan asan uasan
 
 install:
diff --git a/circle.yml b/circle.yml
index 298569d14..218e33bfc 100644
--- a/circle.yml
+++ b/circle.yml
@@ -3,7 +3,7 @@ dependencies:
     - sudo dpkg --add-architecture i386
     - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; sudo apt-get -y -qq update
     - sudo apt-get -y install gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross
-    - sudo apt-get -y install libstdc++-6-dev clang gcc g++ gcc-5 gcc-6
+    - sudo apt-get -y install libstdc++-6-dev clang gcc g++ gcc-5 gcc-6 zlib1g-dev liblzma-dev
     - sudo apt-get -y install linux-libc-dev:i386 libc6-dev-i386
 
 test:

From eb7371f1794378cad706734031279f4afbb29102 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Milan=20=C5=A0ev=C4=8D=C3=ADk?= 
Date: Wed, 19 Apr 2017 20:16:11 +0200
Subject: [PATCH 44/53] Change all SET_TARGET_PROPERTIES to SET_PROPERTY

SET_PROPERTY function can append to lists, whereas previously used
SET_TARGET_PROPERTIES cannot.
---
 build/cmake/contrib/pzstd/CMakeLists.txt |  4 ++--
 build/cmake/lib/CMakeLists.txt           |  4 ++--
 build/cmake/programs/CMakeLists.txt      | 12 ++++++------
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/build/cmake/contrib/pzstd/CMakeLists.txt b/build/cmake/contrib/pzstd/CMakeLists.txt
index b4fbe1689..71def02cd 100644
--- a/build/cmake/contrib/pzstd/CMakeLists.txt
+++ b/build/cmake/contrib/pzstd/CMakeLists.txt
@@ -21,8 +21,8 @@ SET(PZSTD_DIR ${ZSTD_SOURCE_DIR}/contrib/pzstd)
 INCLUDE_DIRECTORIES(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${PZSTD_DIR})
 
 ADD_EXECUTABLE(pzstd ${PZSTD_DIR}/main.cpp ${PZSTD_DIR}/Options.cpp ${PZSTD_DIR}/Pzstd.cpp ${PZSTD_DIR}/SkippableFrame.cpp)
-SET_TARGET_PROPERTIES(pzstd PROPERTIES COMPILE_DEFINITIONS "NDEBUG")
-SET_TARGET_PROPERTIES(pzstd PROPERTIES COMPILE_OPTIONS "-Wno-shadow")
+SET_PROPERTY(TARGET pzstd APPEND PROPERTY COMPILE_DEFINITIONS "NDEBUG")
+SET_PROPERTY(TARGET pzstd APPEND PROPERTY COMPILE_OPTIONS "-Wno-shadow")
 
 SET(THREADS_PREFER_PTHREAD_FLAG ON)
 FIND_PACKAGE(Threads REQUIRED)
diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt
index 50618f4a8..5e945ff12 100644
--- a/build/cmake/lib/CMakeLists.txt
+++ b/build/cmake/lib/CMakeLists.txt
@@ -97,9 +97,9 @@ ENDIF (ZSTD_BUILD_STATIC)
 
 # Add specific compile definitions for MSVC project
 IF (MSVC)
-    SET_TARGET_PROPERTIES(libzstd_shared PROPERTIES COMPILE_DEFINITIONS "ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS")
+    SET_PROPERTY(TARGET libzstd_shared APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS")
     IF (ZSTD_BUILD_STATIC)
-        SET_TARGET_PROPERTIES(libzstd_static PROPERTIES COMPILE_DEFINITIONS "ZSTD_HEAPMODE=0;_CRT_SECURE_NO_WARNINGS")
+        SET_PROPERTY(TARGET libzstd_static APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_HEAPMODE=0;_CRT_SECURE_NO_WARNINGS")
     ENDIF (ZSTD_BUILD_STATIC)
 ENDIF (MSVC)
 
diff --git a/build/cmake/programs/CMakeLists.txt b/build/cmake/programs/CMakeLists.txt
index c3e68e01c..cb6aa9218 100644
--- a/build/cmake/programs/CMakeLists.txt
+++ b/build/cmake/programs/CMakeLists.txt
@@ -49,11 +49,11 @@ IF (UNIX)
 
     ADD_EXECUTABLE(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c)
     TARGET_LINK_LIBRARIES(zstd-frugal libzstd_shared)
-    SET_TARGET_PROPERTIES(zstd-frugal PROPERTIES COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT")
+    SET_PROPERTY(TARGET zstd-frugal APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT")
 ENDIF (UNIX)
 
 IF (ZSTD_MULTITHREAD_SUPPORT)
-    SET_TARGET_PROPERTIES(zstd PROPERTIES COMPILE_DEFINITIONS "ZSTD_MULTITHREAD")
+    SET_PROPERTY(TARGET zstd APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_MULTITHREAD")
 
     SET(THREADS_PREFER_PTHREAD_FLAG ON)
     FIND_PACKAGE(Threads REQUIRED)
@@ -75,8 +75,8 @@ IF (ZSTD_ZLIB_SUPPORT)
 
     IF (ZLIB_FOUND)
         INCLUDE_DIRECTORIES(${ZLIB_INCLUDE_DIRS})
-	TARGET_LINK_LIBRARIES(zstd ${ZLIB_LIBRARIES})
-        SET_TARGET_PROPERTIES(zstd PROPERTIES COMPILE_DEFINITIONS "ZSTD_GZCOMPRESS;ZSTD_GZDECOMPRESS")
+        TARGET_LINK_LIBRARIES(zstd ${ZLIB_LIBRARIES})
+        SET_PROPERTY(TARGET zstd APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_GZCOMPRESS;ZSTD_GZDECOMPRESS")
     ELSE ()
         MESSAGE(SEND_ERROR "zlib library is missing")
     ENDIF ()
@@ -87,8 +87,8 @@ IF (ZSTD_LZMA_SUPPORT)
 
     IF (LIBLZMA_FOUND)
         INCLUDE_DIRECTORIES(${LIBLZMA_INCLUDE_DIRS})
-	TARGET_LINK_LIBRARIES(zstd ${LIBLZMA_LIBRARIES})
-	SET_TARGET_PROPERTIES(zstd PROPERTIES COMPILE_DEFINITIONS "ZSTD_LZMACOMPRESS;ZSTD_LZMADECOMPRESS")
+        TARGET_LINK_LIBRARIES(zstd ${LIBLZMA_LIBRARIES})
+        SET_PROPERTY(TARGET zstd APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_LZMACOMPRESS;ZSTD_LZMADECOMPRESS")
     ELSE ()
         MESSAGE(SEND_ERROR "lzma library is missing")
     ENDIF ()

From e348dad3053c0ffd801a2fc27a7b939570cbf737 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 20 Apr 2017 11:14:13 -0700
Subject: [PATCH 45/53] minor long line reformatting

---
 lib/compress/zstd_compress.c | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 4ad978a4b..a95f4b930 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -510,8 +510,10 @@ static size_t ZSTD_compressLiterals (ZSTD_CCtx* zc,
     {   HUF_repeat repeat = zc->flagStaticHufTable;
         int const preferRepeat = zc->params.cParams.strategy < ZSTD_lazy ? srcSize <= 1024 : 0;
         if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1;
-        cLitSize = singleStream ? HUF_compress1X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, zc->tmpCounters, sizeof(zc->tmpCounters), zc->hufTable, &repeat, preferRepeat)
-                                : HUF_compress4X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, zc->tmpCounters, sizeof(zc->tmpCounters), zc->hufTable, &repeat, preferRepeat);
+        cLitSize = singleStream ? HUF_compress1X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11,
+                                      zc->tmpCounters, sizeof(zc->tmpCounters), zc->hufTable, &repeat, preferRepeat)
+                                : HUF_compress4X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11,
+                                      zc->tmpCounters, sizeof(zc->tmpCounters), zc->hufTable, &repeat, preferRepeat);
         if (repeat != HUF_repeat_none) { hType = set_repeat; }    /* reused the existing table */
         else { zc->flagStaticHufTable = HUF_repeat_check; }       /* now have a table to reuse */
     }
@@ -856,7 +858,14 @@ static unsigned ZSTD_NbCommonBytes (register size_t val)
 #       elif defined(__GNUC__) && (__GNUC__ >= 3)
             return (__builtin_ctzll((U64)val) >> 3);
 #       else
-            static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };
+            static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2,
+                                                     0, 3, 1, 3, 1, 4, 2, 7,
+                                                     0, 2, 3, 6, 1, 5, 3, 5,
+                                                     1, 3, 4, 4, 2, 5, 6, 7,
+                                                     7, 0, 1, 2, 3, 3, 4, 6,
+                                                     2, 6, 5, 5, 3, 4, 5, 6,
+                                                     7, 1, 2, 4, 6, 4, 4, 5,
+                                                     7, 2, 6, 5, 7, 6, 7, 7 };
             return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
 #       endif
         } else { /* 32 bits */
@@ -867,7 +876,10 @@ static unsigned ZSTD_NbCommonBytes (register size_t val)
 #       elif defined(__GNUC__) && (__GNUC__ >= 3)
             return (__builtin_ctz((U32)val) >> 3);
 #       else
-            static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 };
+            static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0,
+                                                     3, 2, 2, 1, 3, 2, 0, 1,
+                                                     3, 3, 1, 2, 2, 2, 2, 0,
+                                                     3, 1, 2, 0, 1, 0, 1, 1 };
             return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
 #       endif
         }

From c17e020c9aedbd421e19e3607a16d87559d29d97 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 20 Apr 2017 12:50:02 -0700
Subject: [PATCH 46/53] disable assert when compiling paramgrill

paramgrill is a benchmark calibration function.
Speed accuracy is critical, it cannot be altered by assert.
---
 lib/compress/zstd_compress.c | 60 ++++++++++++++++++++++++------------
 tests/Makefile               |  4 ++-
 2 files changed, 44 insertions(+), 20 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 891094e46..5f18121b3 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1315,7 +1315,8 @@ void ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx* cctx,
         const BYTE* match = base + matchIndexS;
         hashLong[h2] = hashSmall[h] = current;   /* update hash tables */
 
-        if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) { /* note : by construction, offset_1 <= current */
+        assert(offset_1 <= current);   /* supposed guaranteed by construction */
+        if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) {
             mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4;
             ip++;
             ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mLength-MINMATCH);
@@ -1893,7 +1894,11 @@ size_t ZSTD_HcFindBestMatch_generic (
         }
 
         /* save best solution */
-        if (currentMl > ml) { ml = currentMl; *offsetPtr = current - matchIndex + ZSTD_REP_MOVE; if (ip+currentMl == iLimit) break; /* best possible, and avoid read overflow*/ }
+        if (currentMl > ml) {
+            ml = currentMl;
+            *offsetPtr = current - matchIndex + ZSTD_REP_MOVE;
+            if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */
+        }
 
         if (matchIndex <= minChain) break;
         matchIndex = NEXT_IN_CHAIN(matchIndex, chainMask);
@@ -2038,7 +2043,9 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
 
         /* catch up */
         if (offset) {
-            while ((start>anchor) && (start>base+offset-ZSTD_REP_MOVE) && (start[-1] == start[-1-offset+ZSTD_REP_MOVE]))   /* only search for offset within prefix */
+            while ( (start > anchor)
+                 && (start > base+offset-ZSTD_REP_MOVE)
+                 && (start[-1] == start[-1-offset+ZSTD_REP_MOVE]) )  /* only search for offset within prefix */
                 { start--; matchLength++; }
             offset_2 = offset_1; offset_1 = (U32)(offset - ZSTD_REP_MOVE);
         }
@@ -2204,9 +2211,9 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
                     if (MEM_read32(ip) == MEM_read32(repMatch)) {
                         /* repcode detected */
                         const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
-                        size_t repLength = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32;
-                        int gain2 = (int)(repLength * 4);
-                        int gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 1);
+                        size_t const repLength = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32;
+                        int const gain2 = (int)(repLength * 4);
+                        int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 1);
                         if ((repLength >= EQUAL_READ32) && (gain2 > gain1))
                             matchLength = repLength, offset = 0, start = ip;
                 }   }
@@ -2339,8 +2346,12 @@ typedef void (*ZSTD_blockCompressor) (ZSTD_CCtx* ctx, const void* src, size_t sr
 static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict)
 {
     static const ZSTD_blockCompressor blockCompressor[2][8] = {
-        { ZSTD_compressBlock_fast, ZSTD_compressBlock_doubleFast, ZSTD_compressBlock_greedy, ZSTD_compressBlock_lazy, ZSTD_compressBlock_lazy2, ZSTD_compressBlock_btlazy2, ZSTD_compressBlock_btopt, ZSTD_compressBlock_btopt2 },
-        { ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_doubleFast_extDict, ZSTD_compressBlock_greedy_extDict, ZSTD_compressBlock_lazy_extDict,ZSTD_compressBlock_lazy2_extDict, ZSTD_compressBlock_btlazy2_extDict, ZSTD_compressBlock_btopt_extDict, ZSTD_compressBlock_btopt2_extDict }
+        { ZSTD_compressBlock_fast, ZSTD_compressBlock_doubleFast, ZSTD_compressBlock_greedy,
+          ZSTD_compressBlock_lazy, ZSTD_compressBlock_lazy2, ZSTD_compressBlock_btlazy2,
+          ZSTD_compressBlock_btopt, ZSTD_compressBlock_btopt2 },
+        { ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_doubleFast_extDict, ZSTD_compressBlock_greedy_extDict,
+          ZSTD_compressBlock_lazy_extDict,ZSTD_compressBlock_lazy2_extDict, ZSTD_compressBlock_btlazy2_extDict,
+          ZSTD_compressBlock_btopt_extDict, ZSTD_compressBlock_btopt2_extDict }
     };
 
     return blockCompressor[extDict][(U32)strat];
@@ -2356,7 +2367,7 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCa
     if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) return 0;   /* don't even attempt compression below a certain srcSize */
     ZSTD_resetSeqStore(&(zc->seqStore));
     if (current > zc->nextToUpdate + 384)
-        zc->nextToUpdate = current - MIN(192, (U32)(current - zc->nextToUpdate - 384));   /* update tree not updated after finding very long rep matches */
+        zc->nextToUpdate = current - MIN(192, (U32)(current - zc->nextToUpdate - 384));   /* limited update after finding a very long match */
     blockCompressor(zc, src, srcSize);
     return ZSTD_compressSequences(zc, dst, dstCapacity, srcSize);
 }
@@ -2388,7 +2399,8 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
         U32 const lastBlock = lastFrameChunk & (blockSize >= remaining);
         size_t cSize;
 
-        if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall);   /* not enough space to store compressed block */
+        if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE)
+            return ERROR(dstSize_tooSmall);   /* not enough space to store compressed block */
         if (remaining < blockSize) blockSize = remaining;
 
         /* preemptive overflow correction */
@@ -2647,7 +2659,8 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_CCtx* cctx, const void* dict, size_t
         if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
         if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted);
         /* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */
-        CHECK_E (FSE_buildCTable_wksp(cctx->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog, scratchBuffer, sizeof(scratchBuffer)), dictionary_corrupted);
+        CHECK_E( FSE_buildCTable_wksp(cctx->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog, scratchBuffer, sizeof(scratchBuffer)),
+                 dictionary_corrupted);
         dictPtr += offcodeHeaderSize;
     }
 
@@ -2657,8 +2670,9 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_CCtx* cctx, const void* dict, size_t
         if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
         if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted);
         /* Every match length code must have non-zero probability */
-        CHECK_F (ZSTD_checkDictNCount(matchlengthNCount, matchlengthMaxValue, MaxML));
-        CHECK_E (FSE_buildCTable_wksp(cctx->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog, scratchBuffer, sizeof(scratchBuffer)), dictionary_corrupted);
+        CHECK_F( ZSTD_checkDictNCount(matchlengthNCount, matchlengthMaxValue, MaxML));
+        CHECK_E( FSE_buildCTable_wksp(cctx->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog, scratchBuffer, sizeof(scratchBuffer)),
+                 dictionary_corrupted);
         dictPtr += matchlengthHeaderSize;
     }
 
@@ -2668,8 +2682,9 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_CCtx* cctx, const void* dict, size_t
         if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
         if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted);
         /* Every literal length code must have non-zero probability */
-        CHECK_F (ZSTD_checkDictNCount(litlengthNCount, litlengthMaxValue, MaxLL));
-        CHECK_E(FSE_buildCTable_wksp(cctx->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog, scratchBuffer, sizeof(scratchBuffer)), dictionary_corrupted);
+        CHECK_F( ZSTD_checkDictNCount(litlengthNCount, litlengthMaxValue, MaxLL));
+        CHECK_E( FSE_buildCTable_wksp(cctx->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog, scratchBuffer, sizeof(scratchBuffer)),
+                 dictionary_corrupted);
         dictPtr += litlengthHeaderSize;
     }
 
@@ -2829,7 +2844,8 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
     return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params);
 }
 
-size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict, size_t dictSize, int compressionLevel)
+size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize,
+                               const void* dict, size_t dictSize, int compressionLevel)
 {
     ZSTD_parameters params = ZSTD_getParams(compressionLevel, srcSize, dict ? dictSize : 0);
     params.fParams.contentSizeFlag = 1;
@@ -3053,7 +3069,11 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs)
 /*======   Initialization   ======*/
 
 size_t ZSTD_CStreamInSize(void)  { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; }
-size_t ZSTD_CStreamOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ; }
+
+size_t ZSTD_CStreamOutSize(void)
+{
+    return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ;
+}
 
 static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
 {
@@ -3308,7 +3328,8 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
         /* flush whatever remains */
         size_t srcSize = 0;
         size_t sizeWritten = output->size - output->pos;
-        size_t const notEnded = ZSTD_compressStream_generic(zcs, ostart, &sizeWritten, &srcSize, &srcSize, zsf_end);  /* use a valid src address instead of NULL */
+        size_t const notEnded = ZSTD_compressStream_generic(zcs, ostart, &sizeWritten,
+                                     &srcSize /* use a valid src address instead of NULL */, &srcSize, zsf_end);
         size_t const remainingToFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize;
         op += sizeWritten;
         if (remainingToFlush) {
@@ -3318,7 +3339,8 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
         /* create epilogue */
         zcs->stage = zcss_final;
         zcs->outBuffContentSize = !notEnded ? 0 :
-            ZSTD_compressEnd(zcs->cctx, zcs->outBuff, zcs->outBuffSize, NULL, 0);  /* write epilogue, including final empty block, into outBuff */
+            /* write epilogue, including final empty block, into outBuff */
+            ZSTD_compressEnd(zcs->cctx, zcs->outBuff, zcs->outBuffSize, NULL, 0);
         if (ZSTD_isError(zcs->outBuffContentSize)) return zcs->outBuffContentSize;
     }
 
diff --git a/tests/Makefile b/tests/Makefile
index 70ef51878..f5260e50e 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -26,9 +26,10 @@ PYTHON ?= python3
 TESTARTEFACT := versionsTest namespaceTest
 
 
+DEBUGFLAG= -DZSTD_DEBUG=1
 CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
            -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) \
-           -DZSTD_DEBUG=1
+           $(DEBUGFLAG)
 CFLAGS  ?= -O3
 CFLAGS  += -g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
            -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
@@ -156,6 +157,7 @@ zstreamtest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c zstreamtest.c
 	$(MAKE) -C $(ZSTDDIR) libzstd
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@$(EXT)
 
+paramgrill : DEBUGFLAG =
 paramgrill : $(ZSTD_FILES) $(PRGDIR)/datagen.c paramgrill.c
 	$(CC)      $(FLAGS) $^ -lm -o $@$(EXT)
 

From 1a96bec8dba9d8b8147259a703dd3b7f26584080 Mon Sep 17 00:00:00 2001
From: Michael Maltese 
Date: Thu, 20 Apr 2017 15:35:56 -0700
Subject: [PATCH 47/53] CMake: Set ZSTD_SOURCE_DIR from
 CMAKE_CURRENT_SOURCE_DIR

Instead of CMAKE_SOURCE_DIR, which is not correct when embedding Zstd
within a larger project.
---
 build/cmake/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/cmake/CMakeLists.txt b/build/cmake/CMakeLists.txt
index bf6890818..0aa7b3072 100644
--- a/build/cmake/CMakeLists.txt
+++ b/build/cmake/CMakeLists.txt
@@ -9,7 +9,7 @@
 
 PROJECT(zstd)
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8.9)
-SET(ZSTD_SOURCE_DIR "${CMAKE_SOURCE_DIR}/../..")
+SET(ZSTD_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../..")
 
 #-----------------------------------------------------------------------------
 # Add extra compilation flags

From 7f1fb955662d0df4425db0322b1fd3037edf42f1 Mon Sep 17 00:00:00 2001
From: Michael Maltese 
Date: Thu, 20 Apr 2017 15:46:44 -0700
Subject: [PATCH 48/53] CMake: namespace modules and set CMAKE_MODULE_PATH

---
 build/cmake/CMakeLists.txt                                   | 5 +++--
 ...aCompilationFlags.cmake => AddZstdCompilationFlags.cmake} | 4 ++--
 .../{GetLibraryVersion.cmake => GetZstdLibraryVersion.cmake} | 2 +-
 build/cmake/contrib/gen_html/CMakeLists.txt                  | 4 ++--
 build/cmake/lib/CMakeLists.txt                               | 4 ++--
 5 files changed, 10 insertions(+), 9 deletions(-)
 rename build/cmake/CMakeModules/{AddExtraCompilationFlags.cmake => AddZstdCompilationFlags.cmake} (98%)
 rename build/cmake/CMakeModules/{GetLibraryVersion.cmake => GetZstdLibraryVersion.cmake} (86%)

diff --git a/build/cmake/CMakeLists.txt b/build/cmake/CMakeLists.txt
index 0aa7b3072..5c4eca61c 100644
--- a/build/cmake/CMakeLists.txt
+++ b/build/cmake/CMakeLists.txt
@@ -10,12 +10,13 @@
 PROJECT(zstd)
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8.9)
 SET(ZSTD_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../..")
+LIST(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules")
 
 #-----------------------------------------------------------------------------
 # Add extra compilation flags
 #-----------------------------------------------------------------------------
-INCLUDE(CMakeModules/AddExtraCompilationFlags.cmake)
-ADD_EXTRA_COMPILATION_FLAGS()
+INCLUDE(AddZstdCompilationFlags)
+ADD_ZSTD_COMPILATION_FLAGS()
 
 #-----------------------------------------------------------------------------
 # Options
diff --git a/build/cmake/CMakeModules/AddExtraCompilationFlags.cmake b/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake
similarity index 98%
rename from build/cmake/CMakeModules/AddExtraCompilationFlags.cmake
rename to build/cmake/CMakeModules/AddZstdCompilationFlags.cmake
index e099a01da..177db541c 100644
--- a/build/cmake/CMakeModules/AddExtraCompilationFlags.cmake
+++ b/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake
@@ -19,7 +19,7 @@ function(EnableCompilerFlag _flag _C _CXX)
     endif ()
 endfunction()
 
-MACRO(ADD_EXTRA_COMPILATION_FLAGS)
+MACRO(ADD_ZSTD_COMPILATION_FLAGS)
     if (CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang" OR MINGW) #Not only UNIX but also WIN32 for MinGW
         #Set c++11 by default
         EnableCompilerFlag("-std=c++11" false true)
@@ -95,4 +95,4 @@ MACRO(ADD_EXTRA_COMPILATION_FLAGS)
     set(CMAKE_C_FLAGS_MINSIZEREL "${CMAKE_C_FLAGS_MINSIZEREL}" CACHE STRING "Updated flags" FORCE)
     set(CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO}" CACHE STRING "Updated flags" FORCE)
 
-ENDMACRO(ADD_EXTRA_COMPILATION_FLAGS)
+ENDMACRO(ADD_ZSTD_COMPILATION_FLAGS)
diff --git a/build/cmake/CMakeModules/GetLibraryVersion.cmake b/build/cmake/CMakeModules/GetZstdLibraryVersion.cmake
similarity index 86%
rename from build/cmake/CMakeModules/GetLibraryVersion.cmake
rename to build/cmake/CMakeModules/GetZstdLibraryVersion.cmake
index 95d84a89f..8b6f394da 100644
--- a/build/cmake/CMakeModules/GetLibraryVersion.cmake
+++ b/build/cmake/CMakeModules/GetZstdLibraryVersion.cmake
@@ -1,4 +1,4 @@
-function(GetLibraryVersion _header _major _minor _release)
+function(GetZstdLibraryVersion _header _major _minor _release)
     # Read file content
     FILE(READ ${_header} CONTENT)
 
diff --git a/build/cmake/contrib/gen_html/CMakeLists.txt b/build/cmake/contrib/gen_html/CMakeLists.txt
index dff8c7a11..c10c62b54 100644
--- a/build/cmake/contrib/gen_html/CMakeLists.txt
+++ b/build/cmake/contrib/gen_html/CMakeLists.txt
@@ -11,7 +11,7 @@
 # ################################################################
 
 PROJECT(gen_html)
-INCLUDE(${CMAKE_SOURCE_DIR}/CMakeModules/GetLibraryVersion.cmake)
+INCLUDE(GetZstdLibraryVersion)
 
 SET(CMAKE_INCLUDE_CURRENT_DIR TRUE)
 
@@ -24,7 +24,7 @@ INCLUDE_DIRECTORIES(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${GENHT
 
 ADD_EXECUTABLE(gen_html ${GENHTML_DIR}/gen_html.cpp)
 
-GetLibraryVersion(${LIBRARY_DIR}/zstd.h VMAJOR VMINOR VRELEASE)
+GetZstdLibraryVersion(${LIBRARY_DIR}/zstd.h VMAJOR VMINOR VRELEASE)
 SET(LIBVERSION "${VMAJOR}.${VMINOR}.${VRELEASE}")
 ADD_CUSTOM_TARGET(zstd_manual.html ALL
                   ${GENHTML_BINARY} "${LIBVERSION}" "${LIBRARY_DIR}/zstd.h" "${PROJECT_BINARY_DIR}/zstd_manual.html"
diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt
index 5e945ff12..937ddbc50 100644
--- a/build/cmake/lib/CMakeLists.txt
+++ b/build/cmake/lib/CMakeLists.txt
@@ -11,7 +11,6 @@
 # ################################################################
 
 PROJECT(libzstd)
-INCLUDE(${CMAKE_SOURCE_DIR}/CMakeModules/GetLibraryVersion.cmake)
 
 SET(CMAKE_INCLUDE_CURRENT_DIR TRUE)
 OPTION(ZSTD_BUILD_STATIC "BUILD STATIC LIBRARIES" OFF)
@@ -21,7 +20,8 @@ SET(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
 INCLUDE_DIRECTORIES(${LIBRARY_DIR} ${LIBRARY_DIR}/common)
 
 # Parse version
-GetLibraryVersion(${LIBRARY_DIR}/zstd.h LIBVER_MAJOR LIBVER_MINOR LIBVER_RELEASE)
+INCLUDE(GetZstdLibraryVersion)
+GetZstdLibraryVersion(${LIBRARY_DIR}/zstd.h LIBVER_MAJOR LIBVER_MINOR LIBVER_RELEASE)
 MESSAGE("ZSTD VERSION ${LIBVER_MAJOR}.${LIBVER_MINOR}.${LIBVER_RELEASE}")
 
 SET(Sources

From 554a13dd4b7b61920c9bd8bdc6df81f040f80871 Mon Sep 17 00:00:00 2001
From: Michael Maltese 
Date: Thu, 20 Apr 2017 15:47:31 -0700
Subject: [PATCH 49/53] CMake: various configure_file fixes to use
 CMAKE_CURRENT_SOURCE_DIR

---
 build/cmake/lib/CMakeLists.txt                 | 8 ++++----
 build/cmake/{ => lib}/cmake_uninstall.cmake.in | 0
 2 files changed, 4 insertions(+), 4 deletions(-)
 rename build/cmake/{ => lib}/cmake_uninstall.cmake.in (100%)

diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt
index 937ddbc50..429d49449 100644
--- a/build/cmake/lib/CMakeLists.txt
+++ b/build/cmake/lib/CMakeLists.txt
@@ -138,7 +138,7 @@ IF (UNIX)
     ADD_CUSTOM_TARGET(libzstd.pc ALL
             ${CMAKE_COMMAND} -DIN="${LIBRARY_DIR}/libzstd.pc.in" -DOUT="libzstd.pc"
             -DPREFIX="${PREFIX}" -DLIBDIR="${LIBDIR}" -DINCLUDEDIR="${INCLUDEDIR}" -DVERSION="${VERSION}"
-            -P "${CMAKE_SOURCE_DIR}/lib/pkgconfig.cmake"
+            -P "${CMAKE_CURRENT_SOURCE_DIR}/pkgconfig.cmake"
             COMMENT "Creating pkg-config file")
 
     # install target
@@ -151,10 +151,10 @@ IF (UNIX)
 
     # uninstall target
     CONFIGURE_FILE(
-            "${CMAKE_SOURCE_DIR}/cmake_uninstall.cmake.in"
-            "${CMAKE_BINARY_DIR}/cmake_uninstall.cmake"
+            "${CMAKE_CURRENT_SOURCE_DIR}/cmake_uninstall.cmake.in"
+            "${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake"
             IMMEDIATE @ONLY)
 
     ADD_CUSTOM_TARGET(uninstall
-            COMMAND ${CMAKE_COMMAND} -P ${CMAKE_BINARY_DIR}/cmake_uninstall.cmake)
+            COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake)
 ENDIF (UNIX)
diff --git a/build/cmake/cmake_uninstall.cmake.in b/build/cmake/lib/cmake_uninstall.cmake.in
similarity index 100%
rename from build/cmake/cmake_uninstall.cmake.in
rename to build/cmake/lib/cmake_uninstall.cmake.in

From 377401f161e413962630e4f98a9e4eb55335cd7c Mon Sep 17 00:00:00 2001
From: Michael Maltese 
Date: Thu, 20 Apr 2017 15:54:17 -0700
Subject: [PATCH 50/53] CMake: don't recheck compile flags every time

Doesn't cause a problem when embedded within a larger project, but is
annoying.
---
 .../CMakeModules/AddZstdCompilationFlags.cmake    | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake b/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake
index 177db541c..05ad86439 100644
--- a/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake
+++ b/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake
@@ -2,20 +2,21 @@ include(CheckCXXCompilerFlag)
 include(CheckCCompilerFlag)
 
 function(EnableCompilerFlag _flag _C _CXX)
-    message("Checking flag ${_flag}")
+    string(REGEX REPLACE "\\+" "PLUS" varname "${_flag}")
+    string(REGEX REPLACE "[^A-Za-z0-9]+" "_" varname "${varname}")
+    string(REGEX REPLACE "^_+" "" varname "${varname}")
+    string(TOUPPER "${varname}" varname)
     if (_C)
-        CHECK_C_COMPILER_FLAG(${_flag} C_FLAG)
-        if (C_FLAG)
+        CHECK_C_COMPILER_FLAG(${_flag} C_FLAG_${varname})
+        if (C_FLAG_${varname})
             set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${_flag}" CACHE INTERNAL "C Flags")
         endif ()
-        unset(C_FLAG CACHE)
     endif ()
     if (_CXX)
-        CHECK_CXX_COMPILER_FLAG(${_flag} CXX_FLAG)
-        if (CXX_FLAG)
+        CHECK_CXX_COMPILER_FLAG(${_flag} CXX_FLAG_${varname})
+        if (CXX_FLAG_${varname})
             set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${_flag}" CACHE INTERNAL "CXX Flags")
         endif ()
-        unset(CXX_FLAG CACHE)
     endif ()
 endfunction()
 

From 9eda436733bf41eca2c31e77b3b601aeca215786 Mon Sep 17 00:00:00 2001
From: Michael Maltese 
Date: Thu, 20 Apr 2017 15:55:48 -0700
Subject: [PATCH 51/53] CMake: don't modify global C_FLAGS and CXX_FLAGS

---
 .../AddZstdCompilationFlags.cmake             | 21 ++++---------------
 1 file changed, 4 insertions(+), 17 deletions(-)

diff --git a/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake b/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake
index 05ad86439..e812418e3 100644
--- a/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake
+++ b/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake
@@ -9,13 +9,13 @@ function(EnableCompilerFlag _flag _C _CXX)
     if (_C)
         CHECK_C_COMPILER_FLAG(${_flag} C_FLAG_${varname})
         if (C_FLAG_${varname})
-            set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${_flag}" CACHE INTERNAL "C Flags")
+            set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${_flag}" PARENT_SCOPE)
         endif ()
     endif ()
     if (_CXX)
         CHECK_CXX_COMPILER_FLAG(${_flag} CXX_FLAG_${varname})
         if (CXX_FLAG_${varname})
-            set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${_flag}" CACHE INTERNAL "CXX Flags")
+            set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${_flag}" PARENT_SCOPE)
         endif ()
     endif ()
 endfunction()
@@ -69,8 +69,7 @@ MACRO(ADD_ZSTD_COMPILATION_FLAGS)
         separate_arguments(${flag_var})
         list(REMOVE_DUPLICATES ${flag_var})
         string(REPLACE ";" " " ${flag_var} "${${flag_var}}")
-        set(${flag_var} "${${flag_var}}" CACHE STRING "common build flags" FORCE)
-    ENDFOREACH (flag_var)  
+    ENDFOREACH (flag_var)
 
     if (MSVC)
         # Replace /MT to /MD flag
@@ -81,19 +80,7 @@ MACRO(ADD_ZSTD_COMPILATION_FLAGS)
                  CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO)
             STRING(REGEX REPLACE "/MT" "/MD" ${flag_var} "${${flag_var}}")
             STRING(REGEX REPLACE "/O2" "/Ox" ${flag_var} "${${flag_var}}")
-        ENDFOREACH (flag_var)      
+        ENDFOREACH (flag_var)
     endif ()
 
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}" CACHE STRING "Updated flags" FORCE)
-    set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG}" CACHE STRING "Updated flags" FORCE)
-    set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}" CACHE STRING "Updated flags" FORCE)
-    set(CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL}" CACHE STRING "Updated flags" FORCE)
-    set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}" CACHE STRING "Updated flags" FORCE)
-
-    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}" CACHE STRING "Updated flags" FORCE)
-    set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG}" CACHE STRING "Updated flags" FORCE)
-    set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE}" CACHE STRING "Updated flags" FORCE)
-    set(CMAKE_C_FLAGS_MINSIZEREL "${CMAKE_C_FLAGS_MINSIZEREL}" CACHE STRING "Updated flags" FORCE)
-    set(CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO}" CACHE STRING "Updated flags" FORCE)
-
 ENDMACRO(ADD_ZSTD_COMPILATION_FLAGS)

From d0b1846cf47d3989ce7fb3afb12734c695dddb4b Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 21 Apr 2017 10:59:36 -0700
Subject: [PATCH 52/53] ignore more cmake build artefacts

---
 build/cmake/lib/.gitignore      | 2 ++
 build/cmake/programs/.gitignore | 2 ++
 2 files changed, 4 insertions(+)
 create mode 100644 build/cmake/lib/.gitignore

diff --git a/build/cmake/lib/.gitignore b/build/cmake/lib/.gitignore
new file mode 100644
index 000000000..a4444c8d3
--- /dev/null
+++ b/build/cmake/lib/.gitignore
@@ -0,0 +1,2 @@
+# cmake build artefact
+libzstd.pc
diff --git a/build/cmake/programs/.gitignore b/build/cmake/programs/.gitignore
index f04c5b429..ae3a8a356 100644
--- a/build/cmake/programs/.gitignore
+++ b/build/cmake/programs/.gitignore
@@ -1,3 +1,5 @@
 # produced by make
 zstd
 zstd-frugal
+unzstd
+zstdcat

From 230d7acc7d3d0b08ba2b09acf1bda93bb5c9aa62 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 21 Apr 2017 11:38:13 -0700
Subject: [PATCH 53/53] cli : add support for --threads=# command

updated documentation
add relevant test case
---
 programs/zstd.1    | 23 +++++++++++++----------
 programs/zstd.1.md | 41 +++++++++++++++++++++--------------------
 programs/zstdcli.c |  1 +
 tests/playTests.sh |  1 +
 4 files changed, 36 insertions(+), 30 deletions(-)

diff --git a/programs/zstd.1 b/programs/zstd.1
index b84ab3324..999dc8169 100644
--- a/programs/zstd.1
+++ b/programs/zstd.1
@@ -5,7 +5,7 @@
 \fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files
 .
 .SH "SYNOPSIS"
-\fBzstd\fR [\fIOPTIONS\fR] [\-|] [\-o ]
+\fBzstd\fR [\fIOPTIONS\fR] [\-|\fIINPUT\-FILE\fR] [\-o \fIOUTPUT\-FILE\fR]
 .
 .P
 \fBzstdmt\fR is equivalent to \fBzstd \-T0\fR
@@ -100,8 +100,8 @@ Use FILEs as a training set to create a dictionary\. The training set should con
 unlocks high compression levels 20+ (maximum 22), using a lot more memory\. Note that decompression will also require more memory when using these levels\.
 .
 .TP
-\fB\-T#\fR
-Compress using # threads (default: 1)\. If \fB#\fR is 0, attempt to detect the number of physical CPU cores and compress with that many threads\. This modified does nothing if \fBzstd\fR was compiled without multithread support\.
+\fB\-T#\fR, \fB\-\-threads=#\fR
+Compress using \fB#\fR threads (default: 1)\. If \fB#\fR is 0, attempt to detect and use the number of physical CPU cores\. This modifier does nothing if \fBzstd\fR is compiled without multithread support\.
 .
 .TP
 \fB\-D file\fR
@@ -113,7 +113,7 @@ do not store dictionary ID within frame header (dictionary compression)\. The de
 .
 .TP
 \fB\-o file\fR
-save result into \fBfile\fR (only possible with a single INPUT\-FILE)
+save result into \fBfile\fR (only possible with a single \fIINPUT\-FILE\fR)
 .
 .TP
 \fB\-f\fR, \fB\-\-force\fR
@@ -172,7 +172,7 @@ use FILEs as training set to create a dictionary\. The training set should conta
 .
 .TP
 \fB\-o file\fR
-dictionary saved into \fBfile\fR (default: dictionary)
+dictionary saved into \fBfile\fR (default name: dictionary)
 .
 .TP
 \fB\-\-maxdict=#\fR
@@ -198,15 +198,18 @@ Example: \fB\-\-train \-\-cover=k=64,d=8 FILEs\fR\.
 If \fIsteps\fR is not specified, the default value of 32 is used\. If \fIk\fR is not specified, the \fIk\fR values in [16, 2048] are checked for each value of \fId\fR\. If \fId\fR is not specified, the values checked are [6, 8, \.\.\., 16]\.
 .
 .IP
-Runs the cover dictionary builder for each parameter set and saves the optimal parameters and dictionary\. Prints the optimal parameters and writes the optimal dictionary to the output file\. Supports multithreading if \fBzstd\fR is compiled with threading support\.
+Runs the cover dictionary builder for each parameter set and saves the optimal parameters and dictionary\. Prints optimal parameters and writes optimal dictionary into output file\. Supports multithreading if \fBzstd\fR is compiled with threading support\.
 .
 .IP
 The parameter \fIk\fR is more sensitive than \fId\fR, and is faster to optimize over\. Suggested use is to run with a \fIsteps\fR <= 32 with neither \fIk\fR nor \fId\fR set\. Once it completes, use the value of \fId\fR it selects with a higher \fIsteps\fR (in the range [256, 1024])\.
 .
 .IP
+Examples :
+.
+.IP
 \fBzstd \-\-train \-\-optimize\-cover FILEs\fR
 .
-.br
+.IP
 \fBzstd \-\-train \-\-optimize\-cover=d=d,steps=512 FILEs\fR
 .
 .SH "BENCHMARK"
@@ -233,9 +236,6 @@ set process priority to real\-time
 .
 .SH "ADVANCED COMPRESSION OPTIONS"
 .
-.SS "\-B#:"
-Select the size of each compression job\. This parameter is available only when multi\-threading is enabled\. Default value is \fB4 * windowSize\fR, which means it varies depending on compression level\. \fB\-B#\fR makes it possible to select a custom value\. Note that job size must respect a minimum value which is enforced transparently\. This minimum is either 1 MB, or \fBoverlapSize\fR, whichever is largest\.
-.
 .SS "\-\-zstd[=options]:"
 \fBzstd\fR provides 22 predefined compression levels\. The selected or default predefined compression level can be changed with advanced compression options\. The \fIoptions\fR are provided as a comma\-separated list\. You may specify only the options you want to change and the rest will be taken from the selected or default compression level\. The list of available \fIoptions\fR:
 .
@@ -310,6 +310,9 @@ Determine \fBoverlapSize\fR, amount of data reloaded from previous job\. This pa
 .IP
 The minimum \fIovlog\fR is 0, and the maximum is 9\. 0 means "no overlap", hence completely independent jobs\. 9 means "full overlap", meaning up to \fBwindowSize\fR is reloaded from previous job\. Reducing \fIovlog\fR by 1 reduces the amount of reload by a factor 2\. Default \fIovlog\fR is 6, which means "reload \fBwindowSize / 8\fR"\. Exception : the maximum compression level (22) has a default \fIovlog\fR of 9\.
 .
+.SS "\-B#:"
+Select the size of each compression job\. This parameter is available only when multi\-threading is enabled\. Default value is \fB4 * windowSize\fR, which means it varies depending on compression level\. \fB\-B#\fR makes it possible to select a custom value\. Note that job size must respect a minimum value which is enforced transparently\. This minimum is either 1 MB, or \fBoverlapSize\fR, whichever is largest\.
+.
 .SS "Example"
 The following parameters sets advanced compression options to those of predefined level 19 for files bigger than 256 KB:
 .
diff --git a/programs/zstd.1.md b/programs/zstd.1.md
index 4b918d8fd..f2d04d16f 100644
--- a/programs/zstd.1.md
+++ b/programs/zstd.1.md
@@ -4,7 +4,7 @@ zstd(1) -- zstd, zstdmt, unzstd, zstdcat - Compress or decompress .zst files
 SYNOPSIS
 --------
 
-`zstd` [*OPTIONS*] [-|<INPUT-FILE>] [-o <OUTPUT-FILE>]
+`zstd` [*OPTIONS*] [-|_INPUT-FILE_] [-o _OUTPUT-FILE_]
 
 `zstdmt` is equivalent to `zstd -T0`
 
@@ -101,11 +101,10 @@ the last one takes effect.
 * `--ultra`:
     unlocks high compression levels 20+ (maximum 22), using a lot more memory.
     Note that decompression will also require more memory when using these levels.
-* `-T#`:
-    Compress using # threads (default: 1).
-    If `#` is 0, attempt to detect the number of physical CPU cores and compress with
-    that many threads.
-    This modified does nothing if `zstd` was compiled without multithread support.
+* `-T#`, `--threads=#`:
+    Compress using `#` threads (default: 1).
+    If `#` is 0, attempt to detect and use the number of physical CPU cores.
+    This modifier does nothing if `zstd` is compiled without multithread support.
 * `-D file`:
     use `file` as Dictionary to compress or decompress FILE(s)
 * `--nodictID`:
@@ -113,7 +112,7 @@ the last one takes effect.
     The decoder will have to rely on implicit knowledge about which dictionary to use,
     it won't be able to check if it's correct.
 * `-o file`:
-    save result into `file` (only possible with a single INPUT-FILE)
+    save result into `file` (only possible with a single _INPUT-FILE_)
 * `-f`, `--force`:
     overwrite output without prompting, and (de)compress symbolic links
 * `-c`, `--stdout`:
@@ -164,7 +163,7 @@ Typical gains range from 10% (at 64KB) to x5 better (at <1KB).
     and weight typically 100x the target dictionary size
     (for example, 10 MB for a 100 KB dictionary).
 * `-o file`:
-    dictionary saved into `file` (default: dictionary)
+    dictionary saved into `file` (default name: dictionary)
 * `--maxdict=#`:
     limit dictionary to specified size (default : (112640)
 * `--dictID=#`:
@@ -198,9 +197,9 @@ Typical gains range from 10% (at 64KB) to x5 better (at <1KB).
     value of _d_.
     If _d_ is not specified, the values checked are [6, 8, ..., 16].
 
-    Runs the cover dictionary builder for each parameter set and saves the
-    optimal parameters and dictionary.
-    Prints the optimal parameters and writes the optimal dictionary to the output file.
+    Runs the cover dictionary builder for each parameter set
+    and saves the optimal parameters and dictionary.
+    Prints optimal parameters and writes optimal dictionary into output file.
     Supports multithreading if `zstd` is compiled with threading support.
 
     The parameter _k_ is more sensitive than _d_, and is faster to optimize over.
@@ -208,7 +207,10 @@ Typical gains range from 10% (at 64KB) to x5 better (at <1KB).
     Once it completes, use the value of _d_ it selects with a higher _steps_
     (in the range [256, 1024]).
 
-    `zstd --train --optimize-cover FILEs` 

+    Examples :
+
+    `zstd --train --optimize-cover FILEs`
+
     `zstd --train --optimize-cover=d=d,steps=512 FILEs`
 
 
@@ -229,14 +231,6 @@ BENCHMARK
 
 ADVANCED COMPRESSION OPTIONS
 ----------------------------
-### -B#:
-Select the size of each compression job.
-This parameter is available only when multi-threading is enabled.
-Default value is `4 * windowSize`, which means it varies depending on compression level.
-`-B#` makes it possible to select a custom value.
-Note that job size must respect a minimum value which is enforced transparently.
-This minimum is either 1 MB, or `overlapSize`, whichever is largest.
-
 ### --zstd[=options]:
 `zstd` provides 22 predefined compression levels.
 The selected or default predefined compression level can be changed with
@@ -319,6 +313,13 @@ The list of available _options_:
     Default _ovlog_ is 6, which means "reload `windowSize / 8`".
     Exception : the maximum compression level (22) has a default _ovlog_ of 9.
 
+### -B#:
+Select the size of each compression job.
+This parameter is available only when multi-threading is enabled.
+Default value is `4 * windowSize`, which means it varies depending on compression level.
+`-B#` makes it possible to select a custom value.
+Note that job size must respect a minimum value which is enforced transparently.
+This minimum is either 1 MB, or `overlapSize`, whichever is largest.
 
 ### Example
 The following parameters sets advanced compression options to those of
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 6cbe7383f..76f55de53 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -420,6 +420,7 @@ int main(int argCount, const char* argv[])
                       continue;
                     }
 #endif
+                    if (longCommandWArg(&argument, "--threads=")) { nbThreads = readU32FromChar(&argument); continue; }
                     if (longCommandWArg(&argument, "--memlimit=")) { memLimit = readU32FromChar(&argument); continue; }
                     if (longCommandWArg(&argument, "--memory=")) { memLimit = readU32FromChar(&argument); continue; }
                     if (longCommandWArg(&argument, "--memlimit-decompress=")) { memLimit = readU32FromChar(&argument); continue; }
diff --git a/tests/playTests.sh b/tests/playTests.sh
index deeebee2e..a91e9e8bb 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -471,6 +471,7 @@ then
     $ECHO "\n**** zstdmt round-trip tests **** "
     roundTripTest -g4M "1 -T0"
     roundTripTest -g8M "3 -T2"
+    roundTripTest -g8000K "2 --threads=2"
     fileRoundTripTest -g4M "19 -T2 -B1M"
 else
     $ECHO "\n**** no multithreading, skipping zstdmt tests **** "