From 5397a66b19400255623494acd2caa7847b775700 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 13 Dec 2016 15:21:06 +0100
Subject: [PATCH 01/14] minor BMI version check

---
 lib/common/bitstream.h | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/common/bitstream.h b/lib/common/bitstream.h
index e96798fe4..3a45244f8 100644
--- a/lib/common/bitstream.h
+++ b/lib/common/bitstream.h
@@ -266,7 +266,7 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
         bitD->ptr   = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer);
         bitD->bitContainer = MEM_readLEST(bitD->ptr);
         { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
-          bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0;
+          bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0;  /* ensures bitsConsumed is always set */
           if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ }
     } else {
         bitD->start = (const char*)srcBuffer;
@@ -298,7 +298,7 @@ MEM_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start)
 
 MEM_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits)
 {
-#if defined(__BMI__) && defined(__GNUC__)   /* experimental */
+#if defined(__BMI__) && defined(__GNUC__) && __GNUC__*1000+__GNUC_MINOR__ >= 4008  /* experimental */
 #  if defined(__x86_64__)
     if (sizeof(bitContainer)==8)
         return _bextr_u64(bitContainer, start, nbBits);
@@ -367,10 +367,10 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
 }
 
 /*! BIT_reloadDStream() :
-*   Refill `BIT_DStream_t` from src buffer previously defined (see BIT_initDStream() ).
+*   Refill `bitD` from buffer previously set in BIT_initDStream() .
 *   This function is safe, it guarantees it will not read beyond src buffer.
 *   @return : status of `BIT_DStream_t` internal register.
-              if status == unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */
+              if status == BIT_DStream_unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */
 MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
 {
 	if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8))  /* should not happen => corruption detected */

From e795c8a5f6cf07e913bf0766e44e4b8a5fad9ef0 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 13 Dec 2016 16:39:36 +0100
Subject: [PATCH 02/14] Added ZSTD_initCStream_srcSize(). Added relevant test
 cases in zstreamtest

---
 lib/compress/zstd_compress.c | 14 +++++++++++
 lib/deprecated/zbuff.h       |  4 +---
 lib/zstd.h                   | 19 +++++++--------
 tests/Makefile               |  4 ++--
 tests/zbufftest.c            | 10 ++++----
 tests/zstreamtest.c          | 45 +++++++++++++++++++++++++++---------
 6 files changed, 66 insertions(+), 30 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 6e52ec8ec..3d10fbd96 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2852,6 +2852,8 @@ struct ZSTD_CStream_s {
     ZSTD_cStreamStage stage;
     U32    checksum;
     U32    frameEnded;
+    U64    pledgedSrcSize;
+    U64    inputProcessed;
     ZSTD_parameters params;
     ZSTD_customMem customMem;
 };   /* typedef'd to ZSTD_CStream within "zstd.h" */
@@ -2909,6 +2911,8 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
     zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0;
     zcs->stage = zcss_load;
     zcs->frameEnded = 0;
+    zcs->pledgedSrcSize = pledgedSrcSize;
+    zcs->inputProcessed = 0;
     return 0;   /* ready to go */
 }
 
@@ -2961,6 +2965,12 @@ size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t di
     return ZSTD_initCStream_advanced(zcs, dict, dictSize, params, 0);
 }
 
+size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize)
+{
+    ZSTD_parameters const params = ZSTD_getParams(compressionLevel, pledgedSrcSize, 0);
+    return ZSTD_initCStream_advanced(zcs, NULL, 0, params, pledgedSrcSize);
+}
+
 size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel)
 {
     return ZSTD_initCStream_usingDict(zcs, NULL, 0, compressionLevel);
@@ -3057,6 +3067,7 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
 
     *srcSizePtr = ip - istart;
     *dstCapacityPtr = op - ostart;
+    zcs->inputProcessed += *srcSizePtr;
     if (zcs->frameEnded) return 0;
     {   size_t hintInSize = zcs->inBuffTarget - zcs->inBuffPos;
         if (hintInSize==0) hintInSize = zcs->blockSize;
@@ -3101,6 +3112,9 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
     BYTE* const oend = (BYTE*)(output->dst) + output->size;
     BYTE* op = ostart;
 
+    if ((zcs->pledgedSrcSize) && (zcs->inputProcessed != zcs->pledgedSrcSize))
+        return ERROR(srcSize_wrong);   /* pledgedSrcSize not respected */
+
     if (zcs->stage != zcss_final) {
         /* flush whatever remains */
         size_t srcSize = 0;
diff --git a/lib/deprecated/zbuff.h b/lib/deprecated/zbuff.h
index 8296dc64a..85f973557 100644
--- a/lib/deprecated/zbuff.h
+++ b/lib/deprecated/zbuff.h
@@ -27,7 +27,7 @@ extern "C" {
 *  Dependencies
 ***************************************/
 #include <stddef.h>      /* size_t */
-#include <zstd.h>        /* ZSTD_CStream, ZSTD_DStream, ZSTDLIB_API */
+#include "zstd.h"        /* ZSTD_CStream, ZSTD_DStream, ZSTDLIB_API */
 
 
 /* ***************************************************************
@@ -170,7 +170,6 @@ ZBUFF_DEPRECATED("use ZSTD_DStreamOutSize") size_t ZBUFF_recommendedDOutSize(voi
 
 
 #ifdef ZBUFF_STATIC_LINKING_ONLY
-
 #ifndef ZBUFF_STATIC_H_30298098432
 #define ZBUFF_STATIC_H_30298098432
 
@@ -203,7 +202,6 @@ ZBUFF_DEPRECATED("use ZSTD_initDStream_usingDict") size_t ZBUFF_compressInit_adv
 
 
 #endif    /* ZBUFF_STATIC_H_30298098432 */
-
 #endif    /* ZBUFF_STATIC_LINKING_ONLY */
 
 
diff --git a/lib/zstd.h b/lib/zstd.h
index 44aa99566..a4766e335 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -237,20 +237,20 @@ typedef struct ZSTD_outBuffer_s {
 *
 *  Start a new compression by initializing ZSTD_CStream.
 *  Use ZSTD_initCStream() to start a new compression operation.
-*  Use ZSTD_initCStream_usingDict() for a compression which requires a dictionary.
+*  Use ZSTD_initCStream_usingDict() or ZSTD_initCStream_usingCDict() for a compression which requires a dictionary (experimental section)
 *
 *  Use ZSTD_compressStream() repetitively to consume input stream.
 *  The function will automatically update both `pos` fields.
 *  Note that it may not consume the entire input, in which case `pos < size`,
 *  and it's up to the caller to present again remaining data.
 *  @return : a size hint, preferred nb of bytes to use as input for next function call
-*           (it's just a hint, to help latency a little, any other value will work fine)
-*           (note : the size hint is guaranteed to be <= ZSTD_CStreamInSize() )
 *            or an error code, which can be tested using ZSTD_isError().
+*            Note 1 : it's just a hint, to help latency a little, any other value will work fine.
+*            Note 2 : size hint is guaranteed to be <= ZSTD_CStreamInSize()
 *
-*  At any moment, it's possible to flush whatever data remains within buffer, using ZSTD_flushStream().
+*  At any moment, it's possible to flush whatever data remains within internal buffer, using ZSTD_flushStream().
 *  `output->pos` will be updated.
-*  Note some content might still be left within internal buffer if `output->size` is too small.
+*  Note that some content might still be left within internal buffer if `output->size` is too small.
 *  @return : nb of bytes still present within internal buffer (0 if it's empty)
 *            or an error code, which can be tested using ZSTD_isError().
 *
@@ -259,15 +259,15 @@ typedef struct ZSTD_outBuffer_s {
 *  The epilogue is required for decoders to consider a frame completed.
 *  Similar to ZSTD_flushStream(), it may not be able to flush the full content if `output->size` is too small.
 *  In which case, call again ZSTD_endStream() to complete the flush.
-*  @return : nb of bytes still present within internal buffer (0 if it's empty)
+*  @return : nb of bytes still present within internal buffer (0 if it's empty, hence compression completed)
 *            or an error code, which can be tested using ZSTD_isError().
 *
 * *******************************************************************/
 
-/*=====   Streaming compression functions   ======*/
 typedef struct ZSTD_CStream_s ZSTD_CStream;
 ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void);
 ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
+
 ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
 ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
 ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
@@ -300,10 +300,10 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void);   /**< recommended size for output
 *            The return value is a suggested next input size (a hint to improve latency) that will never load more than the current frame.
 * *******************************************************************************/
 
-/*=====   Streaming decompression functions   =====*/
 typedef struct ZSTD_DStream_s ZSTD_DStream;
 ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void);
 ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);
+
 ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
 ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
 
@@ -490,6 +490,7 @@ unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
 
 /*=====   Advanced Streaming compression functions  =====*/
 ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
+ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);   /**< pledgedSrcSize must be correct */
 ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
 ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
                                              ZSTD_parameters params, unsigned long long pledgedSrcSize);  /**< pledgedSrcSize is optional and can be zero == unknown */
@@ -602,7 +603,7 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
   Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType().
   This information is not required to properly decode a frame.
 
-  == Special case : skippable frames == 
+  == Special case : skippable frames ==
 
   Skippable frames allow integration of user-defined data into a flow of concatenated frames.
   Skippable frames will be ignored (skipped) by a decompressor. The format of skippable frames is as follows :
diff --git a/tests/Makefile b/tests/Makefile
index 6110465f6..d7d25026a 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -100,9 +100,9 @@ fuzzer32 : $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c fuzzer.c
 	$(CC)  -m32  $(FLAGS) $^ -o $@$(EXT)
 
 zbufftest : CPPFLAGS += -I$(ZSTDDIR)/deprecated
-zbufftest : CFLAGS += -Wno-deprecated-declarations
+zbufftest : CFLAGS += -Wno-deprecated-declarations   # required to silence deprecation warnings
 zbufftest : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c zbufftest.c
-	$(CC) $(FLAGS) $^ -o $@$(EXT)   # flag required to silence deprecation warnings
+	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
 zbufftest32 : CPPFLAGS += -I$(ZSTDDIR)/deprecated
 zbufftest32 : CFLAGS += -Wno-deprecated-declarations -m32
diff --git a/tests/zbufftest.c b/tests/zbufftest.c
index 9dc164eaf..14b739233 100644
--- a/tests/zbufftest.c
+++ b/tests/zbufftest.c
@@ -28,8 +28,8 @@
 #include "mem.h"
 #define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_maxCLevel */
 #include "zstd.h"         /* ZSTD_compressBound */
-#define ZBUFF_STATIC_LINKING_ONLY
-#include "zbuff.h"        /* ZBUFF_createCCtx_advanced */
+#define ZBUFF_STATIC_LINKING_ONLY  /* ZBUFF_createCCtx_advanced */
+#include "zbuff.h"        /* ZBUFF_isError */
 #include "datagen.h"      /* RDG_genBuffer */
 #define XXH_STATIC_LINKING_ONLY
 #include "xxhash.h"       /* XXH64_* */
@@ -265,11 +265,11 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
     static const U32 maxSrcLog = 24;
     static const U32 maxSampleLog = 19;
     BYTE* cNoiseBuffer[5];
-    size_t srcBufferSize = (size_t)1<<maxSrcLog;
+    size_t const srcBufferSize = (size_t)1<<maxSrcLog;
     BYTE* copyBuffer;
-    size_t copyBufferSize= srcBufferSize + (1<<maxSampleLog);
+    size_t const copyBufferSize= srcBufferSize + (1<<maxSampleLog);
     BYTE* cBuffer;
-    size_t cBufferSize   = ZSTD_compressBound(srcBufferSize);
+    size_t const cBufferSize   = ZSTD_compressBound(srcBufferSize);
     BYTE* dstBuffer;
     size_t dstBufferSize = srcBufferSize;
     U32 result = 0;
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index c21b9de9b..7783fe11b 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -28,6 +28,7 @@
 #include "mem.h"
 #define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_maxCLevel, ZSTD_customMem */
 #include "zstd.h"         /* ZSTD_compressBound */
+#include "zstd_errors.h"  /* ZSTD_error_srcSize_wrong */
 #include "datagen.h"      /* RDG_genBuffer */
 #define XXH_STATIC_LINKING_ONLY   /* XXH64_state_t */
 #include "xxhash.h"       /* XXH64_* */
@@ -88,16 +89,6 @@ unsigned int FUZ_rand(unsigned int* seedPtr)
     return rand32 >> 5;
 }
 
-/*
-static unsigned FUZ_highbit32(U32 v32)
-{
-    unsigned nbBits = 0;
-    if (v32==0) return 0;
-    for ( ; v32 ; v32>>=1) nbBits++;
-    return nbBits;
-}
-*/
-
 static void* allocFunction(void* opaque, size_t size)
 {
     void* address = malloc(size);
@@ -254,6 +245,38 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     }   }
     DISPLAYLEVEL(4, "OK \n");
 
+    /* _srcSize compression test */
+    DISPLAYLEVEL(4, "test%3i : compress_srcSize %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
+    ZSTD_initCStream_srcSize(zc, 1, CNBufferSize);
+    outBuff.dst = (char*)(compressedBuffer)+cSize;
+    outBuff.size = compressedBufferSize;
+    outBuff.pos = 0;
+    inBuff.src = CNBuffer;
+    inBuff.size = CNBufferSize;
+    inBuff.pos = 0;
+    { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
+      if (ZSTD_isError(r)) goto _output_error; }
+    if (inBuff.pos != inBuff.size) goto _output_error;   /* entire input should be consumed */
+    { size_t const r = ZSTD_endStream(zc, &outBuff);
+      if (r != 0) goto _output_error; }  /* error, or some data not flushed */
+    DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
+
+    /* wrong _srcSize compression test */
+    DISPLAYLEVEL(4, "test%3i : wrong srcSize : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1);
+    ZSTD_initCStream_srcSize(zc, 1, CNBufferSize-1);
+    outBuff.dst = (char*)(compressedBuffer)+cSize;
+    outBuff.size = compressedBufferSize;
+    outBuff.pos = 0;
+    inBuff.src = CNBuffer;
+    inBuff.size = CNBufferSize;
+    inBuff.pos = 0;
+    { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
+      if (ZSTD_isError(r)) goto _output_error; }
+    if (inBuff.pos != inBuff.size) goto _output_error;   /* entire input should be consumed */
+    { size_t const r = ZSTD_endStream(zc, &outBuff);
+      if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error;    /* must fail : wrong srcSize */
+      DISPLAYLEVEL(4, "OK (error detected : %s) \n", ZSTD_getErrorName(r)); }
+
     /* Complex context re-use scenario */
     DISPLAYLEVEL(4, "test%3i : context re-use : ", testNb++);
     ZSTD_freeCStream(zc);
@@ -519,7 +542,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
         {   ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ;
             U32 n;
             for (n=0, cSize=0, totalTestSize=0 ; totalTestSize < maxTestSize ; n++) {
-                /* compress random chunk into random size dst buffer */
+                /* compress random chunks into randomly sized dst buffers */
                 {   size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
                     size_t const srcSize = MIN (maxTestSize-totalTestSize, randomSrcSize);
                     size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize);

From 2b36b238d399078bbdbe31a721ef23b2668fc094 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 13 Dec 2016 17:59:55 +0100
Subject: [PATCH 03/14] changed variable name to estimatedSrcSize, to emphasize
 it does not need to be exact

---
 lib/zstd.h | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/lib/zstd.h b/lib/zstd.h
index a4766e335..b02e16b42 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -406,15 +406,15 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictS
  *  Gives the amount of memory used by a given ZSTD_sizeof_CDict */
 ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
 
-/*! ZSTD_getParams() :
-*   same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of a `ZSTD_compressionParameters`.
-*   All fields of `ZSTD_frameParameters` are set to default (0) */
-ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSize, size_t dictSize);
-
 /*! ZSTD_getCParams() :
-*   @return ZSTD_compressionParameters structure for a selected compression level and srcSize.
-*   `srcSize` value is optional, select 0 if not known */
-ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSize, size_t dictSize);
+*   @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
+*   `estimatedSrcSize` value is optional, select 0 if not known */
+ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
+
+/*! ZSTD_getParams() :
+*   same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`.
+*   All fields of `ZSTD_frameParameters` are set to default (0) */
+ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
 
 /*! ZSTD_checkCParams() :
 *   Ensure param values remain within authorized range */

From eee427ee250d7a2bbb02c54d98c92c06f70d0b2d Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski <inikep@gmail.com>
Date: Tue, 13 Dec 2016 19:14:04 +0100
Subject: [PATCH 04/14] fixed fitblk

---
 zlibWrapper/examples/fitblk.c  | 8 +++-----
 zlibWrapper/zstd_zlibwrapper.c | 4 ++--
 2 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/zlibWrapper/examples/fitblk.c b/zlibWrapper/examples/fitblk.c
index 760e338a2..f389c3a4f 100644
--- a/zlibWrapper/examples/fitblk.c
+++ b/zlibWrapper/examples/fitblk.c
@@ -90,7 +90,7 @@ local int partcompress(FILE *in, z_streamp def)
             flush = Z_FINISH;
         LOG_FITBLK("partcompress1 avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out);
         ret = deflate(def, flush);
-        LOG_FITBLK("partcompress2 avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out);
+        LOG_FITBLK("partcompress2 ret=%d avail_in=%d total_in=%d avail_out=%d total_out=%d\n", ret, (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out);
         assert(ret != Z_STREAM_ERROR);
     } while (def->avail_out != 0 && flush == Z_SYNC_FLUSH);
     return ret;
@@ -106,6 +106,7 @@ local int recompress(z_streamp inf, z_streamp def)
     unsigned char raw[RAWLEN];
 
     flush = Z_NO_FLUSH;
+    LOG_FITBLK("recompress start\n");
     do {
         /* decompress */
         inf->avail_out = RAWLEN;
@@ -125,7 +126,7 @@ local int recompress(z_streamp inf, z_streamp def)
             flush = Z_FINISH;
         LOG_FITBLK("recompress1deflate avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out);
         ret = deflate(def, flush);
-        LOG_FITBLK("recompress2deflate avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out);
+        LOG_FITBLK("recompress2deflate ret=%d avail_in=%d total_in=%d avail_out=%d total_out=%d\n", ret, (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out);
         assert(ret != Z_STREAM_ERROR);
     } while (ret != Z_STREAM_END && def->avail_out != 0);
     return ret;
@@ -165,9 +166,6 @@ int main(int argc, char **argv)
     ret = deflateInit(&def, Z_DEFAULT_COMPRESSION);
     if (ret != Z_OK || blk == NULL)
         quit("out of memory");
-    ret = ZWRAP_setPledgedSrcSize(&def, 1<<16);
-    if (ret != Z_OK)
-        quit("ZWRAP_setPledgedSrcSize");
 
     /* compress from stdin until output full, or no more input */
     def.avail_out = size + EXCESS;
diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c
index 3b23358f3..d26663142 100644
--- a/zlibWrapper/zstd_zlibwrapper.c
+++ b/zlibWrapper/zstd_zlibwrapper.c
@@ -301,7 +301,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush))
         }
     }
 
-    LOG_WRAPPERC("- deflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
+    LOG_WRAPPERC("- deflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
     if (strm->avail_in > 0) {
         zwc->inBuffer.src = strm->next_in;
         zwc->inBuffer.size = strm->avail_in;
@@ -355,7 +355,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush))
         strm->total_out += zwc->outBuffer.pos;
         strm->avail_out -= zwc->outBuffer.pos;
     }
-    LOG_WRAPPERC("- deflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
+    LOG_WRAPPERC("- deflate3 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
     return Z_OK;
 }
 

From 622d741a67fe9f41b9a40bff14a2eb45e3fdf35a Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski <inikep@gmail.com>
Date: Tue, 13 Dec 2016 19:44:07 +0100
Subject: [PATCH 05/14] updated zlib copyright notice

---
 zlibWrapper/examples/minigzip.c | 2 +-
 zlibWrapper/gzclose.c           | 2 +-
 zlibWrapper/gzguts.h            | 2 +-
 zlibWrapper/gzlib.c             | 2 +-
 zlibWrapper/gzread.c            | 2 +-
 zlibWrapper/gzwrite.c           | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/zlibWrapper/examples/minigzip.c b/zlibWrapper/examples/minigzip.c
index 0ddc93f25..521d04711 100644
--- a/zlibWrapper/examples/minigzip.c
+++ b/zlibWrapper/examples/minigzip.c
@@ -3,7 +3,7 @@
 
 /* minigzip.c -- simulate gzip using the zlib compression library
  * Copyright (C) 1995-2006, 2010, 2011 Jean-loup Gailly.
- * For conditions of distribution and use, see copyright notice in zlib.h
+ * For conditions of distribution and use, see http://www.zlib.net/zlib_license.html
  */
 
 /*
diff --git a/zlibWrapper/gzclose.c b/zlibWrapper/gzclose.c
index b5b7480c3..d4493d010 100644
--- a/zlibWrapper/gzclose.c
+++ b/zlibWrapper/gzclose.c
@@ -3,7 +3,7 @@
 
 /* gzclose.c -- zlib gzclose() function
  * Copyright (C) 2004, 2010 Mark Adler
- * For conditions of distribution and use, see copyright notice in zlib.h
+ * For conditions of distribution and use, see http://www.zlib.net/zlib_license.html
  */
 
 #include "gzguts.h"
diff --git a/zlibWrapper/gzguts.h b/zlibWrapper/gzguts.h
index e2538df15..40536de4f 100644
--- a/zlibWrapper/gzguts.h
+++ b/zlibWrapper/gzguts.h
@@ -4,7 +4,7 @@
 
 /* gzguts.h -- zlib internal header definitions for gz* operations
  * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013 Mark Adler
- * For conditions of distribution and use, see copyright notice in zlib.h
+ * For conditions of distribution and use, see http://www.zlib.net/zlib_license.html
  */
 
 #ifdef _LARGEFILE64_SOURCE
diff --git a/zlibWrapper/gzlib.c b/zlibWrapper/gzlib.c
index 172041df5..932319af6 100644
--- a/zlibWrapper/gzlib.c
+++ b/zlibWrapper/gzlib.c
@@ -3,7 +3,7 @@
 
 /* gzlib.c -- zlib functions common to reading and writing gzip files
  * Copyright (C) 2004, 2010, 2011, 2012, 2013 Mark Adler
- * For conditions of distribution and use, see copyright notice in zlib.h
+ * For conditions of distribution and use, see http://www.zlib.net/zlib_license.html
  */
 
 #include "gzguts.h"
diff --git a/zlibWrapper/gzread.c b/zlibWrapper/gzread.c
index 51ffef3e6..bf1a3cc6c 100644
--- a/zlibWrapper/gzread.c
+++ b/zlibWrapper/gzread.c
@@ -3,7 +3,7 @@
 
 /* gzread.c -- zlib functions for reading gzip files
  * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013 Mark Adler
- * For conditions of distribution and use, see copyright notice in zlib.h
+ * For conditions of distribution and use, see http://www.zlib.net/zlib_license.html
  */
 
 #include "gzguts.h"
diff --git a/zlibWrapper/gzwrite.c b/zlibWrapper/gzwrite.c
index d1478d3c1..13c8fb264 100644
--- a/zlibWrapper/gzwrite.c
+++ b/zlibWrapper/gzwrite.c
@@ -3,7 +3,7 @@
 
 /* gzwrite.c -- zlib functions for writing gzip files
  * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013 Mark Adler
- * For conditions of distribution and use, see copyright notice in zlib.h
+ * For conditions of distribution and use, see http://www.zlib.net/zlib_license.html
  */
 
 #include "gzguts.h"

From b23420fa48a9b53eedc82d3ff2e8c5651e2f91b4 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 13 Dec 2016 19:47:17 +0100
Subject: [PATCH 06/14] updated NEWS

---
 NEWS | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/NEWS b/NEWS
index 35e0e5820..9b0001636 100644
--- a/NEWS
+++ b/NEWS
@@ -1,6 +1,6 @@
 v1.1.2
-API : streaming : decompression : changed : implicit reset on starting new frames without init
-API : experimental : added : dictID retrieval functions
+API : streaming : decompression : changed : automatic implicit reset when chain-decoding new frames without init
+API : experimental : added : dictID retrieval functions, and ZSTD_initCStream_srcSize()
 API : zbuff : changed : prototypes now generate deprecation warnings
 lib : improved : faster decompression speed at ultra compression settings and 32-bits mode
 lib : changed : only public ZSTD_ symbols are now exposed

From dc9931205a87fdf6b993b3e215139011d3ef9fee Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 14 Dec 2016 14:53:47 +0100
Subject: [PATCH 07/14] updated manual

---
 doc/zstd_manual.html | 81 ++++++++++++++++----------------------------
 1 file changed, 30 insertions(+), 51 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index 81901ce9f..1badcbd79 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -88,29 +88,21 @@
     note 5 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more. 
 </p></pre><BR>
 
-<h3>Helper functions</h3><pre></pre><b><pre>int         ZSTD_maxCLevel(void);               </b>/*!< maximum compression level available */<b>
+<h3>Helper functions</h3><pre><b>int         ZSTD_maxCLevel(void);               </b>/*!< maximum compression level available */<b>
 size_t      ZSTD_compressBound(size_t srcSize); </b>/*!< maximum compressed size in worst case scenario */<b>
 unsigned    ZSTD_isError(size_t code);          </b>/*!< tells if a `size_t` function result is an error code */<b>
 const char* ZSTD_getErrorName(size_t code);     </b>/*!< provides readable string from an error code */<b>
-</pre></b><BR>
+</b></pre><BR>
 <a name="Chapter4"></a><h2>Explicit memory management</h2><pre></pre>
 
-<h3>Compression context</h3><pre>   When compressing many times,
-   it is recommended to allocate a context just once, and re-use it for each successive compression operation.
-   This will make workload friendlier for system's memory.
-   Use one context per thread for parallel execution in multi-threaded environments. 
-</pre><b><pre>typedef struct ZSTD_CCtx_s ZSTD_CCtx;
-ZSTD_CCtx* ZSTD_createCCtx(void);
-size_t     ZSTD_freeCCtx(ZSTD_CCtx* cctx);
-</pre></b><BR>
 <pre><b>size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel);
 </b><p>    Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()). 
 </p></pre><BR>
 
-<h3>Decompression context</h3><pre></pre><b><pre>typedef struct ZSTD_DCtx_s ZSTD_DCtx;
+<h3>Decompression context</h3><pre><b>typedef struct ZSTD_DCtx_s ZSTD_DCtx;
 ZSTD_DCtx* ZSTD_createDCtx(void);
 size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
-</pre></b><BR>
+</b></pre><BR>
 <pre><b>size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 </b><p>   Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()). 
 </p></pre><BR>
@@ -200,20 +192,20 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
 
   Start a new compression by initializing ZSTD_CStream.
   Use ZSTD_initCStream() to start a new compression operation.
-  Use ZSTD_initCStream_usingDict() for a compression which requires a dictionary.
+  Use ZSTD_initCStream_usingDict() or ZSTD_initCStream_usingCDict() for a compression which requires a dictionary (experimental section)
 
   Use ZSTD_compressStream() repetitively to consume input stream.
   The function will automatically update both `pos` fields.
   Note that it may not consume the entire input, in which case `pos < size`,
   and it's up to the caller to present again remaining data.
   @return : a size hint, preferred nb of bytes to use as input for next function call
-           (it's just a hint, to help latency a little, any other value will work fine)
-           (note : the size hint is guaranteed to be <= ZSTD_CStreamInSize() )
             or an error code, which can be tested using ZSTD_isError().
+            Note 1 : it's just a hint, to help latency a little, any other value will work fine.
+            Note 2 : size hint is guaranteed to be <= ZSTD_CStreamInSize()
 
-  At any moment, it's possible to flush whatever data remains within buffer, using ZSTD_flushStream().
+  At any moment, it's possible to flush whatever data remains within internal buffer, using ZSTD_flushStream().
   `output->pos` will be updated.
-  Note some content might still be left within internal buffer if `output->size` is too small.
+  Note that some content might still be left within internal buffer if `output->size` is too small.
   @return : nb of bytes still present within internal buffer (0 if it's empty)
             or an error code, which can be tested using ZSTD_isError().
 
@@ -222,20 +214,12 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
   The epilogue is required for decoders to consider a frame completed.
   Similar to ZSTD_flushStream(), it may not be able to flush the full content if `output->size` is too small.
   In which case, call again ZSTD_endStream() to complete the flush.
-  @return : nb of bytes still present within internal buffer (0 if it's empty)
+  @return : nb of bytes still present within internal buffer (0 if it's empty, hence compression completed)
             or an error code, which can be tested using ZSTD_isError().
 
  
 <BR></pre>
 
-<h3>Streaming compression functions</h3><pre></pre><b><pre>typedef struct ZSTD_CStream_s ZSTD_CStream;
-ZSTD_CStream* ZSTD_createCStream(void);
-size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
-size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
-size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
-size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
-size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
-</pre></b><BR>
 <pre><b>size_t ZSTD_CStreamInSize(void);    </b>/**< recommended size for input buffer */<b>
 </b></pre><BR>
 <pre><b>size_t ZSTD_CStreamOutSize(void);   </b>/**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */<b>
@@ -261,12 +245,6 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
  
 <BR></pre>
 
-<h3>Streaming decompression functions</h3><pre></pre><b><pre>typedef struct ZSTD_DStream_s ZSTD_DStream;
-ZSTD_DStream* ZSTD_createDStream(void);
-size_t ZSTD_freeDStream(ZSTD_DStream* zds);
-size_t ZSTD_initDStream(ZSTD_DStream* zds);
-size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
-</pre></b><BR>
 <pre><b>size_t ZSTD_DStreamInSize(void);    </b>/*!< recommended size for input buffer */<b>
 </b></pre><BR>
 <pre><b>size_t ZSTD_DStreamOutSize(void);   </b>/*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */<b>
@@ -303,10 +281,10 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
     ZSTD_frameParameters fParams;
 } ZSTD_parameters;
 </b></pre><BR>
-<h3>Custom memory allocation functions</h3><pre></pre><b><pre>typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
+<h3>Custom memory allocation functions</h3><pre><b>typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
 typedef void  (*ZSTD_freeFunction) (void* opaque, void* address);
 typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
-</pre></b><BR>
+</b></pre><BR>
 <a name="Chapter12"></a><h2>Advanced compression functions</h2><pre></pre>
 
 <pre><b>size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams);
@@ -331,14 +309,14 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
 </b><p>  Gives the amount of memory used by a given ZSTD_sizeof_CDict 
 </p></pre><BR>
 
-<pre><b>ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSize, size_t dictSize);
-</b><p>   same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of a `ZSTD_compressionParameters`.
-   All fields of `ZSTD_frameParameters` are set to default (0) 
+<pre><b>ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
+</b><p>   @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
+   `estimatedSrcSize` value is optional, select 0 if not known 
 </p></pre><BR>
 
-<pre><b>ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSize, size_t dictSize);
-</b><p>   @return ZSTD_compressionParameters structure for a selected compression level and srcSize.
-   `srcSize` value is optional, select 0 if not known 
+<pre><b>ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
+</b><p>   same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`.
+   All fields of `ZSTD_frameParameters` are set to default (0) 
 </p></pre><BR>
 
 <pre><b>size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
@@ -409,22 +387,23 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
 
 <a name="Chapter14"></a><h2>Advanced streaming functions</h2><pre></pre>
 
-<h3>Advanced Streaming compression functions</h3><pre></pre><b><pre>ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
+<h3>Advanced Streaming compression functions</h3><pre><b>ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
+size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);   </b>/**< pledgedSrcSize must be correct */<b>
 size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
 size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
                                              ZSTD_parameters params, unsigned long long pledgedSrcSize);  </b>/**< pledgedSrcSize is optional and can be zero == unknown */<b>
 size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);  </b>/**< note : cdict will just be referenced, and must outlive compression session */<b>
 size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);  </b>/**< re-use compression parameters from previous init; skip dictionary loading stage; zcs must be init at least once before */<b>
 size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
-</pre></b><BR>
-<h3>Advanced Streaming decompression functions</h3><pre></pre><b><pre>typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e;
+</b></pre><BR>
+<h3>Advanced Streaming decompression functions</h3><pre><b>typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e;
 ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
 size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
 size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
 size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);  </b>/**< note : ddict will just be referenced, and must outlive decompression session */<b>
 size_t ZSTD_resetDStream(ZSTD_DStream* zds);  </b>/**< re-use decompression parameters from previous init; saves dictionary loading */<b>
 size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
-</pre></b><BR>
+</b></pre><BR>
 <a name="Chapter15"></a><h2>Buffer-less and synchronous inner streaming functions</h2><pre>
   This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
   But it's also a complex one, with many restrictions (documented below).
@@ -461,13 +440,13 @@ size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
   You can then reuse `ZSTD_CCtx` (ZSTD_compressBegin()) to compress some new frame.
 <BR></pre>
 
-<h3>Buffer-less streaming compression functions</h3><pre></pre><b><pre>size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
+<h3>Buffer-less streaming compression functions</h3><pre><b>size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
 size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
 size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize);
 size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize);
 size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
-</pre></b><BR>
+</b></pre><BR>
 <a name="Chapter17"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre>
   A ZSTD_DCtx object is required to track streaming operations.
   Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
@@ -511,7 +490,7 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const vo
   Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType().
   This information is not required to properly decode a frame.
 
-  == Special case : skippable frames == 
+  == Special case : skippable frames ==
 
   Skippable frames allow integration of user-defined data into a flow of concatenated frames.
   Skippable frames will be ignored (skipped) by a decompressor. The format of skippable frames is as follows :
@@ -530,7 +509,7 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const vo
     unsigned checksumFlag;
 } ZSTD_frameParams;
 </b></pre><BR>
-<h3>Buffer-less streaming decompression functions</h3><pre></pre><b><pre>size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize);   </b>/**< doesn't consume input, see details below */<b>
+<h3>Buffer-less streaming decompression functions</h3><pre><b>size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize);   </b>/**< doesn't consume input, see details below */<b>
 size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
 size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
 void   ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
@@ -538,7 +517,7 @@ size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
 size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
 ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
-</pre></b><BR>
+</b></pre><BR>
 <a name="Chapter18"></a><h2>Block functions</h2><pre>
     Block functions produce and decode raw zstd blocks, without frame metadata.
     Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
@@ -563,10 +542,10 @@ ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
         Use ZSTD_insertBlock() in such a case.
 <BR></pre>
 
-<h3>Raw zstd block functions</h3><pre></pre><b><pre>size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx);
+<h3>Raw zstd block functions</h3><pre><b>size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx);
 size_t ZSTD_compressBlock  (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize);  </b>/**< insert block into `dctx` history. Useful for uncompressed blocks */<b>
-</pre></b><BR>
+</b></pre><BR>
 </html>
 </body>

From 8de46ab51ab7c2c54734c7fe9f2c1cfbaa3e3bc3 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Fri, 16 Dec 2016 13:27:30 -0800
Subject: [PATCH 08/14] Export all API functions

---
 build/VS2005/zstdlib/zstdlib.vcproj          |  4 +++
 build/VS2010/libzstd-dll/libzstd-dll.vcxproj |  1 +
 build/VS2010/libzstd/libzstd.vcxproj         |  1 +
 build/cmake/lib/CMakeLists.txt               |  1 +
 lib/common/zstd_common.c                     |  4 ---
 lib/common/zstd_errors.h                     | 18 ++++++++++++--
 lib/deprecated/zbuff_common.c                | 26 ++++++++++++++++++++
 lib/dictBuilder/zdict.h                      | 23 +++++++++--------
 lib/zstd.h                                   | 21 +++++++++-------
 9 files changed, 74 insertions(+), 25 deletions(-)
 create mode 100644 lib/deprecated/zbuff_common.c

diff --git a/build/VS2005/zstdlib/zstdlib.vcproj b/build/VS2005/zstdlib/zstdlib.vcproj
index 1b78986b2..d95212b33 100644
--- a/build/VS2005/zstdlib/zstdlib.vcproj
+++ b/build/VS2005/zstdlib/zstdlib.vcproj
@@ -359,6 +359,10 @@
 				RelativePath="..\..\..\lib\common\xxhash.c"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\lib\deprecated\zbuff_common.c"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\lib\deprecated\zbuff_compress.c"
 				>
diff --git a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj
index f85271970..f1ea5c822 100644
--- a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj
+++ b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj
@@ -29,6 +29,7 @@
     <ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
     <ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
     <ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
+    <ClCompile Include="..\..\..\lib\deprecated\zbuff_common.c" />
     <ClCompile Include="..\..\..\lib\deprecated\zbuff_compress.c" />
     <ClCompile Include="..\..\..\lib\deprecated\zbuff_decompress.c" />
     <ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
diff --git a/build/VS2010/libzstd/libzstd.vcxproj b/build/VS2010/libzstd/libzstd.vcxproj
index f5f30d733..228e83da4 100644
--- a/build/VS2010/libzstd/libzstd.vcxproj
+++ b/build/VS2010/libzstd/libzstd.vcxproj
@@ -29,6 +29,7 @@
     <ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
     <ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
     <ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
+    <ClCompile Include="..\..\..\lib\deprecated\zbuff_common.c" />
     <ClCompile Include="..\..\..\lib\deprecated\zbuff_compress.c" />
     <ClCompile Include="..\..\..\lib\deprecated\zbuff_decompress.c" />
     <ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt
index abcf4ffe2..65942b412 100644
--- a/build/cmake/lib/CMakeLists.txt
+++ b/build/cmake/lib/CMakeLists.txt
@@ -69,6 +69,7 @@ SET(Sources
         ${LIBRARY_DIR}/decompress/zstd_decompress.c
         ${LIBRARY_DIR}/dictBuilder/divsufsort.c
         ${LIBRARY_DIR}/dictBuilder/zdict.c
+        ${LIBRARY_DIR}/deprecated/zbuff_common.c
         ${LIBRARY_DIR}/deprecated/zbuff_compress.c
         ${LIBRARY_DIR}/deprecated/zbuff_decompress.c)
 
diff --git a/lib/common/zstd_common.c b/lib/common/zstd_common.c
index f30128c79..749b2870c 100644
--- a/lib/common/zstd_common.c
+++ b/lib/common/zstd_common.c
@@ -43,10 +43,6 @@ ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); }
 *   provides error code string from enum */
 const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorName(code); }
 
-/* ---   ZBUFF Error Management  (deprecated)   --- */
-unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); }
-const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
-
 
 /*=**************************************************************
 *  Custom allocator
diff --git a/lib/common/zstd_errors.h b/lib/common/zstd_errors.h
index 50dc4f720..949dbd0ff 100644
--- a/lib/common/zstd_errors.h
+++ b/lib/common/zstd_errors.h
@@ -18,6 +18,20 @@ extern "C" {
 #include <stddef.h>   /* size_t */
 
 
+/* =====   ZSTDERRORLIB_API : control library symbols visibility   ===== */
+#if defined(__GNUC__) && (__GNUC__ >= 4)
+#  define ZSTDERRORLIB_VISIBILITY __attribute__ ((visibility ("default")))
+#else
+#  define ZSTDERRORLIB_VISIBILITY
+#endif
+#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
+#  define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBILITY
+#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
+#  define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
+#else
+#  define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBILITY
+#endif
+
 /*-****************************************
 *  error codes list
 ******************************************/
@@ -49,8 +63,8 @@ typedef enum {
 /*! ZSTD_getErrorCode() :
     convert a `size_t` function result into a `ZSTD_ErrorCode` enum type,
     which can be used to compare directly with enum list published into "error_public.h" */
-ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
-const char* ZSTD_getErrorString(ZSTD_ErrorCode code);
+ZSTDERRORLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
+ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code);
 
 
 #if defined (__cplusplus)
diff --git a/lib/deprecated/zbuff_common.c b/lib/deprecated/zbuff_common.c
new file mode 100644
index 000000000..9fff6eb20
--- /dev/null
+++ b/lib/deprecated/zbuff_common.c
@@ -0,0 +1,26 @@
+/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+/*-*************************************
+*  Dependencies
+***************************************/
+#include "error_private.h"
+#include "zbuff.h"
+
+/*-****************************************
+*  ZBUFF Error Management  (deprecated)
+******************************************/
+
+/*! ZBUFF_isError() :
+*   tells if a return value is an error code */
+unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); }
+/*! ZBUFF_getErrorName() :
+*   provides error code string from function result (useful for debugging) */
+const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
+
diff --git a/lib/dictBuilder/zdict.h b/lib/dictBuilder/zdict.h
index 642a43516..d6cf1839e 100644
--- a/lib/dictBuilder/zdict.h
+++ b/lib/dictBuilder/zdict.h
@@ -19,15 +19,18 @@ extern "C" {
 #include <stddef.h>  /* size_t */
 
 
-/*======  Export for Windows  ======*/
-/*!
-*  ZSTD_DLL_EXPORT :
-*  Enable exporting of functions when building a Windows DLL
-*/
-#if defined(_WIN32) && defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
-#  define ZDICTLIB_API __declspec(dllexport)
+/* =====   ZDICTLIB_API : control library symbols visibility   ===== */
+#if defined(__GNUC__) && (__GNUC__ >= 4)
+#  define ZDICTLIB_VISIBILITY __attribute__ ((visibility ("default")))
 #else
-#  define ZDICTLIB_API
+#  define ZDICTLIB_VISIBILITY
+#endif
+#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
+#  define ZDICTLIB_API __declspec(dllexport) ZDICTLIB_VISIBILITY
+#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
+#  define ZDICTLIB_API __declspec(dllimport) ZDICTLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
+#else
+#  define ZDICTLIB_API ZDICTLIB_VISIBILITY
 #endif
 
 
@@ -79,7 +82,7 @@ typedef struct {
               or an error code, which can be tested by ZDICT_isError().
     note : ZDICT_trainFromBuffer_advanced() will send notifications into stderr if instructed to, using notificationLevel>0.
 */
-size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacity,
+ZDICTLIB_API size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacity,
                                 const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
                                 ZDICT_params_t parameters);
 
@@ -97,7 +100,7 @@ size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacit
     starting from its beginning.
     @return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`).
 */
-size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
+ZDICTLIB_API size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
                                         const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples);
 
 
diff --git a/lib/zstd.h b/lib/zstd.h
index b02e16b42..64e782b58 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -20,13 +20,16 @@ extern "C" {
 
 /* =====   ZSTDLIB_API : control library symbols visibility   ===== */
 #if defined(__GNUC__) && (__GNUC__ >= 4)
-#  define ZSTDLIB_API __attribute__ ((visibility ("default")))
-#elif defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
-#  define ZSTDLIB_API __declspec(dllexport)
-#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
-#  define ZSTDLIB_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
+#  define ZSTDLIB_VISIBILITY __attribute__ ((visibility ("default")))
 #else
-#  define ZSTDLIB_API
+#  define ZSTDLIB_VISIBILITY
+#endif
+#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
+#  define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBILITY
+#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
+#  define ZSTDLIB_API __declspec(dllimport) ZSTDLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
+#else
+#  define ZSTDLIB_API ZSTDLIB_VISIBILITY
 #endif
 
 
@@ -463,13 +466,13 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
  *  Provides the dictID stored within dictionary.
  *  if @return == 0, the dictionary is not conformant with Zstandard specification.
  *  It can still be loaded, but as a content-only dictionary. */
-unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
+ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
 
 /*! ZSTD_getDictID_fromDDict() :
  *  Provides the dictID of the dictionary loaded into `ddict`.
  *  If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
  *  Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
-unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
+ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
 
 /*! ZSTD_getDictID_fromFrame() :
  *  Provides the dictID required to decompressed the frame stored within `src`.
@@ -481,7 +484,7 @@ unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
  *  - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
  *  - This is not a Zstandard frame.
  *  When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code. */
-unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
+ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
 
 
 /********************************************************************

From 61e62c014f59757ae4af814ef3a096d060fe4781 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Fri, 16 Dec 2016 13:29:23 -0800
Subject: [PATCH 09/14] Test that all API symbols are exported

---
 .travis.yml      |   2 +-
 appveyor.yml     |   3 +-
 tests/.gitignore |   1 +
 tests/Makefile   |  19 ++++++-
 tests/symbols.c  | 144 +++++++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 164 insertions(+), 5 deletions(-)
 create mode 100644 tests/symbols.c

diff --git a/.travis.yml b/.travis.yml
index 148a98f1b..92c07718f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -12,7 +12,7 @@ matrix:
       os: linux
       sudo: false
 
-    - env: Ubu=12.04cont Cmd="make zlibwrapper && make clean && make -C tests test-zstd-nolegacy && make clean && make cmaketest && make clean && make -C contrib/pzstd googletest pzstd tests check && make -C contrib/pzstd clean"
+    - env: Ubu=12.04cont Cmd="make zlibwrapper && make clean && make -C tests test-symbols && make clean && make -C tests test-zstd-nolegacy && make clean && make cmaketest && make clean && make -C contrib/pzstd googletest pzstd tests check && make -C contrib/pzstd clean"
       os: linux
       sudo: false
       language: cpp
diff --git a/appveyor.yml b/appveyor.yml
index 27c83c04e..1af9da711 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -63,7 +63,8 @@ build_script:
       make -v &&
       cc -v &&
       ECHO make %MAKE_PARAMS% &&
-      make %MAKE_PARAMS%
+      make %MAKE_PARAMS% &&
+      make -C tests test-symbols
     )
   - if [%COMPILER%]==[gcc] if [%PLATFORM%]==[mingw64] (
       COPY programs\zstd.exe bin\zstd.exe &&
diff --git a/tests/.gitignore b/tests/.gitignore
index b558ac258..59a7a1bd2 100644
--- a/tests/.gitignore
+++ b/tests/.gitignore
@@ -12,6 +12,7 @@ paramgrill
 paramgrill32
 roundTripCrash
 longmatch
+symbols
 
 # Tmp test directory
 zstdtest
diff --git a/tests/Makefile b/tests/Makefile
index d7d25026a..b27d1cc60 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -28,7 +28,7 @@ PYTHON ?= python3
 TESTARTEFACT := versionsTest namespaceTest
 
 
-CPPFLAGS= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder -I$(PRGDIR)
+CPPFLAGS= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR)
 CFLAGS ?= -O3
 CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \
           -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef
@@ -127,6 +127,15 @@ roundTripCrash : $(ZSTD_FILES) roundTripCrash.c
 longmatch  : $(ZSTD_FILES) longmatch.c
 	$(CC)      $(FLAGS) $^ -o $@$(EXT)
 
+symbols  : symbols.c
+	$(MAKE) -C $(ZSTDDIR) libzstd
+ifneq (,$(filter Windows%,$(OS)))
+	cp $(ZSTDDIR)/dll/libzstd.dll .
+	$(CC) $(FLAGS) $^ -o $@$(EXT) -DZSTD_DLL_IMPORT=1 libzstd.dll
+else
+	$(CC) $(FLAGS) $^ -o $@$(EXT) -Wl,-rpath=$(ZSTDDIR) $(ZSTDDIR)/libzstd.so
+endif
+
 namespaceTest:
 	if $(CC) namespaceTest.c ../lib/common/xxhash.c -o $@ ; then echo compilation should fail; exit 1 ; fi
 	$(RM) $@
@@ -142,8 +151,9 @@ clean:
         fullbench$(EXT) fullbench32$(EXT) \
         fullbench-lib$(EXT) fullbench-dll$(EXT) \
         fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \
-		zstreamtest$(EXT) zstreamtest32$(EXT) \
-        datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) longmatch$(EXT)
+        zstreamtest$(EXT) zstreamtest32$(EXT) \
+        datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) longmatch$(EXT) \
+        symbols$(EXT)
 	@echo Cleaning completed
 
 
@@ -243,4 +253,7 @@ test-zstream32: zstreamtest32
 test-longmatch: longmatch
 	$(QEMU_SYS) ./longmatch
 
+test-symbols: symbols
+	$(QEMU_SYS) ./symbols
+
 endif
diff --git a/tests/symbols.c b/tests/symbols.c
new file mode 100644
index 000000000..8d03df2fd
--- /dev/null
+++ b/tests/symbols.c
@@ -0,0 +1,144 @@
+#include <stdio.h>
+#include "zstd_errors.h"
+#define ZSTD_STATIC_LINKING_ONLY
+#include "zstd.h"
+#define ZBUFF_DISABLE_DEPRECATE_WARNINGS
+#define ZBUFF_STATIC_LINKING_ONLY
+#include "zbuff.h"
+#define ZDICT_STATIC_LINKING_ONLY
+#include "zdict.h"
+
+static const void *symbols[] = {
+/* zstd.h */
+  &ZSTD_versionNumber,
+  &ZSTD_compress,
+  &ZSTD_decompress,
+  &ZSTD_getDecompressedSize,
+  &ZSTD_maxCLevel,
+  &ZSTD_compressBound,
+  &ZSTD_isError,
+  &ZSTD_getErrorName,
+  &ZSTD_createCCtx,
+  &ZSTD_freeCCtx,
+  &ZSTD_compressCCtx,
+  &ZSTD_createDCtx,
+  &ZSTD_freeDCtx,
+  &ZSTD_decompressDCtx,
+  &ZSTD_compress_usingDict,
+  &ZSTD_decompress_usingDict,
+  &ZSTD_createCDict,
+  &ZSTD_freeCDict,
+  &ZSTD_compress_usingCDict,
+  &ZSTD_createDDict,
+  &ZSTD_freeDDict,
+  &ZSTD_decompress_usingDDict,
+  &ZSTD_createCStream,
+  &ZSTD_freeCStream,
+  &ZSTD_initCStream,
+  &ZSTD_compressStream,
+  &ZSTD_flushStream,
+  &ZSTD_endStream,
+  &ZSTD_CStreamInSize,
+  &ZSTD_CStreamOutSize,
+  &ZSTD_createDStream,
+  &ZSTD_freeDStream,
+  &ZSTD_initDStream,
+  &ZSTD_decompressStream,
+  &ZSTD_DStreamInSize,
+  &ZSTD_DStreamOutSize,
+/* zstd.h: advanced functions */
+  &ZSTD_estimateCCtxSize,
+  &ZSTD_createCCtx_advanced,
+  &ZSTD_sizeof_CCtx,
+  &ZSTD_createCDict_advanced,
+  &ZSTD_sizeof_CDict,
+  &ZSTD_getCParams,
+  &ZSTD_getParams,
+  &ZSTD_checkCParams,
+  &ZSTD_adjustCParams,
+  &ZSTD_compress_advanced,
+  &ZSTD_isFrame,
+  &ZSTD_estimateDCtxSize,
+  &ZSTD_createDCtx_advanced,
+  &ZSTD_sizeof_DCtx,
+  &ZSTD_sizeof_DDict,
+  &ZSTD_getDictID_fromDict,
+  &ZSTD_getDictID_fromDDict,
+  &ZSTD_getDictID_fromFrame,
+  &ZSTD_createCStream_advanced,
+  &ZSTD_initCStream_srcSize,
+  &ZSTD_initCStream_usingDict,
+  &ZSTD_initCStream_advanced,
+  &ZSTD_initCStream_usingCDict,
+  &ZSTD_resetCStream,
+  &ZSTD_sizeof_CStream,
+  &ZSTD_createDStream_advanced,
+  &ZSTD_initDStream_usingDict,
+  &ZSTD_setDStreamParameter,
+  &ZSTD_initDStream_usingDDict,
+  &ZSTD_resetDStream,
+  &ZSTD_sizeof_DStream,
+  &ZSTD_compressBegin,
+  &ZSTD_compressBegin_usingDict,
+  &ZSTD_compressBegin_advanced,
+  &ZSTD_copyCCtx,
+  &ZSTD_compressContinue,
+  &ZSTD_compressEnd,
+  &ZSTD_getFrameParams,
+  &ZSTD_decompressBegin,
+  &ZSTD_decompressBegin_usingDict,
+  &ZSTD_copyDCtx,
+  &ZSTD_nextSrcSizeToDecompress,
+  &ZSTD_decompressContinue,
+  &ZSTD_nextInputType,
+  &ZSTD_getBlockSizeMax,
+  &ZSTD_compressBlock,
+  &ZSTD_decompressBlock,
+  &ZSTD_insertBlock,
+/* zstd_errors.h */
+  &ZSTD_getErrorCode,
+  &ZSTD_getErrorString,
+/* zbuff.h */
+  &ZBUFF_createCCtx,
+  &ZBUFF_freeCCtx,
+  &ZBUFF_compressInit,
+  &ZBUFF_compressInitDictionary,
+  &ZBUFF_compressContinue,
+  &ZBUFF_compressFlush,
+  &ZBUFF_compressEnd,
+  &ZBUFF_createDCtx,
+  &ZBUFF_freeDCtx,
+  &ZBUFF_decompressInit,
+  &ZBUFF_decompressInitDictionary,
+  &ZBUFF_decompressContinue,
+  &ZBUFF_isError,
+  &ZBUFF_getErrorName,
+  &ZBUFF_recommendedCInSize,
+  &ZBUFF_recommendedCOutSize,
+  &ZBUFF_recommendedDInSize,
+  &ZBUFF_recommendedDOutSize,
+/* zbuff.h: advanced functions */
+  &ZBUFF_createCCtx_advanced,
+  &ZBUFF_createDCtx_advanced,
+  &ZBUFF_compressInit_advanced,
+/* zdict.h */
+  &ZDICT_trainFromBuffer,
+  &ZDICT_getDictID,
+  &ZDICT_isError,
+  &ZDICT_getErrorName,
+/* zdict.h: advanced functions */
+  &ZDICT_trainFromBuffer_advanced,
+  &ZDICT_addEntropyTablesFromBuffer,
+  NULL,
+};
+
+int main(int argc, const char** argv) {
+  const void **symbol;
+  (void)argc;
+  (void)argv;
+
+  for (symbol = symbols; *symbol != NULL; ++symbol) {
+    printf("%p\n", *symbol);
+  }
+  return 0;
+}

From d46ecb58a5d537fad672fdc15b3f91da8e0d17be Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sat, 17 Dec 2016 16:28:12 +0100
Subject: [PATCH 10/14] added dll compilation tests

---
 .travis.yml             | 22 ++++++++++++++++++----
 NEWS                    |  3 +++
 lib/dictBuilder/zdict.c |  5 ++---
 lib/zstd.h              |  2 +-
 programs/dibio.c        |  2 +-
 tests/.gitignore        |  2 ++
 tests/Makefile          | 27 ++++++++++++++++++++-------
 7 files changed, 47 insertions(+), 16 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 92c07718f..ef1e67e35 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -69,7 +69,7 @@ matrix:
 
 
     # Ubuntu 14.04 LTS Server Edition 64 bit
-    - env: Ubu=14.04 Cmd="make armtest && make clean && make aarch64test"
+    - env: Ubu=14.04 Cmd="make armtest"
       dist: trusty
       sudo: required
       addons:
@@ -78,7 +78,17 @@ matrix:
             - qemu-system-arm
             - qemu-user-static
             - gcc-arm-linux-gnueabi
-            - libc6-dev-armel-cross 
+            - libc6-dev-armel-cross
+
+    # Ubuntu 14.04 LTS Server Edition 64 bit
+    - env: Ubu=14.04 Cmd="make aarch64test"
+      dist: trusty
+      sudo: required
+      addons:
+        apt:
+          packages:
+            - qemu-system-arm
+            - qemu-user-static
             - gcc-aarch64-linux-gnu
             - libc6-dev-arm64-cross
 
@@ -91,6 +101,7 @@ matrix:
             - qemu-system-ppc
             - qemu-user-static
             - gcc-powerpc-linux-gnu
+            - libc6-dev-armel-cross
 
     - env: Ubu=14.04 Cmd='make -C lib all && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest'
       os: linux
@@ -101,7 +112,10 @@ matrix:
           packages:
             - valgrind
 
-    - env: Ubu=14.04 Cmd="make gpptest && make clean && make gnu90test && make clean && make c99test && make clean && make gnu99test && make clean && make clangtest && make clean && make -C contrib/pzstd googletest32 && make -C contrib/pzstd all32 && make -C contrib/pzstd check && make -C contrib/pzstd clean"
+    - env: Ubu=14.04 Cmd="make gpptest && make clean && make gnu90test && make clean \
+                       && make c99test && make clean && make gnu99test && make clean \
+                       && make clangtest && make clean && make -C contrib/pzstd googletest32 \
+                       && make -C contrib/pzstd all32 && make -C contrib/pzstd check && make -C contrib/pzstd clean"
       os: linux
       dist: trusty
       sudo: required
@@ -127,7 +141,7 @@ matrix:
             - libc6-dev-i386
             - gcc-multilib
 
-    - env: Ubu=14.04 Cmd="make gcc5test && make clean && make gcc6test"
+    - env: Ubu=14.04 Cmd="make gcc5test && make clean && make gcc6test && make clean && make -C tests dll"
       os: linux
       dist: trusty
       sudo: required
diff --git a/NEWS b/NEWS
index 9b0001636..0fc90aff2 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,6 @@
+v1.1.3
+API : fix : all symbols properly exposed in libzstd, by Nick Terrell
+
 v1.1.2
 API : streaming : decompression : changed : automatic implicit reset when chain-decoding new frames without init
 API : experimental : added : dictID retrieval functions, and ZSTD_initCStream_srcSize()
diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c
index ea50b54ad..b539c09fa 100644
--- a/lib/dictBuilder/zdict.c
+++ b/lib/dictBuilder/zdict.c
@@ -36,12 +36,11 @@
 #include <time.h>          /* clock */
 
 #include "mem.h"           /* read */
-#include "error_private.h"
 #include "fse.h"           /* FSE_normalizeCount, FSE_writeNCount */
 #define HUF_STATIC_LINKING_ONLY
-#include "huf.h"
+#include "huf.h"           /* HUF_buildCTable, HUF_writeCTable */
 #include "zstd_internal.h" /* includes zstd.h */
-#include "xxhash.h"
+#include "xxhash.h"        /* XXH64 */
 #include "divsufsort.h"
 #ifndef ZDICT_STATIC_LINKING_ONLY
 #  define ZDICT_STATIC_LINKING_ONLY
diff --git a/lib/zstd.h b/lib/zstd.h
index 64e782b58..7eda69877 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -56,7 +56,7 @@ extern "C" {
 /*------   Version   ------*/
 #define ZSTD_VERSION_MAJOR    1
 #define ZSTD_VERSION_MINOR    1
-#define ZSTD_VERSION_RELEASE  2
+#define ZSTD_VERSION_RELEASE  3
 
 #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
 #define ZSTD_QUOTE(str) #str
diff --git a/programs/dibio.c b/programs/dibio.c
index a793669cb..6fce405b9 100644
--- a/programs/dibio.c
+++ b/programs/dibio.c
@@ -183,7 +183,7 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
     size_t* const fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
     unsigned long long const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
     size_t const maxMem =  DiB_findMaxMem(totalSizeToLoad * MEMMULT) / MEMMULT;
-    size_t benchedSize = MIN (maxMem, (size_t)totalSizeToLoad);
+    size_t benchedSize = (size_t) MIN ((unsigned long long)maxMem, totalSizeToLoad);
     void* const srcBuffer = malloc(benchedSize+NOISELENGTH);
     int result = 0;
 
diff --git a/tests/.gitignore b/tests/.gitignore
index 59a7a1bd2..586c17b6c 100644
--- a/tests/.gitignore
+++ b/tests/.gitignore
@@ -3,10 +3,12 @@ fullbench
 fullbench32
 fuzzer
 fuzzer32
+fuzzer-dll
 zbufftest
 zbufftest32
 zstreamtest
 zstreamtest32
+zstreamtest-dll
 datagen
 paramgrill
 paramgrill32
diff --git a/tests/Makefile b/tests/Makefile
index b27d1cc60..6e37d160e 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -28,12 +28,12 @@ PYTHON ?= python3
 TESTARTEFACT := versionsTest namespaceTest
 
 
-CPPFLAGS= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR)
-CFLAGS ?= -O3
-CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \
+CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR)
+CFLAGS  ?= -O3
+CFLAGS  += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \
           -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef
-CFLAGS += $(MOREFLAGS)
-FLAGS   = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
+CFLAGS  += $(MOREFLAGS)
+FLAGS    = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
 
 
 ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c
@@ -65,6 +65,8 @@ all: fullbench fuzzer zstreamtest paramgrill datagen zbufftest
 
 all32: fullbench32 fuzzer32 zstreamtest32 zbufftest32
 
+dll: fuzzer-dll zstreamtest-dll
+
 
 
 zstd:
@@ -93,11 +95,16 @@ fullbench-dll: $(PRGDIR)/datagen.c fullbench.c
 	$(MAKE) -C $(ZSTDDIR) libzstd
 	$(CC) $(FLAGS) $^ -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(ZSTDDIR)/dll/libzstd.dll
 
-fuzzer  : $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c fuzzer.c
+fuzzer   : $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c fuzzer.c
 	$(CC)      $(FLAGS) $^ -o $@$(EXT)
 
 fuzzer32 : $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c fuzzer.c
-	$(CC)  -m32  $(FLAGS) $^ -o $@$(EXT)
+	$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
+
+fuzzer-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd
+fuzzer-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c fuzzer.c
+	$(MAKE) -C $(ZSTDDIR) libzstd
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@$(EXT)
 
 zbufftest : CPPFLAGS += -I$(ZSTDDIR)/deprecated
 zbufftest : CFLAGS += -Wno-deprecated-declarations   # required to silence deprecation warnings
@@ -115,6 +122,11 @@ zstreamtest  : $(ZSTD_FILES) $(PRGDIR)/datagen.c zstreamtest.c
 zstreamtest32 : $(ZSTD_FILES) $(PRGDIR)/datagen.c zstreamtest.c
 	$(CC) -m32  $(FLAGS) $^ -o $@$(EXT)
 
+zstreamtest-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd
+zstreamtest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c zstreamtest.c
+	$(MAKE) -C $(ZSTDDIR) libzstd
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@$(EXT)
+
 paramgrill : $(ZSTD_FILES) $(PRGDIR)/datagen.c paramgrill.c
 	$(CC)      $(FLAGS) $^ -lm -o $@$(EXT)
 
@@ -151,6 +163,7 @@ clean:
         fullbench$(EXT) fullbench32$(EXT) \
         fullbench-lib$(EXT) fullbench-dll$(EXT) \
         fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \
+        fuzzer-dll$(EXT) zstreamtest-dll$(EXT) \
         zstreamtest$(EXT) zstreamtest32$(EXT) \
         datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) longmatch$(EXT) \
         symbols$(EXT)

From 31ff2a23be05bd01f8dfebb2015eaec34b9aa1a3 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sat, 17 Dec 2016 19:10:10 +0100
Subject: [PATCH 11/14] fix Travis long test list; added zbufftest-dll

---
 .travis.yml    |  6 +++---
 tests/Makefile | 13 ++++++++++---
 2 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index ef1e67e35..6bf99f1bf 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -112,9 +112,9 @@ matrix:
           packages:
             - valgrind
 
-    - env: Ubu=14.04 Cmd="make gpptest && make clean && make gnu90test && make clean \
-                       && make c99test && make clean && make gnu99test && make clean \
-                       && make clangtest && make clean && make -C contrib/pzstd googletest32 \
+    - env: Ubu=14.04 Cmd="make gpptest && make clean && make gnu90test && make clean
+                       && make c99test && make clean && make gnu99test && make clean
+                       && make clangtest && make clean && make -C contrib/pzstd googletest32 
                        && make -C contrib/pzstd all32 && make -C contrib/pzstd check && make -C contrib/pzstd clean"
       os: linux
       dist: trusty
diff --git a/tests/Makefile b/tests/Makefile
index 6e37d160e..abf89f2f6 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -57,7 +57,7 @@ ZSTREAM_TESTTIME = -T2mn
 FUZZERTEST= -T5mn
 ZSTDRTTEST= --test-large-data
 
-.PHONY: default all all32 clean test test32 test-all namespaceTest versionsTest
+.PHONY: default all all32 dll clean test test32 test-all namespaceTest versionsTest
 
 default: fullbench
 
@@ -65,7 +65,7 @@ all: fullbench fuzzer zstreamtest paramgrill datagen zbufftest
 
 all32: fullbench32 fuzzer32 zstreamtest32 zbufftest32
 
-dll: fuzzer-dll zstreamtest-dll
+dll: fuzzer-dll zstreamtest-dll zbufftest-dll
 
 
 
@@ -116,6 +116,13 @@ zbufftest32 : CFLAGS += -Wno-deprecated-declarations -m32
 zbufftest32 : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c zbufftest.c
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
+zbufftest-dll : CPPFLAGS += -I$(ZSTDDIR)/deprecated
+zbufftest-dll : CFLAGS += -Wno-deprecated-declarations   # required to silence deprecation warnings
+zbufftest-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd
+zbufftest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c zbufftest.c
+	$(MAKE) -C $(ZSTDDIR) libzstd
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@$(EXT)
+
 zstreamtest  : $(ZSTD_FILES) $(PRGDIR)/datagen.c zstreamtest.c
 	$(CC)      $(FLAGS) $^ -o $@$(EXT)
 
@@ -163,7 +170,7 @@ clean:
         fullbench$(EXT) fullbench32$(EXT) \
         fullbench-lib$(EXT) fullbench-dll$(EXT) \
         fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \
-        fuzzer-dll$(EXT) zstreamtest-dll$(EXT) \
+        fuzzer-dll$(EXT) zstreamtest-dll$(EXT) zbufftest-dll$(EXT)\
         zstreamtest$(EXT) zstreamtest32$(EXT) \
         datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) longmatch$(EXT) \
         symbols$(EXT)

From 36321d2e329900d09c57c5ee6a4080234f8e333d Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sat, 17 Dec 2016 19:37:55 +0100
Subject: [PATCH 12/14] updated NEWS

---
 NEWS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/NEWS b/NEWS
index 0fc90aff2..9a341781e 100644
--- a/NEWS
+++ b/NEWS
@@ -1,4 +1,5 @@
 v1.1.3
+cli : new : commands for advanced parameters, by Przemyslaw Skibinski
 API : fix : all symbols properly exposed in libzstd, by Nick Terrell
 
 v1.1.2

From 1496c3dc47fc0cf849fb82d7e6b13a1c19fdd34a Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sun, 18 Dec 2016 11:58:23 +0100
Subject: [PATCH 13/14] Fix : size estimation when some samples are very large

---
 lib/dictBuilder/zdict.c |  2 +-
 programs/dibio.c        | 29 +++++++++++++++++++++++++----
 2 files changed, 26 insertions(+), 5 deletions(-)

diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c
index b539c09fa..921e37886 100644
--- a/lib/dictBuilder/zdict.c
+++ b/lib/dictBuilder/zdict.c
@@ -569,7 +569,7 @@ static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters params,
             if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_copyCCtx failed \n"); return; }
     }
     cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_ABSOLUTEMAX, src, srcSize);
-    if (ZSTD_isError(cSize)) { DISPLAYLEVEL(1, "warning : could not compress sample size %u \n", (U32)srcSize); return; }
+    if (ZSTD_isError(cSize)) { DISPLAYLEVEL(3, "warning : could not compress sample size %u \n", (U32)srcSize); return; }
 
     if (cSize) {  /* if == 0; block is not compressible */
         const seqStore_t* seqStorePtr = ZSTD_getSeqStore(esr.zc);
diff --git a/programs/dibio.c b/programs/dibio.c
index 6fce405b9..fa4241b14 100644
--- a/programs/dibio.c
+++ b/programs/dibio.c
@@ -31,7 +31,8 @@
 #define MB *(1 <<20)
 #define GB *(1U<<30)
 
-#define MEMMULT 11
+#define SAMPLESIZE_MAX (128 KB)
+#define MEMMULT 11    /* rough estimation : memory cost to analyze 1 byte of sample */
 static const size_t maxMemory = (sizeof(size_t) == 4) ? (2 GB - 64 MB) : ((size_t)(512 MB) << sizeof(size_t));
 
 #define NOISELENGTH 32
@@ -97,7 +98,7 @@ static unsigned DiB_loadFiles(void* buffer, size_t* bufferSizePtr,
     for (n=0; n<nbFiles; n++) {
         const char* const fileName = fileNamesTable[n];
         unsigned long long const fs64 = UTIL_getFileSize(fileName);
-        size_t const fileSize = (size_t) MIN(fs64, 128 KB);
+        size_t const fileSize = (size_t) MIN(fs64, SAMPLESIZE_MAX);
         if (fileSize > *bufferSizePtr-pos) break;
         {   FILE* const f = fopen(fileName, "rb");
             if (f==NULL) EXM_THROW(10, "zstd: dictBuilder: %s %s ", fileName, strerror(errno));
@@ -163,6 +164,21 @@ static void DiB_saveDict(const char* dictFileName,
 }
 
 
+static int g_tooLargeSamples = 0;
+static U64 DiB_getTotalCappedFileSize(const char** fileNamesTable, unsigned nbFiles)
+{
+    U64 total = 0;
+    unsigned n;
+    for (n=0; n<nbFiles; n++) {
+        U64 const fileSize = UTIL_getFileSize(fileNamesTable[n]);
+        U64 const cappedFileSize = MIN(fileSize, SAMPLESIZE_MAX);
+        total += cappedFileSize;
+        g_tooLargeSamples |= (fileSize > 2*SAMPLESIZE_MAX);
+    }
+    return total;
+}
+
+
 /*! ZDICT_trainFromBuffer_unsafe() :
     Strictly Internal use only !!
     Same as ZDICT_trainFromBuffer_advanced(), but does not control `samplesBuffer`.
@@ -181,7 +197,7 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
 {
     void* const dictBuffer = malloc(maxDictSize);
     size_t* const fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
-    unsigned long long const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
+    unsigned long long const totalSizeToLoad = DiB_getTotalCappedFileSize(fileNamesTable, nbFiles);
     size_t const maxMem =  DiB_findMaxMem(totalSizeToLoad * MEMMULT) / MEMMULT;
     size_t benchedSize = (size_t) MIN ((unsigned long long)maxMem, totalSizeToLoad);
     void* const srcBuffer = malloc(benchedSize+NOISELENGTH);
@@ -190,7 +206,12 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
     /* Checks */
     if ((!fileSizes) || (!srcBuffer) || (!dictBuffer)) EXM_THROW(12, "not enough memory for DiB_trainFiles");   /* should not happen */
     g_displayLevel = params.notificationLevel;
-    if (nbFiles < 5) {
+    if (g_tooLargeSamples) {
+        DISPLAYLEVEL(2, "!  Warning : some samples are very large \n");
+        DISPLAYLEVEL(2, "!  Note that dictionary is only useful for small files or beginning of large files. \n");
+        DISPLAYLEVEL(2, "!  As a consequence, only the first %u bytes of each file are loaded \n", SAMPLESIZE_MAX);
+    }
+    if ((nbFiles < 5) || (totalSizeToLoad < 9 * (unsigned long long)maxDictSize)) {
         DISPLAYLEVEL(2, "!  Warning : nb of samples too low for proper processing ! \n");
         DISPLAYLEVEL(2, "!  Please provide _one file per sample_. \n");
         DISPLAYLEVEL(2, "!  Do not concatenate samples together into a single file, \n");

From d564faa3c6222c9c485f83160c3f2be3b4a82495 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sun, 18 Dec 2016 21:39:15 +0100
Subject: [PATCH 14/14] fix : ZSTD_initCStream_srcSize() correctly set srcSize
 in frame header

---
 lib/compress/zstd_compress.c | 5 +++--
 tests/zstreamtest.c          | 8 ++++++--
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 3d10fbd96..7fd73baa0 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2900,7 +2900,7 @@ size_t ZSTD_CStreamOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSO
 
 size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
 {
-    if (zcs->inBuffSize==0) return ERROR(stage_wrong);   /* zcs has not been init at least once */
+    if (zcs->inBuffSize==0) return ERROR(stage_wrong);   /* zcs has not been init at least once => can't reset */
 
     if (zcs->cdict) CHECK_F(ZSTD_compressBegin_usingCDict(zcs->cctx, zcs->cdict, pledgedSrcSize))
     else CHECK_F(ZSTD_compressBegin_advanced(zcs->cctx, NULL, 0, zcs->params, pledgedSrcSize));
@@ -2967,7 +2967,8 @@ size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t di
 
 size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize)
 {
-    ZSTD_parameters const params = ZSTD_getParams(compressionLevel, pledgedSrcSize, 0);
+    ZSTD_parameters params = ZSTD_getParams(compressionLevel, pledgedSrcSize, 0);
+    if (pledgedSrcSize) params.fParams.contentSizeFlag = 1;
     return ZSTD_initCStream_advanced(zcs, NULL, 0, params, pledgedSrcSize);
 }
 
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 7783fe11b..ce6193085 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -248,7 +248,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     /* _srcSize compression test */
     DISPLAYLEVEL(4, "test%3i : compress_srcSize %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
     ZSTD_initCStream_srcSize(zc, 1, CNBufferSize);
-    outBuff.dst = (char*)(compressedBuffer)+cSize;
+    outBuff.dst = (char*)(compressedBuffer);
     outBuff.size = compressedBufferSize;
     outBuff.pos = 0;
     inBuff.src = CNBuffer;
@@ -259,12 +259,16 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     if (inBuff.pos != inBuff.size) goto _output_error;   /* entire input should be consumed */
     { size_t const r = ZSTD_endStream(zc, &outBuff);
       if (r != 0) goto _output_error; }  /* error, or some data not flushed */
+    { unsigned long long origSize = ZSTD_getDecompressedSize(outBuff.dst, outBuff.pos);
+      DISPLAY("outBuff.pos : %u \n", (U32)outBuff.pos);
+      DISPLAY("origSize = %u \n", (U32)origSize);
+      if ((size_t)origSize != CNBufferSize) goto _output_error; }  /* exact original size must be present */
     DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
 
     /* wrong _srcSize compression test */
     DISPLAYLEVEL(4, "test%3i : wrong srcSize : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1);
     ZSTD_initCStream_srcSize(zc, 1, CNBufferSize-1);
-    outBuff.dst = (char*)(compressedBuffer)+cSize;
+    outBuff.dst = (char*)(compressedBuffer);
     outBuff.size = compressedBufferSize;
     outBuff.pos = 0;
     inBuff.src = CNBuffer;