From dd18d73e7ef1b204bf514095fa8db7f9d6dfdc7b Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 13 Oct 2017 16:32:18 -0700
Subject: [PATCH 01/12] fileio: content size is enabled by default

---
 lib/compress/zstd_compress.c | 4 ++--
 programs/fileio.c            | 1 +
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index e072fe30e..eb6f43d8c 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -391,7 +391,7 @@ size_t ZSTD_CCtxParam_setParameter(
 
     case ZSTD_p_contentSizeFlag :
         /* Content size written in frame header _when known_ (default:1) */
-        DEBUGLOG(5, "set content size flag = %u", (value>0));
+        DEBUGLOG(4, "set content size flag = %u", (value>0));
         params->fParams.contentSizeFlag = value > 0;
         return 0;
 
@@ -401,7 +401,7 @@ size_t ZSTD_CCtxParam_setParameter(
         return 0;
 
     case ZSTD_p_dictIDFlag : /* When applicable, dictionary's dictID is provided in frame header (default:1) */
-        DEBUGLOG(5, "set dictIDFlag = %u", (value>0));
+        DEBUGLOG(4, "set dictIDFlag = %u", (value>0));
         params->fParams.noDictIDFlag = (value == 0);
         return 0;
 
diff --git a/programs/fileio.c b/programs/fileio.c
index 70158f930..427f251b9 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -439,6 +439,7 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
 
 #ifdef ZSTD_NEWAPI
         {   /* frame parameters */
+            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_contentSizeFlag, 1) );
             CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_dictIDFlag, g_dictIDFlag) );
             CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_checksumFlag, g_checksumFlag) );
             (void)srcSize;

From fb4451664125919e0870b256797f826bad576efc Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 13 Oct 2017 17:39:13 -0700
Subject: [PATCH 02/12] ensure fParams.contentSizeFlag starts at 1

such default was failing for ZSTD_compressBegin/ZSTD_compressContinue
fixed too
---
 lib/compress/zstd_compress.c | 19 +++++++++++--------
 lib/zstd.h                   |  8 ++++----
 tests/fullbench.c            |  2 +-
 3 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index eb6f43d8c..b42e6996f 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -78,6 +78,7 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem)
     if (!cctx) return NULL;
     cctx->customMem = customMem;
     cctx->requestedParams.compressionLevel = ZSTD_CLEVEL_DEFAULT;
+    cctx->requestedParams.fParams.contentSizeFlag = 1;
     ZSTD_STATIC_ASSERT(zcss_init==0);
     ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN==(0ULL - 1));
     return cctx;
@@ -1002,7 +1003,7 @@ void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) {
  *  The "context", in this case, refers to the hash and chain tables, entropy
  *  tables, and dictionary offsets.
  *  Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()).
- *  pledgedSrcSize=0 means "empty" if fParams.contentSizeFlag=1
+ *  pledgedSrcSize=0 means "empty".
  *  @return : 0, or an error code */
 static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx,
                             const ZSTD_CCtx* srcCCtx,
@@ -1059,7 +1060,8 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long
     ZSTD_frameParameters fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ };
     ZSTD_buffered_policy_e const zbuff = (ZSTD_buffered_policy_e)(srcCCtx->inBuffSize>0);
     ZSTD_STATIC_ASSERT((U32)ZSTDb_buffered==1);
-    fParams.contentSizeFlag = pledgedSrcSize>0;
+    if (pledgedSrcSize==0) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;
+    fParams.contentSizeFlag = (pledgedSrcSize != ZSTD_CONTENTSIZE_UNKNOWN);
 
     return ZSTD_copyCCtx_internal(dstCCtx, srcCCtx, fParams, pledgedSrcSize, zbuff);
 }
@@ -2077,7 +2079,7 @@ size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t di
     ZSTD_CCtx_params const cctxParams =
             ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params);
     return ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dm_auto, NULL,
-                                       cctxParams, 0, ZSTDb_not_buffered);
+                                       cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, ZSTDb_not_buffered);
 }
 
 size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel)
@@ -2797,18 +2799,18 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
             }
             DEBUGLOG(4, "call ZSTDMT_initCStream_internal as nbThreads=%u", params.nbThreads);
             CHECK_F( ZSTDMT_initCStream_internal(
-                             cctx->mtctx,
-                             prefixDict.dict, prefixDict.dictSize, ZSTD_dm_rawContent,
-                             cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) );
+                        cctx->mtctx,
+                        prefixDict.dict, prefixDict.dictSize, ZSTD_dm_rawContent,
+                        cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) );
             cctx->streamStage = zcss_load;
             cctx->appliedParams.nbThreads = params.nbThreads;
         } else
 #endif
-        {
-            CHECK_F( ZSTD_resetCStream_internal(
+        {   CHECK_F( ZSTD_resetCStream_internal(
                              cctx, prefixDict.dict, prefixDict.dictSize,
                              prefixDict.dictMode, cctx->cdict, params,
                              cctx->pledgedSrcSizePlusOne-1) );
+            assert(cctx->streamStage == zcss_load);
     }   }
 
     /* compression stage */
@@ -3034,5 +3036,6 @@ ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSizeH
     ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, srcSizeHint, dictSize);
     memset(&params, 0, sizeof(params));
     params.cParams = cParams;
+    params.fParams.contentSizeFlag = 1;
     return params;
 }
diff --git a/lib/zstd.h b/lib/zstd.h
index 2194a3b23..7fafdbfc3 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -1000,11 +1000,11 @@ typedef enum {
                               * Special: value 0 means "do not change strategy". */
 
     /* frame parameters */
-    ZSTD_p_contentSizeFlag=200, /* Content size is written into frame header _whenever known_ (default:1)
-                              * note that content size must be known at the beginning,
-                              * it is sent using ZSTD_CCtx_setPledgedSrcSize() */
+    ZSTD_p_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1)
+                              * Content size must be known at the beginning of compression,
+                              * it is provided using ZSTD_CCtx_setPledgedSrcSize() */
     ZSTD_p_checksumFlag,     /* A 32-bits checksum of content is written at end of frame (default:0) */
-    ZSTD_p_dictIDFlag,       /* When applicable, dictID of dictionary is provided in frame header (default:1) */
+    ZSTD_p_dictIDFlag,       /* When applicable, dictionary's ID is written into frame header (default:1) */
 
     /* multi-threading parameters */
     ZSTD_p_nbThreads=400,    /* Select how many threads a compression job can spawn (default:1)
diff --git a/tests/fullbench.c b/tests/fullbench.c
index db00ce217..3f497b034 100644
--- a/tests/fullbench.c
+++ b/tests/fullbench.c
@@ -233,7 +233,7 @@ static ZSTD_CCtx* g_zcc = NULL;
 size_t local_ZSTD_compressContinue(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
 {
     (void)buff2;
-    ZSTD_compressBegin(g_zcc, 1);
+    ZSTD_compressBegin(g_zcc, 1 /* compressionLevel */);
     return ZSTD_compressEnd(g_zcc, dst, dstCapacity, src, srcSize);
 }
 

From 3c1e3f8ec90a09558bda9b860fc1cbfdd322919c Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 13 Oct 2017 18:32:06 -0700
Subject: [PATCH 03/12] contentSizeFlag enabled by default would also fail for
 streaming and MT operations

fixed
---
 lib/compress/zstd_compress.c   | 14 ++++++++------
 lib/compress/zstdmt_compress.c | 34 +++++++++++++++++-----------------
 tests/zstreamtest.c            |  5 +++--
 3 files changed, 28 insertions(+), 25 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index b42e6996f..49b532b94 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2053,6 +2053,8 @@ size_t ZSTD_compressBegin_advanced_internal(
                                     ZSTD_CCtx_params params,
                                     unsigned long long pledgedSrcSize)
 {
+    DEBUGLOG(4, "ZSTD_compressBegin_advanced_internal");
+    DEBUGLOG(4, "contentSizeFlag : %u", params.fParams.contentSizeFlag);
     /* compression parameters verification and optimization */
     CHECK_F( ZSTD_checkCParams(params.cParams) );
     return ZSTD_compressBegin_internal(cctx, dict, dictSize, dictMode, NULL,
@@ -2281,7 +2283,7 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize,
                                       ZSTD_dictMode_e dictMode,
                                       ZSTD_compressionParameters cParams, ZSTD_customMem customMem)
 {
-    DEBUGLOG(5, "ZSTD_createCDict_advanced, mode %u", (U32)dictMode);
+    DEBUGLOG(4, "ZSTD_createCDict_advanced, mode %u", (U32)dictMode);
     if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
 
     {   ZSTD_CDict* const cdict = (ZSTD_CDict*)ZSTD_malloc(sizeof(ZSTD_CDict), customMem);
@@ -2356,9 +2358,9 @@ ZSTD_CDict* ZSTD_initStaticCDict(void* workspace, size_t workspaceSize,
                             + cctxSize;
     ZSTD_CDict* const cdict = (ZSTD_CDict*) workspace;
     void* ptr;
-    DEBUGLOG(5, "(size_t)workspace & 7 : %u", (U32)(size_t)workspace & 7);
+    DEBUGLOG(4, "(size_t)workspace & 7 : %u", (U32)(size_t)workspace & 7);
     if ((size_t)workspace & 7) return NULL;  /* 8-aligned */
-    DEBUGLOG(5, "(workspaceSize < neededSize) : (%u < %u) => %u",
+    DEBUGLOG(4, "(workspaceSize < neededSize) : (%u < %u) => %u",
         (U32)workspaceSize, (U32)neededSize, (U32)(workspaceSize < neededSize));
     if (workspaceSize < neededSize) return NULL;
 
@@ -2394,7 +2396,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced(
     {   ZSTD_CCtx_params params = cctx->requestedParams;
         params.cParams = ZSTD_getCParamsFromCDict(cdict);
         params.fParams = fParams;
-        DEBUGLOG(5, "ZSTD_compressBegin_usingCDict_advanced");
+        DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_advanced");
         return ZSTD_compressBegin_internal(cctx,
                                            NULL, 0, ZSTD_dm_auto,
                                            cdict,
@@ -2409,7 +2411,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced(
 size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
 {
     ZSTD_frameParameters const fParams = { 0 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ };
-    DEBUGLOG(5, "ZSTD_compressBegin_usingCDict : dictIDFlag == %u", !fParams.noDictIDFlag);
+    DEBUGLOG(4, "ZSTD_compressBegin_usingCDict : dictIDFlag == %u", !fParams.noDictIDFlag);
     return ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, 0);
 }
 
@@ -2585,7 +2587,7 @@ size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t di
     ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize);
     ZSTD_CCtx_params const cctxParams =
             ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params);
-    return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL, cctxParams, 0);
+    return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL, cctxParams, ZSTD_CONTENTSIZE_UNKNOWN);
 }
 
 size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize)
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 7831cd3bd..17c69c9df 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -327,6 +327,7 @@ void ZSTDMT_compressChunk(void* jobDescription)
     ZSTD_CCtx* cctx = ZSTDMT_getCCtx(job->cctxPool);
     const void* const src = (const char*)job->srcStart + job->dictSize;
     buffer_t dstBuff = job->dstBuff;
+    DEBUGLOG(5, "ZSTDMT_compressChunk")
     DEBUGLOG(5, "job (first:%u) (last:%u) : dictSize %u, srcSize %u",
                  job->firstChunk, job->lastChunk, (U32)job->dictSize, (U32)job->srcSize);
 
@@ -344,22 +345,22 @@ void ZSTDMT_compressChunk(void* jobDescription)
         job->dstBuff = dstBuff;
     }
 
-    if (job->cdict) {  /* should only happen for first segment */
+    if (job->cdict) {
         size_t const initError = ZSTD_compressBegin_usingCDict_advanced(cctx, job->cdict, job->params.fParams, job->fullFrameSize);
-        DEBUGLOG(5, "using CDict");
+        DEBUGLOG(4, "ZSTDMT_compressChunk, using CDict");
+        assert(job->firstChunk); /* should only happen for first segment */
         if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; }
     } else {  /* srcStart points at reloaded section */
-        if (!job->firstChunk) job->params.fParams.contentSizeFlag = 0;  /* ensure no srcSize control */
-        { ZSTD_CCtx_params jobParams = job->params;
-          size_t const forceWindowError =
-              ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_p_forceMaxWindow, !job->firstChunk);
-          /* Force loading dictionary in "content-only" mode (no header analysis) */
-          size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, job->srcStart, job->dictSize, ZSTD_dm_rawContent, jobParams, job->fullFrameSize);
-            if (ZSTD_isError(initError) || ZSTD_isError(forceWindowError)) {
-                job->cSize = initError;
-                goto _endJob;
-            }
-    }   }
+        ZSTD_CCtx_params jobParams = job->params;
+        size_t const forceWindowError = ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_p_forceMaxWindow, !job->firstChunk);
+        U64 const pledgedSrcSize = job->firstChunk ? job->fullFrameSize : ZSTD_CONTENTSIZE_UNKNOWN;
+        /* load dictionary in "content-only" mode (no header analysis) */
+        size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, job->srcStart, job->dictSize, ZSTD_dm_rawContent, jobParams, pledgedSrcSize);
+        if (ZSTD_isError(initError) || ZSTD_isError(forceWindowError)) {
+            job->cSize = initError;
+            goto _endJob;
+        }
+    }
     if (!job->firstChunk) {  /* flush and overwrite frame header when it's not first segment */
         size_t const hSize = ZSTD_compressContinue(cctx, dstBuff.start, dstBuff.size, src, 0);
         if (ZSTD_isError(hSize)) { job->cSize = hSize; goto _endJob; }
@@ -367,7 +368,7 @@ void ZSTDMT_compressChunk(void* jobDescription)
     }
 
     DEBUGLOG(5, "Compressing : ");
-    DEBUG_PRINTHEX(4, job->srcStart, 12);
+    DEBUG_PRINTHEX(6, job->srcStart, 12);
     job->cSize = (job->lastChunk) ?
                  ZSTD_compressEnd     (cctx, dstBuff.start, dstBuff.size, src, job->srcSize) :
                  ZSTD_compressContinue(cctx, dstBuff.start, dstBuff.size, src, job->srcSize);
@@ -610,6 +611,7 @@ static size_t ZSTDMT_compress_advanced_internal(
     assert(jobParams.nbThreads == 0);
     assert(mtctx->cctxPool->totalCCtx == params.nbThreads);
 
+    DEBUGLOG(4, "ZSTDMT_compress_advanced_internal");
     DEBUGLOG(4, "nbChunks  : %2u   (chunkSize : %u bytes)   ", nbChunks, (U32)avgChunkSize);
     if (nbChunks==1) {   /* fallback to single-thread mode */
         ZSTD_CCtx* const cctx = mtctx->cctxPool->cctx[0];
@@ -641,7 +643,7 @@ static size_t ZSTDMT_compress_advanced_internal(
             mtctx->jobs[u].srcStart = srcStart + frameStartPos - dictSize;
             mtctx->jobs[u].dictSize = dictSize;
             mtctx->jobs[u].srcSize = chunkSize;
-            mtctx->jobs[u].cdict = mtctx->nextJobID==0 ? cdict : NULL;
+            mtctx->jobs[u].cdict = (u==0) ? cdict : NULL;
             mtctx->jobs[u].fullFrameSize = srcSize;
             mtctx->jobs[u].params = jobParams;
             /* do not calculate checksum within sections, but write it in header for first section */
@@ -777,7 +779,6 @@ size_t ZSTDMT_initCStream_internal(
     zcs->params = params;
     zcs->frameContentSize = pledgedSrcSize;
     if (dict) {
-        DEBUGLOG(4,"cdictLocal: %08X", (U32)(size_t)zcs->cdictLocal);
         ZSTD_freeCDict(zcs->cdictLocal);
         zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize,
                                                     ZSTD_dlm_byCopy, dictMode, /* note : a loadPrefix becomes an internal CDict */
@@ -785,7 +786,6 @@ size_t ZSTDMT_initCStream_internal(
         zcs->cdict = zcs->cdictLocal;
         if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
     } else {
-        DEBUGLOG(4,"cdictLocal: %08X", (U32)(size_t)zcs->cdictLocal);
         ZSTD_freeCDict(zcs->cdictLocal);
         zcs->cdictLocal = NULL;
         zcs->cdict = cdict;
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 8b4c83694..36eede293 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -190,9 +190,10 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
 
     /* Basic compression test */
     DISPLAYLEVEL(3, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
-    CHECK_Z( ZSTD_initCStream_usingDict(zc, CNBuffer, dictSize, 1) );
+    CHECK_Z( ZSTD_initCStream_usingDict(zc, CNBuffer, dictSize, 1 /* cLevel */) );
     outBuff.dst = (char*)(compressedBuffer)+cSize;
-    outBuff.size = compressedBufferSize;
+    assert(compressedBufferSize > cSize);
+    outBuff.size = compressedBufferSize - cSize;
     outBuff.pos = 0;
     inBuff.src = CNBuffer;
     inBuff.size = CNBufferSize;

From 213ef3b510b909b7dd95dc472e680ebea25bc450 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 13 Oct 2017 19:01:58 -0700
Subject: [PATCH 04/12] fixed ZSTD_initCStream_advanced() behavior, which
 depends on contentSizeFlag,

and a stream fuzzer test, which was incorrect
(relied on 0 being unconditionnally transformed into `ZSTD_CONTENTSIZE_UNKNOWN`)
---
 lib/compress/zstd_compress.c | 16 ++++++++++++----
 lib/zstd.h                   |  2 +-
 tests/zstreamtest.c          |  5 +++--
 3 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 49b532b94..4ceba415c 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -822,9 +822,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
                                       ZSTD_compResetPolicy_e const crp,
                                       ZSTD_buffered_policy_e const zbuff)
 {
-    DEBUGLOG(4, "ZSTD_resetCCtx_internal");
+    DEBUGLOG(4, "ZSTD_resetCCtx_internal: pledgedSrcSize=%u", (U32)pledgedSrcSize);
     assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
-    DEBUGLOG(4, "pledgedSrcSize: %u", (U32)pledgedSrcSize);
 
     if (crp == ZSTDcrp_continue) {
         if (ZSTD_equivalentParams(params, zc->appliedParams)) {
@@ -2500,12 +2499,15 @@ static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs,
     return 0;   /* ready to go */
 }
 
+/* ZSTD_resetCStream():
+ * pledgedSrcSize == 0 means "unknown" */
 size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
 {
     ZSTD_CCtx_params params = zcs->requestedParams;
-    params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
+    DEBUGLOG(4, "ZSTD_resetCStream: pledgedSrcSize = %u", (U32)pledgedSrcSize);
+    if (pledgedSrcSize==0) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;
+    params.fParams.contentSizeFlag = 1;
     params.cParams = ZSTD_getCParamsFromCCtxParams(params, pledgedSrcSize, 0);
-    DEBUGLOG(4, "ZSTD_resetCStream");
     return ZSTD_resetCStream_internal(zcs, NULL, 0, ZSTD_dm_auto, zcs->cdict, params, pledgedSrcSize);
 }
 
@@ -2572,13 +2574,19 @@ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict)
     return ZSTD_initCStream_usingCDict_advanced(zcs, cdict, fParams, 0);  /* note : will check that cdict != NULL */
 }
 
+/* ZSTD_initCStream_advanced() :
+ * pledgedSrcSize is optional: it can be 0, which means "unknown" if contentSizeFlag is set, or "unknown" if contentSizeFlag is not set. */
 size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
                                  const void* dict, size_t dictSize,
                                  ZSTD_parameters params, unsigned long long pledgedSrcSize)
 {
     ZSTD_CCtx_params const cctxParams =
             ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params);
+    DEBUGLOG(4, "ZSTD_initCStream_advanced: pledgedSrcSize=%u, flag=%u",
+            (U32)pledgedSrcSize, params.fParams.contentSizeFlag);
     CHECK_F( ZSTD_checkCParams(params.cParams) );
+    if ((pledgedSrcSize==0) && (params.fParams.contentSizeFlag==0))
+        pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;
     return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL, cctxParams, pledgedSrcSize);
 }
 
diff --git a/lib/zstd.h b/lib/zstd.h
index 7fafdbfc3..2866866be 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -734,7 +734,7 @@ ZSTDLIB_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspa
 ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);   /**< pledgedSrcSize must be correct, a size of 0 means unknown.  for a frame size of 0 use initCStream_advanced */
 ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.*/
 ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
-                                             ZSTD_parameters params, unsigned long long pledgedSrcSize);  /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. */
+                                             ZSTD_parameters params, unsigned long long pledgedSrcSize);  /**< pledgedSrcSize is optional: it can be 0, which means "unknown" if contentSizeFlag is set, or "unknown" if contentSizeFlag is not set. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. */
 ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);  /**< note : cdict will just be referenced, and must outlive compression session */
 ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize);  /**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters */
 
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 36eede293..3aea22709 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -559,7 +559,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_advanced with pledgedSrcSize=0 and dict : ", testNb++);
     {   ZSTD_parameters params = ZSTD_getParams(5, 0, 0);
         params.fParams.contentSizeFlag = 1;
-        CHECK_Z( ZSTD_initCStream_advanced(zc, dictionary.start, dictionary.filled, params, 0) );
+        CHECK_Z( ZSTD_initCStream_advanced(zc, dictionary.start, dictionary.filled, params, 0 /* pledgedSrcSize==0 means "empty" when params.fParams.contentSizeFlag is set */) );
     } /* cstream advanced shall write content size = 0 */
     inBuff.src = CNBuffer;
     inBuff.size = 0;
@@ -833,10 +833,11 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
             {   size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
                 dict = srcBuffer + dictStart;
             }
-            {   U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? 0 : maxTestSize;
+            {   U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
                 ZSTD_parameters params = ZSTD_getParams(cLevel, pledgedSrcSize, dictSize);
                 params.fParams.checksumFlag = FUZ_rand(&lseed) & 1;
                 params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1;
+                params.fParams.contentSizeFlag = FUZ_rand(&lseed) & 1;
                 CHECK_Z ( ZSTD_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize) );
         }   }
 

From beb9b4b398cb4a9b98f48a49a1c95cfbd030527c Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 13 Oct 2017 19:09:30 -0700
Subject: [PATCH 05/12] fixed ZSTDMT_initCStream() when contentSizeFlag==1 by
 default

and a wrong test in zstreamtest --mt
---
 lib/compress/zstdmt_compress.c | 2 +-
 tests/zstreamtest.c            | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 17c69c9df..197a2c49e 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -852,7 +852,7 @@ size_t ZSTDMT_initCStream(ZSTDMT_CCtx* zcs, int compressionLevel) {
     ZSTD_CCtx_params cctxParams = zcs->params;
     cctxParams.cParams = params.cParams;
     cctxParams.fParams = params.fParams;
-    return ZSTDMT_initCStream_internal(zcs, NULL, 0, ZSTD_dm_auto, NULL, cctxParams, 0);
+    return ZSTDMT_initCStream_internal(zcs, NULL, 0, ZSTD_dm_auto, NULL, cctxParams, ZSTD_CONTENTSIZE_UNKNOWN);
 }
 
 
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 3aea22709..38320e8c4 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -1088,13 +1088,13 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
             {   size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
                 dict = srcBuffer + dictStart;
             }
-            {   U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? 0 : maxTestSize;
+            {   U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
                 ZSTD_parameters params = ZSTD_getParams(cLevel, pledgedSrcSize, dictSize);
                 DISPLAYLEVEL(5, "Init with windowLog = %u and pledgedSrcSize = %u \n",
                     params.cParams.windowLog, (U32)pledgedSrcSize);
                 params.fParams.checksumFlag = FUZ_rand(&lseed) & 1;
                 params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1;
-                params.fParams.contentSizeFlag = pledgedSrcSize>0;
+                params.fParams.contentSizeFlag = FUZ_rand(&lseed) & 1;
                 DISPLAYLEVEL(5, "checksumFlag : %u \n", params.fParams.checksumFlag);
                 CHECK_Z( ZSTDMT_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize) );
                 CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_overlapSectionLog, FUZ_rand(&lseed) % 12) );

From 43914f6a206989432d68bb6abe336e1a662e913d Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 13 Oct 2017 23:47:01 -0700
Subject: [PATCH 06/12] add test for presence of decompressed size field in
 header

previous test was "display" only, it wouldn't trigger an error in case of missing header.
---
 tests/playTests.sh | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/tests/playTests.sh b/tests/playTests.sh
index bc021648c..013182aeb 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -600,7 +600,7 @@ $ECHO "\n**** zstd --list/-l single frame tests ****"
 ./datagen > tmp3
 $ZSTD tmp*
 $ZSTD -l *.zst
-$ZSTD -lv *.zst
+$ZSTD -lv *.zst | grep "Decompressed Size:"  # check that decompressed size is present in header
 $ZSTD --list *.zst
 $ZSTD --list -v *.zst
 
@@ -609,8 +609,6 @@ cat tmp1.zst tmp2.zst > tmp12.zst
 cat tmp12.zst tmp3.zst > tmp123.zst
 $ZSTD -l *.zst
 $ZSTD -lv *.zst
-$ZSTD --list *.zst
-$ZSTD --list -v *.zst
 
 $ECHO "\n**** zstd --list/-l error detection tests ****"
 ! $ZSTD -l tmp1 tmp1.zst
@@ -627,9 +625,9 @@ $ZSTD -lv tmp5.zst
 ! $ZSTD -lv tmp5*
 
 $ECHO "\n**** zstd --list/-l test with no content size field ****"
-./datagen -g1MB | $ZSTD > tmp6.zst
+./datagen -g513K | $ZSTD > tmp6.zst
 $ZSTD -l tmp6.zst
-$ZSTD -lv tmp6.zst
+! $ZSTD -lv tmp6.zst | grep "Decompressed Size:"  # must NOT be present in header
 
 $ECHO "\n**** zstd --list/-l test with no checksum ****"
 $ZSTD -f --no-check tmp1

From 9ef32b3cf1cbf5276f3e6dd0484de1ddd258b4a4 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sat, 14 Oct 2017 00:02:32 -0700
Subject: [PATCH 07/12] minor : zstd -l -v display each file name

---
 programs/fileio.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 427f251b9..01e901ed8 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -1927,7 +1927,7 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){
     return detectError;
 }
 
-static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){
+static void displayInfo(const char* inFileName, const fileInfo_t* info, int displayLevel){
     unsigned const unit = info->compressedSize < (1 MB) ? (1 KB) : (1 MB);
     const char* const unitStr = info->compressedSize < (1 MB) ? "KB" : "MB";
     double const windowSizeUnit = (double)info->windowSize / unit;
@@ -1950,8 +1950,10 @@ static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLev
                     checkString, inFileName);
         }
     } else {
+        DISPLAYOUT("%s \n", inFileName);
         DISPLAYOUT("# Zstandard Frames: %d\n", info->numActualFrames);
-        DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames);
+        if (info->numSkippableFrames)
+            DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames);
         DISPLAYOUT("Window Size: %.2f %2s (%llu B)\n",
                    windowSizeUnit, unitStr,
                    (unsigned long long)info->windowSize);
@@ -1983,7 +1985,6 @@ static fileInfo_t FIO_addFInfo(fileInfo_t fi1, fileInfo_t fi2)
 }
 
 static int FIO_listFile(fileInfo_t* total, const char* inFileName, int displayLevel){
-    /* initialize info to avoid warnings */
     fileInfo_t info;
     memset(&info, 0, sizeof(info));
     {   int const error = getFileInfo(&info, inFileName);
@@ -2023,7 +2024,7 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis
         for (u=0; u<numFiles;u++) {
             error |= FIO_listFile(&total, filenameTable[u], displayLevel);
         }
-        if (numFiles > 1 && displayLevel <= 2) {
+        if (numFiles > 1 && displayLevel <= 2) {   /* display total */
             unsigned const unit = total.compressedSize < (1 MB) ? (1 KB) : (1 MB);
             const char* const unitStr = total.compressedSize < (1 MB) ? "KB" : "MB";
             double const compressedSizeUnit = (double)total.compressedSize / unit;
@@ -2043,8 +2044,7 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis
                         total.numSkippableFrames,
                         compressedSizeUnit, unitStr, decompressedSizeUnit, unitStr,
                         ratio, checkString, total.nbFiles);
-            }
-        }
+        }   }
         return error;
     }
 }

From 5eed8e7a5538be8fc39ab94d1a61e161bbc348b9 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sat, 14 Oct 2017 00:32:06 -0700
Subject: [PATCH 08/12] changed API comments to invite using macro
 ZSTD_CONTENTSIZE_UNKNOWN

to mean "pledgedSrcSize is not known at init time" instead of `0`.
Note that, a few prototypes created and documented with `0` to mean "unknown" still interpret "0" as unknown,
to avoid breaking 3rd party applications which depend on this behavior.
But this value is no longer recommended to mean "unknown".

In some future version, it might be possible to switch "0" to mean "empty",
as is already the case for several prototypes.
The advantage is that pledgedSrcSize field would have same behavior accross entire API,
making it easier to reason about.

Note that all concerned prototypes belong to the "experimental" API section.

srcSize is controlled at end of compression,
so if someone uses "0" to mean "unknown" while it effectively means "empty",
this is immediately caught by the compression function, which generates an error code : ZSTD_ERROR_srcSize_wrong
---
 doc/zstd_manual.html | 26 ++++++++++++++------------
 lib/zstd.h           | 18 ++++++++++--------
 2 files changed, 24 insertions(+), 20 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index b4720adac..7ef86a23d 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -631,10 +631,10 @@ size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMet
 
 <h3>Advanced Streaming compression functions</h3><pre></pre><b><pre>ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
 ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize);    </b>/**< same as ZSTD_initStaticCCtx() */<b>
-size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);   </b>/**< pledgedSrcSize must be correct, a size of 0 means unknown.  for a frame size of 0 use initCStream_advanced */<b>
+size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);   </b>/**< pledgedSrcSize must be correct. If it is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, a size of 0 is interepreted as "unknown". But it may change in some future version to mean "empty". */<b>
 size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); </b>/**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.*/<b>
 size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
-                                             ZSTD_parameters params, unsigned long long pledgedSrcSize);  </b>/**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. */<b>
+                                             ZSTD_parameters params, unsigned long long pledgedSrcSize);  </b>/**< pledgedSrcSize : If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. */<b>
 size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);  </b>/**< note : cdict will just be referenced, and must outlive compression session */<b>
 size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize);  </b>/**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters */<b>
 </pre></b><BR>
@@ -642,9 +642,11 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict*
 </b><p>  start a new compression job, using same parameters from previous job.
   This is typically useful to skip dictionary loading stage, since it will re-use it in-place..
   Note that zcs must be init at least once before using ZSTD_resetCStream().
-  pledgedSrcSize==0 means "srcSize unknown".
+  If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN.
   If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end.
-  @return : 0, or an error code (which can be tested using ZSTD_isError()) 
+  For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs,
+  but it may change to mean "empty" in some future version, so prefer using macro ZSTD_CONTENTSIZE_UNKNOWN.
+ @return : 0, or an error code (which can be tested using ZSTD_isError()) 
 </p></pre><BR>
 
 <h3>Advanced Streaming decompression functions</h3><pre></pre><b><pre>ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
@@ -693,10 +695,10 @@ size_t ZSTD_resetDStream(ZSTD_DStream* zds);  </b>/**< re-use decompression para
 
 <h3>Buffer-less streaming compression functions</h3><pre></pre><b><pre>size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
 size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
-size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */<b>
+size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */<b>
 size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); </b>/**< note: fails if cdict==NULL */<b>
-size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize);   </b>/* compression parameters are already set within cdict. pledgedSrcSize=0 means null-size */<b>
-size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); </b>/**<  note: if pledgedSrcSize can be 0, indicating unknown size.  if it is non-zero, it must be accurate.  for 0 size frames, use compressBegin_advanced */<b>
+size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize);   </b>/* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */<b>
+size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); </b>/**<  note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */<b>
 </pre></b><BR>
 <a name="Chapter19"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre>
   A ZSTD_DCtx object is required to track streaming operations.
@@ -848,11 +850,11 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long
                               * Special: value 0 means "do not change strategy". */
 
     </b>/* frame parameters */<b>
-    ZSTD_p_contentSizeFlag=200, </b>/* Content size is written into frame header _whenever known_ (default:1)<b>
-                              * note that content size must be known at the beginning,
-                              * it is sent using ZSTD_CCtx_setPledgedSrcSize() */
+    ZSTD_p_contentSizeFlag=200, </b>/* Content size will be written into frame header _whenever known_ (default:1)<b>
+                              * Content size must be known at the beginning of compression,
+                              * it is provided using ZSTD_CCtx_setPledgedSrcSize() */
     ZSTD_p_checksumFlag,     </b>/* A 32-bits checksum of content is written at end of frame (default:0) */<b>
-    ZSTD_p_dictIDFlag,       </b>/* When applicable, dictID of dictionary is provided in frame header (default:1) */<b>
+    ZSTD_p_dictIDFlag,       </b>/* When applicable, dictionary's ID is written into frame header (default:1) */<b>
 
     </b>/* multi-threading parameters */<b>
     ZSTD_p_nbThreads=400,    </b>/* Select how many threads a compression job can spawn (default:1)<b>
@@ -913,7 +915,7 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long
  @result : 0, or an error code (which can be tested with ZSTD_isError()).
   Note 1 : 0 means zero, empty.
            In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
-           Note that ZSTD_CONTENTSIZE_UNKNOWN is default value for new compression jobs.
+           ZSTD_CONTENTSIZE_UNKNOWN is default value for any new compression job.
   Note 2 : If all data is provided and consumed in a single round,
            this value is overriden by srcSize instead. 
 </p></pre><BR>
diff --git a/lib/zstd.h b/lib/zstd.h
index 2866866be..e9c4ff9d4 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -731,10 +731,10 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
 /*=====   Advanced Streaming compression functions  =====*/
 ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
 ZSTDLIB_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize);    /**< same as ZSTD_initStaticCCtx() */
-ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);   /**< pledgedSrcSize must be correct, a size of 0 means unknown.  for a frame size of 0 use initCStream_advanced */
+ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);   /**< pledgedSrcSize must be correct. If it is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, a size of 0 is interepreted as "unknown". But it may change in some future version to mean "empty". */
 ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.*/
 ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
-                                             ZSTD_parameters params, unsigned long long pledgedSrcSize);  /**< pledgedSrcSize is optional: it can be 0, which means "unknown" if contentSizeFlag is set, or "unknown" if contentSizeFlag is not set. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. */
+                                             ZSTD_parameters params, unsigned long long pledgedSrcSize);  /**< pledgedSrcSize : If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. */
 ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);  /**< note : cdict will just be referenced, and must outlive compression session */
 ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize);  /**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters */
 
@@ -742,9 +742,11 @@ ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const
  *  start a new compression job, using same parameters from previous job.
  *  This is typically useful to skip dictionary loading stage, since it will re-use it in-place..
  *  Note that zcs must be init at least once before using ZSTD_resetCStream().
- *  pledgedSrcSize==0 means "srcSize unknown".
+ *  If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN.
  *  If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end.
- *  @return : 0, or an error code (which can be tested using ZSTD_isError()) */
+ *  For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs,
+ *  but it may change to mean "empty" in some future version, so prefer using macro ZSTD_CONTENTSIZE_UNKNOWN.
+ * @return : 0, or an error code (which can be tested using ZSTD_isError()) */
 ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
 
 
@@ -800,10 +802,10 @@ ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);  /**< re-use decompress
 /*=====   Buffer-less streaming compression functions  =====*/
 ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
 ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
-ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */
+ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */
 ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */
-ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize);   /* compression parameters are already set within cdict. pledgedSrcSize=0 means null-size */
-ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**<  note: if pledgedSrcSize can be 0, indicating unknown size.  if it is non-zero, it must be accurate.  for 0 size frames, use compressBegin_advanced */
+ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize);   /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */
+ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**<  note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */
 
 ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
@@ -1066,7 +1068,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param
  * @result : 0, or an error code (which can be tested with ZSTD_isError()).
  *  Note 1 : 0 means zero, empty.
  *           In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
- *           Note that ZSTD_CONTENTSIZE_UNKNOWN is default value for new compression jobs.
+ *           ZSTD_CONTENTSIZE_UNKNOWN is default value for any new compression job.
  *  Note 2 : If all data is provided and consumed in a single round,
  *           this value is overriden by srcSize instead. */
 ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);

From fc8d293460d6767558843ac8231c249dcc704382 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sat, 14 Oct 2017 01:21:43 -0700
Subject: [PATCH 09/12] dictionary compression use correct file size estimation

when determining compression parameters
to compress one file only.

For multiple files, it still "bets" that files are going to be small.

There was also a bug recently added in ZSTD_CCtx_loadDictionary_advanced()
making it incapable to use pledgedSrcSize to determine compression parameters.
---
 lib/compress/zstd_compress.c | 2 +-
 programs/fileio.c            | 9 +++++----
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 4ceba415c..c7872e7be 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -531,7 +531,7 @@ size_t ZSTD_CCtx_loadDictionary_advanced(
         cctx->cdict = NULL;
     } else {
         ZSTD_compressionParameters const cParams =
-                ZSTD_getCParamsFromCCtxParams(cctx->requestedParams, 0, dictSize);
+                ZSTD_getCParamsFromCCtxParams(cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1, dictSize);
         cctx->cdictLocal = ZSTD_createCDict_advanced(
                                 dict, dictSize,
                                 dictLoadMethod, dictMode,
diff --git a/programs/fileio.c b/programs/fileio.c
index 01e901ed8..17ad31b32 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -442,7 +442,6 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
             CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_contentSizeFlag, 1) );
             CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_dictIDFlag, g_dictIDFlag) );
             CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_checksumFlag, g_checksumFlag) );
-            (void)srcSize;
             /* compression level */
             CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, cLevel) );
             /* long distance matching */
@@ -468,7 +467,9 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
             DISPLAYLEVEL(5,"set nb threads = %u \n", g_nbThreads);
             CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbThreads, g_nbThreads) );
             /* dictionary */
+            CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) );  /* just to load dictionary with good compression parameters */
             CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) );
+            CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, ZSTD_CONTENTSIZE_UNKNOWN) );  /* reset */
         }
 #elif defined(ZSTD_MULTITHREAD)
         {   ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize);
@@ -1006,7 +1007,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
     size_t dfnSize = FNSPACE;
     char*  dstFileName = (char*)malloc(FNSPACE);
     size_t const suffixSize = suffix ? strlen(suffix) : 0;
-    U64 const srcSize = (nbFiles != 1) ? 0 : UTIL_getFileSize(inFileNamesTable[0]) ;
+    U64 const srcSize = (nbFiles != 1) ? ZSTD_CONTENTSIZE_UNKNOWN : UTIL_getFileSize(inFileNamesTable[0]) ;
     cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, comprParams);
 
     /* init */
@@ -1032,9 +1033,9 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
                 free(dstFileName);
                 dfnSize = ifnSize + 20;
                 dstFileName = (char*)malloc(dfnSize);
-                if (!dstFileName)
+                if (!dstFileName) {
                     EXM_THROW(30, "zstd: %s", strerror(errno));
-            }
+            }   }
             strcpy(dstFileName, inFileNamesTable[u]);
             strcat(dstFileName, suffix);
             missed_files += FIO_compressFilename_dstFile(ress, dstFileName, inFileNamesTable[u], compressionLevel);

From 241c57adffe34a8267dd9526cece87b8751468d2 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 16 Oct 2017 14:01:42 -0700
Subject: [PATCH 10/12] playtest : minor cosmetic changes

---
 tests/playTests.sh | 68 +++++++++++++++++++++++-----------------------
 1 file changed, 34 insertions(+), 34 deletions(-)

diff --git a/tests/playTests.sh b/tests/playTests.sh
index 013182aeb..f008d8998 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -91,7 +91,7 @@ else
     hasMT="true"
 fi
 
-$ECHO "\n**** simple tests **** "
+$ECHO "\n===>  simple tests "
 
 ./datagen > tmp
 $ECHO "test : basic compression "
@@ -166,7 +166,7 @@ $ZSTD -f tmp && die "tmp not present : should have failed"
 test ! -f tmp.zst  # tmp.zst should not be created
 
 
-$ECHO "\n**** Advanced compression parameters **** "
+$ECHO "\n===>  Advanced compression parameters "
 $ECHO "Hello world!" | $ZSTD --zstd=windowLog=21,      - -o tmp.zst && die "wrong parameters not detected!"
 $ECHO "Hello world!" | $ZSTD --zstd=windowLo=21        - -o tmp.zst && die "wrong parameters not detected!"
 $ECHO "Hello world!" | $ZSTD --zstd=windowLog=21,slog  - -o tmp.zst && die "wrong parameters not detected!"
@@ -180,14 +180,14 @@ roundTripTest -g512K " --long --zstd=ldmhlog=20,ldmslen=64,ldmblog=1,ldmhevery=7
 roundTripTest -g512K 19
 
 
-$ECHO "\n**** Pass-Through mode **** "
+$ECHO "\n===>  Pass-Through mode "
 $ECHO "Hello world 1!" | $ZSTD -df
 $ECHO "Hello world 2!" | $ZSTD -dcf
 $ECHO "Hello world 3!" > tmp1
 $ZSTD -dcf tmp1
 
 
-$ECHO "\n**** frame concatenation **** "
+$ECHO "\n===>  frame concatenation "
 
 $ECHO "hello " > hello.tmp
 $ECHO "world!" > world.tmp
@@ -218,7 +218,7 @@ $ECHO "$ECHO foo | $ZSTD | $ZSTD -d > /dev/full"
 $ECHO foo | $ZSTD | $ZSTD -d > /dev/full && die "write error not detected!"
 
 
-$ECHO "\n**** symbolic link test **** "
+$ECHO "\n===>  symbolic link test "
 
 rm -f hello.tmp world.tmp hello.tmp.zst world.tmp.zst
 $ECHO "hello world" > hello.tmp
@@ -233,7 +233,7 @@ rm -f hello.tmp world.tmp hello.tmp.zst world.tmp.zst
 fi
 
 
-$ECHO "\n**** test sparse file support **** "
+$ECHO "\n===>  test sparse file support "
 
 ./datagen -g5M  -P100 > tmpSparse
 $ZSTD tmpSparse -c | $ZSTD -dv -o tmpSparseRegen
@@ -260,7 +260,7 @@ $DIFF tmpSparse2M tmpSparseRegenerated
 rm tmpSparse*
 
 
-$ECHO "\n**** multiple files tests **** "
+$ECHO "\n===>  multiple files tests "
 
 ./datagen -s1        > tmp1 2> $INTOVOID
 ./datagen -s2 -g100K > tmp2 2> $INTOVOID
@@ -283,7 +283,7 @@ $ECHO "compress multiple files including a missing one (notHere) : "
 $ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!"
 
 
-$ECHO "\n**** dictionary tests **** "
+$ECHO "\n===>  dictionary tests "
 
 $ECHO "- test with raw dict (content only) "
 ./datagen > tmpDict
@@ -339,7 +339,7 @@ $ZSTD --train-legacy -q tmp && die "Dictionary training should fail : source is
 rm tmp*
 
 
-$ECHO "\n**** cover dictionary tests **** "
+$ECHO "\n===>  cover dictionary builder : advanced options "
 
 TESTFILE=../programs/zstdcli.c
 ./datagen > tmpDict
@@ -359,7 +359,7 @@ $ECHO "- Create dictionary with size limit"
 $ZSTD --train-cover=steps=8 *.c ../programs/*.c -o tmpDict2 --maxdict=4K
 rm tmp*
 
-$ECHO "\n**** legacy dictionary tests **** "
+$ECHO "\n===>  legacy dictionary builder "
 
 TESTFILE=../programs/zstdcli.c
 ./datagen > tmpDict
@@ -380,7 +380,7 @@ $ZSTD --train-legacy -s9 *.c ../programs/*.c -o tmpDict2 --maxdict=4K
 rm tmp*
 
 
-$ECHO "\n**** integrity tests **** "
+$ECHO "\n===>  integrity tests "
 
 $ECHO "test one file (tmp1.zst) "
 ./datagen > tmp1
@@ -405,13 +405,13 @@ $ZSTD -t tmpSplit.* && die "bad file not detected !"
 
 
 
-$ECHO "\n**** golden files tests **** "
+$ECHO "\n===>  golden files tests "
 
 $ZSTD -t -r files
 $ZSTD -c -r files | $ZSTD -t
 
 
-$ECHO "\n**** benchmark mode tests **** "
+$ECHO "\n===>  benchmark mode tests "
 
 $ECHO "bench one file"
 ./datagen > tmp1
@@ -422,7 +422,7 @@ $ECHO "with recursive and quiet modes"
 $ZSTD -rqi1b1e2 tmp1
 
 
-$ECHO "\n**** gzip compatibility tests **** "
+$ECHO "\n===>  gzip compatibility tests "
 
 GZIPMODE=1
 $ZSTD --format=gzip -V || GZIPMODE=0
@@ -445,7 +445,7 @@ else
 fi
 
 
-$ECHO "\n**** gzip frame tests **** "
+$ECHO "\n===>  gzip frame tests "
 
 if [ $GZIPMODE -eq 1 ]; then
     ./datagen > tmp
@@ -459,7 +459,7 @@ else
 fi
 
 
-$ECHO "\n**** xz compatibility tests **** "
+$ECHO "\n===>  xz compatibility tests "
 
 LZMAMODE=1
 $ZSTD --format=xz -V || LZMAMODE=0
@@ -505,7 +505,7 @@ else
 fi
 
 
-$ECHO "\n**** xz frame tests **** "
+$ECHO "\n===>  xz frame tests "
 
 if [ $LZMAMODE -eq 1 ]; then
     ./datagen > tmp
@@ -520,7 +520,7 @@ else
     $ECHO "xz mode not supported"
 fi
 
-$ECHO "\n**** lz4 compatibility tests **** "
+$ECHO "\n===>  lz4 compatibility tests "
 
 LZ4MODE=1
 $ZSTD --format=lz4 -V || LZ4MODE=0
@@ -543,7 +543,7 @@ else
 fi
 
 
-$ECHO "\n**** lz4 frame tests **** "
+$ECHO "\n===>  lz4 frame tests "
 
 if [ $LZ4MODE -eq 1 ]; then
     ./datagen > tmp
@@ -556,7 +556,7 @@ else
     $ECHO "lz4 mode not supported"
 fi
 
-$ECHO "\n**** zstd round-trip tests **** "
+$ECHO "\n===>  zstd round-trip tests "
 
 roundTripTest
 roundTripTest -g15K       # TableID==3
@@ -569,7 +569,7 @@ roundTripTest -g516K 19   # btopt
 
 fileRoundTripTest -g500K
 
-$ECHO "\n**** zstd long distance matching round-trip tests **** "
+$ECHO "\n===>  zstd long distance matching round-trip tests "
 roundTripTest -g0 "2 --long"
 roundTripTest -g1000K "1 --long"
 roundTripTest -g517K "6 --long"
@@ -580,21 +580,21 @@ fileRoundTripTest -g5M "3 --long"
 
 if [ -n "$hasMT" ]
 then
-    $ECHO "\n**** zstdmt round-trip tests **** "
+    $ECHO "\n===>  zstdmt round-trip tests "
     roundTripTest -g4M "1 -T0"
     roundTripTest -g8M "3 -T2"
     roundTripTest -g8000K "2 --threads=2"
     fileRoundTripTest -g4M "19 -T2 -B1M"
 
-    $ECHO "\n**** zstdmt long distance matching round-trip tests **** "
+    $ECHO "\n===>  zstdmt long distance matching round-trip tests "
     roundTripTest -g8M "3 --long -T2"
 else
-    $ECHO "\n**** no multithreading, skipping zstdmt tests **** "
+    $ECHO "\n===>  no multithreading, skipping zstdmt tests "
 fi
 
 rm tmp*
 
-$ECHO "\n**** zstd --list/-l single frame tests ****"
+$ECHO "\n===>  zstd --list/-l single frame tests "
 ./datagen > tmp1
 ./datagen > tmp2
 ./datagen > tmp3
@@ -604,19 +604,19 @@ $ZSTD -lv *.zst | grep "Decompressed Size:"  # check that decompressed size is p
 $ZSTD --list *.zst
 $ZSTD --list -v *.zst
 
-$ECHO "\n**** zstd --list/-l multiple frame tests ****"
+$ECHO "\n===>  zstd --list/-l multiple frame tests "
 cat tmp1.zst tmp2.zst > tmp12.zst
 cat tmp12.zst tmp3.zst > tmp123.zst
 $ZSTD -l *.zst
 $ZSTD -lv *.zst
 
-$ECHO "\n**** zstd --list/-l error detection tests ****"
+$ECHO "\n===>  zstd --list/-l error detection tests "
 ! $ZSTD -l tmp1 tmp1.zst
 ! $ZSTD --list tmp*
 ! $ZSTD -lv tmp1*
 ! $ZSTD --list -v tmp2 tmp12.zst
 
-$ECHO "\n**** zstd --list/-l test with null files ****"
+$ECHO "\n===>  zstd --list/-l test with null files "
 ./datagen -g0 > tmp5
 $ZSTD tmp5
 $ZSTD -l tmp5.zst
@@ -624,12 +624,12 @@ $ZSTD -l tmp5.zst
 $ZSTD -lv tmp5.zst
 ! $ZSTD -lv tmp5*
 
-$ECHO "\n**** zstd --list/-l test with no content size field ****"
+$ECHO "\n===>  zstd --list/-l test with no content size field "
 ./datagen -g513K | $ZSTD > tmp6.zst
 $ZSTD -l tmp6.zst
 ! $ZSTD -lv tmp6.zst | grep "Decompressed Size:"  # must NOT be present in header
 
-$ECHO "\n**** zstd --list/-l test with no checksum ****"
+$ECHO "\n===>   zstd --list/-l test with no checksum "
 $ZSTD -f --no-check tmp1
 $ZSTD -l tmp1.zst
 $ZSTD -lv tmp1.zst
@@ -637,7 +637,7 @@ $ZSTD -lv tmp1.zst
 rm tmp*
 
 
-$ECHO "\n**** zstd long distance matching tests **** "
+$ECHO "\n===>   zstd long distance matching tests "
 roundTripTest -g0 " --long"
 roundTripTest -g9M "2 --long"
 # Test parameter parsing
@@ -652,7 +652,7 @@ if [ "$1" != "--test-large-data" ]; then
     exit 0
 fi
 
-$ECHO "\n**** large files tests **** "
+$ECHO "\n===>   large files tests "
 
 roundTripTest -g270000000 1
 roundTripTest -g250000000 2
@@ -683,7 +683,7 @@ roundTripTest -g5000000000 -P99 1
 fileRoundTripTest -g4193M -P99 1
 
 
-$ECHO "\n**** zstd long, long distance matching round-trip tests **** "
+$ECHO "\n===>   zstd long, long distance matching round-trip tests "
 roundTripTest -g270000000 "1 --long"
 roundTripTest -g130000000 -P60 "5 --long"
 roundTripTest -g35000000 -P70 "8 --long"
@@ -695,7 +695,7 @@ roundTripTest -g600M -P50 "1 --long --zstd=wlog=29,clog=28"
 
 if [ -n "$hasMT" ]
 then
-    $ECHO "\n**** zstdmt long round-trip tests **** "
+    $ECHO "\n===>   zstdmt long round-trip tests "
     roundTripTest -g80000000 -P99 "19 -T2" " "
     roundTripTest -g5000000000 -P99 "1 -T2" " "
     roundTripTest -g500000000 -P97 "1 -T999" " "

From 13bfe885aa72eadbeed15eaa86f36c265fa68c7b Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 16 Oct 2017 14:06:22 -0700
Subject: [PATCH 11/12] edited ZSTD_initCStream_advanced() comment

---
 lib/compress/zstd_compress.c | 3 ++-
 lib/zstd.h                   | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index c7872e7be..35fa99e6f 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2575,7 +2575,8 @@ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict)
 }
 
 /* ZSTD_initCStream_advanced() :
- * pledgedSrcSize is optional: it can be 0, which means "unknown" if contentSizeFlag is set, or "unknown" if contentSizeFlag is not set. */
+ * pledgedSrcSize : if srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN.
+ * dict is loaded with default parameters ZSTD_dm_auto and ZSTD_dlm_byCopy. */
 size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
                                  const void* dict, size_t dictSize,
                                  ZSTD_parameters params, unsigned long long pledgedSrcSize)
diff --git a/lib/zstd.h b/lib/zstd.h
index e9c4ff9d4..4681d3f2a 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -734,7 +734,7 @@ ZSTDLIB_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspa
 ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);   /**< pledgedSrcSize must be correct. If it is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, a size of 0 is interepreted as "unknown". But it may change in some future version to mean "empty". */
 ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.*/
 ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
-                                             ZSTD_parameters params, unsigned long long pledgedSrcSize);  /**< pledgedSrcSize : If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. */
+                                             ZSTD_parameters params, unsigned long long pledgedSrcSize);  /**< pledgedSrcSize : if srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. */
 ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);  /**< note : cdict will just be referenced, and must outlive compression session */
 ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize);  /**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters */
 

From 32c9f715ae1532d4ea06ee5f811ed61c46e48d31 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 17 Oct 2017 14:07:43 -0700
Subject: [PATCH 12/12] fixed : Visual build compressing stdin with
 multi-threading enabled fails

It was multiple reasons stacked :
- Visual use a different code path, because ZSTD_NEWAPI is not defined
- fileio.c sends `0` as `pledgedSrcSize` to mean `ZSTD_CONTENTSIZE_UNKNOWN`  (fixed)
- ZSTDMT_resetCCtx() interpreted `0` as "empty" instead of "unknown" (fixed)
---
 lib/compress/zstdmt_compress.c | 5 ++++-
 lib/compress/zstdmt_compress.h | 2 +-
 programs/Makefile              | 4 ++--
 programs/fileio.c              | 6 +++---
 4 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 197a2c49e..af7be3167 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -838,9 +838,12 @@ size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx,
 
 
 /* ZSTDMT_resetCStream() :
- * pledgedSrcSize is optional and can be zero == unknown */
+ * pledgedSrcSize can be zero == unknown (for the time being)
+ * prefer using ZSTD_CONTENTSIZE_UNKNOWN,
+ * as `0` might mean "empty" in the future */
 size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* zcs, unsigned long long pledgedSrcSize)
 {
+    if (!pledgedSrcSize) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;
     if (zcs->params.nbThreads==1)
         return ZSTD_resetCStream(zcs->cctxPool->cctx[0], pledgedSrcSize);
     return ZSTDMT_initCStream_internal(zcs, NULL, 0, ZSTD_dm_auto, 0, zcs->params,
diff --git a/lib/compress/zstdmt_compress.h b/lib/compress/zstdmt_compress.h
index 8c59c684f..a7ce7c7dd 100644
--- a/lib/compress/zstdmt_compress.h
+++ b/lib/compress/zstdmt_compress.h
@@ -50,7 +50,7 @@ ZSTDLIB_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
 /* ===   Streaming functions   === */
 
 ZSTDLIB_API size_t ZSTDMT_initCStream(ZSTDMT_CCtx* mtctx, int compressionLevel);
-ZSTDLIB_API size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* mtctx, unsigned long long pledgedSrcSize);    /**< pledgedSrcSize is optional and can be zero == unknown */
+ZSTDLIB_API size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* mtctx, unsigned long long pledgedSrcSize);  /**< if srcSize is not known at reset time, use ZSTD_CONTENTSIZE_UNKNOWN. Note: for compatibility with older programs, 0 means the same as ZSTD_CONTENTSIZE_UNKNOWN, but it may change in the future, to mean "empty" */
 
 ZSTDLIB_API size_t ZSTDMT_compressStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
 
diff --git a/programs/Makefile b/programs/Makefile
index cbb67e9fe..1b21281b4 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -37,8 +37,8 @@ endif
 
 CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
            -I$(ZSTDDIR)/dictBuilder \
-           -DZSTD_NEWAPI \
-           -DXXH_NAMESPACE=ZSTD_   # because xxhash.o already compiled with this macro from library
+           -DXXH_NAMESPACE=ZSTD_ \
+           -DZSTD_NEWAPI
 CFLAGS  ?= -O3
 DEBUGFLAGS+=-Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
             -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
diff --git a/programs/fileio.c b/programs/fileio.c
index 17ad31b32..03e653893 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -817,11 +817,11 @@ static int FIO_compressFilename_internal(cRess_t ress,
     /* init */
 #ifdef ZSTD_NEWAPI
     if (fileSize!=0)  /* when src is stdin, fileSize==0, but is effectively unknown */
-        ZSTD_CCtx_setPledgedSrcSize(ress.cctx, fileSize);  /* note : fileSize==0 means "empty" */
+        ZSTD_CCtx_setPledgedSrcSize(ress.cctx, fileSize);
 #elif defined(ZSTD_MULTITHREAD)
-    CHECK( ZSTDMT_resetCStream(ress.cctx, fileSize) );   /* note : fileSize==0 means "unknown" */
+    CHECK( ZSTDMT_resetCStream(ress.cctx, fileSize ? fileSize : ZSTD_CONTENTSIZE_UNKNOWN) );
 #else
-    CHECK( ZSTD_resetCStream(ress.cctx, fileSize) );   /* note : fileSize==0 means "unknown" */
+    CHECK( ZSTD_resetCStream(ress.cctx, fileSize ? fileSize : ZSTD_CONTENTSIZE_UNKNOWN) );
 #endif
 
     /* Main compression loop */