From fecc721fd9b88bdfbc0619f81a092d8f0ec6a4c0 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 27 Jun 2017 11:46:39 -0700 Subject: [PATCH 1/8] added parameter ZSTD_p_refDictContent --- doc/zstd_manual.html | 20 +++++++++----------- lib/compress/zstd_compress.c | 19 ++++++++++--------- lib/zstd.h | 20 +++++++++----------- tests/zstreamtest.c | 3 ++- 4 files changed, 30 insertions(+), 32 deletions(-) diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html index 6d524ae32..aac3ec383 100644 --- a/doc/zstd_manual.html +++ b/doc/zstd_manual.html @@ -816,13 +816,11 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); ZSTD_p_checksumFlag, /* A 32-bits checksum of content is written at end of frame (default:0) */ ZSTD_p_dictIDFlag, /* When applicable, dictID of dictionary is provided in frame header (default:1) */ - /* dictionary parameters */ - ZSTD_p_refDictContent=300, /* Content of dictionary content will be referenced, instead of copied (default:0). - * This avoids duplicating dictionary content. - * But it also requires that dictionary buffer outlives its users */ - /* Not ready yet ! <=================================== */ - ZSTD_p_dictMode, /* Select how dictionary must be interpreted. Value must be from type ZSTD_dictMode_e. + /* dictionary parameters (must be set before loading) */ + ZSTD_p_dictMode=300, /* Select how dictionary content must be interpreted. Value must be from type ZSTD_dictMode_e. * default : 0==auto : dictionary will be "full" if it respects specification, otherwise it will be "rawContent" */ + ZSTD_p_refDictContent, /* Dictionary content will be referenced, instead of copied (default:0==byCopy). + * It requires that dictionary buffer outlives its users */ /* multi-threading parameters */ ZSTD_p_nbThreads=400, /* Select how many threads a compression job can spawn (default:1) @@ -837,9 +835,9 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); * 0 => no overlap, 6(default) => use 1/8th of windowSize, >=9 => use full windowSize */ /* advanced parameters - may not remain available after API update */ - ZSTD_p_forceMaxWindow=1100, /* Force back-references to remain < windowSize, - * even when referencing into Dictionary content. - * default : 0 when using a CDict, 1 when using a Prefix */ + ZSTD_p_forceMaxWindow=1100, /* Force back-reference distances to remain < windowSize, + * even when referencing into Dictionary content (default:0) */ + } ZSTD_cParameter; 
size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value);
@@ -865,8 +863,8 @@ void   ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
  @result : 0, or an error code (which can be tested with ZSTD_isError()).
   Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
             meaning "return to no-dictionary mode".
-  Note 1 : Dictionary content will be copied internally,
-           except if ZSTD_p_refDictContent is set.
+  Note 1 : `dict` content will be copied internally,
+           except if ZSTD_p_refDictContent is set before loading.
   Note 2 : Loading a dictionary involves building tables, which are dependent on compression parameters.
            For this reason, compression parameters cannot be changed anymore after loading a dictionary.
            It's also a CPU-heavy operation, with non-negligible impact on latency.
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index ea37197ff..b95b4770f 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -89,6 +89,7 @@ struct ZSTD_CCtx_s {
     U32   loadedDictEnd;    /* index of end of dictionary */
     U32   forceWindow;      /* force back-references to respect limit of 1<requestedParams.cParams.strategy = (ZSTD_strategy)value;
         return 0;
 
-#if 0
-    case ZSTD_p_windowSize :   /* to be done later */
-        return ERROR(compressionParameter_unsupported);
-#endif
-
     case ZSTD_p_contentSizeFlag :
         DEBUGLOG(5, "set content size flag = %u", (value>0));
         /* Content size written in frame header _when known_ (default:1) */
@@ -343,10 +339,9 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
         cctx->requestedParams.fParams.noDictIDFlag = (value==0);
         return 0;
 
-    case ZSTD_p_refDictContent :   /* to be done later */
-        return ERROR(compressionParameter_unsupported);
-
+    /* Dictionary parameters */
     case ZSTD_p_dictMode :
+        if (cctx->cdict) return ERROR(stage_wrong);  /* must be set before loading */
         /* restrict dictionary mode, to "rawContent" or "fullDict" only */
         ZSTD_STATIC_ASSERT((U32)ZSTD_dm_fullDict > (U32)ZSTD_dm_rawContent);
         if (value > (unsigned)ZSTD_dm_fullDict)
@@ -354,6 +349,12 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
         cctx->dictMode = (ZSTD_dictMode_e)value;
         return 0;
 
+    case ZSTD_p_refDictContent :
+        if (cctx->cdict) return ERROR(stage_wrong);  /* must be set before loading */
+        /* dictionary content will be referenced, instead of copied */
+        cctx->dictContentByRef = value>0;
+        return 0;
+
     case ZSTD_p_forceMaxWindow :  /* Force back-references to remain < windowSize,
                                    * even when referencing into Dictionary content
                                    * default : 0 when using a CDict, 1 when using a Prefix */
@@ -417,7 +418,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, s
                 ZSTD_getCParams(cctx->compressionLevel, 0, dictSize);
         cctx->cdictLocal = ZSTD_createCDict_advanced(
                                 dict, dictSize,
-                                0 /* byReference */, cctx->dictMode,
+                                cctx->dictContentByRef, cctx->dictMode,
                                 cParams, cctx->customMem);
         cctx->cdict = cctx->cdictLocal;
         if (cctx->cdictLocal == NULL)
diff --git a/lib/zstd.h b/lib/zstd.h
index 08358f0df..49a5b791c 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -956,13 +956,11 @@ typedef enum {
     ZSTD_p_checksumFlag,     /* A 32-bits checksum of content is written at end of frame (default:0) */
     ZSTD_p_dictIDFlag,       /* When applicable, dictID of dictionary is provided in frame header (default:1) */
 
-    /* dictionary parameters */
-    ZSTD_p_refDictContent=300, /* Content of dictionary content will be referenced, instead of copied (default:0).
-                              * This avoids duplicating dictionary content.
-                              * But it also requires that dictionary buffer outlives its users */
-                             /* Not ready yet ! <=================================== */
-    ZSTD_p_dictMode,         /* Select how dictionary must be interpreted. Value must be from type ZSTD_dictMode_e.
+    /* dictionary parameters (must be set before loading) */
+    ZSTD_p_dictMode=300,     /* Select how dictionary content must be interpreted. Value must be from type ZSTD_dictMode_e.
                               * default : 0==auto : dictionary will be "full" if it respects specification, otherwise it will be "rawContent" */
+    ZSTD_p_refDictContent,   /* Dictionary content will be referenced, instead of copied (default:0==byCopy).
+                              * It requires that dictionary buffer outlives its users */
 
     /* multi-threading parameters */
     ZSTD_p_nbThreads=400,    /* Select how many threads a compression job can spawn (default:1)
@@ -977,9 +975,9 @@ typedef enum {
                               * 0 => no overlap, 6(default) => use 1/8th of windowSize, >=9 => use full windowSize */
 
     /* advanced parameters - may not remain available after API update */
-    ZSTD_p_forceMaxWindow=1100, /* Force back-references to remain < windowSize,
-                              * even when referencing into Dictionary content.
-                              * default : 0 when using a CDict, 1 when using a Prefix */
+    ZSTD_p_forceMaxWindow=1100, /* Force back-reference distances to remain < windowSize,
+                              * even when referencing into Dictionary content (default:0) */
+
 } ZSTD_cParameter;
 
 
@@ -1006,8 +1004,8 @@ ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long lo
  * @result : 0, or an error code (which can be tested with ZSTD_isError()).
  *  Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
  *            meaning "return to no-dictionary mode".
- *  Note 1 : Dictionary content will be copied internally,
- *           except if ZSTD_p_refDictContent is set.
+ *  Note 1 : `dict` content will be copied internally,
+ *           except if ZSTD_p_refDictContent is set before loading.
  *  Note 2 : Loading a dictionary involves building tables, which are dependent on compression parameters.
  *           For this reason, compression parameters cannot be changed anymore after loading a dictionary.
  *           It's also a CPU-heavy operation, with non-negligible impact on latency.
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index aae83972a..ca25e891f 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -1294,7 +1294,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
                 ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, pledgedSrcSize, dictSize);
 
                 /* mess with compression parameters */
-                CHECK_Z( ZSTD_CCtx_loadDictionary(zc, NULL, 0) );   /* always cancel previous dict, to make user it's possible to pass compression parameters */
+                CHECK_Z( ZSTD_CCtx_loadDictionary(zc, NULL, 0) );   /* cancel previous dict, to allow new compression parameters */
                 cParams.windowLog += (FUZ_rand(&lseed) & 3) - 1;
                 cParams.hashLog += (FUZ_rand(&lseed) & 3) - 1;
                 cParams.chainLog += (FUZ_rand(&lseed) & 3) - 1;
@@ -1311,6 +1311,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
                 if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_targetLength, cParams.targetLength) );
 
                 /* unconditionally set, to be sync with decoder */
+                if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_refDictContent, FUZ_rand(&lseed) & 1) );
                 CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) );
 
                 if (dict && dictSize) {

From 7d3816183f5332eb2de75ef14ac9371173311bb7 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 27 Jun 2017 13:50:34 -0700
Subject: [PATCH 2/8] exposed ZSTD_MAGIC_DICTIONARY in zstd.h

makes it easier to explain ZSTD_dictMode
---
 doc/zstd_manual.html             |  9 ++++++---
 lib/common/zstd_internal.h       |  1 -
 lib/compress/zstd_compress.c     |  2 +-
 lib/decompress/zstd_decompress.c |  6 +++---
 lib/dictBuilder/zdict.c          |  6 +++---
 lib/zstd.h                       | 10 +++++++---
 6 files changed, 20 insertions(+), 14 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index aac3ec383..69b6fbf63 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -348,7 +348,7 @@ static const ZSTD_customMem ZSTD_defaultCMem = { NULL, NULL, NULL };
 
size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
 
  `src` should point to the start of a ZSTD encoded frame or skippable frame
   `srcSize` must be at least as large as the frame
-  @return : the compressed size of the frame pointed to by `src`,
+  @return : the compressed size of the frame pointed to by `src`,FUZ_rand(&lseed) & 1)
             suitable to pass to `ZSTD_decompress` or similar,
             or an error code if given invalid input. 
 


@@ -482,7 +482,10 @@ size_t ZSTD_estimateDDictSize(size_t dictSize, unsigned byReference);
   It is important that dictBuffer outlives CDict, it must remain read accessible throughout the lifetime of CDict

-
typedef enum { ZSTD_dm_auto=0, ZSTD_dm_rawContent, ZSTD_dm_fullDict } ZSTD_dictMode_e;
+
typedef enum { ZSTD_dm_auto=0,        /* dictionary is "full" if it starts with ZSTD_MAGIC_DICTIONARY, rawContent otherwize */
+               ZSTD_dm_rawContent,    /* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */
+               ZSTD_dm_fullDict       /* refuses to load a dictionary if it does not respect Zstandard's specification */
+} ZSTD_dictMode_e;
 


 
ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize,
                                       unsigned byReference, ZSTD_dictMode_e dictMode,
@@ -886,7 +889,7 @@ void   ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
 


 
 
size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize);   /* Not ready yet ! <===================================== */
-
  Reference a prefix (content-only dictionary) to bootstrap next compression job.
+
  Reference a prefix (raw-content dictionary) for next compression job.
   Decompression will have to use same prefix.
   Prefix is only used once. Tables are discarded at end of compression job.
   If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_CDict.
diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index 4e216d728..bcf93e58d 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -98,7 +98,6 @@
 *  Common constants
 ***************************************/
 #define ZSTD_OPT_NUM    (1<<12)
-#define ZSTD_DICT_MAGIC  0xEC30A437   /* v0.7+ */
 
 #define ZSTD_REP_NUM      3                 /* number of repcodes */
 #define ZSTD_REP_CHECK    (ZSTD_REP_NUM)    /* number of repcodes to check by the optimal parser */
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index b95b4770f..b6b92f32c 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -3181,7 +3181,7 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* cctx,
     if (dictMode==ZSTD_dm_rawContent)
         return ZSTD_loadDictionaryContent(cctx, dict, dictSize);
 
-    if (MEM_readLE32(dict) != ZSTD_DICT_MAGIC) {
+    if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) {
         if (dictMode == ZSTD_dm_auto) {
             DEBUGLOG(5, "raw content dictionary detected");
             return ZSTD_loadDictionaryContent(cctx, dict, dictSize);
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index e823c3dbf..61f766e21 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -1884,7 +1884,7 @@ static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict
 {
     if (dictSize < 8) return ZSTD_refDictContent(dctx, dict, dictSize);
     {   U32 const magic = MEM_readLE32(dict);
-        if (magic != ZSTD_DICT_MAGIC) {
+        if (magic != ZSTD_MAGIC_DICTIONARY) {
             return ZSTD_refDictContent(dctx, dict, dictSize);   /* pure content mode */
     }   }
     dctx->dictID = MEM_readLE32((const char*)dict + 4);
@@ -1964,7 +1964,7 @@ static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict* ddict)
     ddict->entropyPresent = 0;
     if (ddict->dictSize < 8) return 0;
     {   U32 const magic = MEM_readLE32(ddict->dictContent);
-        if (magic != ZSTD_DICT_MAGIC) return 0;   /* pure content mode */
+        if (magic != ZSTD_MAGIC_DICTIONARY) return 0;   /* pure content mode */
     }
     ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + 4);
 
@@ -2083,7 +2083,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)
 unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize)
 {
     if (dictSize < 8) return 0;
-    if (MEM_readLE32(dict) != ZSTD_DICT_MAGIC) return 0;
+    if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) return 0;
     return MEM_readLE32((const char*)dict + 4);
 }
 
diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c
index 943ddde0f..04bfe661d 100644
--- a/lib/dictBuilder/zdict.c
+++ b/lib/dictBuilder/zdict.c
@@ -94,7 +94,7 @@ const char* ZDICT_getErrorName(size_t errorCode) { return ERR_getErrorName(error
 unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize)
 {
     if (dictSize < 8) return 0;
-    if (MEM_readLE32(dictBuffer) != ZSTD_DICT_MAGIC) return 0;
+    if (MEM_readLE32(dictBuffer) != ZSTD_MAGIC_DICTIONARY) return 0;
     return MEM_readLE32((const char*)dictBuffer + 4);
 }
 
@@ -865,7 +865,7 @@ size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity,
     if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) return ERROR(dstSize_tooSmall);
 
     /* dictionary header */
-    MEM_writeLE32(header, ZSTD_DICT_MAGIC);
+    MEM_writeLE32(header, ZSTD_MAGIC_DICTIONARY);
     {   U64 const randomID = XXH64(customDictContent, dictContentSize, 0);
         U32 const compliantID = (randomID % ((1U<<31)-32768)) + 32768;
         U32 const dictID = params.dictID ? params.dictID : compliantID;
@@ -917,7 +917,7 @@ size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictCo
     }
 
     /* add dictionary header (after entropy tables) */
-    MEM_writeLE32(dictBuffer, ZSTD_DICT_MAGIC);
+    MEM_writeLE32(dictBuffer, ZSTD_MAGIC_DICTIONARY);
     {   U64 const randomID = XXH64((char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize, 0);
         U32 const compliantID = (randomID % ((1U<<31)-32768)) + 32768;
         U32 const dictID = params.dictID ? params.dictID : compliantID;
diff --git a/lib/zstd.h b/lib/zstd.h
index 49a5b791c..2b9018707 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -360,6 +360,7 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void);   /*!< recommended size for output
 /* --- Constants ---*/
 #define ZSTD_MAGICNUMBER            0xFD2FB528   /* >= v0.8.0 */
 #define ZSTD_MAGIC_SKIPPABLE_START  0x184D2A50U
+#define ZSTD_MAGIC_DICTIONARY       0xEC30A437   /* v0.7+ */
 
 #define ZSTD_WINDOWLOG_MAX_32  27
 #define ZSTD_WINDOWLOG_MAX_64  27
@@ -379,7 +380,7 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void);   /*!< recommended size for output
 
 #define ZSTD_FRAMEHEADERSIZE_MAX 18    /* for static allocation */
 #define ZSTD_FRAMEHEADERSIZE_MIN  6
-static const size_t ZSTD_frameHeaderSize_prefix = 5;
+static const size_t ZSTD_frameHeaderSize_prefix = 5;  /* minimum input size to know frame header size */
 static const size_t ZSTD_frameHeaderSize_min = ZSTD_FRAMEHEADERSIZE_MIN;
 static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX;
 static const size_t ZSTD_skippableHeaderSize = 8;  /* magic number + skippable frame length */
@@ -573,7 +574,10 @@ ZSTDLIB_API size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter par
 ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
 
 
-typedef enum { ZSTD_dm_auto=0, ZSTD_dm_rawContent, ZSTD_dm_fullDict } ZSTD_dictMode_e;
+typedef enum { ZSTD_dm_auto=0,        /* dictionary is "full" if it starts with ZSTD_MAGIC_DICTIONARY, rawContent otherwize */
+               ZSTD_dm_rawContent,    /* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */
+               ZSTD_dm_fullDict       /* refuses to load a dictionary if it does not respect Zstandard's specification */
+} ZSTD_dictMode_e;
 /*! ZSTD_createCDict_advanced() :
  *  Create a ZSTD_CDict using external alloc and free, and customized compression parameters */
 ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize,
@@ -1027,7 +1031,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, s
 ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
 
 /*! ZSTD_CCtx_refPrefix() :
- *  Reference a prefix (content-only dictionary) to bootstrap next compression job.
+ *  Reference a prefix (raw-content dictionary) for next compression job.
  *  Decompression will have to use same prefix.
  *  Prefix is only used once. Tables are discarded at end of compression job.
  *  If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_CDict.

From b7372933b833ec0d69be2f8a9589fe82fbeb8b96 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 27 Jun 2017 15:49:12 -0700
Subject: [PATCH 3/8] implemented ZSTD_refPrefix()

---
 doc/zstd_manual.html         | 29 ++++++++-----------
 lib/compress/zstd_compress.c | 56 ++++++++++++++++++++++--------------
 lib/zstd.h                   | 27 +++++++----------
 tests/zstreamtest.c          |  6 ++--
 4 files changed, 61 insertions(+), 57 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index 69b6fbf63..f5a1394a7 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -348,7 +348,7 @@ static const ZSTD_customMem ZSTD_defaultCMem = { NULL, NULL, NULL };
 
size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
 
  `src` should point to the start of a ZSTD encoded frame or skippable frame
   `srcSize` must be at least as large as the frame
-  @return : the compressed size of the frame pointed to by `src`,FUZ_rand(&lseed) & 1)
+  @return : the compressed size of the frame pointed to by `src`,
             suitable to pass to `ZSTD_decompress` or similar,
             or an error code if given invalid input. 
 


@@ -806,20 +806,13 @@ void   ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
                               * The higher the value of selected strategy, the more complex it is,
                               * resulting in stronger and slower compression.
                               * Special: value 0 means "do not change strategy". */
-#if 0
-    ZSTD_p_windowSize,       /* Maximum allowed back-reference distance.
-                              * Can be set to a more precise value than windowLog.
-                              * Will be transparently reduced to closest possible inferior value
-                              * (see Zstandard compression format) */
-                             /* Not ready yet ! */
-#endif
 
     /* frame parameters */
     ZSTD_p_contentSizeFlag=200, /* Content size is written into frame header _whenever known_ (default:1) */
     ZSTD_p_checksumFlag,     /* A 32-bits checksum of content is written at end of frame (default:0) */
     ZSTD_p_dictIDFlag,       /* When applicable, dictID of dictionary is provided in frame header (default:1) */
 
-    /* dictionary parameters (must be set before loading) */
+    /* dictionary parameters (must be set before ZSTD_CCtx_loadDictionary) */
     ZSTD_p_dictMode=300,     /* Select how dictionary content must be interpreted. Value must be from type ZSTD_dictMode_e.
                               * default : 0==auto : dictionary will be "full" if it respects specification, otherwise it will be "rawContent" */
     ZSTD_p_refDictContent,   /* Dictionary content will be referenced, instead of copied (default:0==byCopy).
@@ -876,7 +869,7 @@ void   ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
 


 
 
size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
-
  Ref a prepared dictionary, to be used for all next compression jobs.
+
  Reference a prepared dictionary, to be used for all next compression jobs.
   Note that compression parameters are enforced from within CDict,
   and supercede any compression parameter previously set within CCtx.
   The dictionary will remain valid for future compression jobs using same CCtx.
@@ -888,16 +881,18 @@ void   ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
  
 


 
-
size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize);   /* Not ready yet ! <===================================== */
-
  Reference a prefix (raw-content dictionary) for next compression job.
-  Decompression will have to use same prefix.
-  Prefix is only used once. Tables are discarded at end of compression job.
-  If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_CDict.
+
size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize);
+
  Reference a prefix (single-usage dictionary) for next compression job.
+  Decompression need same prefix to properly regenerate data.
+  Prefix is **only used once**. Tables are discarded at end of compression job.
+  Subsequent compression jobs will be done without prefix (if none is explicitly referenced).
+  If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_CDict instead.
  @result : 0, or an error code (which can be tested with ZSTD_isError()).
-  Special : Adding a NULL (or 0-size) dictionary invalidates any previous prefix, meaning "return to no-dictionary mode".
+  Special : Adding any prefix (including NULL) invalidates any previous prefix or dictionary
   Note 1 : Prefix buffer is referenced. It must outlive compression job.
   Note 2 : Referencing a prefix involves building tables, which are dependent on compression parameters.
-           It's a CPU-heavy operation, with non-negligible impact on latency. 
+           It's a CPU-heavy operation, with non-negligible impact on latency.
+  Note 3 : it's possible to alter ZSTD_p_dictMode using ZSTD_CCtx_setParameter() 
 


 
 
typedef enum {
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index b6b92f32c..93400c55c 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -88,8 +88,6 @@ struct ZSTD_CCtx_s {
     U32   hashLog3;         /* dispatch table : larger == faster, more memory */
     U32   loadedDictEnd;    /* index of end of dictionary */
     U32   forceWindow;      /* force back-references to respect limit of 1<streamStage != zcss_init) return ERROR(stage_wrong);
-    return ERROR(compressionParameter_unsupported);
-}
-
 size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
 {
     if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
     cctx->cdict = cdict;
-    return ERROR(compressionParameter_unsupported);
+    cctx->prefix = NULL;   /* exclusive */
+    cctx->prefixSize = 0;
+    return 0;
+}
+
+size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize)
+{
+    if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
+    cctx->cdict = NULL;   /* prefix discards any prior cdict */
+    cctx->prefix = prefix;
+    cctx->prefixSize = prefixSize;
+    return 0;
 }
 
 static void ZSTD_startNewCompression(ZSTD_CCtx* cctx)
@@ -3622,14 +3629,17 @@ size_t ZSTD_CStreamOutSize(void)
 }
 
 static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs,
-                                        ZSTD_parameters params,
-                                        unsigned long long pledgedSrcSize)
+                    const void* dict, size_t dictSize, const ZSTD_CDict* cdict,
+                    ZSTD_parameters params, unsigned long long pledgedSrcSize)
 {
     DEBUGLOG(5, "ZSTD_resetCStream_internal");
+    /* params are supposed to be fully validated at this point */
+    assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
+    assert(!((dict) && (cdict)));  /* either dict or cdict, not both */
 
     CHECK_F( ZSTD_compressBegin_internal(zcs,
-                                        NULL, 0, ZSTD_dm_auto,
-                                        zcs->cdict,
+                                        dict, dictSize, ZSTD_dm_auto,   /* <========= Todo : make dictMode controllable ! */
+                                        cdict,
                                         params, pledgedSrcSize,
                                         ZSTDb_buffered) );
 
@@ -3650,10 +3660,11 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
     if (zcs->compressionLevel != ZSTD_CLEVEL_CUSTOM) {
         params.cParams = ZSTD_getCParams(zcs->compressionLevel, pledgedSrcSize, 0 /* dictSize */);
     }
-    return ZSTD_resetCStream_internal(zcs, params, pledgedSrcSize);
+    return ZSTD_resetCStream_internal(zcs, NULL, 0, zcs->cdict, params, pledgedSrcSize);
 }
 
 /*! ZSTD_initCStream_internal() :
+ *  Note : not static, but hidden (not exposed). Used by zstdmt_compress.c
  *  Assumption 1 : params are valid
  *  Assumption 2 : either dict, or cdict, is defined, not both */
 size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
@@ -3672,7 +3683,7 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
         }
         ZSTD_freeCDict(zcs->cdictLocal);
         zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize,
-                                            0 /* byReference */, ZSTD_dm_auto,
+                                            0 /* byReference */, ZSTD_dm_auto,  /* <======== Todo : make dictMode controllable */
                                             params.cParams, zcs->customMem);
         zcs->cdict = zcs->cdictLocal;
         if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
@@ -3688,7 +3699,7 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
 
     zcs->requestedParams = params;
     zcs->compressionLevel = ZSTD_CLEVEL_CUSTOM;
-    return ZSTD_resetCStream_internal(zcs, params, pledgedSrcSize);
+    return ZSTD_resetCStream_internal(zcs, NULL, 0, zcs->cdict, params, pledgedSrcSize);
 }
 
 /* ZSTD_initCStream_usingCDict_advanced() :
@@ -3925,7 +3936,10 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
         } else
 #endif
         {
-            CHECK_F( ZSTD_resetCStream_internal(cctx, params, cctx->pledgedSrcSizePlusOne-1) );
+            const void* const prefix = cctx->prefix;
+            size_t const prefixSize = cctx->prefixSize;
+            cctx->prefix = NULL; cctx->prefixSize = 0;   /* single usage */
+            CHECK_F( ZSTD_resetCStream_internal(cctx, prefix, prefixSize, cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) );
     }   }
 
 #ifdef ZSTD_MULTITHREAD
diff --git a/lib/zstd.h b/lib/zstd.h
index 2b9018707..700a22f7b 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -947,20 +947,13 @@ typedef enum {
                               * The higher the value of selected strategy, the more complex it is,
                               * resulting in stronger and slower compression.
                               * Special: value 0 means "do not change strategy". */
-#if 0
-    ZSTD_p_windowSize,       /* Maximum allowed back-reference distance.
-                              * Can be set to a more precise value than windowLog.
-                              * Will be transparently reduced to closest possible inferior value
-                              * (see Zstandard compression format) */
-                             /* Not ready yet ! */
-#endif
 
     /* frame parameters */
     ZSTD_p_contentSizeFlag=200, /* Content size is written into frame header _whenever known_ (default:1) */
     ZSTD_p_checksumFlag,     /* A 32-bits checksum of content is written at end of frame (default:0) */
     ZSTD_p_dictIDFlag,       /* When applicable, dictID of dictionary is provided in frame header (default:1) */
 
-    /* dictionary parameters (must be set before loading) */
+    /* dictionary parameters (must be set before ZSTD_CCtx_loadDictionary) */
     ZSTD_p_dictMode=300,     /* Select how dictionary content must be interpreted. Value must be from type ZSTD_dictMode_e.
                               * default : 0==auto : dictionary will be "full" if it respects specification, otherwise it will be "rawContent" */
     ZSTD_p_refDictContent,   /* Dictionary content will be referenced, instead of copied (default:0==byCopy).
@@ -1018,7 +1011,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long lo
 ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
 
 /*! ZSTD_CCtx_refCDict() :
- *  Ref a prepared dictionary, to be used for all next compression jobs.
+ *  Reference a prepared dictionary, to be used for all next compression jobs.
  *  Note that compression parameters are enforced from within CDict,
  *  and supercede any compression parameter previously set within CCtx.
  *  The dictionary will remain valid for future compression jobs using same CCtx.
@@ -1031,16 +1024,18 @@ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, s
 ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
 
 /*! ZSTD_CCtx_refPrefix() :
- *  Reference a prefix (raw-content dictionary) for next compression job.
- *  Decompression will have to use same prefix.
- *  Prefix is only used once. Tables are discarded at end of compression job.
- *  If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_CDict.
+ *  Reference a prefix (single-usage dictionary) for next compression job.
+ *  Decompression need same prefix to properly regenerate data.
+ *  Prefix is **only used once**. Tables are discarded at end of compression job.
+ *  Subsequent compression jobs will be done without prefix (if none is explicitly referenced).
+ *  If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_CDict instead.
  * @result : 0, or an error code (which can be tested with ZSTD_isError()).
- *  Special : Adding a NULL (or 0-size) dictionary invalidates any previous prefix, meaning "return to no-dictionary mode".
+ *  Special : Adding any prefix (including NULL) invalidates any previous prefix or dictionary
  *  Note 1 : Prefix buffer is referenced. It must outlive compression job.
  *  Note 2 : Referencing a prefix involves building tables, which are dependent on compression parameters.
- *           It's a CPU-heavy operation, with non-negligible impact on latency. */
-ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize);   /* Not ready yet ! <===================================== */
+ *           It's a CPU-heavy operation, with non-negligible impact on latency.
+ *  Note 3 : it's possible to alter ZSTD_p_dictMode using ZSTD_CCtx_setParameter() */
+ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize);
 
 
 
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index ca25e891f..0f20d3253 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -1266,11 +1266,11 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
         }
 
         /* compression init */
+        CHECK_Z( ZSTD_CCtx_loadDictionary(zc, NULL, 0) );   /* cancel previous dict /*/
         if ((FUZ_rand(&lseed)&1) /* at beginning, to keep same nb of rand */
             && oldTestLog /* at least one test happened */ && resetAllowed) {
             maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2);
             if (maxTestSize >= srcBufferSize) maxTestSize = srcBufferSize-1;
-            CHECK_Z( ZSTD_CCtx_loadDictionary(zc, NULL, 0) );
             {   int const compressionLevel = (FUZ_rand(&lseed) % 5) + 1;
                 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_compressionLevel, compressionLevel) );
             }
@@ -1294,7 +1294,6 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
                 ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, pledgedSrcSize, dictSize);
 
                 /* mess with compression parameters */
-                CHECK_Z( ZSTD_CCtx_loadDictionary(zc, NULL, 0) );   /* cancel previous dict, to allow new compression parameters */
                 cParams.windowLog += (FUZ_rand(&lseed) & 3) - 1;
                 cParams.hashLog += (FUZ_rand(&lseed) & 3) - 1;
                 cParams.chainLog += (FUZ_rand(&lseed) & 3) - 1;
@@ -1312,7 +1311,8 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
 
                 /* unconditionally set, to be sync with decoder */
                 if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_refDictContent, FUZ_rand(&lseed) & 1) );
-                CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) );
+                if (FUZ_rand(&lseed) & 1) { CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) ); }
+                else { CHECK_Z( ZSTD_CCtx_refPrefix(zc, dict, dictSize) ); }
 
                 if (dict && dictSize) {
                     /* test that compression parameters are correctly rejected after setting a dictionary */

From 2e4274262dbb0c1b549a3f1effc334c38345b841 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 27 Jun 2017 17:09:12 -0700
Subject: [PATCH 4/8] controlled dictMode

---
 lib/compress/zstd_compress.c | 13 +++++++------
 tests/zstreamtest.c          |  2 +-
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 93400c55c..41be22edd 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -3629,7 +3629,8 @@ size_t ZSTD_CStreamOutSize(void)
 }
 
 static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs,
-                    const void* dict, size_t dictSize, const ZSTD_CDict* cdict,
+                    const void* dict, size_t dictSize, ZSTD_dictMode_e dictMode,
+                    const ZSTD_CDict* cdict,
                     ZSTD_parameters params, unsigned long long pledgedSrcSize)
 {
     DEBUGLOG(5, "ZSTD_resetCStream_internal");
@@ -3638,7 +3639,7 @@ static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs,
     assert(!((dict) && (cdict)));  /* either dict or cdict, not both */
 
     CHECK_F( ZSTD_compressBegin_internal(zcs,
-                                        dict, dictSize, ZSTD_dm_auto,   /* <========= Todo : make dictMode controllable ! */
+                                        dict, dictSize, dictMode,
                                         cdict,
                                         params, pledgedSrcSize,
                                         ZSTDb_buffered) );
@@ -3660,7 +3661,7 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
     if (zcs->compressionLevel != ZSTD_CLEVEL_CUSTOM) {
         params.cParams = ZSTD_getCParams(zcs->compressionLevel, pledgedSrcSize, 0 /* dictSize */);
     }
-    return ZSTD_resetCStream_internal(zcs, NULL, 0, zcs->cdict, params, pledgedSrcSize);
+    return ZSTD_resetCStream_internal(zcs, NULL, 0, zcs->dictMode, zcs->cdict, params, pledgedSrcSize);
 }
 
 /*! ZSTD_initCStream_internal() :
@@ -3683,7 +3684,7 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
         }
         ZSTD_freeCDict(zcs->cdictLocal);
         zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize,
-                                            0 /* byReference */, ZSTD_dm_auto,  /* <======== Todo : make dictMode controllable */
+                                            zcs->dictContentByRef, zcs->dictMode,
                                             params.cParams, zcs->customMem);
         zcs->cdict = zcs->cdictLocal;
         if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
@@ -3699,7 +3700,7 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
 
     zcs->requestedParams = params;
     zcs->compressionLevel = ZSTD_CLEVEL_CUSTOM;
-    return ZSTD_resetCStream_internal(zcs, NULL, 0, zcs->cdict, params, pledgedSrcSize);
+    return ZSTD_resetCStream_internal(zcs, NULL, 0, zcs->dictMode, zcs->cdict, params, pledgedSrcSize);
 }
 
 /* ZSTD_initCStream_usingCDict_advanced() :
@@ -3939,7 +3940,7 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
             const void* const prefix = cctx->prefix;
             size_t const prefixSize = cctx->prefixSize;
             cctx->prefix = NULL; cctx->prefixSize = 0;   /* single usage */
-            CHECK_F( ZSTD_resetCStream_internal(cctx, prefix, prefixSize, cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) );
+            CHECK_F( ZSTD_resetCStream_internal(cctx, prefix, prefixSize, cctx->dictMode, cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) );
     }   }
 
 #ifdef ZSTD_MULTITHREAD
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 0f20d3253..f6bb440ce 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -415,7 +415,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
 
     /* CDict scenario */
     DISPLAYLEVEL(3, "test%3i : digested dictionary : ", testNb++);
-    {   ZSTD_CDict* const cdict = ZSTD_createCDict(dictionary.start, dictionary.filled, 1);
+    {   ZSTD_CDict* const cdict = ZSTD_createCDict(dictionary.start, dictionary.filled, 1 /*byRef*/ );
         size_t const initError = ZSTD_initCStream_usingCDict(zc, cdict);
         if (ZSTD_isError(initError)) goto _output_error;
         cSize = 0;

From 1fd2df9c2cfacb84fe7340842ea80b7e8a97cdc1 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 28 Jun 2017 09:57:59 -0700
Subject: [PATCH 5/8] fixed zstreamtest

modifying compression parameters is not allowed after ZSTD_CCtx_loadDictionary()
but it is still allowed after ZSTD_refPrefix()
---
 tests/zstreamtest.c | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index f6bb440ce..4c4479557 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -1311,13 +1311,14 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
 
                 /* unconditionally set, to be sync with decoder */
                 if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_refDictContent, FUZ_rand(&lseed) & 1) );
-                if (FUZ_rand(&lseed) & 1) { CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) ); }
-                else { CHECK_Z( ZSTD_CCtx_refPrefix(zc, dict, dictSize) ); }
-
-                if (dict && dictSize) {
-                    /* test that compression parameters are correctly rejected after setting a dictionary */
-                    size_t const setError = ZSTD_CCtx_setParameter(zc, ZSTD_p_windowLog, cParams.windowLog-1) ;
-                    CHECK(!ZSTD_isError(setError), "ZSTD_CCtx_setParameter should have failed");
+                if (FUZ_rand(&lseed) & 1) {
+                    CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) );
+                    if (dict && dictSize) {
+                        /* test that compression parameters are rejected (correctly) after loading a non-NULL dictionary */
+                        size_t const setError = ZSTD_CCtx_setParameter(zc, ZSTD_p_windowLog, cParams.windowLog-1) ;
+                        CHECK(!ZSTD_isError(setError), "ZSTD_CCtx_setParameter should have failed");
+                }   } else {
+                    CHECK_Z( ZSTD_CCtx_refPrefix(zc, dict, dictSize) );
                 }
 
                 /* mess with frame parameters */

From 33a66390398334dc6a84e6b334194f226986e306 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 28 Jun 2017 11:09:43 -0700
Subject: [PATCH 6/8] fixed ZSTD_refPrefix with Multithread-enabled CCtx

---
 lib/common/zstd_internal.h       | 17 +++++++++-----
 lib/compress/zstd_compress.c     | 15 +++++++------
 lib/compress/zstdmt_compress.c   |  2 +-
 lib/decompress/zstd_decompress.c | 38 +++++++++++++++++++++++---------
 tests/zstreamtest.c              |  5 ++++-
 5 files changed, 52 insertions(+), 25 deletions(-)

diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index bcf93e58d..b2c7cfcdd 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -71,12 +71,17 @@
 
 #if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=2)
 #  include 
-   static unsigned g_debugLevel = ZSTD_DEBUG;
-#  define DEBUGLOG(l, ...) {                          \
-                if (l<=g_debugLevel) {                \
-                    fprintf(stderr, __FILE__ ": ");   \
-                    fprintf(stderr, __VA_ARGS__);     \
-                    fprintf(stderr, " \n");           \
+/* recommended values for ZSTD_DEBUG display levels :
+ * 2 : reserved for currently active debugging path
+ * 3 : events once per object lifetime (CCtx, CDict)
+ * 4 : events once per frame
+ * 5 : events once per block
+ * 6 : events once per sequence (*very* verbose) */
+#  define DEBUGLOG(l, ...) {                         \
+                if (l<=ZSTD_DEBUG) {                 \
+                    fprintf(stderr, __FILE__ ": ");  \
+                    fprintf(stderr, __VA_ARGS__);    \
+                    fprintf(stderr, " \n");          \
             }   }
 #else
 #  define DEBUGLOG(l, ...)      {}    /* disabled */
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 41be22edd..9ea866bf2 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -3924,29 +3924,30 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
 
     if (cctx->streamStage == zcss_init) {
         /* transparent reset */
+        const void* const prefix = cctx->prefix;
+        size_t const prefixSize = cctx->prefixSize;
         ZSTD_parameters params = cctx->requestedParams;
         if (cctx->compressionLevel != ZSTD_CLEVEL_CUSTOM)
             params.cParams = ZSTD_getCParams(cctx->compressionLevel,
-                                    cctx->pledgedSrcSizePlusOne-1, 0 /* dictSize */);
+                                cctx->pledgedSrcSizePlusOne-1, 0 /*dictSize*/);
+        cctx->prefix = NULL; cctx->prefixSize = 0;   /* single usage */
+        assert(prefix==NULL || cctx->cdict==NULL);   /* only one can be set */
 
 #ifdef ZSTD_MULTITHREAD
         if (cctx->nbThreads > 1) {
-            DEBUGLOG(4, "call ZSTDMT_initCStream_internal");
-            CHECK_F( ZSTDMT_initCStream_internal(cctx->mtctx, NULL, 0, cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) );
+            DEBUGLOG(4, "call ZSTDMT_initCStream_internal as nbThreads=%u", cctx->nbThreads);
+            CHECK_F( ZSTDMT_initCStream_internal(cctx->mtctx, prefix, prefixSize, cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) );
             cctx->streamStage = zcss_load;
         } else
 #endif
         {
-            const void* const prefix = cctx->prefix;
-            size_t const prefixSize = cctx->prefixSize;
-            cctx->prefix = NULL; cctx->prefixSize = 0;   /* single usage */
             CHECK_F( ZSTD_resetCStream_internal(cctx, prefix, prefixSize, cctx->dictMode, cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) );
     }   }
 
 #ifdef ZSTD_MULTITHREAD
     if (cctx->nbThreads > 1) {
         size_t const flushMin = ZSTDMT_compressStream_generic(cctx->mtctx, output, input, endOp);
-        DEBUGLOG(4, "ZSTDMT_compressStream_generic : %u", (U32)flushMin);
+        DEBUGLOG(5, "ZSTDMT_compressStream_generic : %u", (U32)flushMin);
         if ( ZSTD_isError(flushMin)
           || (endOp == ZSTD_e_end && flushMin == 0) ) { /* compression completed */
             ZSTD_startNewCompression(cctx);
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 8a167ce1d..d5f08c76f 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -586,7 +586,7 @@ size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
     if (dict) {
         ZSTD_freeCDict(zcs->cdictLocal);
         zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize,
-                                                    0 /* byRef */, ZSTD_dm_auto,
+                                                    0 /* byRef */, ZSTD_dm_auto,   /* note : a loadPrefix becomes an internal CDict */
                                                     params.cParams, zcs->cMem);
         zcs->cdict = zcs->cdictLocal;
         if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 61f766e21..a4844c5b3 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -750,6 +750,7 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
     const BYTE* const istart = (const BYTE* const)src;
     const BYTE* const iend = istart + srcSize;
     const BYTE* ip = istart;
+    DEBUGLOG(5, "ZSTD_decodeSeqHeaders");
 
     /* check */
     if (srcSize < MIN_SEQUENCES_SIZE) return ERROR(srcSize_wrong);
@@ -930,12 +931,18 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState)
         seq.offset = offset;
     }
 
-    seq.matchLength = ML_base[mlCode] + ((mlCode>31) ? BIT_readBitsFast(&seqState->DStream, mlBits) : 0);  /* <=  16 bits */
+    seq.matchLength = ML_base[mlCode]
+                    + ((mlCode>31) ? BIT_readBitsFast(&seqState->DStream, mlBits) : 0);  /* <=  16 bits */
     if (MEM_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&seqState->DStream);
 
-    seq.litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBitsFast(&seqState->DStream, llBits) : 0);    /* <=  16 bits */
-    if (MEM_32bits() ||
-       (totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) ) BIT_reloadDStream(&seqState->DStream);
+    seq.litLength = LL_base[llCode]
+                  + ((llCode>15) ? BIT_readBitsFast(&seqState->DStream, llBits) : 0);    /* <=  16 bits */
+    if (  MEM_32bits()
+      || (totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) )
+       BIT_reloadDStream(&seqState->DStream);
+
+    DEBUGLOG(6, "seq: litL=%u, matchL=%u, offset=%u",
+                (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset);
 
     /* ANS state update */
     FSE_updateState(&seqState->stateLL, &seqState->DStream);    /* <=  9 bits */
@@ -975,7 +982,8 @@ size_t ZSTD_execSequence(BYTE* op,
     /* copy Match */
     if (sequence.offset > (size_t)(oLitEnd - base)) {
         /* offset beyond prefix -> go into extDict */
-        if (sequence.offset > (size_t)(oLitEnd - vBase)) return ERROR(corruption_detected);
+        if (sequence.offset > (size_t)(oLitEnd - vBase))
+            return ERROR(corruption_detected);
         match = dictEnd + (match - base);
         if (match + sequence.matchLength <= dictEnd) {
             memmove(oLitEnd, match, sequence.matchLength);
@@ -1043,9 +1051,12 @@ static size_t ZSTD_decompressSequences(
     const BYTE* const vBase = (const BYTE*) (dctx->vBase);
     const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
     int nbSeq;
+    DEBUGLOG(5, "ZSTD_decompressSequences");
 
     /* Build Decoding Tables */
     {   size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, seqSize);
+        DEBUGLOG(5, "ZSTD_decodeSeqHeaders: size=%u, nbSeq=%i",
+                    (U32)seqHSize, nbSeq);
         if (ZSTD_isError(seqHSize)) return seqHSize;
         ip += seqHSize;
     }
@@ -1064,11 +1075,13 @@ static size_t ZSTD_decompressSequences(
             nbSeq--;
             {   seq_t const sequence = ZSTD_decodeSequence(&seqState);
                 size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd);
+                DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize);
                 if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
                 op += oneSeqSize;
         }   }
 
         /* check if reached exact end */
+        DEBUGLOG(5, "after decode loop, remaining nbSeq : %i", nbSeq);
         if (nbSeq) return ERROR(corruption_detected);
         /* save reps for next block */
         { U32 i; for (i=0; ientropy.rep[i] = (U32)(seqState.prevOffset[i]); }
@@ -1355,11 +1368,13 @@ static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
                       const void* src, size_t srcSize)
 {   /* blockType == blockCompressed */
     const BYTE* ip = (const BYTE*)src;
+    DEBUGLOG(5, "ZSTD_decompressBlock_internal");
 
     if (srcSize >= ZSTD_BLOCKSIZE_MAX) return ERROR(srcSize_wrong);
 
     /* Decode literals section */
     {   size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize);
+        DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : %u", (U32)litCSize);
         if (ZSTD_isError(litCSize)) return litCSize;
         ip += litCSize;
         srcSize -= litCSize;
@@ -1697,6 +1712,7 @@ static int ZSTD_isSkipFrame(ZSTD_DCtx* dctx) { return dctx->stage == ZSTDds_skip
  *            or an error code, which can be tested using ZSTD_isError() */
 size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
 {
+    DEBUGLOG(5, "ZSTD_decompressContinue");
     /* Sanity check */
     if (srcSize != dctx->expected) return ERROR(srcSize_wrong);   /* unauthorized */
     if (dstCapacity) ZSTD_checkContinuity(dctx, dst);
@@ -1756,10 +1772,12 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
         }
     case ZSTDds_decompressLastBlock:
     case ZSTDds_decompressBlock:
+        DEBUGLOG(5, "case ZSTDds_decompressBlock");
         {   size_t rSize;
             switch(dctx->bType)
             {
             case bt_compressed:
+                DEBUGLOG(5, "case bt_compressed");
                 rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize);
                 break;
             case bt_raw :
@@ -2323,7 +2341,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
             }   }
 
             /* Consume header (see ZSTDds_decodeFrameHeader) */
-            DEBUGLOG(5, "Consume header");
+            DEBUGLOG(4, "Consume header");
             CHECK_F(ZSTD_decompressBegin_usingDDict(zds, zds->ddict));
 
             if ((MEM_readLE32(zds->headerBuffer) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {  /* skippable frame */
@@ -2336,7 +2354,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
             }
 
             /* control buffer memory usage */
-            DEBUGLOG(5, "Control max buffer memory usage");
+            DEBUGLOG(4, "Control max buffer memory usage");
             zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
             if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_windowTooLarge);
 
@@ -2346,12 +2364,12 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                 zds->blockSize = blockSize;
                 if ((zds->inBuffSize < blockSize) || (zds->outBuffSize < neededOutSize)) {
                     size_t const bufferSize = blockSize + neededOutSize;
-                    DEBUGLOG(5, "inBuff  : from %u to %u",
+                    DEBUGLOG(4, "inBuff  : from %u to %u",
                                 (U32)zds->inBuffSize, (U32)blockSize);
-                    DEBUGLOG(5, "outBuff : from %u to %u",
+                    DEBUGLOG(4, "outBuff : from %u to %u",
                                 (U32)zds->outBuffSize, (U32)neededOutSize);
                     if (zds->staticSize) {  /* static DCtx */
-                        DEBUGLOG(5, "staticSize : %u", (U32)zds->staticSize);
+                        DEBUGLOG(4, "staticSize : %u", (U32)zds->staticSize);
                         assert(zds->staticSize >= sizeof(ZSTD_DCtx));  /* controlled at init */
                         if (bufferSize > zds->staticSize - sizeof(ZSTD_DCtx))
                             return ERROR(memory_allocation);
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 4c4479557..e4d40c66a 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -1382,8 +1382,10 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
 
         /* multi - fragments decompression test */
         if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) {
+            DISPLAYLEVEL(5, "resetting DCtx (dict:%08X) \n", (U32)(size_t)dict);
             CHECK_Z( ZSTD_resetDStream(zd) );
         } else {
+            DISPLAYLEVEL(5, "using dict of size %u \n", (U32)dictSize);
             CHECK_Z( ZSTD_initDStream_usingDict(zd, dict, dictSize) );
         }
         {   size_t decompressionResult = 1;
@@ -1395,7 +1397,8 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
                 size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
                 inBuff.size = inBuff.pos + readCSrcSize;
                 outBuff.size = inBuff.pos + dstBuffSize;
-                DISPLAYLEVEL(5, "ZSTD_decompressStream input %u bytes \n", (U32)readCSrcSize);
+                DISPLAYLEVEL(5, "ZSTD_decompressStream input %u bytes (pos:%u/%u)\n",
+                            (U32)readCSrcSize, (U32)inBuff.pos, (U32)cSize);
                 decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff);
                 CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult));
                 DISPLAYLEVEL(5, "inBuff.pos = %u \n", (U32)readCSrcSize);

From 62f7efc04a8313bef7ad830089c631f9ff6239c3 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 28 Jun 2017 16:25:13 -0700
Subject: [PATCH 7/8] added a test to ensure ZSTD_refPrefix() doesn't impact
 the following compression job

---
 tests/zstreamtest.c | 49 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)

diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index e4d40c66a..f16cc4b0a 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -522,6 +522,55 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
         DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r));
     }
 
+    DISPLAYLEVEL(3, "test%3i : compress with ZSTD_CCtx_refPrefix : ", testNb++);
+    { size_t const refErr = ZSTD_CCtx_refPrefix(zc, dictionary.start, dictionary.filled);
+      if (ZSTD_isError(refErr)) goto _output_error; }
+    outBuff.dst = compressedBuffer;
+    outBuff.size = compressedBufferSize;
+    outBuff.pos = 0;
+    inBuff.src = CNBuffer;
+    inBuff.size = CNBufferSize;
+    inBuff.pos = 0;
+    { size_t const r = ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end);
+      if (ZSTD_isError(r)) goto _output_error; }
+    if (inBuff.pos != inBuff.size) goto _output_error;  /* entire input should be consumed */
+    cSize = outBuff.pos;
+    DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBufferSize*100);
+
+    DISPLAYLEVEL(3, "test%3i : decompress with dictionary : ", testNb++);
+    {   size_t const r = ZSTD_decompress_usingDict(zd,
+                                        decodedBuffer, CNBufferSize,
+                                        compressedBuffer, cSize,
+                                        dictionary.start, dictionary.filled);
+        if (ZSTD_isError(r)) goto _output_error;  /* must fail : dictionary not used */
+        DISPLAYLEVEL(3, "OK \n");
+    }
+
+    DISPLAYLEVEL(3, "test%3i : decompress without dictionary (should fail): ", testNb++);
+    {   size_t const r = ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize);
+        if (!ZSTD_isError(r)) goto _output_error;  /* must fail : dictionary not used */
+        DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r));
+    }
+
+    DISPLAYLEVEL(3, "test%3i : compress again with ZSTD_compress_generic : ", testNb++);
+    outBuff.dst = compressedBuffer;
+    outBuff.size = compressedBufferSize;
+    outBuff.pos = 0;
+    inBuff.src = CNBuffer;
+    inBuff.size = CNBufferSize;
+    inBuff.pos = 0;
+    { size_t const r = ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end);
+      if (ZSTD_isError(r)) goto _output_error; }
+    if (inBuff.pos != inBuff.size) goto _output_error;  /* entire input should be consumed */
+    cSize = outBuff.pos;
+    DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBufferSize*100);
+
+    DISPLAYLEVEL(3, "test%3i : decompress without dictionary (should work): ", testNb++);
+    {   size_t const r = ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize);
+        if (ZSTD_isError(r)) goto _output_error;  /* must fail : dictionary not used */
+        DISPLAYLEVEL(3, "OK \n");
+    }
+
     /* Empty srcSize */
     DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_advanced with pledgedSrcSize=0 and dict : ", testNb++);
     {   ZSTD_parameters params = ZSTD_getParams(5, 0, 0);

From 037466245fd01669d6a7347e5560000ae2aba44d Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 28 Jun 2017 20:17:22 -0700
Subject: [PATCH 8/8] refactor
 ZSTD_check_compressionLevel_monotonicIncrease_memoryBudget()

use less macro statements
the initial version was meant to work with STATIC_ASSERT
but since it doesn't work and needs assert()
it's possible to rewrite it using normally compiled code
which is better for compiler.

Downside : the error message is less precise.
There is a DEBUGLOG(3,) to compensate.
---
 doc/zstd_manual.html         |  4 ++--
 lib/compress/zstd_compress.c | 38 ++++++++++--------------------------
 2 files changed, 12 insertions(+), 30 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index 683b70e80..cd2b06dd8 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -412,7 +412,7 @@ size_t ZSTD_estimateCCtxSize_advanced(ZSTD_compressionParameters cParams);
 size_t ZSTD_estimateDCtxSize(void);
 
  These functions make it possible to estimate memory usage
   of a future {D,C}Ctx, before its creation.
-  ZSTD_estimateCCtxSize() will provide a budget large for any compression level up to selected one.
+  ZSTD_estimateCCtxSize() will provide a budget large enough for any compression level up to selected one.
   It will also consider src size to be arbitrarily "large", which is worst case.
   If srcSize is known to always be small, ZSTD_estimateCCtxSize_advanced() can provide a tighter estimation.
   ZSTD_estimateCCtxSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
@@ -423,7 +423,7 @@ size_t ZSTD_estimateDCtxSize(void);
 size_t ZSTD_estimateCStreamSize_advanced(ZSTD_compressionParameters cParams);
 size_t ZSTD_estimateDStreamSize(size_t windowSize);
 size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize);
-
  ZSTD_estimateCStreamSize() will provide a budget large for any compression level up to selected one.
+
  ZSTD_estimateCStreamSize() will provide a budget large enough for any compression level up to selected one.
   It will also consider src size to be arbitrarily "large", which is worst case.
   If srcSize is known to always be small, ZSTD_estimateCStreamSize_advanced() can provide a tighter estimation.
   ZSTD_estimateCStreamSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index b396311d3..f492d92bd 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -4115,42 +4115,24 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
 },
 };
 
+#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=1)
 /* This function just controls
  * the monotonic memory budget increase of ZSTD_defaultCParameters[0].
  * Run only once, on first ZSTD_getCParams() usage, when ZSTD_DEBUG is enabled
  */
 MEM_STATIC void ZSTD_check_compressionLevel_monotonicIncrease_memoryBudget(void)
 {
-#   define ZSTD_TABLECOST(h,c) ((1<<(h)) + (1<<(c)))
-#   define ZDCP_FIELD(l,field) (ZSTD_defaultCParameters[0][l].field)
-#   define ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(l) { \
-        assert(ZDCP_FIELD(l,windowLog) <= ZDCP_FIELD(l+1,windowLog) );  \
-        assert(ZSTD_TABLECOST(ZDCP_FIELD(l,hashLog), ZDCP_FIELD(l,chainLog)) <= ZSTD_TABLECOST(ZDCP_FIELD(l+1,hashLog), ZDCP_FIELD(l+1,chainLog)) ); \
+    int level;
+    for (level=1; level