From 31af8290d155091f99c42da7b3787964ca4b37d7 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 26 Jun 2017 15:52:39 -0700 Subject: [PATCH 1/8] ZSTD_estimateCCtx_advanced() ZSTD_estimateCCtx() is now a "simple" function, taking int compressionLevel as single argument. ZSTD_estimateCCtx_advanced() takes a CParams argument, which is both more complete and more complex to generate. --- doc/zstd_manual.html | 15 ++++++++++++--- lib/compress/zstd_compress.c | 16 +++++++++++----- lib/zstd.h | 9 ++++++--- tests/paramgrill.c | 4 ++-- 4 files changed, 31 insertions(+), 13 deletions(-) diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html index 63704e67e..4397faf7a 100644 --- a/doc/zstd_manual.html +++ b/doc/zstd_manual.html @@ -389,6 +389,12 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize); however it does mean that all frame data must be present and valid.


+
size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
+
   `src` should point to the start of a ZSTD frame
+   `srcSize` must be >= ZSTD_frameHeaderSize_prefix.
+   @return : size of the Frame Header 
+

+

Context memory usage


 
 
size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
@@ -401,12 +407,15 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
   Object memory usage can evolve if it's re-used multiple times. 
 


 
-
size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams);
+
size_t ZSTD_estimateCCtxSize(int compressionLevel);
+size_t ZSTD_estimateCCtxSize_advanced(ZSTD_compressionParameters cParams);
 size_t ZSTD_estimateDCtxSize(void);
 
  These functions make it possible to estimate memory usage
-  of a future target object, before its allocation,
-  given a set of parameters, which vary depending on target object.
+  of a future {D,C}Ctx, before its creation.
   The objective is to guide decision before allocation.
+  ZSTD_estimateCCtxSize() will consider src size to be arbitrarily "large".
+  If srcSize is known to be small, ZSTD_estimateCCtxSize_advanced() will provide a better (smaller) estimation.
+  ZSTD_estimateCCtxSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
   Note : CCtx estimation is only correct for single-threaded compression 
 


 
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index e53496091..1bf676926 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -532,7 +532,7 @@ ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, u
 }
 
 
-size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams)
+size_t ZSTD_estimateCCtxSize_advanced(ZSTD_compressionParameters cParams)
 {
     size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog);
     U32    const divider = (cParams.searchLength==3) ? 3 : 4;
@@ -558,9 +558,15 @@ size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams)
     return sizeof(ZSTD_CCtx) + neededSpace;
 }
 
+size_t ZSTD_estimateCCtxSize(int compressionLevel)
+{
+    ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, 0, 0);
+    return ZSTD_estimateCCtxSize_advanced(cParams);
+}
+
 size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams)
 {
-    size_t const CCtxSize = ZSTD_estimateCCtxSize(cParams);
+    size_t const CCtxSize = ZSTD_estimateCCtxSize_advanced(cParams);
     size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog);
     size_t const inBuffSize = ((size_t)1 << cParams.windowLog) + blockSize;
     size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1;
@@ -3355,8 +3361,8 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS
 size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize, unsigned byReference)
 {
     DEBUGLOG(5, "sizeof(ZSTD_CDict) : %u", (U32)sizeof(ZSTD_CDict));
-    DEBUGLOG(5, "CCtx estimate : %u", (U32)ZSTD_estimateCCtxSize(cParams));
-    return sizeof(ZSTD_CDict) + ZSTD_estimateCCtxSize(cParams)
+    DEBUGLOG(5, "CCtx estimate : %u", (U32)ZSTD_estimateCCtxSize_advanced(cParams));
+    return sizeof(ZSTD_CDict) + ZSTD_estimateCCtxSize_advanced(cParams)
            + (byReference ? 0 : dictSize);
 }
 
@@ -3482,7 +3488,7 @@ ZSTD_CDict* ZSTD_initStaticCDict(void* workspace, size_t workspaceSize,
                                  unsigned byReference, ZSTD_dictMode_e dictMode,
                                  ZSTD_compressionParameters cParams)
 {
-    size_t const cctxSize = ZSTD_estimateCCtxSize(cParams);
+    size_t const cctxSize = ZSTD_estimateCCtxSize_advanced(cParams);
     size_t const neededSize = sizeof(ZSTD_CDict) + (byReference ? 0 : dictSize)
                             + cctxSize;
     ZSTD_CDict* const cdict = (ZSTD_CDict*) workspace;
diff --git a/lib/zstd.h b/lib/zstd.h
index 8cf9ba75f..65932088c 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -495,11 +495,14 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
 
 /*! ZSTD_estimate*() :
  *  These functions make it possible to estimate memory usage
- *  of a future target object, before its allocation,
- *  given a set of parameters, which vary depending on target object.
+ *  of a future {D,C}Ctx, before its creation.
  *  The objective is to guide decision before allocation.
+ *  ZSTD_estimateCCtxSize() will consider src size to be arbitrarily "large".
+ *  If srcSize is known to be small, ZSTD_estimateCCtxSize_advanced() will provide a better (smaller) estimation.
+ *  ZSTD_estimateCCtxSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
  *  Note : CCtx estimation is only correct for single-threaded compression */
-ZSTDLIB_API size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams);
+ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel);
+ZSTDLIB_API size_t ZSTD_estimateCCtxSize_advanced(ZSTD_compressionParameters cParams);
 ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void);
 
 /*! ZSTD_estimate?StreamSize() :
diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index 1185c6649..da06ccb52 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -390,8 +390,8 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para
             double W_DMemUsed_note = W_ratioNote * ( 40 + 9*cLevel) - log((double)W_DMemUsed);
             double O_DMemUsed_note = O_ratioNote * ( 40 + 9*cLevel) - log((double)O_DMemUsed);
 
-            size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_estimateCCtxSize(params);
-            size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_estimateCCtxSize(winners[cLevel].params);
+            size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_estimateCCtxSize_advanced(params);
+            size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_estimateCCtxSize_advanced(winners[cLevel].params);
             double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed);
             double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed);
 

From 0c9a915a280d24aa887a4dd19a72195fb8ade5cb Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 26 Jun 2017 16:02:25 -0700
Subject: [PATCH 2/8] ZSTD_estimateCStreamSize_advanced()

---
 doc/zstd_manual.html         | 10 +++++++---
 lib/compress/zstd_compress.c |  7 ++++++-
 lib/zstd.h                   |  8 ++++++--
 tests/fuzzer.c               |  3 +--
 tests/zstreamtest.c          |  2 +-
 5 files changed, 21 insertions(+), 9 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index 4397faf7a..2cd4f78c5 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -414,14 +414,18 @@ size_t ZSTD_estimateDCtxSize(void);
   of a future {D,C}Ctx, before its creation.
   The objective is to guide decision before allocation.
   ZSTD_estimateCCtxSize() will consider src size to be arbitrarily "large".
-  If srcSize is known to be small, ZSTD_estimateCCtxSize_advanced() will provide a better (smaller) estimation.
+  If srcSize is known to be small, ZSTD_estimateCCtxSize_advanced() can provide a tighter estimation.
   ZSTD_estimateCCtxSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
   Note : CCtx estimation is only correct for single-threaded compression 
 


 
-
size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams);
+
size_t ZSTD_estimateCStreamSize(int compressionLevel);
+size_t ZSTD_estimateCStreamSize_advanced(ZSTD_compressionParameters cParams);
 size_t ZSTD_estimateDStreamSize(ZSTD_frameHeader fHeader);
-
  Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
+
  ZSTD_estimateCStreamSize() will consider src size to be arbitrarily "large".
+  If srcSize is known to be small, ZSTD_estimateCStreamSize_advanced() can provide a tighter estimation.
+  ZSTD_estimateCStreamSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
+  Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
          an internal ?Dict will be created, which size is not estimated here.
          In this case, get total size by adding ZSTD_estimate?DictSize 
 


diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 1bf676926..f74f4d8ad 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -564,7 +564,7 @@ size_t ZSTD_estimateCCtxSize(int compressionLevel)
     return ZSTD_estimateCCtxSize_advanced(cParams);
 }
 
-size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams)
+size_t ZSTD_estimateCStreamSize_advanced(ZSTD_compressionParameters cParams)
 {
     size_t const CCtxSize = ZSTD_estimateCCtxSize_advanced(cParams);
     size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog);
@@ -575,6 +575,11 @@ size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams)
     return CCtxSize + streamingSize;
 }
 
+size_t ZSTD_estimateCStreamSize(int compressionLevel) {
+    ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, 0, 0);
+    return ZSTD_estimateCStreamSize_advanced(cParams);
+}
+
 
 static U32 ZSTD_equivalentParams(ZSTD_compressionParameters cParams1,
                                  ZSTD_compressionParameters cParams2)
diff --git a/lib/zstd.h b/lib/zstd.h
index 65932088c..9c6932fbb 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -498,7 +498,7 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
  *  of a future {D,C}Ctx, before its creation.
  *  The objective is to guide decision before allocation.
  *  ZSTD_estimateCCtxSize() will consider src size to be arbitrarily "large".
- *  If srcSize is known to be small, ZSTD_estimateCCtxSize_advanced() will provide a better (smaller) estimation.
+ *  If srcSize is known to be small, ZSTD_estimateCCtxSize_advanced() can provide a tighter estimation.
  *  ZSTD_estimateCCtxSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
  *  Note : CCtx estimation is only correct for single-threaded compression */
 ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel);
@@ -506,10 +506,14 @@ ZSTDLIB_API size_t ZSTD_estimateCCtxSize_advanced(ZSTD_compressionParameters cPa
 ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void);
 
 /*! ZSTD_estimate?StreamSize() :
+ *  ZSTD_estimateCStreamSize() will consider src size to be arbitrarily "large".
+ *  If srcSize is known to be small, ZSTD_estimateCStreamSize_advanced() can provide a tighter estimation.
+ *  ZSTD_estimateCStreamSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
  *  Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
  *         an internal ?Dict will be created, which size is not estimated here.
  *         In this case, get total size by adding ZSTD_estimate?DictSize */
-ZSTDLIB_API size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams);
+ZSTDLIB_API size_t ZSTD_estimateCStreamSize(int compressionLevel);
+ZSTDLIB_API size_t ZSTD_estimateCStreamSize_advanced(ZSTD_compressionParameters cParams);
 ZSTDLIB_API size_t ZSTD_estimateDStreamSize(ZSTD_frameHeader fHeader);
 
 /*! ZSTD_estimate?DictSize() :
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index a3a56d9d8..7d33955f4 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -193,8 +193,7 @@ static int basicUnitTests(U32 seed, double compressibility)
     /* Static CCtx tests */
 #define STATIC_CCTX_LEVEL 3
     DISPLAYLEVEL(4, "test%3i : create static CCtx for level %u :", testNb++, STATIC_CCTX_LEVEL);
-    {   ZSTD_compressionParameters const cParams = ZSTD_getCParams(STATIC_CCTX_LEVEL, 0, 0);
-        size_t const staticCCtxSize = ZSTD_estimateCStreamSize(cParams);
+    {   size_t const staticCCtxSize = ZSTD_estimateCStreamSize(STATIC_CCTX_LEVEL);
         void* const staticCCtxBuffer = malloc(staticCCtxSize);
         size_t const staticDCtxSize = ZSTD_estimateDCtxSize();
         void* const staticDCtxBuffer = malloc(staticDCtxSize);
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 0e14fd23a..7c60b7067 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -210,7 +210,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     /* context size functions */
     DISPLAYLEVEL(3, "test%3i : estimate CStream size : ", testNb++);
     {   ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictSize);
-        size_t const s = ZSTD_estimateCStreamSize(cParams)
+        size_t const s = ZSTD_estimateCStreamSize_advanced(cParams)
                         /* uses ZSTD_initCStream_usingDict() */
                        + ZSTD_estimateCDictSize(cParams, dictSize, 0);
             if (ZSTD_isError(s)) goto _output_error;

From 09ae03a570c1725f261658bdfecb93451c66fc32 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 26 Jun 2017 16:47:32 -0700
Subject: [PATCH 3/8] ZSTD_estimateCDictSize_advanced()

ZSTD_estimateCDictSize() now uses same arguments as ZSTD_createCDict()
ZSTD_estimateCDictSize_advanced() uses same arguments as ZSTD_createCDict_advanced()
---
 doc/zstd_manual.html         |  7 +++++--
 lib/compress/zstd_compress.c | 10 ++++++++--
 lib/zstd.h                   |  5 ++++-
 tests/fuzzer.c               |  4 ++--
 tests/zstreamtest.c          |  2 +-
 5 files changed, 20 insertions(+), 8 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index 2cd4f78c5..eff8ba0aa 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -430,9 +430,12 @@ size_t ZSTD_estimateDStreamSize(ZSTD_frameHeader fHeader);
          In this case, get total size by adding ZSTD_estimate?DictSize 
 


 
-
size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize, unsigned byReference);
+
size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel);
+size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, unsigned byReference);
 size_t ZSTD_estimateDDictSize(size_t dictSize, unsigned byReference);
-
  Note : dictionary created "byReference" are smaller 
+
  ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict().
+  ZSTD_estimateCStreamSize_advanced() makes it possible to control precisely compression parameters, like ZSTD_createCDict_advanced().
+  Note : dictionary created "byReference" are smaller 
 


 
 
Advanced compression functions

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index f74f4d8ad..781cdcd42 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -3361,9 +3361,9 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS
 
 /* =====  Dictionary API  ===== */
 
-/*! ZSTD_estimateCDictSize() :
+/*! ZSTD_estimateCDictSize_advanced() :
  *  Estimate amount of memory that will be needed to create a dictionary with following arguments */
-size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize, unsigned byReference)
+size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, unsigned byReference)
 {
     DEBUGLOG(5, "sizeof(ZSTD_CDict) : %u", (U32)sizeof(ZSTD_CDict));
     DEBUGLOG(5, "CCtx estimate : %u", (U32)ZSTD_estimateCCtxSize_advanced(cParams));
@@ -3371,6 +3371,12 @@ size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSiz
            + (byReference ? 0 : dictSize);
 }
 
+size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel)
+{
+    ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, 0, dictSize);
+    return ZSTD_estimateCDictSize_advanced(dictSize, cParams, 0);
+}
+
 size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict)
 {
     if (cdict==NULL) return 0;   /* support sizeof on NULL */
diff --git a/lib/zstd.h b/lib/zstd.h
index 9c6932fbb..492c3d43c 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -517,8 +517,11 @@ ZSTDLIB_API size_t ZSTD_estimateCStreamSize_advanced(ZSTD_compressionParameters
 ZSTDLIB_API size_t ZSTD_estimateDStreamSize(ZSTD_frameHeader fHeader);
 
 /*! ZSTD_estimate?DictSize() :
+ *  ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict().
+ *  ZSTD_estimateCStreamSize_advanced() makes it possible to control precisely compression parameters, like ZSTD_createCDict_advanced().
  *  Note : dictionary created "byReference" are smaller */
-ZSTDLIB_API size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize, unsigned byReference);
+ZSTDLIB_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel);
+ZSTDLIB_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, unsigned byReference);
 ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize, unsigned byReference);
 
 
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 7d33955f4..0dcba3cea 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -501,7 +501,7 @@ static int basicUnitTests(U32 seed, double compressibility)
 
         DISPLAYLEVEL(4, "test%3i : estimate CDict size : ", testNb++);
         {   ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
-            size_t const estimatedSize = ZSTD_estimateCDictSize(cParams, dictSize, 1 /*byReference*/);
+            size_t const estimatedSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, 1 /*byReference*/);
             DISPLAYLEVEL(4, "OK : %u \n", (U32)estimatedSize);
         }
 
@@ -534,7 +534,7 @@ static int basicUnitTests(U32 seed, double compressibility)
 
         DISPLAYLEVEL(4, "test%3i : compress with static CDict : ", testNb++);
         {   ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
-            size_t const cdictSize = ZSTD_estimateCDictSize(cParams, dictSize, 0);
+            size_t const cdictSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, 0);
             void* const cdictBuffer = malloc(cdictSize);
             if (cdictBuffer==NULL) goto _output_error;
             {   ZSTD_CDict* const cdict = ZSTD_initStaticCDict(cdictBuffer, cdictSize,
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 7c60b7067..2c9cb5204 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -212,7 +212,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     {   ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictSize);
         size_t const s = ZSTD_estimateCStreamSize_advanced(cParams)
                         /* uses ZSTD_initCStream_usingDict() */
-                       + ZSTD_estimateCDictSize(cParams, dictSize, 0);
+                       + ZSTD_estimateCDictSize_advanced(dictSize, cParams, 0);
             if (ZSTD_isError(s)) goto _output_error;
             DISPLAYLEVEL(3, "OK (%u bytes) \n", (U32)s);
     }

From dde10b23fe47dee5d290dffb2a33e0daea76712e Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 26 Jun 2017 17:44:26 -0700
Subject: [PATCH 4/8] refactored ZSTD_estimateDStreamSize()

now uses windowSize as argument.
Also : created ZSTD_estimateDStreamSize_fromFrame()
---
 doc/zstd_manual.html             | 31 ++++++++++++++++++++-----------
 lib/compress/zstd_compress.c     |  5 +++++
 lib/decompress/zstd_decompress.c | 17 +++++++++++++++--
 lib/zstd.h                       | 31 ++++++++++++++++++++-----------
 tests/zstreamtest.c              |  4 ++--
 5 files changed, 62 insertions(+), 26 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index eff8ba0aa..6d524ae32 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -332,7 +332,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
 


 
typedef struct {
     unsigned long long frameContentSize;
-    unsigned windowSize;
+    size_t windowSize;
     unsigned dictID;
     unsigned checksumFlag;
 } ZSTD_frameHeader;
@@ -421,12 +421,17 @@ size_t ZSTD_estimateDCtxSize(void);
 
 
size_t ZSTD_estimateCStreamSize(int compressionLevel);
 size_t ZSTD_estimateCStreamSize_advanced(ZSTD_compressionParameters cParams);
-size_t ZSTD_estimateDStreamSize(ZSTD_frameHeader fHeader);
+size_t ZSTD_estimateDStreamSize(size_t windowSize);
+size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize);
 
  ZSTD_estimateCStreamSize() will consider src size to be arbitrarily "large".
   If srcSize is known to be small, ZSTD_estimateCStreamSize_advanced() can provide a tighter estimation.
   ZSTD_estimateCStreamSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
+  Note : CStream estimation is only correct for single-threaded compression.
+  ZSTD_DStream memory budget depends on window Size.
+  This information can be passed manually, using ZSTD_estimateDStreamSize,
+  or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame();
   Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
-         an internal ?Dict will be created, which size is not estimated here.
+         an internal ?Dict will be created, which additional size is not estimated here.
          In this case, get total size by adding ZSTD_estimate?DictSize 
 


 
@@ -622,6 +627,7 @@ size_t ZSTD_estimateDDictSize(size_t dictSize, unsigned byReference);
 
Advanced streaming functions

 
 
Advanced Streaming compression functions
ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
+ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize);    /**< same as ZSTD_initStaticCCtx() */
 size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);   /**< pledgedSrcSize must be correct, a size of 0 means unknown.  for a frame size of 0 use initCStream_advanced */
 size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. */
 size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
@@ -640,6 +646,7 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict*
 
 
Advanced Streaming decompression functions
typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e;
 ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
+ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize);    /**< same as ZSTD_initStaticDCtx() */
 size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
 size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */
 size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);  /**< note : ddict will just be referenced, and must outlive decompression session */
@@ -694,16 +701,18 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned lo
   A ZSTD_DCtx object can be re-used multiple times.
 
   First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader().
-  It fills a ZSTD_frameParams structure which provide important information to correctly decode the frame,
-  such as the minimum rolling buffer size to allocate to decompress data (`windowSize`),
-  and the dictionary ID used.
+  It fills a ZSTD_frameHeader structure with important information to correctly decode the frame,
+  such as minimum rolling buffer size to allocate to decompress data (`windowSize`),
+  and the dictionary ID in use.
   (Note : content size is optional, it may not be present. 0 means : content size unknown).
   Note that these values could be wrong, either because of data malformation, or because an attacker is spoofing deliberate false information.
   As a consequence, check that values remain within valid application range, especially `windowSize`, before allocation.
-  Each application can set its own limit, depending on local restrictions. For extended interoperability, it is recommended to support at least 8 MB.
-  Frame parameters are extracted from the beginning of the compressed frame.
-  Data fragment must be large enough to ensure successful decoding, typically `ZSTD_frameHeaderSize_max` bytes.
-  @result : 0 : successful decoding, the `ZSTD_frameParams` structure is correctly filled.
+  Each application can set its own limit, depending on local restrictions.
+  For extended interoperability, it is recommended to support windowSize of at least 8 MB.
+  Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough.
+  Data fragment must be large enough to ensure successful decoding.
+  `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough.
+  @result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled.
            >0 : `srcSize` is too small, please provide at least @result bytes on next attempt.
            errorCode, which can be tested using ZSTD_isError().
 
@@ -747,7 +756,7 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned lo
   It also returns Frame Size as fparamsPtr->frameContentSize.
 

 
-
Buffer-less streaming decompression functions
size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize);   /**< doesn't consume input, see details below */
+
Buffer-less streaming decompression functions
size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize);   /**< doesn't consume input */
 size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
 size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
 size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 781cdcd42..ea37197ff 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -3594,6 +3594,11 @@ ZSTD_CStream* ZSTD_createCStream(void)
     return ZSTD_createCStream_advanced(ZSTD_defaultCMem);
 }
 
+ZSTD_CStream* ZSTD_initStaticCStream(void *workspace, size_t workspaceSize)
+{
+    return ZSTD_initStaticCCtx(workspace, workspaceSize);
+}
+
 ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem)
 {   /* CStream and CCtx are now same object */
     return ZSTD_createCCtx_advanced(customMem);
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 95d18d4b5..940c3e033 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -2143,6 +2143,11 @@ ZSTD_DStream* ZSTD_createDStream(void)
     return ZSTD_createDStream_advanced(ZSTD_defaultCMem);
 }
 
+ZSTD_DStream* ZSTD_initStaticDStream(void *workspace, size_t workspaceSize)
+{
+    return ZSTD_initStaticDCtx(workspace, workspaceSize);
+}
+
 ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem)
 {
     return ZSTD_createDCtx_advanced(customMem);
@@ -2214,15 +2219,23 @@ size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds)
     return ZSTD_sizeof_DCtx(zds);
 }
 
-size_t ZSTD_estimateDStreamSize(ZSTD_frameHeader fHeader)
+size_t ZSTD_estimateDStreamSize(size_t windowSize)
 {
-    size_t const windowSize = fHeader.windowSize;
     size_t const blockSize = MIN(windowSize, ZSTD_BLOCKSIZE_MAX);
     size_t const inBuffSize = blockSize;  /* no block can be larger */
     size_t const outBuffSize = windowSize + blockSize + (WILDCOPY_OVERLENGTH * 2);
     return sizeof(ZSTD_DStream) + ZSTD_estimateDCtxSize() + inBuffSize + outBuffSize;
 }
 
+ZSTDLIB_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize)
+{
+    ZSTD_frameHeader fh;
+    size_t const err = ZSTD_getFrameHeader(&fh, src, srcSize);
+    if (ZSTD_isError(err)) return err;
+    if (err>0) return ERROR(srcSize_wrong);
+    return ZSTD_estimateDStreamSize(fh.windowSize);
+}
+
 
 /* *****   Decompression   ***** */
 
diff --git a/lib/zstd.h b/lib/zstd.h
index 492c3d43c..08358f0df 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -412,7 +412,7 @@ typedef struct {
 
 typedef struct {
     unsigned long long frameContentSize;
-    unsigned windowSize;
+    size_t windowSize;
     unsigned dictID;
     unsigned checksumFlag;
 } ZSTD_frameHeader;
@@ -509,12 +509,17 @@ ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void);
  *  ZSTD_estimateCStreamSize() will consider src size to be arbitrarily "large".
  *  If srcSize is known to be small, ZSTD_estimateCStreamSize_advanced() can provide a tighter estimation.
  *  ZSTD_estimateCStreamSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
+ *  Note : CStream estimation is only correct for single-threaded compression.
+ *  ZSTD_DStream memory budget depends on window Size.
+ *  This information can be passed manually, using ZSTD_estimateDStreamSize,
+ *  or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame();
  *  Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
- *         an internal ?Dict will be created, which size is not estimated here.
+ *         an internal ?Dict will be created, which additional size is not estimated here.
  *         In this case, get total size by adding ZSTD_estimate?DictSize */
 ZSTDLIB_API size_t ZSTD_estimateCStreamSize(int compressionLevel);
 ZSTDLIB_API size_t ZSTD_estimateCStreamSize_advanced(ZSTD_compressionParameters cParams);
-ZSTDLIB_API size_t ZSTD_estimateDStreamSize(ZSTD_frameHeader fHeader);
+ZSTDLIB_API size_t ZSTD_estimateDStreamSize(size_t windowSize);
+ZSTDLIB_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize);
 
 /*! ZSTD_estimate?DictSize() :
  *  ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict().
@@ -717,6 +722,7 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
 
 /*=====   Advanced Streaming compression functions  =====*/
 ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
+ZSTDLIB_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize);    /**< same as ZSTD_initStaticCCtx() */
 ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);   /**< pledgedSrcSize must be correct, a size of 0 means unknown.  for a frame size of 0 use initCStream_advanced */
 ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. */
 ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
@@ -737,6 +743,7 @@ ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledg
 /*=====   Advanced Streaming decompression functions  =====*/
 typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e;
 ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
+ZSTDLIB_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize);    /**< same as ZSTD_initStaticDCtx() */
 ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
 ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */
 ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);  /**< note : ddict will just be referenced, and must outlive decompression session */
@@ -803,16 +810,18 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
   A ZSTD_DCtx object can be re-used multiple times.
 
   First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader().
-  It fills a ZSTD_frameParams structure which provide important information to correctly decode the frame,
-  such as the minimum rolling buffer size to allocate to decompress data (`windowSize`),
-  and the dictionary ID used.
+  It fills a ZSTD_frameHeader structure with important information to correctly decode the frame,
+  such as minimum rolling buffer size to allocate to decompress data (`windowSize`),
+  and the dictionary ID in use.
   (Note : content size is optional, it may not be present. 0 means : content size unknown).
   Note that these values could be wrong, either because of data malformation, or because an attacker is spoofing deliberate false information.
   As a consequence, check that values remain within valid application range, especially `windowSize`, before allocation.
-  Each application can set its own limit, depending on local restrictions. For extended interoperability, it is recommended to support at least 8 MB.
-  Frame parameters are extracted from the beginning of the compressed frame.
-  Data fragment must be large enough to ensure successful decoding, typically `ZSTD_frameHeaderSize_max` bytes.
-  @result : 0 : successful decoding, the `ZSTD_frameParams` structure is correctly filled.
+  Each application can set its own limit, depending on local restrictions.
+  For extended interoperability, it is recommended to support windowSize of at least 8 MB.
+  Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough.
+  Data fragment must be large enough to ensure successful decoding.
+  `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough.
+  @result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled.
            >0 : `srcSize` is too small, please provide at least @result bytes on next attempt.
            errorCode, which can be tested using ZSTD_isError().
 
@@ -857,7 +866,7 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
 */
 
 /*=====   Buffer-less streaming decompression functions  =====*/
-ZSTDLIB_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize);   /**< doesn't consume input, see details below */
+ZSTDLIB_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize);   /**< doesn't consume input */
 ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
 ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
 ZSTDLIB_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 2c9cb5204..aae83972a 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -285,8 +285,8 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
         const void* cStart = (char*)compressedBuffer + (skippableFrameSize + 8);
         size_t const gfhError = ZSTD_getFrameHeader(&fhi, cStart, cSize);
         if (gfhError!=0) goto _output_error;
-        DISPLAYLEVEL(5, " (windowSize : %u) ", fhi.windowSize);
-        {   size_t const s = ZSTD_estimateDStreamSize(fhi)
+        DISPLAYLEVEL(5, " (windowSize : %u) ", (U32)fhi.windowSize);
+        {   size_t const s = ZSTD_estimateDStreamSize(fhi.windowSize)
                             /* uses ZSTD_initDStream_usingDict() */
                            + ZSTD_estimateDDictSize(dictSize, 0);
             if (ZSTD_isError(s)) goto _output_error;

From c7fb884eeaa9f03fa25caa1da7ab8f19c203aa6f Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 26 Jun 2017 18:02:23 -0700
Subject: [PATCH 5/8] fixed minor conversion warning

---
 lib/decompress/zstd_decompress.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 940c3e033..e823c3dbf 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -1282,7 +1282,7 @@ static size_t ZSTD_decompressSequencesLong(
     const BYTE* const base = (const BYTE*) (dctx->base);
     const BYTE* const vBase = (const BYTE*) (dctx->vBase);
     const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
-    unsigned const windowSize = dctx->fParams.windowSize;
+    unsigned const windowSize32 = (unsigned)dctx->fParams.windowSize;
     int nbSeq;
 
     /* Build Decoding Tables */
@@ -1312,13 +1312,13 @@ static size_t ZSTD_decompressSequencesLong(
 
         /* prepare in advance */
         for (seqNb=0; (BIT_reloadDStream(&seqState.DStream) <= BIT_DStream_completed) && seqNb
Date: Wed, 28 Jun 2017 13:22:37 -0700
Subject: [PATCH 6/8] adjusted compression levels to guarantee a monotonically
 increasing memory budget

---
 lib/compress/zstd_compress.c | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index ea37197ff..c781232c6 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -3992,30 +3992,30 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
 int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; }
 
 static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL+1] = {
-{   /* "default" */
+{   /* "default" - guarantees a monotonically increasing memory budget */
     /* W,  C,  H,  S,  L, TL, strat */
     { 18, 12, 12,  1,  7, 16, ZSTD_fast    },  /* level  0 - never used */
     { 19, 13, 14,  1,  7, 16, ZSTD_fast    },  /* level  1 */
     { 19, 15, 16,  1,  6, 16, ZSTD_fast    },  /* level  2 */
-    { 20, 16, 17,  1,  5, 16, ZSTD_dfast   },  /* level  3.*/
-    { 20, 18, 18,  1,  5, 16, ZSTD_dfast   },  /* level  4.*/
-    { 20, 15, 18,  3,  5, 16, ZSTD_greedy  },  /* level  5 */
-    { 21, 16, 19,  2,  5, 16, ZSTD_lazy    },  /* level  6 */
-    { 21, 17, 20,  3,  5, 16, ZSTD_lazy    },  /* level  7 */
+    { 20, 16, 17,  1,  5, 16, ZSTD_dfast   },  /* level  3 */
+    { 20, 17, 18,  1,  5, 16, ZSTD_dfast   },  /* level  4 */
+    { 20, 17, 18,  2,  5, 16, ZSTD_greedy  },  /* level  5 */
+    { 21, 17, 19,  2,  5, 16, ZSTD_lazy    },  /* level  6 */
+    { 21, 18, 19,  3,  5, 16, ZSTD_lazy    },  /* level  7 */
     { 21, 18, 20,  3,  5, 16, ZSTD_lazy2   },  /* level  8 */
-    { 21, 20, 20,  3,  5, 16, ZSTD_lazy2   },  /* level  9 */
+    { 21, 19, 20,  3,  5, 16, ZSTD_lazy2   },  /* level  9 */
     { 21, 19, 21,  4,  5, 16, ZSTD_lazy2   },  /* level 10 */
     { 22, 20, 22,  4,  5, 16, ZSTD_lazy2   },  /* level 11 */
     { 22, 20, 22,  5,  5, 16, ZSTD_lazy2   },  /* level 12 */
     { 22, 21, 22,  5,  5, 16, ZSTD_lazy2   },  /* level 13 */
     { 22, 21, 22,  6,  5, 16, ZSTD_lazy2   },  /* level 14 */
-    { 22, 21, 21,  5,  5, 16, ZSTD_btlazy2 },  /* level 15 */
+    { 22, 21, 22,  5,  5, 16, ZSTD_btlazy2 },  /* level 15 */
     { 23, 22, 22,  5,  5, 16, ZSTD_btlazy2 },  /* level 16 */
-    { 23, 21, 22,  4,  5, 24, ZSTD_btopt   },  /* level 17 */
+    { 23, 22, 22,  4,  5, 24, ZSTD_btopt   },  /* level 17 */
     { 23, 22, 22,  5,  4, 32, ZSTD_btopt   },  /* level 18 */
     { 23, 23, 22,  6,  3, 48, ZSTD_btopt   },  /* level 19 */
     { 25, 25, 23,  7,  3, 64, ZSTD_btultra },  /* level 20 */
-    { 26, 26, 23,  7,  3,256, ZSTD_btultra },  /* level 21 */
+    { 26, 26, 24,  7,  3,256, ZSTD_btultra },  /* level 21 */
     { 27, 27, 25,  9,  3,512, ZSTD_btultra },  /* level 22 */
 },
 {   /* for srcSize <= 256 KB */

From 813535105b4883183fb7ebffea454944d0dd7552 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 28 Jun 2017 15:34:56 -0700
Subject: [PATCH 7/8] added function to control monotonic memory budget
 increase of ZSTD_defaultCParameters[0]

It's a runtime test, based on assert(),
played once, on first ZSTD_getCParams() usage,
when ZSTD_DEBUG is enabled.
---
 doc/zstd_manual.html         | 11 +++++----
 lib/compress/zstd_compress.c | 46 ++++++++++++++++++++++++++++++++++++
 lib/zstd.h                   | 11 +++++----
 3 files changed, 58 insertions(+), 10 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index 6d524ae32..bb6f0eac9 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -412,9 +412,9 @@ size_t ZSTD_estimateCCtxSize_advanced(ZSTD_compressionParameters cParams);
 size_t ZSTD_estimateDCtxSize(void);
 
  These functions make it possible to estimate memory usage
   of a future {D,C}Ctx, before its creation.
-  The objective is to guide decision before allocation.
-  ZSTD_estimateCCtxSize() will consider src size to be arbitrarily "large".
-  If srcSize is known to be small, ZSTD_estimateCCtxSize_advanced() can provide a tighter estimation.
+  ZSTD_estimateCCtxSize() will provide a budget large for any compression level up to selected one.
+  It will also consider src size to be arbitrarily "large", which is worst case.
+  If srcSize is known to always be small, ZSTD_estimateCCtxSize_advanced() can provide a tighter estimation.
   ZSTD_estimateCCtxSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
   Note : CCtx estimation is only correct for single-threaded compression 
 


@@ -423,8 +423,9 @@ size_t ZSTD_estimateDCtxSize(void);
 size_t ZSTD_estimateCStreamSize_advanced(ZSTD_compressionParameters cParams);
 size_t ZSTD_estimateDStreamSize(size_t windowSize);
 size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize);
-
  ZSTD_estimateCStreamSize() will consider src size to be arbitrarily "large".
-  If srcSize is known to be small, ZSTD_estimateCStreamSize_advanced() can provide a tighter estimation.
+
  ZSTD_estimateCStreamSize() will provide a budget large for any compression level up to selected one.
+  It will also consider src size to be arbitrarily "large", which is worst case.
+  If srcSize is known to always be small, ZSTD_estimateCStreamSize_advanced() can provide a tighter estimation.
   ZSTD_estimateCStreamSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
   Note : CStream estimation is only correct for single-threaded compression.
   ZSTD_DStream memory budget depends on window Size.
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index c781232c6..c2adc56ab 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -4098,6 +4098,43 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
 },
 };
 
+/* This function just controls
+ * the monotonic memory budget increase of ZSTD_defaultCParameters[0].
+ * Run only once, on first ZSTD_getCParams() usage, when ZSTD_DEBUG is enabled
+ */
+MEM_STATIC void ZSTD_check_compressionLevel_monotonicIncrease_memoryBudget(void)
+{
+#   define ZSTD_TABLECOST(h,c) ((1<<(h)) + (1<<(c)))
+#   define ZDCP_FIELD(l,field) (ZSTD_defaultCParameters[0][l].field)
+#   define ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(l) { \
+        assert(ZDCP_FIELD(l,windowLog) <= ZDCP_FIELD(l+1,windowLog) );  \
+        assert(ZSTD_TABLECOST(ZDCP_FIELD(l,hashLog), ZDCP_FIELD(l,chainLog)) <= ZSTD_TABLECOST(ZDCP_FIELD(l+1,hashLog), ZDCP_FIELD(l+1,chainLog)) ); \
+    }
+
+    ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(1);
+    ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(2);
+    ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(3);
+    ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(4);
+    ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(5);
+    ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(6);
+    ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(7);
+    ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(8);
+    ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(9);
+    ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(10);
+    ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(11);
+    ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(12);
+    ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(13);
+    ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(14);
+    ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(15);
+    ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(16);
+    ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(17);
+    ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(18);
+    ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(19);
+    ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(20);
+    ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(21);
+    assert(ZSTD_maxCLevel()==22);
+}
+
 /*! ZSTD_getCParams() :
 *   @return ZSTD_compressionParameters structure for a selected compression level, `srcSize` and `dictSize`.
 *   Size values are optional, provide 0 if not known or unused */
@@ -4108,6 +4145,15 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long l
     U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB);   /* intentional underflow for srcSizeHint == 0 */
     if (compressionLevel <= 0) compressionLevel = ZSTD_CLEVEL_DEFAULT;   /* 0 == default; no negative compressionLevel yet */
     if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL;
+
+#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=1)
+    static int g_monotonicTest = 1;
+    if (g_monotonicTest) {
+        ZSTD_check_compressionLevel_monotonicIncrease_memoryBudget();
+        g_monotonicTest=0;
+    }
+#endif
+
     { ZSTD_compressionParameters const cp = ZSTD_defaultCParameters[tableID][compressionLevel];
       return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize); }
 }
diff --git a/lib/zstd.h b/lib/zstd.h
index 08358f0df..75bdb4895 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -496,9 +496,9 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
 /*! ZSTD_estimate*() :
  *  These functions make it possible to estimate memory usage
  *  of a future {D,C}Ctx, before its creation.
- *  The objective is to guide decision before allocation.
- *  ZSTD_estimateCCtxSize() will consider src size to be arbitrarily "large".
- *  If srcSize is known to be small, ZSTD_estimateCCtxSize_advanced() can provide a tighter estimation.
+ *  ZSTD_estimateCCtxSize() will provide a budget large enough for any compression level up to selected one.
+ *  It will also consider src size to be arbitrarily "large", which is worst case.
+ *  If srcSize is known to always be small, ZSTD_estimateCCtxSize_advanced() can provide a tighter estimation.
  *  ZSTD_estimateCCtxSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
  *  Note : CCtx estimation is only correct for single-threaded compression */
 ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel);
@@ -506,8 +506,9 @@ ZSTDLIB_API size_t ZSTD_estimateCCtxSize_advanced(ZSTD_compressionParameters cPa
 ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void);
 
 /*! ZSTD_estimate?StreamSize() :
- *  ZSTD_estimateCStreamSize() will consider src size to be arbitrarily "large".
- *  If srcSize is known to be small, ZSTD_estimateCStreamSize_advanced() can provide a tighter estimation.
+ *  ZSTD_estimateCStreamSize() will provide a budget large enough for any compression level up to selected one.
+ *  It will also consider src size to be arbitrarily "large", which is worst case.
+ *  If srcSize is known to always be small, ZSTD_estimateCStreamSize_advanced() can provide a tighter estimation.
  *  ZSTD_estimateCStreamSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
  *  Note : CStream estimation is only correct for single-threaded compression.
  *  ZSTD_DStream memory budget depends on window Size.

From 1ca76039af730c11fc761961a9ccb7de99d4c057 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 28 Jun 2017 15:40:21 -0700
Subject: [PATCH 8/8] fixed -Wdeclaration-after-statement

---
 lib/compress/zstd_compress.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index c2adc56ab..5c2035a24 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -4143,8 +4143,6 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long l
     size_t const addedSize = srcSizeHint ? 0 : 500;
     U64 const rSize = srcSizeHint+dictSize ? srcSizeHint+dictSize+addedSize : (U64)-1;
     U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB);   /* intentional underflow for srcSizeHint == 0 */
-    if (compressionLevel <= 0) compressionLevel = ZSTD_CLEVEL_DEFAULT;   /* 0 == default; no negative compressionLevel yet */
-    if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL;
 
 #if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=1)
     static int g_monotonicTest = 1;
@@ -4154,6 +4152,8 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long l
     }
 #endif
 
+    if (compressionLevel <= 0) compressionLevel = ZSTD_CLEVEL_DEFAULT;   /* 0 == default; no negative compressionLevel yet */
+    if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL;
     { ZSTD_compressionParameters const cp = ZSTD_defaultCParameters[tableID][compressionLevel];
       return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize); }
 }