These functions make it possible to estimate memory usage of a future target object, before its allocation, given a set of parameters, which vary depending on target object. - The objective is to guide decision before allocation. + The objective is to guide decision before allocation. + Note : CCtx estimation is only correct for single-threaded compression
size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams);
@@ -575,17 +576,17 @@ size_t ZSTD_estimateDDictSize(size_t dictSize, unsigned byReference);
ZSTD_p_rawContentDict, /* load dictionary in "content-only" mode (no header analysis) (default:0) */
/* question : should there be an option to load dictionary only in zstd format, rejecting others with an error code ? */
-#if 0
- /* multi-threading parameters (not ready yet !) */
+ /* multi-threading parameters */
ZSTD_p_nbThreads=400, /* Select how many threads a compression job can spawn (default:1)
- * More threads improve speed, but increases also memory usage */
- ZSTDMT_p_jobSize, /* Size of a compression job. Each job is compressed in parallel.
+ * More threads improve speed, but also increase memory usage.
+ * Can only receive a value > 1 if ZSTD_MULTITHREAD is enabled.
+ * Special: value 0 means "do not change nbThreads" */
+ ZSTDMT_p_jobSize, /* Size of a compression job. Each compression job is completed in parallel.
* 0 means default, which is dynamically determined based on compression parameters.
* Job size must be a minimum of overlapSize, or 1 KB, whichever is largest
* The minimum size is automatically and transparently enforced */
ZSTDMT_p_overlapSizeLog, /* Size of previous input reloaded at the beginning of each job.
* 0 => no overlap, 6(default) => use 1/8th of windowSize, >=9 => use full windowSize */
-#endif
/* advanced parameters - may not remain available after API update */
ZSTD_p_forceMaxWindow=1100, /* Force back-references to remain < windowSize,
diff --git a/lib/common/bitstream.h b/lib/common/bitstream.h
index 74eb4b635..f3c9064bb 100644
--- a/lib/common/bitstream.h
+++ b/lib/common/bitstream.h
@@ -58,7 +58,9 @@ extern "C" {
#if defined(BIT_DEBUG) && (BIT_DEBUG>=1)
# include
#else
-# define assert(condition) ((void)0)
+# ifndef assert
+# define assert(condition) ((void)0)
+# endif
#endif
@@ -307,19 +309,19 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24);
/* fall-through */
-
+
case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32);
/* fall-through */
-
+
case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24;
/* fall-through */
-
+
case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16;
/* fall-through */
-
+
case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8;
/* fall-through */
-
+
default: break;
}
{ BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index 6f9b4ef1f..8c627cb7a 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -51,9 +51,36 @@
#define ZSTD_STATIC_LINKING_ONLY
#include "zstd.h"
#ifndef XXH_STATIC_LINKING_ONLY
-# define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
+# define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
+#endif
+#include "xxhash.h" /* XXH_reset, update, digest */
+
+
+/*-*************************************
+* Debug
+***************************************/
+#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=1)
+# include
+#else
+# ifndef assert
+# define assert(condition) ((void)0)
+# endif
+#endif
+
+#define ZSTD_STATIC_ASSERT(c) { enum { ZSTD_static_assert = 1/(int)(!!(c)) }; }
+
+#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=2)
+# include
+ static unsigned g_debugLevel = ZSTD_DEBUG;
+# define DEBUGLOG(l, ...) { \
+ if (l<=g_debugLevel) { \
+ fprintf(stderr, __FILE__ ": "); \
+ fprintf(stderr, __VA_ARGS__); \
+ fprintf(stderr, " \n"); \
+ } }
+#else
+# define DEBUGLOG(l, ...) {} /* disabled */
#endif
-#include "xxhash.h" /* XXH_reset, update, digest */
/*-*************************************
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 8f7c200b2..76d3614c7 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -28,31 +28,6 @@
#include "zstdmt_compress.h"
-/*-*************************************
-* Debug
-***************************************/
-#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=1)
-# include
-#else
-# define assert(condition) ((void)0)
-#endif
-
-#define ZSTD_STATIC_ASSERT(c) { enum { ZSTD_static_assert = 1/(int)(!!(c)) }; }
-
-#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=2)
-# include
- static unsigned g_debugLevel = ZSTD_DEBUG;
-# define DEBUGLOG(l, ...) { \
- if (l<=g_debugLevel) { \
- fprintf(stderr, __FILE__ ": "); \
- fprintf(stderr, __VA_ARGS__); \
- fprintf(stderr, " \n"); \
- } }
-#else
-# define DEBUGLOG(l, ...) {} /* disabled */
-#endif
-
-
/*-*************************************
* Constants
***************************************/
@@ -3622,17 +3597,19 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
ZSTD_inBuffer* input,
ZSTD_EndDirective const flushMode)
{
- U32 someMoreWork = 1;
const char* const istart = (const char*)input->src;
const char* const iend = istart + input->size;
const char* ip = istart + input->pos;
char* const ostart = (char*)output->dst;
char* const oend = ostart + output->size;
char* op = ostart + output->pos;
+ U32 someMoreWork = 1;
- /* expected to be already allocated */
+ /* check expectations */
assert(zcs->inBuff != NULL);
assert(zcs->outBuff!= NULL);
+ assert(output->pos <= output->size);
+ assert(input->pos <= input->size);
while (someMoreWork) {
switch(zcs->streamStage)
@@ -3765,7 +3742,6 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
if (cctx->streamStage == zcss_init) {
/* transparent reset */
ZSTD_parameters params = cctx->requestedParams;
- DEBUGLOG(5, "ZSTD_compress_generic : transparent reset");
if (cctx->compressionLevel != ZSTD_CLEVEL_CUSTOM)
params.cParams = ZSTD_getCParams(cctx->compressionLevel,
cctx->frameContentSize, 0 /* dictSize */);
@@ -3775,7 +3751,7 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
DEBUGLOG(5, "starting ZSTD_compressStream_generic");
CHECK_F( ZSTD_compressStream_generic(cctx, output, input, endOp) );
- DEBUGLOG(5, "completing ZSTD_compress_generic_integral");
+ DEBUGLOG(5, "completing ZSTD_compress_generic");
return cctx->outBuffContentSize - cctx->outBuffFlushedSize; /* remaining to flush */
}
@@ -3788,12 +3764,10 @@ size_t ZSTD_compress_generic_simpleArgs (
ZSTD_outBuffer output = { dst, dstCapacity, *dstPos };
ZSTD_inBuffer input = { src, srcSize, *srcPos };
/* ZSTD_compress_generic() will check validity of dstPos and srcPos */
- size_t const hint = ZSTD_compress_generic(cctx, &output, &input, endOp);
- if (ZSTD_isError(hint)) return hint;
-
+ size_t const cErr = ZSTD_compress_generic(cctx, &output, &input, endOp);
*dstPos = output.pos;
*srcPos = input.pos;
- return hint;
+ return cErr;
}
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 0d6014608..54feb86f5 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -14,25 +14,19 @@
/* ====== Compiler specifics ====== */
#if defined(_MSC_VER)
-# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
+# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
#endif
/* ====== Dependencies ====== */
-#include /* memcpy, memset */
-#include "pool.h" /* threadpool */
-#include "threading.h" /* mutex */
+#include /* memcpy, memset */
+#include "pool.h" /* threadpool */
+#include "threading.h" /* mutex */
#include "zstd_internal.h" /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */
#include "zstdmt_compress.h"
/* ====== Debug ====== */
-#if defined(ZSTDMT_DEBUG) && (ZSTDMT_DEBUG>=1)
-# include
-#else
-# define assert(condition) ((void)0)
-#endif
-
#if defined(ZSTDMT_DEBUG) && (ZSTDMT_DEBUG>=2)
# include
@@ -573,17 +567,15 @@ static void ZSTDMT_waitForAllJobsCompleted(ZSTDMT_CCtx* zcs)
static size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
- const void* dict, size_t dictSize, unsigned updateDict,
- ZSTD_parameters params, unsigned long long pledgedSrcSize)
+ const void* dict, size_t dictSize, unsigned updateDict,
+ ZSTD_parameters params, unsigned long long pledgedSrcSize)
{
- ZSTD_customMem const cmem = { NULL, NULL, NULL };
- DEBUGLOG(3, "Started new compression, with windowLog : %u",
- params.cParams.windowLog);
if (zcs->nbThreads==1)
return ZSTD_initCStream_advanced(zcs->cctxPool->cctx[0],
dict, dictSize,
params, pledgedSrcSize);
- if (zcs->allJobsCompleted == 0) { /* previous job not correctly finished */
+
+ if (zcs->allJobsCompleted == 0) { /* previous compression not correctly finished */
ZSTDMT_waitForAllJobsCompleted(zcs);
ZSTDMT_releaseAllJobResources(zcs);
zcs->allJobsCompleted = 1;
@@ -592,7 +584,8 @@ static size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
if (updateDict) {
ZSTD_freeCDict(zcs->cdict); zcs->cdict = NULL;
if (dict && dictSize) {
- zcs->cdict = ZSTD_createCDict_advanced(dict, dictSize, 0, params.cParams, cmem);
+ zcs->cdict = ZSTD_createCDict_advanced(dict, dictSize, 0 /* byRef */,
+ params.cParams, zcs->cMem);
if (zcs->cdict == NULL) return ERROR(memory_allocation);
} }
zcs->frameContentSize = pledgedSrcSize;
diff --git a/lib/compress/zstdmt_compress.h b/lib/compress/zstdmt_compress.h
index adbb7dfe3..ceff32ef6 100644
--- a/lib/compress/zstdmt_compress.h
+++ b/lib/compress/zstdmt_compress.h
@@ -15,8 +15,8 @@
#endif
-/* Note : All prototypes defined in this file shall be considered experimental.
- * There is no guarantee of API continuity (yet) on any of these prototypes */
+/* Note : All prototypes defined in this file must be considered experimental.
+ * There is no guarantee of API continuity on any of these prototypes */
/* === Dependencies === */
#include /* size_t */
@@ -60,8 +60,10 @@ ZSTDLIB_API size_t ZSTDMT_endStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output);
# define ZSTDMT_SECTION_SIZE_MIN (1U << 20) /* 1 MB - Minimum size of each compression job */
#endif
-ZSTDLIB_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx, const void* dict, size_t dictSize, /**< dict can be released after init, a local copy is preserved within zcs */
- ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */
+ZSTDLIB_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx,
+ const void* dict, size_t dictSize, /* dict can be released after init, a local copy is preserved within zcs */
+ ZSTD_parameters params,
+ unsigned long long pledgedSrcSize); /* pledgedSrcSize is optional and can be zero == unknown */
/* ZSDTMT_parameter :
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 68ec13d1a..bb3eed399 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -70,19 +70,6 @@
#define FSE_isError ERR_isError
#define HUF_isError ERR_isError
-#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=2)
-# include
- static unsigned g_debugLevel = ZSTD_DEBUG;
-# define DEBUGLOG(l, ...) { \
- if (l<=g_debugLevel) { \
- fprintf(stderr, __FILE__ ": "); \
- fprintf(stderr, __VA_ARGS__); \
- fprintf(stderr, " \n"); \
- } }
-#else
-# define DEBUGLOG(l, ...) {} /* disabled */
-#endif
-
/*_*******************************************************
* Memory operations