made debug definitions common within zstd_internal.h

doc/zstd_manual.html

@@ -405,6 +405,7 @@ size_t ZSTD_estimateDCtxSize(void);
   of a future target object, before its allocation,
   given a set of parameters, which vary depending on target object.
   The objective is to guide decision before allocation.
+  Note : CCtx estimation is only correct for single-threaded compression 
 </p></pre><BR>
 
 <pre><b>size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams);
@@ -575,17 +576,17 @@ size_t ZSTD_estimateDDictSize(size_t dictSize, unsigned byReference);
     ZSTD_p_rawContentDict,   </b>/* load dictionary in "content-only" mode (no header analysis) (default:0) */<b>
                              </b>/* question : should there be an option to load dictionary only in zstd format, rejecting others with an error code ? */<b>
 
-#if 0
+    </b>/* multi-threading parameters */<b>
-    </b>/* multi-threading parameters (not ready yet !) */<b>
     ZSTD_p_nbThreads=400,    </b>/* Select how many threads a compression job can spawn (default:1)<b>
-                              * More threads improve speed, but increases also memory usage */
+                              * More threads improve speed, but also increase memory usage.
-    ZSTDMT_p_jobSize,        </b>/* Size of a compression job. Each job is compressed in parallel.<b>
+                              * Can only receive a value > 1 if ZSTD_MULTITHREAD is enabled.
+                              * Special: value 0 means "do not change nbThreads" */
+    ZSTDMT_p_jobSize,        </b>/* Size of a compression job. Each compression job is completed in parallel.<b>
                               * 0 means default, which is dynamically determined based on compression parameters.
                               * Job size must be a minimum of overlapSize, or 1 KB, whichever is largest
                               * The minimum size is automatically and transparently enforced */
     ZSTDMT_p_overlapSizeLog, </b>/* Size of previous input reloaded at the beginning of each job.<b>
                               * 0 => no overlap, 6(default) => use 1/8th of windowSize, >=9 => use full windowSize */
-#endif
 
     </b>/* advanced parameters - may not remain available after API update */<b>
     ZSTD_p_forceMaxWindow=1100, </b>/* Force back-references to remain < windowSize,<b>

lib/common/bitstream.h

@@ -58,7 +58,9 @@ extern "C" {
 #if defined(BIT_DEBUG) && (BIT_DEBUG>=1)
 #  include <assert.h>
 #else
-#  define assert(condition) ((void)0)
+#  ifndef assert
+#    define assert(condition) ((void)0)
+#  endif
 #endif
 
 

lib/common/zstd_internal.h

@@ -51,9 +51,36 @@
 #define ZSTD_STATIC_LINKING_ONLY
 #include "zstd.h"
 #ifndef XXH_STATIC_LINKING_ONLY
-#  define XXH_STATIC_LINKING_ONLY   /* XXH64_state_t */
+#  define XXH_STATIC_LINKING_ONLY  /* XXH64_state_t */
+#endif
+#include "xxhash.h"                /* XXH_reset, update, digest */
+
+
+/*-*************************************
+*  Debug
+***************************************/
+#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=1)
+#  include <assert.h>
+#else
+#  ifndef assert
+#    define assert(condition) ((void)0)
+#  endif
+#endif
+
+#define ZSTD_STATIC_ASSERT(c) { enum { ZSTD_static_assert = 1/(int)(!!(c)) }; }
+
+#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=2)
+#  include <stdio.h>
+   static unsigned g_debugLevel = ZSTD_DEBUG;
+#  define DEBUGLOG(l, ...) {                          \
+                if (l<=g_debugLevel) {                \
+                    fprintf(stderr, __FILE__ ": ");   \
+                    fprintf(stderr, __VA_ARGS__);     \
+                    fprintf(stderr, " \n");           \
+            }   }
+#else
+#  define DEBUGLOG(l, ...)      {}    /* disabled */
 #endif
-#include "xxhash.h"               /* XXH_reset, update, digest */
 
 
 /*-*************************************

lib/compress/zstd_compress.c

@@ -28,31 +28,6 @@
 #include "zstdmt_compress.h"
 
 
-/*-*************************************
-*  Debug
-***************************************/
-#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=1)
-#  include <assert.h>
-#else
-#  define assert(condition) ((void)0)
-#endif
-
-#define ZSTD_STATIC_ASSERT(c) { enum { ZSTD_static_assert = 1/(int)(!!(c)) }; }
-
-#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=2)
-#  include <stdio.h>
-   static unsigned g_debugLevel = ZSTD_DEBUG;
-#  define DEBUGLOG(l, ...) {                          \
-                if (l<=g_debugLevel) {                \
-                    fprintf(stderr, __FILE__ ": ");   \
-                    fprintf(stderr, __VA_ARGS__);     \
-                    fprintf(stderr, " \n");           \
-            }   }
-#else
-#  define DEBUGLOG(l, ...)      {}    /* disabled */
-#endif
-
-
 /*-*************************************
 *  Constants
 ***************************************/
@@ -3622,17 +3597,19 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
                                         ZSTD_inBuffer* input,
                                         ZSTD_EndDirective const flushMode)
 {
-    U32 someMoreWork = 1;
     const char* const istart = (const char*)input->src;
     const char* const iend = istart + input->size;
     const char* ip = istart + input->pos;
     char* const ostart = (char*)output->dst;
     char* const oend = ostart + output->size;
     char* op = ostart + output->pos;
+    U32 someMoreWork = 1;
 
-    /* expected to be already allocated */
+    /* check expectations */
     assert(zcs->inBuff != NULL);
     assert(zcs->outBuff!= NULL);
+    assert(output->pos <= output->size);
+    assert(input->pos <= input->size);
 
     while (someMoreWork) {
         switch(zcs->streamStage)
@@ -3765,7 +3742,6 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
     if (cctx->streamStage == zcss_init) {
         /* transparent reset */
         ZSTD_parameters params = cctx->requestedParams;
-        DEBUGLOG(5, "ZSTD_compress_generic : transparent reset");
         if (cctx->compressionLevel != ZSTD_CLEVEL_CUSTOM)
             params.cParams = ZSTD_getCParams(cctx->compressionLevel,
                                     cctx->frameContentSize, 0 /* dictSize */);
@@ -3775,7 +3751,7 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
     DEBUGLOG(5, "starting ZSTD_compressStream_generic");
     CHECK_F( ZSTD_compressStream_generic(cctx, output, input, endOp) );
 
-    DEBUGLOG(5, "completing ZSTD_compress_generic_integral");
+    DEBUGLOG(5, "completing ZSTD_compress_generic");
     return cctx->outBuffContentSize - cctx->outBuffFlushedSize; /* remaining to flush */
 }
 
@@ -3788,12 +3764,10 @@ size_t ZSTD_compress_generic_simpleArgs (
     ZSTD_outBuffer output = { dst, dstCapacity, *dstPos };
     ZSTD_inBuffer  input  = { src, srcSize, *srcPos };
     /* ZSTD_compress_generic() will check validity of dstPos and srcPos */
-    size_t const hint = ZSTD_compress_generic(cctx, &output, &input, endOp);
+    size_t const cErr = ZSTD_compress_generic(cctx, &output, &input, endOp);
-    if (ZSTD_isError(hint)) return hint;
-
     *dstPos = output.pos;
     *srcPos = input.pos;
-    return hint;
+    return cErr;
 
 }
 

lib/compress/zstdmt_compress.c

@@ -14,25 +14,19 @@
 
 /* ======   Compiler specifics   ====== */
 #if defined(_MSC_VER)
-#  pragma warning(disable : 4204)        /* disable: C4204: non-constant aggregate initializer */
+#  pragma warning(disable : 4204)   /* disable: C4204: non-constant aggregate initializer */
 #endif
 
 
 /* ======   Dependencies   ====== */
-#include <string.h>     /* memcpy, memset */
+#include <string.h>      /* memcpy, memset */
-#include "pool.h"       /* threadpool */
+#include "pool.h"        /* threadpool */
-#include "threading.h"  /* mutex */
+#include "threading.h"   /* mutex */
 #include "zstd_internal.h"  /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */
 #include "zstdmt_compress.h"
 
 
 /* ======   Debug   ====== */
-#if defined(ZSTDMT_DEBUG) && (ZSTDMT_DEBUG>=1)
-#  include <assert.h>
-#else
-#  define assert(condition) ((void)0)
-#endif
-
 #if defined(ZSTDMT_DEBUG) && (ZSTDMT_DEBUG>=2)
 
 #  include <stdio.h>
@@ -573,17 +567,15 @@ static void ZSTDMT_waitForAllJobsCompleted(ZSTDMT_CCtx* zcs)
 
 
 static size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
-                                    const void* dict, size_t dictSize, unsigned updateDict,
+                        const void* dict, size_t dictSize, unsigned updateDict,
-                                    ZSTD_parameters params, unsigned long long pledgedSrcSize)
+                        ZSTD_parameters params, unsigned long long pledgedSrcSize)
 {
-    ZSTD_customMem const cmem = { NULL, NULL, NULL };
-    DEBUGLOG(3, "Started new compression, with windowLog : %u",
-                    params.cParams.windowLog);
     if (zcs->nbThreads==1)
         return ZSTD_initCStream_advanced(zcs->cctxPool->cctx[0],
                                         dict, dictSize,
                                         params, pledgedSrcSize);
-    if (zcs->allJobsCompleted == 0) {   /* previous job not correctly finished */
+
+    if (zcs->allJobsCompleted == 0) {   /* previous compression not correctly finished */
         ZSTDMT_waitForAllJobsCompleted(zcs);
         ZSTDMT_releaseAllJobResources(zcs);
         zcs->allJobsCompleted = 1;
@@ -592,7 +584,8 @@ static size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
     if (updateDict) {
         ZSTD_freeCDict(zcs->cdict); zcs->cdict = NULL;
         if (dict && dictSize) {
-            zcs->cdict = ZSTD_createCDict_advanced(dict, dictSize, 0, params.cParams, cmem);
+            zcs->cdict = ZSTD_createCDict_advanced(dict, dictSize, 0 /* byRef */,
+                                                params.cParams, zcs->cMem);
             if (zcs->cdict == NULL) return ERROR(memory_allocation);
     }   }
     zcs->frameContentSize = pledgedSrcSize;

lib/compress/zstdmt_compress.h

@@ -15,8 +15,8 @@
  #endif
 
 
-/* Note : All prototypes defined in this file shall be considered experimental.
+/* Note : All prototypes defined in this file must be considered experimental.
- *        There is no guarantee of API continuity (yet) on any of these prototypes */
+ *        There is no guarantee of API continuity on any of these prototypes */
 
 /* ===   Dependencies   === */
 #include <stddef.h>   /* size_t */
@@ -60,8 +60,10 @@ ZSTDLIB_API size_t ZSTDMT_endStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output);
 #  define ZSTDMT_SECTION_SIZE_MIN (1U << 20)   /* 1 MB - Minimum size of each compression job */
 #endif
 
-ZSTDLIB_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx, const void* dict, size_t dictSize,  /**< dict can be released after init, a local copy is preserved within zcs */
+ZSTDLIB_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx,
-                                          ZSTD_parameters params, unsigned long long pledgedSrcSize);  /**< pledgedSrcSize is optional and can be zero == unknown */
+                                        const void* dict, size_t dictSize,   /* dict can be released after init, a local copy is preserved within zcs */
+                                        ZSTD_parameters params,
+                                        unsigned long long pledgedSrcSize);  /* pledgedSrcSize is optional and can be zero == unknown */
 
 
 /* ZSDTMT_parameter :

lib/decompress/zstd_decompress.c

@@ -70,19 +70,6 @@
 #define FSE_isError  ERR_isError
 #define HUF_isError  ERR_isError
 
-#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=2)
-#  include <stdio.h>
-   static unsigned g_debugLevel = ZSTD_DEBUG;
-#  define DEBUGLOG(l, ...) {                          \
-                if (l<=g_debugLevel) {                \
-                    fprintf(stderr, __FILE__ ": ");   \
-                    fprintf(stderr, __VA_ARGS__);     \
-                    fprintf(stderr, " \n");           \
-            }   }
-#else
-#  define DEBUGLOG(l, ...)      {}    /* disabled */
-#endif
-
 
 /*_*******************************************************
 *  Memory operations