improved ZSTDMT_compress() memory usage

does not need the input buffer for streaming operations

also : reduced a few tests time length

lib/compress/zstdmt_compress.c

@@ -134,9 +134,11 @@ static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* pool, size_t bSize)
             /* large enough, but not too much */
             return buf;
         /* size conditions not respected : scratch this buffer, create new one */
+        DEBUGLOG(2, "existing buffer does not meet size conditions => freeing");
         ZSTD_free(buf.start, pool->cMem);
     }
     /* create new buffer */
+    DEBUGLOG(2, "create a new buffer");
     {   buffer_t buffer;
         void* const start = ZSTD_malloc(bSize, pool->cMem);
         if (start==NULL) bSize = 0;
@@ -149,12 +151,14 @@ static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* pool, size_t bSize)
 /* store buffer for later re-use, up to pool capacity */
 static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* pool, buffer_t buf)
 {
+    DEBUGLOG(2, "ZSTDMT_releaseBuffer");
     if (buf.start == NULL) return;   /* release on NULL */
     if (pool->nbBuffers < pool->totalBuffers) {
         pool->bTable[pool->nbBuffers++] = buf;   /* store for later re-use */
         return;
     }
     /* Reached bufferPool capacity (should not happen) */
+    DEBUGLOG(2, "buffer pool capacity reached => freeing ");
     ZSTD_free(buf.start, pool->cMem);
 }
 
@@ -635,8 +639,8 @@ size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
     if (zcs->nbThreads==1) {
         DEBUGLOG(4, "single thread mode");
         return ZSTD_initCStream_internal(zcs->cctxPool->cctx[0],
-                                dict, dictSize, cdict,
+                                        dict, dictSize, cdict,
-                                params, pledgedSrcSize);
+                                        params, pledgedSrcSize);
     }
 
     if (zcs->allJobsCompleted == 0) {   /* previous compression not correctly finished */
@@ -671,9 +675,7 @@ size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
     DEBUGLOG(4, "Section Size : %u KB", (U32)(zcs->targetSectionSize>>10));
     zcs->marginSize = zcs->targetSectionSize >> 2;
     zcs->inBuffSize = zcs->targetDictSize + zcs->targetSectionSize + zcs->marginSize;
-    zcs->inBuff.buffer = ZSTDMT_getBuffer(zcs->buffPool, zcs->inBuffSize);
+    zcs->inBuff.buffer = g_nullBuffer;
-    if (zcs->inBuff.buffer.start == NULL) return ERROR(memory_allocation);
-    zcs->inBuff.filled = 0;
     zcs->dictSize = 0;
     zcs->doneJobID = 0;
     zcs->nextJobID = 0;
@@ -899,13 +901,18 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
     }
 
     /* fill input buffer */
-    if ((input->src) && (mtctx->inBuff.buffer.start)) {   /* support NULL input */
+    if (input->src) {   /* support NULL input */
-        size_t const toLoad = MIN(input->size - input->pos, mtctx->inBuffSize - mtctx->inBuff.filled);
+        if (mtctx->inBuff.buffer.start == NULL) {
-        DEBUGLOG(5, "inBuff:%08X;  inBuffSize=%u;  ToCopy=%u", (U32)(size_t)mtctx->inBuff.buffer.start, (U32)mtctx->inBuffSize, (U32)toLoad);
+            mtctx->inBuff.buffer = ZSTDMT_getBuffer(mtctx->buffPool, mtctx->inBuffSize);
-        memcpy((char*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled, (const char*)input->src + input->pos, toLoad);
+            if (mtctx->inBuff.buffer.start == NULL) return ERROR(memory_allocation);
-        input->pos += toLoad;
+            mtctx->inBuff.filled = 0;
-        mtctx->inBuff.filled += toLoad;
+        }
-    }
+        {   size_t const toLoad = MIN(input->size - input->pos, mtctx->inBuffSize - mtctx->inBuff.filled);
+            DEBUGLOG(5, "inBuff:%08X;  inBuffSize=%u;  ToCopy=%u", (U32)(size_t)mtctx->inBuff.buffer.start, (U32)mtctx->inBuffSize, (U32)toLoad);
+            memcpy((char*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled, (const char*)input->src + input->pos, toLoad);
+            input->pos += toLoad;
+            mtctx->inBuff.filled += toLoad;
+    }   }
 
     if ( (mtctx->inBuff.filled >= newJobThreshold)  /* filled enough : let's compress */
       && (mtctx->nextJobID <= mtctx->doneJobID + mtctx->jobIDMask) ) {   /* avoid overwriting job round buffer */

tests/fuzzer.c

@@ -129,7 +129,7 @@ static void* FUZ_mallocDebug(void* counter, size_t size)
 static void FUZ_freeDebug(void* counter, void* address)
 {
     mallocCounter_t* const mcPtr = (mallocCounter_t*)counter;
-    DISPLAYLEVEL(4, "releasing %u KB \n", (U32)(malloc_size(address) >> 10));
+    DISPLAYLEVEL(4, "freeing %u KB \n", (U32)(malloc_size(address) >> 10));
     mcPtr->nbFree += 1;
     mcPtr->currentMalloc -= malloc_size(address);  /* OS-X specific */
     free(address);

tests/playTests.sh

@@ -7,17 +7,17 @@ die() {
 
 roundTripTest() {
     if [ -n "$3" ]; then
-        local_c="$3"
+        cLevel="$3"
-        local_p="$2"
+        proba="$2"
     else
-        local_c="$2"
+        cLevel="$2"
-        local_p=""
+        proba=""
     fi
 
     rm -f tmp1 tmp2
-    $ECHO "roundTripTest: ./datagen $1 $local_p | $ZSTD -v$local_c | $ZSTD -d"
+    $ECHO "roundTripTest: ./datagen $1 $proba | $ZSTD -v$cLevel | $ZSTD -d"
-    ./datagen $1 $local_p | $MD5SUM > tmp1
+    ./datagen $1 $proba | $MD5SUM > tmp1
-    ./datagen $1 $local_p | $ZSTD --ultra -v$local_c | $ZSTD -d  | $MD5SUM > tmp2
+    ./datagen $1 $proba | $ZSTD --ultra -v$cLevel | $ZSTD -d  | $MD5SUM > tmp2
     $DIFF -q tmp1 tmp2
 }
 
@@ -625,16 +625,15 @@ roundTripTest -g35000000 -P75 10
 roundTripTest -g35000000 -P75 11
 roundTripTest -g35000000 -P75 12
 
-roundTripTest -g18000000 -P80 13
+roundTripTest -g18000013 -P80 13
-roundTripTest -g18000000 -P80 14
+roundTripTest -g18000014 -P80 14
-roundTripTest -g18000000 -P80 15
+roundTripTest -g18000015 -P80 15
-roundTripTest -g18000000 -P80 16
+roundTripTest -g18000016 -P80 16
-roundTripTest -g18000000 -P80 17
+roundTripTest -g18000017 -P80 17
+roundTripTest -g18000018 -P94 18
+roundTripTest -g18000019 -P94 19
 
-roundTripTest -g50000000 -P94 18
+roundTripTest -g68000020 -P99 20
-roundTripTest -g50000000 -P94 19
-
-roundTripTest -g99000000 -P99 20
 roundTripTest -g6000000000 -P99 1
 
 fileRoundTripTest -g4193M -P99 1

tests/zstreamtest.c

@@ -95,19 +95,6 @@ unsigned int FUZ_rand(unsigned int* seedPtr)
     return rand32 >> 5;
 }
 
-static void* allocFunction(void* opaque, size_t size)
-{
-    void* address = malloc(size);
-    (void)opaque;
-    return address;
-}
-
-static void freeFunction(void* opaque, void* address)
-{
-    (void)opaque;
-    free(address);
-}
-
 
 /*======================================================
 *   Basic Unit tests
@@ -1543,7 +1530,6 @@ int main(int argc, const char** argv)
     int bigTests = (sizeof(size_t) == 8);
     e_api selected_api = simple_api;
     const char* const programName = argv[0];
-    ZSTD_customMem const customMem = { allocFunction, freeFunction, NULL };
     ZSTD_customMem const customNULL = ZSTD_defaultCMem;
 
     /* Check command line */
@@ -1657,10 +1643,7 @@ int main(int argc, const char** argv)
 
     if (testNb==0) {
         result = basicUnitTests(0, ((double)proba) / 100, customNULL);  /* constant seed for predictability */
-        if (!result) {
+    }
-            DISPLAYLEVEL(3, "Unit tests using customMem :\n")
-            result = basicUnitTests(0, ((double)proba) / 100, customMem);  /* use custom memory allocation functions */
-    }   }
 
     if (!result) {
         switch(selected_api)