Merge pull request #393 from inikep/zlibWrapper

Zlibwrapper

zlibWrapper/README.md

@@ -85,7 +85,7 @@ During streaming compression the compressor never knows how big is data to compr
 Zstandard compression can be improved by providing size of source data to the compressor. By default streaming compressor assumes that data is bigger than 256 KB but it can hurt compression speed on smaller data. 
 The zstd wrapper provides the `ZWRAP_setPledgedSrcSize()` function that allows to change a pledged source size for a given compression stream.
 The function will change zstd compression parameters what may improve compression speed and/or ratio.
-It should be called just after `deflateInit()`. The function is only helpful when data is compressed in blocks. There will be no change in case of `deflateInit()` immediately followed by `deflate(strm, Z_FINISH)`
+It should be called just after `deflateInit()`or `deflateReset()` and before `deflate()` or `deflateSetDictionary()`. The function is only helpful when data is compressed in blocks. There will be no change in case of `deflateInit()` or `deflateReset()`  immediately followed by `deflate(strm, Z_FINISH)`
 as this case is automatically detected.
 
 

zlibWrapper/examples/zwrapbench.c

@@ -282,6 +282,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                 } else if (compressor == BMK_ZWRAP_ZLIB_REUSE || compressor == BMK_ZWRAP_ZSTD_REUSE || compressor == BMK_ZLIB_REUSE) {
                     z_stream def;
                     int ret;
+                    int useSetDict = (dictBuffer != NULL);
                     if (compressor == BMK_ZLIB_REUSE || compressor == BMK_ZWRAP_ZLIB_REUSE) ZWRAP_useZSTDcompression(0);
                     else ZWRAP_useZSTDcompression(1);
                     def.zalloc = Z_NULL;
@@ -296,11 +297,15 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                     do {
                         U32 blockNb;
                         for (blockNb=0; blockNb<nbBlocks; blockNb++) {
-                            ret = deflateReset(&def);
+                            if (ZWRAP_isUsingZSTDcompression())
+                                ret = ZWRAP_deflateReset_keepDict(&def); /* reuse dictionary to make compression faster */
+                            else
+                                ret = deflateReset(&def);
                             if (ret != Z_OK) EXM_THROW(1, "deflateReset failure");
-                            if (dictBuffer) {
+                            if (useSetDict) {
                                 ret = deflateSetDictionary(&def, dictBuffer, dictBufferSize);
                                 if (ret != Z_OK) EXM_THROW(1, "deflateSetDictionary failure");
+                                if (ZWRAP_isUsingZSTDcompression()) useSetDict = 0; /* zstd doesn't require deflateSetDictionary after ZWRAP_deflateReset_keepDict */
                             }
                             def.next_in = (const void*) blockTable[blockNb].srcPtr;
                             def.avail_in = blockTable[blockNb].srcSize;
@@ -433,7 +438,10 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                     do {
                         U32 blockNb;
                         for (blockNb=0; blockNb<nbBlocks; blockNb++) {
-                            ret = inflateReset(&inf);
+                            if (ZWRAP_isUsingZSTDdecompression(&inf))
+                                ret = ZWRAP_inflateReset_keepDict(&inf); /* reuse dictionary to make decompression faster; inflate will return Z_NEED_DICT only for the first time */
+                            else
+                                ret = inflateReset(&inf);
                             if (ret != Z_OK) EXM_THROW(1, "inflateReset failure");
                             inf.next_in = (const void*) blockTable[blockNb].cPtr;
                             inf.avail_in = blockTable[blockNb].cSize;

zlibWrapper/zstd_zlibwrapper.c

@@ -74,6 +74,7 @@ static void ZWRAP_freeFunction(void* opaque, void* address)
 
 
 /* *** Compression *** */
+typedef enum { ZWRAP_useInit, ZWRAP_useReset, ZWRAP_streamEnd } ZWRAP_state_t;
 
 typedef struct {
     ZSTD_CStream* zbc;
@@ -82,7 +83,7 @@ typedef struct {
     z_stream allocFunc; /* copy of zalloc, zfree, opaque */
     ZSTD_inBuffer inBuffer;
     ZSTD_outBuffer outBuffer;
-    int comprState;
+    ZWRAP_state_t comprState;
     unsigned long long pledgedSrcSize;
 } ZWRAP_CCtx;
 
@@ -160,6 +161,7 @@ int ZWRAP_setPledgedSrcSize(z_streamp strm, unsigned long long pledgedSrcSize)
     if (zwc == NULL) return Z_STREAM_ERROR;
     
     zwc->pledgedSrcSize = pledgedSrcSize;
+    zwc->comprState = ZWRAP_useInit;
     return Z_OK;
 }
 
@@ -201,15 +203,26 @@ ZEXTERN int ZEXPORT z_deflateInit2_ OF((z_streamp strm, int level, int method,
 }
 
 
+int ZWRAP_deflateReset_keepDict(z_streamp strm)
+{
+    LOG_WRAPPERC("- ZWRAP_deflateReset_keepDict\n");
+    if (!g_ZWRAP_useZSTDcompression)
+        return deflateReset(strm);
+
+    strm->total_in = 0;
+    strm->total_out = 0;
+    strm->adler = 0;
+    return Z_OK;
+}
+
+
 ZEXTERN int ZEXPORT z_deflateReset OF((z_streamp strm))
 {
     LOG_WRAPPERC("- deflateReset\n");
     if (!g_ZWRAP_useZSTDcompression)
         return deflateReset(strm);
     
-    strm->total_in = 0;
+    ZWRAP_deflateReset_keepDict(strm);
-    strm->total_out = 0;
-    strm->adler = 0;
 
     { ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state;
       if (zwc) zwc->comprState = 0;
@@ -236,7 +249,7 @@ ZEXTERN int ZEXPORT z_deflateSetDictionary OF((z_streamp strm,
         }
         { int res = ZWRAP_initializeCStream(zwc, dictionary, dictLength, 0);
           if (res != Z_OK) return ZWRAPC_finishWithError(zwc, strm, res); }
-        zwc->comprState = Z_NEED_DICT;
+        zwc->comprState = ZWRAP_useReset;
     }
 
     return Z_OK;
@@ -263,14 +276,16 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush))
         if (zwc->zbc == NULL) return ZWRAPC_finishWithError(zwc, strm, 0); 
         res = ZWRAP_initializeCStream(zwc, NULL, 0, (flush == Z_FINISH) ? strm->avail_in : 0);
         if (res != Z_OK) return ZWRAPC_finishWithError(zwc, strm, res);
+        if (flush != Z_FINISH) zwc->comprState = ZWRAP_useReset;
     } else {
         if (strm->total_in == 0) {
-            if (zwc->comprState == Z_NEED_DICT) {
+            if (zwc->comprState == ZWRAP_useReset) {
                 size_t const errorCode = ZSTD_resetCStream(zwc->zbc, (flush == Z_FINISH) ? strm->avail_in : zwc->pledgedSrcSize);
                 if (ZSTD_isError(errorCode)) { LOG_WRAPPERC("ERROR: ZSTD_resetCStream errorCode=%s\n", ZSTD_getErrorName(errorCode)); return ZWRAPC_finishWithError(zwc, strm, 0); }
             } else {
                 int res = ZWRAP_initializeCStream(zwc, NULL, 0, (flush == Z_FINISH) ? strm->avail_in : 0);
                 if (res != Z_OK) return ZWRAPC_finishWithError(zwc, strm, res);
+                if (flush != Z_FINISH) zwc->comprState = ZWRAP_useReset;
             }
         }
     }
@@ -373,12 +388,13 @@ ZEXTERN int ZEXPORT z_deflateParams OF((z_streamp strm,
 
 
 /* *** Decompression *** */
+typedef enum { ZWRAP_ZLIB_STREAM, ZWRAP_ZSTD_STREAM, ZWRAP_UNKNOWN_STREAM } ZWRAP_stream_type;
 
 typedef struct {
     ZSTD_DStream* zbd;
     char headerBuf[16]; /* should be equal or bigger than ZSTD_frameHeaderSize_min */
     int errorCount;
-    int decompState;
+    ZWRAP_state_t decompState;
     ZSTD_inBuffer inBuffer;
     ZSTD_outBuffer outBuffer;
 
@@ -391,9 +407,16 @@ typedef struct {
 } ZWRAP_DCtx;
 
 
+int ZWRAP_isUsingZSTDdecompression(z_streamp strm) 
+{
+    if (strm == NULL) return 0;
+    return (strm->reserved == ZWRAP_ZSTD_STREAM);
+}
+
+
 void ZWRAP_initDCtx(ZWRAP_DCtx* zwd)
 {
-    zwd->errorCount = zwd->decompState = 0;
+    zwd->errorCount = 0;
     zwd->outBuffer.pos = 0;
     zwd->outBuffer.size = 0;
 }
@@ -457,6 +480,7 @@ ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm,
                                      const char *version, int stream_size))
 {
     if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB) {
+        strm->reserved = ZWRAP_ZLIB_STREAM; /* mark as zlib stream */
         return inflateInit(strm);
     }
 
@@ -473,7 +497,7 @@ ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm,
     strm->state = (struct internal_state*) zwd; /* use state which in not used by user */
     strm->total_in = 0;
     strm->total_out = 0;
-    strm->reserved = 1; /* mark as unknown steam */
+    strm->reserved = ZWRAP_UNKNOWN_STREAM; /* mark as unknown steam */
     strm->adler = 0;
     }
 
@@ -499,6 +523,23 @@ ZEXTERN int ZEXPORT z_inflateInit2_ OF((z_streamp strm, int  windowBits,
     }
 }
 
+int ZWRAP_inflateReset_keepDict(z_streamp strm)
+{
+    LOG_WRAPPERD("- ZWRAP_inflateReset_keepDict\n");
+    if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved)
+        return inflateReset(strm);
+
+    {   ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state;
+        if (zwd == NULL) return Z_STREAM_ERROR;
+        ZWRAP_initDCtx(zwd);
+        zwd->decompState = ZWRAP_useReset;
+    }
+    
+    strm->total_in = 0;
+    strm->total_out = 0;
+    return Z_OK;
+}
+
 
 ZEXTERN int ZEXPORT z_inflateReset OF((z_streamp strm))
 {
@@ -506,17 +547,13 @@ ZEXTERN int ZEXPORT z_inflateReset OF((z_streamp strm))
     if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved)
         return inflateReset(strm);
 
-    {   ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state;
+    { int ret = ZWRAP_inflateReset_keepDict(strm);
-        if (zwd == NULL) return Z_STREAM_ERROR;
+      if (ret != Z_OK) return ret; }
-        if (zwd->zbd) { 
+
-            size_t const errorCode = ZSTD_resetDStream(zwd->zbd);
+    { ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state;
-            if (ZSTD_isError(errorCode)) return ZWRAPD_finishWithError(zwd, strm, 0); 
+      if (zwd == NULL) return Z_STREAM_ERROR;    
-        }
+      zwd->decompState = ZWRAP_useInit; }
-        ZWRAP_initDCtx(zwd);
+
-    }
-    
-    strm->total_in = 0;
-    strm->total_out = 0;
     return Z_OK;
 }
 
@@ -553,7 +590,7 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm,
         if (zwd == NULL || zwd->zbd == NULL) return Z_STREAM_ERROR;
         errorCode = ZSTD_initDStream_usingDict(zwd->zbd, dictionary, dictLength);
         if (ZSTD_isError(errorCode)) return ZWRAPD_finishWithError(zwd, strm, 0);
-        zwd->decompState = Z_NEED_DICT; 
+        zwd->decompState = ZWRAP_useReset; 
 
         if (strm->total_in == ZSTD_HEADERSIZE) {
             zwd->inBuffer.src = zwd->headerBuf;
@@ -593,7 +630,7 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
         LOG_WRAPPERD("- inflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
 
         if (zwd == NULL) return Z_STREAM_ERROR;
-        if (zwd->decompState == Z_STREAM_END) return Z_STREAM_END;
+        if (zwd->decompState == ZWRAP_streamEnd) return Z_STREAM_END;
 
         if (strm->total_in < ZLIB_HEADERSIZE) {
             if (strm->total_in == 0 && strm->avail_in >= ZLIB_HEADERSIZE) {
@@ -603,7 +640,7 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
                     else
                         errorCode = inflateInit_(strm, zwd->version, zwd->stream_size);
 
-                    strm->reserved = 0; /* mark as zlib stream */
+                    strm->reserved = ZWRAP_ZLIB_STREAM; /* mark as zlib stream */
                     errorCode = ZWRAP_freeDCtx(zwd);
                     if (ZSTD_isError(errorCode)) goto error;
 
@@ -648,7 +685,7 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
                     strm->next_out = strm2.next_out;
                     strm->avail_out = strm2.avail_out;
 
-                    strm->reserved = 0; /* mark as zlib stream */
+                    strm->reserved = ZWRAP_ZLIB_STREAM; /* mark as zlib stream */
                     errorCode = ZWRAP_freeDCtx(zwd);
                     if (ZSTD_isError(errorCode)) goto error;
 
@@ -660,19 +697,25 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
             }
         }
 
+        strm->reserved = ZWRAP_ZSTD_STREAM; /* mark as zstd steam */
+
         if (flush == Z_INFLATE_SYNC) { strm->msg = "inflateSync is not supported!"; goto error; }
 
         if (!zwd->zbd) {
             zwd->zbd = ZSTD_createDStream_advanced(zwd->customMem);
             if (zwd->zbd == NULL) { LOG_WRAPPERD("ERROR: ZSTD_createDStream_advanced\n"); goto error; }
+            zwd->decompState = ZWRAP_useInit;
         }
 
         if (strm->total_in < ZSTD_HEADERSIZE)
         {
             if (strm->total_in == 0 && strm->avail_in >= ZSTD_HEADERSIZE) {
-                if (zwd->decompState != Z_NEED_DICT) {
+                if (zwd->decompState == ZWRAP_useInit) {
                     errorCode = ZSTD_initDStream(zwd->zbd);
                     if (ZSTD_isError(errorCode)) { LOG_WRAPPERD("ERROR: ZSTD_initDStream errorCode=%s\n", ZSTD_getErrorName(errorCode)); goto error; }
+                } else {
+                    errorCode = ZSTD_resetDStream(zwd->zbd);
+                    if (ZSTD_isError(errorCode)) goto error;
                 }
             } else {
                 srcSize = MIN(strm->avail_in, ZSTD_HEADERSIZE - strm->total_in);
@@ -682,8 +725,13 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
                 strm->avail_in -= srcSize;
                 if (strm->total_in < ZSTD_HEADERSIZE) return Z_OK;
 
-                errorCode = ZSTD_initDStream(zwd->zbd);
+                if (zwd->decompState == ZWRAP_useInit) {
-                if (ZSTD_isError(errorCode)) { LOG_WRAPPERD("ERROR: ZSTD_initDStream errorCode=%s\n", ZSTD_getErrorName(errorCode)); goto error; }
+                    errorCode = ZSTD_initDStream(zwd->zbd);
+                    if (ZSTD_isError(errorCode)) { LOG_WRAPPERD("ERROR: ZSTD_initDStream errorCode=%s\n", ZSTD_getErrorName(errorCode)); goto error; }
+                } else {
+                    errorCode = ZSTD_resetDStream(zwd->zbd);
+                    if (ZSTD_isError(errorCode)) goto error;
+                }
 
                 zwd->inBuffer.src = zwd->headerBuf;
                 zwd->inBuffer.size = ZSTD_HEADERSIZE;
@@ -723,7 +771,7 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
         strm->avail_in -= zwd->inBuffer.pos;
         if (errorCode == 0) { 
             LOG_WRAPPERD("inflate Z_STREAM_END1 avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); 
-            zwd->decompState = Z_STREAM_END; 
+            zwd->decompState = ZWRAP_streamEnd; 
             return Z_STREAM_END;
         }
     }

zlibWrapper/zstd_zlibwrapper.h

@@ -30,32 +30,45 @@ extern "C" {
 const char * zstdVersion(void);
 
 
-/* COMPRESSION */
+/*** COMPRESSION ***/
 /* enables/disables zstd compression during runtime */
 void ZWRAP_useZSTDcompression(int turn_on);
 
-/* check if zstd compression is turned on */
+/* checks if zstd compression is turned on */
 int ZWRAP_isUsingZSTDcompression(void);
 
 /* Changes a pledged source size for a given compression stream.
    It will change ZSTD compression parameters what may improve compression speed and/or ratio.
-   The function should be called just after deflateInit(). It's only helpful when data is compressed in blocks. 
+   The function should be called just after deflateInit() or deflateReset() and before deflate() or deflateSetDictionary().
-   There will be no change in case of deflateInit() immediately followed by deflate(strm, Z_FINISH) 
+   It's only helpful when data is compressed in blocks. 
+   There will be no change in case of deflateInit() or deflateReset() immediately followed by deflate(strm, Z_FINISH) 
    as this case is automatically detected.  */
 int ZWRAP_setPledgedSrcSize(z_streamp strm, unsigned long long pledgedSrcSize);
 
+/* Similar to deflateReset but preserves dictionary set using deflateSetDictionary.
+   It should improve compression speed because there will be less calls to deflateSetDictionary 
+   When using zlib compression this method redirects to deflateReset. */
+int ZWRAP_deflateReset_keepDict(z_streamp strm);
 
-/* DECOMPRESSION */
+
+
+/*** DECOMPRESSION ***/
 typedef enum { ZWRAP_FORCE_ZLIB, ZWRAP_AUTO } ZWRAP_decompress_type;
 
 /* enables/disables automatic recognition of zstd/zlib compressed data during runtime */
 void ZWRAP_setDecompressionType(ZWRAP_decompress_type type);
 
-/* check zstd decompression type */
+/* checks zstd decompression type */
 ZWRAP_decompress_type ZWRAP_getDecompressionType(void);
 
+/* Checks if zstd decompression is used for a given stream.
+   If will return 1 only when inflate() was called and zstd header was detected. */
+int ZWRAP_isUsingZSTDdecompression(z_streamp strm);
 
-
+/* Similar to inflateReset but preserves dictionary set using inflateSetDictionary.
+   inflate() will return Z_NEED_DICT only for the first time what will improve decompression speed.
+   For zlib streams this method redirects to inflateReset. */
+int ZWRAP_inflateReset_keepDict(z_streamp strm);
 
 
 #if defined (__cplusplus)