From b077345f08bde9772f125fbc7bcea97ea516a4df Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 16 Sep 2016 14:06:10 +0200 Subject: [PATCH 01/43] zlibWrapper converted from ZBUFF to ZSTD_CStream --- zlibWrapper/zstd_zlibwrapper.c | 156 +++++++++++++++++++-------------- 1 file changed, 92 insertions(+), 64 deletions(-) diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index 9467950ba..26b8f1f32 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -14,16 +14,14 @@ #include "zstd_zlibwrapper.h" #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_MAGICNUMBER */ #include "zstd.h" -#define ZBUFF_STATIC_LINKING_ONLY /* ZBUFF_createCCtx_advanced */ -#include "zbuff.h" #include "zstd_internal.h" /* defaultCustomMem */ #define Z_INFLATE_SYNC 8 -#define ZWRAP_HEADERSIZE 4 +#define ZWRAP_HEADERSIZE 8 #define ZWRAP_DEFAULT_CLEVEL 5 /* Z_DEFAULT_COMPRESSION is translated to ZWRAP_DEFAULT_CLEVEL for zstd */ -#define LOG_WRAPPER(...) /* printf(__VA_ARGS__) */ +#define LOG_WRAPPER(...) printf(__VA_ARGS__) #define FINISH_WITH_GZ_ERR(msg) { \ @@ -78,18 +76,20 @@ static void ZWRAP_freeFunction(void* opaque, void* address) /* *** Compression *** */ typedef struct { - ZBUFF_CCtx* zbc; + ZSTD_CStream* zbc; size_t bytesLeft; int compressionLevel; ZSTD_customMem customMem; z_stream allocFunc; /* copy of zalloc, zfree, opaque */ + ZSTD_inBuffer inBuffer; + ZSTD_outBuffer outBuffer; } ZWRAP_CCtx; size_t ZWRAP_freeCCtx(ZWRAP_CCtx* zwc) { if (zwc==NULL) return 0; /* support free on NULL */ - ZBUFF_freeCCtx(zwc->zbc); + ZSTD_freeCStream(zwc->zbc); zwc->customMem.customFree(zwc->customMem.opaque, zwc); return 0; } @@ -114,7 +114,7 @@ ZWRAP_CCtx* ZWRAP_createCCtx(z_streamp strm) memcpy(&zwc->customMem, &defaultCustomMem, sizeof(ZSTD_customMem)); } - zwc->zbc = ZBUFF_createCCtx_advanced(zwc->customMem); + zwc->zbc = ZSTD_createCStream_advanced(zwc->customMem); if (zwc->zbc == NULL) { ZWRAP_freeCCtx(zwc); return NULL; } return zwc; } @@ -137,7 +137,7 @@ ZEXTERN int ZEXPORT z_deflateInit_ OF((z_streamp strm, int level, if (level == Z_DEFAULT_COMPRESSION) level = ZWRAP_DEFAULT_CLEVEL; - { size_t const errorCode = ZBUFF_compressInit(zwc->zbc, level); + { size_t const errorCode = ZSTD_initCStream(zwc->zbc, level); if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; } zwc->compressionLevel = level; @@ -168,15 +168,23 @@ ZEXTERN int ZEXPORT z_deflateSetDictionary OF((z_streamp strm, return deflateSetDictionary(strm, dictionary, dictLength); { ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state; - LOG_WRAPPER("- deflateSetDictionary level=%d\n", (int)strm->data_type); - { size_t const errorCode = ZBUFF_compressInitDictionary(zwc->zbc, dictionary, dictLength, zwc->compressionLevel); + LOG_WRAPPER("- deflateSetDictionary level=%d\n", (int)zwc->compressionLevel); + { size_t const errorCode = ZSTD_initCStream_usingDict(zwc->zbc, dictionary, dictLength, zwc->compressionLevel); if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; } } return Z_OK; } - +/* +#define Z_NO_FLUSH 0 +#define Z_PARTIAL_FLUSH 1 +#define Z_SYNC_FLUSH 2 +#define Z_FULL_FLUSH 3 +#define Z_FINISH 4 +#define Z_BLOCK 5 +#define Z_TREES 6 +*/ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) { ZWRAP_CCtx* zwc; @@ -191,48 +199,54 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) LOG_WRAPPER("deflate flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); if (strm->avail_in > 0) { - size_t dstCapacity = strm->avail_out; - size_t srcSize = strm->avail_in; - size_t const errorCode = ZBUFF_compressContinue(zwc->zbc, strm->next_out, &dstCapacity, strm->next_in, &srcSize); - LOG_WRAPPER("ZBUFF_compressContinue srcSize=%d dstCapacity=%d\n", (int)srcSize, (int)dstCapacity); - if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; - strm->next_out += dstCapacity; - strm->total_out += dstCapacity; - strm->avail_out -= dstCapacity; - strm->total_in += srcSize; - strm->next_in += srcSize; - strm->avail_in -= srcSize; + zwc->inBuffer.src = strm->next_in; + zwc->inBuffer.size = strm->avail_in; + zwc->inBuffer.pos = 0; + zwc->outBuffer.dst = strm->next_out; + zwc->outBuffer.size = strm->avail_out; + zwc->outBuffer.pos = 0; + { size_t const errorCode = ZSTD_compressStream(zwc->zbc, &zwc->outBuffer, &zwc->inBuffer); + LOG_WRAPPER("ZSTD_compressStream srcSize=%d dstCapacity=%d\n", (int)zwc->inBuffer.size, (int)zwc->outBuffer.size); + if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; + } + strm->next_out += zwc->outBuffer.pos; + strm->total_out += zwc->outBuffer.pos; + strm->avail_out -= zwc->outBuffer.pos; + strm->total_in += zwc->inBuffer.pos; + strm->next_in += zwc->inBuffer.pos; + strm->avail_in -= zwc->inBuffer.pos; } if (flush == Z_FULL_FLUSH) FINISH_WITH_ERR(strm, "Z_FULL_FLUSH is not supported!"); if (flush == Z_FINISH) { size_t bytesLeft; - size_t dstCapacity = strm->avail_out; + zwc->outBuffer.dst = strm->next_out; + zwc->outBuffer.size = strm->avail_out; + zwc->outBuffer.pos = 0; if (zwc->bytesLeft) { - bytesLeft = ZBUFF_compressFlush(zwc->zbc, strm->next_out, &dstCapacity); - LOG_WRAPPER("ZBUFF_compressFlush avail_out=%d dstCapacity=%d bytesLeft=%d\n", (int)strm->avail_out, (int)dstCapacity, (int)bytesLeft); + bytesLeft = ZSTD_flushStream(zwc->zbc, &zwc->outBuffer); + LOG_WRAPPER("ZSTD_flushStream dstCapacity=%d bytesLeft=%d\n", (int)strm->avail_out, (int)bytesLeft); } else { - bytesLeft = ZBUFF_compressEnd(zwc->zbc, strm->next_out, &dstCapacity); - LOG_WRAPPER("ZBUFF_compressEnd dstCapacity=%d bytesLeft=%d\n", (int)dstCapacity, (int)bytesLeft); + bytesLeft = ZSTD_endStream(zwc->zbc, &zwc->outBuffer); + LOG_WRAPPER("ZSTD_endStream dstCapacity=%d bytesLeft=%d\n", (int)strm->avail_out, (int)bytesLeft); } if (ZSTD_isError(bytesLeft)) return Z_MEM_ERROR; - strm->next_out += dstCapacity; - strm->total_out += dstCapacity; - strm->avail_out -= dstCapacity; + strm->next_out += zwc->outBuffer.pos; + strm->total_out += zwc->outBuffer.pos; + strm->avail_out -= zwc->outBuffer.pos; if (flush == Z_FINISH && bytesLeft == 0) return Z_STREAM_END; zwc->bytesLeft = bytesLeft; } if (flush == Z_SYNC_FLUSH) { size_t bytesLeft; - size_t dstCapacity = strm->avail_out; - bytesLeft = ZBUFF_compressFlush(zwc->zbc, strm->next_out, &dstCapacity); - LOG_WRAPPER("ZBUFF_compressFlush avail_out=%d dstCapacity=%d bytesLeft=%d\n", (int)strm->avail_out, (int)dstCapacity, (int)bytesLeft); + bytesLeft = ZSTD_flushStream(zwc->zbc, &zwc->outBuffer); + LOG_WRAPPER("ZSTD_flushStream dstCapacity=%d bytesLeft=%d\n", (int)strm->avail_out, (int)bytesLeft); if (ZSTD_isError(bytesLeft)) return Z_MEM_ERROR; - strm->next_out += dstCapacity; - strm->total_out += dstCapacity; - strm->avail_out -= dstCapacity; + strm->next_out += zwc->outBuffer.pos; + strm->total_out += zwc->outBuffer.pos; + strm->avail_out -= zwc->outBuffer.pos; zwc->bytesLeft = bytesLeft; } return Z_OK; @@ -283,7 +297,7 @@ ZEXTERN int ZEXPORT z_deflateParams OF((z_streamp strm, /* *** Decompression *** */ typedef struct { - ZBUFF_DCtx* zbd; + ZSTD_DStream* zbd; char headerBuf[ZWRAP_HEADERSIZE]; int errorCount; @@ -293,6 +307,8 @@ typedef struct { int windowBits; ZSTD_customMem customMem; z_stream allocFunc; /* copy of zalloc, zfree, opaque */ + ZSTD_inBuffer inBuffer; + ZSTD_outBuffer outBuffer; } ZWRAP_DCtx; @@ -322,7 +338,7 @@ ZWRAP_DCtx* ZWRAP_createDCtx(z_streamp strm) size_t ZWRAP_freeDCtx(ZWRAP_DCtx* zwd) { if (zwd==NULL) return 0; /* support free on null */ - ZBUFF_freeDCtx(zwd->zbd); + ZSTD_freeDStream(zwd->zbd); if (zwd->version) zwd->customMem.customFree(zwd->customMem.opaque, zwd->version); zwd->customMem.customFree(zwd->customMem.opaque, zwd); return 0; @@ -373,16 +389,20 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm, { size_t errorCode; ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state; if (strm->state == NULL) return Z_MEM_ERROR; - errorCode = ZBUFF_decompressInitDictionary(zwd->zbd, dictionary, dictLength); + errorCode = ZSTD_initDStream_usingDict(zwd->zbd, dictionary, dictLength); if (ZSTD_isError(errorCode)) { ZWRAP_freeDCtx(zwd); strm->state = NULL; return Z_MEM_ERROR; } if (strm->total_in == ZSTD_frameHeaderSize_min) { - size_t dstCapacity = 0; - size_t srcSize = strm->total_in; - errorCode = ZBUFF_decompressContinue(zwd->zbd, strm->next_out, &dstCapacity, zwd->headerBuf, &srcSize); - LOG_WRAPPER("ZBUFF_decompressContinue3 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)srcSize, (int)dstCapacity); - if (dstCapacity > 0 || ZSTD_isError(errorCode)) { - LOG_WRAPPER("ERROR: ZBUFF_decompressContinue %s\n", ZSTD_getErrorName(errorCode)); + zwd->inBuffer.src = zwd->headerBuf; + zwd->inBuffer.size = strm->total_in; + zwd->inBuffer.pos = 0; + zwd->outBuffer.dst = strm->next_out; + zwd->outBuffer.size = 0; + zwd->outBuffer.pos = 0; + errorCode = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer); + LOG_WRAPPER("ZSTD_decompressStream3 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)zwd->inBuffer.size, (int)zwd->outBuffer.size); + if (zwd->outBuffer.size > 0 || ZSTD_isError(errorCode)) { + LOG_WRAPPER("ERROR: ZSTD_decompressStream %s\n", ZSTD_getErrorName(errorCode)); ZWRAP_freeDCtx(zwd); strm->state = NULL; return Z_MEM_ERROR; } @@ -399,7 +419,7 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) return inflate(strm, flush); if (strm->avail_in > 0) { - size_t errorCode, dstCapacity, srcSize; + size_t errorCode, srcSize; ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state; if (strm->state == NULL) return Z_MEM_ERROR; LOG_WRAPPER("inflate avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); @@ -448,38 +468,46 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) return inflate(strm, flush); } - zwd->zbd = ZBUFF_createDCtx_advanced(zwd->customMem); + zwd->zbd = ZSTD_createDStream_advanced(zwd->customMem); if (zwd->zbd == NULL) goto error; - errorCode = ZBUFF_decompressInit(zwd->zbd); + errorCode = ZSTD_initDStream(zwd->zbd); if (ZSTD_isError(errorCode)) goto error; - srcSize = ZWRAP_HEADERSIZE; - dstCapacity = 0; - errorCode = ZBUFF_decompressContinue(zwd->zbd, strm->next_out, &dstCapacity, zwd->headerBuf, &srcSize); - LOG_WRAPPER("ZBUFF_decompressContinue1 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)srcSize, (int)dstCapacity); + zwd->inBuffer.src = zwd->headerBuf; + zwd->inBuffer.size = ZWRAP_HEADERSIZE; + zwd->inBuffer.pos = 0; + zwd->outBuffer.dst = strm->next_out; + zwd->outBuffer.size = 0; + zwd->outBuffer.pos = 0; + errorCode = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer); + LOG_WRAPPER("ZSTD_decompressStream1 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)zwd->inBuffer.size, (int)zwd->outBuffer.size); if (ZSTD_isError(errorCode)) { - LOG_WRAPPER("ERROR: ZBUFF_decompressContinue %s\n", ZSTD_getErrorName(errorCode)); + LOG_WRAPPER("ERROR: ZSTD_decompressStream %s\n", ZSTD_getErrorName(errorCode)); goto error; } if (strm->avail_in == 0) return Z_OK; } - srcSize = strm->avail_in; - dstCapacity = strm->avail_out; - errorCode = ZBUFF_decompressContinue(zwd->zbd, strm->next_out, &dstCapacity, strm->next_in, &srcSize); - LOG_WRAPPER("ZBUFF_decompressContinue2 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)srcSize, (int)dstCapacity); + zwd->inBuffer.src = strm->next_in; + zwd->inBuffer.size = strm->avail_in; + zwd->inBuffer.pos = 0; + zwd->outBuffer.dst = strm->next_out; + zwd->outBuffer.size = strm->avail_out; + zwd->outBuffer.pos = 0; + errorCode = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer); + LOG_WRAPPER("ZSTD_decompressStream2 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)strm->avail_in, (int)strm->avail_out); if (ZSTD_isError(errorCode)) { - LOG_WRAPPER("ERROR: ZBUFF_decompressContinue %s\n", ZSTD_getErrorName(errorCode)); + LOG_WRAPPER("ERROR: ZSTD_decompressStream %s\n", ZSTD_getErrorName(errorCode)); zwd->errorCount++; if (zwd->errorCount<=1) return Z_NEED_DICT; else goto error; } - strm->next_out += dstCapacity; - strm->total_out += dstCapacity; - strm->avail_out -= dstCapacity; - strm->total_in += srcSize; - strm->next_in += srcSize; - strm->avail_in -= srcSize; + strm->next_out += zwd->outBuffer.pos; + strm->total_out += zwd->outBuffer.pos; + strm->avail_out -= zwd->outBuffer.pos; + strm->total_in += zwd->inBuffer.pos; + strm->next_in += zwd->inBuffer.pos; + strm->avail_in -= zwd->inBuffer.pos; if (errorCode == 0) return Z_STREAM_END; return Z_OK; error: From 8fc5848bcb0f1cb0b7ebc6586d2dba0fee14f553 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 16 Sep 2016 17:14:01 +0200 Subject: [PATCH 02/43] inflateSetDictionary uses ZSTD_initDStream_usingDict --- zlibWrapper/zstd_zlibwrapper.c | 54 ++++++++++++++++++++++++---------- 1 file changed, 39 insertions(+), 15 deletions(-) diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index 26b8f1f32..dfb1a97a1 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -18,10 +18,11 @@ #define Z_INFLATE_SYNC 8 -#define ZWRAP_HEADERSIZE 8 +#define ZLIB_HEADERSIZE 4 +#define ZSTD_HEADERSIZE ZSTD_frameHeaderSize_min #define ZWRAP_DEFAULT_CLEVEL 5 /* Z_DEFAULT_COMPRESSION is translated to ZWRAP_DEFAULT_CLEVEL for zstd */ -#define LOG_WRAPPER(...) printf(__VA_ARGS__) +#define LOG_WRAPPER(...) /* printf(__VA_ARGS__) */ #define FINISH_WITH_GZ_ERR(msg) { \ @@ -241,6 +242,9 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) if (flush == Z_SYNC_FLUSH) { size_t bytesLeft; + zwc->outBuffer.dst = strm->next_out; + zwc->outBuffer.size = strm->avail_out; + zwc->outBuffer.pos = 0; bytesLeft = ZSTD_flushStream(zwc->zbc, &zwc->outBuffer); LOG_WRAPPER("ZSTD_flushStream dstCapacity=%d bytesLeft=%d\n", (int)strm->avail_out, (int)bytesLeft); if (ZSTD_isError(bytesLeft)) return Z_MEM_ERROR; @@ -298,7 +302,7 @@ ZEXTERN int ZEXPORT z_deflateParams OF((z_streamp strm, typedef struct { ZSTD_DStream* zbd; - char headerBuf[ZWRAP_HEADERSIZE]; + char headerBuf[16]; /* should be equal or bigger than ZSTD_frameHeaderSize_min */ int errorCount; /* zlib params */ @@ -330,6 +334,8 @@ ZWRAP_DCtx* ZWRAP_createDCtx(z_streamp strm) memset(zwd, 0, sizeof(ZWRAP_DCtx)); memcpy(&zwd->customMem, &defaultCustomMem, sizeof(ZSTD_customMem)); } + zwd->outBuffer.pos = 0; + zwd->outBuffer.size = 0; return zwd; } @@ -392,7 +398,7 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm, errorCode = ZSTD_initDStream_usingDict(zwd->zbd, dictionary, dictLength); if (ZSTD_isError(errorCode)) { ZWRAP_freeDCtx(zwd); strm->state = NULL; return Z_MEM_ERROR; } - if (strm->total_in == ZSTD_frameHeaderSize_min) { + if (strm->total_in == ZSTD_HEADERSIZE) { zwd->inBuffer.src = zwd->headerBuf; zwd->inBuffer.size = strm->total_in; zwd->inBuffer.pos = 0; @@ -401,7 +407,7 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm, zwd->outBuffer.pos = 0; errorCode = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer); LOG_WRAPPER("ZSTD_decompressStream3 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)zwd->inBuffer.size, (int)zwd->outBuffer.size); - if (zwd->outBuffer.size > 0 || ZSTD_isError(errorCode)) { + if (zwd->inBuffer.pos < zwd->outBuffer.size || ZSTD_isError(errorCode)) { LOG_WRAPPER("ERROR: ZSTD_decompressStream %s\n", ZSTD_getErrorName(errorCode)); ZWRAP_freeDCtx(zwd); strm->state = NULL; return Z_MEM_ERROR; @@ -419,18 +425,20 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) return inflate(strm, flush); if (strm->avail_in > 0) { - size_t errorCode, srcSize; + size_t errorCode, srcSize, inPos; ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state; if (strm->state == NULL) return Z_MEM_ERROR; LOG_WRAPPER("inflate avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); - if (strm->total_in < ZWRAP_HEADERSIZE) + // if (((strm->avail_in < ZSTD_HEADERSIZE) || (strm->total_in > 0)) && (strm->total_in < ZLIB_HEADERSIZE)) + if (strm->total_in < ZLIB_HEADERSIZE) { - srcSize = MIN(strm->avail_in, ZWRAP_HEADERSIZE - strm->total_in); + // printf("."); + srcSize = MIN(strm->avail_in, ZLIB_HEADERSIZE - strm->total_in); memcpy(zwd->headerBuf+strm->total_in, strm->next_in, srcSize); strm->total_in += srcSize; strm->next_in += srcSize; strm->avail_in -= srcSize; - if (strm->total_in < ZWRAP_HEADERSIZE) return Z_OK; + if (strm->total_in < ZLIB_HEADERSIZE) return Z_OK; if (MEM_readLE32(zwd->headerBuf) != ZSTD_MAGICNUMBER) { z_stream strm2; @@ -448,7 +456,7 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) /* inflate header */ strm->next_in = (unsigned char*)zwd->headerBuf; - strm->avail_in = ZWRAP_HEADERSIZE; + strm->avail_in = ZLIB_HEADERSIZE; strm->avail_out = 0; errorCode = inflate(strm, Z_NO_FLUSH); LOG_WRAPPER("ZLIB inflate errorCode=%d strm->avail_in=%d\n", (int)errorCode, (int)strm->avail_in); @@ -467,6 +475,18 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) if (flush == Z_INFLATE_SYNC) return inflateSync(strm); return inflate(strm, flush); } + } + + // if (((strm->avail_in < ZSTD_HEADERSIZE) || (strm->total_in > 0)) && (strm->total_in < ZSTD_HEADERSIZE)) + if (strm->total_in < ZSTD_HEADERSIZE) + { + // printf("+"); + srcSize = MIN(strm->avail_in, ZSTD_HEADERSIZE - strm->total_in); + memcpy(zwd->headerBuf+strm->total_in, strm->next_in, srcSize); + strm->total_in += srcSize; + strm->next_in += srcSize; + strm->avail_in -= srcSize; + if (strm->total_in < ZSTD_HEADERSIZE) return Z_OK; zwd->zbd = ZSTD_createDStream_advanced(zwd->customMem); if (zwd->zbd == NULL) goto error; @@ -474,8 +494,9 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) errorCode = ZSTD_initDStream(zwd->zbd); if (ZSTD_isError(errorCode)) goto error; + inPos = zwd->inBuffer.pos; zwd->inBuffer.src = zwd->headerBuf; - zwd->inBuffer.size = ZWRAP_HEADERSIZE; + zwd->inBuffer.size = ZSTD_HEADERSIZE; zwd->inBuffer.pos = 0; zwd->outBuffer.dst = strm->next_out; zwd->outBuffer.size = 0; @@ -486,9 +507,11 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) LOG_WRAPPER("ERROR: ZSTD_decompressStream %s\n", ZSTD_getErrorName(errorCode)); goto error; } - if (strm->avail_in == 0) return Z_OK; + // LOG_WRAPPER("1srcSize=%d inpos=%d inBuffer.pos=%d inBuffer.size=%d outBuffer.pos=%d\n", (int)srcSize, (int)inPos, (int)zwd->inBuffer.pos, (int)zwd->inBuffer.size, (int)zwd->outBuffer.pos); + if (zwd->inBuffer.pos == zwd->inBuffer.size) return Z_OK; } + inPos = 0;//zwd->inBuffer.pos; zwd->inBuffer.src = strm->next_in; zwd->inBuffer.size = strm->avail_in; zwd->inBuffer.pos = 0; @@ -496,6 +519,7 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) zwd->outBuffer.size = strm->avail_out; zwd->outBuffer.pos = 0; errorCode = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer); + // LOG_WRAPPER("2 inpos=%d inBuffer.pos=%d inBuffer.size=%d outBuffer.pos=%d\n", (int)inPos, (int)zwd->inBuffer.pos, (int)zwd->inBuffer.size, (int)zwd->outBuffer.pos); LOG_WRAPPER("ZSTD_decompressStream2 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)strm->avail_in, (int)strm->avail_out); if (ZSTD_isError(errorCode)) { LOG_WRAPPER("ERROR: ZSTD_decompressStream %s\n", ZSTD_getErrorName(errorCode)); @@ -505,9 +529,9 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) strm->next_out += zwd->outBuffer.pos; strm->total_out += zwd->outBuffer.pos; strm->avail_out -= zwd->outBuffer.pos; - strm->total_in += zwd->inBuffer.pos; - strm->next_in += zwd->inBuffer.pos; - strm->avail_in -= zwd->inBuffer.pos; + strm->total_in += zwd->inBuffer.pos - inPos; + strm->next_in += zwd->inBuffer.pos - inPos; + strm->avail_in -= zwd->inBuffer.pos - inPos; if (errorCode == 0) return Z_STREAM_END; return Z_OK; error: From e46bad0b2c7b4fd9d24eda382c012d167643d81e Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 19 Sep 2016 13:24:07 +0200 Subject: [PATCH 03/43] imporved support for Z_FINISH --- zlibWrapper/zstd_zlibwrapper.c | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index dfb1a97a1..bb5bd92ad 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -78,7 +78,6 @@ static void ZWRAP_freeFunction(void* opaque, void* address) typedef struct { ZSTD_CStream* zbc; - size_t bytesLeft; int compressionLevel; ZSTD_customMem customMem; z_stream allocFunc; /* copy of zalloc, zfree, opaque */ @@ -225,21 +224,15 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) zwc->outBuffer.dst = strm->next_out; zwc->outBuffer.size = strm->avail_out; zwc->outBuffer.pos = 0; - if (zwc->bytesLeft) { - bytesLeft = ZSTD_flushStream(zwc->zbc, &zwc->outBuffer); - LOG_WRAPPER("ZSTD_flushStream dstCapacity=%d bytesLeft=%d\n", (int)strm->avail_out, (int)bytesLeft); - } else { - bytesLeft = ZSTD_endStream(zwc->zbc, &zwc->outBuffer); - LOG_WRAPPER("ZSTD_endStream dstCapacity=%d bytesLeft=%d\n", (int)strm->avail_out, (int)bytesLeft); - } + bytesLeft = ZSTD_endStream(zwc->zbc, &zwc->outBuffer); + LOG_WRAPPER("ZSTD_endStream dstCapacity=%d bytesLeft=%d\n", (int)strm->avail_out, (int)bytesLeft); if (ZSTD_isError(bytesLeft)) return Z_MEM_ERROR; strm->next_out += zwc->outBuffer.pos; strm->total_out += zwc->outBuffer.pos; strm->avail_out -= zwc->outBuffer.pos; - if (flush == Z_FINISH && bytesLeft == 0) return Z_STREAM_END; - zwc->bytesLeft = bytesLeft; + if (bytesLeft == 0) return Z_STREAM_END; } - + else if (flush == Z_SYNC_FLUSH) { size_t bytesLeft; zwc->outBuffer.dst = strm->next_out; @@ -251,7 +244,6 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) strm->next_out += zwc->outBuffer.pos; strm->total_out += zwc->outBuffer.pos; strm->avail_out -= zwc->outBuffer.pos; - zwc->bytesLeft = bytesLeft; } return Z_OK; } From 6101687547fad8add87e3555a3952622a32f2d38 Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 19 Sep 2016 14:27:29 +0200 Subject: [PATCH 04/43] improved inflateSync --- zlibWrapper/README.md | 1 + zlibWrapper/zstd_zlibwrapper.c | 27 ++++++++++++++++----------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/zlibWrapper/README.md b/zlibWrapper/README.md index 3a39f00ae..5ea542f23 100644 --- a/zlibWrapper/README.md +++ b/zlibWrapper/README.md @@ -95,6 +95,7 @@ Unsupported methods: - deflateSetHeader - inflateGetDictionary - inflateCopy +- inflateSync - inflateReset - inflateReset2 - inflatePrime diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index bb5bd92ad..3ed842cc4 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -160,6 +160,14 @@ ZEXTERN int ZEXPORT z_deflateInit2_ OF((z_streamp strm, int level, int method, } +ZEXTERN int ZEXPORT z_deflateReset OF((z_streamp strm)) +{ + if (!g_useZSTD) + return deflateReset(strm); + FINISH_WITH_ERR(strm, "deflateReset is not supported!"); +} + + ZEXTERN int ZEXPORT z_deflateSetDictionary OF((z_streamp strm, const Bytef *dictionary, uInt dictLength)) @@ -217,7 +225,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) strm->avail_in -= zwc->inBuffer.pos; } - if (flush == Z_FULL_FLUSH) FINISH_WITH_ERR(strm, "Z_FULL_FLUSH is not supported!"); + if (flush == Z_FULL_FLUSH || flush == Z_BLOCK || flush == Z_TREES) FINISH_WITH_ERR(strm, "Z_FULL_FLUSH, Z_BLOCK and Z_TREES are not supported!"); if (flush == Z_FINISH) { size_t bytesLeft; @@ -233,7 +241,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) if (bytesLeft == 0) return Z_STREAM_END; } else - if (flush == Z_SYNC_FLUSH) { + if (flush == Z_SYNC_FLUSH || flush == Z_PARTIAL_FLUSH) { size_t bytesLeft; zwc->outBuffer.dst = strm->next_out; zwc->outBuffer.size = strm->avail_out; @@ -486,6 +494,8 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) errorCode = ZSTD_initDStream(zwd->zbd); if (ZSTD_isError(errorCode)) goto error; + if (flush == Z_INFLATE_SYNC) { strm->msg = "inflateSync is not supported!"; goto error; } + inPos = zwd->inBuffer.pos; zwd->inBuffer.src = zwd->headerBuf; zwd->inBuffer.size = ZSTD_HEADERSIZE; @@ -553,6 +563,9 @@ ZEXTERN int ZEXPORT z_inflateEnd OF((z_streamp strm)) ZEXTERN int ZEXPORT z_inflateSync OF((z_streamp strm)) { + if (!strm->reserved) + return z_inflateSync(strm); + return z_inflate(strm, Z_INFLATE_SYNC); } @@ -569,14 +582,6 @@ ZEXTERN int ZEXPORT z_deflateCopy OF((z_streamp dest, } -ZEXTERN int ZEXPORT z_deflateReset OF((z_streamp strm)) -{ - if (!g_useZSTD) - return deflateReset(strm); - FINISH_WITH_ERR(strm, "deflateReset is not supported!"); -} - - ZEXTERN int ZEXPORT z_deflateTune OF((z_streamp strm, int good_length, int max_lazy, @@ -622,7 +627,7 @@ ZEXTERN int ZEXPORT z_deflateSetHeader OF((z_streamp strm, -/* Advanced compression functions */ +/* Advanced decompression functions */ #if ZLIB_VERNUM >= 0x1280 ZEXTERN int ZEXPORT z_inflateGetDictionary OF((z_streamp strm, Bytef *dictionary, From 0bb930b12825bcc455ac502f043c352eb33a3f4b Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 19 Sep 2016 14:31:16 +0200 Subject: [PATCH 05/43] added ZWRAP_finish_with_error --- zlibWrapper/zstd_zlibwrapper.c | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index 3ed842cc4..9cd736aa6 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -351,15 +351,23 @@ size_t ZWRAP_freeDCtx(ZWRAP_DCtx* zwd) } +int ZWRAP_finish_with_error(ZWRAP_DCtx* zwd, z_streamp strm, int error) +{ + if (zwd) ZWRAP_freeDCtx(zwd); + strm->state = NULL; + return (error) ? error : Z_DATA_ERROR; +} + + ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm, const char *version, int stream_size)) { ZWRAP_DCtx* zwd = ZWRAP_createDCtx(strm); LOG_WRAPPER("- inflateInit\n"); - if (zwd == NULL) { strm->state = NULL; return Z_MEM_ERROR; } + if (zwd == NULL) return ZWRAP_finish_with_error(zwd, strm, 0); zwd->version = zwd->customMem.customAlloc(zwd->customMem.opaque, strlen(version) + 1); - if (zwd->version == NULL) { ZWRAP_freeDCtx(zwd); strm->state = NULL; return Z_MEM_ERROR; } + if (zwd->version == NULL) return ZWRAP_finish_with_error(zwd, strm, 0); strcpy(zwd->version, version); zwd->stream_size = stream_size; @@ -396,7 +404,7 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm, ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state; if (strm->state == NULL) return Z_MEM_ERROR; errorCode = ZSTD_initDStream_usingDict(zwd->zbd, dictionary, dictLength); - if (ZSTD_isError(errorCode)) { ZWRAP_freeDCtx(zwd); strm->state = NULL; return Z_MEM_ERROR; } + if (ZSTD_isError(errorCode)) return ZWRAP_finish_with_error(zwd, strm, 0); if (strm->total_in == ZSTD_HEADERSIZE) { zwd->inBuffer.src = zwd->headerBuf; @@ -409,8 +417,7 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm, LOG_WRAPPER("ZSTD_decompressStream3 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)zwd->inBuffer.size, (int)zwd->outBuffer.size); if (zwd->inBuffer.pos < zwd->outBuffer.size || ZSTD_isError(errorCode)) { LOG_WRAPPER("ERROR: ZSTD_decompressStream %s\n", ZSTD_getErrorName(errorCode)); - ZWRAP_freeDCtx(zwd); strm->state = NULL; - return Z_MEM_ERROR; + return ZWRAP_finish_with_error(zwd, strm, 0); } } } @@ -452,7 +459,7 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) else errorCode = inflateInit_(strm, zwd->version, zwd->stream_size); LOG_WRAPPER("ZLIB inflateInit errorCode=%d\n", (int)errorCode); - if (errorCode != Z_OK) { ZWRAP_freeDCtx(zwd); strm->state = NULL; return errorCode; } + if (errorCode != Z_OK) return ZWRAP_finish_with_error(zwd, strm, (int)errorCode); /* inflate header */ strm->next_in = (unsigned char*)zwd->headerBuf; @@ -460,7 +467,7 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) strm->avail_out = 0; errorCode = inflate(strm, Z_NO_FLUSH); LOG_WRAPPER("ZLIB inflate errorCode=%d strm->avail_in=%d\n", (int)errorCode, (int)strm->avail_in); - if (errorCode != Z_OK) { ZWRAP_freeDCtx(zwd); strm->state = NULL; return errorCode; } + if (errorCode != Z_OK) return ZWRAP_finish_with_error(zwd, strm, (int)errorCode); if (strm->avail_in > 0) goto error; strm->next_in = strm2.next_in; @@ -537,9 +544,7 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) if (errorCode == 0) return Z_STREAM_END; return Z_OK; error: - ZWRAP_freeDCtx(zwd); - strm->state = NULL; - return Z_MEM_ERROR; + return ZWRAP_finish_with_error(zwd, strm, 0); } return Z_OK; } From c4ab571d89c3b4b30f0685c78cd58f95e2a8d0c9 Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 19 Sep 2016 14:54:13 +0200 Subject: [PATCH 06/43] better memory deallocation in case of error --- zlibWrapper/zstd_zlibwrapper.c | 49 ++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index 9cd736aa6..1709876ac 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -120,6 +120,14 @@ ZWRAP_CCtx* ZWRAP_createCCtx(z_streamp strm) } +int ZWRAPC_finish_with_error(ZWRAP_CCtx* zwc, z_streamp strm, int error) +{ + if (zwc) ZWRAP_freeCCtx(zwc); + if (strm) strm->state = NULL; + return (error) ? error : Z_DATA_ERROR; +} + + ZEXTERN int ZEXPORT z_deflateInit_ OF((z_streamp strm, int level, const char *version, int stream_size)) { @@ -138,7 +146,7 @@ ZEXTERN int ZEXPORT z_deflateInit_ OF((z_streamp strm, int level, level = ZWRAP_DEFAULT_CLEVEL; { size_t const errorCode = ZSTD_initCStream(zwc->zbc, level); - if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; } + if (ZSTD_isError(errorCode)) return ZWRAPC_finish_with_error(zwc, strm, 0); } zwc->compressionLevel = level; strm->state = (struct internal_state*) zwc; /* use state which in not used by user */ @@ -177,22 +185,15 @@ ZEXTERN int ZEXPORT z_deflateSetDictionary OF((z_streamp strm, { ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state; LOG_WRAPPER("- deflateSetDictionary level=%d\n", (int)zwc->compressionLevel); + if (zwc == NULL) return Z_MEM_ERROR; { size_t const errorCode = ZSTD_initCStream_usingDict(zwc->zbc, dictionary, dictLength, zwc->compressionLevel); - if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; } + if (ZSTD_isError(errorCode)) return ZWRAPC_finish_with_error(zwc, strm, 0); } } return Z_OK; } -/* -#define Z_NO_FLUSH 0 -#define Z_PARTIAL_FLUSH 1 -#define Z_SYNC_FLUSH 2 -#define Z_FULL_FLUSH 3 -#define Z_FINISH 4 -#define Z_BLOCK 5 -#define Z_TREES 6 -*/ + ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) { ZWRAP_CCtx* zwc; @@ -204,6 +205,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) } zwc = (ZWRAP_CCtx*) strm->state; + if (zwc == NULL) return Z_MEM_ERROR; LOG_WRAPPER("deflate flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); if (strm->avail_in > 0) { @@ -215,7 +217,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) zwc->outBuffer.pos = 0; { size_t const errorCode = ZSTD_compressStream(zwc->zbc, &zwc->outBuffer, &zwc->inBuffer); LOG_WRAPPER("ZSTD_compressStream srcSize=%d dstCapacity=%d\n", (int)zwc->inBuffer.size, (int)zwc->outBuffer.size); - if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; + if (ZSTD_isError(errorCode)) return ZWRAPC_finish_with_error(zwc, strm, 0); } strm->next_out += zwc->outBuffer.pos; strm->total_out += zwc->outBuffer.pos; @@ -234,7 +236,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) zwc->outBuffer.pos = 0; bytesLeft = ZSTD_endStream(zwc->zbc, &zwc->outBuffer); LOG_WRAPPER("ZSTD_endStream dstCapacity=%d bytesLeft=%d\n", (int)strm->avail_out, (int)bytesLeft); - if (ZSTD_isError(bytesLeft)) return Z_MEM_ERROR; + if (ZSTD_isError(bytesLeft)) return ZWRAPC_finish_with_error(zwc, strm, 0); strm->next_out += zwc->outBuffer.pos; strm->total_out += zwc->outBuffer.pos; strm->avail_out -= zwc->outBuffer.pos; @@ -248,7 +250,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) zwc->outBuffer.pos = 0; bytesLeft = ZSTD_flushStream(zwc->zbc, &zwc->outBuffer); LOG_WRAPPER("ZSTD_flushStream dstCapacity=%d bytesLeft=%d\n", (int)strm->avail_out, (int)bytesLeft); - if (ZSTD_isError(bytesLeft)) return Z_MEM_ERROR; + if (ZSTD_isError(bytesLeft)) return ZWRAPC_finish_with_error(zwc, strm, 0); strm->next_out += zwc->outBuffer.pos; strm->total_out += zwc->outBuffer.pos; strm->avail_out -= zwc->outBuffer.pos; @@ -266,6 +268,7 @@ ZEXTERN int ZEXPORT z_deflateEnd OF((z_streamp strm)) LOG_WRAPPER("- deflateEnd total_in=%d total_out=%d\n", (int)(strm->total_in), (int)(strm->total_out)); { ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state; size_t const errorCode = ZWRAP_freeCCtx(zwc); + strm->state = NULL; if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; } return Z_OK; @@ -351,10 +354,10 @@ size_t ZWRAP_freeDCtx(ZWRAP_DCtx* zwd) } -int ZWRAP_finish_with_error(ZWRAP_DCtx* zwd, z_streamp strm, int error) +int ZWRAPD_finish_with_error(ZWRAP_DCtx* zwd, z_streamp strm, int error) { if (zwd) ZWRAP_freeDCtx(zwd); - strm->state = NULL; + if (strm) strm->state = NULL; return (error) ? error : Z_DATA_ERROR; } @@ -364,10 +367,10 @@ ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm, { ZWRAP_DCtx* zwd = ZWRAP_createDCtx(strm); LOG_WRAPPER("- inflateInit\n"); - if (zwd == NULL) return ZWRAP_finish_with_error(zwd, strm, 0); + if (zwd == NULL) return ZWRAPD_finish_with_error(zwd, strm, 0); zwd->version = zwd->customMem.customAlloc(zwd->customMem.opaque, strlen(version) + 1); - if (zwd->version == NULL) return ZWRAP_finish_with_error(zwd, strm, 0); + if (zwd->version == NULL) return ZWRAPD_finish_with_error(zwd, strm, 0); strcpy(zwd->version, version); zwd->stream_size = stream_size; @@ -404,7 +407,7 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm, ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state; if (strm->state == NULL) return Z_MEM_ERROR; errorCode = ZSTD_initDStream_usingDict(zwd->zbd, dictionary, dictLength); - if (ZSTD_isError(errorCode)) return ZWRAP_finish_with_error(zwd, strm, 0); + if (ZSTD_isError(errorCode)) return ZWRAPD_finish_with_error(zwd, strm, 0); if (strm->total_in == ZSTD_HEADERSIZE) { zwd->inBuffer.src = zwd->headerBuf; @@ -417,7 +420,7 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm, LOG_WRAPPER("ZSTD_decompressStream3 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)zwd->inBuffer.size, (int)zwd->outBuffer.size); if (zwd->inBuffer.pos < zwd->outBuffer.size || ZSTD_isError(errorCode)) { LOG_WRAPPER("ERROR: ZSTD_decompressStream %s\n", ZSTD_getErrorName(errorCode)); - return ZWRAP_finish_with_error(zwd, strm, 0); + return ZWRAPD_finish_with_error(zwd, strm, 0); } } } @@ -459,7 +462,7 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) else errorCode = inflateInit_(strm, zwd->version, zwd->stream_size); LOG_WRAPPER("ZLIB inflateInit errorCode=%d\n", (int)errorCode); - if (errorCode != Z_OK) return ZWRAP_finish_with_error(zwd, strm, (int)errorCode); + if (errorCode != Z_OK) return ZWRAPD_finish_with_error(zwd, strm, (int)errorCode); /* inflate header */ strm->next_in = (unsigned char*)zwd->headerBuf; @@ -467,7 +470,7 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) strm->avail_out = 0; errorCode = inflate(strm, Z_NO_FLUSH); LOG_WRAPPER("ZLIB inflate errorCode=%d strm->avail_in=%d\n", (int)errorCode, (int)strm->avail_in); - if (errorCode != Z_OK) return ZWRAP_finish_with_error(zwd, strm, (int)errorCode); + if (errorCode != Z_OK) return ZWRAPD_finish_with_error(zwd, strm, (int)errorCode); if (strm->avail_in > 0) goto error; strm->next_in = strm2.next_in; @@ -544,7 +547,7 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) if (errorCode == 0) return Z_STREAM_END; return Z_OK; error: - return ZWRAP_finish_with_error(zwd, strm, 0); + return ZWRAPD_finish_with_error(zwd, strm, 0); } return Z_OK; } From 7b546e5da9f1e1f7f78b80815e7899941899d24d Mon Sep 17 00:00:00 2001 From: inikep Date: Tue, 20 Sep 2016 12:49:39 +0200 Subject: [PATCH 07/43] added fitblk.c --- zlibWrapper/Makefile | 11 +- zlibWrapper/examples/fitblk.c | 246 ++++++++++++++++++++++++++++++++++ 2 files changed, 256 insertions(+), 1 deletion(-) create mode 100644 zlibWrapper/examples/fitblk.c diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index 9ad1c01dd..ed296931e 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -24,7 +24,7 @@ LDFLAGS = $(LOC) RM = rm -f -all: clean test testzstd +all: clean test testzstd testfitblk test: example ./example @@ -35,8 +35,14 @@ testdll: example_d testzstd: example_zstd ./example_zstd +testfitblk: fitblk + ./fitblk 10240 <../zstd_compression_format.md + .c.o: $(CC) $(CFLAGS) -c -o $@ $< + +fitblk: $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o + $(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(STATICLIB) example: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(STATICLIB) @@ -47,6 +53,9 @@ example_d: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o example_zstd: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(STATICLIB) +$(EXAMPLE_PATH)/fitblk.o: $(EXAMPLE_PATH)/fitblk.c + $(CC) $(CFLAGS) -I. -c -o $@ $(EXAMPLE_PATH)/fitblk.c + $(EXAMPLE_PATH)/example.o: $(EXAMPLE_PATH)/example.c $(CC) $(CFLAGS) -I. -c -o $@ $(EXAMPLE_PATH)/example.c diff --git a/zlibWrapper/examples/fitblk.c b/zlibWrapper/examples/fitblk.c new file mode 100644 index 000000000..35b067b44 --- /dev/null +++ b/zlibWrapper/examples/fitblk.c @@ -0,0 +1,246 @@ +/* fitblk.c: example of fitting compressed output to a specified size + Not copyrighted -- provided to the public domain + Version 1.1 25 November 2004 Mark Adler */ + +/* Version history: + 1.0 24 Nov 2004 First version + 1.1 25 Nov 2004 Change deflateInit2() to deflateInit() + Use fixed-size, stack-allocated raw buffers + Simplify code moving compression to subroutines + Use assert() for internal errors + Add detailed description of approach + */ + +/* Approach to just fitting a requested compressed size: + + fitblk performs three compression passes on a portion of the input + data in order to determine how much of that input will compress to + nearly the requested output block size. The first pass generates + enough deflate blocks to produce output to fill the requested + output size plus a specfied excess amount (see the EXCESS define + below). The last deflate block may go quite a bit past that, but + is discarded. The second pass decompresses and recompresses just + the compressed data that fit in the requested plus excess sized + buffer. The deflate process is terminated after that amount of + input, which is less than the amount consumed on the first pass. + The last deflate block of the result will be of a comparable size + to the final product, so that the header for that deflate block and + the compression ratio for that block will be about the same as in + the final product. The third compression pass decompresses the + result of the second step, but only the compressed data up to the + requested size minus an amount to allow the compressed stream to + complete (see the MARGIN define below). That will result in a + final compressed stream whose length is less than or equal to the + requested size. Assuming sufficient input and a requested size + greater than a few hundred bytes, the shortfall will typically be + less than ten bytes. + + If the input is short enough that the first compression completes + before filling the requested output size, then that compressed + stream is return with no recompression. + + EXCESS is chosen to be just greater than the shortfall seen in a + two pass approach similar to the above. That shortfall is due to + the last deflate block compressing more efficiently with a smaller + header on the second pass. EXCESS is set to be large enough so + that there is enough uncompressed data for the second pass to fill + out the requested size, and small enough so that the final deflate + block of the second pass will be close in size to the final deflate + block of the third and final pass. MARGIN is chosen to be just + large enough to assure that the final compression has enough room + to complete in all cases. + */ + +#include +#include +#include +//#include "zlib.h" +#include "zstd_zlibwrapper.h" + +#define local static + +/* print nastygram and leave */ +local void quit(char *why) +{ + fprintf(stderr, "fitblk abort: %s\n", why); + exit(1); +} + +#define RAWLEN 4096 /* intermediate uncompressed buffer size */ + +/* compress from file to def until provided buffer is full or end of + input reached; return last deflate() return value, or Z_ERRNO if + there was read error on the file */ +local int partcompress(FILE *in, z_streamp def) +{ + int ret, flush; + unsigned char raw[RAWLEN]; + + flush = Z_NO_FLUSH; + do { + def->avail_in = fread(raw, 1, RAWLEN, in); + printf("def->avail_in=%d\n", def->avail_in); + if (ferror(in)) + return Z_ERRNO; + def->next_in = raw; + if (feof(in)) + flush = Z_FINISH; + ret = deflate(def, flush); + assert(ret != Z_STREAM_ERROR); + } while (def->avail_out != 0 && flush == Z_NO_FLUSH); + return ret; +} + +/* recompress from inf's input to def's output; the input for inf and + the output for def are set in those structures before calling; + return last deflate() return value, or Z_MEM_ERROR if inflate() + was not able to allocate enough memory when it needed to */ +local int recompress(z_streamp inf, z_streamp def) +{ + int ret, flush; + unsigned char raw[RAWLEN]; + + flush = Z_NO_FLUSH; + do { + /* decompress */ + inf->avail_out = RAWLEN; + printf("inf->avail_out=%d\n", inf->avail_out); + + inf->next_out = raw; + ret = inflate(inf, Z_NO_FLUSH); + assert(ret != Z_STREAM_ERROR && ret != Z_DATA_ERROR && + ret != Z_NEED_DICT); + if (ret == Z_MEM_ERROR) + return ret; + + /* compress what was decompresed until done or no room */ + def->avail_in = RAWLEN - inf->avail_out; + def->next_in = raw; + if (inf->avail_out != 0) + flush = Z_FINISH; + ret = deflate(def, flush); + assert(ret != Z_STREAM_ERROR); + } while (ret != Z_STREAM_END && def->avail_out != 0); + return ret; +} + +#define EXCESS 256 /* empirically determined stream overage */ +#define MARGIN 8 /* amount to back off for completion */ + +/* compress from stdin to fixed-size block on stdout */ +int main(int argc, char **argv) +{ + int ret; /* return code */ + unsigned size; /* requested fixed output block size */ + unsigned have; /* bytes written by deflate() call */ + unsigned char *blk; /* intermediate and final stream */ + unsigned char *tmp; /* close to desired size stream */ + z_stream def, inf; /* zlib deflate and inflate states */ + + /* get requested output size */ + if (argc != 2) + quit("need one argument: size of output block"); + ret = strtol(argv[1], argv + 1, 10); + if (argv[1][0] != 0) + quit("argument must be a number"); + if (ret < 8) /* 8 is minimum zlib stream size */ + quit("need positive size of 8 or greater"); + size = (unsigned)ret; + + printf("zlib version %s = 0x%04x, compile flags = 0x%lx\n", + ZLIB_VERSION, ZLIB_VERNUM, zlibCompileFlags()); + if (isUsingZSTD()) printf("zstd version %s\n", zstdVersion()); + + /* allocate memory for buffers and compression engine */ + blk = malloc(size + EXCESS); + def.zalloc = Z_NULL; + def.zfree = Z_NULL; + def.opaque = Z_NULL; + ret = deflateInit(&def, Z_DEFAULT_COMPRESSION); + if (ret != Z_OK || blk == NULL) + quit("out of memory"); + + /* compress from stdin until output full, or no more input */ + def.avail_out = size + EXCESS; + def.next_out = blk; + ret = partcompress(stdin, &def); + if (ret == Z_ERRNO) + quit("error reading input"); + +printf("partcompress def.avail_out=%d\n", def.avail_out); + /* if it all fit, then size was undersubscribed -- done! */ + if (ret == Z_STREAM_END && def.avail_out >= EXCESS) { + /* write block to stdout */ + have = size + EXCESS - def.avail_out; + // if (fwrite(blk, 1, have, stdout) != have || ferror(stdout)) + // quit("error writing output"); + + /* clean up and print results to stderr */ + ret = deflateEnd(&def); + assert(ret != Z_STREAM_ERROR); + free(blk); + fprintf(stderr, + "%u bytes unused out of %u requested (all input)\n", + size - have, size); + return 0; + } + + /* it didn't all fit -- set up for recompression */ + inf.zalloc = Z_NULL; + inf.zfree = Z_NULL; + inf.opaque = Z_NULL; + inf.avail_in = 0; + inf.next_in = Z_NULL; + ret = inflateInit(&inf); + tmp = malloc(size + EXCESS); + if (ret != Z_OK || tmp == NULL) + quit("out of memory"); + ret = deflateReset(&def); + assert(ret != Z_STREAM_ERROR); + + /* do first recompression close to the right amount */ + inf.avail_in = size + EXCESS; + inf.next_in = blk; + def.avail_out = size + EXCESS; + def.next_out = tmp; +printf("recompress1 inf.avail_in=%d def.avail_out=%d\n", inf.avail_in, def.avail_out); + ret = recompress(&inf, &def); +printf("recompress1 inf.avail_in=%d def.avail_out=%d\n", inf.avail_in, def.avail_out); + if (ret == Z_MEM_ERROR) + quit("out of memory"); + + /* set up for next reocmpression */ + ret = inflateReset(&inf); + assert(ret != Z_STREAM_ERROR); + ret = deflateReset(&def); + assert(ret != Z_STREAM_ERROR); + + /* do second and final recompression (third compression) */ + inf.avail_in = size - MARGIN; /* assure stream will complete */ + inf.next_in = tmp; + def.avail_out = size; + def.next_out = blk; +printf("recompress2 inf.avail_in=%d def.avail_out=%d\n", inf.avail_in, def.avail_out); + ret = recompress(&inf, &def); +printf("recompress2 inf.avail_in=%d def.avail_out=%d\n", inf.avail_in, def.avail_out); + if (ret == Z_MEM_ERROR) + quit("out of memory"); + assert(ret == Z_STREAM_END); /* otherwise MARGIN too small */ + + /* done -- write block to stdout */ + have = size - def.avail_out; +// if (fwrite(blk, 1, have, stdout) != have || ferror(stdout)) +// quit("error writing output"); + + /* clean up and print results to stderr */ + free(tmp); + ret = inflateEnd(&inf); + assert(ret != Z_STREAM_ERROR); + ret = deflateEnd(&def); + assert(ret != Z_STREAM_ERROR); + free(blk); + fprintf(stderr, + "%u bytes unused out of %u requested (%lu input)\n", + size - have, size, def.total_in); + return 0; +} From 18f66459d538dfe744e384b893b8eb616ad9772e Mon Sep 17 00:00:00 2001 From: inikep Date: Tue, 20 Sep 2016 12:50:59 +0200 Subject: [PATCH 08/43] use Z_STREAM_ERROR as default error --- zlibWrapper/zstd_zlibwrapper.c | 50 +++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index 1709876ac..3e4d0a45f 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -27,12 +27,12 @@ #define FINISH_WITH_GZ_ERR(msg) { \ (void)msg; \ - return Z_MEM_ERROR; \ + return Z_STREAM_ERROR; \ } #define FINISH_WITH_ERR(strm, message) { \ strm->msg = message; \ - return Z_MEM_ERROR; \ + return Z_STREAM_ERROR; \ } #define FINISH_WITH_NULL_ERR(msg) { \ @@ -124,7 +124,7 @@ int ZWRAPC_finish_with_error(ZWRAP_CCtx* zwc, z_streamp strm, int error) { if (zwc) ZWRAP_freeCCtx(zwc); if (strm) strm->state = NULL; - return (error) ? error : Z_DATA_ERROR; + return (error) ? error : Z_STREAM_ERROR; } @@ -172,6 +172,7 @@ ZEXTERN int ZEXPORT z_deflateReset OF((z_streamp strm)) { if (!g_useZSTD) return deflateReset(strm); + FINISH_WITH_ERR(strm, "deflateReset is not supported!"); } @@ -185,7 +186,7 @@ ZEXTERN int ZEXPORT z_deflateSetDictionary OF((z_streamp strm, { ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state; LOG_WRAPPER("- deflateSetDictionary level=%d\n", (int)zwc->compressionLevel); - if (zwc == NULL) return Z_MEM_ERROR; + if (zwc == NULL) return Z_STREAM_ERROR; { size_t const errorCode = ZSTD_initCStream_usingDict(zwc->zbc, dictionary, dictLength, zwc->compressionLevel); if (ZSTD_isError(errorCode)) return ZWRAPC_finish_with_error(zwc, strm, 0); } } @@ -205,7 +206,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) } zwc = (ZWRAP_CCtx*) strm->state; - if (zwc == NULL) return Z_MEM_ERROR; + if (zwc == NULL) return Z_STREAM_ERROR; LOG_WRAPPER("deflate flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); if (strm->avail_in > 0) { @@ -269,7 +270,7 @@ ZEXTERN int ZEXPORT z_deflateEnd OF((z_streamp strm)) { ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state; size_t const errorCode = ZWRAP_freeCCtx(zwc); strm->state = NULL; - if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; + if (ZSTD_isError(errorCode)) return Z_STREAM_ERROR; } return Z_OK; } @@ -358,7 +359,7 @@ int ZWRAPD_finish_with_error(ZWRAP_DCtx* zwd, z_streamp strm, int error) { if (zwd) ZWRAP_freeDCtx(zwd); if (strm) strm->state = NULL; - return (error) ? error : Z_DATA_ERROR; + return (error) ? error : Z_STREAM_ERROR; } @@ -395,6 +396,18 @@ ZEXTERN int ZEXPORT z_inflateInit2_ OF((z_streamp strm, int windowBits, } +ZEXTERN int ZEXPORT z_inflateReset OF((z_streamp strm)) +{ + if (!strm->reserved) + return inflateReset(strm); + + FINISH_WITH_ERR(strm, "inflateReset is not supported!"); + strm->total_in = 0; + strm->total_out = 0; + return Z_OK; +} + + ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm, const Bytef *dictionary, uInt dictLength)) @@ -405,7 +418,7 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm, LOG_WRAPPER("- inflateSetDictionary\n"); { size_t errorCode; ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state; - if (strm->state == NULL) return Z_MEM_ERROR; + if (strm->state == NULL) return Z_STREAM_ERROR; errorCode = ZSTD_initDStream_usingDict(zwd->zbd, dictionary, dictLength); if (ZSTD_isError(errorCode)) return ZWRAPD_finish_with_error(zwd, strm, 0); @@ -437,7 +450,7 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) if (strm->avail_in > 0) { size_t errorCode, srcSize, inPos; ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state; - if (strm->state == NULL) return Z_MEM_ERROR; + if (strm->state == NULL) return Z_STREAM_ERROR; LOG_WRAPPER("inflate avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); // if (((strm->avail_in < ZSTD_HEADERSIZE) || (strm->total_in > 0)) && (strm->total_in < ZLIB_HEADERSIZE)) if (strm->total_in < ZLIB_HEADERSIZE) @@ -571,8 +584,9 @@ ZEXTERN int ZEXPORT z_inflateEnd OF((z_streamp strm)) ZEXTERN int ZEXPORT z_inflateSync OF((z_streamp strm)) { - if (!strm->reserved) - return z_inflateSync(strm); + if (!strm->reserved) { + return inflateSync(strm); + } return z_inflate(strm, Z_INFLATE_SYNC); } @@ -657,14 +671,6 @@ ZEXTERN int ZEXPORT z_inflateCopy OF((z_streamp dest, } -ZEXTERN int ZEXPORT z_inflateReset OF((z_streamp strm)) -{ - if (!strm->reserved) - return inflateReset(strm); - FINISH_WITH_ERR(strm, "inflateReset is not supported!"); -} - - #if ZLIB_VERNUM >= 0x1240 ZEXTERN int ZEXPORT z_inflateReset2 OF((z_streamp strm, int windowBits)) @@ -750,7 +756,7 @@ ZEXTERN int ZEXPORT z_compress OF((Bytef *dest, uLongf *destLen, { size_t dstCapacity = *destLen; size_t const errorCode = ZSTD_compress(dest, dstCapacity, source, sourceLen, ZWRAP_DEFAULT_CLEVEL); LOG_WRAPPER("z_compress sourceLen=%d dstCapacity=%d\n", (int)sourceLen, (int)dstCapacity); - if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; + if (ZSTD_isError(errorCode)) return Z_STREAM_ERROR; *destLen = errorCode; } return Z_OK; @@ -766,7 +772,7 @@ ZEXTERN int ZEXPORT z_compress2 OF((Bytef *dest, uLongf *destLen, { size_t dstCapacity = *destLen; size_t const errorCode = ZSTD_compress(dest, dstCapacity, source, sourceLen, level); - if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; + if (ZSTD_isError(errorCode)) return Z_STREAM_ERROR; *destLen = errorCode; } return Z_OK; @@ -790,7 +796,7 @@ ZEXTERN int ZEXPORT z_uncompress OF((Bytef *dest, uLongf *destLen, { size_t dstCapacity = *destLen; size_t const errorCode = ZSTD_decompress(dest, dstCapacity, source, sourceLen); - if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; + if (ZSTD_isError(errorCode)) return Z_STREAM_ERROR; *destLen = errorCode; } return Z_OK; From c038c30048c0fc48907c97905c574800508a211f Mon Sep 17 00:00:00 2001 From: inikep Date: Tue, 20 Sep 2016 12:54:26 +0200 Subject: [PATCH 09/43] implemented deflateReset --- zlibWrapper/README.md | 4 ++-- zlibWrapper/zstd_zlibwrapper.c | 11 ++++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/zlibWrapper/README.md b/zlibWrapper/README.md index 5ea542f23..e86da2f58 100644 --- a/zlibWrapper/README.md +++ b/zlibWrapper/README.md @@ -73,10 +73,12 @@ Supported methods: - deflate (with exception of Z_FULL_FLUSH) - deflateSetDictionary - deflateEnd +- deflateReset - deflateBound - inflateInit - inflate - inflateSetDictionary +- inflateReset - compress - compress2 - compressBound @@ -88,7 +90,6 @@ Ignored methods (they do nothing): Unsupported methods: - gzip file access functions - deflateCopy -- deflateReset - deflateTune - deflatePending - deflatePrime @@ -96,7 +97,6 @@ Unsupported methods: - inflateGetDictionary - inflateCopy - inflateSync -- inflateReset - inflateReset2 - inflatePrime - inflateMark diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index 3e4d0a45f..d842ae75f 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -173,7 +173,16 @@ ZEXTERN int ZEXPORT z_deflateReset OF((z_streamp strm)) if (!g_useZSTD) return deflateReset(strm); - FINISH_WITH_ERR(strm, "deflateReset is not supported!"); + { ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state; + LOG_WRAPPER("- z_deflateReset\n"); + if (zwc == NULL) return Z_STREAM_ERROR; + { size_t const errorCode = ZSTD_resetCStream(zwc->zbc, 0); + if (ZSTD_isError(errorCode)) return ZWRAPC_finish_with_error(zwc, strm, 0); } + } + + strm->total_in = 0; + strm->total_out = 0; + return Z_OK; } From 554b3b935c67df0582256a40ffcd231e2b5ade63 Mon Sep 17 00:00:00 2001 From: inikep Date: Tue, 20 Sep 2016 15:18:00 +0200 Subject: [PATCH 10/43] improved logging --- zlibWrapper/Makefile | 1 + zlibWrapper/examples/example.c | 10 ++-- zlibWrapper/examples/fitblk.c | 12 ++-- zlibWrapper/zstd_zlibwrapper.c | 100 ++++++++++++++++++++------------- 4 files changed, 74 insertions(+), 49 deletions(-) diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index ed296931e..f966b4523 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -37,6 +37,7 @@ testzstd: example_zstd testfitblk: fitblk ./fitblk 10240 <../zstd_compression_format.md + #./fitblk 40960 <../zstd_compression_format.md .c.o: $(CC) $(CFLAGS) -c -o $@ $< diff --git a/zlibWrapper/examples/example.c b/zlibWrapper/examples/example.c index bbb2cd5a3..c1f3b46b3 100644 --- a/zlibWrapper/examples/example.c +++ b/zlibWrapper/examples/example.c @@ -45,12 +45,12 @@ } \ } -z_const char hello[] = "hello, hello!"; +z_const char hello[] = "hello, hello! I said hello, hello!"; /* "hello world" would be more standard, but the repeated "hello" * stresses the compression code better, sorry... */ -const char dictionary[] = "hello"; +const char dictionary[] = "hello, hello!"; uLong dictId; /* Adler32 value of the dictionary */ void test_deflate OF((Byte *compr, uLong comprLen)); @@ -156,7 +156,7 @@ void test_gzio(fname, uncompr, uncomprLen) fprintf(stderr, "gzputs err: %s\n", gzerror(file, &err)); exit(1); } - if (gzprintf(file, ", %s!", "hello") != 8) { + if (gzprintf(file, ", %s! I said hello, hello!", "hello") != 8+21) { fprintf(stderr, "gzprintf err: %s\n", gzerror(file, &err)); exit(1); } @@ -182,7 +182,7 @@ void test_gzio(fname, uncompr, uncomprLen) } pos = gzseek(file, -8L, SEEK_CUR); - if (pos != 6 || gztell(file) != pos) { + if (pos != 6+21 || gztell(file) != pos) { fprintf(stderr, "gzseek error, pos=%ld, gztell=%ld\n", (long)pos, (long)gztell(file)); exit(1); @@ -203,7 +203,7 @@ void test_gzio(fname, uncompr, uncomprLen) fprintf(stderr, "gzgets err after gzseek: %s\n", gzerror(file, &err)); exit(1); } - if (strcmp((char*)uncompr, hello + 6)) { + if (strcmp((char*)uncompr, hello + 6+21)) { fprintf(stderr, "bad gzgets after gzseek\n"); exit(1); } else { diff --git a/zlibWrapper/examples/fitblk.c b/zlibWrapper/examples/fitblk.c index 35b067b44..1f12187bd 100644 --- a/zlibWrapper/examples/fitblk.c +++ b/zlibWrapper/examples/fitblk.c @@ -76,18 +76,19 @@ local int partcompress(FILE *in, z_streamp def) int ret, flush; unsigned char raw[RAWLEN]; - flush = Z_NO_FLUSH; + flush = Z_SYNC_FLUSH; do { def->avail_in = fread(raw, 1, RAWLEN, in); - printf("def->avail_in=%d\n", def->avail_in); + printf("partcompress def->avail_in=%d\n", def->avail_in); if (ferror(in)) return Z_ERRNO; def->next_in = raw; if (feof(in)) flush = Z_FINISH; ret = deflate(def, flush); + printf("partcompress def->avail_out=%d\n", def->avail_out); assert(ret != Z_STREAM_ERROR); - } while (def->avail_out != 0 && flush == Z_NO_FLUSH); + } while (def->avail_out != 0 && flush == Z_SYNC_FLUSH); return ret; } @@ -104,7 +105,7 @@ local int recompress(z_streamp inf, z_streamp def) do { /* decompress */ inf->avail_out = RAWLEN; - printf("inf->avail_out=%d\n", inf->avail_out); + printf("recompress inf->avail_out=%d\n", inf->avail_out); inf->next_out = raw; ret = inflate(inf, Z_NO_FLUSH); @@ -120,6 +121,7 @@ local int recompress(z_streamp inf, z_streamp def) flush = Z_FINISH; ret = deflate(def, flush); assert(ret != Z_STREAM_ERROR); + printf("recompress def->avail_out=%d ret=%d\n", def->avail_out, ret); } while (ret != Z_STREAM_END && def->avail_out != 0); return ret; } @@ -167,7 +169,7 @@ int main(int argc, char **argv) if (ret == Z_ERRNO) quit("error reading input"); -printf("partcompress def.avail_out=%d\n", def.avail_out); +printf("partcompress def.total_out=%d ret=%d\n", (int)def.total_out, ret); /* if it all fit, then size was undersubscribed -- done! */ if (ret == Z_STREAM_END && def.avail_out >= EXCESS) { /* write block to stdout */ diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index d842ae75f..28f42526e 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -22,7 +22,8 @@ #define ZSTD_HEADERSIZE ZSTD_frameHeaderSize_min #define ZWRAP_DEFAULT_CLEVEL 5 /* Z_DEFAULT_COMPRESSION is translated to ZWRAP_DEFAULT_CLEVEL for zstd */ -#define LOG_WRAPPER(...) /* printf(__VA_ARGS__) */ +#define LOG_WRAPPERC(...) /*printf(__VA_ARGS__)*/ +#define LOG_WRAPPERD(...) /*printf(__VA_ARGS__)*/ #define FINISH_WITH_GZ_ERR(msg) { \ @@ -122,6 +123,7 @@ ZWRAP_CCtx* ZWRAP_createCCtx(z_streamp strm) int ZWRAPC_finish_with_error(ZWRAP_CCtx* zwc, z_streamp strm, int error) { + LOG_WRAPPERC("- ZWRAPC_finish_with_error=%d\n", error); if (zwc) ZWRAP_freeCCtx(zwc); if (strm) strm->state = NULL; return (error) ? error : Z_STREAM_ERROR; @@ -133,12 +135,11 @@ ZEXTERN int ZEXPORT z_deflateInit_ OF((z_streamp strm, int level, { ZWRAP_CCtx* zwc; + LOG_WRAPPERC("- deflateInit level=%d\n", level); if (!g_useZSTD) { - LOG_WRAPPER("- deflateInit level=%d\n", level); return deflateInit_((strm), (level), version, stream_size); } - LOG_WRAPPER("- deflateInit level=%d\n", level); zwc = ZWRAP_createCCtx(strm); if (zwc == NULL) return Z_MEM_ERROR; @@ -170,11 +171,11 @@ ZEXTERN int ZEXPORT z_deflateInit2_ OF((z_streamp strm, int level, int method, ZEXTERN int ZEXPORT z_deflateReset OF((z_streamp strm)) { + LOG_WRAPPERC("- deflateReset\n"); if (!g_useZSTD) return deflateReset(strm); { ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state; - LOG_WRAPPER("- z_deflateReset\n"); if (zwc == NULL) return Z_STREAM_ERROR; { size_t const errorCode = ZSTD_resetCStream(zwc->zbc, 0); if (ZSTD_isError(errorCode)) return ZWRAPC_finish_with_error(zwc, strm, 0); } @@ -190,11 +191,13 @@ ZEXTERN int ZEXPORT z_deflateSetDictionary OF((z_streamp strm, const Bytef *dictionary, uInt dictLength)) { - if (!g_useZSTD) + if (!g_useZSTD) { + LOG_WRAPPERC("- deflateSetDictionary\n"); return deflateSetDictionary(strm, dictionary, dictLength); + } { ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state; - LOG_WRAPPER("- deflateSetDictionary level=%d\n", (int)zwc->compressionLevel); + LOG_WRAPPERC("- deflateSetDictionary level=%d\n", (int)zwc->compressionLevel); if (zwc == NULL) return Z_STREAM_ERROR; { size_t const errorCode = ZSTD_initCStream_usingDict(zwc->zbc, dictionary, dictLength, zwc->compressionLevel); if (ZSTD_isError(errorCode)) return ZWRAPC_finish_with_error(zwc, strm, 0); } @@ -209,15 +212,17 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) ZWRAP_CCtx* zwc; if (!g_useZSTD) { - int res = deflate(strm, flush); - LOG_WRAPPER("- avail_in=%d total_in=%d total_out=%d\n", (int)strm->avail_in, (int)strm->total_in, (int)strm->total_out); + int res; + LOG_WRAPPERC("- deflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); + res = deflate(strm, flush); + LOG_WRAPPERC("- deflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); return res; } zwc = (ZWRAP_CCtx*) strm->state; if (zwc == NULL) return Z_STREAM_ERROR; - LOG_WRAPPER("deflate flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); + LOG_WRAPPERC("- deflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); if (strm->avail_in > 0) { zwc->inBuffer.src = strm->next_in; zwc->inBuffer.size = strm->avail_in; @@ -226,7 +231,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) zwc->outBuffer.size = strm->avail_out; zwc->outBuffer.pos = 0; { size_t const errorCode = ZSTD_compressStream(zwc->zbc, &zwc->outBuffer, &zwc->inBuffer); - LOG_WRAPPER("ZSTD_compressStream srcSize=%d dstCapacity=%d\n", (int)zwc->inBuffer.size, (int)zwc->outBuffer.size); + LOG_WRAPPERC("deflate ZSTD_compressStream srcSize=%d dstCapacity=%d\n", (int)zwc->inBuffer.size, (int)zwc->outBuffer.size); if (ZSTD_isError(errorCode)) return ZWRAPC_finish_with_error(zwc, strm, 0); } strm->next_out += zwc->outBuffer.pos; @@ -245,12 +250,12 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) zwc->outBuffer.size = strm->avail_out; zwc->outBuffer.pos = 0; bytesLeft = ZSTD_endStream(zwc->zbc, &zwc->outBuffer); - LOG_WRAPPER("ZSTD_endStream dstCapacity=%d bytesLeft=%d\n", (int)strm->avail_out, (int)bytesLeft); + LOG_WRAPPERC("deflate ZSTD_endStream dstCapacity=%d bytesLeft=%d\n", (int)strm->avail_out, (int)bytesLeft); if (ZSTD_isError(bytesLeft)) return ZWRAPC_finish_with_error(zwc, strm, 0); strm->next_out += zwc->outBuffer.pos; strm->total_out += zwc->outBuffer.pos; strm->avail_out -= zwc->outBuffer.pos; - if (bytesLeft == 0) return Z_STREAM_END; + if (bytesLeft == 0) { LOG_WRAPPERC("Z_STREAM_END2 strm->total_in=%d strm->avail_out=%d strm->total_out=%d\n", (int)strm->total_in, (int)strm->avail_out, (int)strm->total_out); return Z_STREAM_END; } } else if (flush == Z_SYNC_FLUSH || flush == Z_PARTIAL_FLUSH) { @@ -259,12 +264,13 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) zwc->outBuffer.size = strm->avail_out; zwc->outBuffer.pos = 0; bytesLeft = ZSTD_flushStream(zwc->zbc, &zwc->outBuffer); - LOG_WRAPPER("ZSTD_flushStream dstCapacity=%d bytesLeft=%d\n", (int)strm->avail_out, (int)bytesLeft); + LOG_WRAPPERC("deflate ZSTD_flushStream dstCapacity=%d bytesLeft=%d\n", (int)strm->avail_out, (int)bytesLeft); if (ZSTD_isError(bytesLeft)) return ZWRAPC_finish_with_error(zwc, strm, 0); strm->next_out += zwc->outBuffer.pos; strm->total_out += zwc->outBuffer.pos; strm->avail_out -= zwc->outBuffer.pos; } + LOG_WRAPPERC("- deflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); return Z_OK; } @@ -272,10 +278,10 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) ZEXTERN int ZEXPORT z_deflateEnd OF((z_streamp strm)) { if (!g_useZSTD) { - LOG_WRAPPER("- deflateEnd\n"); + LOG_WRAPPERC("- deflateEnd\n"); return deflateEnd(strm); } - LOG_WRAPPER("- deflateEnd total_in=%d total_out=%d\n", (int)(strm->total_in), (int)(strm->total_out)); + LOG_WRAPPERC("- deflateEnd total_in=%d total_out=%d\n", (int)(strm->total_in), (int)(strm->total_out)); { ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state; size_t const errorCode = ZWRAP_freeCCtx(zwc); strm->state = NULL; @@ -300,7 +306,7 @@ ZEXTERN int ZEXPORT z_deflateParams OF((z_streamp strm, int strategy)) { if (!g_useZSTD) { - LOG_WRAPPER("- deflateParams level=%d strategy=%d\n", level, strategy); + LOG_WRAPPERC("- deflateParams level=%d strategy=%d\n", level, strategy); return deflateParams(strm, level, strategy); } @@ -366,6 +372,7 @@ size_t ZWRAP_freeDCtx(ZWRAP_DCtx* zwd) int ZWRAPD_finish_with_error(ZWRAP_DCtx* zwd, z_streamp strm, int error) { + LOG_WRAPPERD("- ZWRAPD_finish_with_error=%d\n", error); if (zwd) ZWRAP_freeDCtx(zwd); if (strm) strm->state = NULL; return (error) ? error : Z_STREAM_ERROR; @@ -376,7 +383,7 @@ ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm, const char *version, int stream_size)) { ZWRAP_DCtx* zwd = ZWRAP_createDCtx(strm); - LOG_WRAPPER("- inflateInit\n"); + LOG_WRAPPERD("- inflateInit\n"); if (zwd == NULL) return ZWRAPD_finish_with_error(zwd, strm, 0); zwd->version = zwd->customMem.customAlloc(zwd->customMem.opaque, strlen(version) + 1); @@ -407,10 +414,16 @@ ZEXTERN int ZEXPORT z_inflateInit2_ OF((z_streamp strm, int windowBits, ZEXTERN int ZEXPORT z_inflateReset OF((z_streamp strm)) { + LOG_WRAPPERD("- inflateReset\n"); if (!strm->reserved) return inflateReset(strm); - FINISH_WITH_ERR(strm, "inflateReset is not supported!"); + { ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state; + if (zwd == NULL) return Z_STREAM_ERROR; + { size_t const errorCode = ZSTD_resetDStream(zwd->zbd); + if (ZSTD_isError(errorCode)) return ZWRAPD_finish_with_error(zwd, strm, 0); } + } + strm->total_in = 0; strm->total_out = 0; return Z_OK; @@ -421,10 +434,10 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm, const Bytef *dictionary, uInt dictLength)) { + LOG_WRAPPERD("- inflateSetDictionary\n"); if (!strm->reserved) return inflateSetDictionary(strm, dictionary, dictLength); - LOG_WRAPPER("- inflateSetDictionary\n"); { size_t errorCode; ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state; if (strm->state == NULL) return Z_STREAM_ERROR; @@ -439,9 +452,9 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm, zwd->outBuffer.size = 0; zwd->outBuffer.pos = 0; errorCode = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer); - LOG_WRAPPER("ZSTD_decompressStream3 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)zwd->inBuffer.size, (int)zwd->outBuffer.size); + LOG_WRAPPERD("inflateSetDictionary ZSTD_decompressStream errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)zwd->inBuffer.size, (int)zwd->outBuffer.size); if (zwd->inBuffer.pos < zwd->outBuffer.size || ZSTD_isError(errorCode)) { - LOG_WRAPPER("ERROR: ZSTD_decompressStream %s\n", ZSTD_getErrorName(errorCode)); + LOG_WRAPPERD("ERROR: ZSTD_decompressStream %s\n", ZSTD_getErrorName(errorCode)); return ZWRAPD_finish_with_error(zwd, strm, 0); } } @@ -453,14 +466,19 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm, ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) { - if (!strm->reserved) - return inflate(strm, flush); + int res; + if (!strm->reserved) { + LOG_WRAPPERD("- inflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); + res = inflate(strm, flush); + LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); + return res; + } if (strm->avail_in > 0) { size_t errorCode, srcSize, inPos; ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state; if (strm->state == NULL) return Z_STREAM_ERROR; - LOG_WRAPPER("inflate avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); + LOG_WRAPPERD("- inflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); // if (((strm->avail_in < ZSTD_HEADERSIZE) || (strm->total_in > 0)) && (strm->total_in < ZLIB_HEADERSIZE)) if (strm->total_in < ZLIB_HEADERSIZE) { @@ -483,7 +501,7 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) errorCode = inflateInit2_(strm, zwd->windowBits, zwd->version, zwd->stream_size); else errorCode = inflateInit_(strm, zwd->version, zwd->stream_size); - LOG_WRAPPER("ZLIB inflateInit errorCode=%d\n", (int)errorCode); + LOG_WRAPPERD("ZLIB inflateInit errorCode=%d\n", (int)errorCode); if (errorCode != Z_OK) return ZWRAPD_finish_with_error(zwd, strm, (int)errorCode); /* inflate header */ @@ -491,7 +509,7 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) strm->avail_in = ZLIB_HEADERSIZE; strm->avail_out = 0; errorCode = inflate(strm, Z_NO_FLUSH); - LOG_WRAPPER("ZLIB inflate errorCode=%d strm->avail_in=%d\n", (int)errorCode, (int)strm->avail_in); + LOG_WRAPPERD("ZLIB inflate errorCode=%d strm->avail_in=%d\n", (int)errorCode, (int)strm->avail_in); if (errorCode != Z_OK) return ZWRAPD_finish_with_error(zwd, strm, (int)errorCode); if (strm->avail_in > 0) goto error; @@ -504,8 +522,10 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) errorCode = ZWRAP_freeDCtx(zwd); if (ZSTD_isError(errorCode)) goto error; - if (flush == Z_INFLATE_SYNC) return inflateSync(strm); - return inflate(strm, flush); + if (flush == Z_INFLATE_SYNC) res = inflateSync(strm); + else res = inflate(strm, flush); + LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); + return res; } } @@ -536,13 +556,13 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) zwd->outBuffer.size = 0; zwd->outBuffer.pos = 0; errorCode = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer); - LOG_WRAPPER("ZSTD_decompressStream1 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)zwd->inBuffer.size, (int)zwd->outBuffer.size); + LOG_WRAPPERD("inflate ZSTD_decompressStream1 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)zwd->inBuffer.size, (int)zwd->outBuffer.size); if (ZSTD_isError(errorCode)) { - LOG_WRAPPER("ERROR: ZSTD_decompressStream %s\n", ZSTD_getErrorName(errorCode)); + LOG_WRAPPERD("ERROR: ZSTD_decompressStream %s\n", ZSTD_getErrorName(errorCode)); goto error; } - // LOG_WRAPPER("1srcSize=%d inpos=%d inBuffer.pos=%d inBuffer.size=%d outBuffer.pos=%d\n", (int)srcSize, (int)inPos, (int)zwd->inBuffer.pos, (int)zwd->inBuffer.size, (int)zwd->outBuffer.pos); - if (zwd->inBuffer.pos == zwd->inBuffer.size) return Z_OK; + // LOG_WRAPPERD("1srcSize=%d inpos=%d inBuffer.pos=%d inBuffer.size=%d outBuffer.pos=%d\n", (int)srcSize, (int)inPos, (int)zwd->inBuffer.pos, (int)zwd->inBuffer.size, (int)zwd->outBuffer.pos); + if (zwd->inBuffer.pos != zwd->inBuffer.size) return ZWRAPD_finish_with_error(zwd, strm, 0); /* not consumed */ } inPos = 0;//zwd->inBuffer.pos; @@ -553,10 +573,10 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) zwd->outBuffer.size = strm->avail_out; zwd->outBuffer.pos = 0; errorCode = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer); - // LOG_WRAPPER("2 inpos=%d inBuffer.pos=%d inBuffer.size=%d outBuffer.pos=%d\n", (int)inPos, (int)zwd->inBuffer.pos, (int)zwd->inBuffer.size, (int)zwd->outBuffer.pos); - LOG_WRAPPER("ZSTD_decompressStream2 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)strm->avail_in, (int)strm->avail_out); + // LOG_WRAPPERD("2 inpos=%d inBuffer.pos=%d inBuffer.size=%d outBuffer.pos=%d\n", (int)inPos, (int)zwd->inBuffer.pos, (int)zwd->inBuffer.size, (int)zwd->outBuffer.pos); + LOG_WRAPPERD("inflate ZSTD_decompressStream2 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)strm->avail_in, (int)strm->avail_out); if (ZSTD_isError(errorCode)) { - LOG_WRAPPER("ERROR: ZSTD_decompressStream %s\n", ZSTD_getErrorName(errorCode)); + LOG_WRAPPERD("ERROR: ZSTD_decompressStream %s\n", ZSTD_getErrorName(errorCode)); zwd->errorCount++; if (zwd->errorCount<=1) return Z_NEED_DICT; else goto error; } @@ -566,11 +586,13 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) strm->total_in += zwd->inBuffer.pos - inPos; strm->next_in += zwd->inBuffer.pos - inPos; strm->avail_in -= zwd->inBuffer.pos - inPos; - if (errorCode == 0) return Z_STREAM_END; - return Z_OK; + if (errorCode == 0) { LOG_WRAPPERD("inflate Z_STREAM_END1 strm->total_in=%d strm->avail_out=%d strm->total_out=%d\n", (int)strm->total_in, (int)strm->avail_out, (int)strm->total_out); return Z_STREAM_END; } + goto finish; error: return ZWRAPD_finish_with_error(zwd, strm, 0); } +finish: + LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); return Z_OK; } @@ -581,7 +603,7 @@ ZEXTERN int ZEXPORT z_inflateEnd OF((z_streamp strm)) if (!strm->reserved) return inflateEnd(strm); - LOG_WRAPPER("- inflateEnd total_in=%d total_out=%d\n", (int)(strm->total_in), (int)(strm->total_out)); + LOG_WRAPPERD("- inflateEnd total_in=%d total_out=%d\n", (int)(strm->total_in), (int)(strm->total_out)); { ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state; size_t const errorCode = ZWRAP_freeDCtx(zwd); strm->state = NULL; @@ -764,7 +786,7 @@ ZEXTERN int ZEXPORT z_compress OF((Bytef *dest, uLongf *destLen, { size_t dstCapacity = *destLen; size_t const errorCode = ZSTD_compress(dest, dstCapacity, source, sourceLen, ZWRAP_DEFAULT_CLEVEL); - LOG_WRAPPER("z_compress sourceLen=%d dstCapacity=%d\n", (int)sourceLen, (int)dstCapacity); + LOG_WRAPPERD("z_compress sourceLen=%d dstCapacity=%d\n", (int)sourceLen, (int)dstCapacity); if (ZSTD_isError(errorCode)) return Z_STREAM_ERROR; *destLen = errorCode; } From 86fc8e000332fe27f3a60208442eaca00459db98 Mon Sep 17 00:00:00 2001 From: inikep Date: Tue, 20 Sep 2016 16:22:28 +0200 Subject: [PATCH 11/43] added ZWRAP_DCtx.decompState --- zlibWrapper/Makefile | 2 +- zlibWrapper/examples/fitblk.c | 26 ++++++++++++----------- zlibWrapper/zstd_zlibwrapper.c | 39 +++++++++++++++++++++++----------- 3 files changed, 42 insertions(+), 25 deletions(-) diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index f966b4523..cfdafb6d8 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -37,7 +37,7 @@ testzstd: example_zstd testfitblk: fitblk ./fitblk 10240 <../zstd_compression_format.md - #./fitblk 40960 <../zstd_compression_format.md + ./fitblk 40960 <../zstd_compression_format.md .c.o: $(CC) $(CFLAGS) -c -o $@ $< diff --git a/zlibWrapper/examples/fitblk.c b/zlibWrapper/examples/fitblk.c index 1f12187bd..12e4c8626 100644 --- a/zlibWrapper/examples/fitblk.c +++ b/zlibWrapper/examples/fitblk.c @@ -57,6 +57,7 @@ //#include "zlib.h" #include "zstd_zlibwrapper.h" +#define LOG_FITBLK(...) /*printf(__VA_ARGS__)*/ #define local static /* print nastygram and leave */ @@ -79,14 +80,14 @@ local int partcompress(FILE *in, z_streamp def) flush = Z_SYNC_FLUSH; do { def->avail_in = fread(raw, 1, RAWLEN, in); - printf("partcompress def->avail_in=%d\n", def->avail_in); if (ferror(in)) return Z_ERRNO; def->next_in = raw; if (feof(in)) flush = Z_FINISH; + LOG_FITBLK("partcompress1 avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out); ret = deflate(def, flush); - printf("partcompress def->avail_out=%d\n", def->avail_out); + LOG_FITBLK("partcompress2 avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out); assert(ret != Z_STREAM_ERROR); } while (def->avail_out != 0 && flush == Z_SYNC_FLUSH); return ret; @@ -105,10 +106,10 @@ local int recompress(z_streamp inf, z_streamp def) do { /* decompress */ inf->avail_out = RAWLEN; - printf("recompress inf->avail_out=%d\n", inf->avail_out); - inf->next_out = raw; + LOG_FITBLK("recompress1inflate avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)inf->avail_in, (int)inf->total_in, (int)inf->avail_out, (int)inf->total_out); ret = inflate(inf, Z_NO_FLUSH); + LOG_FITBLK("recompress2inflate avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)inf->avail_in, (int)inf->total_in, (int)inf->avail_out, (int)inf->total_out); assert(ret != Z_STREAM_ERROR && ret != Z_DATA_ERROR && ret != Z_NEED_DICT); if (ret == Z_MEM_ERROR) @@ -119,9 +120,10 @@ local int recompress(z_streamp inf, z_streamp def) def->next_in = raw; if (inf->avail_out != 0) flush = Z_FINISH; + LOG_FITBLK("recompress1deflate avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out); ret = deflate(def, flush); + LOG_FITBLK("recompress2deflate avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out); assert(ret != Z_STREAM_ERROR); - printf("recompress def->avail_out=%d ret=%d\n", def->avail_out, ret); } while (ret != Z_STREAM_END && def->avail_out != 0); return ret; } @@ -149,8 +151,7 @@ int main(int argc, char **argv) quit("need positive size of 8 or greater"); size = (unsigned)ret; - printf("zlib version %s = 0x%04x, compile flags = 0x%lx\n", - ZLIB_VERSION, ZLIB_VERNUM, zlibCompileFlags()); + printf("zlib version %s\n", ZLIB_VERSION); if (isUsingZSTD()) printf("zstd version %s\n", zstdVersion()); /* allocate memory for buffers and compression engine */ @@ -165,11 +166,12 @@ int main(int argc, char **argv) /* compress from stdin until output full, or no more input */ def.avail_out = size + EXCESS; def.next_out = blk; + LOG_FITBLK("partcompress1 total_in=%d total_out=%d\n", (int)def.total_in, (int)def.total_out); ret = partcompress(stdin, &def); + LOG_FITBLK("partcompress2 total_in=%d total_out=%d\n", (int)def.total_in, (int)def.total_out); if (ret == Z_ERRNO) quit("error reading input"); -printf("partcompress def.total_out=%d ret=%d\n", (int)def.total_out, ret); /* if it all fit, then size was undersubscribed -- done! */ if (ret == Z_STREAM_END && def.avail_out >= EXCESS) { /* write block to stdout */ @@ -205,9 +207,9 @@ printf("partcompress def.total_out=%d ret=%d\n", (int)def.total_out, ret); inf.next_in = blk; def.avail_out = size + EXCESS; def.next_out = tmp; -printf("recompress1 inf.avail_in=%d def.avail_out=%d\n", inf.avail_in, def.avail_out); + LOG_FITBLK("recompress1 inf.total_in=%d def.total_out=%d\n", (int)inf.total_in, (int)def.total_out); ret = recompress(&inf, &def); -printf("recompress1 inf.avail_in=%d def.avail_out=%d\n", inf.avail_in, def.avail_out); + LOG_FITBLK("recompress1 inf.total_in=%d def.total_out=%d\n", (int)inf.total_in, (int)def.total_out); if (ret == Z_MEM_ERROR) quit("out of memory"); @@ -222,9 +224,9 @@ printf("recompress1 inf.avail_in=%d def.avail_out=%d\n", inf.avail_in, def.avail inf.next_in = tmp; def.avail_out = size; def.next_out = blk; -printf("recompress2 inf.avail_in=%d def.avail_out=%d\n", inf.avail_in, def.avail_out); + LOG_FITBLK("recompress2 inf.total_in=%d def.total_out=%d\n", (int)inf.total_in, (int)def.total_out); ret = recompress(&inf, &def); -printf("recompress2 inf.avail_in=%d def.avail_out=%d\n", inf.avail_in, def.avail_out); + LOG_FITBLK("recompress2 inf.total_in=%d def.total_out=%d\n", (int)inf.total_in, (int)def.total_out); if (ret == Z_MEM_ERROR) quit("out of memory"); assert(ret == Z_STREAM_END); /* otherwise MARGIN too small */ diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index 28f42526e..90ec590d0 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -23,7 +23,7 @@ #define ZWRAP_DEFAULT_CLEVEL 5 /* Z_DEFAULT_COMPRESSION is translated to ZWRAP_DEFAULT_CLEVEL for zstd */ #define LOG_WRAPPERC(...) /*printf(__VA_ARGS__)*/ -#define LOG_WRAPPERD(...) /*printf(__VA_ARGS__)*/ +#define LOG_WRAPPERD(...) /*printf(__VA_ARGS__)*/ #define FINISH_WITH_GZ_ERR(msg) { \ @@ -323,6 +323,9 @@ typedef struct { ZSTD_DStream* zbd; char headerBuf[16]; /* should be equal or bigger than ZSTD_frameHeaderSize_min */ int errorCount; + int decompState; + ZSTD_inBuffer inBuffer; + ZSTD_outBuffer outBuffer; /* zlib params */ int stream_size; @@ -330,11 +333,16 @@ typedef struct { int windowBits; ZSTD_customMem customMem; z_stream allocFunc; /* copy of zalloc, zfree, opaque */ - ZSTD_inBuffer inBuffer; - ZSTD_outBuffer outBuffer; } ZWRAP_DCtx; +void ZWRAP_initDCtx(ZWRAP_DCtx* zwd) +{ + zwd->errorCount = zwd->decompState = 0; + zwd->outBuffer.pos = 0; + zwd->outBuffer.size = 0; +} + ZWRAP_DCtx* ZWRAP_createDCtx(z_streamp strm) { ZWRAP_DCtx* zwd; @@ -353,9 +361,8 @@ ZWRAP_DCtx* ZWRAP_createDCtx(z_streamp strm) memset(zwd, 0, sizeof(ZWRAP_DCtx)); memcpy(&zwd->customMem, &defaultCustomMem, sizeof(ZSTD_customMem)); } - zwd->outBuffer.pos = 0; - zwd->outBuffer.size = 0; + ZWRAP_initDCtx(zwd); return zwd; } @@ -422,6 +429,7 @@ ZEXTERN int ZEXPORT z_inflateReset OF((z_streamp strm)) if (zwd == NULL) return Z_STREAM_ERROR; { size_t const errorCode = ZSTD_resetDStream(zwd->zbd); if (ZSTD_isError(errorCode)) return ZWRAPD_finish_with_error(zwd, strm, 0); } + ZWRAP_initDCtx(zwd); } strm->total_in = 0; @@ -470,7 +478,7 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) if (!strm->reserved) { LOG_WRAPPERD("- inflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); res = inflate(strm, flush); - LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); + LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, res); return res; } @@ -479,6 +487,9 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state; if (strm->state == NULL) return Z_STREAM_ERROR; LOG_WRAPPERD("- inflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); + + if (zwd->decompState == Z_STREAM_END) return Z_STREAM_END; + // if (((strm->avail_in < ZSTD_HEADERSIZE) || (strm->total_in > 0)) && (strm->total_in < ZLIB_HEADERSIZE)) if (strm->total_in < ZLIB_HEADERSIZE) { @@ -524,7 +535,7 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) if (flush == Z_INFLATE_SYNC) res = inflateSync(strm); else res = inflate(strm, flush); - LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); + LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, res); return res; } } @@ -558,7 +569,7 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) errorCode = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer); LOG_WRAPPERD("inflate ZSTD_decompressStream1 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)zwd->inBuffer.size, (int)zwd->outBuffer.size); if (ZSTD_isError(errorCode)) { - LOG_WRAPPERD("ERROR: ZSTD_decompressStream %s\n", ZSTD_getErrorName(errorCode)); + LOG_WRAPPERD("ERROR: ZSTD_decompressStream1 %s\n", ZSTD_getErrorName(errorCode)); goto error; } // LOG_WRAPPERD("1srcSize=%d inpos=%d inBuffer.pos=%d inBuffer.size=%d outBuffer.pos=%d\n", (int)srcSize, (int)inPos, (int)zwd->inBuffer.pos, (int)zwd->inBuffer.size, (int)zwd->outBuffer.pos); @@ -573,26 +584,30 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) zwd->outBuffer.size = strm->avail_out; zwd->outBuffer.pos = 0; errorCode = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer); - // LOG_WRAPPERD("2 inpos=%d inBuffer.pos=%d inBuffer.size=%d outBuffer.pos=%d\n", (int)inPos, (int)zwd->inBuffer.pos, (int)zwd->inBuffer.size, (int)zwd->outBuffer.pos); LOG_WRAPPERD("inflate ZSTD_decompressStream2 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)strm->avail_in, (int)strm->avail_out); if (ZSTD_isError(errorCode)) { - LOG_WRAPPERD("ERROR: ZSTD_decompressStream %s\n", ZSTD_getErrorName(errorCode)); zwd->errorCount++; + LOG_WRAPPERD("ERROR: ZSTD_decompressStream2 %s zwd->errorCount=%d\n", ZSTD_getErrorName(errorCode), zwd->errorCount); if (zwd->errorCount<=1) return Z_NEED_DICT; else goto error; } + LOG_WRAPPERD("inflate inpos=%d inBuffer.pos=%d inBuffer.size=%d outBuffer.pos=%d outBuffer.size=%d o\n", (int)inPos, (int)zwd->inBuffer.pos, (int)zwd->inBuffer.size, (int)zwd->outBuffer.pos, (int)zwd->outBuffer.size); strm->next_out += zwd->outBuffer.pos; strm->total_out += zwd->outBuffer.pos; strm->avail_out -= zwd->outBuffer.pos; strm->total_in += zwd->inBuffer.pos - inPos; strm->next_in += zwd->inBuffer.pos - inPos; strm->avail_in -= zwd->inBuffer.pos - inPos; - if (errorCode == 0) { LOG_WRAPPERD("inflate Z_STREAM_END1 strm->total_in=%d strm->avail_out=%d strm->total_out=%d\n", (int)strm->total_in, (int)strm->avail_out, (int)strm->total_out); return Z_STREAM_END; } + if (errorCode == 0) { + LOG_WRAPPERD("inflate Z_STREAM_END1 avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); + zwd->decompState = Z_STREAM_END; + return Z_STREAM_END; + } goto finish; error: return ZWRAPD_finish_with_error(zwd, strm, 0); } finish: - LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); + LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, Z_OK); return Z_OK; } From 694130015b1698eb9ae3bb1c67ee95e777b737e6 Mon Sep 17 00:00:00 2001 From: inikep Date: Tue, 20 Sep 2016 16:40:50 +0200 Subject: [PATCH 12/43] implemented inflateReset2 --- zlibWrapper/README.md | 2 +- zlibWrapper/zstd_zlibwrapper.c | 31 ++++++++++++++++++++----------- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/zlibWrapper/README.md b/zlibWrapper/README.md index e86da2f58..e90a5189a 100644 --- a/zlibWrapper/README.md +++ b/zlibWrapper/README.md @@ -79,6 +79,7 @@ Supported methods: - inflate - inflateSetDictionary - inflateReset +- inflateReset2 - compress - compress2 - compressBound @@ -97,7 +98,6 @@ Unsupported methods: - inflateGetDictionary - inflateCopy - inflateSync -- inflateReset2 - inflatePrime - inflateMark - inflateGetHeader diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index 90ec590d0..d3f95fb9c 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -413,6 +413,7 @@ ZEXTERN int ZEXPORT z_inflateInit2_ OF((z_streamp strm, int windowBits, int ret = z_inflateInit_ (strm, version, stream_size); if (ret == Z_OK) { ZWRAP_DCtx* zwd = (ZWRAP_DCtx*)strm->state; + if (zwd == NULL) return Z_STREAM_ERROR; zwd->windowBits = windowBits; } return ret; @@ -438,6 +439,25 @@ ZEXTERN int ZEXPORT z_inflateReset OF((z_streamp strm)) } +#if ZLIB_VERNUM >= 0x1240 +ZEXTERN int ZEXPORT z_inflateReset2 OF((z_streamp strm, + int windowBits)) +{ + if (!strm->reserved) + return inflateReset2(strm, windowBits); + + { int ret = z_inflateReset (strm); + if (ret == Z_OK) { + ZWRAP_DCtx* zwd = (ZWRAP_DCtx*)strm->state; + if (zwd == NULL) return Z_STREAM_ERROR; + zwd->windowBits = windowBits; + } + return ret; + } +} +#endif + + ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm, const Bytef *dictionary, uInt dictLength)) @@ -717,17 +737,6 @@ ZEXTERN int ZEXPORT z_inflateCopy OF((z_streamp dest, } -#if ZLIB_VERNUM >= 0x1240 -ZEXTERN int ZEXPORT z_inflateReset2 OF((z_streamp strm, - int windowBits)) -{ - if (!strm->reserved) - return inflateReset2(strm, windowBits); - FINISH_WITH_ERR(strm, "inflateReset2 is not supported!"); -} -#endif - - #if ZLIB_VERNUM >= 0x1240 ZEXTERN long ZEXPORT z_inflateMark OF((z_streamp strm)) { From 3fa1b748d62d95c029b0ebfce913bdaf84aa2c91 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 21 Sep 2016 13:51:57 +0200 Subject: [PATCH 13/43] improved deflateEnd and inflateEnd --- zlibWrapper/README.md | 2 +- zlibWrapper/zstd_zlibwrapper.c | 21 ++++++++++++--------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/zlibWrapper/README.md b/zlibWrapper/README.md index e90a5189a..c2637fe6f 100644 --- a/zlibWrapper/README.md +++ b/zlibWrapper/README.md @@ -70,7 +70,7 @@ After enabling zstd compression not all native zlib functions are supported. Whe Supported methods: - deflateInit -- deflate (with exception of Z_FULL_FLUSH) +- deflate (with exception of Z_FULL_FLUSH, Z_BLOCK, and Z_TREES) - deflateSetDictionary - deflateEnd - deflateReset diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index d3f95fb9c..faa43c14e 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -282,9 +282,11 @@ ZEXTERN int ZEXPORT z_deflateEnd OF((z_streamp strm)) return deflateEnd(strm); } LOG_WRAPPERC("- deflateEnd total_in=%d total_out=%d\n", (int)(strm->total_in), (int)(strm->total_out)); - { ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state; - size_t const errorCode = ZWRAP_freeCCtx(zwc); + { size_t errorCode; + ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state; + if (zwc == NULL) return Z_OK; /* structures are already freed */ strm->state = NULL; + errorCode = ZWRAP_freeCCtx(zwc); if (ZSTD_isError(errorCode)) return Z_STREAM_ERROR; } return Z_OK; @@ -468,7 +470,7 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm, { size_t errorCode; ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state; - if (strm->state == NULL) return Z_STREAM_ERROR; + if (zwd == NULL) return Z_STREAM_ERROR; errorCode = ZSTD_initDStream_usingDict(zwd->zbd, dictionary, dictLength); if (ZSTD_isError(errorCode)) return ZWRAPD_finish_with_error(zwd, strm, 0); @@ -505,7 +507,7 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) if (strm->avail_in > 0) { size_t errorCode, srcSize, inPos; ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state; - if (strm->state == NULL) return Z_STREAM_ERROR; + if (zwd == NULL) return Z_STREAM_ERROR; LOG_WRAPPERD("- inflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); if (zwd->decompState == Z_STREAM_END) return Z_STREAM_END; @@ -634,17 +636,18 @@ finish: ZEXTERN int ZEXPORT z_inflateEnd OF((z_streamp strm)) { - int ret = Z_OK; if (!strm->reserved) return inflateEnd(strm); LOG_WRAPPERD("- inflateEnd total_in=%d total_out=%d\n", (int)(strm->total_in), (int)(strm->total_out)); - { ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state; - size_t const errorCode = ZWRAP_freeDCtx(zwd); + { size_t errorCode; + ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state; + if (zwd == NULL) return Z_OK; /* structures are already freed */ strm->state = NULL; - if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; + errorCode = ZWRAP_freeDCtx(zwd); + if (ZSTD_isError(errorCode)) return Z_STREAM_ERROR; } - return ret; + return Z_OK; } From 146ef58ff8130a5f6c6edc73b594b05dc82c1502 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 21 Sep 2016 14:05:01 +0200 Subject: [PATCH 14/43] added ZWRAPC_finish_with_error_message and ZWRAPD_finish_with_error_message --- zlibWrapper/zstd_zlibwrapper.c | 53 ++++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index faa43c14e..51a362839 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -31,11 +31,6 @@ return Z_STREAM_ERROR; \ } -#define FINISH_WITH_ERR(strm, message) { \ - strm->msg = message; \ - return Z_STREAM_ERROR; \ -} - #define FINISH_WITH_NULL_ERR(msg) { \ (void)msg; \ return NULL; \ @@ -130,6 +125,16 @@ int ZWRAPC_finish_with_error(ZWRAP_CCtx* zwc, z_streamp strm, int error) } +int ZWRAPC_finish_with_error_message(z_streamp strm, char* message) +{ + ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state; + strm->msg = message; + if (zwc == NULL) return Z_STREAM_ERROR; + + return ZWRAPC_finish_with_error(zwc, strm, 0); +} + + ZEXTERN int ZEXPORT z_deflateInit_ OF((z_streamp strm, int level, const char *version, int stream_size)) { @@ -242,7 +247,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) strm->avail_in -= zwc->inBuffer.pos; } - if (flush == Z_FULL_FLUSH || flush == Z_BLOCK || flush == Z_TREES) FINISH_WITH_ERR(strm, "Z_FULL_FLUSH, Z_BLOCK and Z_TREES are not supported!"); + if (flush == Z_FULL_FLUSH || flush == Z_BLOCK || flush == Z_TREES) return ZWRAPC_finish_with_error_message(strm, "Z_FULL_FLUSH, Z_BLOCK and Z_TREES are not supported!"); if (flush == Z_FINISH) { size_t bytesLeft; @@ -388,6 +393,16 @@ int ZWRAPD_finish_with_error(ZWRAP_DCtx* zwd, z_streamp strm, int error) } +int ZWRAPD_finish_with_error_message(z_streamp strm, char* message) +{ + ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state; + strm->msg = message; + if (zwd == NULL) return Z_STREAM_ERROR; + + return ZWRAPD_finish_with_error(zwd, strm, 0); +} + + ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm, const char *version, int stream_size)) { @@ -669,7 +684,7 @@ ZEXTERN int ZEXPORT z_deflateCopy OF((z_streamp dest, { if (!g_useZSTD) return deflateCopy(dest, source); - FINISH_WITH_ERR(source, "deflateCopy is not supported!"); + return ZWRAPC_finish_with_error_message(source, "deflateCopy is not supported!"); } @@ -681,7 +696,7 @@ ZEXTERN int ZEXPORT z_deflateTune OF((z_streamp strm, { if (!g_useZSTD) return deflateTune(strm, good_length, max_lazy, nice_length, max_chain); - FINISH_WITH_ERR(strm, "deflateTune is not supported!"); + return ZWRAPC_finish_with_error_message(strm, "deflateTune is not supported!"); } @@ -692,7 +707,7 @@ ZEXTERN int ZEXPORT z_deflatePending OF((z_streamp strm, { if (!g_useZSTD) return deflatePending(strm, pending, bits); - FINISH_WITH_ERR(strm, "deflatePending is not supported!"); + return ZWRAPC_finish_with_error_message(strm, "deflatePending is not supported!"); } #endif @@ -703,7 +718,7 @@ ZEXTERN int ZEXPORT z_deflatePrime OF((z_streamp strm, { if (!g_useZSTD) return deflatePrime(strm, bits, value); - FINISH_WITH_ERR(strm, "deflatePrime is not supported!"); + return ZWRAPC_finish_with_error_message(strm, "deflatePrime is not supported!"); } @@ -712,7 +727,7 @@ ZEXTERN int ZEXPORT z_deflateSetHeader OF((z_streamp strm, { if (!g_useZSTD) return deflateSetHeader(strm, head); - FINISH_WITH_ERR(strm, "deflateSetHeader is not supported!"); + return ZWRAPC_finish_with_error_message(strm, "deflateSetHeader is not supported!"); } @@ -726,7 +741,7 @@ ZEXTERN int ZEXPORT z_inflateGetDictionary OF((z_streamp strm, { if (!strm->reserved) return inflateGetDictionary(strm, dictionary, dictLength); - FINISH_WITH_ERR(strm, "inflateGetDictionary is not supported!"); + return ZWRAPD_finish_with_error_message(strm, "inflateGetDictionary is not supported!"); } #endif @@ -736,7 +751,7 @@ ZEXTERN int ZEXPORT z_inflateCopy OF((z_streamp dest, { if (!g_useZSTD) return inflateCopy(dest, source); - FINISH_WITH_ERR(source, "inflateCopy is not supported!"); + return ZWRAPD_finish_with_error_message(source, "inflateCopy is not supported!"); } @@ -745,7 +760,7 @@ ZEXTERN long ZEXPORT z_inflateMark OF((z_streamp strm)) { if (!strm->reserved) return inflateMark(strm); - FINISH_WITH_ERR(strm, "inflateMark is not supported!"); + return ZWRAPD_finish_with_error_message(strm, "inflateMark is not supported!"); } #endif @@ -756,7 +771,7 @@ ZEXTERN int ZEXPORT z_inflatePrime OF((z_streamp strm, { if (!strm->reserved) return inflatePrime(strm, bits, value); - FINISH_WITH_ERR(strm, "inflatePrime is not supported!"); + return ZWRAPD_finish_with_error_message(strm, "inflatePrime is not supported!"); } @@ -765,7 +780,7 @@ ZEXTERN int ZEXPORT z_inflateGetHeader OF((z_streamp strm, { if (!strm->reserved) return inflateGetHeader(strm, head); - FINISH_WITH_ERR(strm, "inflateGetHeader is not supported!"); + return ZWRAPD_finish_with_error_message(strm, "inflateGetHeader is not supported!"); } @@ -776,7 +791,7 @@ ZEXTERN int ZEXPORT z_inflateBackInit_ OF((z_streamp strm, int windowBits, { if (!strm->reserved) return inflateBackInit_(strm, windowBits, window, version, stream_size); - FINISH_WITH_ERR(strm, "inflateBackInit is not supported!"); + return ZWRAPD_finish_with_error_message(strm, "inflateBackInit is not supported!"); } @@ -786,7 +801,7 @@ ZEXTERN int ZEXPORT z_inflateBack OF((z_streamp strm, { if (!strm->reserved) return inflateBack(strm, in, in_desc, out, out_desc); - FINISH_WITH_ERR(strm, "inflateBack is not supported!"); + return ZWRAPD_finish_with_error_message(strm, "inflateBack is not supported!"); } @@ -794,7 +809,7 @@ ZEXTERN int ZEXPORT z_inflateBackEnd OF((z_streamp strm)) { if (!strm->reserved) return inflateBackEnd(strm); - FINISH_WITH_ERR(strm, "inflateBackEnd is not supported!"); + return ZWRAPD_finish_with_error_message(strm, "inflateBackEnd is not supported!"); } From 230a61fff2fc34eb485842ff272db6dd4afe9ea6 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 21 Sep 2016 16:46:35 +0200 Subject: [PATCH 15/43] added ZSTD_setPledgedSrcSize --- zlibWrapper/zstd_zlibwrapper.c | 21 +++++++++++++++++++-- zlibWrapper/zstd_zlibwrapper.h | 10 ++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index 51a362839..0c09e6fb9 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -53,6 +53,7 @@ const char * zstdVersion(void) { return ZSTD_VERSION_STRING; } ZEXTERN const char * ZEXPORT z_zlibVersion OF((void)) { return zlibVersion(); } + static void* ZWRAP_allocFunction(void* opaque, size_t size) { z_streamp strm = (z_streamp) opaque; @@ -79,6 +80,7 @@ typedef struct { z_stream allocFunc; /* copy of zalloc, zfree, opaque */ ZSTD_inBuffer inBuffer; ZSTD_outBuffer outBuffer; + unsigned long long pledgedSrcSize; } ZWRAP_CCtx; @@ -110,6 +112,7 @@ ZWRAP_CCtx* ZWRAP_createCCtx(z_streamp strm) memcpy(&zwc->customMem, &defaultCustomMem, sizeof(ZSTD_customMem)); } + zwc->pledgedSrcSize = 1<<16; zwc->zbc = ZSTD_createCStream_advanced(zwc->customMem); if (zwc->zbc == NULL) { ZWRAP_freeCCtx(zwc); return NULL; } return zwc; @@ -135,6 +138,16 @@ int ZWRAPC_finish_with_error_message(z_streamp strm, char* message) } +int ZSTD_setPledgedSrcSize(z_streamp strm, unsigned long long pledgedSrcSize) +{ + ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state; + if (zwc == NULL) return Z_STREAM_ERROR; + + zwc->pledgedSrcSize = pledgedSrcSize; + return Z_OK; +} + + ZEXTERN int ZEXPORT z_deflateInit_ OF((z_streamp strm, int level, const char *version, int stream_size)) { @@ -151,7 +164,10 @@ ZEXTERN int ZEXPORT z_deflateInit_ OF((z_streamp strm, int level, if (level == Z_DEFAULT_COMPRESSION) level = ZWRAP_DEFAULT_CLEVEL; - { size_t const errorCode = ZSTD_initCStream(zwc->zbc, level); + { ZSTD_parameters const params = ZSTD_getParams(level, zwc->pledgedSrcSize, 0); /* use the 4th table which is adapted for srcSize <= 16KB */ + size_t errorCode; + LOG_WRAPPERC("windowLog=%d chainLog=%d hashLog=%d searchLog=%d searchLength=%d strategy=%d\n", params.cParams.windowLog, params.cParams.chainLog, params.cParams.hashLog, params.cParams.searchLog, params.cParams.searchLength, params.cParams.strategy); + errorCode = ZSTD_initCStream_advanced(zwc->zbc, NULL, 0, params, zwc->pledgedSrcSize /*pledgedSrcSize*/); if (ZSTD_isError(errorCode)) return ZWRAPC_finish_with_error(zwc, strm, 0); } zwc->compressionLevel = level; @@ -182,7 +198,8 @@ ZEXTERN int ZEXPORT z_deflateReset OF((z_streamp strm)) { ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state; if (zwc == NULL) return Z_STREAM_ERROR; - { size_t const errorCode = ZSTD_resetCStream(zwc->zbc, 0); + { size_t const errorCode = ZSTD_resetCStream(zwc->zbc, zwc->pledgedSrcSize); + printf("zwc->pledgedSrcSize=%d\n", (int)zwc->pledgedSrcSize); if (ZSTD_isError(errorCode)) return ZWRAPC_finish_with_error(zwc, strm, 0); } } diff --git a/zlibWrapper/zstd_zlibwrapper.h b/zlibWrapper/zstd_zlibwrapper.h index 24247b2ca..f7763b2f0 100644 --- a/zlibWrapper/zstd_zlibwrapper.h +++ b/zlibWrapper/zstd_zlibwrapper.h @@ -26,10 +26,20 @@ extern "C" { #endif #endif +/* enables/disables zstd compression during runtime */ void useZSTD(int turn_on); + +/* check if zstd compression is turned on */ int isUsingZSTD(void); + +/* returns a string with version of zstd library */ const char * zstdVersion(void); +/* Changes a pledged source size for a given stream. + The function should be called after deflateInit(). + After this function deflateReset() should be called. */ +int ZSTD_setPledgedSrcSize(z_streamp strm, unsigned long long pledgedSrcSize); + #if defined (__cplusplus) } From 7e7925710d8ade0edd3f2c9248c5e05223992de3 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 21 Sep 2016 17:17:29 +0200 Subject: [PATCH 16/43] tests with ZSTD_setPledgedSrcSize --- zlibWrapper/examples/fitblk.c | 8 +++++++- zlibWrapper/zstd_zlibwrapper.c | 2 +- zlibWrapper/zstd_zlibwrapper.h | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/zlibWrapper/examples/fitblk.c b/zlibWrapper/examples/fitblk.c index 12e4c8626..2666b2456 100644 --- a/zlibWrapper/examples/fitblk.c +++ b/zlibWrapper/examples/fitblk.c @@ -162,13 +162,19 @@ int main(int argc, char **argv) ret = deflateInit(&def, Z_DEFAULT_COMPRESSION); if (ret != Z_OK || blk == NULL) quit("out of memory"); + ret = ZSTD_setPledgedSrcSize(&def, 1<<16); + if (ret != Z_OK) + quit("ZSTD_setPledgedSrcSize"); + ret = deflateReset(&def); + if (ret != Z_OK) + quit("deflateReset"); /* compress from stdin until output full, or no more input */ def.avail_out = size + EXCESS; def.next_out = blk; LOG_FITBLK("partcompress1 total_in=%d total_out=%d\n", (int)def.total_in, (int)def.total_out); ret = partcompress(stdin, &def); - LOG_FITBLK("partcompress2 total_in=%d total_out=%d\n", (int)def.total_in, (int)def.total_out); + printf("partcompress total_in=%d total_out=%d\n", (int)def.total_in, (int)def.total_out); if (ret == Z_ERRNO) quit("error reading input"); diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index 0c09e6fb9..d921f5476 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -112,7 +112,7 @@ ZWRAP_CCtx* ZWRAP_createCCtx(z_streamp strm) memcpy(&zwc->customMem, &defaultCustomMem, sizeof(ZSTD_customMem)); } - zwc->pledgedSrcSize = 1<<16; + zwc->pledgedSrcSize = 0; zwc->zbc = ZSTD_createCStream_advanced(zwc->customMem); if (zwc->zbc == NULL) { ZWRAP_freeCCtx(zwc); return NULL; } return zwc; diff --git a/zlibWrapper/zstd_zlibwrapper.h b/zlibWrapper/zstd_zlibwrapper.h index f7763b2f0..149d5ace5 100644 --- a/zlibWrapper/zstd_zlibwrapper.h +++ b/zlibWrapper/zstd_zlibwrapper.h @@ -35,7 +35,7 @@ int isUsingZSTD(void); /* returns a string with version of zstd library */ const char * zstdVersion(void); -/* Changes a pledged source size for a given stream. +/* Changes a pledged source size for a given compression stream. The function should be called after deflateInit(). After this function deflateReset() should be called. */ int ZSTD_setPledgedSrcSize(z_streamp strm, unsigned long long pledgedSrcSize); From 61abecc41762a3d1bb82dfa0586297d1f83dba71 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 21 Sep 2016 19:30:29 +0200 Subject: [PATCH 17/43] added ZWRAP_initializeCStream --- zlibWrapper/examples/fitblk.c | 3 --- zlibWrapper/zstd_zlibwrapper.c | 48 +++++++++++++++++++++++++--------- 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/zlibWrapper/examples/fitblk.c b/zlibWrapper/examples/fitblk.c index 2666b2456..17b422668 100644 --- a/zlibWrapper/examples/fitblk.c +++ b/zlibWrapper/examples/fitblk.c @@ -165,9 +165,6 @@ int main(int argc, char **argv) ret = ZSTD_setPledgedSrcSize(&def, 1<<16); if (ret != Z_OK) quit("ZSTD_setPledgedSrcSize"); - ret = deflateReset(&def); - if (ret != Z_OK) - quit("deflateReset"); /* compress from stdin until output full, or no more input */ def.avail_out = size + EXCESS; diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index d921f5476..58e76ec91 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -87,7 +87,7 @@ typedef struct { size_t ZWRAP_freeCCtx(ZWRAP_CCtx* zwc) { if (zwc==NULL) return 0; /* support free on NULL */ - ZSTD_freeCStream(zwc->zbc); + if (zwc->zbc) ZSTD_freeCStream(zwc->zbc); zwc->customMem.customFree(zwc->customMem.opaque, zwc); return 0; } @@ -112,13 +112,29 @@ ZWRAP_CCtx* ZWRAP_createCCtx(z_streamp strm) memcpy(&zwc->customMem, &defaultCustomMem, sizeof(ZSTD_customMem)); } - zwc->pledgedSrcSize = 0; - zwc->zbc = ZSTD_createCStream_advanced(zwc->customMem); - if (zwc->zbc == NULL) { ZWRAP_freeCCtx(zwc); return NULL; } return zwc; } +int ZWRAP_initializeCStream(ZWRAP_CCtx* zwc) +{ + if (zwc == NULL) return Z_STREAM_ERROR; + + if (zwc->zbc == NULL) { + zwc->zbc = ZSTD_createCStream_advanced(zwc->customMem); + if (zwc->zbc == NULL) return Z_STREAM_ERROR; + + { ZSTD_parameters const params = ZSTD_getParams(zwc->compressionLevel, zwc->pledgedSrcSize, 0); + size_t errorCode; + LOG_WRAPPERC("windowLog=%d chainLog=%d hashLog=%d searchLog=%d searchLength=%d strategy=%d\n", params.cParams.windowLog, params.cParams.chainLog, params.cParams.hashLog, params.cParams.searchLog, params.cParams.searchLength, params.cParams.strategy); + errorCode = ZSTD_initCStream_advanced(zwc->zbc, NULL, 0, params, zwc->pledgedSrcSize); + if (ZSTD_isError(errorCode)) return Z_STREAM_ERROR; } + } + + return Z_OK; +} + + int ZWRAPC_finish_with_error(ZWRAP_CCtx* zwc, z_streamp strm, int error) { LOG_WRAPPERC("- ZWRAPC_finish_with_error=%d\n", error); @@ -164,12 +180,6 @@ ZEXTERN int ZEXPORT z_deflateInit_ OF((z_streamp strm, int level, if (level == Z_DEFAULT_COMPRESSION) level = ZWRAP_DEFAULT_CLEVEL; - { ZSTD_parameters const params = ZSTD_getParams(level, zwc->pledgedSrcSize, 0); /* use the 4th table which is adapted for srcSize <= 16KB */ - size_t errorCode; - LOG_WRAPPERC("windowLog=%d chainLog=%d hashLog=%d searchLog=%d searchLength=%d strategy=%d\n", params.cParams.windowLog, params.cParams.chainLog, params.cParams.hashLog, params.cParams.searchLog, params.cParams.searchLength, params.cParams.strategy); - errorCode = ZSTD_initCStream_advanced(zwc->zbc, NULL, 0, params, zwc->pledgedSrcSize /*pledgedSrcSize*/); - if (ZSTD_isError(errorCode)) return ZWRAPC_finish_with_error(zwc, strm, 0); } - zwc->compressionLevel = level; strm->state = (struct internal_state*) zwc; /* use state which in not used by user */ strm->total_in = 0; @@ -197,9 +207,12 @@ ZEXTERN int ZEXPORT z_deflateReset OF((z_streamp strm)) return deflateReset(strm); { ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state; - if (zwc == NULL) return Z_STREAM_ERROR; + if (!zwc) return Z_STREAM_ERROR; + if (zwc->zbc == NULL) { + int res = ZWRAP_initializeCStream(zwc); + if (res != Z_OK) return ZWRAPC_finish_with_error(zwc, strm, res); + } { size_t const errorCode = ZSTD_resetCStream(zwc->zbc, zwc->pledgedSrcSize); - printf("zwc->pledgedSrcSize=%d\n", (int)zwc->pledgedSrcSize); if (ZSTD_isError(errorCode)) return ZWRAPC_finish_with_error(zwc, strm, 0); } } @@ -220,7 +233,11 @@ ZEXTERN int ZEXPORT z_deflateSetDictionary OF((z_streamp strm, { ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state; LOG_WRAPPERC("- deflateSetDictionary level=%d\n", (int)zwc->compressionLevel); - if (zwc == NULL) return Z_STREAM_ERROR; + if (!zwc) return Z_STREAM_ERROR; + if (zwc->zbc == NULL) { + int res = ZWRAP_initializeCStream(zwc); + if (res != Z_OK) return ZWRAPC_finish_with_error(zwc, strm, res); + } { size_t const errorCode = ZSTD_initCStream_usingDict(zwc->zbc, dictionary, dictLength, zwc->compressionLevel); if (ZSTD_isError(errorCode)) return ZWRAPC_finish_with_error(zwc, strm, 0); } } @@ -244,6 +261,11 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) zwc = (ZWRAP_CCtx*) strm->state; if (zwc == NULL) return Z_STREAM_ERROR; + if (zwc->zbc == NULL) { + int res = ZWRAP_initializeCStream(zwc); + if (res != Z_OK) return ZWRAPC_finish_with_error(zwc, strm, res); + } + LOG_WRAPPERC("- deflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); if (strm->avail_in > 0) { zwc->inBuffer.src = strm->next_in; From adc4c1640fb8e615ff32fc281b6a0c8d67940184 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 21 Sep 2016 19:39:25 +0200 Subject: [PATCH 18/43] changed naming convention --- zlibWrapper/zstd_zlibwrapper.c | 78 +++++++++++++++++----------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index 58e76ec91..c0bae799a 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -135,22 +135,22 @@ int ZWRAP_initializeCStream(ZWRAP_CCtx* zwc) } -int ZWRAPC_finish_with_error(ZWRAP_CCtx* zwc, z_streamp strm, int error) +int ZWRAPC_finishWithError(ZWRAP_CCtx* zwc, z_streamp strm, int error) { - LOG_WRAPPERC("- ZWRAPC_finish_with_error=%d\n", error); + LOG_WRAPPERC("- ZWRAPC_finishWithError=%d\n", error); if (zwc) ZWRAP_freeCCtx(zwc); if (strm) strm->state = NULL; return (error) ? error : Z_STREAM_ERROR; } -int ZWRAPC_finish_with_error_message(z_streamp strm, char* message) +int ZWRAPC_finishWithErrorMsg(z_streamp strm, char* message) { ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state; strm->msg = message; if (zwc == NULL) return Z_STREAM_ERROR; - return ZWRAPC_finish_with_error(zwc, strm, 0); + return ZWRAPC_finishWithError(zwc, strm, 0); } @@ -210,10 +210,10 @@ ZEXTERN int ZEXPORT z_deflateReset OF((z_streamp strm)) if (!zwc) return Z_STREAM_ERROR; if (zwc->zbc == NULL) { int res = ZWRAP_initializeCStream(zwc); - if (res != Z_OK) return ZWRAPC_finish_with_error(zwc, strm, res); + if (res != Z_OK) return ZWRAPC_finishWithError(zwc, strm, res); } { size_t const errorCode = ZSTD_resetCStream(zwc->zbc, zwc->pledgedSrcSize); - if (ZSTD_isError(errorCode)) return ZWRAPC_finish_with_error(zwc, strm, 0); } + if (ZSTD_isError(errorCode)) return ZWRAPC_finishWithError(zwc, strm, 0); } } strm->total_in = 0; @@ -236,10 +236,10 @@ ZEXTERN int ZEXPORT z_deflateSetDictionary OF((z_streamp strm, if (!zwc) return Z_STREAM_ERROR; if (zwc->zbc == NULL) { int res = ZWRAP_initializeCStream(zwc); - if (res != Z_OK) return ZWRAPC_finish_with_error(zwc, strm, res); + if (res != Z_OK) return ZWRAPC_finishWithError(zwc, strm, res); } { size_t const errorCode = ZSTD_initCStream_usingDict(zwc->zbc, dictionary, dictLength, zwc->compressionLevel); - if (ZSTD_isError(errorCode)) return ZWRAPC_finish_with_error(zwc, strm, 0); } + if (ZSTD_isError(errorCode)) return ZWRAPC_finishWithError(zwc, strm, 0); } } return Z_OK; @@ -263,7 +263,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) if (zwc->zbc == NULL) { int res = ZWRAP_initializeCStream(zwc); - if (res != Z_OK) return ZWRAPC_finish_with_error(zwc, strm, res); + if (res != Z_OK) return ZWRAPC_finishWithError(zwc, strm, res); } LOG_WRAPPERC("- deflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); @@ -276,7 +276,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) zwc->outBuffer.pos = 0; { size_t const errorCode = ZSTD_compressStream(zwc->zbc, &zwc->outBuffer, &zwc->inBuffer); LOG_WRAPPERC("deflate ZSTD_compressStream srcSize=%d dstCapacity=%d\n", (int)zwc->inBuffer.size, (int)zwc->outBuffer.size); - if (ZSTD_isError(errorCode)) return ZWRAPC_finish_with_error(zwc, strm, 0); + if (ZSTD_isError(errorCode)) return ZWRAPC_finishWithError(zwc, strm, 0); } strm->next_out += zwc->outBuffer.pos; strm->total_out += zwc->outBuffer.pos; @@ -286,7 +286,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) strm->avail_in -= zwc->inBuffer.pos; } - if (flush == Z_FULL_FLUSH || flush == Z_BLOCK || flush == Z_TREES) return ZWRAPC_finish_with_error_message(strm, "Z_FULL_FLUSH, Z_BLOCK and Z_TREES are not supported!"); + if (flush == Z_FULL_FLUSH || flush == Z_BLOCK || flush == Z_TREES) return ZWRAPC_finishWithErrorMsg(strm, "Z_FULL_FLUSH, Z_BLOCK and Z_TREES are not supported!"); if (flush == Z_FINISH) { size_t bytesLeft; @@ -295,7 +295,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) zwc->outBuffer.pos = 0; bytesLeft = ZSTD_endStream(zwc->zbc, &zwc->outBuffer); LOG_WRAPPERC("deflate ZSTD_endStream dstCapacity=%d bytesLeft=%d\n", (int)strm->avail_out, (int)bytesLeft); - if (ZSTD_isError(bytesLeft)) return ZWRAPC_finish_with_error(zwc, strm, 0); + if (ZSTD_isError(bytesLeft)) return ZWRAPC_finishWithError(zwc, strm, 0); strm->next_out += zwc->outBuffer.pos; strm->total_out += zwc->outBuffer.pos; strm->avail_out -= zwc->outBuffer.pos; @@ -309,7 +309,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) zwc->outBuffer.pos = 0; bytesLeft = ZSTD_flushStream(zwc->zbc, &zwc->outBuffer); LOG_WRAPPERC("deflate ZSTD_flushStream dstCapacity=%d bytesLeft=%d\n", (int)strm->avail_out, (int)bytesLeft); - if (ZSTD_isError(bytesLeft)) return ZWRAPC_finish_with_error(zwc, strm, 0); + if (ZSTD_isError(bytesLeft)) return ZWRAPC_finishWithError(zwc, strm, 0); strm->next_out += zwc->outBuffer.pos; strm->total_out += zwc->outBuffer.pos; strm->avail_out -= zwc->outBuffer.pos; @@ -423,22 +423,22 @@ size_t ZWRAP_freeDCtx(ZWRAP_DCtx* zwd) } -int ZWRAPD_finish_with_error(ZWRAP_DCtx* zwd, z_streamp strm, int error) +int ZWRAPD_finishWithError(ZWRAP_DCtx* zwd, z_streamp strm, int error) { - LOG_WRAPPERD("- ZWRAPD_finish_with_error=%d\n", error); + LOG_WRAPPERD("- ZWRAPD_finishWithError=%d\n", error); if (zwd) ZWRAP_freeDCtx(zwd); if (strm) strm->state = NULL; return (error) ? error : Z_STREAM_ERROR; } -int ZWRAPD_finish_with_error_message(z_streamp strm, char* message) +int ZWRAPD_finishWithErrorMsg(z_streamp strm, char* message) { ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state; strm->msg = message; if (zwd == NULL) return Z_STREAM_ERROR; - return ZWRAPD_finish_with_error(zwd, strm, 0); + return ZWRAPD_finishWithError(zwd, strm, 0); } @@ -447,10 +447,10 @@ ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm, { ZWRAP_DCtx* zwd = ZWRAP_createDCtx(strm); LOG_WRAPPERD("- inflateInit\n"); - if (zwd == NULL) return ZWRAPD_finish_with_error(zwd, strm, 0); + if (zwd == NULL) return ZWRAPD_finishWithError(zwd, strm, 0); zwd->version = zwd->customMem.customAlloc(zwd->customMem.opaque, strlen(version) + 1); - if (zwd->version == NULL) return ZWRAPD_finish_with_error(zwd, strm, 0); + if (zwd->version == NULL) return ZWRAPD_finishWithError(zwd, strm, 0); strcpy(zwd->version, version); zwd->stream_size = stream_size; @@ -485,7 +485,7 @@ ZEXTERN int ZEXPORT z_inflateReset OF((z_streamp strm)) { ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state; if (zwd == NULL) return Z_STREAM_ERROR; { size_t const errorCode = ZSTD_resetDStream(zwd->zbd); - if (ZSTD_isError(errorCode)) return ZWRAPD_finish_with_error(zwd, strm, 0); } + if (ZSTD_isError(errorCode)) return ZWRAPD_finishWithError(zwd, strm, 0); } ZWRAP_initDCtx(zwd); } @@ -526,7 +526,7 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm, ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state; if (zwd == NULL) return Z_STREAM_ERROR; errorCode = ZSTD_initDStream_usingDict(zwd->zbd, dictionary, dictLength); - if (ZSTD_isError(errorCode)) return ZWRAPD_finish_with_error(zwd, strm, 0); + if (ZSTD_isError(errorCode)) return ZWRAPD_finishWithError(zwd, strm, 0); if (strm->total_in == ZSTD_HEADERSIZE) { zwd->inBuffer.src = zwd->headerBuf; @@ -539,7 +539,7 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm, LOG_WRAPPERD("inflateSetDictionary ZSTD_decompressStream errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)zwd->inBuffer.size, (int)zwd->outBuffer.size); if (zwd->inBuffer.pos < zwd->outBuffer.size || ZSTD_isError(errorCode)) { LOG_WRAPPERD("ERROR: ZSTD_decompressStream %s\n", ZSTD_getErrorName(errorCode)); - return ZWRAPD_finish_with_error(zwd, strm, 0); + return ZWRAPD_finishWithError(zwd, strm, 0); } } } @@ -589,7 +589,7 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) else errorCode = inflateInit_(strm, zwd->version, zwd->stream_size); LOG_WRAPPERD("ZLIB inflateInit errorCode=%d\n", (int)errorCode); - if (errorCode != Z_OK) return ZWRAPD_finish_with_error(zwd, strm, (int)errorCode); + if (errorCode != Z_OK) return ZWRAPD_finishWithError(zwd, strm, (int)errorCode); /* inflate header */ strm->next_in = (unsigned char*)zwd->headerBuf; @@ -597,7 +597,7 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) strm->avail_out = 0; errorCode = inflate(strm, Z_NO_FLUSH); LOG_WRAPPERD("ZLIB inflate errorCode=%d strm->avail_in=%d\n", (int)errorCode, (int)strm->avail_in); - if (errorCode != Z_OK) return ZWRAPD_finish_with_error(zwd, strm, (int)errorCode); + if (errorCode != Z_OK) return ZWRAPD_finishWithError(zwd, strm, (int)errorCode); if (strm->avail_in > 0) goto error; strm->next_in = strm2.next_in; @@ -649,7 +649,7 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) goto error; } // LOG_WRAPPERD("1srcSize=%d inpos=%d inBuffer.pos=%d inBuffer.size=%d outBuffer.pos=%d\n", (int)srcSize, (int)inPos, (int)zwd->inBuffer.pos, (int)zwd->inBuffer.size, (int)zwd->outBuffer.pos); - if (zwd->inBuffer.pos != zwd->inBuffer.size) return ZWRAPD_finish_with_error(zwd, strm, 0); /* not consumed */ + if (zwd->inBuffer.pos != zwd->inBuffer.size) return ZWRAPD_finishWithError(zwd, strm, 0); /* not consumed */ } inPos = 0;//zwd->inBuffer.pos; @@ -680,7 +680,7 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) } goto finish; error: - return ZWRAPD_finish_with_error(zwd, strm, 0); + return ZWRAPD_finishWithError(zwd, strm, 0); } finish: LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, Z_OK); @@ -723,7 +723,7 @@ ZEXTERN int ZEXPORT z_deflateCopy OF((z_streamp dest, { if (!g_useZSTD) return deflateCopy(dest, source); - return ZWRAPC_finish_with_error_message(source, "deflateCopy is not supported!"); + return ZWRAPC_finishWithErrorMsg(source, "deflateCopy is not supported!"); } @@ -735,7 +735,7 @@ ZEXTERN int ZEXPORT z_deflateTune OF((z_streamp strm, { if (!g_useZSTD) return deflateTune(strm, good_length, max_lazy, nice_length, max_chain); - return ZWRAPC_finish_with_error_message(strm, "deflateTune is not supported!"); + return ZWRAPC_finishWithErrorMsg(strm, "deflateTune is not supported!"); } @@ -746,7 +746,7 @@ ZEXTERN int ZEXPORT z_deflatePending OF((z_streamp strm, { if (!g_useZSTD) return deflatePending(strm, pending, bits); - return ZWRAPC_finish_with_error_message(strm, "deflatePending is not supported!"); + return ZWRAPC_finishWithErrorMsg(strm, "deflatePending is not supported!"); } #endif @@ -757,7 +757,7 @@ ZEXTERN int ZEXPORT z_deflatePrime OF((z_streamp strm, { if (!g_useZSTD) return deflatePrime(strm, bits, value); - return ZWRAPC_finish_with_error_message(strm, "deflatePrime is not supported!"); + return ZWRAPC_finishWithErrorMsg(strm, "deflatePrime is not supported!"); } @@ -766,7 +766,7 @@ ZEXTERN int ZEXPORT z_deflateSetHeader OF((z_streamp strm, { if (!g_useZSTD) return deflateSetHeader(strm, head); - return ZWRAPC_finish_with_error_message(strm, "deflateSetHeader is not supported!"); + return ZWRAPC_finishWithErrorMsg(strm, "deflateSetHeader is not supported!"); } @@ -780,7 +780,7 @@ ZEXTERN int ZEXPORT z_inflateGetDictionary OF((z_streamp strm, { if (!strm->reserved) return inflateGetDictionary(strm, dictionary, dictLength); - return ZWRAPD_finish_with_error_message(strm, "inflateGetDictionary is not supported!"); + return ZWRAPD_finishWithErrorMsg(strm, "inflateGetDictionary is not supported!"); } #endif @@ -790,7 +790,7 @@ ZEXTERN int ZEXPORT z_inflateCopy OF((z_streamp dest, { if (!g_useZSTD) return inflateCopy(dest, source); - return ZWRAPD_finish_with_error_message(source, "inflateCopy is not supported!"); + return ZWRAPD_finishWithErrorMsg(source, "inflateCopy is not supported!"); } @@ -799,7 +799,7 @@ ZEXTERN long ZEXPORT z_inflateMark OF((z_streamp strm)) { if (!strm->reserved) return inflateMark(strm); - return ZWRAPD_finish_with_error_message(strm, "inflateMark is not supported!"); + return ZWRAPD_finishWithErrorMsg(strm, "inflateMark is not supported!"); } #endif @@ -810,7 +810,7 @@ ZEXTERN int ZEXPORT z_inflatePrime OF((z_streamp strm, { if (!strm->reserved) return inflatePrime(strm, bits, value); - return ZWRAPD_finish_with_error_message(strm, "inflatePrime is not supported!"); + return ZWRAPD_finishWithErrorMsg(strm, "inflatePrime is not supported!"); } @@ -819,7 +819,7 @@ ZEXTERN int ZEXPORT z_inflateGetHeader OF((z_streamp strm, { if (!strm->reserved) return inflateGetHeader(strm, head); - return ZWRAPD_finish_with_error_message(strm, "inflateGetHeader is not supported!"); + return ZWRAPD_finishWithErrorMsg(strm, "inflateGetHeader is not supported!"); } @@ -830,7 +830,7 @@ ZEXTERN int ZEXPORT z_inflateBackInit_ OF((z_streamp strm, int windowBits, { if (!strm->reserved) return inflateBackInit_(strm, windowBits, window, version, stream_size); - return ZWRAPD_finish_with_error_message(strm, "inflateBackInit is not supported!"); + return ZWRAPD_finishWithErrorMsg(strm, "inflateBackInit is not supported!"); } @@ -840,7 +840,7 @@ ZEXTERN int ZEXPORT z_inflateBack OF((z_streamp strm, { if (!strm->reserved) return inflateBack(strm, in, in_desc, out, out_desc); - return ZWRAPD_finish_with_error_message(strm, "inflateBack is not supported!"); + return ZWRAPD_finishWithErrorMsg(strm, "inflateBack is not supported!"); } @@ -848,7 +848,7 @@ ZEXTERN int ZEXPORT z_inflateBackEnd OF((z_streamp strm)) { if (!strm->reserved) return inflateBackEnd(strm); - return ZWRAPD_finish_with_error_message(strm, "inflateBackEnd is not supported!"); + return ZWRAPD_finishWithErrorMsg(strm, "inflateBackEnd is not supported!"); } From dfef5ddc9e9cd693d0fcc8e63ac25b9c199ac143 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 22 Sep 2016 10:23:26 +0200 Subject: [PATCH 19/43] added zwrapbench.c --- zlibWrapper/examples/zwrapbench.c | 734 ++++++++++++++++++++++++++++++ zlibWrapper/zstd_zlibwrapper.h | 4 +- 2 files changed, 736 insertions(+), 2 deletions(-) create mode 100644 zlibWrapper/examples/zwrapbench.c diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c new file mode 100644 index 000000000..f548e9b82 --- /dev/null +++ b/zlibWrapper/examples/zwrapbench.c @@ -0,0 +1,734 @@ +/** + * Copyright (c) 2016-present, Yann Collet, Przemyslaw Skibinski, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + + +/* ************************************* +* Includes +***************************************/ +#include "util.h" /* Compiler options, UTIL_GetFileSize, UTIL_sleep */ +#include /* malloc, free */ +#include /* memset */ +#include /* fprintf, fopen, ftello64 */ +#include /* clock_t, clock, CLOCKS_PER_SEC */ + +#include "mem.h" +#define ZSTD_STATIC_LINKING_ONLY +#include "zstd.h" +#include "datagen.h" /* RDG_genBuffer */ +#include "xxhash.h" + + + +/*-************************************ +* Tuning parameters +**************************************/ +#ifndef ZSTDCLI_CLEVEL_DEFAULT +# define ZSTDCLI_CLEVEL_DEFAULT 3 +#endif + + + +/*-************************************ +* Constants +**************************************/ +#define COMPRESSOR_NAME "zlibWrapper for zstd command line interface" +#ifndef ZSTD_VERSION +# define ZSTD_VERSION "v" ZSTD_VERSION_STRING +#endif +#define AUTHOR "Yann Collet" +#define WELCOME_MESSAGE "*** %s %i-bits %s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(size_t)*8), ZSTD_VERSION, AUTHOR + +#ifndef ZSTD_GIT_COMMIT +# define ZSTD_GIT_COMMIT_STRING "" +#else +# define ZSTD_GIT_COMMIT_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_GIT_COMMIT) +#endif + +#define NBLOOPS 3 +#define TIMELOOP_MICROSEC 1*1000000ULL /* 1 second */ +#define ACTIVEPERIOD_MICROSEC 70*1000000ULL /* 70 seconds */ +#define COOLPERIOD_SEC 10 + +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31)); + +static U32 g_compressibilityDefault = 50; + + +/* ************************************* +* console display +***************************************/ +#define DEFAULT_DISPLAY_LEVEL 2 +#define DISPLAY(...) fprintf(displayOut, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } +static U32 g_displayLevel = DEFAULT_DISPLAY_LEVEL; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */ +static FILE* displayOut; + +#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ + if ((clock() - g_time > refreshRate) || (g_displayLevel>=4)) \ + { g_time = clock(); DISPLAY(__VA_ARGS__); \ + if (g_displayLevel>=4) fflush(stdout); } } +static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100; +static clock_t g_time = 0; + + +/* ************************************* +* Exceptions +***************************************/ +#ifndef DEBUG +# define DEBUG 0 +#endif +#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); +#define EXM_THROW(error, ...) \ +{ \ + DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ + DISPLAYLEVEL(1, "Error %i : ", error); \ + DISPLAYLEVEL(1, __VA_ARGS__); \ + DISPLAYLEVEL(1, "\n"); \ + exit(error); \ +} + + +/* ************************************* +* Benchmark Parameters +***************************************/ +static U32 g_nbIterations = NBLOOPS; +static size_t g_blockSize = 0; +int g_additionalParam = 0; + +void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; } + +void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; } + +void BMK_SetNbIterations(unsigned nbLoops) +{ + g_nbIterations = nbLoops; + DISPLAYLEVEL(3, "- test >= %u seconds per compression / decompression -\n", g_nbIterations); +} + +void BMK_SetBlockSize(size_t blockSize) +{ + g_blockSize = blockSize; + DISPLAYLEVEL(2, "using blocks of size %u KB \n", (U32)(blockSize>>10)); +} + + +/* ******************************************************** +* Bench functions +**********************************************************/ +typedef struct +{ + const char* srcPtr; + size_t srcSize; + char* cPtr; + size_t cRoom; + size_t cSize; + char* resPtr; + size_t resSize; +} blockParam_t; + + +#define MIN(a,b) ((a)<(b) ? (a) : (b)) +#define MAX(a,b) ((a)>(b) ? (a) : (b)) + +static int BMK_benchMem(const void* srcBuffer, size_t srcSize, + const char* displayName, int cLevel, + const size_t* fileSizes, U32 nbFiles, + const void* dictBuffer, size_t dictBufferSize) +{ + size_t const blockSize = (g_blockSize>=32 ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ; + size_t const avgSize = MIN(g_blockSize, (srcSize / nbFiles)); + U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; + blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t)); + size_t const maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */ + void* const compressedBuffer = malloc(maxCompressedSize); + void* const resultBuffer = malloc(srcSize); + ZSTD_CCtx* const ctx = ZSTD_createCCtx(); + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + U32 nbBlocks; + UTIL_time_t ticksPerSecond; + + /* checks */ + if (!compressedBuffer || !resultBuffer || !blockTable || !ctx || !dctx) + EXM_THROW(31, "allocation error : not enough memory"); + + /* init */ + if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* can only display 17 characters */ + UTIL_initTimer(&ticksPerSecond); + + /* Init blockTable data */ + { const char* srcPtr = (const char*)srcBuffer; + char* cPtr = (char*)compressedBuffer; + char* resPtr = (char*)resultBuffer; + U32 fileNb; + for (nbBlocks=0, fileNb=0; fileNb ACTIVEPERIOD_MICROSEC) { + DISPLAYLEVEL(2, "\rcooling down ... \r"); + UTIL_sleep(COOLPERIOD_SEC); + UTIL_getTime(&coolTime); + } + + /* Compression */ + DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize); + if (!cCompleted) memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */ + + UTIL_sleepMilli(1); /* give processor time to other processes */ + UTIL_waitForNextTick(ticksPerSecond); + UTIL_getTime(&clockStart); + + if (!cCompleted) { /* still some time to do compression tests */ + ZSTD_parameters const zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize); + ZSTD_customMem const cmem = { NULL, NULL, NULL }; + U32 nbLoops = 0; + ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, zparams, cmem); + if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure"); + do { + U32 blockNb; + for (blockNb=0; blockNbmaxTime; + } } + + cSize = 0; + { U32 blockNb; for (blockNb=0; blockNb%10u (%5.3f),%6.1f MB/s\r", + marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio, + (double)srcSize / fastestC ); + + (void)fastestD; (void)crcOrig; /* unused when decompression disabled */ +#if 1 + /* Decompression */ + if (!dCompleted) memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */ + + UTIL_sleepMilli(1); /* give processor time to other processes */ + UTIL_waitForNextTick(ticksPerSecond); + UTIL_getTime(&clockStart); + + if (!dCompleted) { + U32 nbLoops = 0; + ZSTD_DDict* ddict = ZSTD_createDDict(dictBuffer, dictBufferSize); + if (!ddict) EXM_THROW(2, "ZSTD_createDDict() allocation failure"); + do { + U32 blockNb; + for (blockNb=0; blockNbmaxTime; + } } + + markNb = (markNb+1) % NB_MARKS; + DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r", + marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio, + (double)srcSize / fastestC, + (double)srcSize / fastestD ); + + /* CRC Checking */ + { U64 const crcCheck = XXH64(resultBuffer, srcSize, 0); + if (crcOrig!=crcCheck) { + size_t u; + DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck); + for (u=0; u u) break; + bacc += blockTable[segNb].srcSize; + } + pos = (U32)(u - bacc); + bNb = pos / (128 KB); + DISPLAY("(block %u, sub %u, pos %u) \n", segNb, bNb, pos); + break; + } + if (u==srcSize-1) { /* should never happen */ + DISPLAY("no difference detected\n"); + } } + break; + } } /* CRC Checking */ +#endif + } /* for (testNb = 1; testNb <= (g_nbIterations + !g_nbIterations); testNb++) */ + + if (g_displayLevel == 1) { + double cSpeed = (double)srcSize / fastestC; + double dSpeed = (double)srcSize / fastestD; + if (g_additionalParam) + DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, g_additionalParam); + else + DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName); + } + DISPLAYLEVEL(2, "%2i#\n", cLevel); + } /* Bench */ + + /* clean up */ + free(blockTable); + free(compressedBuffer); + free(resultBuffer); + ZSTD_freeCCtx(ctx); + ZSTD_freeDCtx(dctx); + return 0; +} + + +static size_t BMK_findMaxMem(U64 requiredMem) +{ + size_t const step = 64 MB; + BYTE* testmem = NULL; + + requiredMem = (((requiredMem >> 26) + 1) << 26); + requiredMem += step; + if (requiredMem > maxMemory) requiredMem = maxMemory; + + do { + testmem = (BYTE*)malloc((size_t)requiredMem); + requiredMem -= step; + } while (!testmem); + + free(testmem); + return (size_t)(requiredMem); +} + +static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, + const char* displayName, int cLevel, int cLevelLast, + const size_t* fileSizes, unsigned nbFiles, + const void* dictBuffer, size_t dictBufferSize) +{ + int l; + + const char* pch = strrchr(displayName, '\\'); /* Windows */ + if (!pch) pch = strrchr(displayName, '/'); /* Linux */ + if (pch) displayName = pch+1; + + SET_HIGH_PRIORITY; + + if (g_displayLevel == 1 && !g_additionalParam) + DISPLAY("bench %s %s: input %u bytes, %u iterations, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10)); + + if (cLevelLast < cLevel) cLevelLast = cLevel; + + for (l=cLevel; l <= cLevelLast; l++) { + BMK_benchMem(srcBuffer, benchedSize, + displayName, l, + fileSizes, nbFiles, + dictBuffer, dictBufferSize); + } +} + + +/*! BMK_loadFiles() : + Loads `buffer` with content of files listed within `fileNamesTable`. + At most, fills `buffer` entirely */ +static void BMK_loadFiles(void* buffer, size_t bufferSize, + size_t* fileSizes, + const char** fileNamesTable, unsigned nbFiles) +{ + size_t pos = 0, totalSize = 0; + unsigned n; + for (n=0; n bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */ + { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f); + if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]); + pos += readSize; } + fileSizes[n] = (size_t)fileSize; + totalSize += (size_t)fileSize; + fclose(f); + } + + if (totalSize == 0) EXM_THROW(12, "no data to bench"); +} + +static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, + const char* dictFileName, int cLevel, int cLevelLast) +{ + void* srcBuffer; + size_t benchedSize; + void* dictBuffer = NULL; + size_t dictBufferSize = 0; + size_t* fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t)); + U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles); + char mfName[20] = {0}; + + if (!fileSizes) EXM_THROW(12, "not enough memory for fileSizes"); + + /* Load dictionary */ + if (dictFileName != NULL) { + U64 dictFileSize = UTIL_getFileSize(dictFileName); + if (dictFileSize > 64 MB) EXM_THROW(10, "dictionary file %s too large", dictFileName); + dictBufferSize = (size_t)dictFileSize; + dictBuffer = malloc(dictBufferSize); + if (dictBuffer==NULL) EXM_THROW(11, "not enough memory for dictionary (%u bytes)", (U32)dictBufferSize); + BMK_loadFiles(dictBuffer, dictBufferSize, fileSizes, &dictFileName, 1); + } + + /* Memory allocation & restrictions */ + benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3; + if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad; + if (benchedSize < totalSizeToLoad) + DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20)); + srcBuffer = malloc(benchedSize); + if (!srcBuffer) EXM_THROW(12, "not enough memory"); + + /* Load input buffer */ + BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles); + + /* Bench */ + snprintf (mfName, sizeof(mfName), " %u files", nbFiles); + { const char* displayName = (nbFiles > 1) ? mfName : fileNamesTable[0]; + BMK_benchCLevel(srcBuffer, benchedSize, + displayName, cLevel, cLevelLast, + fileSizes, nbFiles, + dictBuffer, dictBufferSize); + } + + /* clean up */ + free(srcBuffer); + free(dictBuffer); + free(fileSizes); +} + + +static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility) +{ + char name[20] = {0}; + size_t benchedSize = 10000000; + void* const srcBuffer = malloc(benchedSize); + + /* Memory allocation */ + if (!srcBuffer) EXM_THROW(21, "not enough memory"); + + /* Fill input buffer */ + RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0); + + /* Bench */ + snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100)); + BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0); + + /* clean up */ + free(srcBuffer); +} + + +int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, + const char* dictFileName, int cLevel, int cLevelLast) +{ + double const compressibility = (double)g_compressibilityDefault / 100; + + if (nbFiles == 0) + BMK_syntheticTest(cLevel, cLevelLast, compressibility); + else + BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast); + return 0; +} + + + + +/*-************************************ +* Command Line +**************************************/ +static int usage(const char* programName) +{ + DISPLAY(WELCOME_MESSAGE); + DISPLAY( "Usage :\n"); + DISPLAY( " %s [args] [FILE(s)] [-o file]\n", programName); + DISPLAY( "\n"); + DISPLAY( "FILE : a filename\n"); + DISPLAY( " with no FILE, or when FILE is - , read standard input\n"); + DISPLAY( "Arguments :\n"); + DISPLAY( " -D file: use `file` as Dictionary \n"); + DISPLAY( " -h/-H : display help/long help and exit\n"); + DISPLAY( " -V : display Version number and exit\n"); + DISPLAY( " -v : verbose mode; specify multiple times to increase log level (default:%d)\n", DEFAULT_DISPLAY_LEVEL); + DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n"); +#ifdef UTIL_HAS_CREATEFILELIST + DISPLAY( " -r : operate recursively on directories\n"); +#endif + DISPLAY( "\n"); + DISPLAY( "Benchmark arguments :\n"); + DISPLAY( " -b# : benchmark file(s), using # compression level (default : 1) \n"); + DISPLAY( " -e# : test all compression levels from -bX to # (default: 1)\n"); + DISPLAY( " -i# : minimum evaluation time in seconds (default : 3s)\n"); + DISPLAY( " -B# : cut file into independent blocks of size # (default: no block)\n"); + return 0; +} + +static int badusage(const char* programName) +{ + DISPLAYLEVEL(1, "Incorrect parameters\n"); + if (g_displayLevel >= 1) usage(programName); + return 1; +} + +static void waitEnter(void) +{ + int unused; + DISPLAY("Press enter to continue...\n"); + unused = getchar(); + (void)unused; +} + +/*! readU32FromChar() : + @return : unsigned integer value reach from input in `char` format + Will also modify `*stringPtr`, advancing it to position where it stopped reading. + Note : this function can overflow if digit string > MAX_UINT */ +static unsigned readU32FromChar(const char** stringPtr) +{ + unsigned result = 0; + while ((**stringPtr >='0') && (**stringPtr <='9')) + result *= 10, result += **stringPtr - '0', (*stringPtr)++ ; + return result; +} + + +#define CLEAN_RETURN(i) { operationResult = (i); goto _end; } + +int main(int argCount, char** argv) +{ + int argNb, + main_pause=0, + nextEntryIsDictionary=0, + operationResult=0, + nextArgumentIsFile=0; + int cLevel = ZSTDCLI_CLEVEL_DEFAULT; + int cLevelLast = 1; + unsigned recursive = 0; + const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */ + unsigned filenameIdx = 0; + const char* programName = argv[0]; + const char* dictFileName = NULL; + char* dynNameSpace = NULL; +#ifdef UTIL_HAS_CREATEFILELIST + const char** fileNamesTable = NULL; + char* fileNamesBuf = NULL; + unsigned fileNamesNb; +#endif + + /* init */ + if (filenameTable==NULL) { DISPLAY("zstd: %s \n", strerror(errno)); exit(1); } + displayOut = stderr; + + /* Pick out program name from path. Don't rely on stdlib because of conflicting behavior */ + { size_t pos; + for (pos = (int)strlen(programName); pos > 0; pos--) { if (programName[pos] == '/') { pos++; break; } } + programName += pos; + } + + /* command switches */ + for(argNb=1; argNb='0') && (*argument<='9')) { + BMK_setAdditionalParam(readU32FromChar(&argument)); + } else + main_pause=1; + break; + /* unknown command */ + default : CLEAN_RETURN(badusage(programName)); + } + } + continue; + } /* if (argument[0]=='-') */ + + } /* if (nextArgumentIsAFile==0) */ + + if (nextEntryIsDictionary) { + nextEntryIsDictionary = 0; + dictFileName = argument; + continue; + } + + /* add filename to list */ + filenameTable[filenameIdx++] = argument; + } + + /* Welcome message (if verbose) */ + DISPLAYLEVEL(3, WELCOME_MESSAGE); + +#ifdef UTIL_HAS_CREATEFILELIST + if (recursive) { + fileNamesTable = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, &fileNamesNb); + if (fileNamesTable) { + unsigned u; + for (u=0; u Date: Thu, 22 Sep 2016 10:23:58 +0200 Subject: [PATCH 20/43] improved zlibWrapper\Makefile --- Makefile | 2 +- zlibWrapper/Makefile | 46 ++--- zlibWrapper/examples/fitblk_original.c | 233 +++++++++++++++++++++++++ 3 files changed, 258 insertions(+), 23 deletions(-) create mode 100644 zlibWrapper/examples/fitblk_original.c diff --git a/Makefile b/Makefile index 7860ce1db..a122ffc7b 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,7 @@ zstd: zlibwrapper: $(MAKE) -C $(ZSTDDIR) all - $(MAKE) -C $(ZWRAPDIR) all + $(MAKE) -C $(ZWRAPDIR) test test_zstd test: $(MAKE) -C $(TESTDIR) $@ diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index cfdafb6d8..4cc149434 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -1,8 +1,8 @@ # Makefile for example of using zstd wrapper for zlib # -# make - compiles statically and dynamically linked examples/example.c -# make test testdll - compiles and runs statically and dynamically linked examples/example.c -# make LOC=-DZWRAP_USE_ZSTD=1 - compiles statically and dynamically linked examples/example.c with zstd compression turned on +# make - compiles statically and dynamically linked examples +# make test test_d - runs statically and dynamically linked examples +# make LOC=-DZWRAP_USE_ZSTD=1 - compiles statically and dynamically linked examples with zstd compression turned on # Paths to static and dynamic zlib and zstd libraries @@ -13,38 +13,37 @@ IMPLIB = $(ZLIBDIR)/libz.dll.a ../lib/libzstd.a else STATICLIB = -static -lz ../lib/libzstd.a IMPLIB = -lz ../lib/libzstd.a +ZLIBDIR = . endif ZLIBWRAPPER_PATH = . EXAMPLE_PATH = examples +PROGRAMS_PATH = ../programs CC ?= gcc -CFLAGS = $(LOC) -I../lib -I../lib/common -I$(ZLIBDIR) -I$(ZLIBWRAPPER_PATH) -O3 -std=gnu90 +CFLAGS = $(LOC) -I$(PROGRAMS_PATH) -I../lib -I../lib/common -I$(ZLIBWRAPPER_PATH) -I$(ZLIBDIR) -O3 -std=gnu90 CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef LDFLAGS = $(LOC) RM = rm -f -all: clean test testzstd testfitblk +all: clean fitblk example example_d zwrapbench -test: example +test: example fitblk ./example - -testdll: example_d - ./example_d - -testzstd: example_zstd - ./example_zstd - -testfitblk: fitblk ./fitblk 10240 <../zstd_compression_format.md ./fitblk 40960 <../zstd_compression_format.md +test_d: example_d + ./example_d + +test_zstd: example_zstd fitblk_zstd + ./example_zstd + ./fitblk_zstd 10240 <../zstd_compression_format.md + ./fitblk_zstd 40960 <../zstd_compression_format.md + .c.o: $(CC) $(CFLAGS) -c -o $@ $< -fitblk: $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o - $(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(STATICLIB) - example: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(STATICLIB) @@ -54,11 +53,14 @@ example_d: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o example_zstd: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(STATICLIB) -$(EXAMPLE_PATH)/fitblk.o: $(EXAMPLE_PATH)/fitblk.c - $(CC) $(CFLAGS) -I. -c -o $@ $(EXAMPLE_PATH)/fitblk.c +fitblk: $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o + $(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(STATICLIB) -$(EXAMPLE_PATH)/example.o: $(EXAMPLE_PATH)/example.c - $(CC) $(CFLAGS) -I. -c -o $@ $(EXAMPLE_PATH)/example.c +fitblk_zstd: $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o + $(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(STATICLIB) + +zwrapbench: $(EXAMPLE_PATH)/zwrapbench.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(PROGRAMS_PATH)/datagen.o + $(CC) $(LDFLAGS) -o $@ $^ $(STATICLIB) $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h $(CC) $(CFLAGS) -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c @@ -67,5 +69,5 @@ $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwra $(CC) $(CFLAGS) -DZWRAP_USE_ZSTD=1 -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c clean: - -$(RM) $(ZLIBWRAPPER_PATH)/*.o $(EXAMPLE_PATH)/*.o *.o *.exe foo.gz example example_d example_zstd + -$(RM) $(ZLIBWRAPPER_PATH)/*.o $(EXAMPLE_PATH)/*.o *.o *.exe foo.gz example example_d example_zstd fitblk fitblk_zstd @echo Cleaning completed diff --git a/zlibWrapper/examples/fitblk_original.c b/zlibWrapper/examples/fitblk_original.c new file mode 100644 index 000000000..c61de5c99 --- /dev/null +++ b/zlibWrapper/examples/fitblk_original.c @@ -0,0 +1,233 @@ +/* fitblk.c: example of fitting compressed output to a specified size + Not copyrighted -- provided to the public domain + Version 1.1 25 November 2004 Mark Adler */ + +/* Version history: + 1.0 24 Nov 2004 First version + 1.1 25 Nov 2004 Change deflateInit2() to deflateInit() + Use fixed-size, stack-allocated raw buffers + Simplify code moving compression to subroutines + Use assert() for internal errors + Add detailed description of approach + */ + +/* Approach to just fitting a requested compressed size: + + fitblk performs three compression passes on a portion of the input + data in order to determine how much of that input will compress to + nearly the requested output block size. The first pass generates + enough deflate blocks to produce output to fill the requested + output size plus a specfied excess amount (see the EXCESS define + below). The last deflate block may go quite a bit past that, but + is discarded. The second pass decompresses and recompresses just + the compressed data that fit in the requested plus excess sized + buffer. The deflate process is terminated after that amount of + input, which is less than the amount consumed on the first pass. + The last deflate block of the result will be of a comparable size + to the final product, so that the header for that deflate block and + the compression ratio for that block will be about the same as in + the final product. The third compression pass decompresses the + result of the second step, but only the compressed data up to the + requested size minus an amount to allow the compressed stream to + complete (see the MARGIN define below). That will result in a + final compressed stream whose length is less than or equal to the + requested size. Assuming sufficient input and a requested size + greater than a few hundred bytes, the shortfall will typically be + less than ten bytes. + + If the input is short enough that the first compression completes + before filling the requested output size, then that compressed + stream is return with no recompression. + + EXCESS is chosen to be just greater than the shortfall seen in a + two pass approach similar to the above. That shortfall is due to + the last deflate block compressing more efficiently with a smaller + header on the second pass. EXCESS is set to be large enough so + that there is enough uncompressed data for the second pass to fill + out the requested size, and small enough so that the final deflate + block of the second pass will be close in size to the final deflate + block of the third and final pass. MARGIN is chosen to be just + large enough to assure that the final compression has enough room + to complete in all cases. + */ + +#include +#include +#include +#include "zlib.h" + +#define local static + +/* print nastygram and leave */ +local void quit(char *why) +{ + fprintf(stderr, "fitblk abort: %s\n", why); + exit(1); +} + +#define RAWLEN 4096 /* intermediate uncompressed buffer size */ + +/* compress from file to def until provided buffer is full or end of + input reached; return last deflate() return value, or Z_ERRNO if + there was read error on the file */ +local int partcompress(FILE *in, z_streamp def) +{ + int ret, flush; + unsigned char raw[RAWLEN]; + + flush = Z_NO_FLUSH; + do { + def->avail_in = fread(raw, 1, RAWLEN, in); + if (ferror(in)) + return Z_ERRNO; + def->next_in = raw; + if (feof(in)) + flush = Z_FINISH; + ret = deflate(def, flush); + assert(ret != Z_STREAM_ERROR); + } while (def->avail_out != 0 && flush == Z_NO_FLUSH); + return ret; +} + +/* recompress from inf's input to def's output; the input for inf and + the output for def are set in those structures before calling; + return last deflate() return value, or Z_MEM_ERROR if inflate() + was not able to allocate enough memory when it needed to */ +local int recompress(z_streamp inf, z_streamp def) +{ + int ret, flush; + unsigned char raw[RAWLEN]; + + flush = Z_NO_FLUSH; + do { + /* decompress */ + inf->avail_out = RAWLEN; + inf->next_out = raw; + ret = inflate(inf, Z_NO_FLUSH); + assert(ret != Z_STREAM_ERROR && ret != Z_DATA_ERROR && + ret != Z_NEED_DICT); + if (ret == Z_MEM_ERROR) + return ret; + + /* compress what was decompresed until done or no room */ + def->avail_in = RAWLEN - inf->avail_out; + def->next_in = raw; + if (inf->avail_out != 0) + flush = Z_FINISH; + ret = deflate(def, flush); + assert(ret != Z_STREAM_ERROR); + } while (ret != Z_STREAM_END && def->avail_out != 0); + return ret; +} + +#define EXCESS 256 /* empirically determined stream overage */ +#define MARGIN 8 /* amount to back off for completion */ + +/* compress from stdin to fixed-size block on stdout */ +int main(int argc, char **argv) +{ + int ret; /* return code */ + unsigned size; /* requested fixed output block size */ + unsigned have; /* bytes written by deflate() call */ + unsigned char *blk; /* intermediate and final stream */ + unsigned char *tmp; /* close to desired size stream */ + z_stream def, inf; /* zlib deflate and inflate states */ + + /* get requested output size */ + if (argc != 2) + quit("need one argument: size of output block"); + ret = strtol(argv[1], argv + 1, 10); + if (argv[1][0] != 0) + quit("argument must be a number"); + if (ret < 8) /* 8 is minimum zlib stream size */ + quit("need positive size of 8 or greater"); + size = (unsigned)ret; + + /* allocate memory for buffers and compression engine */ + blk = malloc(size + EXCESS); + def.zalloc = Z_NULL; + def.zfree = Z_NULL; + def.opaque = Z_NULL; + ret = deflateInit(&def, Z_DEFAULT_COMPRESSION); + if (ret != Z_OK || blk == NULL) + quit("out of memory"); + + /* compress from stdin until output full, or no more input */ + def.avail_out = size + EXCESS; + def.next_out = blk; + ret = partcompress(stdin, &def); + if (ret == Z_ERRNO) + quit("error reading input"); + + /* if it all fit, then size was undersubscribed -- done! */ + if (ret == Z_STREAM_END && def.avail_out >= EXCESS) { + /* write block to stdout */ + have = size + EXCESS - def.avail_out; + if (fwrite(blk, 1, have, stdout) != have || ferror(stdout)) + quit("error writing output"); + + /* clean up and print results to stderr */ + ret = deflateEnd(&def); + assert(ret != Z_STREAM_ERROR); + free(blk); + fprintf(stderr, + "%u bytes unused out of %u requested (all input)\n", + size - have, size); + return 0; + } + + /* it didn't all fit -- set up for recompression */ + inf.zalloc = Z_NULL; + inf.zfree = Z_NULL; + inf.opaque = Z_NULL; + inf.avail_in = 0; + inf.next_in = Z_NULL; + ret = inflateInit(&inf); + tmp = malloc(size + EXCESS); + if (ret != Z_OK || tmp == NULL) + quit("out of memory"); + ret = deflateReset(&def); + assert(ret != Z_STREAM_ERROR); + + /* do first recompression close to the right amount */ + inf.avail_in = size + EXCESS; + inf.next_in = blk; + def.avail_out = size + EXCESS; + def.next_out = tmp; + ret = recompress(&inf, &def); + if (ret == Z_MEM_ERROR) + quit("out of memory"); + + /* set up for next reocmpression */ + ret = inflateReset(&inf); + assert(ret != Z_STREAM_ERROR); + ret = deflateReset(&def); + assert(ret != Z_STREAM_ERROR); + + /* do second and final recompression (third compression) */ + inf.avail_in = size - MARGIN; /* assure stream will complete */ + inf.next_in = tmp; + def.avail_out = size; + def.next_out = blk; + ret = recompress(&inf, &def); + if (ret == Z_MEM_ERROR) + quit("out of memory"); + assert(ret == Z_STREAM_END); /* otherwise MARGIN too small */ + + /* done -- write block to stdout */ + have = size - def.avail_out; + if (fwrite(blk, 1, have, stdout) != have || ferror(stdout)) + quit("error writing output"); + + /* clean up and print results to stderr */ + free(tmp); + ret = inflateEnd(&inf); + assert(ret != Z_STREAM_ERROR); + ret = deflateEnd(&def); + assert(ret != Z_STREAM_ERROR); + free(blk); + fprintf(stderr, + "%u bytes unused out of %u requested (%lu input)\n", + size - have, size, def.total_in); + return 0; +} From d755717941d4a54622631303d9f75d94efe2de60 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 22 Sep 2016 11:52:00 +0200 Subject: [PATCH 21/43] added setZWRAPdecompressionType --- zlibWrapper/Makefile | 7 +- zlibWrapper/README.md | 2 +- zlibWrapper/examples/example.c | 6 +- zlibWrapper/examples/fitblk.c | 2 +- zlibWrapper/zstd_zlibwrapper.c | 139 +++++++++++++++++++-------------- zlibWrapper/zstd_zlibwrapper.h | 27 +++++-- 6 files changed, 111 insertions(+), 72 deletions(-) diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index 4cc149434..fd2b69654 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -1,8 +1,8 @@ # Makefile for example of using zstd wrapper for zlib # # make - compiles statically and dynamically linked examples -# make test test_d - runs statically and dynamically linked examples # make LOC=-DZWRAP_USE_ZSTD=1 - compiles statically and dynamically linked examples with zstd compression turned on +# make test test_d - runs statically and dynamically linked examples # Paths to static and dynamic zlib and zstd libraries @@ -36,10 +36,11 @@ test: example fitblk test_d: example_d ./example_d -test_zstd: example_zstd fitblk_zstd +test_zstd: example_zstd fitblk_zstd zwrapbench ./example_zstd ./fitblk_zstd 10240 <../zstd_compression_format.md ./fitblk_zstd 40960 <../zstd_compression_format.md + ./zwrapbench ../zstd_compression_format.md .c.o: $(CC) $(CFLAGS) -c -o $@ $< @@ -62,6 +63,8 @@ fitblk_zstd: $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o zwrapbench: $(EXAMPLE_PATH)/zwrapbench.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(PROGRAMS_PATH)/datagen.o $(CC) $(LDFLAGS) -o $@ $^ $(STATICLIB) +$(EXAMPLE_PATH)/zwrapbench.o: $(EXAMPLE_PATH)/zwrapbench.c + $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h $(CC) $(CFLAGS) -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c diff --git a/zlibWrapper/README.md b/zlibWrapper/README.md index c2637fe6f..90ee47e60 100644 --- a/zlibWrapper/README.md +++ b/zlibWrapper/README.md @@ -34,7 +34,7 @@ The linking should be changed to: After embedding the zstd wrapper within your project the zstd library is turned off by default. Your project should work as before with zlib. There are two options to enable zstd compression: - compilation with ```-DZWRAP_USE_ZSTD=1``` (or using ```#define ZWRAP_USE_ZSTD 1``` before ```#include "zstd_zlibwrapper.h"```) -- using the ```void useZSTD(int turn_on)``` function (declared in ```#include "zstd_zlibwrapper.h"```) +- using the ```void useZSTDcompression(int turn_on)``` function (declared in ```#include "zstd_zlibwrapper.h"```) There is no switch for zstd decompression because zlib and zstd streams are automatically detected and decompressed using a proper library. diff --git a/zlibWrapper/examples/example.c b/zlibWrapper/examples/example.c index c1f3b46b3..735aa6bb3 100644 --- a/zlibWrapper/examples/example.c +++ b/zlibWrapper/examples/example.c @@ -583,7 +583,7 @@ int main(argc, argv) printf("zlib version %s = 0x%04x, compile flags = 0x%lx\n", ZLIB_VERSION, ZLIB_VERNUM, zlibCompileFlags()); - if (isUsingZSTD()) printf("zstd version %s\n", zstdVersion()); + if (isUsingZSTDcompression()) printf("zstd version %s\n", zstdVersion()); compr = (Byte*)calloc((uInt)comprLen, 1); uncompr = (Byte*)calloc((uInt)uncomprLen, 1); @@ -600,7 +600,7 @@ int main(argc, argv) #else test_compress(compr, comprLen, uncompr, uncomprLen); - if (!isUsingZSTD()) + if (!isUsingZSTDcompression()) test_gzio((argc > 1 ? argv[1] : TESTFILE), uncompr, uncomprLen); #endif @@ -611,7 +611,7 @@ int main(argc, argv) test_large_deflate(compr, comprLen, uncompr, uncomprLen); test_large_inflate(compr, comprLen, uncompr, uncomprLen); - if (!isUsingZSTD()) { + if (!isUsingZSTDcompression()) { test_flush(compr, &comprLen); test_sync(compr, comprLen, uncompr, uncomprLen); } diff --git a/zlibWrapper/examples/fitblk.c b/zlibWrapper/examples/fitblk.c index 17b422668..4e5a38315 100644 --- a/zlibWrapper/examples/fitblk.c +++ b/zlibWrapper/examples/fitblk.c @@ -152,7 +152,7 @@ int main(int argc, char **argv) size = (unsigned)ret; printf("zlib version %s\n", ZLIB_VERSION); - if (isUsingZSTD()) printf("zstd version %s\n", zstdVersion()); + if (isUsingZSTDcompression()) printf("zstd version %s\n", zstdVersion()); /* allocate memory for buffers and compression engine */ blk = malloc(size + EXCESS); diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index c0bae799a..3160b2567 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -36,21 +36,30 @@ return NULL; \ } +const char * zstdVersion(void) { return ZSTD_VERSION_STRING; } + +ZEXTERN const char * ZEXPORT z_zlibVersion OF((void)) { return zlibVersion(); } + + + #ifndef ZWRAP_USE_ZSTD #define ZWRAP_USE_ZSTD 0 #endif -static int g_useZSTD = ZWRAP_USE_ZSTD; /* 0 = don't use ZSTD */ +static int g_useZSTDcompression = ZWRAP_USE_ZSTD; /* 0 = don't use ZSTD */ + +void useZSTDcompression(int turn_on) { g_useZSTDcompression = turn_on; } + +int isUsingZSTDcompression(void) { return g_useZSTDcompression; } -void useZSTD(int turn_on) { g_useZSTD = turn_on; } +static ZWRAP_decompress_type g_ZWRAPdecompressionType = ZWRAP_AUTO; -int isUsingZSTD(void) { return g_useZSTD; } +void setZWRAPdecompressionType(ZWRAP_decompress_type type) { g_ZWRAPdecompressionType = type; }; -const char * zstdVersion(void) { return ZSTD_VERSION_STRING; } +ZWRAP_decompress_type getZWRAPdecompressionType(void) { return g_ZWRAPdecompressionType; } -ZEXTERN const char * ZEXPORT z_zlibVersion OF((void)) { return zlibVersion(); } @@ -170,7 +179,7 @@ ZEXTERN int ZEXPORT z_deflateInit_ OF((z_streamp strm, int level, ZWRAP_CCtx* zwc; LOG_WRAPPERC("- deflateInit level=%d\n", level); - if (!g_useZSTD) { + if (!g_useZSTDcompression) { return deflateInit_((strm), (level), version, stream_size); } @@ -193,7 +202,7 @@ ZEXTERN int ZEXPORT z_deflateInit2_ OF((z_streamp strm, int level, int method, int strategy, const char *version, int stream_size)) { - if (!g_useZSTD) + if (!g_useZSTDcompression) return deflateInit2_(strm, level, method, windowBits, memLevel, strategy, version, stream_size); return z_deflateInit_ (strm, level, version, stream_size); @@ -203,7 +212,7 @@ ZEXTERN int ZEXPORT z_deflateInit2_ OF((z_streamp strm, int level, int method, ZEXTERN int ZEXPORT z_deflateReset OF((z_streamp strm)) { LOG_WRAPPERC("- deflateReset\n"); - if (!g_useZSTD) + if (!g_useZSTDcompression) return deflateReset(strm); { ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state; @@ -226,7 +235,7 @@ ZEXTERN int ZEXPORT z_deflateSetDictionary OF((z_streamp strm, const Bytef *dictionary, uInt dictLength)) { - if (!g_useZSTD) { + if (!g_useZSTDcompression) { LOG_WRAPPERC("- deflateSetDictionary\n"); return deflateSetDictionary(strm, dictionary, dictLength); } @@ -250,7 +259,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) { ZWRAP_CCtx* zwc; - if (!g_useZSTD) { + if (!g_useZSTDcompression) { int res; LOG_WRAPPERC("- deflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); res = deflate(strm, flush); @@ -321,7 +330,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) ZEXTERN int ZEXPORT z_deflateEnd OF((z_streamp strm)) { - if (!g_useZSTD) { + if (!g_useZSTDcompression) { LOG_WRAPPERC("- deflateEnd\n"); return deflateEnd(strm); } @@ -340,7 +349,7 @@ ZEXTERN int ZEXPORT z_deflateEnd OF((z_streamp strm)) ZEXTERN uLong ZEXPORT z_deflateBound OF((z_streamp strm, uLong sourceLen)) { - if (!g_useZSTD) + if (!g_useZSTDcompression) return deflateBound(strm, sourceLen); return ZSTD_compressBound(sourceLen); @@ -351,7 +360,7 @@ ZEXTERN int ZEXPORT z_deflateParams OF((z_streamp strm, int level, int strategy)) { - if (!g_useZSTD) { + if (!g_useZSTDcompression) { LOG_WRAPPERC("- deflateParams level=%d strategy=%d\n", level, strategy); return deflateParams(strm, level, strategy); } @@ -445,6 +454,11 @@ int ZWRAPD_finishWithErrorMsg(z_streamp strm, char* message) ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm, const char *version, int stream_size)) { + if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB) { + return inflateInit(strm); + } + + { ZWRAP_DCtx* zwd = ZWRAP_createDCtx(strm); LOG_WRAPPERD("- inflateInit\n"); if (zwd == NULL) return ZWRAPD_finishWithError(zwd, strm, 0); @@ -458,6 +472,7 @@ ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm, strm->total_in = 0; strm->total_out = 0; strm->reserved = 1; /* mark as unknown steam */ + } return Z_OK; } @@ -466,6 +481,11 @@ ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm, ZEXTERN int ZEXPORT z_inflateInit2_ OF((z_streamp strm, int windowBits, const char *version, int stream_size)) { + if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB) { + return inflateInit2_(strm, windowBits, version, stream_size); + } + + { int ret = z_inflateInit_ (strm, version, stream_size); if (ret == Z_OK) { ZWRAP_DCtx* zwd = (ZWRAP_DCtx*)strm->state; @@ -473,13 +493,14 @@ ZEXTERN int ZEXPORT z_inflateInit2_ OF((z_streamp strm, int windowBits, zwd->windowBits = windowBits; } return ret; + } } ZEXTERN int ZEXPORT z_inflateReset OF((z_streamp strm)) { LOG_WRAPPERD("- inflateReset\n"); - if (!strm->reserved) + if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved) return inflateReset(strm); { ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state; @@ -499,7 +520,7 @@ ZEXTERN int ZEXPORT z_inflateReset OF((z_streamp strm)) ZEXTERN int ZEXPORT z_inflateReset2 OF((z_streamp strm, int windowBits)) { - if (!strm->reserved) + if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved) return inflateReset2(strm, windowBits); { int ret = z_inflateReset (strm); @@ -519,7 +540,7 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm, uInt dictLength)) { LOG_WRAPPERD("- inflateSetDictionary\n"); - if (!strm->reserved) + if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved) return inflateSetDictionary(strm, dictionary, dictLength); { size_t errorCode; @@ -551,7 +572,7 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm, ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) { int res; - if (!strm->reserved) { + if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved) { LOG_WRAPPERD("- inflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); res = inflate(strm, flush); LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, res); @@ -690,7 +711,7 @@ finish: ZEXTERN int ZEXPORT z_inflateEnd OF((z_streamp strm)) { - if (!strm->reserved) + if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved) return inflateEnd(strm); LOG_WRAPPERD("- inflateEnd total_in=%d total_out=%d\n", (int)(strm->total_in), (int)(strm->total_out)); @@ -707,7 +728,7 @@ ZEXTERN int ZEXPORT z_inflateEnd OF((z_streamp strm)) ZEXTERN int ZEXPORT z_inflateSync OF((z_streamp strm)) { - if (!strm->reserved) { + if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved) { return inflateSync(strm); } @@ -721,7 +742,7 @@ ZEXTERN int ZEXPORT z_inflateSync OF((z_streamp strm)) ZEXTERN int ZEXPORT z_deflateCopy OF((z_streamp dest, z_streamp source)) { - if (!g_useZSTD) + if (!g_useZSTDcompression) return deflateCopy(dest, source); return ZWRAPC_finishWithErrorMsg(source, "deflateCopy is not supported!"); } @@ -733,7 +754,7 @@ ZEXTERN int ZEXPORT z_deflateTune OF((z_streamp strm, int nice_length, int max_chain)) { - if (!g_useZSTD) + if (!g_useZSTDcompression) return deflateTune(strm, good_length, max_lazy, nice_length, max_chain); return ZWRAPC_finishWithErrorMsg(strm, "deflateTune is not supported!"); } @@ -744,7 +765,7 @@ ZEXTERN int ZEXPORT z_deflatePending OF((z_streamp strm, unsigned *pending, int *bits)) { - if (!g_useZSTD) + if (!g_useZSTDcompression) return deflatePending(strm, pending, bits); return ZWRAPC_finishWithErrorMsg(strm, "deflatePending is not supported!"); } @@ -755,7 +776,7 @@ ZEXTERN int ZEXPORT z_deflatePrime OF((z_streamp strm, int bits, int value)) { - if (!g_useZSTD) + if (!g_useZSTDcompression) return deflatePrime(strm, bits, value); return ZWRAPC_finishWithErrorMsg(strm, "deflatePrime is not supported!"); } @@ -764,7 +785,7 @@ ZEXTERN int ZEXPORT z_deflatePrime OF((z_streamp strm, ZEXTERN int ZEXPORT z_deflateSetHeader OF((z_streamp strm, gz_headerp head)) { - if (!g_useZSTD) + if (!g_useZSTDcompression) return deflateSetHeader(strm, head); return ZWRAPC_finishWithErrorMsg(strm, "deflateSetHeader is not supported!"); } @@ -778,7 +799,7 @@ ZEXTERN int ZEXPORT z_inflateGetDictionary OF((z_streamp strm, Bytef *dictionary, uInt *dictLength)) { - if (!strm->reserved) + if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved) return inflateGetDictionary(strm, dictionary, dictLength); return ZWRAPD_finishWithErrorMsg(strm, "inflateGetDictionary is not supported!"); } @@ -788,7 +809,7 @@ ZEXTERN int ZEXPORT z_inflateGetDictionary OF((z_streamp strm, ZEXTERN int ZEXPORT z_inflateCopy OF((z_streamp dest, z_streamp source)) { - if (!g_useZSTD) + if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !source->reserved) return inflateCopy(dest, source); return ZWRAPD_finishWithErrorMsg(source, "inflateCopy is not supported!"); } @@ -797,7 +818,7 @@ ZEXTERN int ZEXPORT z_inflateCopy OF((z_streamp dest, #if ZLIB_VERNUM >= 0x1240 ZEXTERN long ZEXPORT z_inflateMark OF((z_streamp strm)) { - if (!strm->reserved) + if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved) return inflateMark(strm); return ZWRAPD_finishWithErrorMsg(strm, "inflateMark is not supported!"); } @@ -808,7 +829,7 @@ ZEXTERN int ZEXPORT z_inflatePrime OF((z_streamp strm, int bits, int value)) { - if (!strm->reserved) + if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved) return inflatePrime(strm, bits, value); return ZWRAPD_finishWithErrorMsg(strm, "inflatePrime is not supported!"); } @@ -817,7 +838,7 @@ ZEXTERN int ZEXPORT z_inflatePrime OF((z_streamp strm, ZEXTERN int ZEXPORT z_inflateGetHeader OF((z_streamp strm, gz_headerp head)) { - if (!strm->reserved) + if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved) return inflateGetHeader(strm, head); return ZWRAPD_finishWithErrorMsg(strm, "inflateGetHeader is not supported!"); } @@ -828,7 +849,7 @@ ZEXTERN int ZEXPORT z_inflateBackInit_ OF((z_streamp strm, int windowBits, const char *version, int stream_size)) { - if (!strm->reserved) + if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved) return inflateBackInit_(strm, windowBits, window, version, stream_size); return ZWRAPD_finishWithErrorMsg(strm, "inflateBackInit is not supported!"); } @@ -838,7 +859,7 @@ ZEXTERN int ZEXPORT z_inflateBack OF((z_streamp strm, in_func in, void FAR *in_desc, out_func out, void FAR *out_desc)) { - if (!strm->reserved) + if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved) return inflateBack(strm, in, in_desc, out, out_desc); return ZWRAPD_finishWithErrorMsg(strm, "inflateBack is not supported!"); } @@ -846,7 +867,7 @@ ZEXTERN int ZEXPORT z_inflateBack OF((z_streamp strm, ZEXTERN int ZEXPORT z_inflateBackEnd OF((z_streamp strm)) { - if (!strm->reserved) + if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved) return inflateBackEnd(strm); return ZWRAPD_finishWithErrorMsg(strm, "inflateBackEnd is not supported!"); } @@ -862,7 +883,7 @@ ZEXTERN uLong ZEXPORT z_zlibCompileFlags OF((void)) { return zlibCompileFlags(); ZEXTERN int ZEXPORT z_compress OF((Bytef *dest, uLongf *destLen, const Bytef *source, uLong sourceLen)) { - if (!g_useZSTD) + if (!g_useZSTDcompression) return compress(dest, destLen, source, sourceLen); { size_t dstCapacity = *destLen; @@ -879,7 +900,7 @@ ZEXTERN int ZEXPORT z_compress2 OF((Bytef *dest, uLongf *destLen, const Bytef *source, uLong sourceLen, int level)) { - if (!g_useZSTD) + if (!g_useZSTDcompression) return compress2(dest, destLen, source, sourceLen, level); { size_t dstCapacity = *destLen; @@ -893,7 +914,7 @@ ZEXTERN int ZEXPORT z_compress2 OF((Bytef *dest, uLongf *destLen, ZEXTERN uLong ZEXPORT z_compressBound OF((uLong sourceLen)) { - if (!g_useZSTD) + if (!g_useZSTDcompression) return compressBound(sourceLen); return ZSTD_compressBound(sourceLen); @@ -919,7 +940,7 @@ ZEXTERN int ZEXPORT z_uncompress OF((Bytef *dest, uLongf *destLen, /* gzip file access functions */ ZEXTERN gzFile ZEXPORT z_gzopen OF((const char *path, const char *mode)) { - if (!g_useZSTD) + if (!g_useZSTDcompression) return gzopen(path, mode); FINISH_WITH_NULL_ERR("gzopen is not supported!"); } @@ -927,7 +948,7 @@ ZEXTERN gzFile ZEXPORT z_gzopen OF((const char *path, const char *mode)) ZEXTERN gzFile ZEXPORT z_gzdopen OF((int fd, const char *mode)) { - if (!g_useZSTD) + if (!g_useZSTDcompression) return gzdopen(fd, mode); FINISH_WITH_NULL_ERR("gzdopen is not supported!"); } @@ -936,7 +957,7 @@ ZEXTERN gzFile ZEXPORT z_gzdopen OF((int fd, const char *mode)) #if ZLIB_VERNUM >= 0x1240 ZEXTERN int ZEXPORT z_gzbuffer OF((gzFile file, unsigned size)) { - if (!g_useZSTD) + if (!g_useZSTDcompression) return gzbuffer(file, size); FINISH_WITH_GZ_ERR("gzbuffer is not supported!"); } @@ -944,7 +965,7 @@ ZEXTERN int ZEXPORT z_gzbuffer OF((gzFile file, unsigned size)) ZEXTERN z_off_t ZEXPORT z_gzoffset OF((gzFile file)) { - if (!g_useZSTD) + if (!g_useZSTDcompression) return gzoffset(file); FINISH_WITH_GZ_ERR("gzoffset is not supported!"); } @@ -952,7 +973,7 @@ ZEXTERN z_off_t ZEXPORT z_gzoffset OF((gzFile file)) ZEXTERN int ZEXPORT z_gzclose_r OF((gzFile file)) { - if (!g_useZSTD) + if (!g_useZSTDcompression) return gzclose_r(file); FINISH_WITH_GZ_ERR("gzclose_r is not supported!"); } @@ -960,7 +981,7 @@ ZEXTERN int ZEXPORT z_gzclose_r OF((gzFile file)) ZEXTERN int ZEXPORT z_gzclose_w OF((gzFile file)) { - if (!g_useZSTD) + if (!g_useZSTDcompression) return gzclose_w(file); FINISH_WITH_GZ_ERR("gzclose_w is not supported!"); } @@ -969,7 +990,7 @@ ZEXTERN int ZEXPORT z_gzclose_w OF((gzFile file)) ZEXTERN int ZEXPORT z_gzsetparams OF((gzFile file, int level, int strategy)) { - if (!g_useZSTD) + if (!g_useZSTDcompression) return gzsetparams(file, level, strategy); FINISH_WITH_GZ_ERR("gzsetparams is not supported!"); } @@ -977,7 +998,7 @@ ZEXTERN int ZEXPORT z_gzsetparams OF((gzFile file, int level, int strategy)) ZEXTERN int ZEXPORT z_gzread OF((gzFile file, voidp buf, unsigned len)) { - if (!g_useZSTD) + if (!g_useZSTDcompression) return gzread(file, buf, len); FINISH_WITH_GZ_ERR("gzread is not supported!"); } @@ -986,7 +1007,7 @@ ZEXTERN int ZEXPORT z_gzread OF((gzFile file, voidp buf, unsigned len)) ZEXTERN int ZEXPORT z_gzwrite OF((gzFile file, voidpc buf, unsigned len)) { - if (!g_useZSTD) + if (!g_useZSTDcompression) return gzwrite(file, buf, len); FINISH_WITH_GZ_ERR("gzwrite is not supported!"); } @@ -998,7 +1019,7 @@ ZEXTERN int ZEXPORTVA z_gzprintf Z_ARG((gzFile file, const char *format, ...)) ZEXTERN int ZEXPORTVA z_gzprintf OF((gzFile file, const char *format, ...)) #endif { - if (!g_useZSTD) { + if (!g_useZSTDcompression) { int ret; char buf[1024]; va_list args; @@ -1015,7 +1036,7 @@ ZEXTERN int ZEXPORTVA z_gzprintf OF((gzFile file, const char *format, ...)) ZEXTERN int ZEXPORT z_gzputs OF((gzFile file, const char *s)) { - if (!g_useZSTD) + if (!g_useZSTDcompression) return gzputs(file, s); FINISH_WITH_GZ_ERR("gzputs is not supported!"); } @@ -1023,7 +1044,7 @@ ZEXTERN int ZEXPORT z_gzputs OF((gzFile file, const char *s)) ZEXTERN char * ZEXPORT z_gzgets OF((gzFile file, char *buf, int len)) { - if (!g_useZSTD) + if (!g_useZSTDcompression) return gzgets(file, buf, len); FINISH_WITH_NULL_ERR("gzgets is not supported!"); } @@ -1031,7 +1052,7 @@ ZEXTERN char * ZEXPORT z_gzgets OF((gzFile file, char *buf, int len)) ZEXTERN int ZEXPORT z_gzputc OF((gzFile file, int c)) { - if (!g_useZSTD) + if (!g_useZSTDcompression) return gzputc(file, c); FINISH_WITH_GZ_ERR("gzputc is not supported!"); } @@ -1043,7 +1064,7 @@ ZEXTERN int ZEXPORT z_gzgetc_ OF((gzFile file)) ZEXTERN int ZEXPORT z_gzgetc OF((gzFile file)) #endif { - if (!g_useZSTD) + if (!g_useZSTDcompression) return gzgetc(file); FINISH_WITH_GZ_ERR("gzgetc is not supported!"); } @@ -1051,7 +1072,7 @@ ZEXTERN int ZEXPORT z_gzgetc OF((gzFile file)) ZEXTERN int ZEXPORT z_gzungetc OF((int c, gzFile file)) { - if (!g_useZSTD) + if (!g_useZSTDcompression) return gzungetc(c, file); FINISH_WITH_GZ_ERR("gzungetc is not supported!"); } @@ -1059,7 +1080,7 @@ ZEXTERN int ZEXPORT z_gzungetc OF((int c, gzFile file)) ZEXTERN int ZEXPORT z_gzflush OF((gzFile file, int flush)) { - if (!g_useZSTD) + if (!g_useZSTDcompression) return gzflush(file, flush); FINISH_WITH_GZ_ERR("gzflush is not supported!"); } @@ -1067,7 +1088,7 @@ ZEXTERN int ZEXPORT z_gzflush OF((gzFile file, int flush)) ZEXTERN z_off_t ZEXPORT z_gzseek OF((gzFile file, z_off_t offset, int whence)) { - if (!g_useZSTD) + if (!g_useZSTDcompression) return gzseek(file, offset, whence); FINISH_WITH_GZ_ERR("gzseek is not supported!"); } @@ -1075,7 +1096,7 @@ ZEXTERN z_off_t ZEXPORT z_gzseek OF((gzFile file, z_off_t offset, int whence)) ZEXTERN int ZEXPORT z_gzrewind OF((gzFile file)) { - if (!g_useZSTD) + if (!g_useZSTDcompression) return gzrewind(file); FINISH_WITH_GZ_ERR("gzrewind is not supported!"); } @@ -1083,7 +1104,7 @@ ZEXTERN int ZEXPORT z_gzrewind OF((gzFile file)) ZEXTERN z_off_t ZEXPORT z_gztell OF((gzFile file)) { - if (!g_useZSTD) + if (!g_useZSTDcompression) return gztell(file); FINISH_WITH_GZ_ERR("gztell is not supported!"); } @@ -1091,7 +1112,7 @@ ZEXTERN z_off_t ZEXPORT z_gztell OF((gzFile file)) ZEXTERN int ZEXPORT z_gzeof OF((gzFile file)) { - if (!g_useZSTD) + if (!g_useZSTDcompression) return gzeof(file); FINISH_WITH_GZ_ERR("gzeof is not supported!"); } @@ -1099,7 +1120,7 @@ ZEXTERN int ZEXPORT z_gzeof OF((gzFile file)) ZEXTERN int ZEXPORT z_gzdirect OF((gzFile file)) { - if (!g_useZSTD) + if (!g_useZSTDcompression) return gzdirect(file); FINISH_WITH_GZ_ERR("gzdirect is not supported!"); } @@ -1107,7 +1128,7 @@ ZEXTERN int ZEXPORT z_gzdirect OF((gzFile file)) ZEXTERN int ZEXPORT z_gzclose OF((gzFile file)) { - if (!g_useZSTD) + if (!g_useZSTDcompression) return gzclose(file); FINISH_WITH_GZ_ERR("gzclose is not supported!"); } @@ -1115,7 +1136,7 @@ ZEXTERN int ZEXPORT z_gzclose OF((gzFile file)) ZEXTERN const char * ZEXPORT z_gzerror OF((gzFile file, int *errnum)) { - if (!g_useZSTD) + if (!g_useZSTDcompression) return gzerror(file, errnum); FINISH_WITH_NULL_ERR("gzerror is not supported!"); } @@ -1123,7 +1144,7 @@ ZEXTERN const char * ZEXPORT z_gzerror OF((gzFile file, int *errnum)) ZEXTERN void ZEXPORT z_gzclearerr OF((gzFile file)) { - if (!g_useZSTD) + if (!g_useZSTDcompression) gzclearerr(file); } diff --git a/zlibWrapper/zstd_zlibwrapper.h b/zlibWrapper/zstd_zlibwrapper.h index db7fc3e24..e8114c4b8 100644 --- a/zlibWrapper/zstd_zlibwrapper.h +++ b/zlibWrapper/zstd_zlibwrapper.h @@ -26,21 +26,36 @@ extern "C" { #endif #endif -/* enables/disables zstd compression during runtime */ -void useZSTD(int turn_on); - -/* check if zstd compression is turned on */ -int isUsingZSTD(void); - /* returns a string with version of zstd library */ const char * zstdVersion(void); + +/* COMPRESSION */ +/* enables/disables zstd compression during runtime */ +void useZSTDcompression(int turn_on); + +/* check if zstd compression is turned on */ +int isUsingZSTDcompression(void); + /* Changes a pledged source size for a given compression stream. It will change ZSTD compression parameters what may improve compression speed and/or ratio. The function should be called just after deflateInit(). */ int ZSTD_setPledgedSrcSize(z_streamp strm, unsigned long long pledgedSrcSize); +/* DECOMPRESSION */ +typedef enum { ZWRAP_FORCE_ZLIB, ZWRAP_FORCE_ZSTD, ZWRAP_AUTO } ZWRAP_decompress_type; + +/* enables/disables automatic recognition of zstd/zlib compressed data during runtime */ +void setZWRAPdecompressionType(ZWRAP_decompress_type type); + +/* check zstd decompression type */ +ZWRAP_decompress_type getZWRAPdecompressionType(void); + + + + + #if defined (__cplusplus) } #endif From 54320ce9051fee64657dfb98ae37f9c03fc1a878 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 22 Sep 2016 11:52:53 +0200 Subject: [PATCH 22/43] zwrapbench tests zlib --- zlibWrapper/examples/zwrapbench.c | 147 ++++++++++++++++++++++-------- 1 file changed, 108 insertions(+), 39 deletions(-) diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c index f548e9b82..c41c2ea5d 100644 --- a/zlibWrapper/examples/zwrapbench.c +++ b/zlibWrapper/examples/zwrapbench.c @@ -23,6 +23,8 @@ #include "datagen.h" /* RDG_genBuffer */ #include "xxhash.h" +#include "zlib.h" + /*-************************************ @@ -37,7 +39,7 @@ /*-************************************ * Constants **************************************/ -#define COMPRESSOR_NAME "zlibWrapper for zstd command line interface" +#define COMPRESSOR_NAME "Zstandard wrapper for zlib command line interface" #ifndef ZSTD_VERSION # define ZSTD_VERSION "v" ZSTD_VERSION_STRING #endif @@ -136,6 +138,8 @@ typedef struct size_t resSize; } blockParam_t; +typedef enum { BMK_ZSTD, BMK_ZLIB } BMK_compressor; + #define MIN(a,b) ((a)<(b) ? (a) : (b)) #define MAX(a,b) ((a)>(b) ? (a) : (b)) @@ -143,7 +147,7 @@ typedef struct static int BMK_benchMem(const void* srcBuffer, size_t srcSize, const char* displayName, int cLevel, const size_t* fileSizes, U32 nbFiles, - const void* dictBuffer, size_t dictBufferSize) + const void* dictBuffer, size_t dictBufferSize, BMK_compressor compressor) { size_t const blockSize = (g_blockSize>=32 ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ; size_t const avgSize = MIN(g_blockSize, (srcSize / nbFiles)); @@ -225,24 +229,53 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, UTIL_getTime(&clockStart); if (!cCompleted) { /* still some time to do compression tests */ - ZSTD_parameters const zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize); - ZSTD_customMem const cmem = { NULL, NULL, NULL }; U32 nbLoops = 0; - ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, zparams, cmem); - if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure"); - do { - U32 blockNb; - for (blockNb=0; blockNb Date: Thu, 22 Sep 2016 14:42:32 +0200 Subject: [PATCH 23/43] zwrapbench benchmarks zlibWrapper --- zlibWrapper/Makefile | 26 +++++----- zlibWrapper/examples/zwrapbench.c | 80 +++++++++++++++++++++++++++---- zlibWrapper/zstd_zlibwrapper.c | 2 +- 3 files changed, 87 insertions(+), 21 deletions(-) diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index fd2b69654..5329e7e2c 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -8,19 +8,20 @@ # Paths to static and dynamic zlib and zstd libraries # Use "make ZLIBDIR=path/to/zlib" to select a path to library ifdef ZLIBDIR -STATICLIB = $(ZLIBDIR)/libz.a ../lib/libzstd.a -IMPLIB = $(ZLIBDIR)/libz.dll.a ../lib/libzstd.a +STATICLIB = $(ZLIBDIR)/libz.a $(ZSTDLIBDIR)/libzstd.a +IMPLIB = $(ZLIBDIR)/libz.dll.a $(ZSTDLIBDIR)/libzstd.a else -STATICLIB = -static -lz ../lib/libzstd.a -IMPLIB = -lz ../lib/libzstd.a +STATICLIB = -static -lz $(ZSTDLIBDIR)/libzstd.a +IMPLIB = -lz $(ZSTDLIBDIR)/libzstd.a ZLIBDIR = . endif +ZSTDLIBDIR = ../lib ZLIBWRAPPER_PATH = . EXAMPLE_PATH = examples PROGRAMS_PATH = ../programs CC ?= gcc -CFLAGS = $(LOC) -I$(PROGRAMS_PATH) -I../lib -I../lib/common -I$(ZLIBWRAPPER_PATH) -I$(ZLIBDIR) -O3 -std=gnu90 +CFLAGS = $(LOC) -I$(PROGRAMS_PATH) -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH) -I$(ZLIBDIR) -O3 -std=gnu90 CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef LDFLAGS = $(LOC) RM = rm -f @@ -45,22 +46,22 @@ test_zstd: example_zstd fitblk_zstd zwrapbench .c.o: $(CC) $(CFLAGS) -c -o $@ $< -example: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o +example: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(ZSTDLIBDIR)/libzstd.a $(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(STATICLIB) example_d: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(IMPLIB) -example_zstd: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o +example_zstd: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(ZSTDLIBDIR)/libzstd.a $(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(STATICLIB) -fitblk: $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o +fitblk: $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(ZSTDLIBDIR)/libzstd.a $(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(STATICLIB) -fitblk_zstd: $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o +fitblk_zstd: $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(ZSTDLIBDIR)/libzstd.a $(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(STATICLIB) -zwrapbench: $(EXAMPLE_PATH)/zwrapbench.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(PROGRAMS_PATH)/datagen.o +zwrapbench: $(EXAMPLE_PATH)/zwrapbench.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(PROGRAMS_PATH)/datagen.o $(ZSTDLIBDIR)/libzstd.a $(CC) $(LDFLAGS) -o $@ $^ $(STATICLIB) $(EXAMPLE_PATH)/zwrapbench.o: $(EXAMPLE_PATH)/zwrapbench.c @@ -71,6 +72,9 @@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $ $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h $(CC) $(CFLAGS) -DZWRAP_USE_ZSTD=1 -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c +$(ZSTDLIBDIR)/libzstd.a: + $(MAKE) -C $(ZSTDLIBDIR) all + clean: - -$(RM) $(ZLIBWRAPPER_PATH)/*.o $(EXAMPLE_PATH)/*.o *.o *.exe foo.gz example example_d example_zstd fitblk fitblk_zstd + -$(RM) $(ZLIBWRAPPER_PATH)/*.o $(EXAMPLE_PATH)/*.o *.o *.exe foo.gz example example_d example_zstd fitblk fitblk_zstd zwrapbench @echo Cleaning completed diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c index c41c2ea5d..3f4fe05b0 100644 --- a/zlibWrapper/examples/zwrapbench.c +++ b/zlibWrapper/examples/zwrapbench.c @@ -23,7 +23,7 @@ #include "datagen.h" /* RDG_genBuffer */ #include "xxhash.h" -#include "zlib.h" +#include "zstd_zlibwrapper.h" @@ -138,7 +138,7 @@ typedef struct size_t resSize; } blockParam_t; -typedef enum { BMK_ZSTD, BMK_ZLIB } BMK_compressor; +typedef enum { BMK_ZSTD, BMK_ZSTD2, BMK_ZLIB, BMK_ZWRAP_ZLIB, BMK_ZWRAP_ZSTD } BMK_compressor; #define MIN(a,b) ((a)<(b) ? (a) : (b)) @@ -235,6 +235,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, ZSTD_customMem const cmem = { NULL, NULL, NULL }; ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, zparams, cmem); if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure"); + do { U32 blockNb; for (blockNb=0; blockNb Z_BEST_COMPRESSION) cLevelLast = Z_BEST_COMPRESSION; + DISPLAY("benchmarking zlib %s\n", ZLIB_VERSION); for (l=cLevel; l <= cLevelLast; l++) { BMK_benchMem(srcBuffer, benchedSize, @@ -452,12 +514,12 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, dictBuffer, dictBufferSize, BMK_ZLIB); } - DISPLAY("benchmarking zstd %s\n", ZSTD_VERSION_STRING); + DISPLAY("benchmarking zlibWrapper with zlib %s\n", ZLIB_VERSION); for (l=cLevel; l <= cLevelLast; l++) { BMK_benchMem(srcBuffer, benchedSize, displayName, l, fileSizes, nbFiles, - dictBuffer, dictBufferSize, BMK_ZSTD); + dictBuffer, dictBufferSize, BMK_ZWRAP_ZLIB); } } @@ -602,8 +664,8 @@ static int usage(const char* programName) #endif DISPLAY( "\n"); DISPLAY( "Benchmark arguments :\n"); - DISPLAY( " -b# : benchmark file(s), using # compression level (default : 1) \n"); - DISPLAY( " -e# : test all compression levels from -bX to # (default: 1)\n"); + DISPLAY( " -b# : benchmark file(s), using # compression level (default : %d) \n", ZSTDCLI_CLEVEL_DEFAULT); + DISPLAY( " -e# : test all compression levels from -bX to # (default: %d)\n", ZSTDCLI_CLEVEL_DEFAULT); DISPLAY( " -i# : minimum evaluation time in seconds (default : 3s)\n"); DISPLAY( " -B# : cut file into independent blocks of size # (default: no block)\n"); return 0; diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index 3160b2567..11ba94180 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -139,7 +139,7 @@ int ZWRAP_initializeCStream(ZWRAP_CCtx* zwc) errorCode = ZSTD_initCStream_advanced(zwc->zbc, NULL, 0, params, zwc->pledgedSrcSize); if (ZSTD_isError(errorCode)) return Z_STREAM_ERROR; } } - + return Z_OK; } From f71828f2c455ec46f58bd669c358db402062e19d Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 22 Sep 2016 15:55:01 +0200 Subject: [PATCH 24/43] zwrapbench: testing speed of ZSTD_decompressStream --- zlibWrapper/.gitignore | 2 +- zlibWrapper/Makefile | 2 +- zlibWrapper/examples/zwrapbench.c | 31 ++++++++++++++++++++++++++++--- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/zlibWrapper/.gitignore b/zlibWrapper/.gitignore index bf3f3d87d..f197c8cc6 100644 --- a/zlibWrapper/.gitignore +++ b/zlibWrapper/.gitignore @@ -26,4 +26,4 @@ foo.gz # Misc files *.bat *.zip -examples/example2.c +*.txt diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index 5329e7e2c..3d6131aa1 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -41,7 +41,7 @@ test_zstd: example_zstd fitblk_zstd zwrapbench ./example_zstd ./fitblk_zstd 10240 <../zstd_compression_format.md ./fitblk_zstd 40960 <../zstd_compression_format.md - ./zwrapbench ../zstd_compression_format.md + ./zwrapbench -qb1e5 ../zstd_compression_format.md .c.o: $(CC) $(CFLAGS) -c -o $@ $< diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c index 3f4fe05b0..318a890c0 100644 --- a/zlibWrapper/examples/zwrapbench.c +++ b/zlibWrapper/examples/zwrapbench.c @@ -281,9 +281,9 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, if (compressor == BMK_ZLIB || compressor == BMK_ZWRAP_ZLIB) useZSTDcompression(0); else useZSTDcompression(1); do { + z_stream def; U32 blockNb; for (blockNb=0; blockNb Date: Thu, 22 Sep 2016 17:59:10 +0200 Subject: [PATCH 25/43] zwrapbench: testing reusing of a context --- zlibWrapper/examples/zwrapbench.c | 124 +++++++++++++++++++++++++----- zlibWrapper/zstd_zlibwrapper.c | 37 ++++----- 2 files changed, 123 insertions(+), 38 deletions(-) diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c index 318a890c0..4659cbb92 100644 --- a/zlibWrapper/examples/zwrapbench.c +++ b/zlibWrapper/examples/zwrapbench.c @@ -138,7 +138,7 @@ typedef struct size_t resSize; } blockParam_t; -typedef enum { BMK_ZSTD, BMK_ZSTD2, BMK_ZLIB, BMK_ZWRAP_ZLIB, BMK_ZWRAP_ZSTD } BMK_compressor; +typedef enum { BMK_ZSTD, BMK_ZSTD_STREAM, BMK_ZLIB, BMK_ZWRAP_ZLIB, BMK_ZWRAP_ZSTD, BMK_ZLIB_REUSE, BMK_ZWRAP_ZLIB_REUSE, BMK_ZWRAP_ZSTD_REUSE } BMK_compressor; #define MIN(a,b) ((a)<(b) ? (a) : (b)) @@ -249,19 +249,20 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, nbLoops++; } while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop); ZSTD_freeCDict(cdict); - } else if (compressor == BMK_ZSTD2) { + } else if (compressor == BMK_ZSTD_STREAM) { ZSTD_parameters const zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize); ZSTD_inBuffer inBuffer; ZSTD_outBuffer outBuffer; ZSTD_CStream* zbc = ZSTD_createCStream(); + size_t rSize; if (zbc == NULL) EXM_THROW(1, "ZSTD_createCStream() allocation failure"); - + rSize = ZSTD_initCStream_advanced(zbc, NULL, 0, zparams, avgSize); + if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_initCStream_advanced() failed : %s", ZSTD_getErrorName(rSize)); do { U32 blockNb; for (blockNb=0; blockNb Z_BEST_COMPRESSION) cLevelLast = Z_BEST_COMPRESSION; @@ -539,13 +611,29 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, dictBuffer, dictBufferSize, BMK_ZLIB); } - DISPLAY("benchmarking zlibWrapper with zlib %s\n", ZLIB_VERSION); + DISPLAY("benchmarking zlib %s (reusing a context)\n", ZLIB_VERSION); + for (l=cLevel; l <= cLevelLast; l++) { + BMK_benchMem(srcBuffer, benchedSize, + displayName, l, + fileSizes, nbFiles, + dictBuffer, dictBufferSize, BMK_ZLIB_REUSE); + } + + DISPLAY("benchmarking zlib %s (using zlibWrapper)\n", ZLIB_VERSION); for (l=cLevel; l <= cLevelLast; l++) { BMK_benchMem(srcBuffer, benchedSize, displayName, l, fileSizes, nbFiles, dictBuffer, dictBufferSize, BMK_ZWRAP_ZLIB); } + + DISPLAY("benchmarking zlib %s (zlibWrapper with reusing a context)\n", ZLIB_VERSION); + for (l=cLevel; l <= cLevelLast; l++) { + BMK_benchMem(srcBuffer, benchedSize, + displayName, l, + fileSizes, nbFiles, + dictBuffer, dictBufferSize, BMK_ZWRAP_ZLIB_REUSE); + } } diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index 11ba94180..c9a04b8b0 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -425,7 +425,7 @@ ZWRAP_DCtx* ZWRAP_createDCtx(z_streamp strm) size_t ZWRAP_freeDCtx(ZWRAP_DCtx* zwd) { if (zwd==NULL) return 0; /* support free on null */ - ZSTD_freeDStream(zwd->zbd); + if (zwd->zbd) ZSTD_freeDStream(zwd->zbd); if (zwd->version) zwd->customMem.customFree(zwd->customMem.opaque, zwd->version); zwd->customMem.customFree(zwd->customMem.opaque, zwd); return 0; @@ -505,8 +505,10 @@ ZEXTERN int ZEXPORT z_inflateReset OF((z_streamp strm)) { ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state; if (zwd == NULL) return Z_STREAM_ERROR; - { size_t const errorCode = ZSTD_resetDStream(zwd->zbd); - if (ZSTD_isError(errorCode)) return ZWRAPD_finishWithError(zwd, strm, 0); } + if (zwd->zbd) { + size_t const errorCode = ZSTD_resetDStream(zwd->zbd); + if (ZSTD_isError(errorCode)) return ZWRAPD_finishWithError(zwd, strm, 0); + } ZWRAP_initDCtx(zwd); } @@ -545,7 +547,7 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm, { size_t errorCode; ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state; - if (zwd == NULL) return Z_STREAM_ERROR; + if (zwd == NULL || zwd->zbd == NULL) return Z_STREAM_ERROR; errorCode = ZSTD_initDStream_usingDict(zwd->zbd, dictionary, dictLength); if (ZSTD_isError(errorCode)) return ZWRAPD_finishWithError(zwd, strm, 0); @@ -580,17 +582,15 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) } if (strm->avail_in > 0) { - size_t errorCode, srcSize, inPos; + size_t errorCode, srcSize; ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state; if (zwd == NULL) return Z_STREAM_ERROR; LOG_WRAPPERD("- inflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); if (zwd->decompState == Z_STREAM_END) return Z_STREAM_END; - // if (((strm->avail_in < ZSTD_HEADERSIZE) || (strm->total_in > 0)) && (strm->total_in < ZLIB_HEADERSIZE)) if (strm->total_in < ZLIB_HEADERSIZE) { - // printf("."); srcSize = MIN(strm->avail_in, ZLIB_HEADERSIZE - strm->total_in); memcpy(zwd->headerBuf+strm->total_in, strm->next_in, srcSize); strm->total_in += srcSize; @@ -637,10 +637,13 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) } } - // if (((strm->avail_in < ZSTD_HEADERSIZE) || (strm->total_in > 0)) && (strm->total_in < ZSTD_HEADERSIZE)) + if (!zwd->zbd) { + zwd->zbd = ZSTD_createDStream_advanced(zwd->customMem); + if (zwd->zbd == NULL) { LOG_WRAPPERD("ERROR: ZSTD_createDStream_advanced\n"); goto error; } + } + if (strm->total_in < ZSTD_HEADERSIZE) { - // printf("+"); srcSize = MIN(strm->avail_in, ZSTD_HEADERSIZE - strm->total_in); memcpy(zwd->headerBuf+strm->total_in, strm->next_in, srcSize); strm->total_in += srcSize; @@ -648,15 +651,11 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) strm->avail_in -= srcSize; if (strm->total_in < ZSTD_HEADERSIZE) return Z_OK; - zwd->zbd = ZSTD_createDStream_advanced(zwd->customMem); - if (zwd->zbd == NULL) goto error; - errorCode = ZSTD_initDStream(zwd->zbd); - if (ZSTD_isError(errorCode)) goto error; + if (ZSTD_isError(errorCode)) { LOG_WRAPPERD("ERROR: ZSTD_initDStream errorCode=%s\n", ZSTD_getErrorName(errorCode)); goto error; } if (flush == Z_INFLATE_SYNC) { strm->msg = "inflateSync is not supported!"; goto error; } - inPos = zwd->inBuffer.pos; zwd->inBuffer.src = zwd->headerBuf; zwd->inBuffer.size = ZSTD_HEADERSIZE; zwd->inBuffer.pos = 0; @@ -669,11 +668,9 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) LOG_WRAPPERD("ERROR: ZSTD_decompressStream1 %s\n", ZSTD_getErrorName(errorCode)); goto error; } - // LOG_WRAPPERD("1srcSize=%d inpos=%d inBuffer.pos=%d inBuffer.size=%d outBuffer.pos=%d\n", (int)srcSize, (int)inPos, (int)zwd->inBuffer.pos, (int)zwd->inBuffer.size, (int)zwd->outBuffer.pos); if (zwd->inBuffer.pos != zwd->inBuffer.size) return ZWRAPD_finishWithError(zwd, strm, 0); /* not consumed */ } - inPos = 0;//zwd->inBuffer.pos; zwd->inBuffer.src = strm->next_in; zwd->inBuffer.size = strm->avail_in; zwd->inBuffer.pos = 0; @@ -687,13 +684,13 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) LOG_WRAPPERD("ERROR: ZSTD_decompressStream2 %s zwd->errorCount=%d\n", ZSTD_getErrorName(errorCode), zwd->errorCount); if (zwd->errorCount<=1) return Z_NEED_DICT; else goto error; } - LOG_WRAPPERD("inflate inpos=%d inBuffer.pos=%d inBuffer.size=%d outBuffer.pos=%d outBuffer.size=%d o\n", (int)inPos, (int)zwd->inBuffer.pos, (int)zwd->inBuffer.size, (int)zwd->outBuffer.pos, (int)zwd->outBuffer.size); + LOG_WRAPPERD("inflate inBuffer.pos=%d inBuffer.size=%d outBuffer.pos=%d outBuffer.size=%d o\n", (int)zwd->inBuffer.pos, (int)zwd->inBuffer.size, (int)zwd->outBuffer.pos, (int)zwd->outBuffer.size); strm->next_out += zwd->outBuffer.pos; strm->total_out += zwd->outBuffer.pos; strm->avail_out -= zwd->outBuffer.pos; - strm->total_in += zwd->inBuffer.pos - inPos; - strm->next_in += zwd->inBuffer.pos - inPos; - strm->avail_in -= zwd->inBuffer.pos - inPos; + strm->total_in += zwd->inBuffer.pos; + strm->next_in += zwd->inBuffer.pos; + strm->avail_in -= zwd->inBuffer.pos; if (errorCode == 0) { LOG_WRAPPERD("inflate Z_STREAM_END1 avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); zwd->decompState = Z_STREAM_END; From 252c20dd3419753f80570f4e2a69856d4c7db938 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 23 Sep 2016 09:08:40 +0200 Subject: [PATCH 26/43] a new ZWRAP API --- zlibWrapper/README.md | 2 +- zlibWrapper/examples/example.c | 6 +- zlibWrapper/examples/fitblk.c | 6 +- zlibWrapper/examples/zwrapbench.c | 57 +++++++++---------- zlibWrapper/zstd_zlibwrapper.c | 92 +++++++++++++++---------------- zlibWrapper/zstd_zlibwrapper.h | 12 ++-- 6 files changed, 88 insertions(+), 87 deletions(-) diff --git a/zlibWrapper/README.md b/zlibWrapper/README.md index 90ee47e60..88174fbec 100644 --- a/zlibWrapper/README.md +++ b/zlibWrapper/README.md @@ -34,7 +34,7 @@ The linking should be changed to: After embedding the zstd wrapper within your project the zstd library is turned off by default. Your project should work as before with zlib. There are two options to enable zstd compression: - compilation with ```-DZWRAP_USE_ZSTD=1``` (or using ```#define ZWRAP_USE_ZSTD 1``` before ```#include "zstd_zlibwrapper.h"```) -- using the ```void useZSTDcompression(int turn_on)``` function (declared in ```#include "zstd_zlibwrapper.h"```) +- using the ```void ZWRAP_useZSTDcompression(int turn_on)``` function (declared in ```#include "zstd_zlibwrapper.h"```) There is no switch for zstd decompression because zlib and zstd streams are automatically detected and decompressed using a proper library. diff --git a/zlibWrapper/examples/example.c b/zlibWrapper/examples/example.c index 735aa6bb3..20ed81d57 100644 --- a/zlibWrapper/examples/example.c +++ b/zlibWrapper/examples/example.c @@ -583,7 +583,7 @@ int main(argc, argv) printf("zlib version %s = 0x%04x, compile flags = 0x%lx\n", ZLIB_VERSION, ZLIB_VERNUM, zlibCompileFlags()); - if (isUsingZSTDcompression()) printf("zstd version %s\n", zstdVersion()); + if (ZWRAP_isUsingZSTDcompression()) printf("zstd version %s\n", zstdVersion()); compr = (Byte*)calloc((uInt)comprLen, 1); uncompr = (Byte*)calloc((uInt)uncomprLen, 1); @@ -600,7 +600,7 @@ int main(argc, argv) #else test_compress(compr, comprLen, uncompr, uncomprLen); - if (!isUsingZSTDcompression()) + if (!ZWRAP_isUsingZSTDcompression()) test_gzio((argc > 1 ? argv[1] : TESTFILE), uncompr, uncomprLen); #endif @@ -611,7 +611,7 @@ int main(argc, argv) test_large_deflate(compr, comprLen, uncompr, uncomprLen); test_large_inflate(compr, comprLen, uncompr, uncomprLen); - if (!isUsingZSTDcompression()) { + if (!ZWRAP_isUsingZSTDcompression()) { test_flush(compr, &comprLen); test_sync(compr, comprLen, uncompr, uncomprLen); } diff --git a/zlibWrapper/examples/fitblk.c b/zlibWrapper/examples/fitblk.c index 4e5a38315..e3fda3c80 100644 --- a/zlibWrapper/examples/fitblk.c +++ b/zlibWrapper/examples/fitblk.c @@ -152,7 +152,7 @@ int main(int argc, char **argv) size = (unsigned)ret; printf("zlib version %s\n", ZLIB_VERSION); - if (isUsingZSTDcompression()) printf("zstd version %s\n", zstdVersion()); + if (ZWRAP_isUsingZSTDcompression()) printf("zstd version %s\n", zstdVersion()); /* allocate memory for buffers and compression engine */ blk = malloc(size + EXCESS); @@ -162,9 +162,9 @@ int main(int argc, char **argv) ret = deflateInit(&def, Z_DEFAULT_COMPRESSION); if (ret != Z_OK || blk == NULL) quit("out of memory"); - ret = ZSTD_setPledgedSrcSize(&def, 1<<16); + ret = ZWRAP_setPledgedSrcSize(&def, 1<<16); if (ret != Z_OK) - quit("ZSTD_setPledgedSrcSize"); + quit("ZWRAP_setPledgedSrcSize"); /* compress from stdin until output full, or no more input */ def.avail_out = size + EXCESS; diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c index 4659cbb92..fe747f50e 100644 --- a/zlibWrapper/examples/zwrapbench.c +++ b/zlibWrapper/examples/zwrapbench.c @@ -281,16 +281,16 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, } else if (compressor == BMK_ZWRAP_ZLIB_REUSE || compressor == BMK_ZWRAP_ZSTD_REUSE || compressor == BMK_ZLIB_REUSE) { z_stream def; int ret; - if (compressor == BMK_ZLIB_REUSE || compressor == BMK_ZWRAP_ZLIB_REUSE) useZSTDcompression(0); - else useZSTDcompression(1); + if (compressor == BMK_ZLIB_REUSE || compressor == BMK_ZWRAP_ZLIB_REUSE) ZWRAP_useZSTDcompression(0); + else ZWRAP_useZSTDcompression(1); def.zalloc = Z_NULL; def.zfree = Z_NULL; def.opaque = Z_NULL; ret = deflateInit(&def, cLevel); if (ret != Z_OK) EXM_THROW(1, "deflateInit failure"); - if (isUsingZSTDcompression()) { - ret = ZSTD_setPledgedSrcSize(&def, avgSize); - if (ret != Z_OK) EXM_THROW(1, "ZSTD_setPledgedSrcSize failure"); + if (ZWRAP_isUsingZSTDcompression()) { + ret = ZWRAP_setPledgedSrcSize(&def, avgSize); + if (ret != Z_OK) EXM_THROW(1, "ZWRAP_setPledgedSrcSize failure"); } do { U32 blockNb; @@ -313,8 +313,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, if (ret != Z_OK) EXM_THROW(1, "deflateEnd failure"); } else { z_stream def; - if (compressor == BMK_ZLIB || compressor == BMK_ZWRAP_ZLIB) useZSTDcompression(0); - else useZSTDcompression(1); + if (compressor == BMK_ZLIB || compressor == BMK_ZWRAP_ZLIB) ZWRAP_useZSTDcompression(0); + else ZWRAP_useZSTDcompression(1); do { U32 blockNb; for (blockNb=0; blockNb Z_BEST_COMPRESSION) cLevelLast = Z_BEST_COMPRESSION; + DISPLAY("\n"); DISPLAY("benchmarking zlib %s\n", ZLIB_VERSION); - for (l=cLevel; l <= cLevelLast; l++) { - BMK_benchMem(srcBuffer, benchedSize, - displayName, l, - fileSizes, nbFiles, - dictBuffer, dictBufferSize, BMK_ZLIB); - } - - DISPLAY("benchmarking zlib %s (reusing a context)\n", ZLIB_VERSION); for (l=cLevel; l <= cLevelLast; l++) { BMK_benchMem(srcBuffer, benchedSize, displayName, l, @@ -619,20 +612,28 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, dictBuffer, dictBufferSize, BMK_ZLIB_REUSE); } + DISPLAY("benchmarking zlib %s (zlib not reusing a context)\n", ZLIB_VERSION); + for (l=cLevel; l <= cLevelLast; l++) { + BMK_benchMem(srcBuffer, benchedSize, + displayName, l, + fileSizes, nbFiles, + dictBuffer, dictBufferSize, BMK_ZLIB); + } + DISPLAY("benchmarking zlib %s (using zlibWrapper)\n", ZLIB_VERSION); for (l=cLevel; l <= cLevelLast; l++) { BMK_benchMem(srcBuffer, benchedSize, displayName, l, fileSizes, nbFiles, - dictBuffer, dictBufferSize, BMK_ZWRAP_ZLIB); + dictBuffer, dictBufferSize, BMK_ZWRAP_ZLIB_REUSE); } - DISPLAY("benchmarking zlib %s (zlibWrapper with reusing a context)\n", ZLIB_VERSION); + DISPLAY("benchmarking zlib %s (zlibWrapper not reusing a context)\n", ZLIB_VERSION); for (l=cLevel; l <= cLevelLast; l++) { BMK_benchMem(srcBuffer, benchedSize, displayName, l, fileSizes, nbFiles, - dictBuffer, dictBufferSize, BMK_ZWRAP_ZLIB_REUSE); + dictBuffer, dictBufferSize, BMK_ZWRAP_ZLIB); } } diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index c9a04b8b0..b1af8eb55 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -46,19 +46,19 @@ ZEXTERN const char * ZEXPORT z_zlibVersion OF((void)) { return zlibVersion(); } #define ZWRAP_USE_ZSTD 0 #endif -static int g_useZSTDcompression = ZWRAP_USE_ZSTD; /* 0 = don't use ZSTD */ +static int g_ZWRAP_useZSTDcompression = ZWRAP_USE_ZSTD; /* 0 = don't use ZSTD */ -void useZSTDcompression(int turn_on) { g_useZSTDcompression = turn_on; } +void ZWRAP_useZSTDcompression(int turn_on) { g_ZWRAP_useZSTDcompression = turn_on; } -int isUsingZSTDcompression(void) { return g_useZSTDcompression; } +int ZWRAP_isUsingZSTDcompression(void) { return g_ZWRAP_useZSTDcompression; } static ZWRAP_decompress_type g_ZWRAPdecompressionType = ZWRAP_AUTO; -void setZWRAPdecompressionType(ZWRAP_decompress_type type) { g_ZWRAPdecompressionType = type; }; +void ZWRAP_setDecompressionType(ZWRAP_decompress_type type) { g_ZWRAPdecompressionType = type; }; -ZWRAP_decompress_type getZWRAPdecompressionType(void) { return g_ZWRAPdecompressionType; } +ZWRAP_decompress_type ZWRAP_getDecompressionType(void) { return g_ZWRAPdecompressionType; } @@ -163,7 +163,7 @@ int ZWRAPC_finishWithErrorMsg(z_streamp strm, char* message) } -int ZSTD_setPledgedSrcSize(z_streamp strm, unsigned long long pledgedSrcSize) +int ZWRAP_setPledgedSrcSize(z_streamp strm, unsigned long long pledgedSrcSize) { ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state; if (zwc == NULL) return Z_STREAM_ERROR; @@ -179,7 +179,7 @@ ZEXTERN int ZEXPORT z_deflateInit_ OF((z_streamp strm, int level, ZWRAP_CCtx* zwc; LOG_WRAPPERC("- deflateInit level=%d\n", level); - if (!g_useZSTDcompression) { + if (!g_ZWRAP_useZSTDcompression) { return deflateInit_((strm), (level), version, stream_size); } @@ -202,7 +202,7 @@ ZEXTERN int ZEXPORT z_deflateInit2_ OF((z_streamp strm, int level, int method, int strategy, const char *version, int stream_size)) { - if (!g_useZSTDcompression) + if (!g_ZWRAP_useZSTDcompression) return deflateInit2_(strm, level, method, windowBits, memLevel, strategy, version, stream_size); return z_deflateInit_ (strm, level, version, stream_size); @@ -212,7 +212,7 @@ ZEXTERN int ZEXPORT z_deflateInit2_ OF((z_streamp strm, int level, int method, ZEXTERN int ZEXPORT z_deflateReset OF((z_streamp strm)) { LOG_WRAPPERC("- deflateReset\n"); - if (!g_useZSTDcompression) + if (!g_ZWRAP_useZSTDcompression) return deflateReset(strm); { ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state; @@ -235,7 +235,7 @@ ZEXTERN int ZEXPORT z_deflateSetDictionary OF((z_streamp strm, const Bytef *dictionary, uInt dictLength)) { - if (!g_useZSTDcompression) { + if (!g_ZWRAP_useZSTDcompression) { LOG_WRAPPERC("- deflateSetDictionary\n"); return deflateSetDictionary(strm, dictionary, dictLength); } @@ -259,7 +259,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) { ZWRAP_CCtx* zwc; - if (!g_useZSTDcompression) { + if (!g_ZWRAP_useZSTDcompression) { int res; LOG_WRAPPERC("- deflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); res = deflate(strm, flush); @@ -330,7 +330,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) ZEXTERN int ZEXPORT z_deflateEnd OF((z_streamp strm)) { - if (!g_useZSTDcompression) { + if (!g_ZWRAP_useZSTDcompression) { LOG_WRAPPERC("- deflateEnd\n"); return deflateEnd(strm); } @@ -349,7 +349,7 @@ ZEXTERN int ZEXPORT z_deflateEnd OF((z_streamp strm)) ZEXTERN uLong ZEXPORT z_deflateBound OF((z_streamp strm, uLong sourceLen)) { - if (!g_useZSTDcompression) + if (!g_ZWRAP_useZSTDcompression) return deflateBound(strm, sourceLen); return ZSTD_compressBound(sourceLen); @@ -360,7 +360,7 @@ ZEXTERN int ZEXPORT z_deflateParams OF((z_streamp strm, int level, int strategy)) { - if (!g_useZSTDcompression) { + if (!g_ZWRAP_useZSTDcompression) { LOG_WRAPPERC("- deflateParams level=%d strategy=%d\n", level, strategy); return deflateParams(strm, level, strategy); } @@ -739,7 +739,7 @@ ZEXTERN int ZEXPORT z_inflateSync OF((z_streamp strm)) ZEXTERN int ZEXPORT z_deflateCopy OF((z_streamp dest, z_streamp source)) { - if (!g_useZSTDcompression) + if (!g_ZWRAP_useZSTDcompression) return deflateCopy(dest, source); return ZWRAPC_finishWithErrorMsg(source, "deflateCopy is not supported!"); } @@ -751,7 +751,7 @@ ZEXTERN int ZEXPORT z_deflateTune OF((z_streamp strm, int nice_length, int max_chain)) { - if (!g_useZSTDcompression) + if (!g_ZWRAP_useZSTDcompression) return deflateTune(strm, good_length, max_lazy, nice_length, max_chain); return ZWRAPC_finishWithErrorMsg(strm, "deflateTune is not supported!"); } @@ -762,7 +762,7 @@ ZEXTERN int ZEXPORT z_deflatePending OF((z_streamp strm, unsigned *pending, int *bits)) { - if (!g_useZSTDcompression) + if (!g_ZWRAP_useZSTDcompression) return deflatePending(strm, pending, bits); return ZWRAPC_finishWithErrorMsg(strm, "deflatePending is not supported!"); } @@ -773,7 +773,7 @@ ZEXTERN int ZEXPORT z_deflatePrime OF((z_streamp strm, int bits, int value)) { - if (!g_useZSTDcompression) + if (!g_ZWRAP_useZSTDcompression) return deflatePrime(strm, bits, value); return ZWRAPC_finishWithErrorMsg(strm, "deflatePrime is not supported!"); } @@ -782,7 +782,7 @@ ZEXTERN int ZEXPORT z_deflatePrime OF((z_streamp strm, ZEXTERN int ZEXPORT z_deflateSetHeader OF((z_streamp strm, gz_headerp head)) { - if (!g_useZSTDcompression) + if (!g_ZWRAP_useZSTDcompression) return deflateSetHeader(strm, head); return ZWRAPC_finishWithErrorMsg(strm, "deflateSetHeader is not supported!"); } @@ -880,7 +880,7 @@ ZEXTERN uLong ZEXPORT z_zlibCompileFlags OF((void)) { return zlibCompileFlags(); ZEXTERN int ZEXPORT z_compress OF((Bytef *dest, uLongf *destLen, const Bytef *source, uLong sourceLen)) { - if (!g_useZSTDcompression) + if (!g_ZWRAP_useZSTDcompression) return compress(dest, destLen, source, sourceLen); { size_t dstCapacity = *destLen; @@ -897,7 +897,7 @@ ZEXTERN int ZEXPORT z_compress2 OF((Bytef *dest, uLongf *destLen, const Bytef *source, uLong sourceLen, int level)) { - if (!g_useZSTDcompression) + if (!g_ZWRAP_useZSTDcompression) return compress2(dest, destLen, source, sourceLen, level); { size_t dstCapacity = *destLen; @@ -911,7 +911,7 @@ ZEXTERN int ZEXPORT z_compress2 OF((Bytef *dest, uLongf *destLen, ZEXTERN uLong ZEXPORT z_compressBound OF((uLong sourceLen)) { - if (!g_useZSTDcompression) + if (!g_ZWRAP_useZSTDcompression) return compressBound(sourceLen); return ZSTD_compressBound(sourceLen); @@ -937,7 +937,7 @@ ZEXTERN int ZEXPORT z_uncompress OF((Bytef *dest, uLongf *destLen, /* gzip file access functions */ ZEXTERN gzFile ZEXPORT z_gzopen OF((const char *path, const char *mode)) { - if (!g_useZSTDcompression) + if (!g_ZWRAP_useZSTDcompression) return gzopen(path, mode); FINISH_WITH_NULL_ERR("gzopen is not supported!"); } @@ -945,7 +945,7 @@ ZEXTERN gzFile ZEXPORT z_gzopen OF((const char *path, const char *mode)) ZEXTERN gzFile ZEXPORT z_gzdopen OF((int fd, const char *mode)) { - if (!g_useZSTDcompression) + if (!g_ZWRAP_useZSTDcompression) return gzdopen(fd, mode); FINISH_WITH_NULL_ERR("gzdopen is not supported!"); } @@ -954,7 +954,7 @@ ZEXTERN gzFile ZEXPORT z_gzdopen OF((int fd, const char *mode)) #if ZLIB_VERNUM >= 0x1240 ZEXTERN int ZEXPORT z_gzbuffer OF((gzFile file, unsigned size)) { - if (!g_useZSTDcompression) + if (!g_ZWRAP_useZSTDcompression) return gzbuffer(file, size); FINISH_WITH_GZ_ERR("gzbuffer is not supported!"); } @@ -962,7 +962,7 @@ ZEXTERN int ZEXPORT z_gzbuffer OF((gzFile file, unsigned size)) ZEXTERN z_off_t ZEXPORT z_gzoffset OF((gzFile file)) { - if (!g_useZSTDcompression) + if (!g_ZWRAP_useZSTDcompression) return gzoffset(file); FINISH_WITH_GZ_ERR("gzoffset is not supported!"); } @@ -970,7 +970,7 @@ ZEXTERN z_off_t ZEXPORT z_gzoffset OF((gzFile file)) ZEXTERN int ZEXPORT z_gzclose_r OF((gzFile file)) { - if (!g_useZSTDcompression) + if (!g_ZWRAP_useZSTDcompression) return gzclose_r(file); FINISH_WITH_GZ_ERR("gzclose_r is not supported!"); } @@ -978,7 +978,7 @@ ZEXTERN int ZEXPORT z_gzclose_r OF((gzFile file)) ZEXTERN int ZEXPORT z_gzclose_w OF((gzFile file)) { - if (!g_useZSTDcompression) + if (!g_ZWRAP_useZSTDcompression) return gzclose_w(file); FINISH_WITH_GZ_ERR("gzclose_w is not supported!"); } @@ -987,7 +987,7 @@ ZEXTERN int ZEXPORT z_gzclose_w OF((gzFile file)) ZEXTERN int ZEXPORT z_gzsetparams OF((gzFile file, int level, int strategy)) { - if (!g_useZSTDcompression) + if (!g_ZWRAP_useZSTDcompression) return gzsetparams(file, level, strategy); FINISH_WITH_GZ_ERR("gzsetparams is not supported!"); } @@ -995,7 +995,7 @@ ZEXTERN int ZEXPORT z_gzsetparams OF((gzFile file, int level, int strategy)) ZEXTERN int ZEXPORT z_gzread OF((gzFile file, voidp buf, unsigned len)) { - if (!g_useZSTDcompression) + if (!g_ZWRAP_useZSTDcompression) return gzread(file, buf, len); FINISH_WITH_GZ_ERR("gzread is not supported!"); } @@ -1004,7 +1004,7 @@ ZEXTERN int ZEXPORT z_gzread OF((gzFile file, voidp buf, unsigned len)) ZEXTERN int ZEXPORT z_gzwrite OF((gzFile file, voidpc buf, unsigned len)) { - if (!g_useZSTDcompression) + if (!g_ZWRAP_useZSTDcompression) return gzwrite(file, buf, len); FINISH_WITH_GZ_ERR("gzwrite is not supported!"); } @@ -1016,7 +1016,7 @@ ZEXTERN int ZEXPORTVA z_gzprintf Z_ARG((gzFile file, const char *format, ...)) ZEXTERN int ZEXPORTVA z_gzprintf OF((gzFile file, const char *format, ...)) #endif { - if (!g_useZSTDcompression) { + if (!g_ZWRAP_useZSTDcompression) { int ret; char buf[1024]; va_list args; @@ -1033,7 +1033,7 @@ ZEXTERN int ZEXPORTVA z_gzprintf OF((gzFile file, const char *format, ...)) ZEXTERN int ZEXPORT z_gzputs OF((gzFile file, const char *s)) { - if (!g_useZSTDcompression) + if (!g_ZWRAP_useZSTDcompression) return gzputs(file, s); FINISH_WITH_GZ_ERR("gzputs is not supported!"); } @@ -1041,7 +1041,7 @@ ZEXTERN int ZEXPORT z_gzputs OF((gzFile file, const char *s)) ZEXTERN char * ZEXPORT z_gzgets OF((gzFile file, char *buf, int len)) { - if (!g_useZSTDcompression) + if (!g_ZWRAP_useZSTDcompression) return gzgets(file, buf, len); FINISH_WITH_NULL_ERR("gzgets is not supported!"); } @@ -1049,7 +1049,7 @@ ZEXTERN char * ZEXPORT z_gzgets OF((gzFile file, char *buf, int len)) ZEXTERN int ZEXPORT z_gzputc OF((gzFile file, int c)) { - if (!g_useZSTDcompression) + if (!g_ZWRAP_useZSTDcompression) return gzputc(file, c); FINISH_WITH_GZ_ERR("gzputc is not supported!"); } @@ -1061,7 +1061,7 @@ ZEXTERN int ZEXPORT z_gzgetc_ OF((gzFile file)) ZEXTERN int ZEXPORT z_gzgetc OF((gzFile file)) #endif { - if (!g_useZSTDcompression) + if (!g_ZWRAP_useZSTDcompression) return gzgetc(file); FINISH_WITH_GZ_ERR("gzgetc is not supported!"); } @@ -1069,7 +1069,7 @@ ZEXTERN int ZEXPORT z_gzgetc OF((gzFile file)) ZEXTERN int ZEXPORT z_gzungetc OF((int c, gzFile file)) { - if (!g_useZSTDcompression) + if (!g_ZWRAP_useZSTDcompression) return gzungetc(c, file); FINISH_WITH_GZ_ERR("gzungetc is not supported!"); } @@ -1077,7 +1077,7 @@ ZEXTERN int ZEXPORT z_gzungetc OF((int c, gzFile file)) ZEXTERN int ZEXPORT z_gzflush OF((gzFile file, int flush)) { - if (!g_useZSTDcompression) + if (!g_ZWRAP_useZSTDcompression) return gzflush(file, flush); FINISH_WITH_GZ_ERR("gzflush is not supported!"); } @@ -1085,7 +1085,7 @@ ZEXTERN int ZEXPORT z_gzflush OF((gzFile file, int flush)) ZEXTERN z_off_t ZEXPORT z_gzseek OF((gzFile file, z_off_t offset, int whence)) { - if (!g_useZSTDcompression) + if (!g_ZWRAP_useZSTDcompression) return gzseek(file, offset, whence); FINISH_WITH_GZ_ERR("gzseek is not supported!"); } @@ -1093,7 +1093,7 @@ ZEXTERN z_off_t ZEXPORT z_gzseek OF((gzFile file, z_off_t offset, int whence)) ZEXTERN int ZEXPORT z_gzrewind OF((gzFile file)) { - if (!g_useZSTDcompression) + if (!g_ZWRAP_useZSTDcompression) return gzrewind(file); FINISH_WITH_GZ_ERR("gzrewind is not supported!"); } @@ -1101,7 +1101,7 @@ ZEXTERN int ZEXPORT z_gzrewind OF((gzFile file)) ZEXTERN z_off_t ZEXPORT z_gztell OF((gzFile file)) { - if (!g_useZSTDcompression) + if (!g_ZWRAP_useZSTDcompression) return gztell(file); FINISH_WITH_GZ_ERR("gztell is not supported!"); } @@ -1109,7 +1109,7 @@ ZEXTERN z_off_t ZEXPORT z_gztell OF((gzFile file)) ZEXTERN int ZEXPORT z_gzeof OF((gzFile file)) { - if (!g_useZSTDcompression) + if (!g_ZWRAP_useZSTDcompression) return gzeof(file); FINISH_WITH_GZ_ERR("gzeof is not supported!"); } @@ -1117,7 +1117,7 @@ ZEXTERN int ZEXPORT z_gzeof OF((gzFile file)) ZEXTERN int ZEXPORT z_gzdirect OF((gzFile file)) { - if (!g_useZSTDcompression) + if (!g_ZWRAP_useZSTDcompression) return gzdirect(file); FINISH_WITH_GZ_ERR("gzdirect is not supported!"); } @@ -1125,7 +1125,7 @@ ZEXTERN int ZEXPORT z_gzdirect OF((gzFile file)) ZEXTERN int ZEXPORT z_gzclose OF((gzFile file)) { - if (!g_useZSTDcompression) + if (!g_ZWRAP_useZSTDcompression) return gzclose(file); FINISH_WITH_GZ_ERR("gzclose is not supported!"); } @@ -1133,7 +1133,7 @@ ZEXTERN int ZEXPORT z_gzclose OF((gzFile file)) ZEXTERN const char * ZEXPORT z_gzerror OF((gzFile file, int *errnum)) { - if (!g_useZSTDcompression) + if (!g_ZWRAP_useZSTDcompression) return gzerror(file, errnum); FINISH_WITH_NULL_ERR("gzerror is not supported!"); } @@ -1141,7 +1141,7 @@ ZEXTERN const char * ZEXPORT z_gzerror OF((gzFile file, int *errnum)) ZEXTERN void ZEXPORT z_gzclearerr OF((gzFile file)) { - if (!g_useZSTDcompression) + if (!g_ZWRAP_useZSTDcompression) gzclearerr(file); } diff --git a/zlibWrapper/zstd_zlibwrapper.h b/zlibWrapper/zstd_zlibwrapper.h index e8114c4b8..258cb234d 100644 --- a/zlibWrapper/zstd_zlibwrapper.h +++ b/zlibWrapper/zstd_zlibwrapper.h @@ -32,25 +32,25 @@ const char * zstdVersion(void); /* COMPRESSION */ /* enables/disables zstd compression during runtime */ -void useZSTDcompression(int turn_on); +void ZWRAP_useZSTDcompression(int turn_on); /* check if zstd compression is turned on */ -int isUsingZSTDcompression(void); +int ZWRAP_isUsingZSTDcompression(void); /* Changes a pledged source size for a given compression stream. It will change ZSTD compression parameters what may improve compression speed and/or ratio. The function should be called just after deflateInit(). */ -int ZSTD_setPledgedSrcSize(z_streamp strm, unsigned long long pledgedSrcSize); +int ZWRAP_setPledgedSrcSize(z_streamp strm, unsigned long long pledgedSrcSize); /* DECOMPRESSION */ -typedef enum { ZWRAP_FORCE_ZLIB, ZWRAP_FORCE_ZSTD, ZWRAP_AUTO } ZWRAP_decompress_type; +typedef enum { ZWRAP_FORCE_ZLIB, ZWRAP_AUTO } ZWRAP_decompress_type; /* enables/disables automatic recognition of zstd/zlib compressed data during runtime */ -void setZWRAPdecompressionType(ZWRAP_decompress_type type); +void ZWRAP_setDecompressionType(ZWRAP_decompress_type type); /* check zstd decompression type */ -ZWRAP_decompress_type getZWRAPdecompressionType(void); +ZWRAP_decompress_type ZWRAP_getDecompressionType(void); From cf3ec08840b84e58ab79d4eee8ef46440e9db30a Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 23 Sep 2016 10:30:26 +0200 Subject: [PATCH 27/43] ZWRAP_setPledgedSrcSize not required with Z_FINISH --- zlibWrapper/examples/zwrapbench.c | 10 +++------- zlibWrapper/zstd_zlibwrapper.c | 26 +++++++++++--------------- 2 files changed, 14 insertions(+), 22 deletions(-) diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c index fe747f50e..f7ed821e3 100644 --- a/zlibWrapper/examples/zwrapbench.c +++ b/zlibWrapper/examples/zwrapbench.c @@ -261,7 +261,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, do { U32 blockNb; for (blockNb=0; blockNbzbc = ZSTD_createCStream_advanced(zwc->customMem); if (zwc->zbc == NULL) return Z_STREAM_ERROR; - { ZSTD_parameters const params = ZSTD_getParams(zwc->compressionLevel, zwc->pledgedSrcSize, 0); + if (!pledgedSrcSize) pledgedSrcSize = zwc->pledgedSrcSize; + { ZSTD_parameters const params = ZSTD_getParams(zwc->compressionLevel, pledgedSrcSize, 0); size_t errorCode; LOG_WRAPPERC("windowLog=%d chainLog=%d hashLog=%d searchLog=%d searchLength=%d strategy=%d\n", params.cParams.windowLog, params.cParams.chainLog, params.cParams.hashLog, params.cParams.searchLog, params.cParams.searchLength, params.cParams.strategy); - errorCode = ZSTD_initCStream_advanced(zwc->zbc, NULL, 0, params, zwc->pledgedSrcSize); + errorCode = ZSTD_initCStream_advanced(zwc->zbc, NULL, 0, params, pledgedSrcSize); if (ZSTD_isError(errorCode)) return Z_STREAM_ERROR; } } @@ -214,16 +215,6 @@ ZEXTERN int ZEXPORT z_deflateReset OF((z_streamp strm)) LOG_WRAPPERC("- deflateReset\n"); if (!g_ZWRAP_useZSTDcompression) return deflateReset(strm); - - { ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state; - if (!zwc) return Z_STREAM_ERROR; - if (zwc->zbc == NULL) { - int res = ZWRAP_initializeCStream(zwc); - if (res != Z_OK) return ZWRAPC_finishWithError(zwc, strm, res); - } - { size_t const errorCode = ZSTD_resetCStream(zwc->zbc, zwc->pledgedSrcSize); - if (ZSTD_isError(errorCode)) return ZWRAPC_finishWithError(zwc, strm, 0); } - } strm->total_in = 0; strm->total_out = 0; @@ -244,7 +235,7 @@ ZEXTERN int ZEXPORT z_deflateSetDictionary OF((z_streamp strm, LOG_WRAPPERC("- deflateSetDictionary level=%d\n", (int)zwc->compressionLevel); if (!zwc) return Z_STREAM_ERROR; if (zwc->zbc == NULL) { - int res = ZWRAP_initializeCStream(zwc); + int res = ZWRAP_initializeCStream(zwc, 0); if (res != Z_OK) return ZWRAPC_finishWithError(zwc, strm, res); } { size_t const errorCode = ZSTD_initCStream_usingDict(zwc->zbc, dictionary, dictLength, zwc->compressionLevel); @@ -271,8 +262,13 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) if (zwc == NULL) return Z_STREAM_ERROR; if (zwc->zbc == NULL) { - int res = ZWRAP_initializeCStream(zwc); + int res = ZWRAP_initializeCStream(zwc, (flush == Z_FINISH) ? strm->avail_in : 0); if (res != Z_OK) return ZWRAPC_finishWithError(zwc, strm, res); + } else { + if (strm->total_in == 0) { + size_t const errorCode = ZSTD_resetCStream(zwc->zbc, (flush == Z_FINISH) ? strm->avail_in : zwc->pledgedSrcSize); + if (ZSTD_isError(errorCode)) return ZWRAPC_finishWithError(zwc, strm, 0); + } } LOG_WRAPPERC("- deflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); From 4602e530215a71957669da17c7d7e74ec30fa07d Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 23 Sep 2016 10:43:37 +0200 Subject: [PATCH 28/43] added valgrindTest for zlibWrapper --- .travis.yml | 75 ++------------------------------------------ Makefile | 1 - zlibWrapper/Makefile | 8 +++++ 3 files changed, 10 insertions(+), 74 deletions(-) diff --git a/.travis.yml b/.travis.yml index 80cae76fc..1bfbb631a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,88 +3,17 @@ compiler: gcc matrix: fast_finish: true include: - # OS X Mavericks - - os: osx - env: PLATFORM="OS X Mavericks" CMD="make gnu90test && make clean && make test && make clean && make travis-install" - # Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes) - - os: linux - sudo: false - env: PLATFORM="Ubuntu 12.04 container" CMD="make test && make clean && make travis-install" - - os: linux - sudo: false - language: cpp - install: - - export CXX="g++-4.8" CC="gcc-4.8" - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - gcc-4.8 - - g++-4.8 - env: PLATFORM="Ubuntu 12.04 container" CMD="make -C tests test-zstd_nolegacy && make clean && make zlibwrapper && make clean && make cmaketest && make clean && make -C contrib/pzstd pzstd && make -C contrib/pzstd googletest && make -C contrib/pzstd test && make -C contrib/pzstd clean" - - os: linux - sudo: false - env: PLATFORM="Ubuntu 12.04 container" CMD="make usan" - - os: linux - sudo: false - env: PLATFORM="Ubuntu 12.04 container" CMD="make asan" - # Standard Ubuntu 12.04 LTS Server Edition 64 bit - os: linux sudo: required - env: PLATFORM="Ubuntu 12.04" CMD="make armtest" - addons: - apt: - packages: - - gcc-arm-linux-gnueabi - - libc6-dev-armel-cross - - linux-libc-dev-armel-cross - - binfmt-support - - qemu - - qemu-user-static - - os: linux - sudo: required - env: PLATFORM="Ubuntu 12.04" CMD="make -C tests versionsTest" - - os: linux - sudo: required - env: PLATFORM="Ubuntu 12.04" CMD="make asan32" - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - libc6-dev-i386 - - gcc-multilib - - os: linux - sudo: required - env: PLATFORM="Ubuntu 12.04" CMD="make -C tests valgrindTest" + env: PLATFORM="Ubuntu 12.04" CMD="make -C zlibWrapper test valgrindTest" addons: apt: packages: - valgrind - # Ubuntu 14.04 LTS Server Edition 64 bit - os: linux dist: trusty sudo: required - env: PLATFORM="Ubuntu 14.04" CMD="make gpptest && make clean && make gnu90test && make clean && make c99test && make clean && make gnu99test && make clean && make clangtest" - addons: - apt: - packages: - - libc6-dev-i386 - - g++-multilib - - os: linux - dist: trusty - sudo: required - env: PLATFORM="Ubuntu 14.04" CMD="make -C tests test32" - addons: - apt: - packages: - - libc6-dev-i386 - - gcc-multilib - - os: linux - dist: trusty - sudo: required - env: PLATFORM="Ubuntu 14.04" CMD="make zlibwrapper && make clean && make gcc5test && make clean && make gcc6test && sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu && make clean && make ppctest" + env: PLATFORM="Ubuntu 14.04" CMD="make zlibwrapper" addons: apt: sources: diff --git a/Makefile b/Makefile index d3ab6e021..f355891ae 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,6 @@ zstd: cp $(PRGDIR)/zstd . zlibwrapper: - $(MAKE) -C $(ZSTDDIR) all $(MAKE) -C $(ZWRAPDIR) test test_zstd test: diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index 3d6131aa1..e83c5bc02 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -43,6 +43,14 @@ test_zstd: example_zstd fitblk_zstd zwrapbench ./fitblk_zstd 40960 <../zstd_compression_format.md ./zwrapbench -qb1e5 ../zstd_compression_format.md +valgrindTest: VALGRIND = valgrind --leak-check=full --error-exitcode=1 +valgrindTest: example_zstd fitblk_zstd zwrapbench + @echo "\n ---- valgrind tests ----" + $(VALGRIND) ./example_zstd + $(VALGRIND) ./fitblk_zstd 10240 <../zstd_compression_format.md + $(VALGRIND) ./fitblk_zstd 40960 <../zstd_compression_format.md + $(VALGRIND) ./zwrapbench -qb1e5 ../zstd_compression_format.md + .c.o: $(CC) $(CFLAGS) -c -o $@ $< From f77a1132a73da64d55c14222138c8794f615626f Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 23 Sep 2016 12:01:38 +0200 Subject: [PATCH 29/43] improved valgrind tests --- .travis.yml | 2 +- zlibWrapper/Makefile | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1bfbb631a..be8a90afe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ matrix: include: - os: linux sudo: required - env: PLATFORM="Ubuntu 12.04" CMD="make -C zlibWrapper test valgrindTest" + env: PLATFORM="Ubuntu 12.04" CMD='make -C lib all && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest' addons: apt: packages: diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index e83c5bc02..46671ea6e 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -21,7 +21,8 @@ ZLIBWRAPPER_PATH = . EXAMPLE_PATH = examples PROGRAMS_PATH = ../programs CC ?= gcc -CFLAGS = $(LOC) -I$(PROGRAMS_PATH) -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH) -I$(ZLIBDIR) -O3 -std=gnu90 +CFLAGS ?= -O3 +CFLAGS += $(LOC) -I$(PROGRAMS_PATH) -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH) -I$(ZLIBDIR) -std=gnu90 CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef LDFLAGS = $(LOC) RM = rm -f @@ -33,6 +34,8 @@ test: example fitblk ./example ./fitblk 10240 <../zstd_compression_format.md ./fitblk 40960 <../zstd_compression_format.md + ./zwrapbench -qb1e5 ../zstd_compression_format.md + ./zwrapbench -qb1e5B1K ../zstd_compression_format.md test_d: example_d ./example_d @@ -42,14 +45,20 @@ test_zstd: example_zstd fitblk_zstd zwrapbench ./fitblk_zstd 10240 <../zstd_compression_format.md ./fitblk_zstd 40960 <../zstd_compression_format.md ./zwrapbench -qb1e5 ../zstd_compression_format.md + ./zwrapbench -qb1e5B1K ../zstd_compression_format.md valgrindTest: VALGRIND = valgrind --leak-check=full --error-exitcode=1 -valgrindTest: example_zstd fitblk_zstd zwrapbench +valgrindTest: STATICLIB = $(IMPLIB) +valgrindTest: clean example fitblk example_zstd fitblk_zstd zwrapbench @echo "\n ---- valgrind tests ----" + $(VALGRIND) ./example + $(VALGRIND) ./fitblk 10240 <../zstd_compression_format.md + $(VALGRIND) ./fitblk 40960 <../zstd_compression_format.md $(VALGRIND) ./example_zstd $(VALGRIND) ./fitblk_zstd 10240 <../zstd_compression_format.md $(VALGRIND) ./fitblk_zstd 40960 <../zstd_compression_format.md $(VALGRIND) ./zwrapbench -qb1e5 ../zstd_compression_format.md + $(VALGRIND) ./zwrapbench -qb1e5B1K ../zstd_compression_format.md .c.o: $(CC) $(CFLAGS) -c -o $@ $< From 68cd4766c922dfc9dc1839313c201df2c3a2711d Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 23 Sep 2016 12:42:21 +0200 Subject: [PATCH 30/43] initialization of strm->adler --- Makefile | 2 +- zlibWrapper/Makefile | 18 +++++++----------- zlibWrapper/examples/zwrapbench.c | 2 +- zlibWrapper/zstd_zlibwrapper.c | 6 ++++-- 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index f355891ae..ac0c583f4 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,7 @@ zstd: cp $(PRGDIR)/zstd . zlibwrapper: - $(MAKE) -C $(ZWRAPDIR) test test_zstd + $(MAKE) -C $(ZWRAPDIR) test test: $(MAKE) -C $(TESTDIR) $@ diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index 46671ea6e..e595a046c 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -30,31 +30,27 @@ RM = rm -f all: clean fitblk example example_d zwrapbench -test: example fitblk +test: example fitblk example_zstd fitblk_zstd zwrapbench ./example + ./example_zstd ./fitblk 10240 <../zstd_compression_format.md ./fitblk 40960 <../zstd_compression_format.md + ./fitblk_zstd 10240 <../zstd_compression_format.md + ./fitblk_zstd 40960 <../zstd_compression_format.md ./zwrapbench -qb1e5 ../zstd_compression_format.md ./zwrapbench -qb1e5B1K ../zstd_compression_format.md test_d: example_d ./example_d -test_zstd: example_zstd fitblk_zstd zwrapbench - ./example_zstd - ./fitblk_zstd 10240 <../zstd_compression_format.md - ./fitblk_zstd 40960 <../zstd_compression_format.md - ./zwrapbench -qb1e5 ../zstd_compression_format.md - ./zwrapbench -qb1e5B1K ../zstd_compression_format.md - -valgrindTest: VALGRIND = valgrind --leak-check=full --error-exitcode=1 -valgrindTest: STATICLIB = $(IMPLIB) +valgrindTest: VALGRIND = valgrind --track-origins=yes --leak-check=full --error-exitcode=1 +valgrindTest: STATICLIB = -lz $(ZSTDLIBDIR)/libzstd.so valgrindTest: clean example fitblk example_zstd fitblk_zstd zwrapbench @echo "\n ---- valgrind tests ----" $(VALGRIND) ./example + $(VALGRIND) ./example_zstd $(VALGRIND) ./fitblk 10240 <../zstd_compression_format.md $(VALGRIND) ./fitblk 40960 <../zstd_compression_format.md - $(VALGRIND) ./example_zstd $(VALGRIND) ./fitblk_zstd 10240 <../zstd_compression_format.md $(VALGRIND) ./fitblk_zstd 40960 <../zstd_compression_format.md $(VALGRIND) ./zwrapbench -qb1e5 ../zstd_compression_format.md diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c index f7ed821e3..69f372d5d 100644 --- a/zlibWrapper/examples/zwrapbench.c +++ b/zlibWrapper/examples/zwrapbench.c @@ -560,7 +560,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, SET_HIGH_PRIORITY; if (g_displayLevel == 1 && !g_additionalParam) - DISPLAY("bench %s %s: input %u bytes, %u iterations, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10)); + DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10)); if (cLevelLast < cLevel) cLevelLast = cLevel; diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index 2ad3ee0b7..7a825899c 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -22,8 +22,8 @@ #define ZSTD_HEADERSIZE ZSTD_frameHeaderSize_min #define ZWRAP_DEFAULT_CLEVEL 5 /* Z_DEFAULT_COMPRESSION is translated to ZWRAP_DEFAULT_CLEVEL for zstd */ -#define LOG_WRAPPERC(...) /*printf(__VA_ARGS__)*/ -#define LOG_WRAPPERD(...) /*printf(__VA_ARGS__)*/ +#define LOG_WRAPPERC(...) /* printf(__VA_ARGS__) */ +#define LOG_WRAPPERD(...) /* printf(__VA_ARGS__) */ #define FINISH_WITH_GZ_ERR(msg) { \ @@ -194,6 +194,7 @@ ZEXTERN int ZEXPORT z_deflateInit_ OF((z_streamp strm, int level, strm->state = (struct internal_state*) zwc; /* use state which in not used by user */ strm->total_in = 0; strm->total_out = 0; + strm->adler = 0; return Z_OK; } @@ -468,6 +469,7 @@ ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm, strm->total_in = 0; strm->total_out = 0; strm->reserved = 1; /* mark as unknown steam */ + strm->adler = 0; } return Z_OK; From b88accfb5fc013e7891d87c700d4f26e14eb7656 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 23 Sep 2016 13:38:02 +0200 Subject: [PATCH 31/43] use valgrind with a dynamic zstd library --- zlibWrapper/Makefile | 44 +++++++++++++------------------ zlibWrapper/examples/zwrapbench.c | 1 + 2 files changed, 20 insertions(+), 25 deletions(-) diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index e595a046c..a612f2593 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -6,23 +6,17 @@ # Paths to static and dynamic zlib and zstd libraries -# Use "make ZLIBDIR=path/to/zlib" to select a path to library -ifdef ZLIBDIR -STATICLIB = $(ZLIBDIR)/libz.a $(ZSTDLIBDIR)/libzstd.a -IMPLIB = $(ZLIBDIR)/libz.dll.a $(ZSTDLIBDIR)/libzstd.a -else -STATICLIB = -static -lz $(ZSTDLIBDIR)/libzstd.a -IMPLIB = -lz $(ZSTDLIBDIR)/libzstd.a -ZLIBDIR = . -endif +# Use "make ZLIB_LIBRARY=path/to/zlib" to select a path to library +ZLIB_LIBRARY ?= -lz ZSTDLIBDIR = ../lib +ZSTDLIBRARY = $(ZSTDLIBDIR)/libzstd.a ZLIBWRAPPER_PATH = . EXAMPLE_PATH = examples PROGRAMS_PATH = ../programs CC ?= gcc CFLAGS ?= -O3 -CFLAGS += $(LOC) -I$(PROGRAMS_PATH) -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH) -I$(ZLIBDIR) -std=gnu90 +CFLAGS += $(LOC) -I$(PROGRAMS_PATH) -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH) -std=gnu90 CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef LDFLAGS = $(LOC) RM = rm -f @@ -43,8 +37,8 @@ test: example fitblk example_zstd fitblk_zstd zwrapbench test_d: example_d ./example_d -valgrindTest: VALGRIND = valgrind --track-origins=yes --leak-check=full --error-exitcode=1 -valgrindTest: STATICLIB = -lz $(ZSTDLIBDIR)/libzstd.so +valgrindTest: VALGRIND = LD_LIBRARY_PATH=$(ZSTDLIBDIR) valgrind --track-origins=yes --leak-check=full --error-exitcode=1 +valgrindTest: ZSTDLIBRARY = $(ZSTDLIBDIR)/libzstd.so valgrindTest: clean example fitblk example_zstd fitblk_zstd zwrapbench @echo "\n ---- valgrind tests ----" $(VALGRIND) ./example @@ -59,23 +53,20 @@ valgrindTest: clean example fitblk example_zstd fitblk_zstd zwrapbench .c.o: $(CC) $(CFLAGS) -c -o $@ $< -example: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(ZSTDLIBDIR)/libzstd.a - $(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(STATICLIB) +example: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(ZSTDLIBRARY) + $(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(ZSTDLIBRARY) $(ZLIB_LIBRARY) -example_d: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o - $(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(IMPLIB) +example_zstd: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(ZSTDLIBRARY) + $(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(ZSTDLIBRARY) $(ZLIB_LIBRARY) -example_zstd: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(ZSTDLIBDIR)/libzstd.a - $(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(STATICLIB) +fitblk: $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(ZSTDLIBRARY) + $(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(ZSTDLIBRARY) $(ZLIB_LIBRARY) -fitblk: $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(ZSTDLIBDIR)/libzstd.a - $(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(STATICLIB) +fitblk_zstd: $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(ZSTDLIBRARY) + $(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(ZSTDLIBRARY) $(ZLIB_LIBRARY) -fitblk_zstd: $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(ZSTDLIBDIR)/libzstd.a - $(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(STATICLIB) - -zwrapbench: $(EXAMPLE_PATH)/zwrapbench.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(PROGRAMS_PATH)/datagen.o $(ZSTDLIBDIR)/libzstd.a - $(CC) $(LDFLAGS) -o $@ $^ $(STATICLIB) +zwrapbench: $(EXAMPLE_PATH)/zwrapbench.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(PROGRAMS_PATH)/datagen.o $(ZSTDLIBRARY) + $(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/zwrapbench.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(PROGRAMS_PATH)/datagen.o $(ZSTDLIBRARY) $(ZLIB_LIBRARY) $(EXAMPLE_PATH)/zwrapbench.o: $(EXAMPLE_PATH)/zwrapbench.c @@ -88,6 +79,9 @@ $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwra $(ZSTDLIBDIR)/libzstd.a: $(MAKE) -C $(ZSTDLIBDIR) all +$(ZSTDLIBDIR)/libzstd.so: + $(MAKE) -C $(ZSTDLIBDIR) all + clean: -$(RM) $(ZLIBWRAPPER_PATH)/*.o $(EXAMPLE_PATH)/*.o *.o *.exe foo.gz example example_d example_zstd fitblk fitblk_zstd zwrapbench @echo Cleaning completed diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c index 69f372d5d..b925cf7cb 100644 --- a/zlibWrapper/examples/zwrapbench.c +++ b/zlibWrapper/examples/zwrapbench.c @@ -16,6 +16,7 @@ #include /* memset */ #include /* fprintf, fopen, ftello64 */ #include /* clock_t, clock, CLOCKS_PER_SEC */ +#include /* toupper */ #include "mem.h" #define ZSTD_STATIC_LINKING_ONLY From 57b9708054d019aba2e559c3eb257a9276ac20b0 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 23 Sep 2016 14:59:46 +0200 Subject: [PATCH 32/43] faster inflate() autodetection of zlib/zstd --- .travis.yml | 76 +++++++++++++++- zlibWrapper/Makefile | 15 ++-- zlibWrapper/zstd_zlibwrapper.c | 159 +++++++++++++++++++-------------- 3 files changed, 172 insertions(+), 78 deletions(-) diff --git a/.travis.yml b/.travis.yml index be8a90afe..58728d89c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,9 +3,63 @@ compiler: gcc matrix: fast_finish: true include: + # OS X Mavericks + - os: osx + env: PLATFORM="OS X Mavericks" CMD="make gnu90test && make clean && make test && make clean && make travis-install" + # Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes) + - os: linux + sudo: false + env: PLATFORM="Ubuntu 12.04 container" CMD="make test && make clean && make travis-install" + - os: linux + sudo: false + language: cpp + install: + - export CXX="g++-4.8" CC="gcc-4.8" + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - gcc-4.8 + - g++-4.8 + env: PLATFORM="Ubuntu 12.04 container" CMD="make zlibwrapper && make clean && make -C tests test-zstd_nolegacy && make clean && make clean && make cmaketest && make clean && make -C contrib/pzstd pzstd && make -C contrib/pzstd googletest && make -C contrib/pzstd test && make -C contrib/pzstd clean" + - os: linux + sudo: false + env: PLATFORM="Ubuntu 12.04 container" CMD="make usan" + - os: linux + sudo: false + env: PLATFORM="Ubuntu 12.04 container" CMD="make asan" + # Standard Ubuntu 12.04 LTS Server Edition 64 bit - os: linux sudo: required - env: PLATFORM="Ubuntu 12.04" CMD='make -C lib all && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest' + env: PLATFORM="Ubuntu 12.04" CMD="make armtest" + addons: + apt: + packages: + - gcc-arm-linux-gnueabi + - libc6-dev-armel-cross + - linux-libc-dev-armel-cross + - binfmt-support + - qemu + - qemu-user-static + - os: linux + sudo: required + env: PLATFORM="Ubuntu 12.04" CMD="make -C tests versionsTest" + - os: linux + sudo: required + env: PLATFORM="Ubuntu 12.04" CMD="make asan32" + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - libc6-dev-i386 + - gcc-multilib + # Ubuntu 14.04 LTS Server Edition 64 bit + - os: linux + dist: trusty + sudo: required + env: PLATFORM="Ubuntu 14.04" CMD="make -C lib all && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest" addons: apt: packages: @@ -13,7 +67,25 @@ matrix: - os: linux dist: trusty sudo: required - env: PLATFORM="Ubuntu 14.04" CMD="make zlibwrapper" + env: PLATFORM="Ubuntu 14.04" CMD="make gpptest && make clean && make gnu90test && make clean && make c99test && make clean && make gnu99test && make clean && make clangtest" + addons: + apt: + packages: + - libc6-dev-i386 + - g++-multilib + - os: linux + dist: trusty + sudo: required + env: PLATFORM="Ubuntu 14.04" CMD="make -C tests test32" + addons: + apt: + packages: + - libc6-dev-i386 + - gcc-multilib + - os: linux + dist: trusty + sudo: required + env: PLATFORM="Ubuntu 14.04" CMD="make gcc5test && make clean && make gcc6test && sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu && make clean && make ppctest" addons: apt: sources: diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index a612f2593..ea025b951 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -1,8 +1,8 @@ # Makefile for example of using zstd wrapper for zlib # -# make - compiles statically and dynamically linked examples -# make LOC=-DZWRAP_USE_ZSTD=1 - compiles statically and dynamically linked examples with zstd compression turned on -# make test test_d - runs statically and dynamically linked examples +# make - compiles examples +# make LOC=-DZWRAP_USE_ZSTD=1 - compiles examples with zstd compression turned on +# make test - runs examples # Paths to static and dynamic zlib and zstd libraries @@ -22,7 +22,7 @@ LDFLAGS = $(LOC) RM = rm -f -all: clean fitblk example example_d zwrapbench +all: clean fitblk example zwrapbench test: example fitblk example_zstd fitblk_zstd zwrapbench ./example @@ -34,11 +34,8 @@ test: example fitblk example_zstd fitblk_zstd zwrapbench ./zwrapbench -qb1e5 ../zstd_compression_format.md ./zwrapbench -qb1e5B1K ../zstd_compression_format.md -test_d: example_d - ./example_d - +#valgrindTest: ZSTDLIBRARY = $(ZSTDLIBDIR)/libzstd.so valgrindTest: VALGRIND = LD_LIBRARY_PATH=$(ZSTDLIBDIR) valgrind --track-origins=yes --leak-check=full --error-exitcode=1 -valgrindTest: ZSTDLIBRARY = $(ZSTDLIBDIR)/libzstd.so valgrindTest: clean example fitblk example_zstd fitblk_zstd zwrapbench @echo "\n ---- valgrind tests ----" $(VALGRIND) ./example @@ -83,5 +80,5 @@ $(ZSTDLIBDIR)/libzstd.so: $(MAKE) -C $(ZSTDLIBDIR) all clean: - -$(RM) $(ZLIBWRAPPER_PATH)/*.o $(EXAMPLE_PATH)/*.o *.o *.exe foo.gz example example_d example_zstd fitblk fitblk_zstd zwrapbench + -$(RM) $(ZLIBWRAPPER_PATH)/*.o $(EXAMPLE_PATH)/*.o *.o *.exe foo.gz example example_zstd fitblk fitblk_zstd zwrapbench @echo Cleaning completed diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index 7a825899c..76f075108 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -548,6 +548,7 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm, if (zwd == NULL || zwd->zbd == NULL) return Z_STREAM_ERROR; errorCode = ZSTD_initDStream_usingDict(zwd->zbd, dictionary, dictLength); if (ZSTD_isError(errorCode)) return ZWRAPD_finishWithError(zwd, strm, 0); + zwd->decompState = Z_NEED_DICT; if (strm->total_in == ZSTD_HEADERSIZE) { zwd->inBuffer.src = zwd->headerBuf; @@ -571,6 +572,7 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm, ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) { + ZWRAP_DCtx* zwd; int res; if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved) { LOG_WRAPPERD("- inflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); @@ -581,60 +583,79 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) if (strm->avail_in > 0) { size_t errorCode, srcSize; - ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state; - if (zwd == NULL) return Z_STREAM_ERROR; + zwd = (ZWRAP_DCtx*) strm->state; LOG_WRAPPERD("- inflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); + if (zwd == NULL) return Z_STREAM_ERROR; if (zwd->decompState == Z_STREAM_END) return Z_STREAM_END; - if (strm->total_in < ZLIB_HEADERSIZE) - { - srcSize = MIN(strm->avail_in, ZLIB_HEADERSIZE - strm->total_in); - memcpy(zwd->headerBuf+strm->total_in, strm->next_in, srcSize); - strm->total_in += srcSize; - strm->next_in += srcSize; - strm->avail_in -= srcSize; - if (strm->total_in < ZLIB_HEADERSIZE) return Z_OK; + if (strm->total_in < ZLIB_HEADERSIZE) { + if (strm->total_in == 0 && strm->avail_in >= ZLIB_HEADERSIZE) { + if (MEM_readLE32(strm->next_in) != ZSTD_MAGICNUMBER) { + if (zwd->windowBits) + errorCode = inflateInit2_(strm, zwd->windowBits, zwd->version, zwd->stream_size); + else + errorCode = inflateInit_(strm, zwd->version, zwd->stream_size); - if (MEM_readLE32(zwd->headerBuf) != ZSTD_MAGICNUMBER) { - z_stream strm2; - strm2.next_in = strm->next_in; - strm2.avail_in = strm->avail_in; - strm2.next_out = strm->next_out; - strm2.avail_out = strm->avail_out; + strm->reserved = 0; /* mark as zlib stream */ + errorCode = ZWRAP_freeDCtx(zwd); + if (ZSTD_isError(errorCode)) goto error; - if (zwd->windowBits) - errorCode = inflateInit2_(strm, zwd->windowBits, zwd->version, zwd->stream_size); - else - errorCode = inflateInit_(strm, zwd->version, zwd->stream_size); - LOG_WRAPPERD("ZLIB inflateInit errorCode=%d\n", (int)errorCode); - if (errorCode != Z_OK) return ZWRAPD_finishWithError(zwd, strm, (int)errorCode); + if (flush == Z_INFLATE_SYNC) res = inflateSync(strm); + else res = inflate(strm, flush); + LOG_WRAPPERD("- inflate3 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, res); + return res; + } + } else { + srcSize = MIN(strm->avail_in, ZLIB_HEADERSIZE - strm->total_in); + memcpy(zwd->headerBuf+strm->total_in, strm->next_in, srcSize); + strm->total_in += srcSize; + strm->next_in += srcSize; + strm->avail_in -= srcSize; + if (strm->total_in < ZLIB_HEADERSIZE) return Z_OK; - /* inflate header */ - strm->next_in = (unsigned char*)zwd->headerBuf; - strm->avail_in = ZLIB_HEADERSIZE; - strm->avail_out = 0; - errorCode = inflate(strm, Z_NO_FLUSH); - LOG_WRAPPERD("ZLIB inflate errorCode=%d strm->avail_in=%d\n", (int)errorCode, (int)strm->avail_in); - if (errorCode != Z_OK) return ZWRAPD_finishWithError(zwd, strm, (int)errorCode); - if (strm->avail_in > 0) goto error; + if (MEM_readLE32(zwd->headerBuf) != ZSTD_MAGICNUMBER) { + z_stream strm2; + strm2.next_in = strm->next_in; + strm2.avail_in = strm->avail_in; + strm2.next_out = strm->next_out; + strm2.avail_out = strm->avail_out; - strm->next_in = strm2.next_in; - strm->avail_in = strm2.avail_in; - strm->next_out = strm2.next_out; - strm->avail_out = strm2.avail_out; + if (zwd->windowBits) + errorCode = inflateInit2_(strm, zwd->windowBits, zwd->version, zwd->stream_size); + else + errorCode = inflateInit_(strm, zwd->version, zwd->stream_size); + LOG_WRAPPERD("ZLIB inflateInit errorCode=%d\n", (int)errorCode); + if (errorCode != Z_OK) return ZWRAPD_finishWithError(zwd, strm, (int)errorCode); - strm->reserved = 0; /* mark as zlib stream */ - errorCode = ZWRAP_freeDCtx(zwd); - if (ZSTD_isError(errorCode)) goto error; + /* inflate header */ + strm->next_in = (unsigned char*)zwd->headerBuf; + strm->avail_in = ZLIB_HEADERSIZE; + strm->avail_out = 0; + errorCode = inflate(strm, Z_NO_FLUSH); + LOG_WRAPPERD("ZLIB inflate errorCode=%d strm->avail_in=%d\n", (int)errorCode, (int)strm->avail_in); + if (errorCode != Z_OK) return ZWRAPD_finishWithError(zwd, strm, (int)errorCode); + if (strm->avail_in > 0) goto error; - if (flush == Z_INFLATE_SYNC) res = inflateSync(strm); - else res = inflate(strm, flush); - LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, res); - return res; + strm->next_in = strm2.next_in; + strm->avail_in = strm2.avail_in; + strm->next_out = strm2.next_out; + strm->avail_out = strm2.avail_out; + + strm->reserved = 0; /* mark as zlib stream */ + errorCode = ZWRAP_freeDCtx(zwd); + if (ZSTD_isError(errorCode)) goto error; + + if (flush == Z_INFLATE_SYNC) res = inflateSync(strm); + else res = inflate(strm, flush); + LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, res); + return res; + } } } + if (flush == Z_INFLATE_SYNC) { strm->msg = "inflateSync is not supported!"; goto error; } + if (!zwd->zbd) { zwd->zbd = ZSTD_createDStream_advanced(zwd->customMem); if (zwd->zbd == NULL) { LOG_WRAPPERD("ERROR: ZSTD_createDStream_advanced\n"); goto error; } @@ -642,31 +663,36 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) if (strm->total_in < ZSTD_HEADERSIZE) { - srcSize = MIN(strm->avail_in, ZSTD_HEADERSIZE - strm->total_in); - memcpy(zwd->headerBuf+strm->total_in, strm->next_in, srcSize); - strm->total_in += srcSize; - strm->next_in += srcSize; - strm->avail_in -= srcSize; - if (strm->total_in < ZSTD_HEADERSIZE) return Z_OK; + if (strm->total_in == 0 && strm->avail_in >= ZSTD_HEADERSIZE) { + if (zwd->decompState != Z_NEED_DICT) { + errorCode = ZSTD_initDStream(zwd->zbd); + if (ZSTD_isError(errorCode)) { LOG_WRAPPERD("ERROR: ZSTD_initDStream errorCode=%s\n", ZSTD_getErrorName(errorCode)); goto error; } + } + } else { + srcSize = MIN(strm->avail_in, ZSTD_HEADERSIZE - strm->total_in); + memcpy(zwd->headerBuf+strm->total_in, strm->next_in, srcSize); + strm->total_in += srcSize; + strm->next_in += srcSize; + strm->avail_in -= srcSize; + if (strm->total_in < ZSTD_HEADERSIZE) return Z_OK; - errorCode = ZSTD_initDStream(zwd->zbd); - if (ZSTD_isError(errorCode)) { LOG_WRAPPERD("ERROR: ZSTD_initDStream errorCode=%s\n", ZSTD_getErrorName(errorCode)); goto error; } + errorCode = ZSTD_initDStream(zwd->zbd); + if (ZSTD_isError(errorCode)) { LOG_WRAPPERD("ERROR: ZSTD_initDStream errorCode=%s\n", ZSTD_getErrorName(errorCode)); goto error; } - if (flush == Z_INFLATE_SYNC) { strm->msg = "inflateSync is not supported!"; goto error; } - - zwd->inBuffer.src = zwd->headerBuf; - zwd->inBuffer.size = ZSTD_HEADERSIZE; - zwd->inBuffer.pos = 0; - zwd->outBuffer.dst = strm->next_out; - zwd->outBuffer.size = 0; - zwd->outBuffer.pos = 0; - errorCode = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer); - LOG_WRAPPERD("inflate ZSTD_decompressStream1 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)zwd->inBuffer.size, (int)zwd->outBuffer.size); - if (ZSTD_isError(errorCode)) { - LOG_WRAPPERD("ERROR: ZSTD_decompressStream1 %s\n", ZSTD_getErrorName(errorCode)); - goto error; + zwd->inBuffer.src = zwd->headerBuf; + zwd->inBuffer.size = ZSTD_HEADERSIZE; + zwd->inBuffer.pos = 0; + zwd->outBuffer.dst = strm->next_out; + zwd->outBuffer.size = 0; + zwd->outBuffer.pos = 0; + errorCode = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer); + LOG_WRAPPERD("inflate ZSTD_decompressStream1 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)zwd->inBuffer.size, (int)zwd->outBuffer.size); + if (ZSTD_isError(errorCode)) { + LOG_WRAPPERD("ERROR: ZSTD_decompressStream1 %s\n", ZSTD_getErrorName(errorCode)); + goto error; + } + if (zwd->inBuffer.pos != zwd->inBuffer.size) return ZWRAPD_finishWithError(zwd, strm, 0); /* not consumed */ } - if (zwd->inBuffer.pos != zwd->inBuffer.size) return ZWRAPD_finishWithError(zwd, strm, 0); /* not consumed */ } zwd->inBuffer.src = strm->next_in; @@ -694,13 +720,12 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) zwd->decompState = Z_STREAM_END; return Z_STREAM_END; } - goto finish; -error: - return ZWRAPD_finishWithError(zwd, strm, 0); } -finish: LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, Z_OK); return Z_OK; + +error: + return ZWRAPD_finishWithError(zwd, strm, 0); } From e82c811946fff7bde1aa7e82f0ad4ca2f3f7220e Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 23 Sep 2016 16:20:13 +0200 Subject: [PATCH 33/43] updated zlibWrapper\README.md --- .travis.yml | 2 +- zlibWrapper/README.md | 32 +++++++++++++++++++++++++++----- zlibWrapper/zstd_zlibwrapper.c | 6 ++++-- 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 58728d89c..116519645 100644 --- a/.travis.yml +++ b/.travis.yml @@ -59,7 +59,7 @@ matrix: - os: linux dist: trusty sudo: required - env: PLATFORM="Ubuntu 14.04" CMD="make -C lib all && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest" + env: PLATFORM="Ubuntu 14.04" CMD='make -C lib all && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest' addons: apt: packages: diff --git a/zlibWrapper/README.md b/zlibWrapper/README.md index 88174fbec..2ce9060dc 100644 --- a/zlibWrapper/README.md +++ b/zlibWrapper/README.md @@ -23,10 +23,10 @@ Let's assume that your project that uses zlib is compiled with: To compile the zstd wrapper with your project you have to do the following: - change all references with ```#include "zlib.h"``` to ```#include "zstd_zlibwrapper.h"``` -- compile your project with zlib_wrapper.c and a static or dynamic zstd library +- compile your project with `zstd_zlibwrapper.c` and a static or dynamic zstd library The linking should be changed to: -```gcc project.o zlib_wrapper.o -lz -lzstd``` +```gcc project.o zstd_zlibwrapper.o -lz -lzstd``` #### Enabling zstd compression within your project @@ -35,7 +35,29 @@ After embedding the zstd wrapper within your project the zstd library is turned Your project should work as before with zlib. There are two options to enable zstd compression: - compilation with ```-DZWRAP_USE_ZSTD=1``` (or using ```#define ZWRAP_USE_ZSTD 1``` before ```#include "zstd_zlibwrapper.h"```) - using the ```void ZWRAP_useZSTDcompression(int turn_on)``` function (declared in ```#include "zstd_zlibwrapper.h"```) -There is no switch for zstd decompression because zlib and zstd streams are automatically detected and decompressed using a proper library. + +During decompression zlib and zstd streams are automatically detected and decompressed using a proper library. +This behavior can be changed using ZWRAP_setDecompressionType(ZWRAP_FORCE_ZLIB) what will make zlib decompression slightly faster. + + +#### Performace of Zstandard wrapper for zlib + +The zstd distribution contains a tool called `zwrapbench` which can measure speed and ratio of zlib, zstd and the wrapper. +The benchmark is conducted using given filenames or synthetic data if filenames are not provided. +The files are read into memory and joined together. +It makes benchmark more precise as it eliminates I/O overhead. +Many filenames can be supplied as multiple parameters, parameters with wildcards or names of directories can be used as parameters with the -r option. +One can select compression levels starting from -b and ending with -e. The -i parameter selects minimal time used for each of tested levels. +With -B option bigger files can be divided into smaller, independently compressed blocks. +The benchmark tool can be compiled with `make zwrapbench` using [zlibWrapper/Makefile](this Makefile). + + +#### Improving speed of streaming compression + +Zstandard compression can be improved by providing size of source data to compressor. By default compressor assumes that files are bigger than 256 KB but it can hurt compression speed on smaller files. +The zstd wrapper provides the `int ZWRAP_setPledgedSrcSize(z_streamp strm, unsigned long long pledgedSrcSize)` function that allows to change a pledged source size for a given compression stream. +The function should be called just after deflateInit(). The function is only helpful when data is compressed in blocks. There will be no change in case of deflateInit() immediately followed by deflate(strm, Z_FINISH) +as this case is automatically detected. #### Example @@ -52,7 +74,7 @@ after inflateSync(): hello, hello! inflate with dictionary: hello, hello! ``` Then we have changed ```#include "zlib.h"``` to ```#include "zstd_zlibwrapper.h"```, compiled the [example.c](examples/example.c) file -with ```-DZWRAP_USE_ZSTD=1``` and linked with additional ```zlib_wrapper.o -lzstd```. +with ```-DZWRAP_USE_ZSTD=1``` and linked with additional ```zstd_zlibwrapper.o -lzstd```. We were forced to turn off the following functions: ```test_gzio```, ```test_flush```, ```test_sync``` which use currently unsupported features. After running it shows the following results: ``` @@ -66,7 +88,7 @@ The script used for compilation can be found at [zlibWrapper/Makefile](Makefile) #### Compatibility issues -After enabling zstd compression not all native zlib functions are supported. When calling unsupported methods they print error message and return an error value. +After enabling zstd compression not all native zlib functions are supported. When calling unsupported methods they put error message into strm->msg and return Z_STREAM_ERROR. Supported methods: - deflateInit diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index 76f075108..99e2e2e52 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -581,7 +581,9 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) return res; } - if (strm->avail_in > 0) { + if (strm->avail_in <= 0) return Z_OK; + + { size_t errorCode, srcSize; zwd = (ZWRAP_DCtx*) strm->state; LOG_WRAPPERD("- inflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); @@ -691,7 +693,7 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) LOG_WRAPPERD("ERROR: ZSTD_decompressStream1 %s\n", ZSTD_getErrorName(errorCode)); goto error; } - if (zwd->inBuffer.pos != zwd->inBuffer.size) return ZWRAPD_finishWithError(zwd, strm, 0); /* not consumed */ + if (zwd->inBuffer.pos != zwd->inBuffer.size) goto error; /* not consumed */ } } From 2bb83e827144879c40dc2e5291ca80f28e2effa2 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 23 Sep 2016 18:59:53 +0200 Subject: [PATCH 34/43] zlibWrapper\README.md: Reusing contexts --- zlibWrapper/README.md | 44 +++++++++++++++++++++++++------ zlibWrapper/examples/zwrapbench.c | 4 +-- zlibWrapper/zstd_zlibwrapper.c | 8 +++--- 3 files changed, 43 insertions(+), 13 deletions(-) diff --git a/zlibWrapper/README.md b/zlibWrapper/README.md index 2ce9060dc..5747e38fa 100644 --- a/zlibWrapper/README.md +++ b/zlibWrapper/README.md @@ -37,29 +37,57 @@ Your project should work as before with zlib. There are two options to enable zs - using the ```void ZWRAP_useZSTDcompression(int turn_on)``` function (declared in ```#include "zstd_zlibwrapper.h"```) During decompression zlib and zstd streams are automatically detected and decompressed using a proper library. -This behavior can be changed using ZWRAP_setDecompressionType(ZWRAP_FORCE_ZLIB) what will make zlib decompression slightly faster. +This behavior can be changed using `ZWRAP_setDecompressionType(ZWRAP_FORCE_ZLIB)` what will make zlib decompression slightly faster. -#### Performace of Zstandard wrapper for zlib +#### The measurement of performace of Zstandard wrapper for zlib -The zstd distribution contains a tool called `zwrapbench` which can measure speed and ratio of zlib, zstd and the wrapper. +The zstd distribution contains a tool called `zwrapbench` which can measure speed and ratio of zlib, zstd, and the wrapper. The benchmark is conducted using given filenames or synthetic data if filenames are not provided. The files are read into memory and joined together. It makes benchmark more precise as it eliminates I/O overhead. Many filenames can be supplied as multiple parameters, parameters with wildcards or names of directories can be used as parameters with the -r option. -One can select compression levels starting from -b and ending with -e. The -i parameter selects minimal time used for each of tested levels. -With -B option bigger files can be divided into smaller, independently compressed blocks. -The benchmark tool can be compiled with `make zwrapbench` using [zlibWrapper/Makefile](this Makefile). +One can select compression levels starting from `-b` and ending with `-e`. The `-i` parameter selects minimal time used for each of tested levels. +With `-B` option bigger files can be divided into smaller, independently compressed blocks. +The benchmark tool can be compiled with `make zwrapbench` using [zlibWrapper/Makefile](Makefile). #### Improving speed of streaming compression Zstandard compression can be improved by providing size of source data to compressor. By default compressor assumes that files are bigger than 256 KB but it can hurt compression speed on smaller files. -The zstd wrapper provides the `int ZWRAP_setPledgedSrcSize(z_streamp strm, unsigned long long pledgedSrcSize)` function that allows to change a pledged source size for a given compression stream. -The function should be called just after deflateInit(). The function is only helpful when data is compressed in blocks. There will be no change in case of deflateInit() immediately followed by deflate(strm, Z_FINISH) +The zstd wrapper provides the `ZWRAP_setPledgedSrcSize()` function that allows to change a pledged source size for a given compression stream. +The function should be called just after `deflateInit()`. The function is only helpful when data is compressed in blocks. There will be no change in case of `deflateInit()` immediately followed by `deflate(strm, Z_FINISH)` as this case is automatically detected. +#### Reusing contexts + +The ordinary zlib compression of two files/streams: +- for the 1st file calls `deflateInit`, `deflate`, `...`, `deflate`, `defalateEnd` +- for the 2nd file calls `deflateInit`, `deflate`, `...`, `deflate`, `defalateEnd` + +The speed of compression can be improved with reusing a context with following steps: +- initialize a context with `deflateInit` +- for the 1st file call `deflate`, `...`, `deflate` +- for the 2nd file call `deflateReset`, `deflate`, `...`, `deflate` +- free a context with `deflateEnd` + +We made experiments using `zwrapbench` with zstd and zlib compression (both at level 3) in 4 KB blocks. +The input data was decompressed git repository downloaded from https://github.com/git/git/archive/master.zip +The table below shows that reusing contexts has minor influnce on zlib but for zstd it gives 15% better compression speed and 6% better decompression speed. + +| Compression type | Compression | Decompress.| Compr. size | Ratio | +| ------------------------------------------------- | ------------| -----------| ----------- | ----- | +| zlib 1.2.8 | 54.77 MB/s | 176.2 MB/s | 8928115 | 2.910 | +| zlib 1.2.8 not reusing a context | 54.98 MB/s | 174.6 MB/s | 8928115 | 2.910 | +| zlib 1.2.8 using zlibWrapper | 55.16 MB/s | 176.8 MB/s | 8928115 | 2.910 | +| zlib 1.2.8 with zlibWrapper not reusing a context | 54.11 MB/s | 174.5 MB/s | 8928115 | 2.910 | +| zstd 1.1.0 using ZSTD_CCtx | 108.03 MB/s | 319.8 MB/s | 8962336 | 2.899 | +| zstd 1.1.0 using ZSTD_CStream | 107.34 MB/s | 307.3 MB/s | 8981368 | 2.893 | +| zstd 1.1.0 using zlibWrapper | 107.52 MB/s | 297.3 MB/s | 8981368 | 2.893 | +| zstd 1.1.0 with zlibWrapper not reusing a context | 91.45 MB/s | 279.8 MB/s | 8981368 | 2.893 | + + #### Example We have take the file ```test/example.c``` from [the zlib library distribution](http://zlib.net/) and copied it to [zlibWrapper/examples/example.c](examples/example.c). After compilation and execution it shows the following results: diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c index b925cf7cb..f0ae8cd97 100644 --- a/zlibWrapper/examples/zwrapbench.c +++ b/zlibWrapper/examples/zwrapbench.c @@ -292,7 +292,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, /* if (ZWRAP_isUsingZSTDcompression()) { ret = ZWRAP_setPledgedSrcSize(&def, avgSize); if (ret != Z_OK) EXM_THROW(1, "ZWRAP_setPledgedSrcSize failure"); - }*/ + } */ do { U32 blockNb; for (blockNb=0; blockNbzbc == NULL) { @@ -136,7 +137,7 @@ int ZWRAP_initializeCStream(ZWRAP_CCtx* zwc, unsigned long long pledgedSrcSize) if (!pledgedSrcSize) pledgedSrcSize = zwc->pledgedSrcSize; { ZSTD_parameters const params = ZSTD_getParams(zwc->compressionLevel, pledgedSrcSize, 0); size_t errorCode; - LOG_WRAPPERC("windowLog=%d chainLog=%d hashLog=%d searchLog=%d searchLength=%d strategy=%d\n", params.cParams.windowLog, params.cParams.chainLog, params.cParams.hashLog, params.cParams.searchLog, params.cParams.searchLength, params.cParams.strategy); + LOG_WRAPPERC("pledgedSrcSize=%d windowLog=%d chainLog=%d hashLog=%d searchLog=%d searchLength=%d strategy=%d\n", (int)pledgedSrcSize, params.cParams.windowLog, params.cParams.chainLog, params.cParams.hashLog, params.cParams.searchLog, params.cParams.searchLength, params.cParams.strategy); errorCode = ZSTD_initCStream_advanced(zwc->zbc, NULL, 0, params, pledgedSrcSize); if (ZSTD_isError(errorCode)) return Z_STREAM_ERROR; } } @@ -219,6 +220,7 @@ ZEXTERN int ZEXPORT z_deflateReset OF((z_streamp strm)) strm->total_in = 0; strm->total_out = 0; + strm->adler = 0; return Z_OK; } @@ -260,7 +262,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) } zwc = (ZWRAP_CCtx*) strm->state; - if (zwc == NULL) return Z_STREAM_ERROR; + if (zwc == NULL) { LOG_WRAPPERC("zwc == NULL\n"); return Z_STREAM_ERROR; } if (zwc->zbc == NULL) { int res = ZWRAP_initializeCStream(zwc, (flush == Z_FINISH) ? strm->avail_in : 0); @@ -268,7 +270,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) } else { if (strm->total_in == 0) { size_t const errorCode = ZSTD_resetCStream(zwc->zbc, (flush == Z_FINISH) ? strm->avail_in : zwc->pledgedSrcSize); - if (ZSTD_isError(errorCode)) return ZWRAPC_finishWithError(zwc, strm, 0); + if (ZSTD_isError(errorCode)) { LOG_WRAPPERC("ERROR: ZSTD_resetCStream errorCode=%s\n", ZSTD_getErrorName(errorCode)); return ZWRAPC_finishWithError(zwc, strm, 0); } } } From cd2f6b680bab8882d663d29fc3024fb54f9acbdc Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 23 Sep 2016 20:03:17 +0200 Subject: [PATCH 35/43] zlibWrapper\README.md: minor tweaks --- zlibWrapper/README.md | 25 ++++++++++++++----------- zlibWrapper/zstd_zlibwrapper.c | 27 ++++++++++----------------- 2 files changed, 24 insertions(+), 28 deletions(-) diff --git a/zlibWrapper/README.md b/zlibWrapper/README.md index 5747e38fa..70cfb0e2b 100644 --- a/zlibWrapper/README.md +++ b/zlibWrapper/README.md @@ -54,37 +54,40 @@ The benchmark tool can be compiled with `make zwrapbench` using [zlibWrapper/Mak #### Improving speed of streaming compression -Zstandard compression can be improved by providing size of source data to compressor. By default compressor assumes that files are bigger than 256 KB but it can hurt compression speed on smaller files. +During streaming compression the compressor never knows how big is data to compress. +Zstandard compression can be improved by providing size of source data to the compressor. By default streaming compressor assumes that data is bigger than 256 KB but it can hurt compression speed on smaller data. The zstd wrapper provides the `ZWRAP_setPledgedSrcSize()` function that allows to change a pledged source size for a given compression stream. -The function should be called just after `deflateInit()`. The function is only helpful when data is compressed in blocks. There will be no change in case of `deflateInit()` immediately followed by `deflate(strm, Z_FINISH)` +The function will change zstd compression parameters what may improve compression speed and/or ratio. +It should be called just after `deflateInit()`. The function is only helpful when data is compressed in blocks. There will be no change in case of `deflateInit()` immediately followed by `deflate(strm, Z_FINISH)` as this case is automatically detected. #### Reusing contexts -The ordinary zlib compression of two files/streams: +The ordinary zlib compression of two files/streams allocates two contexts: - for the 1st file calls `deflateInit`, `deflate`, `...`, `deflate`, `defalateEnd` - for the 2nd file calls `deflateInit`, `deflate`, `...`, `deflate`, `defalateEnd` -The speed of compression can be improved with reusing a context with following steps: -- initialize a context with `deflateInit` +The speed of compression can be improved with reusing a single context with following steps: +- initialize the context with `deflateInit` - for the 1st file call `deflate`, `...`, `deflate` - for the 2nd file call `deflateReset`, `deflate`, `...`, `deflate` -- free a context with `deflateEnd` +- free the context with `deflateEnd` -We made experiments using `zwrapbench` with zstd and zlib compression (both at level 3) in 4 KB blocks. -The input data was decompressed git repository downloaded from https://github.com/git/git/archive/master.zip -The table below shows that reusing contexts has minor influnce on zlib but for zstd it gives 15% better compression speed and 6% better decompression speed. +To check the difference we made experiments using `zwrapbench` with zstd and zlib compression (both at level 3) with 4 KB blocks. +The input data was git repository downloaded from https://github.com/git/git/archive/master.zip and converted to uncompressed tarball. +The table below shows that reusing contexts has a minor influence on zlib but it gives improvement for zstd. +In our example (the last 2 lines) is gives 15% better compression speed and 6% better decompression speed. | Compression type | Compression | Decompress.| Compr. size | Ratio | | ------------------------------------------------- | ------------| -----------| ----------- | ----- | | zlib 1.2.8 | 54.77 MB/s | 176.2 MB/s | 8928115 | 2.910 | | zlib 1.2.8 not reusing a context | 54.98 MB/s | 174.6 MB/s | 8928115 | 2.910 | -| zlib 1.2.8 using zlibWrapper | 55.16 MB/s | 176.8 MB/s | 8928115 | 2.910 | +| zlib 1.2.8 with zlibWrapper and reusing a context | 55.16 MB/s | 176.8 MB/s | 8928115 | 2.910 | | zlib 1.2.8 with zlibWrapper not reusing a context | 54.11 MB/s | 174.5 MB/s | 8928115 | 2.910 | | zstd 1.1.0 using ZSTD_CCtx | 108.03 MB/s | 319.8 MB/s | 8962336 | 2.899 | | zstd 1.1.0 using ZSTD_CStream | 107.34 MB/s | 307.3 MB/s | 8981368 | 2.893 | -| zstd 1.1.0 using zlibWrapper | 107.52 MB/s | 297.3 MB/s | 8981368 | 2.893 | +| zstd 1.1.0 with zlibWrapper and reusing a context | 107.52 MB/s | 297.3 MB/s | 8981368 | 2.893 | | zstd 1.1.0 with zlibWrapper not reusing a context | 91.45 MB/s | 279.8 MB/s | 8981368 | 2.893 | diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index 63dc44ff3..e4906a277 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -25,20 +25,8 @@ #define LOG_WRAPPERC(...) /* printf(__VA_ARGS__) */ #define LOG_WRAPPERD(...) /* printf(__VA_ARGS__) */ - -#define FINISH_WITH_GZ_ERR(msg) { \ - (void)msg; \ - return Z_STREAM_ERROR; \ -} - -#define FINISH_WITH_NULL_ERR(msg) { \ - (void)msg; \ - return NULL; \ -} - -const char * zstdVersion(void) { return ZSTD_VERSION_STRING; } - -ZEXTERN const char * ZEXPORT z_zlibVersion OF((void)) { return zlibVersion(); } +#define FINISH_WITH_GZ_ERR(msg) { (void)msg; return Z_STREAM_ERROR; } +#define FINISH_WITH_NULL_ERR(msg) { (void)msg; return NULL; } @@ -62,6 +50,11 @@ ZWRAP_decompress_type ZWRAP_getDecompressionType(void) { return g_ZWRAPdecompres +const char * zstdVersion(void) { return ZSTD_VERSION_STRING; } + +ZEXTERN const char * ZEXPORT z_zlibVersion OF((void)) { return zlibVersion(); } + + static void* ZWRAP_allocFunction(void* opaque, size_t size) { @@ -257,7 +250,6 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) int res; LOG_WRAPPERC("- deflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); res = deflate(strm, flush); - LOG_WRAPPERC("- deflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); return res; } @@ -397,6 +389,7 @@ void ZWRAP_initDCtx(ZWRAP_DCtx* zwd) zwd->outBuffer.size = 0; } + ZWRAP_DCtx* ZWRAP_createDCtx(z_streamp strm) { ZWRAP_DCtx* zwd; @@ -585,8 +578,7 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) if (strm->avail_in <= 0) return Z_OK; - { - size_t errorCode, srcSize; + { size_t errorCode, srcSize; zwd = (ZWRAP_DCtx*) strm->state; LOG_WRAPPERD("- inflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); @@ -762,6 +754,7 @@ ZEXTERN int ZEXPORT z_inflateSync OF((z_streamp strm)) + /* Advanced compression functions */ ZEXTERN int ZEXPORT z_deflateCopy OF((z_streamp dest, z_streamp source)) From 611cd094d11d9b1881a299fa535b96f9966788b9 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 23 Sep 2016 21:14:37 +0200 Subject: [PATCH 36/43] typo in pzstd --- contrib/pzstd/Options.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/pzstd/Options.cpp b/contrib/pzstd/Options.cpp index 5562ee18f..b503def15 100644 --- a/contrib/pzstd/Options.cpp +++ b/contrib/pzstd/Options.cpp @@ -103,7 +103,7 @@ void usage() { std::fprintf(stderr, " -V, --version : display version number and exit\n"); std::fprintf(stderr, " -v, --verbose : verbose mode; specify multiple times to increase log level (default:2)\n"); std::fprintf(stderr, " -q, --quiet : suppress warnings; specify twice to suppress errors too\n"); - std::fprintf(stderr, " -c, --stdout : force wrtie to standard output, even if it is the console\n"); + std::fprintf(stderr, " -c, --stdout : force write to standard output, even if it is the console\n"); #ifdef UTIL_HAS_CREATEFILELIST std::fprintf(stderr, " -r : operate recursively on directories\n"); #endif From 2fb7e6b15d93ff8870600a946be2acb2d8d43cc6 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 23 Sep 2016 21:32:16 +0200 Subject: [PATCH 37/43] zlibWrapper\README.md: reordering --- zlibWrapper/README.md | 56 +++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/zlibWrapper/README.md b/zlibWrapper/README.md index 70cfb0e2b..6a7cdd2fa 100644 --- a/zlibWrapper/README.md +++ b/zlibWrapper/README.md @@ -40,6 +40,33 @@ During decompression zlib and zstd streams are automatically detected and decomp This behavior can be changed using `ZWRAP_setDecompressionType(ZWRAP_FORCE_ZLIB)` what will make zlib decompression slightly faster. +#### Example +We have take the file ```test/example.c``` from [the zlib library distribution](http://zlib.net/) and copied it to [zlibWrapper/examples/example.c](examples/example.c). +After compilation and execution it shows the following results: +``` +zlib version 1.2.8 = 0x1280, compile flags = 0x65 +uncompress(): hello, hello! +gzread(): hello, hello! +gzgets() after gzseek: hello! +inflate(): hello, hello! +large_inflate(): OK +after inflateSync(): hello, hello! +inflate with dictionary: hello, hello! +``` +Then we have changed ```#include "zlib.h"``` to ```#include "zstd_zlibwrapper.h"```, compiled the [example.c](examples/example.c) file +with ```-DZWRAP_USE_ZSTD=1``` and linked with additional ```zstd_zlibwrapper.o -lzstd```. +We were forced to turn off the following functions: ```test_gzio```, ```test_flush```, ```test_sync``` which use currently unsupported features. +After running it shows the following results: +``` +zlib version 1.2.8 = 0x1280, compile flags = 0x65 +uncompress(): hello, hello! +inflate(): hello, hello! +large_inflate(): OK +inflate with dictionary: hello, hello! +``` +The script used for compilation can be found at [zlibWrapper/Makefile](Makefile). + + #### The measurement of performace of Zstandard wrapper for zlib The zstd distribution contains a tool called `zwrapbench` which can measure speed and ratio of zlib, zstd, and the wrapper. @@ -77,7 +104,7 @@ The speed of compression can be improved with reusing a single context with foll To check the difference we made experiments using `zwrapbench` with zstd and zlib compression (both at level 3) with 4 KB blocks. The input data was git repository downloaded from https://github.com/git/git/archive/master.zip and converted to uncompressed tarball. The table below shows that reusing contexts has a minor influence on zlib but it gives improvement for zstd. -In our example (the last 2 lines) is gives 15% better compression speed and 6% better decompression speed. +In our example (the last 2 lines) it gives 15% better compression speed and 6% better decompression speed. | Compression type | Compression | Decompress.| Compr. size | Ratio | | ------------------------------------------------- | ------------| -----------| ----------- | ----- | @@ -91,33 +118,6 @@ In our example (the last 2 lines) is gives 15% better compression speed and 6% b | zstd 1.1.0 with zlibWrapper not reusing a context | 91.45 MB/s | 279.8 MB/s | 8981368 | 2.893 | -#### Example -We have take the file ```test/example.c``` from [the zlib library distribution](http://zlib.net/) and copied it to [zlibWrapper/examples/example.c](examples/example.c). -After compilation and execution it shows the following results: -``` -zlib version 1.2.8 = 0x1280, compile flags = 0x65 -uncompress(): hello, hello! -gzread(): hello, hello! -gzgets() after gzseek: hello! -inflate(): hello, hello! -large_inflate(): OK -after inflateSync(): hello, hello! -inflate with dictionary: hello, hello! -``` -Then we have changed ```#include "zlib.h"``` to ```#include "zstd_zlibwrapper.h"```, compiled the [example.c](examples/example.c) file -with ```-DZWRAP_USE_ZSTD=1``` and linked with additional ```zstd_zlibwrapper.o -lzstd```. -We were forced to turn off the following functions: ```test_gzio```, ```test_flush```, ```test_sync``` which use currently unsupported features. -After running it shows the following results: -``` -zlib version 1.2.8 = 0x1280, compile flags = 0x65 -uncompress(): hello, hello! -inflate(): hello, hello! -large_inflate(): OK -inflate with dictionary: hello, hello! -``` -The script used for compilation can be found at [zlibWrapper/Makefile](Makefile). - - #### Compatibility issues After enabling zstd compression not all native zlib functions are supported. When calling unsupported methods they put error message into strm->msg and return Z_STREAM_ERROR. From cbd7bdca1ecd6961a082b64427c601270226b580 Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 26 Sep 2016 20:41:52 +0200 Subject: [PATCH 38/43] improved zwrapbench tests --- zlibWrapper/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index ea025b951..07a2490c0 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -31,8 +31,8 @@ test: example fitblk example_zstd fitblk_zstd zwrapbench ./fitblk 40960 <../zstd_compression_format.md ./fitblk_zstd 10240 <../zstd_compression_format.md ./fitblk_zstd 40960 <../zstd_compression_format.md - ./zwrapbench -qb1e5 ../zstd_compression_format.md ./zwrapbench -qb1e5B1K ../zstd_compression_format.md + ./zwrapbench -qb1e5r ../lib ../programs ../tests #valgrindTest: ZSTDLIBRARY = $(ZSTDLIBDIR)/libzstd.so valgrindTest: VALGRIND = LD_LIBRARY_PATH=$(ZSTDLIBDIR) valgrind --track-origins=yes --leak-check=full --error-exitcode=1 @@ -44,8 +44,8 @@ valgrindTest: clean example fitblk example_zstd fitblk_zstd zwrapbench $(VALGRIND) ./fitblk 40960 <../zstd_compression_format.md $(VALGRIND) ./fitblk_zstd 10240 <../zstd_compression_format.md $(VALGRIND) ./fitblk_zstd 40960 <../zstd_compression_format.md - $(VALGRIND) ./zwrapbench -qb1e5 ../zstd_compression_format.md $(VALGRIND) ./zwrapbench -qb1e5B1K ../zstd_compression_format.md + $(VALGRIND) ./zwrapbench -qb1e5 ../lib ../programs ../tests .c.o: $(CC) $(CFLAGS) -c -o $@ $< From 67a1f4d72af0749f43eb0f98ce975f989c2624fb Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 26 Sep 2016 20:49:18 +0200 Subject: [PATCH 39/43] improved behavior of deflateReset --- zlibWrapper/zstd_zlibwrapper.c | 53 ++++++++++++++++++++-------------- zlibWrapper/zstd_zlibwrapper.h | 4 ++- 2 files changed, 35 insertions(+), 22 deletions(-) diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index e4906a277..766570f2b 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -20,7 +20,7 @@ #define Z_INFLATE_SYNC 8 #define ZLIB_HEADERSIZE 4 #define ZSTD_HEADERSIZE ZSTD_frameHeaderSize_min -#define ZWRAP_DEFAULT_CLEVEL 5 /* Z_DEFAULT_COMPRESSION is translated to ZWRAP_DEFAULT_CLEVEL for zstd */ +#define ZWRAP_DEFAULT_CLEVEL 3 /* Z_DEFAULT_COMPRESSION is translated to ZWRAP_DEFAULT_CLEVEL for zstd */ #define LOG_WRAPPERC(...) /* printf(__VA_ARGS__) */ #define LOG_WRAPPERD(...) /* printf(__VA_ARGS__) */ @@ -82,6 +82,7 @@ typedef struct { z_stream allocFunc; /* copy of zalloc, zfree, opaque */ ZSTD_inBuffer inBuffer; ZSTD_outBuffer outBuffer; + int comprState; unsigned long long pledgedSrcSize; } ZWRAP_CCtx; @@ -118,22 +119,17 @@ ZWRAP_CCtx* ZWRAP_createCCtx(z_streamp strm) } -int ZWRAP_initializeCStream(ZWRAP_CCtx* zwc, unsigned long long pledgedSrcSize) +int ZWRAP_initializeCStream(ZWRAP_CCtx* zwc, const void* dict, size_t dictSize, unsigned long long pledgedSrcSize) { LOG_WRAPPERC("- ZWRAP_initializeCStream=%p\n", zwc); - if (zwc == NULL) return Z_STREAM_ERROR; - - if (zwc->zbc == NULL) { - zwc->zbc = ZSTD_createCStream_advanced(zwc->customMem); - if (zwc->zbc == NULL) return Z_STREAM_ERROR; - - if (!pledgedSrcSize) pledgedSrcSize = zwc->pledgedSrcSize; - { ZSTD_parameters const params = ZSTD_getParams(zwc->compressionLevel, pledgedSrcSize, 0); - size_t errorCode; - LOG_WRAPPERC("pledgedSrcSize=%d windowLog=%d chainLog=%d hashLog=%d searchLog=%d searchLength=%d strategy=%d\n", (int)pledgedSrcSize, params.cParams.windowLog, params.cParams.chainLog, params.cParams.hashLog, params.cParams.searchLog, params.cParams.searchLength, params.cParams.strategy); - errorCode = ZSTD_initCStream_advanced(zwc->zbc, NULL, 0, params, pledgedSrcSize); - if (ZSTD_isError(errorCode)) return Z_STREAM_ERROR; } - } + if (zwc == NULL || zwc->zbc == NULL) return Z_STREAM_ERROR; + + if (!pledgedSrcSize) pledgedSrcSize = zwc->pledgedSrcSize; + { ZSTD_parameters const params = ZSTD_getParams(zwc->compressionLevel, pledgedSrcSize, dictSize); + size_t errorCode; + LOG_WRAPPERC("pledgedSrcSize=%d windowLog=%d chainLog=%d hashLog=%d searchLog=%d searchLength=%d strategy=%d\n", (int)pledgedSrcSize, params.cParams.windowLog, params.cParams.chainLog, params.cParams.hashLog, params.cParams.searchLog, params.cParams.searchLength, params.cParams.strategy); + errorCode = ZSTD_initCStream_advanced(zwc->zbc, dict, dictSize, params, pledgedSrcSize); + if (ZSTD_isError(errorCode)) return Z_STREAM_ERROR; } return Z_OK; } @@ -214,6 +210,10 @@ ZEXTERN int ZEXPORT z_deflateReset OF((z_streamp strm)) strm->total_in = 0; strm->total_out = 0; strm->adler = 0; + + { ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state; + if (zwc) zwc->comprState = 0; + } return Z_OK; } @@ -231,11 +231,13 @@ ZEXTERN int ZEXPORT z_deflateSetDictionary OF((z_streamp strm, LOG_WRAPPERC("- deflateSetDictionary level=%d\n", (int)zwc->compressionLevel); if (!zwc) return Z_STREAM_ERROR; if (zwc->zbc == NULL) { - int res = ZWRAP_initializeCStream(zwc, 0); + int res; + zwc->zbc = ZSTD_createCStream_advanced(zwc->customMem); + if (zwc->zbc == NULL) return ZWRAPC_finishWithError(zwc, strm, res); + res = ZWRAP_initializeCStream(zwc, dictionary, dictLength, 0); if (res != Z_OK) return ZWRAPC_finishWithError(zwc, strm, res); + zwc->comprState = Z_NEED_DICT; } - { size_t const errorCode = ZSTD_initCStream_usingDict(zwc->zbc, dictionary, dictLength, zwc->compressionLevel); - if (ZSTD_isError(errorCode)) return ZWRAPC_finishWithError(zwc, strm, 0); } } return Z_OK; @@ -257,12 +259,20 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) if (zwc == NULL) { LOG_WRAPPERC("zwc == NULL\n"); return Z_STREAM_ERROR; } if (zwc->zbc == NULL) { - int res = ZWRAP_initializeCStream(zwc, (flush == Z_FINISH) ? strm->avail_in : 0); + int res; + zwc->zbc = ZSTD_createCStream_advanced(zwc->customMem); + if (zwc->zbc == NULL) return ZWRAPC_finishWithError(zwc, strm, res); + res = ZWRAP_initializeCStream(zwc, NULL, 0, (flush == Z_FINISH) ? strm->avail_in : 0); if (res != Z_OK) return ZWRAPC_finishWithError(zwc, strm, res); } else { if (strm->total_in == 0) { - size_t const errorCode = ZSTD_resetCStream(zwc->zbc, (flush == Z_FINISH) ? strm->avail_in : zwc->pledgedSrcSize); - if (ZSTD_isError(errorCode)) { LOG_WRAPPERC("ERROR: ZSTD_resetCStream errorCode=%s\n", ZSTD_getErrorName(errorCode)); return ZWRAPC_finishWithError(zwc, strm, 0); } + if (zwc->comprState == Z_NEED_DICT) { + size_t const errorCode = ZSTD_resetCStream(zwc->zbc, (flush == Z_FINISH) ? strm->avail_in : zwc->pledgedSrcSize); + if (ZSTD_isError(errorCode)) { LOG_WRAPPERC("ERROR: ZSTD_resetCStream errorCode=%s\n", ZSTD_getErrorName(errorCode)); return ZWRAPC_finishWithError(zwc, strm, 0); } + } else { + int res = ZWRAP_initializeCStream(zwc, NULL, 0, (flush == Z_FINISH) ? strm->avail_in : 0); + if (res != Z_OK) return ZWRAPC_finishWithError(zwc, strm, res); + } } } @@ -409,6 +419,7 @@ ZWRAP_DCtx* ZWRAP_createDCtx(z_streamp strm) memcpy(&zwd->customMem, &defaultCustomMem, sizeof(ZSTD_customMem)); } + MEM_STATIC_ASSERT(sizeof(zwd->headerBuf) >= ZSTD_frameHeaderSize_min); /* if compilation fails here, assertion is false */ ZWRAP_initDCtx(zwd); return zwd; } diff --git a/zlibWrapper/zstd_zlibwrapper.h b/zlibWrapper/zstd_zlibwrapper.h index 258cb234d..5447b3a91 100644 --- a/zlibWrapper/zstd_zlibwrapper.h +++ b/zlibWrapper/zstd_zlibwrapper.h @@ -39,7 +39,9 @@ int ZWRAP_isUsingZSTDcompression(void); /* Changes a pledged source size for a given compression stream. It will change ZSTD compression parameters what may improve compression speed and/or ratio. - The function should be called just after deflateInit(). */ + The function should be called just after deflateInit(). It's only helpful when data is compressed in blocks. + There will be no change in case of deflateInit() immediately followed by deflate(strm, Z_FINISH) + as this case is automatically detected. */ int ZWRAP_setPledgedSrcSize(z_streamp strm, unsigned long long pledgedSrcSize); From c941d396c0d4c1c0ffeca358e04945bbee3e18bf Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 26 Sep 2016 22:11:08 +0200 Subject: [PATCH 40/43] updated results in zlibWrapper\README.md --- zlibWrapper/README.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/zlibWrapper/README.md b/zlibWrapper/README.md index 6a7cdd2fa..a3d161f27 100644 --- a/zlibWrapper/README.md +++ b/zlibWrapper/README.md @@ -101,21 +101,21 @@ The speed of compression can be improved with reusing a single context with foll - for the 2nd file call `deflateReset`, `deflate`, `...`, `deflate` - free the context with `deflateEnd` -To check the difference we made experiments using `zwrapbench` with zstd and zlib compression (both at level 3) with 4 KB blocks. -The input data was git repository downloaded from https://github.com/git/git/archive/master.zip and converted to uncompressed tarball. +To check the difference we made experiments using `zwrapbench -ri6b6` with zstd and zlib compression (both at level 6). +The input data was decompressed git repository downloaded from https://github.com/git/git/archive/master.zip that contains 2979 files. The table below shows that reusing contexts has a minor influence on zlib but it gives improvement for zstd. -In our example (the last 2 lines) it gives 15% better compression speed and 6% better decompression speed. +In our example (the last 2 lines) it gives 4% better compression speed and 5% better decompression speed. | Compression type | Compression | Decompress.| Compr. size | Ratio | | ------------------------------------------------- | ------------| -----------| ----------- | ----- | -| zlib 1.2.8 | 54.77 MB/s | 176.2 MB/s | 8928115 | 2.910 | -| zlib 1.2.8 not reusing a context | 54.98 MB/s | 174.6 MB/s | 8928115 | 2.910 | -| zlib 1.2.8 with zlibWrapper and reusing a context | 55.16 MB/s | 176.8 MB/s | 8928115 | 2.910 | -| zlib 1.2.8 with zlibWrapper not reusing a context | 54.11 MB/s | 174.5 MB/s | 8928115 | 2.910 | -| zstd 1.1.0 using ZSTD_CCtx | 108.03 MB/s | 319.8 MB/s | 8962336 | 2.899 | -| zstd 1.1.0 using ZSTD_CStream | 107.34 MB/s | 307.3 MB/s | 8981368 | 2.893 | -| zstd 1.1.0 with zlibWrapper and reusing a context | 107.52 MB/s | 297.3 MB/s | 8981368 | 2.893 | -| zstd 1.1.0 with zlibWrapper not reusing a context | 91.45 MB/s | 279.8 MB/s | 8981368 | 2.893 | +| zlib 1.2.8 | 30.51 MB/s | 219.3 MB/s | 6819783 | 3.459 | +| zlib 1.2.8 not reusing a context | 30.22 MB/s | 218.1 MB/s | 6819783 | 3.459 | +| zlib 1.2.8 with zlibWrapper and reusing a context | 30.40 MB/s | 218.9 MB/s | 6819783 | 3.459 | +| zlib 1.2.8 with zlibWrapper not reusing a context | 30.28 MB/s | 218.1 MB/s | 6819783 | 3.459 | +| zstd 1.1.0 using ZSTD_CCtx | 68.35 MB/s | 430.9 MB/s | 6868521 | 3.435 | +| zstd 1.1.0 using ZSTD_CStream | 66.63 MB/s | 422.3 MB/s | 6868521 | 3.435 | +| zstd 1.1.0 with zlibWrapper and reusing a context | 54.01 MB/s | 403.2 MB/s | 6763482 | 3.488 | +| zstd 1.1.0 with zlibWrapper not reusing a context | 51.59 MB/s | 383.7 MB/s | 6763482 | 3.488 | #### Compatibility issues From a03b7a7f1bbcc940aee072cd6893d38a4270b5a7 Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 26 Sep 2016 22:11:55 +0200 Subject: [PATCH 41/43] zwrapbench: improved tests with a dictionary --- zlibWrapper/examples/zwrapbench.c | 22 ++++++++++++++++++++-- zlibWrapper/zstd_zlibwrapper.c | 9 ++++----- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c index f0ae8cd97..99f91739b 100644 --- a/zlibWrapper/examples/zwrapbench.c +++ b/zlibWrapper/examples/zwrapbench.c @@ -257,7 +257,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, ZSTD_CStream* zbc = ZSTD_createCStream(); size_t rSize; if (zbc == NULL) EXM_THROW(1, "ZSTD_createCStream() allocation failure"); - rSize = ZSTD_initCStream_advanced(zbc, NULL, 0, zparams, avgSize); + rSize = ZSTD_initCStream_advanced(zbc, dictBuffer, dictBufferSize, zparams, avgSize); if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_initCStream_advanced() failed : %s", ZSTD_getErrorName(rSize)); do { U32 blockNb; @@ -298,6 +298,10 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, for (blockNb=0; blockNbcompressionLevel); if (!zwc) return Z_STREAM_ERROR; if (zwc->zbc == NULL) { - int res; zwc->zbc = ZSTD_createCStream_advanced(zwc->customMem); - if (zwc->zbc == NULL) return ZWRAPC_finishWithError(zwc, strm, res); - res = ZWRAP_initializeCStream(zwc, dictionary, dictLength, 0); - if (res != Z_OK) return ZWRAPC_finishWithError(zwc, strm, res); - zwc->comprState = Z_NEED_DICT; + if (zwc->zbc == NULL) return ZWRAPC_finishWithError(zwc, strm, 0); } + { int res = ZWRAP_initializeCStream(zwc, dictionary, dictLength, 0); + if (res != Z_OK) return ZWRAPC_finishWithError(zwc, strm, res); } + zwc->comprState = Z_NEED_DICT; } return Z_OK; From ad468ab25c4fc823a2f012c29e94c2a4f7cea8c3 Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 26 Sep 2016 22:24:04 +0200 Subject: [PATCH 42/43] updated zlibWrapper\Makefile --- zlibWrapper/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index 07a2490c0..42a8d32d5 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -16,7 +16,7 @@ EXAMPLE_PATH = examples PROGRAMS_PATH = ../programs CC ?= gcc CFLAGS ?= -O3 -CFLAGS += $(LOC) -I$(PROGRAMS_PATH) -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH) -std=gnu90 +CFLAGS += $(LOC) -I$(PROGRAMS_PATH) -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH) -std=gnu99 CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef LDFLAGS = $(LOC) RM = rm -f @@ -31,7 +31,7 @@ test: example fitblk example_zstd fitblk_zstd zwrapbench ./fitblk 40960 <../zstd_compression_format.md ./fitblk_zstd 10240 <../zstd_compression_format.md ./fitblk_zstd 40960 <../zstd_compression_format.md - ./zwrapbench -qb1e5B1K ../zstd_compression_format.md + ./zwrapbench -qb3B1K ../zstd_compression_format.md ./zwrapbench -qb1e5r ../lib ../programs ../tests #valgrindTest: ZSTDLIBRARY = $(ZSTDLIBDIR)/libzstd.so @@ -44,7 +44,7 @@ valgrindTest: clean example fitblk example_zstd fitblk_zstd zwrapbench $(VALGRIND) ./fitblk 40960 <../zstd_compression_format.md $(VALGRIND) ./fitblk_zstd 10240 <../zstd_compression_format.md $(VALGRIND) ./fitblk_zstd 40960 <../zstd_compression_format.md - $(VALGRIND) ./zwrapbench -qb1e5B1K ../zstd_compression_format.md + $(VALGRIND) ./zwrapbench -qb3B1K ../zstd_compression_format.md $(VALGRIND) ./zwrapbench -qb1e5 ../lib ../programs ../tests .c.o: From 60dddc2109039a59cce960d339616dbe09d95fc9 Mon Sep 17 00:00:00 2001 From: inikep Date: Mon, 26 Sep 2016 22:47:39 +0200 Subject: [PATCH 43/43] zlibWrapper: minor tweaks --- zlibWrapper/Makefile | 4 ++-- zlibWrapper/README.md | 2 +- zlibWrapper/zstd_zlibwrapper.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index 42a8d32d5..69c976fa5 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -32,7 +32,7 @@ test: example fitblk example_zstd fitblk_zstd zwrapbench ./fitblk_zstd 10240 <../zstd_compression_format.md ./fitblk_zstd 40960 <../zstd_compression_format.md ./zwrapbench -qb3B1K ../zstd_compression_format.md - ./zwrapbench -qb1e5r ../lib ../programs ../tests + ./zwrapbench -rqb1e5 ../lib ../programs ../tests #valgrindTest: ZSTDLIBRARY = $(ZSTDLIBDIR)/libzstd.so valgrindTest: VALGRIND = LD_LIBRARY_PATH=$(ZSTDLIBDIR) valgrind --track-origins=yes --leak-check=full --error-exitcode=1 @@ -45,7 +45,7 @@ valgrindTest: clean example fitblk example_zstd fitblk_zstd zwrapbench $(VALGRIND) ./fitblk_zstd 10240 <../zstd_compression_format.md $(VALGRIND) ./fitblk_zstd 40960 <../zstd_compression_format.md $(VALGRIND) ./zwrapbench -qb3B1K ../zstd_compression_format.md - $(VALGRIND) ./zwrapbench -qb1e5 ../lib ../programs ../tests + $(VALGRIND) ./zwrapbench -rqb1e5 ../lib ../programs ../tests .c.o: $(CC) $(CFLAGS) -c -o $@ $< diff --git a/zlibWrapper/README.md b/zlibWrapper/README.md index a3d161f27..163009a8b 100644 --- a/zlibWrapper/README.md +++ b/zlibWrapper/README.md @@ -102,7 +102,7 @@ The speed of compression can be improved with reusing a single context with foll - free the context with `deflateEnd` To check the difference we made experiments using `zwrapbench -ri6b6` with zstd and zlib compression (both at level 6). -The input data was decompressed git repository downloaded from https://github.com/git/git/archive/master.zip that contains 2979 files. +The input data was decompressed git repository downloaded from https://github.com/git/git/archive/master.zip which contains 2979 files. The table below shows that reusing contexts has a minor influence on zlib but it gives improvement for zstd. In our example (the last 2 lines) it gives 4% better compression speed and 5% better decompression speed. diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index 524ca7868..3464c600d 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -260,7 +260,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) if (zwc->zbc == NULL) { int res; zwc->zbc = ZSTD_createCStream_advanced(zwc->customMem); - if (zwc->zbc == NULL) return ZWRAPC_finishWithError(zwc, strm, res); + if (zwc->zbc == NULL) return ZWRAPC_finishWithError(zwc, strm, 0); res = ZWRAP_initializeCStream(zwc, NULL, 0, (flush == Z_FINISH) ? strm->avail_in : 0); if (res != Z_OK) return ZWRAPC_finishWithError(zwc, strm, res); } else {