From 2ed3ba2a35a83765b35235aa77f1d68f183273ba Mon Sep 17 00:00:00 2001 From: Dmitry Krot Date: Sat, 13 Aug 2016 22:02:45 +0300 Subject: [PATCH 01/14] Fix issue with SYNC_FLUSH support by zlib wrapper --- zlibWrapper/zstd_zlibwrapper.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index 34de73a6d..e23a235c9 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -227,7 +227,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) if (flush == Z_FULL_FLUSH) FINISH_WITH_ERR(strm, "Z_FULL_FLUSH is not supported!"); - if (flush == Z_FINISH || flush == Z_FULL_FLUSH) { + if (flush == Z_FINISH) { size_t bytesLeft; size_t dstCapacity = strm->avail_out; if (zwc->bytesLeft) { @@ -246,6 +246,18 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) if (flush == Z_FINISH && bytesLeft == 0) return Z_STREAM_END; zwc->bytesLeft = bytesLeft; } + + if (flush == Z_SYNC_FLUSH) { + size_t bytesLeft; + size_t dstCapacity = strm->avail_out; + bytesLeft = ZBUFF_compressFlush(zwc->zbc, strm->next_out, &dstCapacity); + LOG_WRAPPER("ZBUFF_compressFlush avail_out=%d dstCapacity=%d bytesLeft=%d\n", (int)strm->avail_out, (int)dstCapacity, (int)bytesLeft); + if (ZSTD_isError(bytesLeft)) return Z_MEM_ERROR; + strm->next_out += dstCapacity; + strm->total_out += dstCapacity; + strm->avail_out -= dstCapacity; + zwc->bytesLeft = bytesLeft; + } return Z_OK; } From b3950eef20ddb98ed8946c3eb20d0f3f3d37986f Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 16 Aug 2016 15:10:30 +0200 Subject: [PATCH 02/14] support for Z_SYNC_FLUSH, by @dak-evanti-ru --- NEWS | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS b/NEWS index 20765a3d7..77edad474 100644 --- a/NEWS +++ b/NEWS @@ -3,6 +3,7 @@ Changed : --ultra now enables levels beyond 19 Changed : -i# now selects benchmark time in second Fixed : ZSTD_compress* can now compress > 4 GB in a single pass, reported by Nick Terrell Fixed : speed regression on specific patterns (#272) +Fixed : support for Z_SYNC_FLUSH, by Dmitry Krot (#291) v0.8.0 Improved : better speed on clang and gcc -O2, thanks to Eric Biggers From 5a0c8e24395079f8e8cdc90aa1659cd5ab1b7427 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 12 Aug 2016 01:20:36 +0200 Subject: [PATCH 03/14] new streaming API (compression) --- lib/compress/zstd_compress.c | 272 ++++++++++++++++++++++++++++++++++- lib/zstd.h | 47 +++++- 2 files changed, 316 insertions(+), 3 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index e218b6374..d14a1cde8 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -32,7 +32,7 @@ */ -/* ******************************************************* +/*-******************************************************* * Compiler specifics *********************************************************/ #ifdef _MSC_VER /* Visual Studio */ @@ -55,7 +55,7 @@ #include "mem.h" #define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ #include "xxhash.h" /* XXH_reset, update, digest */ -#define FSE_STATIC_LINKING_ONLY +#define FSE_STATIC_LINKING_ONLY /* FSE_encodeSymbol */ #include "fse.h" #define HUF_STATIC_LINKING_ONLY #include "huf.h" @@ -2758,6 +2758,274 @@ ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, +/*-====== Streaming ======-*/ + +typedef enum { zcss_init, zcss_load, zcss_flush, zcss_final } ZSTD_cStreamStage; + +struct ZSTD_CStream_s { + ZSTD_CCtx* zc; + char* inBuff; + size_t inBuffSize; + size_t inToCompress; + size_t inBuffPos; + size_t inBuffTarget; + size_t blockSize; + char* outBuff; + size_t outBuffSize; + size_t outBuffContentSize; + size_t outBuffFlushedSize; + ZSTD_cStreamStage stage; + U32 checksum; + U32 frameEnded; + ZSTD_customMem customMem; +}; /* typedef'd to ZSTD_CStream within "zstd.h" */ + +ZSTD_CStream* ZSTD_createCStream(void) +{ + return ZSTD_createCStream_advanced(defaultCustomMem); +} + +ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem) +{ + ZSTD_CStream* zcs; + + if (!customMem.customAlloc && !customMem.customFree) + customMem = defaultCustomMem; + + if (!customMem.customAlloc || !customMem.customFree) + return NULL; + + zcs = (ZSTD_CStream*)customMem.customAlloc(customMem.opaque, sizeof(ZSTD_CStream)); + if (zcs==NULL) return NULL; + memset(zcs, 0, sizeof(ZSTD_CStream)); + memcpy(&zcs->customMem, &customMem, sizeof(ZSTD_customMem)); + zcs->zc = ZSTD_createCCtx_advanced(customMem); + if (zcs->zc == NULL) { ZSTD_freeCStream(zcs); return NULL; } + return zcs; +} + +size_t ZSTD_freeCStream(ZSTD_CStream* zcs) +{ + if (zcs==NULL) return 0; /* support free on NULL */ + ZSTD_freeCCtx(zcs->zc); + if (zcs->inBuff) zcs->customMem.customFree(zcs->customMem.opaque, zcs->inBuff); + if (zcs->outBuff) zcs->customMem.customFree(zcs->customMem.opaque, zcs->outBuff); + zcs->customMem.customFree(zcs->customMem.opaque, zcs); + return 0; +} + + +/* Initialization */ + +size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + ZSTD_parameters params, unsigned long long pledgedSrcSize) +{ + /* allocate buffers */ + { size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog; + if (zcs->inBuffSize < neededInBuffSize) { + zcs->inBuffSize = neededInBuffSize; + zcs->customMem.customFree(zcs->customMem.opaque, zcs->inBuff); /* should not be necessary */ + zcs->inBuff = (char*)zcs->customMem.customAlloc(zcs->customMem.opaque, neededInBuffSize); + if (zcs->inBuff == NULL) return ERROR(memory_allocation); + } + zcs->blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, neededInBuffSize); + } + if (zcs->outBuffSize < ZSTD_compressBound(zcs->blockSize)+1) { + zcs->outBuffSize = ZSTD_compressBound(zcs->blockSize)+1; + zcs->customMem.customFree(zcs->customMem.opaque, zcs->outBuff); /* should not be necessary */ + zcs->outBuff = (char*)zcs->customMem.customAlloc(zcs->customMem.opaque, zcs->outBuffSize); + if (zcs->outBuff == NULL) return ERROR(memory_allocation); + } + + { size_t const errorCode = ZSTD_compressBegin_advanced(zcs->zc, dict, dictSize, params, pledgedSrcSize); + if (ZSTD_isError(errorCode)) return errorCode; } + + zcs->inToCompress = 0; + zcs->inBuffPos = 0; + zcs->inBuffTarget = zcs->blockSize; + zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0; + zcs->stage = zcss_load; + zcs->checksum = params.fParams.checksumFlag > 0; + zcs->frameEnded = 0; + return 0; /* ready to go */ +} + +size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel) +{ + ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize); + return ZSTD_initCStream_advanced(zcs, dict, dictSize, params, 0); +} + +size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel) +{ + return ZSTD_initCStream_usingDict(zcs, NULL, 0, compressionLevel); +} + + +/* Compression */ + +typedef enum { zsf_gather, zsf_flush, zsf_end } ZSTD_flush_e; + +MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + size_t const length = MIN(dstCapacity, srcSize); + memcpy(dst, src, length); + return length; +} + +static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, + void* dst, size_t* dstCapacityPtr, + const void* src, size_t* srcSizePtr, + ZSTD_flush_e const flush) +{ + U32 someMoreWork = 1; + const char* const istart = (const char*)src; + const char* const iend = istart + *srcSizePtr; + const char* ip = istart; + char* const ostart = (char*)dst; + char* const oend = ostart + *dstCapacityPtr; + char* op = ostart; + + while (someMoreWork) { + switch(zcs->stage) + { + case zcss_init: return ERROR(init_missing); /* call ZBUFF_compressInit() first ! */ + + case zcss_load: + /* complete inBuffer */ + { size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos; + size_t const loaded = ZSTD_limitCopy(zcs->inBuff + zcs->inBuffPos, toLoad, ip, iend-ip); + zcs->inBuffPos += loaded; + ip += loaded; + if ( (zcs->inBuffPos==zcs->inToCompress) || (!flush && (toLoad != loaded)) ) { + someMoreWork = 0; break; /* not enough input to get a full block : stop there, wait for more */ + } } + /* compress current block (note : this stage cannot be stopped in the middle) */ + { void* cDst; + size_t cSize; + size_t const iSize = zcs->inBuffPos - zcs->inToCompress; + size_t oSize = oend-op; + if (oSize >= ZSTD_compressBound(iSize)) + cDst = op; /* compress directly into output buffer (avoid flush stage) */ + else + cDst = zcs->outBuff, oSize = zcs->outBuffSize; + cSize = (flush == zsf_end) ? + ZSTD_compressEnd(zcs->zc, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize) : + ZSTD_compressContinue(zcs->zc, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize); + if (ZSTD_isError(cSize)) return cSize; + if (flush == zsf_end) zcs->frameEnded = 1; + /* prepare next block */ + zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize; + if (zcs->inBuffTarget > zcs->inBuffSize) + zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize; /* note : inBuffSize >= blockSize */ + zcs->inToCompress = zcs->inBuffPos; + if (cDst == op) { op += cSize; break; } /* no need to flush */ + zcs->outBuffContentSize = cSize; + zcs->outBuffFlushedSize = 0; + zcs->stage = zcss_flush; /* pass-through to flush stage */ + } + + case zcss_flush: + { size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; + size_t const flushed = ZSTD_limitCopy(op, oend-op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush); + op += flushed; + zcs->outBuffFlushedSize += flushed; + if (toFlush!=flushed) { someMoreWork = 0; break; } /* dst too small to store flushed data : stop there */ + zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0; + zcs->stage = zcss_load; + break; + } + + case zcss_final: + someMoreWork = 0; /* do nothing */ + break; + + default: + return ERROR(GENERIC); /* impossible */ + } + } + + *srcSizePtr = ip - istart; + *dstCapacityPtr = op - ostart; + if (zcs->frameEnded) return 0; + { size_t hintInSize = zcs->inBuffTarget - zcs->inBuffPos; + if (hintInSize==0) hintInSize = zcs->blockSize; + return hintInSize; + } +} + +size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_wCursor* output, ZSTD_rCursor* input) +{ + size_t sizeRead = input->size; + size_t sizeWritten = output->size; + size_t const result = ZSTD_compressStream_generic(zcs, output->ptr, &sizeWritten, input->ptr, &sizeRead, zsf_gather); + input->ptr = (const char*)(input->ptr) + sizeRead; + input->size -= sizeRead; + output->ptr = (char*)(output->ptr) + sizeWritten; + output->size -= sizeWritten; + output->nbBytesWritten += sizeWritten; + return result; +} + + +/* Finalize */ + +/*! ZSTD_flushStream() : +* @return : amount of data remaining to flush */ +size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_wCursor* output) +{ + size_t srcSize = 0; + size_t sizeWritten = output->size; + size_t const result = ZSTD_compressStream_generic(zcs, output->ptr, &sizeWritten, &srcSize, &srcSize, zsf_flush); /* use a valid src address instead of NULL */ + output->ptr = (char*)(output->ptr) + sizeWritten; + output->size -= sizeWritten; + output->nbBytesWritten += sizeWritten; + if (ZSTD_isError(result)) return result; + return zcs->outBuffContentSize - zcs->outBuffFlushedSize; /* remaining to flush */ +} + + +size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_wCursor* output) +{ + BYTE* const ostart = (BYTE*)(output->ptr); + BYTE* const oend = ostart + output->size; + BYTE* op = ostart; + + if (zcs->stage != zcss_final) { + /* flush whatever remains */ + size_t srcSize = 0; + size_t sizeWritten = output->size; + size_t const notEnded = ZSTD_compressStream_generic(zcs, ostart, &sizeWritten, &srcSize, &srcSize, zsf_end); /* use a valid src address instead of NULL */ + size_t const remainingToFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; + op += sizeWritten; + if (remainingToFlush) { + output->ptr = op; + output->size -= sizeWritten; + output->nbBytesWritten += sizeWritten; + return remainingToFlush + ZSTD_BLOCKHEADERSIZE /* final empty block */ + (zcs->checksum * 4); + } + /* create epilogue */ + zcs->stage = zcss_final; + zcs->outBuffContentSize = !notEnded ? 0 : + ZSTD_compressEnd(zcs->zc, zcs->outBuff, zcs->outBuffSize, NULL, 0); /* write epilogue, including final empty block, into outBuff */ + } + + /* flush epilogue */ + { size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; + size_t const flushed = ZSTD_limitCopy(op, oend-op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush); + op += flushed; + zcs->outBuffFlushedSize += flushed; + output->ptr = op; + output->size = oend-op; + output->nbBytesWritten += op-ostart; + if (toFlush==flushed) zcs->stage = zcss_init; /* end reached */ + return toFlush - flushed; + } +} + + + /*-===== Pre-defined compression levels =====-*/ #define ZSTD_DEFAULT_CLEVEL 1 diff --git a/lib/zstd.h b/lib/zstd.h index 7bd8a05c4..3c563e38f 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -323,7 +323,52 @@ ZSTDLIB_API size_t ZSTD_sizeofDCtx(const ZSTD_DCtx* dctx); /* ****************************************************************** -* Buffer-less streaming functions (synchronous mode) +* Streaming +********************************************************************/ + +typedef struct ZSTD_readCursor_s { + const void* ptr; /* position of cursor - update to new position */ + size_t size; /* remaining buffer size to read - update preserves end of buffer */ +} ZSTD_rCursor; + +typedef struct ZSTD_writeCursor_s { + void* ptr; /* position of cursor - update to new position */ + size_t size; /* remaining buffer size to write - update preserves end of buffer */ + size_t nbBytesWritten; /* already written bytes - update adds bytes newly written (accumulator) */ +} ZSTD_wCursor; + + +/*====== compression ======*/ + +typedef struct ZSTD_CStream_s ZSTD_CStream; +ZSTD_CStream* ZSTD_createCStream(void); +size_t ZSTD_freeCStream(ZSTD_CStream* zcs); + +size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); +size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_wCursor* output, ZSTD_rCursor* input); +size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_wCursor* output); +size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_wCursor* output); + +/* advanced */ +ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); +size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); +size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, + ZSTD_parameters params, unsigned long long pledgedSrcSize); + + +/*====== decompression ======*/ + +typedef struct ZSTD_DStream_s ZSTD_DStream; +ZSTD_DStream* ZSTD_createDStream(void); +size_t ZSTD_freeDStream(ZSTD_DStream* zds); + +size_t ZSTD_initDStream(ZSTD_DStream* zds); +size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_wCursor* output, ZSTD_rCursor* input); + + + +/* ****************************************************************** +* Buffer-less and synchronous inner streaming functions ********************************************************************/ /* This is an advanced API, giving full control over buffer management, for users which need direct control over memory. * But it's also a complex one, with a lot of restrictions (documented below). From 104e5b072dadfe5dc0fc65d2a5b6671f0f28893c Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 12 Aug 2016 13:04:27 +0200 Subject: [PATCH 04/14] added : streaming decompression API --- lib/compress/zstd_compress.c | 13 +- lib/decompress/zstd_decompress.c | 240 +++++++++++++++++++++++++++++++ lib/zstd.h | 10 ++ 3 files changed, 259 insertions(+), 4 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index d14a1cde8..6e8f0fba7 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2758,7 +2758,9 @@ ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, -/*-====== Streaming ======-*/ +/* ****************************************************************** +* Streaming +********************************************************************/ typedef enum { zcss_init, zcss_load, zcss_flush, zcss_final } ZSTD_cStreamStage; @@ -2815,7 +2817,10 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs) } -/* Initialization */ +/*====== Initialization ======*/ + +size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; } +size_t ZSTD_CStreamOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ; } size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, @@ -2863,7 +2868,7 @@ size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel) } -/* Compression */ +/*====== Compression ======*/ typedef enum { zsf_gather, zsf_flush, zsf_end } ZSTD_flush_e; @@ -2969,7 +2974,7 @@ size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_wCursor* output, ZSTD_rCursor } -/* Finalize */ +/*====== Finalize ======*/ /*! ZSTD_flushStream() : * @return : amount of data remaining to flush */ diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 052ccd601..57ca21112 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1301,3 +1301,243 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, dst, dstCapacity, src, srcSize); } + + +/*===================================== +* Streaming decompression +*====================================*/ + +typedef enum { zdss_init, zdss_loadHeader, + zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage; + +/* *** Resource management *** */ +struct ZSTD_DStream_s { + ZSTD_DCtx* zd; + ZSTD_frameParams fParams; + ZSTD_dStreamStage stage; + char* inBuff; + size_t inBuffSize; + size_t inPos; + char* outBuff; + size_t outBuffSize; + size_t outStart; + size_t outEnd; + size_t blockSize; + BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; + size_t lhSize; + ZSTD_customMem customMem; +}; /* typedef'd to ZSTD_DStream within "zstd.h" */ + + +ZSTD_DStream* ZSTD_createDStream(void) +{ + return ZSTD_createDStream_advanced(defaultCustomMem); +} + +ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem) +{ + ZSTD_DStream* zds; + + if (!customMem.customAlloc && !customMem.customFree) + customMem = defaultCustomMem; + + if (!customMem.customAlloc || !customMem.customFree) + return NULL; + + zds = (ZSTD_DStream*)customMem.customAlloc(customMem.opaque, sizeof(ZSTD_DStream)); + if (zds==NULL) return NULL; + memset(zds, 0, sizeof(ZSTD_DStream)); + memcpy(&zds->customMem, &customMem, sizeof(ZSTD_customMem)); + zds->zd = ZSTD_createDCtx_advanced(customMem); + if (zds->zd == NULL) { ZSTD_freeDStream(zds); return NULL; } + zds->stage = zdss_init; + return zds; +} + +size_t ZSTD_freeDStream(ZSTD_DStream* zds) +{ + if (zds==NULL) return 0; /* support free on null */ + ZSTD_freeDCtx(zds->zd); + if (zds->inBuff) zds->customMem.customFree(zds->customMem.opaque, zds->inBuff); + if (zds->outBuff) zds->customMem.customFree(zds->customMem.opaque, zds->outBuff); + zds->customMem.customFree(zds->customMem.opaque, zds); + return 0; +} + + +/* *** Initialization *** */ + +size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX + ZSTD_blockHeaderSize; } +size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; } + +size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize) +{ + zds->stage = zdss_loadHeader; + zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; + return ZSTD_decompressBegin_usingDict(zds->zd, dict, dictSize); +} + +size_t ZSTD_initDStream(ZSTD_DStream* zds) +{ + return ZSTD_initDStream_usingDict(zds, NULL, 0); +} + + +/* *** Decompression *** */ + +MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + size_t const length = MIN(dstCapacity, srcSize); + memcpy(dst, src, length); + return length; +} + + +size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_wCursor* output, ZSTD_rCursor* input) +{ + const char* const istart = (const char*)(input->ptr); + const char* const iend = istart + input->size; + const char* ip = istart; + char* const ostart = (char*)(output->ptr); + char* const oend = ostart + output->size; + char* op = ostart; + U32 someMoreWork = 1; + + while (someMoreWork) { + switch(zds->stage) + { + case zdss_init : + return ERROR(init_missing); + + case zdss_loadHeader : + { size_t const hSize = ZSTD_getFrameParams(&(zds->fParams), zds->headerBuffer, zds->lhSize); + if (ZSTD_isError(hSize)) return hSize; + if (hSize != 0) { /* need more input */ + size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */ + if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */ + memcpy(zds->headerBuffer + zds->lhSize, ip, iend-ip); + zds->lhSize += iend-ip; + input->ptr = iend; + input->size = 0; + return (hSize - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ + } + memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad; + break; + } } + + /* Consume header */ + { size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->zd); /* == ZSTD_frameHeaderSize_min */ + size_t const h1Result = ZSTD_decompressContinue(zds->zd, NULL, 0, zds->headerBuffer, h1Size); + if (ZSTD_isError(h1Result)) return h1Result; /* should not happen : already checked */ + if (h1Size < zds->lhSize) { /* long header */ + size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->zd); + size_t const h2Result = ZSTD_decompressContinue(zds->zd, NULL, 0, zds->headerBuffer+h1Size, h2Size); + if (ZSTD_isError(h2Result)) return h2Result; + } } + + zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); + + /* Frame header instruct buffer sizes */ + { size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); + size_t const neededOutSize = zds->fParams.windowSize + blockSize; + zds->blockSize = blockSize; + if (zds->inBuffSize < blockSize) { + zds->customMem.customFree(zds->customMem.opaque, zds->inBuff); + zds->inBuffSize = blockSize; + zds->inBuff = (char*)zds->customMem.customAlloc(zds->customMem.opaque, blockSize); + if (zds->inBuff == NULL) return ERROR(memory_allocation); + } + if (zds->outBuffSize < neededOutSize) { + zds->customMem.customFree(zds->customMem.opaque, zds->outBuff); + zds->outBuffSize = neededOutSize; + zds->outBuff = (char*)zds->customMem.customAlloc(zds->customMem.opaque, neededOutSize); + if (zds->outBuff == NULL) return ERROR(memory_allocation); + } } + zds->stage = zdss_read; + /* pass-through */ + + case zdss_read: + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->zd); + if (neededInSize==0) { /* end of frame */ + zds->stage = zdss_init; + someMoreWork = 0; + break; + } + if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ + const int isSkipFrame = ZSTD_isSkipFrame(zds->zd); + size_t const decodedSize = ZSTD_decompressContinue(zds->zd, + zds->outBuff + zds->outStart, (isSkipFrame ? 0 : zds->outBuffSize - zds->outStart), + ip, neededInSize); + if (ZSTD_isError(decodedSize)) return decodedSize; + ip += neededInSize; + if (!decodedSize && !isSkipFrame) break; /* this was just a header */ + zds->outEnd = zds->outStart + decodedSize; + zds->stage = zdss_flush; + break; + } + if (ip==iend) { someMoreWork = 0; break; } /* no more input */ + zds->stage = zdss_load; + /* pass-through */ + } + + case zdss_load: + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->zd); + size_t const toLoad = neededInSize - zds->inPos; /* should always be <= remaining space within inBuff */ + size_t loadedSize; + if (toLoad > zds->inBuffSize - zds->inPos) return ERROR(corruption_detected); /* should never happen */ + loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, iend-ip); + ip += loadedSize; + zds->inPos += loadedSize; + if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */ + + /* decode loaded input */ + { const int isSkipFrame = ZSTD_isSkipFrame(zds->zd); + size_t const decodedSize = ZSTD_decompressContinue(zds->zd, + zds->outBuff + zds->outStart, zds->outBuffSize - zds->outStart, + zds->inBuff, neededInSize); + if (ZSTD_isError(decodedSize)) return decodedSize; + zds->inPos = 0; /* input is consumed */ + if (!decodedSize && !isSkipFrame) { zds->stage = zdss_read; break; } /* this was just a header */ + zds->outEnd = zds->outStart + decodedSize; + zds->stage = zdss_flush; + /* pass-through */ + } } + + case zdss_flush: + { size_t const toFlushSize = zds->outEnd - zds->outStart; + size_t const flushedSize = ZSTD_limitCopy(op, oend-op, zds->outBuff + zds->outStart, toFlushSize); + op += flushedSize; + zds->outStart += flushedSize; + if (flushedSize == toFlushSize) { /* flush completed */ + zds->stage = zdss_read; + if (zds->outStart + zds->blockSize > zds->outBuffSize) + zds->outStart = zds->outEnd = 0; + break; + } + /* cannot flush everything */ + someMoreWork = 0; + break; + } + default: return ERROR(GENERIC); /* impossible */ + } } + + /* result */ + input->ptr = ip; + input->size = (size_t)(iend-ip); + output->ptr = op; + output->size = (size_t) (oend-op); + output->nbBytesWritten += (size_t)(op-ostart); + { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->zd); + if (!nextSrcSizeHint) return (zds->outEnd != zds->outStart); /* return 0 only if fully flushed too */ + nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->zd) == ZSTDnit_block); + if (zds->inPos > nextSrcSizeHint) return ERROR(GENERIC); /* should never happen */ + nextSrcSizeHint -= zds->inPos; /* already loaded*/ + return nextSrcSizeHint; + } +} + + + + + + diff --git a/lib/zstd.h b/lib/zstd.h index 3c563e38f..ff2f627e7 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -344,6 +344,9 @@ typedef struct ZSTD_CStream_s ZSTD_CStream; ZSTD_CStream* ZSTD_createCStream(void); size_t ZSTD_freeCStream(ZSTD_CStream* zcs); +size_t ZSTD_CStreamInSize(void); /*!< recommended size for input buffer */ +size_t ZSTD_CStreamOutSize(void); /*!< recommended size for output buffer */ + size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_wCursor* output, ZSTD_rCursor* input); size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_wCursor* output); @@ -362,9 +365,16 @@ typedef struct ZSTD_DStream_s ZSTD_DStream; ZSTD_DStream* ZSTD_createDStream(void); size_t ZSTD_freeDStream(ZSTD_DStream* zds); +size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ +size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer */ + size_t ZSTD_initDStream(ZSTD_DStream* zds); size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_wCursor* output, ZSTD_rCursor* input); +/* advanced */ +ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); +size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); + /* ****************************************************************** From d7883a2c0e3b75e9aa5215bc7aa247a97c1cfb5e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 12 Aug 2016 16:48:02 +0200 Subject: [PATCH 05/14] added zstreamtest --- programs/.gitignore | 2 + programs/Makefile | 18 +- programs/zstreamtest.c | 658 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 676 insertions(+), 2 deletions(-) create mode 100644 programs/zstreamtest.c diff --git a/programs/.gitignore b/programs/.gitignore index adf78081d..369a27cc1 100644 --- a/programs/.gitignore +++ b/programs/.gitignore @@ -9,6 +9,8 @@ fuzzer fuzzer32 zbufftest zbufftest32 +zstreamtest +zstreamtest32 datagen paramgrill paramgrill32 diff --git a/programs/Makefile b/programs/Makefile index 9a9fcd346..285d7f9e8 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -152,6 +152,13 @@ zbufftest32 : CFLAGS += -m32 zbufftest32 : EXT := 32$(EXT) zbufftest32 : zbufftest +zstreamtest : $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c zstreamtest.c + $(CC) $(FLAGS) $^ -o $@$(EXT) + +zstreamtest32 : CFLAGS += -m32 +zstreamtest32 : EXT := 32$(EXT) +zstreamtest32 : zstreamtest + paramgrill : $(ZSTD_FILES) datagen.c paramgrill.c $(CC) $(FLAGS) $^ -lm -o $@$(EXT) @@ -229,9 +236,9 @@ ifneq (,$(filter $(HOST_OS),MSYS POSIX)) zstd-playTests: datagen ZSTD=$(ZSTD) ./playTests.sh $(ZSTDRTTEST) -test: test-zstd test-fullbench test-fuzzer test-zbuff +test: test-zstd test-fullbench test-fuzzer test-zbuff test-zstream -test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32 +test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32 test-zstream32 test-all: test test32 valgrindTest @@ -263,4 +270,11 @@ test-zbuff: zbufftest test-zbuff32: zbufftest32 ./zbufftest32 $(ZBUFFTEST) + +test-zstream: zstreamtest + ./zstreamtest $(ZBUFFTEST) + +test-zstream32: zstreamtest32 + ./zstreamtest32 $(ZBUFFTEST) + endif diff --git a/programs/zstreamtest.c b/programs/zstreamtest.c new file mode 100644 index 000000000..7ef58fe7e --- /dev/null +++ b/programs/zstreamtest.c @@ -0,0 +1,658 @@ +/* + Fuzzer test tool for zstd streaming API + Copyright (C) Yann Collet 2016 + + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - ZSTD homepage : https://www.zstd.net/ +*/ + +/*-************************************ +* Compiler specific +**************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# define _CRT_SECURE_NO_WARNINGS /* fgets */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ +#endif + + +/*-************************************ +* Includes +**************************************/ +#include /* free */ +#include /* fgets, sscanf */ +#include /* timeb */ +#include /* strcmp */ +#include "mem.h" +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel */ +#include "zstd.h" /* ZSTD_compressBound */ +#include "datagen.h" /* RDG_genBuffer */ +#define XXH_STATIC_LINKING_ONLY +#include "xxhash.h" /* XXH64_* */ + + +/*-************************************ +* Constants +**************************************/ +#define KB *(1U<<10) +#define MB *(1U<<20) +#define GB *(1U<<30) + +static const U32 nbTestsDefault = 10000; +#define COMPRESSIBLE_NOISE_LENGTH (10 MB) +#define FUZ_COMPRESSIBILITY_DEFAULT 50 +static const U32 prime1 = 2654435761U; +static const U32 prime2 = 2246822519U; + + + +/*-************************************ +* Display Macros +**************************************/ +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } +static U32 g_displayLevel = 2; + +#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ + if ((FUZ_GetMilliSpan(g_displayTime) > g_refreshRate) || (g_displayLevel>=4)) \ + { g_displayTime = FUZ_GetMilliStart(); DISPLAY(__VA_ARGS__); \ + if (g_displayLevel>=4) fflush(stdout); } } +static const U32 g_refreshRate = 150; +static U32 g_displayTime = 0; + +static U32 g_testTime = 0; + + +/*-******************************************************* +* Fuzzer functions +*********************************************************/ +#define MAX(a,b) ((a)>(b)?(a):(b)) + +static U32 FUZ_GetMilliStart(void) +{ + struct timeb tb; + U32 nCount; + ftime( &tb ); + nCount = (U32) (((tb.time & 0xFFFFF) * 1000) + tb.millitm); + return nCount; +} + + +static U32 FUZ_GetMilliSpan(U32 nTimeStart) +{ + U32 const nCurrent = FUZ_GetMilliStart(); + U32 nSpan = nCurrent - nTimeStart; + if (nTimeStart > nCurrent) + nSpan += 0x100000 * 1000; + return nSpan; +} + +/*! FUZ_rand() : + @return : a 27 bits random value, from a 32-bits `seed`. + `seed` is also modified */ +# define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r))) +unsigned int FUZ_rand(unsigned int* seedPtr) +{ + U32 rand32 = *seedPtr; + rand32 *= prime1; + rand32 += prime2; + rand32 = FUZ_rotl32(rand32, 13); + *seedPtr = rand32; + return rand32 >> 5; +} + + +/* +static unsigned FUZ_highbit32(U32 v32) +{ + unsigned nbBits = 0; + if (v32==0) return 0; + for ( ; v32 ; v32>>=1) nbBits++; + return nbBits; +} +*/ + +static void* allocFunction(void* opaque, size_t size) +{ + void* address = malloc(size); + (void)opaque; + return address; +} + +static void freeFunction(void* opaque, void* address) +{ + (void)opaque; + free(address); +} + +static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem customMem) +{ + int testResult = 0; + size_t CNBufferSize = COMPRESSIBLE_NOISE_LENGTH; + void* CNBuffer = malloc(CNBufferSize); + size_t const skippableFrameSize = 11; + size_t const compressedBufferSize = (8 + skippableFrameSize) + ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH); + void* compressedBuffer = malloc(compressedBufferSize); + size_t const decodedBufferSize = CNBufferSize; + void* decodedBuffer = malloc(decodedBufferSize); + size_t cSize; + U32 testNb=0; + ZSTD_CStream* zc = ZSTD_createCStream_advanced(customMem); + ZSTD_DStream* zd = ZSTD_createDStream_advanced(customMem); + ZSTD_rCursor rCursor; + ZSTD_wCursor wCursor; + + /* Create compressible test buffer */ + if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd) { + DISPLAY("Not enough memory, aborting\n"); + goto _output_error; + } + RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., seed); + + /* generate skippable frame */ + MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START); + MEM_writeLE32(((char*)compressedBuffer)+4, (U32)skippableFrameSize); + cSize = skippableFrameSize + 8; + + /* Basic compression test */ + DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); + ZSTD_initCStream_usingDict(zc, CNBuffer, 128 KB, 1); + wCursor.ptr = (char*)(compressedBuffer)+cSize; + wCursor.size = compressedBufferSize; + wCursor.nbBytesWritten = 0; + rCursor.ptr = CNBuffer; + rCursor.size = CNBufferSize; + { size_t const r = ZSTD_compressStream(zc, &wCursor, &rCursor); + if (ZSTD_isError(r)) goto _output_error; } + if (rCursor.size != 0) goto _output_error; /* entire input should be consumed */ + { size_t const r = ZSTD_endStream(zc, &wCursor); + if (r != 0) goto _output_error; } /*< error, or some data not flushed */ + cSize += wCursor.nbBytesWritten; + DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); + + /* skippable frame test */ + DISPLAYLEVEL(4, "test%3i : decompress skippable frame : ", testNb++); + ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB); + rCursor.ptr = compressedBuffer; + rCursor.size = cSize; + wCursor.ptr = decodedBuffer; + wCursor.size = CNBufferSize; + wCursor.nbBytesWritten = 0; + { size_t const r = ZSTD_decompressStream(zd, &wCursor, &rCursor); + if (r != 0) goto _output_error; } + if (wCursor.nbBytesWritten != 0) goto _output_error; /* skippable frame len is 0 */ + DISPLAYLEVEL(4, "OK \n"); + + /* Basic decompression test */ + DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); + ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB); + { size_t const r = ZSTD_decompressStream(zd, &wCursor, &rCursor); + if (r != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */ + if (wCursor.nbBytesWritten != CNBufferSize) goto _output_error; /* should regenerate the same amount */ + if (rCursor.size != 0) goto _output_error; /* should have read the entire frame */ + DISPLAYLEVEL(4, "OK \n"); + + /* check regenerated data is byte exact */ + DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++); + { size_t i; + for (i=0; i "); DISPLAY(__VA_ARGS__); \ + DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; } + +static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility) +{ + static const U32 maxSrcLog = 24; + static const U32 maxSampleLog = 19; + BYTE* cNoiseBuffer[5]; + size_t srcBufferSize = (size_t)1<= testNb) DISPLAYUPDATE(2, "/%6u ", nbTests); + FUZ_rand(&coreSeed); + lseed = coreSeed ^ prime1; + + /* states full reset (unsynchronized) */ + /* some issues only happen when reusing states in a specific sequence of parameters */ + if ((FUZ_rand(&lseed) & 0xFF) == 131) { ZSTD_freeCStream(zc); zc = ZSTD_createCStream(); } + if ((FUZ_rand(&lseed) & 0xFF) == 132) { ZSTD_freeDStream(zd); zd = ZSTD_createDStream(); } + + /* srcBuffer selection [0-4] */ + { U32 buffNb = FUZ_rand(&lseed) & 0x7F; + if (buffNb & 7) buffNb=2; /* most common : compressible (P) */ + else { + buffNb >>= 3; + if (buffNb & 7) { + const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */ + buffNb = tnb[buffNb >> 3]; + } else { + const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */ + buffNb = tnb[buffNb >> 3]; + } } + srcBuffer = cNoiseBuffer[buffNb]; + } + + /* compression init */ + { U32 const testLog = FUZ_rand(&lseed) % maxSrcLog; + U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (testLog/3))) + 1; + maxTestSize = FUZ_rLogLength(&lseed, testLog); + dictSize = (FUZ_rand(&lseed)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0; + /* random dictionary selection */ + { size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize); + dict = srcBuffer + dictStart; + } + { ZSTD_parameters params = ZSTD_getParams(cLevel, 0, dictSize); + params.fParams.checksumFlag = FUZ_rand(&lseed) & 1; + params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1; + { size_t const initError = ZSTD_initCStream_advanced(zc, dict, dictSize, params, 0); + CHECK (ZSTD_isError(initError),"init error : %s", ZSTD_getErrorName(initError)); + } } } + + /* multi-segments compression test */ + XXH64_reset(&xxhState, 0); + nbChunks = (FUZ_rand(&lseed) & 127) + 2; + { ZSTD_wCursor wCursor = { cBuffer, cBufferSize, 0 } ; + for (n=0, cSize=0, totalTestSize=0 ; (n= remainingToFlush); + wCursor.size = adjustedDstSize; + remainingToFlush = ZSTD_endStream(zc, &wCursor); + CHECK (ZSTD_isError(remainingToFlush), "flush error : %s", ZSTD_getErrorName(remainingToFlush)); + CHECK (enoughDstSize && remainingToFlush, "ZSTD_endStream() not fully flushed (%u remaining), but enough space available", (U32)remainingToFlush); + } } + crcOrig = XXH64_digest(&xxhState); + cSize = wCursor.nbBytesWritten; + } + + /* multi - fragments decompression test */ + ZSTD_initDStream_usingDict(zd, dict, dictSize); + { size_t decompressionResult = 1; + ZSTD_rCursor rCursor = { cBuffer, cSize }; + ZSTD_wCursor wCursor = { dstBuffer, dstBufferSize, 0 }; + for (totalCSize = 0, totalGenSize = 0 ; decompressionResult ; ) { + size_t const readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize); + rCursor.size = readCSrcSize; + wCursor.size = dstBuffSize; + decompressionResult = ZSTD_decompressStream(zd, &wCursor, &rCursor); + CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult)); + totalCSize += readCSrcSize - rCursor.size; + } + CHECK (decompressionResult != 0, "frame not fully decoded"); + CHECK (wCursor.nbBytesWritten != totalTestSize, "decompressed data : wrong size") + CHECK (totalCSize != cSize, "compressed data should be fully read") + { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0); + if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize); + CHECK (crcDest!=crcOrig, "decompressed data corrupted"); + } } + + /*===== noisy/erroneous src decompression test =====*/ + + /* add some noise */ + { U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2; + U32 nn; for (nn=0; nn='0') && (*argument<='9')) { + nbTests *= 10; + nbTests += *argument - '0'; + argument++; + } + break; + + case 'T': + argument++; + nbTests=0; g_testTime=0; + while ((*argument>='0') && (*argument<='9')) { + g_testTime *= 10; + g_testTime += *argument - '0'; + argument++; + } + if (*argument=='m') g_testTime *=60, argument++; + if (*argument=='n') argument++; + g_testTime *= 1000; + break; + + case 's': + argument++; + seed=0; + seedset=1; + while ((*argument>='0') && (*argument<='9')) { + seed *= 10; + seed += *argument - '0'; + argument++; + } + break; + + case 't': + argument++; + testNb=0; + while ((*argument>='0') && (*argument<='9')) { + testNb *= 10; + testNb += *argument - '0'; + argument++; + } + break; + + case 'P': /* compressibility % */ + argument++; + proba=0; + while ((*argument>='0') && (*argument<='9')) { + proba *= 10; + proba += *argument - '0'; + argument++; + } + if (proba<0) proba=0; + if (proba>100) proba=100; + break; + + default: + return FUZ_usage(programName); + } + } } } /* for(argNb=1; argNb Date: Fri, 12 Aug 2016 18:42:25 +0200 Subject: [PATCH 06/14] added streaming_compression.c example --- examples/.gitignore | 1 + examples/Makefile | 16 ++-- examples/streaming_compression.c | 143 +++++++++++++++++++++++++++++++ 3 files changed, 155 insertions(+), 5 deletions(-) create mode 100644 examples/streaming_compression.c diff --git a/examples/.gitignore b/examples/.gitignore index 9d241dba6..3b7352c92 100644 --- a/examples/.gitignore +++ b/examples/.gitignore @@ -3,6 +3,7 @@ simple_compression simple_decompression dictionary_compression dictionary_decompression +streaming_compression #test artefact tmp* diff --git a/examples/Makefile b/examples/Makefile index 6ba8655b4..de1f0d7bb 100644 --- a/examples/Makefile +++ b/examples/Makefile @@ -31,7 +31,8 @@ LDFLAGS+= -lzstd default: all all: simple_compression simple_decompression \ - dictionary_compression dictionary_decompression + dictionary_compression dictionary_decompression \ + streaming_compression simple_compression : simple_compression.c $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ @@ -45,19 +46,24 @@ dictionary_compression : dictionary_compression.c dictionary_decompression : dictionary_decompression.c $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ +streaming_compression : streaming_compression.c + $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ + clean: @rm -f core *.o tmp* result* *.zst \ simple_compression simple_decompression \ - dictionary_compression dictionary_decompression + dictionary_compression dictionary_decompression \ + streaming_compression @echo Cleaning completed test: all cp README.md tmp + @echo starting simple compression ./simple_compression tmp - @echo starting simple_decompression ./simple_decompression tmp.zst - @echo dictionary compression + @echo starting streaming compression + ./streaming_compression tmp + @echo starting dictionary compression ./dictionary_compression tmp README.md - @echo dictionary decompression ./dictionary_decompression tmp.zst README.md @echo tests completed diff --git a/examples/streaming_compression.c b/examples/streaming_compression.c new file mode 100644 index 000000000..9082e38a8 --- /dev/null +++ b/examples/streaming_compression.c @@ -0,0 +1,143 @@ +/* + Streaming compression + Educational program using zstd library + Copyright (C) Yann Collet 2016 + + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - zstd homepage : http://www.zstd.net/ +*/ + +#include // malloc, exit +#include // fprintf, perror +#include // strerror +#include // errno +#define ZSTD_STATIC_LINKING_ONLY // streaming API defined as "experimental" for the time being +#include // presumes zstd library is installed + + +static void* malloc_orDie(size_t size) +{ + void* const buff = malloc(size); + if (buff) return buff; + /* error */ + perror(NULL); + exit(1); +} + +static FILE* fopen_orDie(const char *filename, const char *instruction) +{ + FILE* const inFile = fopen(filename, instruction); + if (inFile) return inFile; + /* error */ + perror(filename); + exit(3); +} + +static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file) +{ + size_t const readSize = fread(buffer, 1, sizeToRead, file); + if (readSize == sizeToRead) return readSize; /* good */ + if (feof(file)) return readSize; /* good, reached end of file */ + /* error */ + perror("fread"); + exit(4); +} + +static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file) +{ + size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file); + if (writtenSize == sizeToWrite) return sizeToWrite; /* good */ + /* error */ + perror("fwrite"); + exit(5); +} + +static size_t fclose_orDie(FILE* file) +{ + if (!fclose(file)) return 0; + /* error */ + perror("fclose"); + exit(6); +} + + +static void compressFile_orDie(const char* fname, const char* outName, int cLevel) +{ + FILE* const fin = fopen_orDie(fname, "rb"); + FILE* const fout = fopen_orDie(outName, "wb"); + size_t const buffInSize = ZSTD_CStreamInSize();; + void* const buffIn = malloc_orDie(buffInSize); + size_t const buffOutSize = ZSTD_CStreamOutSize();; + void* const buffOut = malloc_orDie(buffOutSize); + size_t read, toRead = buffInSize; + + ZSTD_CStream* const cstream = ZSTD_createCStream(); + if (cstream==NULL) { fprintf(stderr, "ZSTD_createCStream() error \n"); exit(10); } + size_t const initResult = ZSTD_initCStream(cstream, cLevel); + if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initCStream() error \n"); exit(11); } + + while( (read = fread_orDie(buffIn, toRead, fin)) ) { + ZSTD_rCursor cursin = { buffIn, read }; + while (cursin.size) { + ZSTD_wCursor cursout = { buffOut, buffOutSize, 0 }; + toRead = ZSTD_compressStream(cstream, &cursout , &cursin); + fwrite_orDie(buffOut, cursout.nbBytesWritten, fout); + } + } + + ZSTD_wCursor cursout = { buffOut, buffOutSize, 0 }; + size_t const remainingToFlush = ZSTD_endStream(cstream, &cursout); + if (remainingToFlush) { fprintf(stderr, "not fully flushed"); exit(12); } + fwrite_orDie(buffOut, cursout.nbBytesWritten, fout); + + fclose_orDie(fout); + fclose_orDie(fin); + free(buffIn); + free(buffOut); +} + + +static const char* createOutFilename_orDie(const char* filename) +{ + size_t const inL = strlen(filename); + size_t const outL = inL + 5; + void* outSpace = malloc_orDie(outL); + memset(outSpace, 0, outL); + strcat(outSpace, filename); + strcat(outSpace, ".zst"); + return (const char*)outSpace; +} + +int main(int argc, const char** argv) +{ + const char* const exeName = argv[0]; + const char* const inFilename = argv[1]; + + if (argc!=2) { + printf("wrong arguments\n"); + printf("usage:\n"); + printf("%s FILE\n", exeName); + return 1; + } + + const char* const outFilename = createOutFilename_orDie(inFilename); + compressFile_orDie(inFilename, outFilename, 1); + + return 0; +} From 9f9f1fcb52877ff51a3ae6a6de381ddb6a5c1e69 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 12 Aug 2016 18:56:27 +0200 Subject: [PATCH 07/14] added streaming_decompression example --- examples/.gitignore | 1 + examples/Makefile | 9 ++- examples/streaming_compression.c | 4 +- examples/streaming_decompression.c | 116 +++++++++++++++++++++++++++++ 4 files changed, 126 insertions(+), 4 deletions(-) create mode 100644 examples/streaming_decompression.c diff --git a/examples/.gitignore b/examples/.gitignore index 3b7352c92..1c98e1884 100644 --- a/examples/.gitignore +++ b/examples/.gitignore @@ -4,6 +4,7 @@ simple_decompression dictionary_compression dictionary_decompression streaming_compression +streaming_decompression #test artefact tmp* diff --git a/examples/Makefile b/examples/Makefile index de1f0d7bb..a9d608778 100644 --- a/examples/Makefile +++ b/examples/Makefile @@ -32,7 +32,7 @@ default: all all: simple_compression simple_decompression \ dictionary_compression dictionary_decompression \ - streaming_compression + streaming_compression streaming_decompression simple_compression : simple_compression.c $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ @@ -49,11 +49,14 @@ dictionary_decompression : dictionary_decompression.c streaming_compression : streaming_compression.c $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ +streaming_decompression : streaming_decompression.c + $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ + clean: @rm -f core *.o tmp* result* *.zst \ simple_compression simple_decompression \ dictionary_compression dictionary_decompression \ - streaming_compression + streaming_compression streaming_decompression @echo Cleaning completed test: all @@ -61,8 +64,10 @@ test: all @echo starting simple compression ./simple_compression tmp ./simple_decompression tmp.zst + ./streaming_decompression tmp.zst @echo starting streaming compression ./streaming_compression tmp + ./streaming_decompression tmp.zst @echo starting dictionary compression ./dictionary_compression tmp README.md ./dictionary_decompression tmp.zst README.md diff --git a/examples/streaming_compression.c b/examples/streaming_compression.c index 9082e38a8..f99e040c0 100644 --- a/examples/streaming_compression.c +++ b/examples/streaming_compression.c @@ -24,7 +24,7 @@ */ #include // malloc, exit -#include // fprintf, perror +#include // fprintf, perror, feof #include // strerror #include // errno #define ZSTD_STATIC_LINKING_ONLY // streaming API defined as "experimental" for the time being @@ -36,7 +36,7 @@ static void* malloc_orDie(size_t size) void* const buff = malloc(size); if (buff) return buff; /* error */ - perror(NULL); + perror("malloc:"); exit(1); } diff --git a/examples/streaming_decompression.c b/examples/streaming_decompression.c new file mode 100644 index 000000000..1176cabcb --- /dev/null +++ b/examples/streaming_decompression.c @@ -0,0 +1,116 @@ +/* + Streaming compression + Educational program using zstd library + Copyright (C) Yann Collet 2016 + + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - zstd homepage : http://www.zstd.net/ +*/ + +#include // malloc, exit +#include // fprintf, perror, feof +#include // strerror +#include // errno +#define ZSTD_STATIC_LINKING_ONLY // streaming API defined as "experimental" for the time being +#include // presumes zstd library is installed + + +static void* malloc_orDie(size_t size) +{ + void* const buff = malloc(size); + if (buff) return buff; + /* error */ + perror("malloc:"); + exit(1); +} + +static FILE* fopen_orDie(const char *filename, const char *instruction) +{ + FILE* const inFile = fopen(filename, instruction); + if (inFile) return inFile; + /* error */ + perror(filename); + exit(3); +} + +static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file) +{ + size_t const readSize = fread(buffer, 1, sizeToRead, file); + if (readSize == sizeToRead) return readSize; /* good */ + if (feof(file)) return readSize; /* good, reached end of file */ + /* error */ + perror("fread"); + exit(4); +} + +static size_t fclose_orDie(FILE* file) +{ + if (!fclose(file)) return 0; + /* error */ + perror("fclose"); + exit(6); +} + + +static void decompressFile_orDie(const char* fname) +{ + FILE* const fin = fopen_orDie(fname, "rb"); + size_t const buffInSize = ZSTD_DStreamInSize();; + void* const buffIn = malloc_orDie(buffInSize); + size_t const buffOutSize = ZSTD_DStreamOutSize();; + void* const buffOut = malloc_orDie(buffOutSize); + size_t read, toRead = buffInSize; + + ZSTD_DStream* const dstream = ZSTD_createDStream(); + if (dstream==NULL) { fprintf(stderr, "ZSTD_createDStream() error \n"); exit(10); } + size_t const initResult = ZSTD_initDStream(dstream); + if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initDStream() error \n"); exit(11); } + + while( (read = fread_orDie(buffIn, toRead, fin)) ) { + ZSTD_rCursor cursin = { buffIn, read }; + while (cursin.size) { + ZSTD_wCursor cursout = { buffOut, buffOutSize, 0 }; + toRead = ZSTD_decompressStream(dstream, &cursout , &cursin); + /* note : data is just "sinked" into buffOut */ + } + } + + fclose_orDie(fin); + free(buffIn); + free(buffOut); +} + + +int main(int argc, const char** argv) +{ + const char* const exeName = argv[0]; + const char* const inFilename = argv[1]; + + if (argc!=2) { + printf("wrong arguments\n"); + printf("usage:\n"); + printf("%s FILE\n", exeName); + return 1; + } + + decompressFile_orDie(inFilename); + printf("%s correctly decoded (in memory). \n", inFilename); + + return 0; +} From fb5e38516497d3ada812040d2b7bb91388fc77ad Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 13 Aug 2016 20:49:47 +0200 Subject: [PATCH 08/14] minor changes --- programs/fileio.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index b04ee3b04..01e08cfbe 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -22,8 +22,7 @@ - zstd homepage : http://www.zstd.net */ /* - Note : this is stand-alone program. - It is not part of ZSTD compression library, it is a user program of ZSTD library. + Note : this file is part of zstd command line, which is not library. The license of ZSTD library is BSD. The license of this file is GPLv2. */ @@ -173,10 +172,9 @@ static FILE* FIO_openSrcFile(const char* srcFileName) SET_BINARY_MODE(stdin); } else { f = fopen(srcFileName, "rb"); + if ( f==NULL ) DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno)); } - if ( f==NULL ) DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno)); - return f; } @@ -212,9 +210,9 @@ static FILE* FIO_openDstFile(const char* dstFileName) while ((ch!=EOF) && (ch!='\n')) ch = getchar(); /* flush rest of input line */ } } } f = fopen( dstFileName, "wb" ); + if (f==NULL) DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno)); } - if (f==NULL) DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno)); return f; } From 6263ba5451597690c19578993a304e7ef4a21129 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 13 Aug 2016 23:45:45 +0200 Subject: [PATCH 09/14] fileio uses new streaming API --- programs/Makefile | 10 +++---- programs/fileio.c | 75 +++++++++++++++++++++++------------------------ 2 files changed, 42 insertions(+), 43 deletions(-) diff --git a/programs/Makefile b/programs/Makefile index 285d7f9e8..a51d438b5 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -94,7 +94,7 @@ all32: cleano32 all $(ZSTDDIR)/decompress/zstd_decompress.o: CFLAGS += $(ALIGN_LOOP) -zstd : $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZBUFF_FILES) $(ZDICT_FILES) \ +zstd : $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \ zstdcli.c fileio.c bench.c datagen.c dibio.c $(CC) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT) @@ -116,15 +116,15 @@ zstd-pgo : clean zstd rm zstd $(MAKE) zstd MOREFLAGS=-fprofile-use -zstd-frugal: $(ZSTD_FILES) $(ZBUFF_FILES) zstdcli.c fileio.c +zstd-frugal: $(ZSTD_FILES) zstdcli.c fileio.c $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_LEGACY_SUPPORT=0 $^ -o zstd$(EXT) zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) \ - $(ZSTDDIR)/compress/zbuff_compress.c zstdcli.c fileio.c + zstdcli.c fileio.c $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT) zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) \ - $(ZSTDDIR)/decompress/zbuff_decompress.c zstdcli.c fileio.c + zstdcli.c fileio.c $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT) zstd-small: clean @@ -152,7 +152,7 @@ zbufftest32 : CFLAGS += -m32 zbufftest32 : EXT := 32$(EXT) zbufftest32 : zbufftest -zstreamtest : $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c zstreamtest.c +zstreamtest : $(ZSTD_FILES) datagen.c zstreamtest.c $(CC) $(FLAGS) $^ -o $@$(EXT) zstreamtest32 : CFLAGS += -m32 diff --git a/programs/fileio.c b/programs/fileio.c index 01e08cfbe..e0bba2087 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -58,8 +58,6 @@ #include "fileio.h" #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */ #include "zstd.h" -#define ZBUFF_STATIC_LINKING_ONLY -#include "zbuff.h" #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) # include "zstd_legacy.h" /* ZSTD_isLegacy */ @@ -263,7 +261,7 @@ typedef struct { size_t dstBufferSize; void* dictBuffer; size_t dictBufferSize; - ZBUFF_CCtx* ctx; + ZSTD_CStream* cctx; FILE* dstFile; FILE* srcFile; } cRess_t; @@ -273,11 +271,11 @@ static cRess_t FIO_createCResources(const char* dictFileName) cRess_t ress; memset(&ress, 0, sizeof(ress)); - ress.ctx = ZBUFF_createCCtx(); - if (ress.ctx == NULL) EXM_THROW(30, "zstd: allocation error : can't create ZBUFF context"); - ress.srcBufferSize = ZBUFF_recommendedCInSize(); + ress.cctx = ZSTD_createCStream(); + if (ress.cctx == NULL) EXM_THROW(30, "zstd: allocation error : can't create ZSTD_CStream"); + ress.srcBufferSize = ZSTD_CStreamInSize(); ress.srcBuffer = malloc(ress.srcBufferSize); - ress.dstBufferSize = ZBUFF_recommendedCOutSize(); + ress.dstBufferSize = ZSTD_CStreamOutSize(); ress.dstBuffer = malloc(ress.dstBufferSize); if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(31, "zstd: allocation error : not enough memory"); @@ -293,8 +291,8 @@ static void FIO_freeCResources(cRess_t ress) free(ress.srcBuffer); free(ress.dstBuffer); free(ress.dictBuffer); - errorCode = ZBUFF_freeCCtx(ress.ctx); - if (ZBUFF_isError(errorCode)) EXM_THROW(38, "zstd: error : can't release ZBUFF context resource : %s", ZBUFF_getErrorName(errorCode)); + errorCode = ZSTD_freeCStream(ress.cctx); + if (ZSTD_isError(errorCode)) EXM_THROW(38, "zstd: error : can't release ZSTD_CStream : %s", ZSTD_getErrorName(errorCode)); } @@ -322,8 +320,8 @@ static int FIO_compressFilename_internal(cRess_t ress, params.cParams.windowLog = g_maxWLog; params.cParams = ZSTD_adjustCParams(params.cParams, fileSize, ress.dictBufferSize); } - { size_t const errorCode = ZBUFF_compressInit_advanced(ress.ctx, ress.dictBuffer, ress.dictBufferSize, params, fileSize); - if (ZBUFF_isError(errorCode)) EXM_THROW(21, "Error initializing compression : %s", ZBUFF_getErrorName(errorCode)); + { size_t const errorCode = ZSTD_initCStream_advanced(ress.cctx, ress.dictBuffer, ress.dictBufferSize, params, fileSize); + if (ZSTD_isError(errorCode)) EXM_THROW(21, "Error initializing compression : %s", ZSTD_getErrorName(errorCode)); } } /* Main compression loop */ @@ -335,30 +333,30 @@ static int FIO_compressFilename_internal(cRess_t ress, DISPLAYUPDATE(2, "\rRead : %u MB ", (U32)(readsize>>20)); /* Compress using buffered streaming */ - { size_t usedInSize = inSize; - size_t cSize = ress.dstBufferSize; - { size_t const result = ZBUFF_compressContinue(ress.ctx, ress.dstBuffer, &cSize, ress.srcBuffer, &usedInSize); - if (ZBUFF_isError(result)) EXM_THROW(23, "Compression error : %s ", ZBUFF_getErrorName(result)); } - if (inSize != usedInSize) + { ZSTD_rCursor rCursor = { ress.srcBuffer, inSize }; + ZSTD_wCursor wCursor = { ress.dstBuffer, ress.dstBufferSize, 0 }; + { size_t const result = ZSTD_compressStream(ress.cctx, &wCursor, &rCursor); + if (ZSTD_isError(result)) EXM_THROW(23, "Compression error : %s ", ZSTD_getErrorName(result)); } + if (rCursor.size != 0) /* inBuff should be entirely consumed since buffer sizes are recommended ones */ EXM_THROW(24, "Compression error : input block not fully consumed"); /* Write cBlock */ - { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, cSize, dstFile); - if (sizeCheck!=cSize) EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName); } - compressedfilesize += cSize; + { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, wCursor.nbBytesWritten, dstFile); + if (sizeCheck!=wCursor.nbBytesWritten) EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName); } + compressedfilesize += wCursor.nbBytesWritten; } DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (U32)(readsize>>20), (double)compressedfilesize/readsize*100); } /* End of Frame */ - { size_t cSize = ress.dstBufferSize; - size_t const result = ZBUFF_compressEnd(ress.ctx, ress.dstBuffer, &cSize); + { ZSTD_wCursor wCursor = { ress.dstBuffer, ress.dstBufferSize, 0 }; + size_t const result = ZSTD_endStream(ress.cctx, &wCursor); if (result!=0) EXM_THROW(26, "Compression error : cannot create frame end"); - { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, cSize, dstFile); - if (sizeCheck!=cSize) EXM_THROW(27, "Write error : cannot write frame end into %s", dstFileName); } - compressedfilesize += cSize; + { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, wCursor.nbBytesWritten, dstFile); + if (sizeCheck!=wCursor.nbBytesWritten) EXM_THROW(27, "Write error : cannot write frame end into %s", dstFileName); } + compressedfilesize += wCursor.nbBytesWritten; } /* Status */ @@ -494,7 +492,7 @@ typedef struct { size_t dstBufferSize; void* dictBuffer; size_t dictBufferSize; - ZBUFF_DCtx* dctx; + ZSTD_DStream* dctx; FILE* dstFile; } dRess_t; @@ -504,11 +502,11 @@ static dRess_t FIO_createDResources(const char* dictFileName) memset(&ress, 0, sizeof(ress)); /* Allocation */ - ress.dctx = ZBUFF_createDCtx(); - if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZBUFF decompression context"); - ress.srcBufferSize = ZBUFF_recommendedDInSize(); + ress.dctx = ZSTD_createDStream(); + if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZSTD_DStream"); + ress.srcBufferSize = ZSTD_DStreamInSize(); ress.srcBuffer = malloc(ress.srcBufferSize); - ress.dstBufferSize = ZBUFF_recommendedDOutSize(); + ress.dstBufferSize = ZSTD_DStreamOutSize(); ress.dstBuffer = malloc(ress.dstBufferSize); if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory"); @@ -520,8 +518,8 @@ static dRess_t FIO_createDResources(const char* dictFileName) static void FIO_freeDResources(dRess_t ress) { - size_t const errorCode = ZBUFF_freeDCtx(ress.dctx); - if (ZBUFF_isError(errorCode)) EXM_THROW(69, "Error : can't free ZBUFF context resource : %s", ZBUFF_getErrorName(errorCode)); + size_t const errorCode = ZSTD_freeDStream(ress.dctx); + if (ZSTD_isError(errorCode)) EXM_THROW(69, "Error : can't free ZSTD_DStream context resource : %s", ZSTD_getErrorName(errorCode)); free(ress.srcBuffer); free(ress.dstBuffer); free(ress.dictBuffer); @@ -613,7 +611,7 @@ unsigned long long FIO_decompressFrame(dRess_t ress, size_t readSize; U32 storedSkips = 0; - ZBUFF_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize); + ZSTD_initDStream_usingDict(ress.dctx, ress.dictBuffer, ress.dictBufferSize); /* Header loading (optional, saves one loop) */ { size_t const toLoad = 9 - alreadyLoaded; /* assumption : 9 >= alreadyLoaded */ @@ -623,14 +621,15 @@ unsigned long long FIO_decompressFrame(dRess_t ress, /* Main decompression Loop */ while (1) { - size_t inSize=readSize, decodedSize=ress.dstBufferSize; - size_t const toRead = ZBUFF_decompressContinue(ress.dctx, ress.dstBuffer, &decodedSize, ress.srcBuffer, &inSize); - if (ZBUFF_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFF_getErrorName(toRead)); - readSize -= inSize; + ZSTD_rCursor rCursor = { ress.srcBuffer, readSize }; + ZSTD_wCursor wCursor = { ress.dstBuffer, ress.dstBufferSize, 0 }; + size_t const toRead = ZSTD_decompressStream(ress.dctx, &wCursor, &rCursor ); + if (ZSTD_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZSTD_getErrorName(toRead)); + readSize = rCursor.size; /* Write block */ - storedSkips = FIO_fwriteSparse(foutput, ress.dstBuffer, decodedSize, storedSkips); - frameSize += decodedSize; + storedSkips = FIO_fwriteSparse(foutput, ress.dstBuffer, wCursor.nbBytesWritten, storedSkips); + frameSize += wCursor.nbBytesWritten; DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(frameSize>>20) ); if (toRead == 0) break; /* end of frame */ From 655393cc72d405eb77a403ac92c65397fa5f848f Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 14 Aug 2016 00:16:20 +0200 Subject: [PATCH 10/14] updated doc for streaming API --- NEWS | 1 + lib/README.md | 14 +++--- lib/zstd.h | 116 ++++++++++++++++++++++++++++++++++++-------------- 3 files changed, 94 insertions(+), 37 deletions(-) diff --git a/NEWS b/NEWS index 77edad474..36bd15242 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,5 @@ v0.8.1 +New streaming API Changed : --ultra now enables levels beyond 19 Changed : -i# now selects benchmark time in second Fixed : ZSTD_compress* can now compress > 4 GB in a single pass, reported by Nick Terrell diff --git a/lib/README.md b/lib/README.md index a3087f05e..f6dbc4181 100644 --- a/lib/README.md +++ b/lib/README.md @@ -37,15 +37,18 @@ Other optional functionalities provided are : - `legacy/` : source code to decompress previous versions of zstd, starting from `v0.1`. This module also depends on `common/` and `decompress/` . - Note that it's required to compile the library with `ZSTD_LEGACY_SUPPORT = 1` . + Library compilation must include directive `ZSTD_LEGACY_SUPPORT = 1` . The main API can be consulted in `legacy/zstd_legacy.h`. - Advanced API from each version can be found in its relevant header file. - For example, advanced API for version `v0.4` is in `zstd_v04.h` . + Advanced API from each version can be found in their relevant header file. + For example, advanced API for version `v0.4` is in `legacy/zstd_v04.h` . -#### Streaming API +#### Obsolete streaming API -Streaming is currently provided by `common/zbuff.h`. +Streaming is now provided within `zstd.h`. +Older streaming API is still provided within `common/zbuff.h`. +It is considered obsolete, and will be removed in a future version. +Consider migrating towards newer streaming API. #### Miscellaneous @@ -55,3 +58,4 @@ The other files are not source code. There are : - LICENSE : contains the BSD license text - Makefile : script to compile or install zstd library (static and dynamic) - libzstd.pc.in : for pkg-config (`make install`) + - README.md : this file diff --git a/lib/zstd.h b/lib/zstd.h index ff2f627e7..718416d4b 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -36,7 +36,7 @@ extern "C" { #endif -/*====== Dependency ======*/ +/*====== Dependency ======*/ #include /* size_t */ @@ -52,7 +52,7 @@ extern "C" { #endif -/*====== Version ======*/ +/*======= Version =======*/ #define ZSTD_VERSION_MAJOR 0 #define ZSTD_VERSION_MINOR 8 #define ZSTD_VERSION_RELEASE 1 @@ -84,15 +84,13 @@ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, * potentially larger than what local system can handle as a single memory segment. * In which case, it's necessary to use streaming mode to decompress data. * note 2 : decompressed size is an optional field, that may not be present. -* When `return==0`, consider data to decompress could have any size. -* In which case, it's necessary to use streaming mode to decompress data, -* or rely on application's implied limits. -* (For example, it may know that its own data is necessarily cut into blocks <= 16 KB). +* When `return==0`, data to decompress can have any size. +* In which case, it's necessary to use streaming mode to decompress data. +* Optionally, application may rely on its own implied limits. +* (For example, application own data could be necessarily cut into blocks <= 16 KB). * note 3 : decompressed size could be wrong or intentionally modified ! * Always ensure result fits within application's authorized limits ! -* Each application can have its own set of conditions. -* If the intention is to decompress public data compressed by zstd command line utility, -* it is recommended to support at least 8 MB for extended compatibility. +* Each application can set its own limits. * note 4 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more. */ unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); @@ -100,7 +98,7 @@ unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); `compressedSize` : must be the _exact_ size of compressed input, otherwise decompression will fail. `dstCapacity` must be equal or larger than originalSize (see ZSTD_getDecompressedSize() ). If originalSize is unknown, and if there is no implied application-specific limitations, - it's necessary to use streaming mode to decompress data. + it's preferable to use streaming mode to decompress data. @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), or an errorCode if it fails (which can be tested using ZSTD_isError()) */ ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity, @@ -141,7 +139,7 @@ ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapa ***************************/ /*! ZSTD_compress_usingDict() : * Compression using a predefined Dictionary (see dictBuilder/zdict.h). -* Note : This function load the dictionary, resulting in a significant startup time. */ +* Note : This function load the dictionary, resulting in significant startup delay. */ ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, @@ -151,7 +149,7 @@ ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, /*! ZSTD_decompress_usingDict() : * Decompression using a predefined Dictionary (see dictBuilder/zdict.h). * Dictionary must be identical to the one used during compression. -* Note : This function load the dictionary, resulting in a significant startup time */ +* Note : This function load the dictionary, resulting in significant startup delay */ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, @@ -163,7 +161,7 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, ****************************/ /*! ZSTD_createCDict() : * Create a digested dictionary, ready to start compression operation without startup delay. -* `dict` can be released after creation */ +* `dict` can be released after ZSTD_CDict creation */ typedef struct ZSTD_CDict_s ZSTD_CDict; ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel); ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict); @@ -232,19 +230,19 @@ static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable f typedef enum { ZSTD_fast, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt } ZSTD_strategy; /*< from faster to stronger */ typedef struct { - unsigned windowLog; /*< largest match distance : larger == more compression, more memory needed during decompression */ - unsigned chainLog; /*< fully searched segment : larger == more compression, slower, more memory (useless for fast) */ - unsigned hashLog; /*< dispatch table : larger == faster, more memory */ - unsigned searchLog; /*< nb of searches : larger == more compression, slower */ - unsigned searchLength; /*< match length searched : larger == faster decompression, sometimes less compression */ - unsigned targetLength; /*< acceptable match size for optimal parser (only) : larger == more compression, slower */ + unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */ + unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */ + unsigned hashLog; /**< dispatch table : larger == faster, more memory */ + unsigned searchLog; /**< nb of searches : larger == more compression, slower */ + unsigned searchLength; /**< match length searched : larger == faster decompression, sometimes less compression */ + unsigned targetLength; /**< acceptable match size for optimal parser (only) : larger == more compression, slower */ ZSTD_strategy strategy; } ZSTD_compressionParameters; typedef struct { - unsigned contentSizeFlag; /*< 1: content size will be in frame header (if known). */ - unsigned checksumFlag; /*< 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */ - unsigned noDictIDFlag; /*< 1: no dict ID will be saved into frame header (if dictionary compression) */ + unsigned contentSizeFlag; /**< 1: content size will be in frame header (if known). */ + unsigned checksumFlag; /**< 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */ + unsigned noDictIDFlag; /**< 1: no dict ID will be saved into frame header (if dictionary compression) */ } ZSTD_frameParameters; typedef struct { @@ -327,25 +325,59 @@ ZSTDLIB_API size_t ZSTD_sizeofDCtx(const ZSTD_DCtx* dctx); ********************************************************************/ typedef struct ZSTD_readCursor_s { - const void* ptr; /* position of cursor - update to new position */ - size_t size; /* remaining buffer size to read - update preserves end of buffer */ + const void* ptr; /**< position of cursor - update to new position */ + size_t size; /**< remaining buffer size to read - update preserves end of buffer */ } ZSTD_rCursor; typedef struct ZSTD_writeCursor_s { - void* ptr; /* position of cursor - update to new position */ - size_t size; /* remaining buffer size to write - update preserves end of buffer */ - size_t nbBytesWritten; /* already written bytes - update adds bytes newly written (accumulator) */ + void* ptr; /**< position of cursor - update to new position */ + size_t size; /**< remaining buffer size to write - update preserves end of buffer */ + size_t nbBytesWritten; /**< already written bytes - update adds bytes newly written (accumulator) */ } ZSTD_wCursor; /*====== compression ======*/ +/*-*********************************************************************** +* Streaming compression - howto +* +* A ZSTD_CStream object is required to track streaming operation. +* Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources. +* ZSTD_CStream objects can be reused multiple times on consecutive compression operations. +* +* Start by initializing ZSTD_CStream. +* Use ZSTD_initCStream() to start a new compression operation. +* Use ZSTD_initCStream_usingDict() for a compression which requires a dictionary. +* +* Use ZSTD_compressStream() repetitively to consume input stream. +* The function will automatically advance cursors. +* Note that it may not consume the entire input, in which case `input->size > 0`, +* and it's up to the caller to present again remaining data. +* @return : a hint to preferred nb of bytes to use as input for next function call (it's just a hint, to improve latency) +* or an error code, which can be tested using ZSTD_isError(). +* +* At any moment, it's possible to flush whatever data remains within buffer, using ZSTD_flushStream(). +* Cursor will be updated. +* Note some content might still be left within internal buffer if `output->size` is too small. +* @return : nb of bytes still present within internal buffer (0 if it's empty) +* or an error code, which can be tested using ZSTD_isError(). +* +* ZSTD_endStream() instructs to finish a frame. +* It will perform a flush and write frame epilogue. +* The epilogue is required for decoders to consider a frame completed. +* Similar to ZSTD_flushStream(), it may not be able to flush the full content if `output->size` is too small. +* In which case, call again ZSTD_endStream() to complete the flush. +* @return : nb of bytes still present within internal buffer (0 if it's empty) +* or an error code, which can be tested using ZSTD_isError(). +* +* *******************************************************************/ + typedef struct ZSTD_CStream_s ZSTD_CStream; ZSTD_CStream* ZSTD_createCStream(void); size_t ZSTD_freeCStream(ZSTD_CStream* zcs); -size_t ZSTD_CStreamInSize(void); /*!< recommended size for input buffer */ -size_t ZSTD_CStreamOutSize(void); /*!< recommended size for output buffer */ +size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ +size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer */ size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_wCursor* output, ZSTD_rCursor* input); @@ -361,6 +393,27 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dic /*====== decompression ======*/ +/*-*************************************************************************** +* Streaming decompression howto +* +* A ZSTD_DStream object is required to track streaming operations. +* Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. +* ZSTD_DStream objects can be re-init multiple times. +* +* Use ZSTD_initDStream() to start a new decompression operation, +* or ZSTD_initDStream_usingDict() if decompression requires a dictionary. +* +* Use ZSTD_decompressStream() repetitively to consume your input. +* The function will update cursors. +* Note that it may not consume the entire input (input->size > 0), +* in which case it's up to the caller to present remaining input again. +* @return : 0 when a frame is completely decoded and fully flushed, +* 1 when there is still some data left within internal buffer to flush, +* >1 when more data is expected, with value being a suggested next input size (it's just a hint, which helps latency, any size is accepted), +* or an error code, which can be tested using ZSTD_isError(). +* +* *******************************************************************************/ + typedef struct ZSTD_DStream_s ZSTD_DStream; ZSTD_DStream* ZSTD_createDStream(void); size_t ZSTD_freeDStream(ZSTD_DStream* zds); @@ -381,9 +434,8 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di * Buffer-less and synchronous inner streaming functions ********************************************************************/ /* This is an advanced API, giving full control over buffer management, for users which need direct control over memory. -* But it's also a complex one, with a lot of restrictions (documented below). -* For an easier streaming API, look into common/zbuff.h -* which removes all restrictions by allocating and managing its own internal buffer */ +* But it's also a complex one, with many restrictions (documented below). +* Prefer using normal streaming API for an easier experience */ ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); From 94ca85d01b7f582de40375307fef45bffb499649 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 14 Aug 2016 01:19:12 +0200 Subject: [PATCH 11/14] fixed Visual warning --- programs/fileio.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index e0bba2087..66ccbc784 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -40,8 +40,11 @@ /* ************************************* * Compiler Options ***************************************/ -#define _POSIX_SOURCE 1 /* enable %llu on Windows */ -#define _CRT_SECURE_NO_WARNINGS /* removes Visual warning on strerror() */ +#ifdef _MSC_VER /* Visual */ +# define _POSIX_SOURCE 1 /* enable %llu on Windows */ +# define _CRT_SECURE_NO_WARNINGS /* removes Visual warning on strerror() */ +# pragma warning(disable : 4204) /* non-constant aggregate initializer */ +#endif /*-************************************* From 53e17fbd5e9f84d6fae9a6a221f7420233698d94 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 17 Aug 2016 01:39:22 +0200 Subject: [PATCH 12/14] updated streaming API --- lib/compress/zstd_compress.c | 43 +++++------ lib/decompress/zstd_decompress.c | 20 ++--- lib/zstd.h | 37 ++++----- programs/fileio.c | 37 +++++---- programs/zstreamtest.c | 129 ++++++++++++++++--------------- 5 files changed, 128 insertions(+), 138 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 6e8f0fba7..6de2e900c 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2960,16 +2960,15 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, } } -size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_wCursor* output, ZSTD_rCursor* input) +size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input) { - size_t sizeRead = input->size; - size_t sizeWritten = output->size; - size_t const result = ZSTD_compressStream_generic(zcs, output->ptr, &sizeWritten, input->ptr, &sizeRead, zsf_gather); - input->ptr = (const char*)(input->ptr) + sizeRead; - input->size -= sizeRead; - output->ptr = (char*)(output->ptr) + sizeWritten; - output->size -= sizeWritten; - output->nbBytesWritten += sizeWritten; + size_t sizeRead = input->size - input->pos; + size_t sizeWritten = output->size - output->pos; + size_t const result = ZSTD_compressStream_generic(zcs, + (char*)(output->dst) + output->pos, &sizeWritten, + (const char*)(input->src) + input->pos, &sizeRead, zsf_gather); + input->pos += sizeRead; + output->pos += sizeWritten; return result; } @@ -2978,36 +2977,32 @@ size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_wCursor* output, ZSTD_rCursor /*! ZSTD_flushStream() : * @return : amount of data remaining to flush */ -size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_wCursor* output) +size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) { size_t srcSize = 0; - size_t sizeWritten = output->size; - size_t const result = ZSTD_compressStream_generic(zcs, output->ptr, &sizeWritten, &srcSize, &srcSize, zsf_flush); /* use a valid src address instead of NULL */ - output->ptr = (char*)(output->ptr) + sizeWritten; - output->size -= sizeWritten; - output->nbBytesWritten += sizeWritten; + size_t sizeWritten = output->size - output->pos; + size_t const result = ZSTD_compressStream_generic(zcs, output->dst + output->pos, &sizeWritten, &srcSize, &srcSize, zsf_flush); /* use a valid src address instead of NULL */ + output->pos += sizeWritten; if (ZSTD_isError(result)) return result; return zcs->outBuffContentSize - zcs->outBuffFlushedSize; /* remaining to flush */ } -size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_wCursor* output) +size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) { - BYTE* const ostart = (BYTE*)(output->ptr); - BYTE* const oend = ostart + output->size; + BYTE* const ostart = (BYTE*)(output->dst) + output->pos; + BYTE* const oend = (BYTE*)(output->dst) + output->size; BYTE* op = ostart; if (zcs->stage != zcss_final) { /* flush whatever remains */ size_t srcSize = 0; - size_t sizeWritten = output->size; + size_t sizeWritten = output->size - output->pos; size_t const notEnded = ZSTD_compressStream_generic(zcs, ostart, &sizeWritten, &srcSize, &srcSize, zsf_end); /* use a valid src address instead of NULL */ size_t const remainingToFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; op += sizeWritten; if (remainingToFlush) { - output->ptr = op; - output->size -= sizeWritten; - output->nbBytesWritten += sizeWritten; + output->pos += sizeWritten; return remainingToFlush + ZSTD_BLOCKHEADERSIZE /* final empty block */ + (zcs->checksum * 4); } /* create epilogue */ @@ -3021,9 +3016,7 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_wCursor* output) size_t const flushed = ZSTD_limitCopy(op, oend-op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush); op += flushed; zcs->outBuffFlushedSize += flushed; - output->ptr = op; - output->size = oend-op; - output->nbBytesWritten += op-ostart; + output->pos += op-ostart; if (toFlush==flushed) zcs->stage = zcss_init; /* end reached */ return toFlush - flushed; } diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 57ca21112..74e166add 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1393,13 +1393,13 @@ MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, } -size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_wCursor* output, ZSTD_rCursor* input) +size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input) { - const char* const istart = (const char*)(input->ptr); - const char* const iend = istart + input->size; + const char* const istart = (const char*)(input->src) + input->pos; + const char* const iend = (const char*)(input->src) + input->size; const char* ip = istart; - char* const ostart = (char*)(output->ptr); - char* const oend = ostart + output->size; + char* const ostart = (char*)(output->dst) + output->pos; + char* const oend = (char*)(output->dst) + output->size; char* op = ostart; U32 someMoreWork = 1; @@ -1417,8 +1417,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_wCursor* output, ZSTD_rCurs if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */ memcpy(zds->headerBuffer + zds->lhSize, ip, iend-ip); zds->lhSize += iend-ip; - input->ptr = iend; - input->size = 0; + input->pos = input->size; return (hSize - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ } memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad; @@ -1522,11 +1521,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_wCursor* output, ZSTD_rCurs } } /* result */ - input->ptr = ip; - input->size = (size_t)(iend-ip); - output->ptr = op; - output->size = (size_t) (oend-op); - output->nbBytesWritten += (size_t)(op-ostart); + input->pos += (size_t)(ip-istart); + output->pos += (size_t)(op-ostart); { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->zd); if (!nextSrcSizeHint) return (zds->outEnd != zds->outStart); /* return 0 only if fully flushed too */ nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->zd) == ZSTDnit_block); diff --git a/lib/zstd.h b/lib/zstd.h index 718416d4b..056940ff1 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -324,16 +324,17 @@ ZSTDLIB_API size_t ZSTD_sizeofDCtx(const ZSTD_DCtx* dctx); * Streaming ********************************************************************/ -typedef struct ZSTD_readCursor_s { - const void* ptr; /**< position of cursor - update to new position */ - size_t size; /**< remaining buffer size to read - update preserves end of buffer */ -} ZSTD_rCursor; +typedef struct ZSTD_inBuffer_s { + const void* src; /**< start of input buffer */ + size_t size; /**< size of input buffer */ + size_t pos; /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */ +} ZSTD_inBuffer; -typedef struct ZSTD_writeCursor_s { - void* ptr; /**< position of cursor - update to new position */ - size_t size; /**< remaining buffer size to write - update preserves end of buffer */ - size_t nbBytesWritten; /**< already written bytes - update adds bytes newly written (accumulator) */ -} ZSTD_wCursor; +typedef struct ZSTD_outBuffer_s { + void* dst; /**< start of output buffer */ + size_t size; /**< size of output buffer */ + size_t pos; /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */ +} ZSTD_outBuffer; /*====== compression ======*/ @@ -350,14 +351,14 @@ typedef struct ZSTD_writeCursor_s { * Use ZSTD_initCStream_usingDict() for a compression which requires a dictionary. * * Use ZSTD_compressStream() repetitively to consume input stream. -* The function will automatically advance cursors. -* Note that it may not consume the entire input, in which case `input->size > 0`, +* The function will automatically update both `pos`. +* Note that it may not consume the entire input, in which case `pos < size`, * and it's up to the caller to present again remaining data. * @return : a hint to preferred nb of bytes to use as input for next function call (it's just a hint, to improve latency) * or an error code, which can be tested using ZSTD_isError(). * * At any moment, it's possible to flush whatever data remains within buffer, using ZSTD_flushStream(). -* Cursor will be updated. +* `output->pos` will be updated. * Note some content might still be left within internal buffer if `output->size` is too small. * @return : nb of bytes still present within internal buffer (0 if it's empty) * or an error code, which can be tested using ZSTD_isError(). @@ -380,9 +381,9 @@ size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer */ size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); -size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_wCursor* output, ZSTD_rCursor* input); -size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_wCursor* output); -size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_wCursor* output); +size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); +size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); +size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); /* advanced */ ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); @@ -404,8 +405,8 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dic * or ZSTD_initDStream_usingDict() if decompression requires a dictionary. * * Use ZSTD_decompressStream() repetitively to consume your input. -* The function will update cursors. -* Note that it may not consume the entire input (input->size > 0), +* The function will update both `pos`. +* Note that it may not consume the entire input (pos < size), * in which case it's up to the caller to present remaining input again. * @return : 0 when a frame is completely decoded and fully flushed, * 1 when there is still some data left within internal buffer to flush, @@ -422,7 +423,7 @@ size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer */ size_t ZSTD_initDStream(ZSTD_DStream* zds); -size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_wCursor* output, ZSTD_rCursor* input); +size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); /* advanced */ ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); diff --git a/programs/fileio.c b/programs/fileio.c index 66ccbc784..04d5a71ac 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -336,30 +336,30 @@ static int FIO_compressFilename_internal(cRess_t ress, DISPLAYUPDATE(2, "\rRead : %u MB ", (U32)(readsize>>20)); /* Compress using buffered streaming */ - { ZSTD_rCursor rCursor = { ress.srcBuffer, inSize }; - ZSTD_wCursor wCursor = { ress.dstBuffer, ress.dstBufferSize, 0 }; - { size_t const result = ZSTD_compressStream(ress.cctx, &wCursor, &rCursor); + { ZSTD_inBuffer inBuff = { ress.srcBuffer, inSize, 0 }; + ZSTD_outBuffer outBuff= { ress.dstBuffer, ress.dstBufferSize, 0 }; + { size_t const result = ZSTD_compressStream(ress.cctx, &outBuff, &inBuff); if (ZSTD_isError(result)) EXM_THROW(23, "Compression error : %s ", ZSTD_getErrorName(result)); } - if (rCursor.size != 0) + if (inBuff.pos != inBuff.size) /* inBuff should be entirely consumed since buffer sizes are recommended ones */ EXM_THROW(24, "Compression error : input block not fully consumed"); /* Write cBlock */ - { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, wCursor.nbBytesWritten, dstFile); - if (sizeCheck!=wCursor.nbBytesWritten) EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName); } - compressedfilesize += wCursor.nbBytesWritten; + { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile); + if (sizeCheck!=outBuff.pos) EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName); } + compressedfilesize += outBuff.pos; } DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (U32)(readsize>>20), (double)compressedfilesize/readsize*100); } /* End of Frame */ - { ZSTD_wCursor wCursor = { ress.dstBuffer, ress.dstBufferSize, 0 }; - size_t const result = ZSTD_endStream(ress.cctx, &wCursor); + { ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 }; + size_t const result = ZSTD_endStream(ress.cctx, &outBuff); if (result!=0) EXM_THROW(26, "Compression error : cannot create frame end"); - { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, wCursor.nbBytesWritten, dstFile); - if (sizeCheck!=wCursor.nbBytesWritten) EXM_THROW(27, "Write error : cannot write frame end into %s", dstFileName); } - compressedfilesize += wCursor.nbBytesWritten; + { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile); + if (sizeCheck!=outBuff.pos) EXM_THROW(27, "Write error : cannot write frame end into %s", dstFileName); } + compressedfilesize += outBuff.pos; } /* Status */ @@ -624,19 +624,18 @@ unsigned long long FIO_decompressFrame(dRess_t ress, /* Main decompression Loop */ while (1) { - ZSTD_rCursor rCursor = { ress.srcBuffer, readSize }; - ZSTD_wCursor wCursor = { ress.dstBuffer, ress.dstBufferSize, 0 }; - size_t const toRead = ZSTD_decompressStream(ress.dctx, &wCursor, &rCursor ); + ZSTD_inBuffer inBuff = { ress.srcBuffer, readSize, 0 }; + ZSTD_outBuffer outBuff= { ress.dstBuffer, ress.dstBufferSize, 0 }; + size_t const toRead = ZSTD_decompressStream(ress.dctx, &outBuff, &inBuff ); if (ZSTD_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZSTD_getErrorName(toRead)); - readSize = rCursor.size; /* Write block */ - storedSkips = FIO_fwriteSparse(foutput, ress.dstBuffer, wCursor.nbBytesWritten, storedSkips); - frameSize += wCursor.nbBytesWritten; + storedSkips = FIO_fwriteSparse(foutput, ress.dstBuffer, outBuff.pos, storedSkips); + frameSize += outBuff.pos; DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(frameSize>>20) ); if (toRead == 0) break; /* end of frame */ - if (readSize) EXM_THROW(37, "Decoding error : should consume entire input"); + if (inBuff.size != inBuff.pos) EXM_THROW(37, "Decoding error : should consume entire input"); /* Fill input buffer */ if (toRead > ress.srcBufferSize) EXM_THROW(38, "too large block"); diff --git a/programs/zstreamtest.c b/programs/zstreamtest.c index 7ef58fe7e..7f136b1b5 100644 --- a/programs/zstreamtest.c +++ b/programs/zstreamtest.c @@ -155,8 +155,8 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo U32 testNb=0; ZSTD_CStream* zc = ZSTD_createCStream_advanced(customMem); ZSTD_DStream* zd = ZSTD_createDStream_advanced(customMem); - ZSTD_rCursor rCursor; - ZSTD_wCursor wCursor; + ZSTD_inBuffer inBuff; + ZSTD_outBuffer outBuff; /* Create compressible test buffer */ if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd) { @@ -173,39 +173,41 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo /* Basic compression test */ DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); ZSTD_initCStream_usingDict(zc, CNBuffer, 128 KB, 1); - wCursor.ptr = (char*)(compressedBuffer)+cSize; - wCursor.size = compressedBufferSize; - wCursor.nbBytesWritten = 0; - rCursor.ptr = CNBuffer; - rCursor.size = CNBufferSize; - { size_t const r = ZSTD_compressStream(zc, &wCursor, &rCursor); + outBuff.dst = (char*)(compressedBuffer)+cSize; + outBuff.size = compressedBufferSize; + outBuff.pos = 0; + inBuff.src = CNBuffer; + inBuff.size = CNBufferSize; + inBuff.pos = 0; + { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff); if (ZSTD_isError(r)) goto _output_error; } - if (rCursor.size != 0) goto _output_error; /* entire input should be consumed */ - { size_t const r = ZSTD_endStream(zc, &wCursor); + if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ + { size_t const r = ZSTD_endStream(zc, &outBuff); if (r != 0) goto _output_error; } /*< error, or some data not flushed */ - cSize += wCursor.nbBytesWritten; + cSize += outBuff.pos; DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); /* skippable frame test */ DISPLAYLEVEL(4, "test%3i : decompress skippable frame : ", testNb++); ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB); - rCursor.ptr = compressedBuffer; - rCursor.size = cSize; - wCursor.ptr = decodedBuffer; - wCursor.size = CNBufferSize; - wCursor.nbBytesWritten = 0; - { size_t const r = ZSTD_decompressStream(zd, &wCursor, &rCursor); + inBuff.src = compressedBuffer; + inBuff.size = cSize; + inBuff.pos = 0; + outBuff.dst = decodedBuffer; + outBuff.size = CNBufferSize; + outBuff.pos = 0; + { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff); if (r != 0) goto _output_error; } - if (wCursor.nbBytesWritten != 0) goto _output_error; /* skippable frame len is 0 */ + if (outBuff.pos != 0) goto _output_error; /* skippable frame len is 0 */ DISPLAYLEVEL(4, "OK \n"); /* Basic decompression test */ DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB); - { size_t const r = ZSTD_decompressStream(zd, &wCursor, &rCursor); + { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff); if (r != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */ - if (wCursor.nbBytesWritten != CNBufferSize) goto _output_error; /* should regenerate the same amount */ - if (rCursor.size != 0) goto _output_error; /* should have read the entire frame */ + if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */ + if (inBuff.pos != inBuff.size) goto _output_error; /* should have read the entire frame */ DISPLAYLEVEL(4, "OK \n"); /* check regenerated data is byte exact */ @@ -220,26 +222,27 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo DISPLAYLEVEL(4, "test%3i : decompress byte-by-byte : ", testNb++); { size_t r = 1; ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB); - rCursor.ptr = compressedBuffer; - wCursor.ptr = decodedBuffer; - wCursor.nbBytesWritten = 0; + inBuff.src = compressedBuffer; + outBuff.dst = decodedBuffer; + inBuff.pos = 0; + outBuff.pos = 0; while (r) { /* skippable frame */ - rCursor.size = 1; - wCursor.size = 1; - r = ZSTD_decompressStream(zd, &wCursor, &rCursor); + inBuff.size = inBuff.pos + 1; + outBuff.size = outBuff.pos + 1; + r = ZSTD_decompressStream(zd, &outBuff, &inBuff); if (ZSTD_isError(r)) goto _output_error; } ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB); r=1; while (r) { /* normal frame */ - rCursor.size = 1; - wCursor.size = 1; - r = ZSTD_decompressStream(zd, &wCursor, &rCursor); + inBuff.size = inBuff.pos + 1; + outBuff.size = outBuff.pos + 1; + r = ZSTD_decompressStream(zd, &outBuff, &inBuff); if (ZSTD_isError(r)) goto _output_error; } } - if (wCursor.nbBytesWritten != CNBufferSize) goto _output_error; /* should regenerate the same amount */ - if ((size_t)(rCursor.ptr - compressedBuffer) != cSize) goto _output_error; /* should have read the entire frame */ + if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */ + if (inBuff.pos != cSize) goto _output_error; /* should have read the entire frame */ DISPLAYLEVEL(4, "OK \n"); /* check regenerated data is byte exact */ @@ -345,7 +348,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres const BYTE* srcBuffer; const BYTE* dict; size_t maxTestSize, dictSize; - size_t cSize, totalTestSize, totalCSize, totalGenSize; + size_t cSize, totalTestSize, totalGenSize; U32 n, nbChunks; XXH64_state_t xxhState; U64 crcOrig; @@ -395,31 +398,30 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres /* multi-segments compression test */ XXH64_reset(&xxhState, 0); nbChunks = (FUZ_rand(&lseed) & 127) + 2; - { ZSTD_wCursor wCursor = { cBuffer, cBufferSize, 0 } ; + { ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ; for (n=0, cSize=0, totalTestSize=0 ; (n= remainingToFlush); - wCursor.size = adjustedDstSize; - remainingToFlush = ZSTD_endStream(zc, &wCursor); + outBuff.size = outBuff.pos + adjustedDstSize; + remainingToFlush = ZSTD_endStream(zc, &outBuff); CHECK (ZSTD_isError(remainingToFlush), "flush error : %s", ZSTD_getErrorName(remainingToFlush)); CHECK (enoughDstSize && remainingToFlush, "ZSTD_endStream() not fully flushed (%u remaining), but enough space available", (U32)remainingToFlush); } } crcOrig = XXH64_digest(&xxhState); - cSize = wCursor.nbBytesWritten; + cSize = outBuff.pos; } /* multi - fragments decompression test */ ZSTD_initDStream_usingDict(zd, dict, dictSize); { size_t decompressionResult = 1; - ZSTD_rCursor rCursor = { cBuffer, cSize }; - ZSTD_wCursor wCursor = { dstBuffer, dstBufferSize, 0 }; - for (totalCSize = 0, totalGenSize = 0 ; decompressionResult ; ) { + ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 }; + ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 }; + for (totalGenSize = 0 ; decompressionResult ; ) { size_t const readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog); size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize); - rCursor.size = readCSrcSize; - wCursor.size = dstBuffSize; - decompressionResult = ZSTD_decompressStream(zd, &wCursor, &rCursor); + inBuff.size = inBuff.pos + readCSrcSize; + outBuff.size = inBuff.pos + dstBuffSize; + decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff); CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult)); - totalCSize += readCSrcSize - rCursor.size; } CHECK (decompressionResult != 0, "frame not fully decoded"); - CHECK (wCursor.nbBytesWritten != totalTestSize, "decompressed data : wrong size") - CHECK (totalCSize != cSize, "compressed data should be fully read") + CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size") + CHECK (inBuff.pos != cSize, "compressed data should be fully read") { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0); if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize); CHECK (crcDest!=crcOrig, "decompressed data corrupted"); @@ -475,15 +476,15 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres /* try decompression on noisy data */ ZSTD_initDStream(zd); - { ZSTD_rCursor rCursor = { cBuffer, cSize }; - ZSTD_wCursor wCursor = { dstBuffer, dstBufferSize, 0 }; - while (wCursor.nbBytesWritten < dstBufferSize) { + { ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 }; + ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 }; + while (outBuff.pos < dstBufferSize) { size_t const randomCSrcSize = FUZ_randomLength(&lseed, maxSampleLog); size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); - size_t const adjustedDstSize = MIN(dstBufferSize - wCursor.nbBytesWritten, randomDstSize); - wCursor.size = adjustedDstSize; - rCursor.size = randomCSrcSize; - { size_t const decompressError = ZSTD_decompressStream(zd, &wCursor, &rCursor); + size_t const adjustedDstSize = MIN(dstBufferSize - outBuff.pos, randomDstSize); + outBuff.size = outBuff.pos + adjustedDstSize; + inBuff.size = inBuff.pos + randomCSrcSize; + { size_t const decompressError = ZSTD_decompressStream(zd, &outBuff, &inBuff); if (ZSTD_isError(decompressError)) break; /* error correctly detected */ } } } } DISPLAY("\r%u fuzzer tests completed \n", testNb); From 2065879793b745530f21594b521d740f351d18dd Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 17 Aug 2016 01:48:43 +0200 Subject: [PATCH 13/14] updated examples --- examples/streaming_compression.c | 16 ++++++++-------- examples/streaming_decompression.c | 11 ++++++----- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/examples/streaming_compression.c b/examples/streaming_compression.c index f99e040c0..e2568836a 100644 --- a/examples/streaming_compression.c +++ b/examples/streaming_compression.c @@ -93,18 +93,18 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initCStream() error \n"); exit(11); } while( (read = fread_orDie(buffIn, toRead, fin)) ) { - ZSTD_rCursor cursin = { buffIn, read }; - while (cursin.size) { - ZSTD_wCursor cursout = { buffOut, buffOutSize, 0 }; - toRead = ZSTD_compressStream(cstream, &cursout , &cursin); - fwrite_orDie(buffOut, cursout.nbBytesWritten, fout); + ZSTD_inBuffer input = { buffIn, read, 0 }; + while (input.pos < input.size) { + ZSTD_outBuffer output = { buffOut, buffOutSize, 0 }; + toRead = ZSTD_compressStream(cstream, &output , &input); + fwrite_orDie(buffOut, output.pos, fout); } } - ZSTD_wCursor cursout = { buffOut, buffOutSize, 0 }; - size_t const remainingToFlush = ZSTD_endStream(cstream, &cursout); + ZSTD_outBuffer output = { buffOut, buffOutSize, 0 }; + size_t const remainingToFlush = ZSTD_endStream(cstream, &output); if (remainingToFlush) { fprintf(stderr, "not fully flushed"); exit(12); } - fwrite_orDie(buffOut, cursout.nbBytesWritten, fout); + fwrite_orDie(buffOut, output.pos, fout); fclose_orDie(fout); fclose_orDie(fin); diff --git a/examples/streaming_decompression.c b/examples/streaming_decompression.c index 1176cabcb..a9efd7a2d 100644 --- a/examples/streaming_decompression.c +++ b/examples/streaming_decompression.c @@ -83,11 +83,12 @@ static void decompressFile_orDie(const char* fname) if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initDStream() error \n"); exit(11); } while( (read = fread_orDie(buffIn, toRead, fin)) ) { - ZSTD_rCursor cursin = { buffIn, read }; - while (cursin.size) { - ZSTD_wCursor cursout = { buffOut, buffOutSize, 0 }; - toRead = ZSTD_decompressStream(dstream, &cursout , &cursin); - /* note : data is just "sinked" into buffOut */ + ZSTD_inBuffer input = { buffIn, read, 0 }; + while (input.pos < input.size) { + ZSTD_outBuffer output = { buffOut, buffOutSize, 0 }; + toRead = ZSTD_decompressStream(dstream, &output , &input); + /* note : data is just "sinked" into buffOut + a more complete example would write it to disk or stdout */ } } From c41190223025ae7f28a5bfe188c493d1e90189e8 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 17 Aug 2016 01:50:54 +0200 Subject: [PATCH 14/14] fixed g++ conversion warning --- lib/compress/zstd_compress.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 6de2e900c..00b1408ca 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2981,7 +2981,10 @@ size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) { size_t srcSize = 0; size_t sizeWritten = output->size - output->pos; - size_t const result = ZSTD_compressStream_generic(zcs, output->dst + output->pos, &sizeWritten, &srcSize, &srcSize, zsf_flush); /* use a valid src address instead of NULL */ + size_t const result = ZSTD_compressStream_generic(zcs, + (char*)(output->dst) + output->pos, &sizeWritten, + &srcSize, &srcSize, /* use a valid src address instead of NULL */ + zsf_flush); output->pos += sizeWritten; if (ZSTD_isError(result)) return result; return zcs->outBuffContentSize - zcs->outBuffFlushedSize; /* remaining to flush */