diff --git a/NEWS b/NEWS index ccbf15eb5..d4abd36f9 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,9 @@ +v0.6.0 +Stronger high compression modes, thanks to Przemyslaw Skibinski +API : ZSTD_getFrameParams() provides size of decompressed content +New : highest compression modes require `--ultra` command to fully unleash their capacity +Fixed : zstd cli return error code > 0 and removes dst file artifact when decompression fails, thanks to Chip Turner + v0.5.1 New : Optimal parsing => Very high compression modes, thanks to Przemyslaw Skibinski Changed : Dictionary builder integrated into libzstd and zstd cli diff --git a/lib/zbuff.c b/lib/zbuff.c index 4c1eb2cf2..386b47d5a 100644 --- a/lib/zbuff.c +++ b/lib/zbuff.c @@ -40,17 +40,18 @@ ***************************************/ #include #include "error_private.h" -#include "zstd_static.h" +#include "zstd_internal.h" /* MIN, ZSTD_blockHeaderSize */ +#include "zstd_static.h" /* ZSTD_BLOCKSIZE_MAX */ #include "zbuff_static.h" /* ************************************* * Constants ***************************************/ -static size_t ZBUFF_blockHeaderSize = 3; -static size_t ZBUFF_endFrameSize = 3; +static size_t const ZBUFF_endFrameSize = ZSTD_BLOCKHEADERSIZE; -/** ************************************************ + +/*_************************************************** * Streaming compression * * A ZBUFF_CCtx object is required to track streaming operation. @@ -59,28 +60,28 @@ static size_t ZBUFF_endFrameSize = 3; * ZBUFF_CCtx objects can be reused multiple times. * * Use ZBUFF_compressContinue() repetitively to consume your input. -* *srcSizePtr and *maxDstSizePtr can be any size. -* The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr. +* *srcSizePtr and *dstCapacityPtr can be any size. +* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. * Note that it may not consume the entire input, in which case it's up to the caller to call again the function with remaining input. -* The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or change dst . +* The content of dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters or change dst . * @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency) * or an error code, which can be tested using ZBUFF_isError(). * * ZBUFF_compressFlush() can be used to instruct ZBUFF to compress and output whatever remains within its buffer. -* Note that it will not output more than *maxDstSizePtr. +* Note that it will not output more than *dstCapacityPtr. * Therefore, some content might still be left into its internal buffer if dst buffer is too small. * @return : nb of bytes still present into internal buffer (0 if it's empty) * or an error code, which can be tested using ZBUFF_isError(). * * ZBUFF_compressEnd() instructs to finish a frame. * It will perform a flush and write frame epilogue. -* Similar to ZBUFF_compressFlush(), it may not be able to output the entire internal buffer content if *maxDstSizePtr is too small. +* Similar to ZBUFF_compressFlush(), it may not be able to output the entire internal buffer content if *dstCapacityPtr is too small. * @return : nb of bytes still present into internal buffer (0 if it's empty) * or an error code, which can be tested using ZBUFF_isError(). * * Hint : recommended buffer sizes (not compulsory) -* input : 128 KB block size is the internal unit, it improves latency to use this value. -* output : ZSTD_compressBound(128 KB) + 3 + 3 : ensures it's always possible to write/flush/end a full block at best speed. +* input : ZSTD_BLOCKSIZE_MAX (128 KB), internal unit size, it improves latency to use this value. +* output : ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + ZBUFF_endFrameSize : ensures it's always possible to write/flush/end a full block at best speed. * **************************************************/ typedef enum { ZBUFFcs_init, ZBUFFcs_load, ZBUFFcs_flush } ZBUFF_cStage; @@ -88,13 +89,13 @@ typedef enum { ZBUFFcs_init, ZBUFFcs_load, ZBUFFcs_flush } ZBUFF_cStage; /* *** Ressources *** */ struct ZBUFF_CCtx_s { ZSTD_CCtx* zc; - char* inBuff; + char* inBuff; size_t inBuffSize; size_t inToCompress; size_t inBuffPos; size_t inBuffTarget; size_t blockSize; - char* outBuff; + char* outBuff; size_t outBuffSize; size_t outBuffContentSize; size_t outBuffFlushedSize; @@ -123,8 +124,6 @@ size_t ZBUFF_freeCCtx(ZBUFF_CCtx* zbc) /* *** Initialization *** */ -#define MIN(a,b) ( ((a)<(b)) ? (a) : (b) ) -#define BLOCKSIZE (128 * 1024) /* a bit too "magic", should come from reference */ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, ZSTD_parameters params) { size_t neededInBuffSize; @@ -139,7 +138,7 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, const void* dict, size_t dic zbc->inBuff = (char*)malloc(neededInBuffSize); if (zbc->inBuff == NULL) return ERROR(memory_allocation); } - zbc->blockSize = MIN(BLOCKSIZE, zbc->inBuffSize); + zbc->blockSize = MIN(ZSTD_BLOCKSIZE_MAX, zbc->inBuffSize); if (zbc->outBuffSize < ZSTD_compressBound(zbc->blockSize)+1) { zbc->outBuffSize = ZSTD_compressBound(zbc->blockSize)+1; free(zbc->outBuff); /* should not be necessary */ @@ -172,15 +171,15 @@ ZSTDLIB_API size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* zbc, const void* dic /* *** Compression *** */ -static size_t ZBUFF_limitCopy(void* dst, size_t maxDstSize, const void* src, size_t srcSize) +static size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) { - size_t length = MIN(maxDstSize, srcSize); + size_t length = MIN(dstCapacity, srcSize); memcpy(dst, src, length); return length; } static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc, - void* dst, size_t* maxDstSizePtr, + void* dst, size_t* dstCapacityPtr, const void* src, size_t* srcSizePtr, int flush) /* aggregate : wait for full block before compressing */ { @@ -190,7 +189,7 @@ static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc, const char* const iend = istart + *srcSizePtr; char* const ostart = (char*)dst; char* op = ostart; - char* const oend = ostart + *maxDstSizePtr; + char* const oend = ostart + *dstCapacityPtr; while (notDone) { switch(zbc->stage) @@ -250,7 +249,7 @@ static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc, } *srcSizePtr = ip - istart; - *maxDstSizePtr = op - ostart; + *dstCapacityPtr = op - ostart; { size_t hintInSize = zbc->inBuffTarget - zbc->inBuffPos; if (hintInSize==0) hintInSize = zbc->blockSize; @@ -259,30 +258,30 @@ static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc, } size_t ZBUFF_compressContinue(ZBUFF_CCtx* zbc, - void* dst, size_t* maxDstSizePtr, + void* dst, size_t* dstCapacityPtr, const void* src, size_t* srcSizePtr) { - return ZBUFF_compressContinue_generic(zbc, dst, maxDstSizePtr, src, srcSizePtr, 0); + return ZBUFF_compressContinue_generic(zbc, dst, dstCapacityPtr, src, srcSizePtr, 0); } /* *** Finalize *** */ -size_t ZBUFF_compressFlush(ZBUFF_CCtx* zbc, void* dst, size_t* maxDstSizePtr) +size_t ZBUFF_compressFlush(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr) { size_t srcSize = 0; - ZBUFF_compressContinue_generic(zbc, dst, maxDstSizePtr, &srcSize, &srcSize, 1); /* use a valid src address instead of NULL, as some sanitizer don't like it */ + ZBUFF_compressContinue_generic(zbc, dst, dstCapacityPtr, &srcSize, &srcSize, 1); /* use a valid src address instead of NULL, as some sanitizer don't like it */ return zbc->outBuffContentSize - zbc->outBuffFlushedSize; } -size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* maxDstSizePtr) +size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr) { BYTE* const ostart = (BYTE*)dst; BYTE* op = ostart; - BYTE* const oend = ostart + *maxDstSizePtr; - size_t outSize = *maxDstSizePtr; + BYTE* const oend = ostart + *dstCapacityPtr; + size_t outSize = *dstCapacityPtr; size_t epilogueSize, remaining; ZBUFF_compressFlush(zbc, dst, &outSize); /* flush any remaining inBuff */ op += outSize; @@ -293,43 +292,42 @@ size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* maxDstSizePtr) remaining = ZBUFF_compressFlush(zbc, op, &outSize); /* attempt to flush epilogue into dst */ op += outSize; if (!remaining) zbc->stage = ZBUFFcs_init; /* close only if nothing left to flush */ - *maxDstSizePtr = op-ostart; /* tells how many bytes were written */ + *dstCapacityPtr = op-ostart; /* tells how many bytes were written */ return remaining; } - -/** ************************************************ -* Streaming decompression +/*-*************************************************************************** +* Streaming decompression howto * -* A ZBUFF_DCtx object is required to track streaming operation. +* A ZBUFF_DCtx object is required to track streaming operations. * Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources. -* Use ZBUFF_decompressInit() to start a new decompression operation. -* ZBUFF_DCtx objects can be reused multiple times. +* Use ZBUFF_decompressInit() to start a new decompression operation, +* or ZBUFF_decompressInitDictionary() if decompression requires a dictionary. +* Note that ZBUFF_DCtx objects can be re-init multiple times. * * Use ZBUFF_decompressContinue() repetitively to consume your input. -* *srcSizePtr and *maxDstSizePtr can be any size. -* The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr. -* Note that it may not consume the entire input, in which case it's up to the caller to call again the function with remaining input. -* The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or change dst . -* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency) -* or 0 when a frame is completely decoded +* *srcSizePtr and *dstCapacityPtr can be any size. +* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. +* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. +* The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change @dst. +* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency), +* or 0 when a frame is completely decoded, * or an error code, which can be tested using ZBUFF_isError(). * -* Hint : recommended buffer sizes (not compulsory) -* output : 128 KB block size is the internal unit, it ensures it's always possible to write a full block when it's decoded. -* input : just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . -* **************************************************/ +* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize() and ZBUFF_recommendedDOutSize() +* output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. +* input : ZBUFF_recommendedDInSize == 128KB + 3; +* just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . +* *******************************************************************************/ -typedef enum { ZBUFFds_init, ZBUFFds_readHeader, ZBUFFds_loadHeader, ZBUFFds_decodeHeader, +typedef enum { ZBUFFds_init, ZBUFFds_readHeader, ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFF_dStage; /* *** Resource management *** */ - -#define ZSTD_frameHeaderSize_max 5 /* too magical, should come from reference */ struct ZBUFF_DCtx_s { ZSTD_DCtx* zc; - ZSTD_parameters params; + ZSTD_frameParams fParams; char* inBuff; size_t inBuffSize; size_t inPos; @@ -337,9 +335,7 @@ struct ZBUFF_DCtx_s { size_t outBuffSize; size_t outStart; size_t outEnd; - size_t hPos; ZBUFF_dStage stage; - unsigned char headerBuffer[ZSTD_frameHeaderSize_max]; }; /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */ @@ -369,7 +365,7 @@ size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbc) size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* zbc, const void* dict, size_t dictSize) { zbc->stage = ZBUFFds_readHeader; - zbc->hPos = zbc->inPos = zbc->outStart = zbc->outEnd = 0; + zbc->inPos = zbc->outStart = zbc->outEnd = 0; return ZSTD_decompressBegin_usingDict(zbc->zc, dict, dictSize); } @@ -381,14 +377,16 @@ size_t ZBUFF_decompressInit(ZBUFF_DCtx* zbc) /* *** Decompression *** */ -size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr) +size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, + void* dst, size_t* dstCapacityPtr, + const void* src, size_t* srcSizePtr) { const char* const istart = (const char*)src; const char* ip = istart; const char* const iend = istart + *srcSizePtr; char* const ostart = (char*)dst; char* op = ostart; - char* const oend = ostart + *maxDstSizePtr; + char* const oend = ostart + *dstCapacityPtr; U32 notDone = 1; while (notDone) { @@ -399,69 +397,36 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePt case ZBUFFds_readHeader : /* read header from src */ - { - size_t headerSize = ZSTD_getFrameParams(&(zbc->params), src, *srcSizePtr); + { size_t const headerSize = ZSTD_getFrameParams(&(zbc->fParams), src, *srcSizePtr); if (ZSTD_isError(headerSize)) return headerSize; if (headerSize) { - /* not enough input to decode header : tell how many bytes would be necessary */ - memcpy(zbc->headerBuffer+zbc->hPos, src, *srcSizePtr); - zbc->hPos += *srcSizePtr; - *maxDstSizePtr = 0; - zbc->stage = ZBUFFds_loadHeader; - return headerSize - zbc->hPos; - } - zbc->stage = ZBUFFds_decodeHeader; - break; - } + /* not enough input to decode header : needs headerSize > *srcSizePtr */ + *dstCapacityPtr = 0; + *srcSizePtr = 0; + return headerSize; + } } - case ZBUFFds_loadHeader: - /* complete header from src */ + /* Frame header provides buffer sizes */ + { size_t const neededInSize = ZSTD_BLOCKSIZE_MAX; /* a block is never > ZSTD_BLOCKSIZE_MAX */ + if (zbc->inBuffSize < neededInSize) { + free(zbc->inBuff); + zbc->inBuffSize = neededInSize; + zbc->inBuff = (char*)malloc(neededInSize); + if (zbc->inBuff == NULL) return ERROR(memory_allocation); + } } { - size_t headerSize = ZBUFF_limitCopy( - zbc->headerBuffer + zbc->hPos, ZSTD_frameHeaderSize_max - zbc->hPos, - src, *srcSizePtr); - zbc->hPos += headerSize; - ip += headerSize; - headerSize = ZSTD_getFrameParams(&(zbc->params), zbc->headerBuffer, zbc->hPos); - if (ZSTD_isError(headerSize)) return headerSize; - if (headerSize) { - /* not enough input to decode header : tell how many bytes would be necessary */ - *maxDstSizePtr = 0; - return headerSize - zbc->hPos; - } - // zbc->stage = ZBUFFds_decodeHeader; break; /* useless : stage follows */ - } - - case ZBUFFds_decodeHeader: - /* apply header to create / resize buffers */ - { - size_t neededOutSize = (size_t)1 << zbc->params.windowLog; - size_t neededInSize = BLOCKSIZE; /* a block is never > BLOCKSIZE */ - if (zbc->inBuffSize < neededInSize) { - free(zbc->inBuff); - zbc->inBuffSize = neededInSize; - zbc->inBuff = (char*)malloc(neededInSize); - if (zbc->inBuff == NULL) return ERROR(memory_allocation); - } - if (zbc->outBuffSize < neededOutSize) { - free(zbc->outBuff); - zbc->outBuffSize = neededOutSize; - zbc->outBuff = (char*)malloc(neededOutSize); - if (zbc->outBuff == NULL) return ERROR(memory_allocation); - } } - if (zbc->hPos) { - /* some data already loaded into headerBuffer : transfer into inBuff */ - memcpy(zbc->inBuff, zbc->headerBuffer, zbc->hPos); - zbc->inPos = zbc->hPos; - zbc->hPos = 0; - zbc->stage = ZBUFFds_load; - break; - } - zbc->stage = ZBUFFds_read; + size_t const neededOutSize = (size_t)1 << zbc->fParams.windowLog; + if (zbc->outBuffSize < neededOutSize) { + free(zbc->outBuff); + zbc->outBuffSize = neededOutSize; + zbc->outBuff = (char*)malloc(neededOutSize); + if (zbc->outBuff == NULL) return ERROR(memory_allocation); + } } + zbc->stage = ZBUFFds_read; case ZBUFFds_read: { - size_t neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc); + size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc); if (neededInSize==0) { /* end of frame */ zbc->stage = ZBUFFds_init; notDone = 0; @@ -469,7 +434,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePt } if ((size_t)(iend-ip) >= neededInSize) { /* directly decode from src */ - size_t decodedSize = ZSTD_decompressContinue(zbc->zc, + size_t const decodedSize = ZSTD_decompressContinue(zbc->zc, zbc->outBuff + zbc->outStart, zbc->outBuffSize - zbc->outStart, ip, neededInSize); if (ZSTD_isError(decodedSize)) return decodedSize; @@ -485,8 +450,8 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePt case ZBUFFds_load: { - size_t neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc); - size_t toLoad = neededInSize - zbc->inPos; /* should always be <= remaining space within inBuff */ + size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc); + size_t const toLoad = neededInSize - zbc->inPos; /* should always be <= remaining space within inBuff */ size_t loadedSize; if (toLoad > zbc->inBuffSize - zbc->inPos) return ERROR(corruption_detected); /* should never happen */ loadedSize = ZBUFF_limitCopy(zbc->inBuff + zbc->inPos, toLoad, ip, iend-ip); @@ -494,7 +459,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePt zbc->inPos += loadedSize; if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */ { - size_t decodedSize = ZSTD_decompressContinue(zbc->zc, + size_t const decodedSize = ZSTD_decompressContinue(zbc->zc, zbc->outBuff + zbc->outStart, zbc->outBuffSize - zbc->outStart, zbc->inBuff, neededInSize); if (ZSTD_isError(decodedSize)) return decodedSize; @@ -506,13 +471,13 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePt } } case ZBUFFds_flush: { - size_t toFlushSize = zbc->outEnd - zbc->outStart; - size_t flushedSize = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outStart, toFlushSize); + size_t const toFlushSize = zbc->outEnd - zbc->outStart; + size_t const flushedSize = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outStart, toFlushSize); op += flushedSize; zbc->outStart += flushedSize; if (flushedSize == toFlushSize) { zbc->stage = ZBUFFds_read; - if (zbc->outStart + BLOCKSIZE > zbc->outBuffSize) + if (zbc->outStart + ZSTD_BLOCKSIZE_MAX > zbc->outBuffSize) zbc->outStart = zbc->outEnd = 0; break; } @@ -523,12 +488,12 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePt default: return ERROR(GENERIC); /* impossible */ } } + /* result */ *srcSizePtr = ip-istart; - *maxDstSizePtr = op-ostart; - + *dstCapacityPtr = op-ostart; { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zbc->zc); - if (nextSrcSizeHint > ZBUFF_blockHeaderSize) nextSrcSizeHint+= ZBUFF_blockHeaderSize; /* get next block header too */ + if (nextSrcSizeHint > ZSTD_blockHeaderSize) nextSrcSizeHint+= ZSTD_blockHeaderSize; /* get following block header too */ nextSrcSizeHint -= zbc->inPos; /* already loaded*/ return nextSrcSizeHint; } @@ -542,7 +507,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePt unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); } const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); } -size_t ZBUFF_recommendedCInSize(void) { return BLOCKSIZE; } -size_t ZBUFF_recommendedCOutSize(void) { return ZSTD_compressBound(BLOCKSIZE) + ZBUFF_blockHeaderSize + ZBUFF_endFrameSize; } -size_t ZBUFF_recommendedDInSize(void) { return BLOCKSIZE + ZBUFF_blockHeaderSize /* block header size*/ ; } -size_t ZBUFF_recommendedDOutSize(void) { return BLOCKSIZE; } +size_t ZBUFF_recommendedCInSize(void) { return ZSTD_BLOCKSIZE_MAX; } +size_t ZBUFF_recommendedCOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + ZBUFF_endFrameSize; } +size_t ZBUFF_recommendedDInSize(void) { return ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize /* block header size*/ ; } +size_t ZBUFF_recommendedDOutSize(void) { return ZSTD_BLOCKSIZE_MAX; } diff --git a/lib/zbuff.h b/lib/zbuff.h index d3275b7df..7a914a4a7 100644 --- a/lib/zbuff.h +++ b/lib/zbuff.h @@ -75,7 +75,7 @@ ZSTDLIB_API size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* dstC ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr); /*-************************************************* -* Streaming compression +* Streaming compression - howto * * A ZBUFF_CCtx object is required to track streaming operation. * Use ZBUFF_createCCtx() and ZBUFF_freeCCtx() to create/release resources. @@ -127,26 +127,27 @@ ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, const void* src, size_t* srcSizePtr); /*-*************************************************************************** -* Streaming decompression +* Streaming decompression howto * * A ZBUFF_DCtx object is required to track streaming operations. * Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources. * Use ZBUFF_decompressInit() to start a new decompression operation, * or ZBUFF_decompressInitDictionary() if decompression requires a dictionary. -* Note that ZBUFF_DCtx objects can be reused multiple times. +* Note that ZBUFF_DCtx objects can be re-init multiple times. * * Use ZBUFF_decompressContinue() repetitively to consume your input. * *srcSizePtr and *dstCapacityPtr can be any size. * The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. * Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. -* The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters or change @dst. -* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency) -* or 0 when a frame is completely decoded +* The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change @dst. +* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency), +* or 0 when a frame is completely decoded, * or an error code, which can be tested using ZBUFF_isError(). * -* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize() / ZBUFF_recommendedDOutSize() +* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize() and ZBUFF_recommendedDOutSize() * output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. -* input : ZBUFF_recommendedDInSize==128Kb+3; just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . +* input : ZBUFF_recommendedDInSize == 128KB + 3; +* just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . * *******************************************************************************/ diff --git a/lib/zdict.c b/lib/zdict.c index 2b3d3ae8a..a7c8090a8 100644 --- a/lib/zdict.c +++ b/lib/zdict.c @@ -587,7 +587,7 @@ typedef struct { ZSTD_CCtx* ref; ZSTD_CCtx* zc; - void* workPlace; /* must be BLOCKSIZE allocated */ + void* workPlace; /* must be ZSTD_BLOCKSIZE_MAX allocated */ } EStats_ress_t; @@ -599,9 +599,9 @@ static void ZDICT_countEStats(EStats_ress_t esr, const U32* u32Ptr; seqStore_t seqStore; - if (srcSize > BLOCKSIZE) srcSize = BLOCKSIZE; /* protection vs large samples */ + if (srcSize > ZSTD_BLOCKSIZE_MAX) srcSize = ZSTD_BLOCKSIZE_MAX; /* protection vs large samples */ ZSTD_copyCCtx(esr.zc, esr.ref); - ZSTD_compressBlock(esr.zc, esr.workPlace, BLOCKSIZE, src, srcSize); + ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_MAX, src, srcSize); seqStore = ZSTD_copySeqStore(esr.zc); /* count stats */ @@ -654,7 +654,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, for (u=0; u<=MaxLL; u++) litlengthCount[u]=1; esr.ref = ZSTD_createCCtx(); esr.zc = ZSTD_createCCtx(); - esr.workPlace = malloc(BLOCKSIZE); + esr.workPlace = malloc(ZSTD_BLOCKSIZE_MAX); if (!esr.ref || !esr.zc || !esr.workPlace) { eSize = ERROR(memory_allocation); DISPLAYLEVEL(1, "Not enough memory"); diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index 7c8570345..f0518b046 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -104,7 +104,7 @@ struct ZSTD_CCtx_s size_t workSpaceSize; size_t blockSize; size_t hbSize; - char headerBuffer[ZSTD_frameHeaderSize_max]; + char headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; seqStore_t seqStore; /* sequences storage ptrs */ U32* hashTable; @@ -170,7 +170,7 @@ void ZSTD_validateParams(ZSTD_parameters* params) size_t ZSTD_sizeofCCtx(ZSTD_parameters params) /* hidden interface, for paramagrill */ { /* copy / pasted from ZSTD_resetCCtx_advanced */ - const size_t blockSize = MIN(BLOCKSIZE, (size_t)1 << params.windowLog); + const size_t blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params.windowLog); const U32 contentLog = (params.strategy == ZSTD_fast) ? 1 : params.contentLog; const U32 divider = (params.searchLength==3) ? 3 : 4; const size_t maxNbSeq = blockSize / divider; @@ -187,7 +187,7 @@ size_t ZSTD_sizeofCCtx(ZSTD_parameters params) /* hidden interface, for parama static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, ZSTD_parameters params) { /* note : params considered validated here */ - const size_t blockSize = MIN(BLOCKSIZE, (size_t)1 << params.windowLog); + const size_t blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params.windowLog); const U32 contentLog = (params.strategy == ZSTD_fast) ? 1 : params.contentLog; const U32 divider = (params.searchLength==3) ? 3 : 4; const size_t maxNbSeq = blockSize / divider; @@ -289,7 +289,7 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx) } -/*! ZSTD_reduceIndex +/*! ZSTD_reduceIndex() : * rescale indexes to avoid future overflow (indexes are U32) */ static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue) @@ -310,6 +310,37 @@ static void ZSTD_reduceIndex (ZSTD_CCtx* zc, * Block entropic compression *********************************************************/ +/* Frame format description + Frame Header - [ Block Header - Block ] - Frame End + 1) Frame Header + - 4 bytes - Magic Number : ZSTD_MAGICNUMBER (defined within zstd_static.h) + - 1 byte - Frame Descriptor + 2) Block Header + - 3 bytes, starting with a 2-bits descriptor + Uncompressed, Compressed, Frame End, unused + 3) Block + See Block Format Description + 4) Frame End + - 3 bytes, compatible with Block Header +*/ + + +/* Frame descriptor + + 1 byte, using : + bit 0-3 : windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN (see zstd_internal.h) + bit 4 : minmatch 4(0) or 3(1) + bit 5 : reserved (must be zero) + bit 6-7 : Frame content size : unknown, 1 byte, 2 bytes, 8 bytes + + Optional : content size (0, 1, 2 or 8 bytes) + 0 : unknown + 1 : 0-255 bytes + 2 : 256 - 65535+256 + 8 : up to 16 exa +*/ + + /* Block format description Block = Literal Section - Sequences Section @@ -396,11 +427,11 @@ static void ZSTD_reduceIndex (ZSTD_CCtx* zc, FSE_ENCODING_DYNAMIC : read NCount */ -size_t ZSTD_noCompressBlock (void* dst, size_t maxDstSize, const void* src, size_t srcSize) +size_t ZSTD_noCompressBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize) { BYTE* const ostart = (BYTE* const)dst; - if (srcSize + ZSTD_blockHeaderSize > maxDstSize) return ERROR(dstSize_tooSmall); + if (srcSize + ZSTD_blockHeaderSize > dstCapacity) return ERROR(dstSize_tooSmall); memcpy(ostart + ZSTD_blockHeaderSize, src, srcSize); /* Build header */ @@ -413,12 +444,12 @@ size_t ZSTD_noCompressBlock (void* dst, size_t maxDstSize, const void* src, size } -static size_t ZSTD_noCompressLiterals (void* dst, size_t maxDstSize, const void* src, size_t srcSize) +static size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize) { BYTE* const ostart = (BYTE* const)dst; const U32 flSize = 1 + (srcSize>31) + (srcSize>4095); - if (srcSize + flSize > maxDstSize) return ERROR(dstSize_tooSmall); + if (srcSize + flSize > dstCapacity) return ERROR(dstSize_tooSmall); switch(flSize) { @@ -441,12 +472,12 @@ static size_t ZSTD_noCompressLiterals (void* dst, size_t maxDstSize, const void* return srcSize + flSize; } -static size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t maxDstSize, const void* src, size_t srcSize) +static size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize) { BYTE* const ostart = (BYTE* const)dst; U32 flSize = 1 + (srcSize>31) + (srcSize>4095); - (void)maxDstSize; /* maxDstSize guaranteed to be >=4, hence large enough */ + (void)dstCapacity; /* dstCapacity guaranteed to be >=4, hence large enough */ switch(flSize) { @@ -473,7 +504,7 @@ static size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t maxDstSize, const size_t ZSTD_minGain(size_t srcSize) { return (srcSize >> 6) + 2; } static size_t ZSTD_compressLiterals (ZSTD_CCtx* zc, - void* dst, size_t maxDstSize, + void* dst, size_t dstCapacity, const void* src, size_t srcSize) { const size_t minGain = ZSTD_minGain(srcSize); @@ -483,19 +514,19 @@ static size_t ZSTD_compressLiterals (ZSTD_CCtx* zc, U32 hType = IS_HUF; size_t clitSize; - if (maxDstSize < lhSize+1) return ERROR(dstSize_tooSmall); /* not enough space for compression */ + if (dstCapacity < lhSize+1) return ERROR(dstSize_tooSmall); /* not enough space for compression */ if (zc->flagStaticTables && (lhSize==3)) { hType = IS_PCH; singleStream = 1; - clitSize = HUF_compress1X_usingCTable(ostart+lhSize, maxDstSize-lhSize, src, srcSize, zc->hufTable); + clitSize = HUF_compress1X_usingCTable(ostart+lhSize, dstCapacity-lhSize, src, srcSize, zc->hufTable); } else { - clitSize = singleStream ? HUF_compress1X(ostart+lhSize, maxDstSize-lhSize, src, srcSize, 255, 12) - : HUF_compress2 (ostart+lhSize, maxDstSize-lhSize, src, srcSize, 255, 12); + clitSize = singleStream ? HUF_compress1X(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 12) + : HUF_compress2 (ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 12); } - if ((clitSize==0) || (clitSize >= srcSize - minGain)) return ZSTD_noCompressLiterals(dst, maxDstSize, src, srcSize); - if (clitSize==1) return ZSTD_compressRleLiteralsBlock(dst, maxDstSize, src, srcSize); + if ((clitSize==0) || (clitSize >= srcSize - minGain)) return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); + if (clitSize==1) return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize); /* Build header */ switch(lhSize) @@ -528,7 +559,7 @@ static size_t ZSTD_compressLiterals (ZSTD_CCtx* zc, #define LITERAL_NOENTROPY 63 /* don't even attempt to compress literals below this threshold (cheap heuristic) */ size_t ZSTD_compressSequences(ZSTD_CCtx* zc, - void* dst, size_t maxDstSize, + void* dst, size_t dstCapacity, size_t srcSize) { const seqStore_t* seqStorePtr = &(zc->seqStore); @@ -548,7 +579,7 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* zc, BYTE* const offCodeTable = seqStorePtr->offCodeStart; BYTE* const ostart = (BYTE*)dst; BYTE* op = ostart; - BYTE* const oend = ostart + maxDstSize; + BYTE* const oend = ostart + dstCapacity; const size_t nbSeq = llPtr - llTable; const size_t minGain = ZSTD_minGain(srcSize); const size_t maxCSize = srcSize - minGain; @@ -561,9 +592,9 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* zc, const size_t minLitSize = zc->flagStaticTables ? 6 : LITERAL_NOENTROPY; if (litSize <= minLitSize) - cSize = ZSTD_noCompressLiterals(op, maxDstSize, op_lit_start, litSize); + cSize = ZSTD_noCompressLiterals(op, dstCapacity, op_lit_start, litSize); else - cSize = ZSTD_compressLiterals(zc, op, maxDstSize, op_lit_start, litSize); + cSize = ZSTD_compressLiterals(zc, op, dstCapacity, op_lit_start, litSize); if (ZSTD_isError(cSize)) return cSize; op += cSize; } @@ -1951,17 +1982,17 @@ static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int } -static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t maxDstSize, const void* src, size_t srcSize) +static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { ZSTD_blockCompressor blockCompressor = ZSTD_selectBlockCompressor(zc->params.strategy, zc->lowLimit < zc->dictLimit); if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) return 0; /* don't even attempt compression below a certain srcSize */ blockCompressor(zc, src, srcSize); - return ZSTD_compressSequences(zc, dst, maxDstSize, srcSize); + return ZSTD_compressSequences(zc, dst, dstCapacity, srcSize); } static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, - void* dst, size_t maxDstSize, + void* dst, size_t dstCapacity, const void* src, size_t srcSize) { size_t blockSize = zc->blockSize; @@ -1980,7 +2011,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, while (remaining) { size_t cSize; - if (maxDstSize < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */ + if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */ if (remaining < blockSize) blockSize = remaining; if ((U32)(ip+blockSize - zc->base) > zc->loadedDictEnd + maxDist) { /* enforce maxDist */ @@ -1989,11 +2020,11 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, if (zc->dictLimit < zc->lowLimit) zc->dictLimit = zc->lowLimit; } - cSize = ZSTD_compressBlock_internal(zc, op+ZSTD_blockHeaderSize, maxDstSize-ZSTD_blockHeaderSize, ip, blockSize); + cSize = ZSTD_compressBlock_internal(zc, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, ip, blockSize); if (ZSTD_isError(cSize)) return cSize; if (cSize == 0) { /* block is not compressible */ - cSize = ZSTD_noCompressBlock(op, maxDstSize, ip, blockSize); + cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize); if (ZSTD_isError(cSize)) return cSize; } else { op[0] = (BYTE)(cSize>>16); @@ -2004,7 +2035,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, } remaining -= blockSize; - maxDstSize -= cSize; + dstCapacity -= cSize; ip += blockSize; op += cSize; } @@ -2088,12 +2119,12 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* zc, } -size_t ZSTD_compressBlock(ZSTD_CCtx* zc, void* dst, size_t maxDstSize, const void* src, size_t srcSize) +size_t ZSTD_compressBlock(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { - if (srcSize > BLOCKSIZE) return ERROR(srcSize_wrong); + if (srcSize > ZSTD_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); zc->params.searchLength = MINMATCH; /* force ZSTD_btopt to MINMATCH in block mode */ ZSTD_LOG_BLOCK("%p: ZSTD_compressBlock searchLength=%d\n", zc->base, zc->params.searchLength); - return ZSTD_compressContinue_internal(zc, dst, maxDstSize, src, srcSize, 0); + return ZSTD_compressContinue_internal(zc, dst, dstCapacity, src, srcSize, 0); } @@ -2141,10 +2172,10 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t /* Dictionary format : Magic == ZSTD_DICT_MAGIC (4 bytes) - Huff0 CTable (256 * 4 bytes) => to be changed to read from writeCTable + HUF_writeCTable(256) Dictionary content */ -/*! ZSTD_loadDictEntropyStats +/*! ZSTD_loadDictEntropyStats() : @return : size read from dictionary */ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* zc, const void* dict, size_t dictSize) { @@ -2185,49 +2216,66 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* zc, const void* dict, size_t return hufHeaderSize + offcodeHeaderSize + matchlengthHeaderSize + litlengthHeaderSize; } - +/** ZSTD_compress_insertDictionary() : +* @return : 0, or an error code */ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, size_t dictSize) { - if (dict && (dictSize>4)) { - U32 magic = MEM_readLE32(dict); - size_t eSize; - if (magic != ZSTD_DICT_MAGIC) - return ZSTD_loadDictionaryContent(zc, dict, dictSize); + if ((dict==NULL) || (dictSize<=4)) return 0; - eSize = ZSTD_loadDictEntropyStats(zc, (const char*)dict+4, dictSize-4) + 4; - if (ZSTD_isError(eSize)) return eSize; - return ZSTD_loadDictionaryContent(zc, (const char*)dict+eSize, dictSize-eSize); + /* default : dict is pure content */ + if (MEM_readLE32(dict) != ZSTD_DICT_MAGIC) return ZSTD_loadDictionaryContent(zc, dict, dictSize); + + /* known magic number : dict is parsed for entropy stats and content */ + { size_t const eSize = ZSTD_loadDictEntropyStats(zc, (const char*)dict+4 /* skip magic */, dictSize-4) + 4; + if (ZSTD_isError(eSize)) return eSize; + return ZSTD_loadDictionaryContent(zc, (const char*)dict+eSize, dictSize-eSize); } - return 0; } -/*! ZSTD_compressBegin_advanced +/*! ZSTD_compressBegin_advanced() : * @return : 0, or an error code */ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* zc, const void* dict, size_t dictSize, ZSTD_parameters params) { - size_t errorCode; - ZSTD_validateParams(¶ms); - errorCode = ZSTD_resetCCtx_advanced(zc, params); - if (ZSTD_isError(errorCode)) return errorCode; + { size_t const errorCode = ZSTD_resetCCtx_advanced(zc, params); + if (ZSTD_isError(errorCode)) return errorCode; } + + /* Write Frame Header into ctx headerBuffer */ + MEM_writeLE32(zc->headerBuffer, ZSTD_MAGICNUMBER); + { + BYTE* const op = (BYTE*)zc->headerBuffer; + U32 const fcsSize[4] = { 0, 1, 2, 8 }; + U32 const fcsId = (params.srcSize>0) + (params.srcSize>=256) + (params.srcSize>=65536+256); /* 0-3 */ + BYTE fdescriptor = (BYTE)(params.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN); /* windowLog : 4 KB - 128 MB */ + fdescriptor |= (BYTE)((params.searchLength==3)<<4); /* mml : 3-4 */ + fdescriptor |= (BYTE)(fcsId << 6); + op[4] = fdescriptor; + switch(fcsId) + { + default: /* impossible */ + case 0 : break; + case 1 : op[5] = (BYTE)(params.srcSize); break; + case 2 : MEM_writeLE16(op+5, (U16)(params.srcSize-256)); break; + case 3 : MEM_writeLE64(op+5, (U64)(params.srcSize)); break; + } + zc->hbSize = ZSTD_frameHeaderSize_min + fcsSize[fcsId]; + } - MEM_writeLE32(zc->headerBuffer, ZSTD_MAGICNUMBER); /* Write Header */ - ((BYTE*)zc->headerBuffer)[4] = (BYTE)(params.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN + ((params.searchLength==3)<<4)); - zc->hbSize = ZSTD_frameHeaderSize_min; zc->stage = 0; - return ZSTD_compress_insertDictionary(zc, dict, dictSize); } size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* zc, const void* dict, size_t dictSize, int compressionLevel) { + ZSTD_parameters params = ZSTD_getParams(compressionLevel, dictSize); + params.srcSize = 0; ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin_usingDict compressionLevel=%d\n", zc->base, compressionLevel); - return ZSTD_compressBegin_advanced(zc, dict, dictSize, ZSTD_getParams(compressionLevel, MAX(128 KB, dictSize))); + return ZSTD_compressBegin_advanced(zc, dict, dictSize, params); } size_t ZSTD_compressBegin(ZSTD_CCtx* zc, int compressionLevel) @@ -2237,10 +2285,10 @@ size_t ZSTD_compressBegin(ZSTD_CCtx* zc, int compressionLevel) } -/*! ZSTD_compressEnd -* Write frame epilogue +/*! ZSTD_compressEnd() : +* Write frame epilogue. * @return : nb of bytes written into dst (or an error code) */ -size_t ZSTD_compressEnd(ZSTD_CCtx* zc, void* dst, size_t maxDstSize) +size_t ZSTD_compressEnd(ZSTD_CCtx* zc, void* dst, size_t dstCapacity) { BYTE* op = (BYTE*)dst; size_t hbSize = 0; @@ -2248,15 +2296,15 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* zc, void* dst, size_t maxDstSize) /* empty frame */ if (zc->stage==0) { hbSize = zc->hbSize; - if (maxDstSize <= hbSize) return ERROR(dstSize_tooSmall); + if (dstCapacity <= hbSize) return ERROR(dstSize_tooSmall); zc->stage = 1; memcpy(dst, zc->headerBuffer, hbSize); - maxDstSize -= hbSize; + dstCapacity -= hbSize; op += hbSize; } /* frame epilogue */ - if (maxDstSize < 3) return ERROR(dstSize_tooSmall); + if (dstCapacity < 3) return ERROR(dstSize_tooSmall); op[0] = (BYTE)(bt_end << 6); op[1] = 0; op[2] = 0; @@ -2266,16 +2314,16 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* zc, void* dst, size_t maxDstSize) size_t ZSTD_compress_usingPreparedCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, - void* dst, size_t maxDstSize, + void* dst, size_t dstCapacity, const void* src, size_t srcSize) { size_t outSize; size_t errorCode = ZSTD_copyCCtx(cctx, preparedCCtx); if (ZSTD_isError(errorCode)) return errorCode; - errorCode = ZSTD_compressContinue(cctx, dst, maxDstSize, src, srcSize); + errorCode = ZSTD_compressContinue(cctx, dst, dstCapacity, src, srcSize); if (ZSTD_isError(errorCode)) return errorCode; outSize = errorCode; - errorCode = ZSTD_compressEnd(cctx, (char*)dst+outSize, maxDstSize-outSize); + errorCode = ZSTD_compressEnd(cctx, (char*)dst+outSize, dstCapacity-outSize); if (ZSTD_isError(errorCode)) return errorCode; outSize += errorCode; return outSize; @@ -2283,51 +2331,50 @@ size_t ZSTD_compress_usingPreparedCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* prepare size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, - void* dst, size_t maxDstSize, + void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict,size_t dictSize, ZSTD_parameters params) { BYTE* const ostart = (BYTE*)dst; BYTE* op = ostart; - size_t oSize; /* Init */ - oSize = ZSTD_compressBegin_advanced(ctx, dict, dictSize, params); - if(ZSTD_isError(oSize)) return oSize; + { size_t const errorCode = ZSTD_compressBegin_advanced(ctx, dict, dictSize, params); + if(ZSTD_isError(errorCode)) return errorCode; } /* body (compression) */ - oSize = ZSTD_compressContinue (ctx, op, maxDstSize, src, srcSize); + { size_t const oSize = ZSTD_compressContinue (ctx, op, dstCapacity, src, srcSize); if(ZSTD_isError(oSize)) return oSize; op += oSize; - maxDstSize -= oSize; + dstCapacity -= oSize; } /* Close frame */ - oSize = ZSTD_compressEnd(ctx, op, maxDstSize); + { size_t const oSize = ZSTD_compressEnd(ctx, op, dstCapacity); if(ZSTD_isError(oSize)) return oSize; - op += oSize; + op += oSize; } return (op - ostart); } -size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, const void* dict, size_t dictSize, int compressionLevel) +size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict, size_t dictSize, int compressionLevel) { ZSTD_LOG_BLOCK("%p: ZSTD_compress_usingDict srcSize=%d dictSize=%d compressionLevel=%d\n", ctx->base, (int)srcSize, (int)dictSize, compressionLevel); - return ZSTD_compress_advanced(ctx, dst, maxDstSize, src, srcSize, dict, dictSize, ZSTD_getParams(compressionLevel, srcSize)); + return ZSTD_compress_advanced(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, ZSTD_getParams(compressionLevel, srcSize)); } -size_t ZSTD_compressCCtx (ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel) +size_t ZSTD_compressCCtx (ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel) { ZSTD_LOG_BLOCK("%p: ZSTD_compressCCtx srcSize=%d compressionLevel=%d\n", ctx->base, (int)srcSize, compressionLevel); - return ZSTD_compress_advanced(ctx, dst, maxDstSize, src, srcSize, NULL, 0, ZSTD_getParams(compressionLevel, srcSize)); + return ZSTD_compress_advanced(ctx, dst, dstCapacity, src, srcSize, NULL, 0, ZSTD_getParams(compressionLevel, srcSize)); } -size_t ZSTD_compress(void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel) +size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel) { size_t result; ZSTD_CCtx ctxBody; memset(&ctxBody, 0, sizeof(ctxBody)); - result = ZSTD_compressCCtx(&ctxBody, dst, maxDstSize, src, srcSize, compressionLevel); + result = ZSTD_compressCCtx(&ctxBody, dst, dstCapacity, src, srcSize, compressionLevel); free(ctxBody.workSpace); /* can't free ctxBody, since it's on stack; just free heap content */ return result; } diff --git a/lib/zstd_decompress.c b/lib/zstd_decompress.c index 201e67516..bb7ad43d3 100644 --- a/lib/zstd_decompress.c +++ b/lib/zstd_decompress.c @@ -94,13 +94,13 @@ typedef struct } blockProperties_t; -/* ******************************************************* +/*_******************************************************* * Memory operations **********************************************************/ static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); } -/* ************************************* +/*-************************************* * Error Management ***************************************/ unsigned ZSTD_versionNumber (void) { return ZSTD_VERSION_NUMBER; } @@ -118,7 +118,7 @@ ZSTD_ErrorCode ZSTD_getError(size_t code) { return ERR_getError(code); } const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); } -/* ************************************************************* +/*-************************************************************* * Context management ***************************************************************/ typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader, @@ -136,15 +136,15 @@ struct ZSTD_DCtx_s const void* dictEnd; size_t expected; size_t headerSize; - ZSTD_parameters params; + ZSTD_frameParams fParams; blockType_t bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */ ZSTD_dStage stage; U32 flagStaticTables; const BYTE* litPtr; size_t litBufSize; size_t litSize; - BYTE litBuffer[BLOCKSIZE + WILDCOPY_OVERLENGTH]; - BYTE headerBuffer[ZSTD_frameHeaderSize_max]; + BYTE litBuffer[ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH]; + BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; }; /* typedef'd to ZSTD_DCtx within "zstd_static.h" */ size_t sizeofDCtx (void) { return sizeof(ZSTD_DCtx); } @@ -159,7 +159,7 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) dctx->dictEnd = NULL; dctx->hufTableX4[0] = HufLog; dctx->flagStaticTables = 0; - dctx->params.searchLength = MINMATCH; /* overwritten by frame but forces ZSTD_btopt to MINMATCH in block mode */ + dctx->fParams.mml = MINMATCH; /* overwritten by frame but forces ZSTD_btopt to MINMATCH in block mode */ ZSTD_LOG_BLOCK("%p: ZSTD_decompressBegin searchLength=%d\n", dctx->base, dctx->params.searchLength); return 0; } @@ -181,19 +181,19 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx) void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) { memcpy(dstDCtx, srcDCtx, - sizeof(ZSTD_DCtx) - (BLOCKSIZE+WILDCOPY_OVERLENGTH + ZSTD_frameHeaderSize_max)); /* no need to copy workspace */ + sizeof(ZSTD_DCtx) - (ZSTD_BLOCKSIZE_MAX+WILDCOPY_OVERLENGTH + ZSTD_frameHeaderSize_max)); /* no need to copy workspace */ } -/* ************************************************************* +/*-************************************************************* * Decompression section ***************************************************************/ /* Frame format description Frame Header - [ Block Header - Block ] - Frame End 1) Frame Header - - 4 bytes - Magic Number : ZSTD_MAGICNUMBER (defined within zstd_internal.h) - - 1 byte - Window Descriptor + - 4 bytes - Magic Number : ZSTD_MAGICNUMBER (defined within zstd_static.h) + - 1 byte - Frame Descriptor 2) Block Header - 3 bytes, starting with a 2-bits descriptor Uncompressed, Compressed, Frame End, unused @@ -203,7 +203,24 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) - 3 bytes, compatible with Block Header */ -/* Block format description + +/* Frame descriptor + + 1 byte, using : + bit 0-3 : windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN (see zstd_internal.h) + bit 4 : minmatch 4(0) or 3(1) + bit 5 : reserved (must be zero) + bit 6-7 : Frame content size : unknown, 1 byte, 2 bytes, 8 bytes + + Optional : content size (0, 1, 2 or 8 bytes) + 0 : unknown + 1 : 0-255 bytes + 2 : 256 - 65535+256 + 8 : up to 16 exa +*/ + + +/* Compressed Block, format description Block = Literal Section - Sequences Section Prerequisite : size of (compressed) block, maximum size of regenerated data @@ -269,47 +286,60 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) TO DO */ +static const size_t ZSTD_fcs_fieldSize[4] = { 0, 1, 2, 8 }; -/** ZSTD_decodeFrameHeader_Part1() : -* decode the 1st part of the Frame Header, which tells Frame Header size. -* srcSize must be == ZSTD_frameHeaderSize_min. -* @return : the full size of the Frame Header */ -static size_t ZSTD_decodeFrameHeader_Part1(ZSTD_DCtx* zc, const void* src, size_t srcSize) +/** ZSTD_frameHeaderSize() : +* srcSize must be >= ZSTD_frameHeaderSize_min. +* @return : size of the Frame Header */ +static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) { - U32 magicNumber; - if (srcSize != ZSTD_frameHeaderSize_min) - return ERROR(srcSize_wrong); - magicNumber = MEM_readLE32(src); - if (magicNumber != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown); - zc->headerSize = ZSTD_frameHeaderSize_min; - return zc->headerSize; + U32 fcsId; + if (srcSize < ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); + fcsId = (((const BYTE*)src)[4]) >> 6; + return ZSTD_frameHeaderSize_min + ZSTD_fcs_fieldSize[fcsId]; } -size_t ZSTD_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcSize) +/** ZSTD_getFrameParams() : +* decode Frame Header, or provide expected `srcSize`. +* @return : 0, `fparamsPtr` is correctly filled, +* >0, `srcSize` is too small, result is expected `srcSize`, +* or an error code, which can be tested using ZSTD_isError() */ +size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize) { - U32 magicNumber; - if (srcSize < ZSTD_frameHeaderSize_min) return ZSTD_frameHeaderSize_max; - magicNumber = MEM_readLE32(src); - if (magicNumber != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown); - memset(params, 0, sizeof(*params)); - params->windowLog = (((const BYTE*)src)[4] & 15) + ZSTD_WINDOWLOG_ABSOLUTEMIN; - params->searchLength = (((const BYTE*)src)[4] & 16) ? MINMATCH-1 : MINMATCH; - if ((((const BYTE*)src)[4] >> 5) != 0) return ERROR(frameParameter_unsupported); /* reserved 3 bits */ + const BYTE* ip = (const BYTE*)src; + + if (srcSize < ZSTD_frameHeaderSize_min) return ZSTD_frameHeaderSize_min; + if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown); + + /* ensure there is enough `srcSize` to fully read/decode frame header */ + { size_t const fhsize = ZSTD_frameHeaderSize(src, srcSize); + if (srcSize < fhsize) return fhsize; } + + memset(fparamsPtr, 0, sizeof(*fparamsPtr)); + { BYTE const frameDesc = ip[4]; + fparamsPtr->windowLog = (frameDesc & 0xF) + ZSTD_WINDOWLOG_ABSOLUTEMIN; + fparamsPtr->mml = (frameDesc & 0x10) ? MINMATCH-1 : MINMATCH; + if ((frameDesc & 0x20) != 0) return ERROR(frameParameter_unsupported); /* reserved 1 bit */ + switch(frameDesc >> 6) /* fcsId */ + { + default: /* impossible */ + case 0 : fparamsPtr->frameContentSize = 0; break; + case 1 : fparamsPtr->frameContentSize = ip[5]; break; + case 2 : fparamsPtr->frameContentSize = MEM_readLE16(ip+5)+256; break; + case 3 : fparamsPtr->frameContentSize = MEM_readLE64(ip+5); break; + } } return 0; } -/** ZSTD_decodeFrameHeader_Part2() : -* decode the full Frame Header. -* srcSize must be the size provided by ZSTD_decodeFrameHeader_Part1(). + +/** ZSTD_decodeFrameHeader() : +* `srcSize` must be the size provided by ZSTD_frameHeaderSize(). * @return : 0, or an error code, which can be tested using ZSTD_isError() */ -static size_t ZSTD_decodeFrameHeader_Part2(ZSTD_DCtx* zc, const void* src, size_t srcSize) +static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* zc, const void* src, size_t srcSize) { - size_t result; - if (srcSize != zc->headerSize) - return ERROR(srcSize_wrong); - result = ZSTD_getFrameParams(&(zc->params), src, srcSize); - if ((MEM_32bits()) && (zc->params.windowLog > 25)) return ERROR(frameParameter_unsupportedBy32bits); + size_t result = ZSTD_getFrameParams(&(zc->fParams), src, srcSize); + if ((MEM_32bits()) && (zc->fParams.windowLog > 25)) return ERROR(frameParameter_unsupportedBy32bits); return result; } @@ -381,7 +411,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, litCSize = ((istart[2] & 3) << 16) + (istart[3] << 8) + istart[4]; break; } - if (litSize > BLOCKSIZE) return ERROR(corruption_detected); + if (litSize > ZSTD_BLOCKSIZE_MAX) return ERROR(corruption_detected); if (HUF_isError(singleStream ? HUF_decompress1X2(dctx->litBuffer, litSize, istart+lhSize, litCSize) : @@ -389,7 +419,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, return ERROR(corruption_detected); dctx->litPtr = dctx->litBuffer; - dctx->litBufSize = BLOCKSIZE+8; + dctx->litBufSize = ZSTD_BLOCKSIZE_MAX+8; dctx->litSize = litSize; return litCSize + lhSize; } @@ -412,7 +442,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, if (HUF_isError(errorCode)) return ERROR(corruption_detected); dctx->litPtr = dctx->litBuffer; - dctx->litBufSize = BLOCKSIZE+WILDCOPY_OVERLENGTH; + dctx->litBufSize = ZSTD_BLOCKSIZE_MAX+WILDCOPY_OVERLENGTH; dctx->litSize = litSize; return litCSize + lhSize; } @@ -438,7 +468,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, if (litSize+lhSize > srcSize) return ERROR(corruption_detected); memcpy(dctx->litBuffer, istart+lhSize, litSize); dctx->litPtr = dctx->litBuffer; - dctx->litBufSize = BLOCKSIZE+8; + dctx->litBufSize = ZSTD_BLOCKSIZE_MAX+8; dctx->litSize = litSize; return lhSize+litSize; } @@ -465,10 +495,10 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, litSize = ((istart[0] & 15) << 16) + (istart[1] << 8) + istart[2]; break; } - if (litSize > BLOCKSIZE) return ERROR(corruption_detected); + if (litSize > ZSTD_BLOCKSIZE_MAX) return ERROR(corruption_detected); memset(dctx->litBuffer, istart[lhSize], litSize); dctx->litPtr = dctx->litBuffer; - dctx->litBufSize = BLOCKSIZE+WILDCOPY_OVERLENGTH; + dctx->litBufSize = ZSTD_BLOCKSIZE_MAX+WILDCOPY_OVERLENGTH; dctx->litSize = litSize; return lhSize+1; } @@ -622,16 +652,14 @@ typedef struct { static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState, const U32 mls) { - size_t litLength; - size_t offset; - size_t matchLength; const BYTE* dumps = seqState->dumps; const BYTE* const de = seqState->dumpsEnd; + size_t litLength, offset; /* Literal length */ litLength = FSE_peakSymbol(&(seqState->stateLL)); if (litLength == MaxLL) { - U32 add = *dumps++; + const U32 add = *dumps++; if (add < 255) litLength += add; else { litLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no risk : dumps is always followed by seq tables > 1 byte */ @@ -647,9 +675,8 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState, const U32 mls) 1 /*fake*/, 1, 2, 4, 8, 0x10, 0x20, 0x40, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000, 0x10000, 0x20000, 0x40000, 0x80000, 0x100000, 0x200000, 0x400000, 0x800000, 0x1000000, 0x2000000, 0x4000000, /*fake*/ 1, 1, 1, 1 }; - U32 offsetCode = FSE_peakSymbol(&(seqState->stateOffb)); /* <= maxOff, by table construction */ - U32 nbBits = offsetCode - 1; - if (offsetCode==0) nbBits = 0; /* cmove */ + const U32 offsetCode = FSE_peakSymbol(&(seqState->stateOffb)); /* <= maxOff, by table construction */ + const U32 nbBits = offsetCode ? offsetCode-1 : 0; offset = offsetPrefix[offsetCode] + BIT_readBits(&(seqState->DStream), nbBits); if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream)); #if ZSTD_REP_NUM == 4 @@ -715,23 +742,25 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState, const U32 mls) if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream)); /* MatchLength */ - matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream)); - if (matchLength == MaxML) { - U32 add = *dumps++; - if (add < 255) matchLength += add; - else { - matchLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */ - if (matchLength&1) matchLength>>=1, dumps += 3; - else matchLength = (U16)(matchLength)>>1, dumps += 2; + { + size_t matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream)); + if (matchLength == MaxML) { + const U32 add = *dumps++; + if (add < 255) matchLength += add; + else { + matchLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */ + if (matchLength&1) matchLength>>=1, dumps += 3; + else matchLength = (U16)(matchLength)>>1, dumps += 2; + } + if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */ } - if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */ + matchLength += mls; + seq->matchLength = matchLength; } - matchLength += mls; /* save result */ seq->litLength = litLength; seq->offset = offset; - seq->matchLength = matchLength; seqState->dumps = dumps; #if 0 /* debug */ @@ -841,7 +870,7 @@ static size_t ZSTD_decompressSequences( const BYTE* const base = (const BYTE*) (dctx->base); const BYTE* const vBase = (const BYTE*) (dctx->vBase); const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); - const U32 mls = dctx->params.searchLength; + const U32 mls = dctx->fParams.mml; /* Build Decoding Tables */ errorCode = ZSTD_decodeSeqHeaders(&nbSeq, &dumps, &dumpsLength, @@ -912,7 +941,7 @@ static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, const BYTE* ip = (const BYTE*)src; size_t litCSize; - if (srcSize >= BLOCKSIZE) return ERROR(srcSize_wrong); + if (srcSize >= ZSTD_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); ZSTD_LOG_BLOCK("%p: ZSTD_decompressBlock_internal searchLength=%d\n", dctx->base, dctx->params.searchLength); @@ -935,9 +964,9 @@ size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, } -/*! ZSTD_decompress_continueDCtx -* dctx must have been properly initialized */ -static size_t ZSTD_decompress_continueDCtx(ZSTD_DCtx* dctx, +/*! ZSTD_decompress_continueDCtx() : +* `dctx` must have been properly initialized */ +static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize) { @@ -951,7 +980,7 @@ static size_t ZSTD_decompress_continueDCtx(ZSTD_DCtx* dctx, /* Frame Header */ { - size_t frameHeaderSize; + size_t frameHeaderSize, errorCode; if (srcSize < ZSTD_frameHeaderSize_min+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) { @@ -960,12 +989,12 @@ static size_t ZSTD_decompress_continueDCtx(ZSTD_DCtx* dctx, return ZSTD_decompressLegacy(dst, maxDstSize, src, srcSize, magicNumber); } #endif - frameHeaderSize = ZSTD_decodeFrameHeader_Part1(dctx, src, ZSTD_frameHeaderSize_min); + frameHeaderSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min); if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize; if (srcSize < frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); + errorCode = ZSTD_decodeFrameHeader(dctx, src, frameHeaderSize); + if (ZSTD_isError(errorCode)) return errorCode; ip += frameHeaderSize; remainingSize -= frameHeaderSize; - frameHeaderSize = ZSTD_decodeFrameHeader_Part2(dctx, src, frameHeaderSize); - if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize; } /* Loop on each block */ @@ -1015,7 +1044,7 @@ size_t ZSTD_decompress_usingPreparedDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* refDC { ZSTD_copyDCtx(dctx, refDCtx); ZSTD_checkContinuity(dctx, dst); - return ZSTD_decompress_continueDCtx(dctx, dst, maxDstSize, src, srcSize); + return ZSTD_decompressFrame(dctx, dst, maxDstSize, src, srcSize); } @@ -1027,7 +1056,7 @@ size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, ZSTD_decompressBegin_usingDict(dctx, dict, dictSize); ZSTD_LOG_BLOCK("%p: ZSTD_decompressBegin_usingDict searchLength=%d\n", dctx->base, dctx->params.searchLength); ZSTD_checkContinuity(dctx, dst); - return ZSTD_decompress_continueDCtx(dctx, dst, maxDstSize, src, srcSize); + return ZSTD_decompressFrame(dctx, dst, maxDstSize, src, srcSize); } @@ -1036,6 +1065,7 @@ size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const return ZSTD_decompress_usingDict(dctx, dst, maxDstSize, src, srcSize, NULL, 0); } + size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t srcSize) { #if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE==1) @@ -1052,7 +1082,7 @@ size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t src } -/* ****************************** +/*_****************************** * Streaming Decompression API ********************************/ size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) @@ -1073,7 +1103,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co { /* get frame header size */ if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */ - dctx->headerSize = ZSTD_decodeFrameHeader_Part1(dctx, src, ZSTD_frameHeaderSize_min); + dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min); if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize; memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min); if (dctx->headerSize > ZSTD_frameHeaderSize_min) { @@ -1088,7 +1118,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co /* get frame header */ size_t result; memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_min, src, dctx->expected); - result = ZSTD_decodeFrameHeader_Part2(dctx, dctx->headerBuffer, dctx->headerSize); + result = ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize); if (ZSTD_isError(result)) return result; dctx->expected = ZSTD_blockHeaderSize; dctx->stage = ZSTDds_decodeBlockHeader; diff --git a/lib/zstd_internal.h b/lib/zstd_internal.h index 43179c570..dbce471ab 100644 --- a/lib/zstd_internal.h +++ b/lib/zstd_internal.h @@ -27,7 +27,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - - zstd source repository : https://github.com/Cyan4973/zstd + - zstd homepage : https://www.zstd.net */ #ifndef ZSTD_CCOMMON_H_MODULE #define ZSTD_CCOMMON_H_MODULE @@ -78,11 +78,8 @@ #define MB *(1 <<20) #define GB *(1U<<30) -#define BLOCKSIZE (128 KB) /* define, for static allocation */ - -static const size_t ZSTD_blockHeaderSize = 3; -static const size_t ZSTD_frameHeaderSize_min = 5; -#define ZSTD_frameHeaderSize_max 5 /* define, for static allocation */ +#define ZSTD_BLOCKHEADERSIZE 3 /* because C standard does not allow a static const value to be defined using another static const value .... :( */ +static const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE; #define BIT7 128 #define BIT6 64 diff --git a/lib/zstd_static.h b/lib/zstd_static.h index 4f23fca98..cb140a494 100644 --- a/lib/zstd_static.h +++ b/lib/zstd_static.h @@ -93,8 +93,8 @@ ZSTDLIB_API unsigned ZSTD_maxCLevel (void); /*! ZSTD_getParams() : * @return ZSTD_parameters structure for a selected compression level and srcSize. -* `srcSizeHint` value is optional, select 0 if not known */ -ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSizeHint); +* `srcSize` value is optional, select 0 if not known */ +ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSize); /*! ZSTD_validateParams() : * correct params value to remain within authorized range */ @@ -112,7 +112,7 @@ ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, * Same as ZSTD_compress_usingDict, but using a reference context `preparedCCtx`, where dictionary has been loaded. * It avoids reloading the dictionary each time. * `preparedCCtx` must have been properly initialized using ZSTD_compressBegin_usingDict() or ZSTD_compressBegin_advanced(). -* Requires 2 contexts : 1 for reference, which will not be modified, and 1 to run the compression operation */ +* Requires 2 contexts : 1 for reference (preparedCCtx) which will not be modified, and 1 to run the compression operation (cctx) */ ZSTDLIB_API size_t ZSTD_compress_usingPreparedCCtx( ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, void* dst, size_t dstCapacity, @@ -124,19 +124,19 @@ ZSTDLIB_API size_t ZSTD_compress_usingPreparedCCtx( * Same as ZSTD_decompress_usingDict, but using a reference context `preparedDCtx`, where dictionary has been loaded. * It avoids reloading the dictionary each time. * `preparedDCtx` must have been properly initialized using ZSTD_decompressBegin_usingDict(). -* Requires 2 contexts : 1 for reference, which will not be modified, and 1 to run the decompression operation */ +* Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */ ZSTDLIB_API size_t ZSTD_decompress_usingPreparedDCtx( - ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize); + ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); /* ************************************** * Streaming functions (direct mode) ****************************************/ ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); -ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict,size_t dictSize, int compressionLevel); -ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict,size_t dictSize, ZSTD_parameters params); +ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); +ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params); ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx); ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); @@ -165,13 +165,17 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci You can then reuse ZSTD_CCtx to compress some new frame. */ +typedef struct { U64 frameContentSize; U32 windowLog; U32 mml; } ZSTD_frameParams; + +#define ZSTD_FRAMEHEADERSIZE_MAX 13 /* for static allocation */ +static const size_t ZSTD_frameHeaderSize_min = 5; +static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX; +ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */ ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); -ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcSize); - ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); @@ -182,15 +186,19 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it. A ZSTD_DCtx object can be re-used multiple times. - First typical operation is to retrieve frame parameters, using ZSTD_getFrameParams(). - This operation is independent, and just needs enough input data to properly decode the frame header. - Objective is to retrieve *params.windowlog, to know minimum amount of memory required during decoding. - Result : 0 when successful, it means the ZSTD_parameters structure has been filled. - >0 : means there is not enough data into src. Provides the expected size to successfully decode header. + First optional operation is to retrieve frame parameters, using ZSTD_getFrameParams(). + It can provide the minimum size of rolling buffer required to properly decompress data, + and optionally the final size of uncompressed content. + (Note : content size is an optional info that may not be present. 0 means : content size unknown) + Frame parameters are extracted from the beginning of compressed frame. + The amount of data to read is variable, from ZSTD_frameHeaderSize_min to ZSTD_frameHeaderSize_max (so if `srcSize` >= ZSTD_frameHeaderSize_max, it will always work) + If `srcSize` is too small for operation to succeed, function will return the minimum size it requires to produce a result. + Result : 0 when successful, it means the ZSTD_frameParams structure has been filled. + >0 : means there is not enough data into `src`. Provides the expected size to successfully decode header. errorCode, which can be tested using ZSTD_isError() - Start decompression, with ZSTD_decompressBegin() or ZSTD_decompressBegin_usingDict() - Alternatively, you can copy a prepared context, using ZSTD_copyDCtx() + Start decompression, with ZSTD_decompressBegin() or ZSTD_decompressBegin_usingDict(). + Alternatively, you can copy a prepared context, using ZSTD_copyDCtx(). Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively. ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue(). @@ -198,7 +206,7 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds ZSTD_decompressContinue() needs previous data blocks during decompression, up to (1 << windowlog). They should preferably be located contiguously, prior to current block. Alternatively, a round buffer is also possible. - @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst'. + @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity) It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header. A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero. @@ -210,10 +218,10 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds * Block functions ****************************************/ /*! Block functions produce and decode raw zstd blocks, without frame metadata. - User will have to take in charge required information to regenerate data, such as block sizes. + User will have to take in charge required information to regenerate data, such as compressed and content sizes. A few rules to respect : - - Uncompressed block size must be <= 128 KB + - Uncompressed block size must be <= ZSTD_BLOCKSIZE_MAX (128 KB) - Compressing or decompressing requires a context structure + Use ZSTD_createCCtx() and ZSTD_createDCtx() - It is necessary to init context before starting @@ -227,6 +235,7 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds + ZSTD_decompressBlock() doesn't accept uncompressed data as input !! */ +#define ZSTD_BLOCKSIZE_MAX (128 * 1024) /* define, for static allocation */ size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); diff --git a/programs/Makefile b/programs/Makefile index 4a650c488..ca2ba99fc 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -112,11 +112,11 @@ zstd-frugal: $(ZSTD_FILES) $(ZSTDDIR)/zbuff.c zstdcli.c fileio.c zstd-small: clean CFLAGS="-Os -s" $(MAKE) zstd-frugal -fullbench : $(ZSTD_FILES) \ +fullbench : $(ZSTD_FILES) $(ZSTDDIR)/zbuff.c \ datagen.c fullbench.c $(CC) $(FLAGS) $^ -o $@$(EXT) -fullbench32: $(ZSTD_FILES) \ +fullbench32: $(ZSTD_FILES) $(ZSTDDIR)/zbuff.c \ datagen.c fullbench.c $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) diff --git a/programs/bench.c b/programs/bench.c index 40146c10b..ff01ca687 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -40,11 +40,6 @@ # define _LARGEFILE64_SOURCE #endif -/* S_ISREG & gettimeofday() are not supported by MSVC */ -#if defined(_MSC_VER) || defined(_WIN32) -# define BMK_LEGACY_TIMER 1 -#endif - /* ************************************* * Includes @@ -54,12 +49,17 @@ #include /* fprintf, fopen, ftello64 */ #include /* stat64 */ #include /* stat64 */ +#include /* clock_t, clock, CLOCKS_PER_SEC */ -/* Use ftime() if gettimeofday() is not available */ -#if defined(BMK_LEGACY_TIMER) -# include /* timeb, ftime */ +/* sleep : posix - windows - others */ +#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__))) +# include +# define BMK_sleep(s) sleep(s) +#elif defined(_WIN32) +# include +# define BMK_sleep(s) Sleep(1000*s) #else -# include /* gettimeofday */ +# define BMK_sleep(s) /* disabled */ #endif #include "mem.h" @@ -88,7 +88,10 @@ # define ZSTD_VERSION "" #endif -#define NBLOOPS 3 +#define NBLOOPS 3 +#define TIMELOOP_S 1 +#define ACTIVEPERIOD_S 70 +#define COOLPERIOD_S 10 #define KB *(1 <<10) #define MB *(1 <<20) @@ -127,15 +130,15 @@ static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result /* ************************************* * Benchmark Parameters ***************************************/ -static int nbIterations = NBLOOPS; +static U32 g_nbIterations = NBLOOPS; static size_t g_blockSize = 0; void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; } -void BMK_SetNbIterations(int nbLoops) +void BMK_SetNbIterations(unsigned nbLoops) { - nbIterations = nbLoops; - DISPLAYLEVEL(2, "- %i iterations -\n", nbIterations); + g_nbIterations = nbLoops; + DISPLAYLEVEL(2, "- %i iterations -\n", g_nbIterations); } void BMK_SetBlockSize(size_t blockSize) @@ -148,44 +151,11 @@ void BMK_SetBlockSize(size_t blockSize) /* ******************************************************** * Private functions **********************************************************/ - -#if defined(BMK_LEGACY_TIMER) - -static int BMK_GetMilliStart(void) +static clock_t BMK_clockSpan( clock_t clockStart ) { - /* Based on Legacy ftime() - * Rolls over every ~ 12.1 days (0x100000/24/60/60) - * Use GetMilliSpan to correct for rollover */ - struct timeb tb; - int nCount; - ftime( &tb ); - nCount = (int) (tb.millitm + (tb.time & 0xfffff) * 1000); - return nCount; + return clock() - clockStart; /* works even if overflow, span limited to <= ~30mn */ } -#else - -static int BMK_GetMilliStart(void) -{ - /* Based on newer gettimeofday() - * Use GetMilliSpan to correct for rollover */ - struct timeval tv; - int nCount; - gettimeofday(&tv, NULL); - nCount = (int) (tv.tv_usec/1000 + (tv.tv_sec & 0xfffff) * 1000); - return nCount; -} - -#endif - - -static int BMK_GetMilliSpan( int nTimeStart ) -{ - int nSpan = BMK_GetMilliStart() - nTimeStart; - if ( nSpan < 0 ) - nSpan += 0x100000 * 1000; - return nSpan; -} static U64 BMK_getFileSize(const char* infilename) { @@ -259,14 +229,14 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, /* Init blockTable data */ { - U32 fileNb; const char* srcPtr = (const char*)srcBuffer; char* cPtr = (char*)compressedBuffer; char* resPtr = (char*)resultBuffer; + U32 fileNb; for (fileNb=0; fileNb ACTIVEPERIOD_S * CLOCKS_PER_SEC) { + DISPLAY("\rcooling down ... \r"); + BMK_sleep(COOLPERIOD_S); + coolTime = clock(); + } /* Compression */ DISPLAYLEVEL(2, "%2i-%-17.17s :%10u ->\r", loopNb, displayName, (U32)srcSize); - memset(compressedBuffer, 0xE5, maxCompressedSize); + memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */ nbLoops = 0; - milliTime = BMK_GetMilliStart(); - while (BMK_GetMilliStart() == milliTime); - milliTime = BMK_GetMilliStart(); - while (BMK_GetMilliSpan(milliTime) < timeloop) { + clockStart = clock(); + while (clock() == clockStart); + clockStart = clock(); + while (BMK_clockSpan(clockStart) < clockLoop) { ZSTD_compressBegin_advanced(refCtx, dictBuffer, dictBufferSize, ZSTD_getParams(cLevel, MAX(dictBufferSize, largestBlockSize))); for (blockNb=0; blockNb%10i (%5.3f),%6.1f MB/s\r", loopNb, displayName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000.); + DISPLAYLEVEL(2, "%2i-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r", + loopNb, displayName, (U32)srcSize, (U32)cSize, ratio, + (double)srcSize / 1000000. / (fastestC / CLOCKS_PER_SEC) ); #if 1 /* Decompression */ memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */ nbLoops = 0; - milliTime = BMK_GetMilliStart(); - while (BMK_GetMilliStart() == milliTime); - milliTime = BMK_GetMilliStart(); + clockStart = clock(); + while (clock() == clockStart); + clockStart = clock(); - for ( ; BMK_GetMilliSpan(milliTime) < timeloop; nbLoops++) { + for ( ; BMK_clockSpan(clockStart) < clockLoop; nbLoops++) { ZSTD_decompressBegin_usingDict(refDCtx, dictBuffer, dictBufferSize); for (blockNb=0; blockNb%10i (%5.3f),%6.1f MB/s ,%6.1f MB/s\r", loopNb, displayName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.); + clockSpan = BMK_clockSpan(clockStart); + if ((double)clockSpan < fastestD*nbLoops) fastestD = (double)clockSpan / nbLoops; + DISPLAYLEVEL(2, "%2i-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r", + loopNb, displayName, (U32)srcSize, (U32)cSize, ratio, + (double)srcSize / 1000000. / (fastestC / CLOCKS_PER_SEC), + (double)srcSize / 1000000. / (fastestD / CLOCKS_PER_SEC) ); /* CRC Checking */ _findError: @@ -439,7 +422,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, memset(&total, 0, sizeof(total)); if (g_displayLevel == 1 && !additionalParam) - DISPLAY("bench %s: input %u bytes, %i iterations, %u KB blocks\n", ZSTD_VERSION, (U32)benchedSize, nbIterations, (U32)(g_blockSize>>10)); + DISPLAY("bench %s: input %u bytes, %i iterations, %u KB blocks\n", ZSTD_VERSION, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10)); if (cLevelLast < cLevel) cLevelLast = cLevel; @@ -480,12 +463,11 @@ static U64 BMK_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles) static void BMK_loadFiles(void* buffer, size_t bufferSize, size_t* fileSizes, - const char** fileNamesTable, unsigned nbFiles) + const char** fileNamesTable, unsigned const nbFiles) { - BYTE* buff = (BYTE*)buffer; size_t pos = 0; - unsigned n; + unsigned n; for (n=0; n bufferSize-pos) fileSize = bufferSize-pos; - readSize = fread(buff+pos, 1, (size_t)fileSize, f); + readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f); if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]); pos += readSize; fileSizes[n] = (size_t)fileSize; @@ -577,7 +559,7 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, int additionalParam, d int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, int cLevel, int cLevelLast, int additionalParam) { - double compressibility = (double)g_compressibilityDefault / 100; + double const compressibility = (double)g_compressibilityDefault / 100; if (nbFiles == 0) BMK_syntheticTest(cLevel, cLevelLast, additionalParam, compressibility); diff --git a/programs/bench.h b/programs/bench.h index 5e6d3b18d..bc5ffa423 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -30,7 +30,7 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, int cLevel, int cLevelLast, int additionalParam); /* Set Parameters */ -void BMK_SetNbIterations(int nbLoops); +void BMK_SetNbIterations(unsigned nbLoops); void BMK_SetBlockSize(size_t blockSize); void BMK_setNotificationLevel(unsigned level); diff --git a/programs/fileio.c b/programs/fileio.c index 028c7db45..907d990a1 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -64,7 +64,7 @@ #include /* stat64 */ #include "mem.h" #include "fileio.h" -#include "zstd_static.h" /* ZSTD_magicNumber */ +#include "zstd_static.h" /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */ #include "zbuff_static.h" #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) @@ -126,6 +126,7 @@ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */ +void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; } #define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ if ((FIO_GetMilliSpan(g_time) > refreshRate) || (g_displayLevel>=4)) \ @@ -142,7 +143,8 @@ static clock_t g_time = 0; ***************************************/ static U32 g_overwrite = 0; void FIO_overwriteMode(void) { g_overwrite=1; } -void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; } +static U32 g_maxWLog = 23; +void FIO_setMaxWLog(unsigned maxWLog) { g_maxWLog = maxWLog; } /*-************************************* @@ -239,10 +241,10 @@ static FILE* FIO_openDstFile(const char* dstFileName) } -/*!FIO_loadFile -* creates a buffer, pointed by *bufferPtr, -* loads "filename" content into it -* up to MAX_DICT_SIZE bytes +/*! FIO_loadFile() : +* creates a buffer, pointed by `*bufferPtr`, +* loads `filename` content into it, +* up to MAX_DICT_SIZE bytes */ static size_t FIO_loadFile(void** bufferPtr, const char* fileName) { @@ -274,7 +276,7 @@ static size_t FIO_loadFile(void** bufferPtr, const char* fileName) } -/* ********************************************************************** +/*-********************************************************************** * Compression ************************************************************************/ typedef struct { @@ -321,7 +323,7 @@ static void FIO_freeCResources(cRess_t ress) /*! FIO_compressFilename_internal() : - * same as FIO_compressFilename_extRess(), with ress.desFile already opened + * same as FIO_compressFilename_extRess(), with `ress.desFile` already opened. * @return : 0 : compression completed correctly, * 1 : missing or pb opening srcFileName */ @@ -335,10 +337,14 @@ static int FIO_compressFilename_internal(cRess_t ress, U64 compressedfilesize = 0; size_t dictSize = ress.dictBufferSize; size_t sizeCheck, errorCode; + ZSTD_parameters params; /* init */ filesize = MAX(FIO_getFileSize(srcFileName),dictSize); - errorCode = ZBUFF_compressInit_advanced(ress.ctx, ress.dictBuffer, ress.dictBufferSize, ZSTD_getParams(cLevel, filesize)); + params = ZSTD_getParams(cLevel, filesize); + params.srcSize = filesize; + if (g_maxWLog) if (params.windowLog > g_maxWLog) params.windowLog = g_maxWLog; + errorCode = ZBUFF_compressInit_advanced(ress.ctx, ress.dictBuffer, ress.dictBufferSize, params); if (ZBUFF_isError(errorCode)) EXM_THROW(21, "Error initializing compression : %s", ZBUFF_getErrorName(errorCode)); /* Main compression loop */ @@ -389,7 +395,7 @@ static int FIO_compressFilename_internal(cRess_t ress, /*! FIO_compressFilename_internal() : - * same as FIO_compressFilename_extRess(), with ress.desFile already opened + * same as FIO_compressFilename_extRess(), with ress.destFile already opened (typically stdout) * @return : 0 : compression completed correctly, * 1 : missing or pb opening srcFileName */ @@ -427,6 +433,7 @@ static int FIO_compressFilename_extRess(cRess_t ress, if (ress.dstFile==0) { fclose(ress.srcFile); return 1; } result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, cLevel); + if (result!=0) remove(dstFileName); /* remove operation artefact */ fclose(ress.srcFile); /* no pb to expect : only reading */ if (fclose(ress.dstFile)) EXM_THROW(28, "Write error : cannot properly close %s", dstFileName); @@ -482,7 +489,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile ress.dstFile = stdout; for (u=0; u>20) ); - if (toRead == 0) break; + if (toRead == 0) break; /* end of frame */ if (readSize) EXM_THROW(38, "Decoding error : should consume entire input"); /* Fill input buffer */ if (toRead > ress.srcBufferSize) EXM_THROW(34, "too large block"); readSize = fread(ress.srcBuffer, 1, toRead, finput); - if (readSize != toRead) EXM_THROW(35, "Read error"); + if (readSize != toRead) + EXM_THROW(35, "Read error"); } return frameSize; @@ -633,13 +651,17 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) static int FIO_decompressFile_extRess(dRess_t ress, const char* dstFileName, const char* srcFileName) { + int result; ress.dstFile = FIO_openDstFile(dstFileName); if (ress.dstFile==0) return 1; - FIO_decompressSrcFile(ress, srcFileName); + result = FIO_decompressSrcFile(ress, srcFileName); + if (result != 0) { + remove(dstFileName); + } if (fclose(ress.dstFile)) EXM_THROW(38, "Write error : cannot properly close %s", dstFileName); - return 0; + return result; } diff --git a/programs/fileio.h b/programs/fileio.h index ee3cf2278..d5aae449b 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -41,14 +41,15 @@ extern "C" { #endif -/* ************************************* +/*-************************************* * Parameters ***************************************/ void FIO_overwriteMode(void); void FIO_setNotificationLevel(unsigned level); +void FIO_setMaxWLog(unsigned maxWLog); /**< if `maxWLog` == 0, no max enforced */ -/* ************************************* +/*-************************************* * Single File functions ***************************************/ /** FIO_compressFilename() : @@ -60,7 +61,7 @@ int FIO_compressFilename (const char* outfilename, const char* infilename, const int FIO_decompressFilename (const char* outfilename, const char* infilename, const char* dictFileName); -/* ************************************* +/*-************************************* * Multiple File functions ***************************************/ /** FIO_compressMultipleFilenames() : diff --git a/programs/fullbench.c b/programs/fullbench.c index f6a2321ec..24cf95d39 100644 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -1,6 +1,6 @@ /* fullbench.c - Detailed bench program for zstd - Copyright (C) Yann Collet 2014-2015 + Copyright (C) Yann Collet 2014-2016 GPL v2 License @@ -19,11 +19,10 @@ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. You can contact the author at : - - zstd source repository : https://github.com/Cyan4973/zstd - - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c + - zstd homepage : http://www.zstd.net */ -/************************************** +/*_************************************ * Compiler Options **************************************/ /* Disable some Visual warning messages */ @@ -38,13 +37,8 @@ # define _LARGEFILE64_SOURCE #endif -/* S_ISREG & gettimeofday() are not supported by MSVC */ -#if defined(_MSC_VER) || defined(_WIN32) -# define BMK_LEGACY_TIMER 1 -#endif - -/************************************** +/*_************************************ * Includes **************************************/ #include /* malloc */ @@ -52,21 +46,16 @@ #include /* stat64 */ #include /* stat64 */ #include /* strcmp */ - -/* Use ftime() if gettimeofday() is not available on your target */ -#if defined(BMK_LEGACY_TIMER) -# include /* timeb, ftime */ -#else -# include /* gettimeofday */ -#endif +#include /* clock_t, clock, CLOCKS_PER_SEC */ #include "mem.h" #include "zstd_static.h" #include "fse_static.h" +#include "zbuff.h" #include "datagen.h" -/************************************** +/*_************************************ * Compiler Options **************************************/ /* S_ISREG & gettimeofday() are not supported by MSVC */ @@ -75,7 +64,7 @@ #endif -/************************************** +/*_************************************ * Constants **************************************/ #define PROGRAM_DESCRIPTION "Zstandard speed analyzer" @@ -90,98 +79,63 @@ #define MB *(1<<20) #define NBLOOPS 6 -#define TIMELOOP 2500 +#define TIMELOOP_S 2 #define KNUTH 2654435761U #define MAX_MEM (1984 MB) #define COMPRESSIBILITY_DEFAULT 0.50 -static const size_t sampleSize = 10000000; +static const size_t g_sampleSize = 10000000; -/************************************** +/*_************************************ * Macros **************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -/************************************** +/*_************************************ * Benchmark Parameters **************************************/ -static int nbIterations = NBLOOPS; +static U32 g_nbIterations = NBLOOPS; static double g_compressibility = COMPRESSIBILITY_DEFAULT; -void BMK_SetNbIterations(int nbLoops) +static void BMK_SetNbIterations(U32 nbLoops) { - nbIterations = nbLoops; - DISPLAY("- %i iterations -\n", nbIterations); + g_nbIterations = nbLoops; + DISPLAY("- %i iterations -\n", g_nbIterations); } -/********************************************************* +/*_******************************************************* * Private functions *********************************************************/ - -#if defined(BMK_LEGACY_TIMER) - -static int BMK_GetMilliStart(void) +static clock_t BMK_clockSpan( clock_t clockStart ) { - /* Based on Legacy ftime() - * Rolls over every ~ 12.1 days (0x100000/24/60/60) - * Use GetMilliSpan to correct for rollover */ - struct timeb tb; - int nCount; - ftime( &tb ); - nCount = (int) (tb.millitm + (tb.time & 0xfffff) * 1000); - return nCount; -} - -#else - -static int BMK_GetMilliStart(void) -{ - /* Based on newer gettimeofday() - * Use GetMilliSpan to correct for rollover */ - struct timeval tv; - int nCount; - gettimeofday(&tv, NULL); - nCount = (int) (tv.tv_usec/1000 + (tv.tv_sec & 0xfffff) * 1000); - return nCount; -} - -#endif - - -static int BMK_GetMilliSpan( int nTimeStart ) -{ - int nSpan = BMK_GetMilliStart() - nTimeStart; - if ( nSpan < 0 ) - nSpan += 0x100000 * 1000; - return nSpan; + return clock() - clockStart; /* works even if overflow, span limited to <= ~30mn */ } static size_t BMK_findMaxMem(U64 requiredMem) { - size_t step = 64 MB; - BYTE* testmem=NULL; + const size_t step = 64 MB; + void* testmem = NULL; requiredMem = (((requiredMem >> 26) + 1) << 26); if (requiredMem > MAX_MEM) requiredMem = MAX_MEM; - requiredMem += 2*step; - while (!testmem) - { + requiredMem += step; + do { + testmem = malloc ((size_t)requiredMem); requiredMem -= step; - testmem = (BYTE*) malloc ((size_t)requiredMem); - } + } while (!testmem); free (testmem); - return (size_t) (requiredMem - step); + return (size_t) requiredMem; } -static U64 BMK_GetFileSize(char* infilename) +static U64 BMK_GetFileSize(const char* infilename) { int r; #if defined(_MSC_VER) @@ -196,42 +150,39 @@ static U64 BMK_GetFileSize(char* infilename) } -/********************************************************* +/*_******************************************************* * Benchmark wrappers *********************************************************/ typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t; -typedef struct -{ +typedef struct { blockType_t blockType; U32 unusedBits; U32 origSize; } blockProperties_t; -static size_t g_cSize = 0; -static ZSTD_DCtx* g_dctxPtr = NULL; - -extern size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr); -extern size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr, FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb, const void* src, size_t srcSize); - size_t local_ZSTD_compress(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize) { (void)buff2; return ZSTD_compress(dst, dstSize, src, srcSize, 1); } +static size_t g_cSize = 0; size_t local_ZSTD_decompress(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize) { (void)src; (void)srcSize; return ZSTD_decompress(dst, dstSize, buff2, g_cSize); } +static ZSTD_DCtx* g_zdc = NULL; extern size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* ctx, const void* src, size_t srcSize); size_t local_ZSTD_decodeLiteralsBlock(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize) { (void)src; (void)srcSize; (void)dst; (void)dstSize; - return ZSTD_decodeLiteralsBlock((ZSTD_DCtx*)g_dctxPtr, buff2, g_cSize); + return ZSTD_decodeLiteralsBlock((ZSTD_DCtx*)g_zdc, buff2, g_cSize); } +extern size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr); +extern size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr, FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb, const void* src, size_t srcSize); size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize) { U32 DTableML[FSE_DTABLE_SIZE_U32(10)], DTableLL[FSE_DTABLE_SIZE_U32(10)], DTableOffb[FSE_DTABLE_SIZE_U32(9)]; /* MLFSELog, LLFSELog and OffFSELog are not public values */ @@ -243,20 +194,76 @@ size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const } +static ZBUFF_CCtx* g_zbcc = NULL; +size_t local_ZBUFF_compress(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize) +{ + size_t compressedSize; + size_t srcRead = srcSize, dstWritten = dstCapacity; + (void)buff2; + ZBUFF_compressInit(g_zbcc, 1); + ZBUFF_compressContinue(g_zbcc, dst, &dstWritten, src, &srcRead); + compressedSize = dstWritten; + dstWritten = dstCapacity-compressedSize; + ZBUFF_compressEnd(g_zbcc, ((char*)dst)+compressedSize, &dstWritten); + compressedSize += dstWritten; + return compressedSize; +} -/********************************************************* +static ZBUFF_DCtx* g_zbdc = NULL; +static size_t local_ZBUFF_decompress(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize) +{ + size_t srcRead = g_cSize, dstWritten = dstCapacity; + (void)src; (void)srcSize; + ZBUFF_decompressInit(g_zbdc); + ZBUFF_decompressContinue(g_zbdc, dst, &dstWritten, buff2, &srcRead); + return dstWritten; +} + +static ZSTD_CCtx* g_zcc = NULL; +size_t local_ZSTD_compressContinue(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize) +{ + size_t compressedSize; + (void)buff2; + ZSTD_compressBegin(g_zcc, 1); + compressedSize = ZSTD_compressContinue(g_zcc, dst, dstCapacity, src, srcSize); + compressedSize += ZSTD_compressEnd(g_zcc, ((char*)dst)+compressedSize, dstCapacity-compressedSize); + return compressedSize; +} + +size_t local_ZSTD_decompressContinue(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize) +{ + size_t regeneratedSize = 0; + const BYTE* ip = (const BYTE*)buff2; + const BYTE* const iend = ip + g_cSize; + BYTE* op = (BYTE*)dst; + size_t remainingCapacity = dstCapacity; + + (void)src; (void)srcSize; + ZSTD_decompressBegin(g_zdc); + while (ip < iend) { + size_t const iSize = ZSTD_nextSrcSizeToDecompress(g_zdc); + size_t const decodedSize = ZSTD_decompressContinue(g_zdc, op, remainingCapacity, ip, iSize); + ip += iSize; + regeneratedSize += decodedSize; + op += decodedSize; + remainingCapacity -= decodedSize; + } + + return regeneratedSize; +} + + +/*_******************************************************* * Bench functions *********************************************************/ -size_t benchMem(void* src, size_t srcSize, U32 benchNb) +static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) { BYTE* dstBuff; size_t dstBuffSize; BYTE* buff2; - int loopNb; const char* benchName; size_t (*benchFunction)(void* dst, size_t dstSize, void* verifBuff, const void* src, size_t srcSize); double bestTime = 100000000.; - size_t errorCode = 0; /* Selection */ switch(benchNb) @@ -264,15 +271,27 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb) case 1: benchFunction = local_ZSTD_compress; benchName = "ZSTD_compress"; break; - case 11: + case 2: benchFunction = local_ZSTD_decompress; benchName = "ZSTD_decompress"; break; + case 11: + benchFunction = local_ZSTD_compressContinue; benchName = "ZSTD_compressContinue"; + break; + case 12: + benchFunction = local_ZSTD_decompressContinue; benchName = "ZSTD_decompressContinue"; + break; case 31: benchFunction = local_ZSTD_decodeLiteralsBlock; benchName = "ZSTD_decodeLiteralsBlock"; break; case 32: benchFunction = local_ZSTD_decodeSeqHeaders; benchName = "ZSTD_decodeSeqHeaders"; break; + case 41: + benchFunction = local_ZBUFF_compress; benchName = "ZBUFF_compressContinue"; + break; + case 42: + benchFunction = local_ZBUFF_decompress; benchName = "ZBUFF_decompressContinue"; + break; default : return 0; } @@ -281,9 +300,7 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb) dstBuffSize = ZSTD_compressBound(srcSize); dstBuff = (BYTE*)malloc(dstBuffSize); buff2 = (BYTE*)malloc(dstBuffSize); - g_dctxPtr = ZSTD_createDCtx(); - if ((!dstBuff) || (!buff2)) - { + if ((!dstBuff) || (!buff2)) { DISPLAY("\nError: not enough memory!\n"); free(dstBuff); free(buff2); return 12; @@ -292,45 +309,66 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb) /* Preparation */ switch(benchNb) { - case 11: + case 2: + g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1); + break; + case 11 : + if (g_zcc==NULL) g_zcc = ZSTD_createCCtx(); + break; + case 12 : + if (g_zdc==NULL) g_zdc = ZSTD_createDCtx(); g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1); break; case 31: /* ZSTD_decodeLiteralsBlock */ - { - blockProperties_t bp; + if (g_zdc==NULL) g_zdc = ZSTD_createDCtx(); + { blockProperties_t bp; + ZSTD_frameParams zfp; + size_t frameHeaderSize, skippedSize; g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1); - ZSTD_getcBlockSize(dstBuff+4, dstBuffSize, &bp); /* Get 1st block type */ - if (bp.blockType != bt_compressed) - { + frameHeaderSize = ZSTD_getFrameParams(&zfp, dstBuff, ZSTD_frameHeaderSize_min); + if (frameHeaderSize==0) frameHeaderSize = ZSTD_frameHeaderSize_min; + ZSTD_getcBlockSize(dstBuff+frameHeaderSize, dstBuffSize, &bp); /* Get 1st block type */ + if (bp.blockType != bt_compressed) { DISPLAY("ZSTD_decodeLiteralsBlock : impossible to test on this sample (not compressible)\n"); goto _cleanOut; } - memcpy(buff2, dstBuff+8, g_cSize-8); + skippedSize = frameHeaderSize + 3 /* ZSTD_blockHeaderSize */; + memcpy(buff2, dstBuff+skippedSize, g_cSize-skippedSize); srcSize = srcSize > 128 KB ? 128 KB : srcSize; /* speed relative to block */ break; } case 32: /* ZSTD_decodeSeqHeaders */ - { - blockProperties_t bp; + if (g_zdc==NULL) g_zdc = ZSTD_createDCtx(); + { blockProperties_t bp; + ZSTD_frameParams zfp; const BYTE* ip = dstBuff; const BYTE* iend; - size_t blockSize; + size_t frameHeaderSize, cBlockSize; ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1); /* it would be better to use direct block compression here */ - ip += 5; /* Skip frame Header */ - blockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); /* Get 1st block type */ - if (bp.blockType != bt_compressed) - { + g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1); + frameHeaderSize = ZSTD_getFrameParams(&zfp, dstBuff, ZSTD_frameHeaderSize_min); + if (frameHeaderSize==0) frameHeaderSize = ZSTD_frameHeaderSize_min; + ip += frameHeaderSize; /* Skip frame Header */ + cBlockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); /* Get 1st block type */ + if (bp.blockType != bt_compressed) { DISPLAY("ZSTD_decodeSeqHeaders : impossible to test on this sample (not compressible)\n"); goto _cleanOut; } - iend = ip + 3 + blockSize; /* End of first block */ - ip += 3; /* skip block header */ - ip += ZSTD_decodeLiteralsBlock(g_dctxPtr, ip, iend-ip); /* skip literal segment */ + iend = ip + 3 /* ZSTD_blockHeaderSize */ + cBlockSize; /* End of first block */ + ip += 3 /* ZSTD_blockHeaderSize */; /* skip block header */ + ip += ZSTD_decodeLiteralsBlock(g_zdc, ip, iend-ip); /* skip literal segment */ g_cSize = iend-ip; memcpy(buff2, ip, g_cSize); /* copy rest of block (it starts by SeqHeader) */ srcSize = srcSize > 128 KB ? 128 KB : srcSize; /* speed relative to block */ break; } + case 41 : + if (g_zbcc==NULL) g_zbcc = ZBUFF_createCCtx(); + break; + case 42 : + if (g_zbdc==NULL) g_zbdc = ZBUFF_createDCtx(); + g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1); + break; /* test functions */ /* by convention, test functions can be added > 100 */ @@ -340,53 +378,44 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb) { size_t i; for (i=0; i %s !! \n", benchName, ZSTD_getErrorName(errorCode)); exit(1); } - nbRounds++; + clockStart = clock(); + while (clock() == clockStart); + clockStart = clock(); + for (nbRounds=0; BMK_clockSpan(clockStart) < timeLoop; nbRounds++) { + benchResult = benchFunction(dstBuff, dstBuffSize, buff2, src, srcSize); + if (ZSTD_isError(benchResult)) { DISPLAY("ERROR ! %s() => %s !! \n", benchName, ZSTD_getErrorName(benchResult)); exit(1); } } - milliTime = BMK_GetMilliSpan(milliTime); - - averageTime = (double)milliTime / nbRounds; + averageTime = (((double)BMK_clockSpan(clockStart)) / CLOCKS_PER_SEC) / nbRounds; if (averageTime < bestTime) bestTime = averageTime; - DISPLAY("%2i- %-30.30s : %7.1f MB/s (%9u)\r", loopNb, benchName, (double)srcSize / bestTime / 1000., (U32)errorCode); - } - - DISPLAY("%2u- %-30.30s : %7.1f MB/s (%9u)\n", benchNb, benchName, (double)srcSize / bestTime / 1000., (U32)errorCode); + DISPLAY("%2i- %-30.30s : %7.1f MB/s (%9u)\r", loopNb, benchName, (double)srcSize / (1 MB) / bestTime, (U32)benchResult); + }} + DISPLAY("%2u\n", benchNb); _cleanOut: free(dstBuff); free(buff2); - ZSTD_freeDCtx(g_dctxPtr); return 0; } -int benchSample(U32 benchNb) +static int benchSample(U32 benchNb) { - char* origBuff; - size_t benchedSize = sampleSize; + size_t const benchedSize = g_sampleSize; const char* name = "Sample 10MiB"; /* Allocation */ - origBuff = (char*) malloc((size_t)benchedSize); - if(!origBuff) - { - DISPLAY("\nError: not enough memory!\n"); - return 12; - } + void* origBuff = malloc(benchedSize); + if (!origBuff) { DISPLAY("\nError: not enough memory!\n"); return 12; } /* Fill buffer */ RDG_genBuffer(origBuff, benchedSize, g_compressibility, 0.0, 0); @@ -404,58 +433,41 @@ int benchSample(U32 benchNb) } -int benchFiles(char** fileNamesTable, int nbFiles, U32 benchNb) +static int benchFiles(const char** fileNamesTable, const int nbFiles, U32 benchNb) { - int fileIdx=0; - /* Loop for each file */ - while (fileIdx inFileSize) benchedSize = (size_t)inFileSize; if (benchedSize < inFileSize) - { - DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20)); - } + DISPLAY("Not enough memory for '%s' full size; testing %u MB only...\n", inFileName, (U32)(benchedSize>>20)); /* Alloc */ - origBuff = (char*) malloc((size_t)benchedSize); - if(!origBuff) - { - DISPLAY("\nError: not enough memory!\n"); - fclose(inFile); - return 12; - } + origBuff = malloc(benchedSize); + if (!origBuff) { DISPLAY("\nError: not enough memory!\n"); fclose(inFile); return 12; } /* Fill input buffer */ DISPLAY("Loading %s... \r", inFileName); - readSize = fread(origBuff, 1, benchedSize, inFile); - fclose(inFile); - - if(readSize != benchedSize) { - DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); - free(origBuff); - return 13; - } + size_t readSize = fread(origBuff, 1, benchedSize, inFile); + fclose(inFile); + if (readSize != benchedSize) { + DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); + free(origBuff); + return 13; + } } /* bench */ DISPLAY("\r%79s\r", ""); @@ -464,13 +476,15 @@ int benchFiles(char** fileNamesTable, int nbFiles, U32 benchNb) benchMem(origBuff, benchedSize, benchNb); else for (benchNb=0; benchNb<100; benchNb++) benchMem(origBuff, benchedSize, benchNb); + + free(origBuff); } return 0; } -int usage(char* exename) +static int usage(const char* exename) { DISPLAY( "Usage :\n"); DISPLAY( " %s [arg] file1 file2 ... fileX\n", exename); @@ -479,8 +493,9 @@ int usage(char* exename) return 0; } -int usage_advanced(void) +static int usage_advanced(const char* exename) { + usage(exename); DISPLAY( "\nAdvanced options :\n"); DISPLAY( " -b# : test only function # \n"); DISPLAY( " -i# : iteration loops [1-9](default : %i)\n", NBLOOPS); @@ -488,45 +503,38 @@ int usage_advanced(void) return 0; } -int badusage(char* exename) +static int badusage(const char* exename) { DISPLAY("Wrong parameters\n"); usage(exename); - return 0; + return 1; } -int main(int argc, char** argv) +int main(int argc, const char** argv) { - int i, - filenamesStart=0, - result; - char* exename=argv[0]; - char* input_filename=0; + int i, filenamesStart=0, result; + const char* exename = argv[0]; + const char* input_filename = NULL; U32 benchNb = 0, main_pause = 0; - // Welcome message DISPLAY(WELCOME_MESSAGE); + if (argc<1) return badusage(exename); - if (argc<1) { badusage(exename); return 1; } + for(i=1; i= '0') && (argument[1]<= '9')) - { + while ((argument[1]>= '0') && (argument[1]<= '9')) { benchNb *= 10; benchNb += argument[1] - '0'; argument++; @@ -544,20 +551,17 @@ int main(int argc, char** argv) /* Modify Nb Iterations */ case 'i': - if ((argument[1] >='0') && (argument[1] <='9')) - { + if ((argument[1] >='0') && (argument[1] <='9')) { int iters = argument[1] - '0'; BMK_SetNbIterations(iters); argument++; } break; - /* Select specific algorithm to bench */ + /* Select compressibility of synthetic sample */ case 'P': - { - U32 proba32 = 0; - while ((argument[1]>= '0') && (argument[1]<= '9')) - { + { U32 proba32 = 0; + while ((argument[1]>= '0') && (argument[1]<= '9')) { proba32 *= 10; proba32 += argument[1] - '0'; argument++; @@ -567,7 +571,7 @@ int main(int argc, char** argv) break; /* Unknown command */ - default : badusage(exename); return 1; + default : return badusage(exename); } } continue; @@ -577,9 +581,10 @@ int main(int argc, char** argv) if (!input_filename) { input_filename=argument; filenamesStart=i; continue; } } - if (filenamesStart==0) + if (filenamesStart==0) /* no input file */ result = benchSample(benchNb); - else result = benchFiles(argv+filenamesStart, argc-filenamesStart, benchNb); + else + result = benchFiles(argv+filenamesStart, argc-filenamesStart, benchNb); if (main_pause) { int unused; printf("press enter...\n"); unused = getchar(); (void)unused; } diff --git a/programs/fuzzer.c b/programs/fuzzer.c index 697695b38..55e0c7d70 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -513,8 +513,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed"); /* compression failure test : too small dest buffer */ - if (cSize > 3) - { + if (cSize > 3) { const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */ const size_t tooSmallSize = cSize - missing; static const U32 endMark = 0x4DC2B1A9; @@ -526,6 +525,13 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow"); } + /* decompression header test */ + { ZSTD_frameParams dParams; + size_t const check = ZSTD_getFrameParams(&dParams, cBuffer, cSize); + CHECK(ZSTD_isError(check), "Frame Parameters extraction failed"); + CHECK(dParams.frameContentSize != sampleSize, "Frame content size incorrect"); + } + /* successfull decompression tests*/ dSupSize = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1; dSize = ZSTD_decompress(dstBuffer, sampleSize + dSupSize, cBuffer, cSize); diff --git a/programs/playTests.sh b/programs/playTests.sh index ec625ee84..891ab9d4e 100755 --- a/programs/playTests.sh +++ b/programs/playTests.sh @@ -36,7 +36,10 @@ $ZSTD --decompress tmpCompressed --stdout > tmpResult $ZSTD -q tmp && die "overwrite check failed!" $ZSTD -q -f tmp $ZSTD -q --force tmp - +$ZSTD -df tmp && die "should have refused : wrong extension" +cp tmp tmp2.zst +$ZSTD -df tmp2.zst && die "should have failed : wrong format" +rm tmp2.zst echo "\n**** frame concatenation **** " diff --git a/programs/zstdcli.c b/programs/zstdcli.c index c986caee3..6b6b92d25 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -132,6 +132,7 @@ static int usage_advanced(const char* programName) DISPLAY( " -v : verbose mode\n"); DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n"); DISPLAY( " -c : force write to standard output, even if it is the console\n"); + DISPLAY( "--ultra : enable ultra modes (requires more memory to decompress)\n"); #ifndef ZSTD_NODICT DISPLAY( "Dictionary builder :\n"); DISPLAY( "--train : create a dictionary from a training set of files \n"); @@ -220,6 +221,7 @@ int main(int argCount, const char** argv) if (!strcmp(argument, "--train")) { dictBuild=1; outFileName=g_defaultDictName; continue; } if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; continue; } if (!strcmp(argument, "--keep")) { continue; } /* does nothing, since preserving input is default; for gzip/xz compatibility */ + if (!strcmp(argument, "--ultra")) { FIO_setMaxWLog(0); continue; } /* '-' means stdin/stdout */ if (!strcmp(argument, "-")){ diff --git a/visual/2013/fullbench/fullbench.vcxproj b/visual/2013/fullbench/fullbench.vcxproj index 3797960f4..afee2370c 100644 --- a/visual/2013/fullbench/fullbench.vcxproj +++ b/visual/2013/fullbench/fullbench.vcxproj @@ -161,6 +161,7 @@ + @@ -175,6 +176,8 @@ + + diff --git a/visual/2013/fullbench/fullbench.vcxproj.filters b/visual/2013/fullbench/fullbench.vcxproj.filters index 3a82000fb..1d5f9c59b 100644 --- a/visual/2013/fullbench/fullbench.vcxproj.filters +++ b/visual/2013/fullbench/fullbench.vcxproj.filters @@ -33,6 +33,9 @@ Fichiers sources + + Fichiers sources + @@ -68,5 +71,11 @@ Fichiers d%27en-tête + + Fichiers d%27en-tête + + + Fichiers d%27en-tête + \ No newline at end of file