From d1b26849e504bc72a2e3b1e9d247be37f9218d7b Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 15 Mar 2016 01:24:33 +0100 Subject: [PATCH 1/3] Frame content size supported --- lib/zstd_compress.c | 148 +++++++++++++++++++++++------------------- lib/zstd_decompress.c | 11 ++-- 2 files changed, 85 insertions(+), 74 deletions(-) diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index 469410faf..b1b106b2b 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -425,11 +425,11 @@ static void ZSTD_reduceIndex (ZSTD_CCtx* zc, FSE_ENCODING_DYNAMIC : read NCount */ -size_t ZSTD_noCompressBlock (void* dst, size_t maxDstSize, const void* src, size_t srcSize) +size_t ZSTD_noCompressBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize) { BYTE* const ostart = (BYTE* const)dst; - if (srcSize + ZSTD_blockHeaderSize > maxDstSize) return ERROR(dstSize_tooSmall); + if (srcSize + ZSTD_blockHeaderSize > dstCapacity) return ERROR(dstSize_tooSmall); memcpy(ostart + ZSTD_blockHeaderSize, src, srcSize); /* Build header */ @@ -442,12 +442,12 @@ size_t ZSTD_noCompressBlock (void* dst, size_t maxDstSize, const void* src, size } -static size_t ZSTD_noCompressLiterals (void* dst, size_t maxDstSize, const void* src, size_t srcSize) +static size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize) { BYTE* const ostart = (BYTE* const)dst; const U32 flSize = 1 + (srcSize>31) + (srcSize>4095); - if (srcSize + flSize > maxDstSize) return ERROR(dstSize_tooSmall); + if (srcSize + flSize > dstCapacity) return ERROR(dstSize_tooSmall); switch(flSize) { @@ -470,12 +470,12 @@ static size_t ZSTD_noCompressLiterals (void* dst, size_t maxDstSize, const void* return srcSize + flSize; } -static size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t maxDstSize, const void* src, size_t srcSize) +static size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize) { BYTE* const ostart = (BYTE* const)dst; U32 flSize = 1 + (srcSize>31) + (srcSize>4095); - (void)maxDstSize; /* maxDstSize guaranteed to be >=4, hence large enough */ + (void)dstCapacity; /* dstCapacity guaranteed to be >=4, hence large enough */ switch(flSize) { @@ -502,7 +502,7 @@ static size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t maxDstSize, const size_t ZSTD_minGain(size_t srcSize) { return (srcSize >> 6) + 2; } static size_t ZSTD_compressLiterals (ZSTD_CCtx* zc, - void* dst, size_t maxDstSize, + void* dst, size_t dstCapacity, const void* src, size_t srcSize) { const size_t minGain = ZSTD_minGain(srcSize); @@ -512,19 +512,19 @@ static size_t ZSTD_compressLiterals (ZSTD_CCtx* zc, U32 hType = IS_HUF; size_t clitSize; - if (maxDstSize < lhSize+1) return ERROR(dstSize_tooSmall); /* not enough space for compression */ + if (dstCapacity < lhSize+1) return ERROR(dstSize_tooSmall); /* not enough space for compression */ if (zc->flagStaticTables && (lhSize==3)) { hType = IS_PCH; singleStream = 1; - clitSize = HUF_compress1X_usingCTable(ostart+lhSize, maxDstSize-lhSize, src, srcSize, zc->hufTable); + clitSize = HUF_compress1X_usingCTable(ostart+lhSize, dstCapacity-lhSize, src, srcSize, zc->hufTable); } else { - clitSize = singleStream ? HUF_compress1X(ostart+lhSize, maxDstSize-lhSize, src, srcSize, 255, 12) - : HUF_compress2 (ostart+lhSize, maxDstSize-lhSize, src, srcSize, 255, 12); + clitSize = singleStream ? HUF_compress1X(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 12) + : HUF_compress2 (ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 12); } - if ((clitSize==0) || (clitSize >= srcSize - minGain)) return ZSTD_noCompressLiterals(dst, maxDstSize, src, srcSize); - if (clitSize==1) return ZSTD_compressRleLiteralsBlock(dst, maxDstSize, src, srcSize); + if ((clitSize==0) || (clitSize >= srcSize - minGain)) return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); + if (clitSize==1) return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize); /* Build header */ switch(lhSize) @@ -557,7 +557,7 @@ static size_t ZSTD_compressLiterals (ZSTD_CCtx* zc, #define LITERAL_NOENTROPY 63 /* don't even attempt to compress literals below this threshold (cheap heuristic) */ size_t ZSTD_compressSequences(ZSTD_CCtx* zc, - void* dst, size_t maxDstSize, + void* dst, size_t dstCapacity, size_t srcSize) { const seqStore_t* seqStorePtr = &(zc->seqStore); @@ -577,7 +577,7 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* zc, BYTE* const offCodeTable = seqStorePtr->offCodeStart; BYTE* const ostart = (BYTE*)dst; BYTE* op = ostart; - BYTE* const oend = ostart + maxDstSize; + BYTE* const oend = ostart + dstCapacity; const size_t nbSeq = llPtr - llTable; const size_t minGain = ZSTD_minGain(srcSize); const size_t maxCSize = srcSize - minGain; @@ -590,9 +590,9 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* zc, const size_t minLitSize = zc->flagStaticTables ? 6 : LITERAL_NOENTROPY; if (litSize <= minLitSize) - cSize = ZSTD_noCompressLiterals(op, maxDstSize, op_lit_start, litSize); + cSize = ZSTD_noCompressLiterals(op, dstCapacity, op_lit_start, litSize); else - cSize = ZSTD_compressLiterals(zc, op, maxDstSize, op_lit_start, litSize); + cSize = ZSTD_compressLiterals(zc, op, dstCapacity, op_lit_start, litSize); if (ZSTD_isError(cSize)) return cSize; op += cSize; } @@ -1940,17 +1940,17 @@ static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int } -static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t maxDstSize, const void* src, size_t srcSize) +static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { ZSTD_blockCompressor blockCompressor = ZSTD_selectBlockCompressor(zc->params.strategy, zc->lowLimit < zc->dictLimit); if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) return 0; /* don't even attempt compression below a certain srcSize */ blockCompressor(zc, src, srcSize); - return ZSTD_compressSequences(zc, dst, maxDstSize, srcSize); + return ZSTD_compressSequences(zc, dst, dstCapacity, srcSize); } static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, - void* dst, size_t maxDstSize, + void* dst, size_t dstCapacity, const void* src, size_t srcSize) { size_t blockSize = zc->blockSize; @@ -1969,7 +1969,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, while (remaining) { size_t cSize; - if (maxDstSize < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */ + if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */ if (remaining < blockSize) blockSize = remaining; if ((U32)(ip+blockSize - zc->base) > zc->loadedDictEnd + maxDist) { /* enforce maxDist */ @@ -1978,11 +1978,11 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, if (zc->dictLimit < zc->lowLimit) zc->dictLimit = zc->lowLimit; } - cSize = ZSTD_compressBlock_internal(zc, op+ZSTD_blockHeaderSize, maxDstSize-ZSTD_blockHeaderSize, ip, blockSize); + cSize = ZSTD_compressBlock_internal(zc, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, ip, blockSize); if (ZSTD_isError(cSize)) return cSize; if (cSize == 0) { /* block is not compressible */ - cSize = ZSTD_noCompressBlock(op, maxDstSize, ip, blockSize); + cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize); if (ZSTD_isError(cSize)) return cSize; } else { op[0] = (BYTE)(cSize>>16); @@ -1993,7 +1993,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, } remaining -= blockSize; - maxDstSize -= cSize; + dstCapacity -= cSize; ip += blockSize; op += cSize; } @@ -2077,12 +2077,12 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* zc, } -size_t ZSTD_compressBlock(ZSTD_CCtx* zc, void* dst, size_t maxDstSize, const void* src, size_t srcSize) +size_t ZSTD_compressBlock(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { if (srcSize > BLOCKSIZE) return ERROR(srcSize_wrong); zc->params.searchLength = MINMATCH; /* force ZSTD_btopt to MINMATCH in block mode */ ZSTD_LOG_BLOCK("%p: ZSTD_compressBlock searchLength=%d\n", zc->base, zc->params.searchLength); - return ZSTD_compressContinue_internal(zc, dst, maxDstSize, src, srcSize, 0); + return ZSTD_compressContinue_internal(zc, dst, dstCapacity, src, srcSize, 0); } @@ -2174,20 +2174,20 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* zc, const void* dict, size_t return hufHeaderSize + offcodeHeaderSize + matchlengthHeaderSize + litlengthHeaderSize; } - +/** ZSTD_compress_insertDictionary() : +* @return : 0, or an error code */ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, size_t dictSize) { - if (dict && (dictSize>4)) { - U32 magic = MEM_readLE32(dict); - size_t eSize; - if (magic != ZSTD_DICT_MAGIC) - return ZSTD_loadDictionaryContent(zc, dict, dictSize); + if ((dict==NULL) || (dictSize<=4)) return 0; - eSize = ZSTD_loadDictEntropyStats(zc, (const char*)dict+4, dictSize-4) + 4; - if (ZSTD_isError(eSize)) return eSize; - return ZSTD_loadDictionaryContent(zc, (const char*)dict+eSize, dictSize-eSize); + /* default : dict is pure content */ + if (MEM_readLE32(dict) != ZSTD_DICT_MAGIC) return ZSTD_loadDictionaryContent(zc, dict, dictSize); + + /* known magic number : dict is parsed for entropy stats and content */ + { size_t const eSize = ZSTD_loadDictEntropyStats(zc, (const char*)dict+4 /* skip magic */, dictSize-4) + 4; + if (ZSTD_isError(eSize)) return eSize; + return ZSTD_loadDictionaryContent(zc, (const char*)dict+eSize, dictSize-eSize); } - return 0; } @@ -2197,27 +2197,42 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* zc, const void* dict, size_t dictSize, ZSTD_parameters params) { - size_t errorCode; - ZSTD_validateParams(¶ms); - errorCode = ZSTD_resetCCtx_advanced(zc, params); - if (ZSTD_isError(errorCode)) return errorCode; + { size_t const errorCode = ZSTD_resetCCtx_advanced(zc, params); + if (ZSTD_isError(errorCode)) return errorCode; } + + /* Write Frame Header into ctx headerBuffer */ + MEM_writeLE32(zc->headerBuffer, ZSTD_MAGICNUMBER); + { + U32 const fcsSize[4] = { 0, 1, 2, 8 }; + U32 const fcsId = (params.srcSize>0) + (params.srcSize>=256) + (params.srcSize>=65536+256); /* 0-3 */ + BYTE fdescriptor = (BYTE)(params.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN); /* windowLog : 4 KB - 128 MB */ + fdescriptor |= (BYTE)((params.searchLength==3)<<4); /* mml : 3-4 */ + fdescriptor |= (BYTE)(fcsId << 6); + ((BYTE*)zc->headerBuffer)[4] = fdescriptor; + switch(fcsId) + { + default: /* impossible */ + case 0 : break; + case 1 : ((BYTE*)zc->headerBuffer)[5] = (BYTE)(params.srcSize); break; + case 2 : MEM_writeLE16(((BYTE*)zc->headerBuffer)+5, (U16)(params.srcSize-256)); break; + case 3 : MEM_writeLE64(((BYTE*)zc->headerBuffer)+5, (U64)(params.srcSize)); break; + } + zc->hbSize = ZSTD_frameHeaderSize_min + fcsSize[fcsId]; + } - /* Write Frame Header */ - MEM_writeLE32(zc->headerBuffer, ZSTD_MAGICNUMBER); /* Write Header */ - ((BYTE*)zc->headerBuffer)[4] = (BYTE)(params.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN + ((params.searchLength==3)<<4)); - zc->hbSize = ZSTD_frameHeaderSize_min; zc->stage = 0; - return ZSTD_compress_insertDictionary(zc, dict, dictSize); } size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* zc, const void* dict, size_t dictSize, int compressionLevel) { + ZSTD_parameters params = ZSTD_getParams(compressionLevel, dictSize); + params.srcSize = 0; ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin_usingDict compressionLevel=%d\n", zc->base, compressionLevel); - return ZSTD_compressBegin_advanced(zc, dict, dictSize, ZSTD_getParams(compressionLevel, MAX(128 KB, dictSize))); + return ZSTD_compressBegin_advanced(zc, dict, dictSize, params); } size_t ZSTD_compressBegin(ZSTD_CCtx* zc, int compressionLevel) @@ -2230,7 +2245,7 @@ size_t ZSTD_compressBegin(ZSTD_CCtx* zc, int compressionLevel) /*! ZSTD_compressEnd() : * Write frame epilogue. * @return : nb of bytes written into dst (or an error code) */ -size_t ZSTD_compressEnd(ZSTD_CCtx* zc, void* dst, size_t maxDstSize) +size_t ZSTD_compressEnd(ZSTD_CCtx* zc, void* dst, size_t dstCapacity) { BYTE* op = (BYTE*)dst; size_t hbSize = 0; @@ -2238,15 +2253,15 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* zc, void* dst, size_t maxDstSize) /* empty frame */ if (zc->stage==0) { hbSize = zc->hbSize; - if (maxDstSize <= hbSize) return ERROR(dstSize_tooSmall); + if (dstCapacity <= hbSize) return ERROR(dstSize_tooSmall); zc->stage = 1; memcpy(dst, zc->headerBuffer, hbSize); - maxDstSize -= hbSize; + dstCapacity -= hbSize; op += hbSize; } /* frame epilogue */ - if (maxDstSize < 3) return ERROR(dstSize_tooSmall); + if (dstCapacity < 3) return ERROR(dstSize_tooSmall); op[0] = (BYTE)(bt_end << 6); op[1] = 0; op[2] = 0; @@ -2256,16 +2271,16 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* zc, void* dst, size_t maxDstSize) size_t ZSTD_compress_usingPreparedCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, - void* dst, size_t maxDstSize, + void* dst, size_t dstCapacity, const void* src, size_t srcSize) { size_t outSize; size_t errorCode = ZSTD_copyCCtx(cctx, preparedCCtx); if (ZSTD_isError(errorCode)) return errorCode; - errorCode = ZSTD_compressContinue(cctx, dst, maxDstSize, src, srcSize); + errorCode = ZSTD_compressContinue(cctx, dst, dstCapacity, src, srcSize); if (ZSTD_isError(errorCode)) return errorCode; outSize = errorCode; - errorCode = ZSTD_compressEnd(cctx, (char*)dst+outSize, maxDstSize-outSize); + errorCode = ZSTD_compressEnd(cctx, (char*)dst+outSize, dstCapacity-outSize); if (ZSTD_isError(errorCode)) return errorCode; outSize += errorCode; return outSize; @@ -2273,51 +2288,50 @@ size_t ZSTD_compress_usingPreparedCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* prepare size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, - void* dst, size_t maxDstSize, + void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict,size_t dictSize, ZSTD_parameters params) { BYTE* const ostart = (BYTE*)dst; BYTE* op = ostart; - size_t oSize; /* Init */ - oSize = ZSTD_compressBegin_advanced(ctx, dict, dictSize, params); - if(ZSTD_isError(oSize)) return oSize; + { size_t const errorCode = ZSTD_compressBegin_advanced(ctx, dict, dictSize, params); + if(ZSTD_isError(errorCode)) return errorCode; } /* body (compression) */ - oSize = ZSTD_compressContinue (ctx, op, maxDstSize, src, srcSize); + { size_t const oSize = ZSTD_compressContinue (ctx, op, dstCapacity, src, srcSize); if(ZSTD_isError(oSize)) return oSize; op += oSize; - maxDstSize -= oSize; + dstCapacity -= oSize; } /* Close frame */ - oSize = ZSTD_compressEnd(ctx, op, maxDstSize); + { size_t const oSize = ZSTD_compressEnd(ctx, op, dstCapacity); if(ZSTD_isError(oSize)) return oSize; - op += oSize; + op += oSize; } return (op - ostart); } -size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, const void* dict, size_t dictSize, int compressionLevel) +size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict, size_t dictSize, int compressionLevel) { ZSTD_LOG_BLOCK("%p: ZSTD_compress_usingDict srcSize=%d dictSize=%d compressionLevel=%d\n", ctx->base, (int)srcSize, (int)dictSize, compressionLevel); - return ZSTD_compress_advanced(ctx, dst, maxDstSize, src, srcSize, dict, dictSize, ZSTD_getParams(compressionLevel, srcSize)); + return ZSTD_compress_advanced(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, ZSTD_getParams(compressionLevel, srcSize)); } -size_t ZSTD_compressCCtx (ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel) +size_t ZSTD_compressCCtx (ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel) { ZSTD_LOG_BLOCK("%p: ZSTD_compressCCtx srcSize=%d compressionLevel=%d\n", ctx->base, (int)srcSize, compressionLevel); - return ZSTD_compress_advanced(ctx, dst, maxDstSize, src, srcSize, NULL, 0, ZSTD_getParams(compressionLevel, srcSize)); + return ZSTD_compress_advanced(ctx, dst, dstCapacity, src, srcSize, NULL, 0, ZSTD_getParams(compressionLevel, srcSize)); } -size_t ZSTD_compress(void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel) +size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel) { size_t result; ZSTD_CCtx ctxBody; memset(&ctxBody, 0, sizeof(ctxBody)); - result = ZSTD_compressCCtx(&ctxBody, dst, maxDstSize, src, srcSize, compressionLevel); + result = ZSTD_compressCCtx(&ctxBody, dst, dstCapacity, src, srcSize, compressionLevel); free(ctxBody.workSpace); /* can't free ctxBody, since it's on stack; just free heap content */ return result; } diff --git a/lib/zstd_decompress.c b/lib/zstd_decompress.c index 6e595228f..736e0a275 100644 --- a/lib/zstd_decompress.c +++ b/lib/zstd_decompress.c @@ -307,15 +307,13 @@ static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) * or an error code, which can be tested using ZSTD_isError() */ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize) { - U32 magicNumber, fcsId; const BYTE* ip = (const BYTE*)src; BYTE frameDesc; if (srcSize < ZSTD_frameHeaderSize_min) return ZSTD_frameHeaderSize_min; - magicNumber = MEM_readLE32(src); - if (magicNumber != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown); + if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown); - { size_t fhsize = ZSTD_frameHeaderSize(src, srcSize); + { size_t const fhsize = ZSTD_frameHeaderSize(src, srcSize); if (srcSize < fhsize) return fhsize; } memset(fparamsPtr, 0, sizeof(*fparamsPtr)); @@ -323,13 +321,12 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t fparamsPtr->windowLog = (frameDesc & 0xF) + ZSTD_WINDOWLOG_ABSOLUTEMIN; fparamsPtr->mml = (frameDesc & 0x10) ? MINMATCH-1 : MINMATCH; if ((frameDesc & 0x20) != 0) return ERROR(frameParameter_unsupported); /* reserved 1 bit */ - fcsId = frameDesc >> 6; - switch(fcsId) + switch(frameDesc >> 6) /* fcsId */ { default: /* impossible */ case 0 : fparamsPtr->frameContentSize = 0; break; case 1 : fparamsPtr->frameContentSize = ip[5]; break; - case 2 : fparamsPtr->frameContentSize = MEM_readLE16(ip+5); break; + case 2 : fparamsPtr->frameContentSize = MEM_readLE16(ip+5)+256; break; case 3 : fparamsPtr->frameContentSize = MEM_readLE64(ip+5); break; } return 0; From 1c2c2bcaffb95a9516b907aec35c2ae7a8183b09 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 15 Mar 2016 01:33:36 +0100 Subject: [PATCH 2/3] minor refactoring --- lib/zstd_compress.c | 9 +++++---- lib/zstd_decompress.c | 33 ++++++++++++++++----------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index b1b106b2b..6afcddeb8 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -2205,19 +2205,20 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* zc, /* Write Frame Header into ctx headerBuffer */ MEM_writeLE32(zc->headerBuffer, ZSTD_MAGICNUMBER); { + BYTE* const op = (BYTE*)zc->headerBuffer; U32 const fcsSize[4] = { 0, 1, 2, 8 }; U32 const fcsId = (params.srcSize>0) + (params.srcSize>=256) + (params.srcSize>=65536+256); /* 0-3 */ BYTE fdescriptor = (BYTE)(params.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN); /* windowLog : 4 KB - 128 MB */ fdescriptor |= (BYTE)((params.searchLength==3)<<4); /* mml : 3-4 */ fdescriptor |= (BYTE)(fcsId << 6); - ((BYTE*)zc->headerBuffer)[4] = fdescriptor; + op[4] = fdescriptor; switch(fcsId) { default: /* impossible */ case 0 : break; - case 1 : ((BYTE*)zc->headerBuffer)[5] = (BYTE)(params.srcSize); break; - case 2 : MEM_writeLE16(((BYTE*)zc->headerBuffer)+5, (U16)(params.srcSize-256)); break; - case 3 : MEM_writeLE64(((BYTE*)zc->headerBuffer)+5, (U64)(params.srcSize)); break; + case 1 : op[5] = (BYTE)(params.srcSize); break; + case 2 : MEM_writeLE16(op+5, (U16)(params.srcSize-256)); break; + case 3 : MEM_writeLE64(op+5, (U64)(params.srcSize)); break; } zc->hbSize = ZSTD_frameHeaderSize_min + fcsSize[fcsId]; } diff --git a/lib/zstd_decompress.c b/lib/zstd_decompress.c index 736e0a275..4007541c7 100644 --- a/lib/zstd_decompress.c +++ b/lib/zstd_decompress.c @@ -303,39 +303,38 @@ static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) /** ZSTD_getFrameParams() : * decode Frame Header, or provide expected `srcSize`. * @return : 0, `fparamsPtr` is correctly filled, -* >0, not enough srcSize, provide expected `srcSize`, +* >0, `srcSize` is too small, result is expected `srcSize`, * or an error code, which can be tested using ZSTD_isError() */ -size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize) +size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t const srcSize) { const BYTE* ip = (const BYTE*)src; - BYTE frameDesc; if (srcSize < ZSTD_frameHeaderSize_min) return ZSTD_frameHeaderSize_min; if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown); + /* ensure there is enough `srcSize` to fully read/decode frame header */ { size_t const fhsize = ZSTD_frameHeaderSize(src, srcSize); if (srcSize < fhsize) return fhsize; } memset(fparamsPtr, 0, sizeof(*fparamsPtr)); - frameDesc = ip[4]; - fparamsPtr->windowLog = (frameDesc & 0xF) + ZSTD_WINDOWLOG_ABSOLUTEMIN; - fparamsPtr->mml = (frameDesc & 0x10) ? MINMATCH-1 : MINMATCH; - if ((frameDesc & 0x20) != 0) return ERROR(frameParameter_unsupported); /* reserved 1 bit */ - switch(frameDesc >> 6) /* fcsId */ - { - default: /* impossible */ - case 0 : fparamsPtr->frameContentSize = 0; break; - case 1 : fparamsPtr->frameContentSize = ip[5]; break; - case 2 : fparamsPtr->frameContentSize = MEM_readLE16(ip+5)+256; break; - case 3 : fparamsPtr->frameContentSize = MEM_readLE64(ip+5); break; - } + { BYTE const frameDesc = ip[4]; + fparamsPtr->windowLog = (frameDesc & 0xF) + ZSTD_WINDOWLOG_ABSOLUTEMIN; + fparamsPtr->mml = (frameDesc & 0x10) ? MINMATCH-1 : MINMATCH; + if ((frameDesc & 0x20) != 0) return ERROR(frameParameter_unsupported); /* reserved 1 bit */ + switch(frameDesc >> 6) /* fcsId */ + { + default: /* impossible */ + case 0 : fparamsPtr->frameContentSize = 0; break; + case 1 : fparamsPtr->frameContentSize = ip[5]; break; + case 2 : fparamsPtr->frameContentSize = MEM_readLE16(ip+5)+256; break; + case 3 : fparamsPtr->frameContentSize = MEM_readLE64(ip+5); break; + } } return 0; } /** ZSTD_decodeFrameHeader() : -* decode Frame Header. -* srcSize must be the size provided by ZSTD_decodeFrameHeader_Part1(). +* `srcSize` must be the size provided by ZSTD_frameHeaderSize(). * @return : 0, or an error code, which can be tested using ZSTD_isError() */ static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* zc, const void* src, size_t srcSize) { From 09b21ee2f67b4fd1dd2bfccb538d38f7811b8ad6 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 15 Mar 2016 12:56:03 +0100 Subject: [PATCH 3/3] fix fileio bug with new zbuff simplified zbuff decoding --- lib/zbuff.c | 115 ++++++++++++++---------------------------- lib/zstd_decompress.c | 2 +- programs/fileio.c | 37 +++++++++----- 3 files changed, 62 insertions(+), 92 deletions(-) diff --git a/lib/zbuff.c b/lib/zbuff.c index 69341df8e..a08060cf4 100644 --- a/lib/zbuff.c +++ b/lib/zbuff.c @@ -40,7 +40,7 @@ ***************************************/ #include #include "error_private.h" -#include "zstd_static.h" +#include "zstd_internal.h" #include "zbuff_static.h" @@ -50,7 +50,7 @@ static size_t ZBUFF_blockHeaderSize = 3; static size_t ZBUFF_endFrameSize = 3; -/** ************************************************ +/*_************************************************** * Streaming compression * * A ZBUFF_CCtx object is required to track streaming operation. @@ -88,13 +88,13 @@ typedef enum { ZBUFFcs_init, ZBUFFcs_load, ZBUFFcs_flush } ZBUFF_cStage; /* *** Ressources *** */ struct ZBUFF_CCtx_s { ZSTD_CCtx* zc; - char* inBuff; + char* inBuff; size_t inBuffSize; size_t inToCompress; size_t inBuffPos; size_t inBuffTarget; size_t blockSize; - char* outBuff; + char* outBuff; size_t outBuffSize; size_t outBuffContentSize; size_t outBuffFlushedSize; @@ -123,8 +123,6 @@ size_t ZBUFF_freeCCtx(ZBUFF_CCtx* zbc) /* *** Initialization *** */ -#define MIN(a,b) ( ((a)<(b)) ? (a) : (b) ) -#define BLOCKSIZE (128 * 1024) /* a bit too "magic", should come from reference */ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, ZSTD_parameters params) { size_t neededInBuffSize; @@ -172,9 +170,9 @@ ZSTDLIB_API size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* zbc, const void* dic /* *** Compression *** */ -static size_t ZBUFF_limitCopy(void* dst, size_t maxDstSize, const void* src, size_t srcSize) +static size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) { - size_t length = MIN(maxDstSize, srcSize); + size_t length = MIN(dstCapacity, srcSize); memcpy(dst, src, length); return length; } @@ -322,12 +320,10 @@ size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* maxDstSizePtr) * just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . * *******************************************************************************/ -typedef enum { ZBUFFds_init, ZBUFFds_readHeader, ZBUFFds_loadHeader, ZBUFFds_decodeHeader, +typedef enum { ZBUFFds_init, ZBUFFds_readHeader, ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFF_dStage; /* *** Resource management *** */ - -#define ZSTD_frameHeaderSize_max 5 /* too magical, should come from reference */ struct ZBUFF_DCtx_s { ZSTD_DCtx* zc; ZSTD_frameParams fParams; @@ -338,9 +334,7 @@ struct ZBUFF_DCtx_s { size_t outBuffSize; size_t outStart; size_t outEnd; - size_t hPos; ZBUFF_dStage stage; - unsigned char headerBuffer[ZSTD_frameHeaderSize_max]; }; /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */ @@ -370,7 +364,7 @@ size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbc) size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* zbc, const void* dict, size_t dictSize) { zbc->stage = ZBUFFds_readHeader; - zbc->hPos = zbc->inPos = zbc->outStart = zbc->outEnd = 0; + zbc->inPos = zbc->outStart = zbc->outEnd = 0; return ZSTD_decompressBegin_usingDict(zbc->zc, dict, dictSize); } @@ -402,69 +396,36 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, case ZBUFFds_readHeader : /* read header from src */ - { - size_t headerSize = ZSTD_getFrameParams(&(zbc->fParams), src, *srcSizePtr); + { size_t const headerSize = ZSTD_getFrameParams(&(zbc->fParams), src, *srcSizePtr); if (ZSTD_isError(headerSize)) return headerSize; if (headerSize) { - /* not enough input to decode header : tell how many bytes would be necessary */ - memcpy(zbc->headerBuffer+zbc->hPos, src, *srcSizePtr); - zbc->hPos += *srcSizePtr; + /* not enough input to decode header : needs headerSize > *srcSizePtr */ *dstCapacityPtr = 0; - zbc->stage = ZBUFFds_loadHeader; - return headerSize - zbc->hPos; - } - zbc->stage = ZBUFFds_decodeHeader; - break; - } + *srcSizePtr = 0; + return headerSize; + } } - case ZBUFFds_loadHeader: - /* complete header from src */ + /* Frame header provides buffer sizes */ + { size_t const neededInSize = BLOCKSIZE; /* a block is never > BLOCKSIZE */ + if (zbc->inBuffSize < neededInSize) { + free(zbc->inBuff); + zbc->inBuffSize = neededInSize; + zbc->inBuff = (char*)malloc(neededInSize); + if (zbc->inBuff == NULL) return ERROR(memory_allocation); + } } { - size_t headerSize = ZBUFF_limitCopy( - zbc->headerBuffer + zbc->hPos, ZSTD_frameHeaderSize_max - zbc->hPos, - src, *srcSizePtr); - zbc->hPos += headerSize; - ip += headerSize; - headerSize = ZSTD_getFrameParams(&(zbc->fParams), zbc->headerBuffer, zbc->hPos); - if (ZSTD_isError(headerSize)) return headerSize; - if (headerSize) { - /* not enough input to decode header : tell how many bytes would be necessary */ - *dstCapacityPtr = 0; - return headerSize - zbc->hPos; - } - // zbc->stage = ZBUFFds_decodeHeader; break; /* useless : stage follows */ - } - - case ZBUFFds_decodeHeader: - /* apply header to create / resize buffers */ - { - size_t neededOutSize = (size_t)1 << zbc->fParams.windowLog; - size_t neededInSize = BLOCKSIZE; /* a block is never > BLOCKSIZE */ - if (zbc->inBuffSize < neededInSize) { - free(zbc->inBuff); - zbc->inBuffSize = neededInSize; - zbc->inBuff = (char*)malloc(neededInSize); - if (zbc->inBuff == NULL) return ERROR(memory_allocation); - } - if (zbc->outBuffSize < neededOutSize) { - free(zbc->outBuff); - zbc->outBuffSize = neededOutSize; - zbc->outBuff = (char*)malloc(neededOutSize); - if (zbc->outBuff == NULL) return ERROR(memory_allocation); - } } - if (zbc->hPos) { - /* some data already loaded into headerBuffer : transfer into inBuff */ - memcpy(zbc->inBuff, zbc->headerBuffer, zbc->hPos); - zbc->inPos = zbc->hPos; - zbc->hPos = 0; - zbc->stage = ZBUFFds_load; - break; - } - zbc->stage = ZBUFFds_read; + size_t const neededOutSize = (size_t)1 << zbc->fParams.windowLog; + if (zbc->outBuffSize < neededOutSize) { + free(zbc->outBuff); + zbc->outBuffSize = neededOutSize; + zbc->outBuff = (char*)malloc(neededOutSize); + if (zbc->outBuff == NULL) return ERROR(memory_allocation); + } } + zbc->stage = ZBUFFds_read; case ZBUFFds_read: { - size_t neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc); + size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc); if (neededInSize==0) { /* end of frame */ zbc->stage = ZBUFFds_init; notDone = 0; @@ -472,7 +433,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, } if ((size_t)(iend-ip) >= neededInSize) { /* directly decode from src */ - size_t decodedSize = ZSTD_decompressContinue(zbc->zc, + size_t const decodedSize = ZSTD_decompressContinue(zbc->zc, zbc->outBuff + zbc->outStart, zbc->outBuffSize - zbc->outStart, ip, neededInSize); if (ZSTD_isError(decodedSize)) return decodedSize; @@ -488,8 +449,8 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, case ZBUFFds_load: { - size_t neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc); - size_t toLoad = neededInSize - zbc->inPos; /* should always be <= remaining space within inBuff */ + size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc); + size_t const toLoad = neededInSize - zbc->inPos; /* should always be <= remaining space within inBuff */ size_t loadedSize; if (toLoad > zbc->inBuffSize - zbc->inPos) return ERROR(corruption_detected); /* should never happen */ loadedSize = ZBUFF_limitCopy(zbc->inBuff + zbc->inPos, toLoad, ip, iend-ip); @@ -497,7 +458,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, zbc->inPos += loadedSize; if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */ { - size_t decodedSize = ZSTD_decompressContinue(zbc->zc, + size_t const decodedSize = ZSTD_decompressContinue(zbc->zc, zbc->outBuff + zbc->outStart, zbc->outBuffSize - zbc->outStart, zbc->inBuff, neededInSize); if (ZSTD_isError(decodedSize)) return decodedSize; @@ -509,8 +470,8 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, } } case ZBUFFds_flush: { - size_t toFlushSize = zbc->outEnd - zbc->outStart; - size_t flushedSize = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outStart, toFlushSize); + size_t const toFlushSize = zbc->outEnd - zbc->outStart; + size_t const flushedSize = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outStart, toFlushSize); op += flushedSize; zbc->outStart += flushedSize; if (flushedSize == toFlushSize) { @@ -526,12 +487,12 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, default: return ERROR(GENERIC); /* impossible */ } } + /* result */ *srcSizePtr = ip-istart; *dstCapacityPtr = op-ostart; - { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zbc->zc); - if (nextSrcSizeHint > ZBUFF_blockHeaderSize) nextSrcSizeHint+= ZBUFF_blockHeaderSize; /* get next block header too */ + if (nextSrcSizeHint > ZBUFF_blockHeaderSize) nextSrcSizeHint+= ZBUFF_blockHeaderSize; /* get following block header too */ nextSrcSizeHint -= zbc->inPos; /* already loaded*/ return nextSrcSizeHint; } diff --git a/lib/zstd_decompress.c b/lib/zstd_decompress.c index 4007541c7..685e6de44 100644 --- a/lib/zstd_decompress.c +++ b/lib/zstd_decompress.c @@ -305,7 +305,7 @@ static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) * @return : 0, `fparamsPtr` is correctly filled, * >0, `srcSize` is too small, result is expected `srcSize`, * or an error code, which can be tested using ZSTD_isError() */ -size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t const srcSize) +size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize) { const BYTE* ip = (const BYTE*)src; diff --git a/programs/fileio.c b/programs/fileio.c index c0ec891e3..907d990a1 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -64,7 +64,7 @@ #include /* stat64 */ #include "mem.h" #include "fileio.h" -#include "zstd_static.h" /* ZSTD_magicNumber */ +#include "zstd_static.h" /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */ #include "zbuff_static.h" #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) @@ -242,7 +242,7 @@ static FILE* FIO_openDstFile(const char* dstFileName) /*! FIO_loadFile() : -* creates a buffer, pointed by *bufferPtr, +* creates a buffer, pointed by `*bufferPtr`, * loads `filename` content into it, * up to MAX_DICT_SIZE bytes */ @@ -342,6 +342,7 @@ static int FIO_compressFilename_internal(cRess_t ress, /* init */ filesize = MAX(FIO_getFileSize(srcFileName),dictSize); params = ZSTD_getParams(cLevel, filesize); + params.srcSize = filesize; if (g_maxWLog) if (params.windowLog > g_maxWLog) params.windowLog = g_maxWLog; errorCode = ZBUFF_compressInit_advanced(ress.ctx, ress.dictBuffer, ress.dictBufferSize, params); if (ZBUFF_isError(errorCode)) EXM_THROW(21, "Error initializing compression : %s", ZBUFF_getErrorName(errorCode)); @@ -394,7 +395,7 @@ static int FIO_compressFilename_internal(cRess_t ress, /*! FIO_compressFilename_internal() : - * same as FIO_compressFilename_extRess(), with ress.desFile already opened + * same as FIO_compressFilename_extRess(), with ress.destFile already opened (typically stdout) * @return : 0 : compression completed correctly, * 1 : missing or pb opening srcFileName */ @@ -432,10 +433,7 @@ static int FIO_compressFilename_extRess(cRess_t ress, if (ress.dstFile==0) { fclose(ress.srcFile); return 1; } result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, cLevel); - - if (result != 0) { - remove(dstFileName); - } + if (result!=0) remove(dstFileName); /* remove operation artefact */ fclose(ress.srcFile); /* no pb to expect : only reading */ if (fclose(ress.dstFile)) EXM_THROW(28, "Write error : cannot properly close %s", dstFileName); @@ -556,35 +554,46 @@ static void FIO_freeDResources(dRess_t ress) } +/** FIO_decompressFrame() : + @return : size of decoded frame +*/ unsigned long long FIO_decompressFrame(dRess_t ress, FILE* foutput, FILE* finput, size_t alreadyLoaded) { U64 frameSize = 0; - size_t readSize=alreadyLoaded; + size_t readSize; + + ZBUFF_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize); + + /* Complete Header loading */ + { size_t const toLoad = ZSTD_frameHeaderSize_max - alreadyLoaded; /* assumption : alreadyLoaded <= ZSTD_frameHeaderSize_max */ + size_t const checkSize = fread(((char*)ress.srcBuffer) + alreadyLoaded, 1, toLoad, finput); + if (checkSize != toLoad) EXM_THROW(32, "Read error"); + } + readSize = ZSTD_frameHeaderSize_max; /* Main decompression Loop */ - ZBUFF_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize); while (1) { /* Decode */ - size_t sizeCheck; size_t inSize=readSize, decodedSize=ress.dstBufferSize; size_t toRead = ZBUFF_decompressContinue(ress.dctx, ress.dstBuffer, &decodedSize, ress.srcBuffer, &inSize); if (ZBUFF_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFF_getErrorName(toRead)); readSize -= inSize; /* Write block */ - sizeCheck = fwrite(ress.dstBuffer, 1, decodedSize, foutput); - if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block to destination file"); + { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, decodedSize, foutput); + if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block into destination"); } frameSize += decodedSize; DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(frameSize>>20) ); - if (toRead == 0) break; + if (toRead == 0) break; /* end of frame */ if (readSize) EXM_THROW(38, "Decoding error : should consume entire input"); /* Fill input buffer */ if (toRead > ress.srcBufferSize) EXM_THROW(34, "too large block"); readSize = fread(ress.srcBuffer, 1, toRead, finput); - if (readSize != toRead) EXM_THROW(35, "Read error"); + if (readSize != toRead) + EXM_THROW(35, "Read error"); } return frameSize;