ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

zbuff supports byte-by-byte decompression scenarios

Browse Source
This commit is contained in:
Yann Collet
2016-05-10 14:14:19 +02:00
parent 918b934a26
commit bd39d54576
4 changed files with 81 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/decompress/zbuff_decompress.c
+32 -16
View File
@@ -64,14 +64,14 @@
* just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
* *******************************************************************************/
typedef enum { ZBUFFds_init, ZBUFFds_readHeader,
typedef enum { ZBUFFds_init, ZBUFFds_loadHeader,
ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFF_dStage;
/* *** Resource management *** */
struct ZBUFF_DCtx_s {
ZSTD_DCtx* zd;
ZSTD_frameParams fParams;
size_t blockSize;
ZBUFF_dStage stage;
char* inBuff;
size_t inBuffSize;
size_t inPos;
@@ -79,7 +79,9 @@ struct ZBUFF_DCtx_s {
size_t outBuffSize;
size_t outStart;
size_t outEnd;
ZBUFF_dStage stage;
size_t blockSize;
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
size_t lhSize;
}; /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */
@@ -108,8 +110,8 @@ size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbd)
size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* zbd, const void* dict, size_t dictSize)
{
zbd->stage = ZBUFFds_readHeader;
zbd->inPos = zbd->outStart = zbd->outEnd = 0;
zbd->stage = ZBUFFds_loadHeader;
zbd->lhSize = zbd->inPos = zbd->outStart = zbd->outEnd = 0;
return ZSTD_decompressBegin_usingDict(zbd->zd, dict, dictSize);
}
@@ -139,15 +141,29 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
case ZBUFFds_init :
return ERROR(init_missing);
case ZBUFFds_readHeader :
/* read header from src */
{ size_t const headerSize = ZSTD_getFrameParams(&(zbd->fParams), src, *srcSizePtr);
if (ZSTD_isError(headerSize)) return headerSize;
if (headerSize) {
/* not enough input to decode header : needs headerSize > *srcSizePtr */
*dstCapacityPtr = 0;
*srcSizePtr = 0;
return headerSize;
case ZBUFFds_loadHeader :
{ size_t const hSize = ZSTD_getFrameParams(&(zbd->fParams), zbd->headerBuffer, zbd->lhSize);
if (hSize != 0) {
size_t const toLoad = hSize - zbd->lhSize; /* if hSize!=0, hSize > zbd->lhSize */
if (ZSTD_isError(hSize)) return hSize;
if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */
memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip);
zbd->lhSize += iend-ip; ip = iend; notDone = 0;
*dstCapacityPtr = 0;
return (hSize - zbd->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */
}
memcpy(zbd->headerBuffer + zbd->lhSize, ip, toLoad); zbd->lhSize = hSize; ip += toLoad;
break;
} }
/* Consume header */
{ size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zbd->zd); /* == ZSTD_frameHeaderSize_min */
size_t const h1Result = ZSTD_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer, h1Size);
if (ZSTD_isError(h1Result)) return h1Result;
if (h1Size < zbd->lhSize) { /* long header */
size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zbd->zd);
size_t const h2Result = ZSTD_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer+h1Size, h2Size);
if (ZSTD_isError(h2Result)) return h2Result;
} }
/* Frame header instruct buffer sizes */
@@ -175,8 +191,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
notDone = 0;
break;
}
if ((size_t)(iend-ip) >= neededInSize) {
/* directly decode from src */
if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */
size_t const decodedSize = ZSTD_decompressContinue(zbd->zd,
zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart,
ip, neededInSize);
@@ -200,6 +215,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
ip += loadedSize;
zbd->inPos += loadedSize;
if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */
/* decode loaded input */
{ size_t const decodedSize = ZSTD_decompressContinue(zbd->zd,
zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart,
lib/decompress/zstd_decompress.c
+19 -23
View File
@@ -657,7 +657,8 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
}
FORCE_INLINE size_t ZSTD_execSequence(BYTE* op,
//FORCE_INLINE
size_t ZSTD_execSequence(BYTE* op,
BYTE* const oend, seq_t sequence,
const BYTE** litPtr, const BYTE* const litLimit_8,
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
@@ -964,31 +965,29 @@ size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx)
return dctx->expected;
}
size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{
/* Sanity check */
if (srcSize != dctx->expected) return ERROR(srcSize_wrong);
ZSTD_checkContinuity(dctx, dst);
if (dstCapacity) ZSTD_checkContinuity(dctx, dst);
/* Decompress : frame header; part 1 */
switch (dctx->stage)
{
case ZSTDds_getFrameHeaderSize :
{
if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */
dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min);
if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize;
memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min);
if (dctx->headerSize > ZSTD_frameHeaderSize_min) {
dctx->expected = dctx->headerSize - ZSTD_frameHeaderSize_min;
dctx->stage = ZSTDds_decodeFrameHeader;
return 0;
}
dctx->expected = 0; /* not necessary to copy more */
if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */
dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min);
if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize;
memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min);
if (dctx->headerSize > ZSTD_frameHeaderSize_min) {
dctx->expected = dctx->headerSize - ZSTD_frameHeaderSize_min;
dctx->stage = ZSTDds_decodeFrameHeader;
return 0;
}
dctx->expected = 0; /* not necessary to copy more */
case ZSTDds_decodeFrameHeader:
{
size_t result;
{ size_t result;
memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_min, src, dctx->expected);
result = ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize);
if (ZSTD_isError(result)) return result;
@@ -997,8 +996,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co
return 0;
}
case ZSTDds_decodeBlockHeader:
{
blockProperties_t bp;
{ blockProperties_t bp;
size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
if (ZSTD_isError(cBlockSize)) return cBlockSize;
if (bp.blockType == bt_end) {
@@ -1012,16 +1010,14 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co
return 0;
}
case ZSTDds_decompressBlock:
{
/* Decompress : block content */
size_t rSize;
{ size_t rSize;
switch(dctx->bType)
{
case bt_compressed:
rSize = ZSTD_decompressBlock_internal(dctx, dst, maxDstSize, src, srcSize);
rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize);
break;
case bt_raw :
rSize = ZSTD_copyRawBlock(dst, maxDstSize, src, srcSize);
rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize);
break;
case bt_rle :
return ERROR(GENERIC); /* not yet handled */