ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

new frame end, 32-bits checksums

Browse Source
This commit is contained in:
Yann Collet
2016-07-28 00:55:43 +02:00
parent d4180cad9c
commit c991cc1828
6 changed files with 160 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/common/zstd_internal.h
+1 -1
View File
@@ -88,7 +88,7 @@ static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 };
#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */
static const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE;
typedef enum { bt_raw, bt_rle, bt_compressed, bt_end } blockType_e;
typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e;
#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */
#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */) /* for a non-null block */
lib/compress/zbuff_compress.c
+4 -2
View File
@@ -95,6 +95,7 @@ struct ZBUFF_CCtx_s {
size_t outBuffContentSize;
size_t outBuffFlushedSize;
ZBUFF_cStage stage;
U32 checksum;
ZSTD_customMem customMem;
}; /* typedef'd tp ZBUFF_CCtx within "zstd_buffered.h" */
@@ -164,6 +165,7 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
zbc->inBuffTarget = zbc->blockSize;
zbc->outBuffContentSize = zbc->outBuffFlushedSize = 0;
zbc->stage = ZBUFFcs_load;
zbc->checksum = params.fParams.checksumFlag > 0;
return 0; /* ready to go */
}
@@ -300,11 +302,11 @@ size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr)
op += outSize;
if (remainingToFlush) {
*dstCapacityPtr = op-ostart;
return remainingToFlush + ZBUFF_endFrameSize;
return remainingToFlush + ZBUFF_endFrameSize + (zbc->checksum * 4);
}
/* create epilogue */
zbc->stage = ZBUFFcs_final;
zbc->outBuffContentSize = ZSTD_compressEnd(zbc->zc, zbc->outBuff, zbc->outBuffSize); /* epilogue into outBuff */
zbc->outBuffContentSize = ZSTD_compressEnd(zbc->zc, zbc->outBuff, zbc->outBuffSize); /* epilogue into outBuff */
}
/* flush epilogue */
lib/compress/zstd_compress.c
+42 -20
View File
@@ -2227,9 +2227,17 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCa
}
/*! ZSTD_compress_generic() :
* Compress a chunk of data into one or multiple blocks.
* All blocks will be terminated, all input will be consumed.
* Function will issue an error if there is not enough `dstCapacity` to hold the compressed content.
* Frame is supposed already started (header already produced)
* @return : compressed size, or an error code
*/
static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
const void* src, size_t srcSize,
U32 lastFrameChunk)
{
size_t blockSize = cctx->blockSize;
size_t remaining = srcSize;
@@ -2244,6 +2252,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
XXH64_update(&cctx->xxhState, src, srcSize);
while (remaining) {
U32 const lastBlock = lastFrameChunk & (blockSize >= remaining);
size_t cSize;
ZSTD_statsResetFreqs(stats); /* debug only */
@@ -2261,12 +2270,15 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
if (ZSTD_isError(cSize)) return cSize;
if (cSize == 0) { /* block is not compressible */
cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize);
if (ZSTD_isError(cSize)) return cSize;
U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw)<<1) + (U32)(blockSize << 3);
if (blockSize + ZSTD_blockHeaderSize > dstCapacity) return ERROR(dstSize_tooSmall);
MEM_writeLE32(op, cBlockHeader24); /* no pb, 4th byte will be overwritten */
memcpy(op + ZSTD_blockHeaderSize, ip, blockSize);
cSize = ZSTD_blockHeaderSize+blockSize;
} else {
U32 const cBlockHeader24 = (U32)bt_compressed + (U32)(cSize << 2);
U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3);
MEM_writeLE24(op, cBlockHeader24);
cSize += 3;
cSize += ZSTD_blockHeaderSize;
}
remaining -= blockSize;
@@ -2275,6 +2287,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
op += cSize;
}
if (lastFrameChunk) cctx->stage = ZSTDcs_ending;
ZSTD_statsPrint(stats, cctx->params.cParams.searchLength); /* debug only */
return op-ostart;
}
@@ -2322,7 +2335,7 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* zc,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
U32 frame)
U32 frame, U32 lastFrameChunk)
{
const BYTE* const ip = (const BYTE*) src;
size_t fhSize = 0;
@@ -2372,7 +2385,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* zc,
zc->nextSrc = ip + srcSize;
{ size_t const cSize = frame ?
ZSTD_compress_generic (zc, dst, dstCapacity, src, srcSize) :
ZSTD_compress_generic (zc, dst, dstCapacity, src, srcSize, lastFrameChunk) :
ZSTD_compressBlock_internal (zc, dst, dstCapacity, src, srcSize);
if (ZSTD_isError(cSize)) return cSize;
return cSize + fhSize;
@@ -2384,7 +2397,7 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* zc,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
{
return ZSTD_compressContinue_internal(zc, dst, dstCapacity, src, srcSize, 1);
return ZSTD_compressContinue_internal(zc, dst, dstCapacity, src, srcSize, 1, 0);
}
@@ -2398,7 +2411,7 @@ size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const
size_t const blockSizeMax = ZSTD_getBlockSizeMax(cctx);
if (srcSize > blockSizeMax) return ERROR(srcSize_wrong);
ZSTD_LOG_BLOCK("%p: ZSTD_compressBlock searchLength=%d\n", cctx->base, cctx->params.cParams.searchLength);
return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0);
return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0, 0);
}
@@ -2572,13 +2585,14 @@ size_t ZSTD_compressBegin(ZSTD_CCtx* zc, int compressionLevel)
* @return : nb of bytes written into dst (or an error code) */
size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity)
{
BYTE* op = (BYTE*)dst;
BYTE* const ostart = (BYTE*)dst;
BYTE* op = ostart;
size_t fhSize = 0;
if (cctx->stage==ZSTDcs_created) return ERROR(stage_wrong); /*< not even init ! */
if (cctx->stage == ZSTDcs_created) return ERROR(stage_wrong); /*< not even init ! */
/* special case : empty frame */
if (cctx->stage==ZSTDcs_init) {
if (cctx->stage == ZSTDcs_init) {
fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->params, 0, 0);
if (ZSTD_isError(fhSize)) return fhSize;
dstCapacity -= fhSize;
@@ -2586,16 +2600,24 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity)
cctx->stage = ZSTDcs_ongoing;
}
/* frame epilogue */
if (dstCapacity < ZSTD_blockHeaderSize) return ERROR(dstSize_tooSmall);
{ U32 const checksum = cctx->params.fParams.checksumFlag ?
(U32)(XXH64_digest(&cctx->xxhState) >> 11) :
0;
MEM_writeLE24(op, (U32)bt_end + (checksum << 2));
if (cctx->stage != ZSTDcs_ending) {
/* write one last empty block, make it the "last" block */
U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw)<<1) + 0;
if (dstCapacity<4) return ERROR(dstSize_tooSmall);
MEM_writeLE32(op, cBlockHeader24);
op += ZSTD_blockHeaderSize;
dstCapacity -= ZSTD_blockHeaderSize;
}
if (cctx->params.fParams.checksumFlag) {
U32 const checksum = (U32) XXH64_digest(&cctx->xxhState);
if (dstCapacity<4) return ERROR(dstSize_tooSmall);
MEM_writeLE32(op, checksum);
op += 4;
}
cctx->stage = ZSTDcs_created; /* return to "created but no init" status */
return ZSTD_blockHeaderSize+fhSize;
return op-ostart;
}
@@ -2635,7 +2657,7 @@ static size_t ZSTD_compress_internal (ZSTD_CCtx* ctx,
if(ZSTD_isError(errorCode)) return errorCode; }
/* body (compression) */
{ size_t const oSize = ZSTD_compressContinue (ctx, op, dstCapacity, src, srcSize);
{ size_t const oSize = ZSTD_compressContinue_internal(ctx, op, dstCapacity, src, srcSize, 1, 1);
if(ZSTD_isError(oSize)) return oSize;
op += oSize;
dstCapacity -= oSize; }
lib/decompress/zstd_decompress.c
+69 -34
View File
@@ -105,6 +105,7 @@ static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); }
***************************************************************/
typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader,
ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock,
ZSTDds_decompressLastBlock, ZSTDds_checkChecksum,
ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage;
struct ZSTD_DCtx_s
@@ -131,6 +132,7 @@ struct ZSTD_DCtx_s
ZSTD_customMem customMem;
size_t litBufSize;
size_t litSize;
size_t rleSize;
BYTE litBuffer[ZSTD_BLOCKSIZE_ABSOLUTEMAX + WILDCOPY_OVERLENGTH];
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
}; /* typedef'd to ZSTD_DCtx within "zstd_static.h" */
@@ -318,6 +320,7 @@ static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t sr
typedef struct
{
blockType_e blockType;
U32 lastBlock;
U32 origSize;
} blockProperties_t;
@@ -327,11 +330,12 @@ size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bp
{
if (srcSize < ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
{ U32 const cBlockHeader = MEM_readLE24(src);
U32 const cSize = cBlockHeader >> 2;
bpPtr->blockType = (blockType_e)(cBlockHeader & 3);
U32 const cSize = cBlockHeader >> 3;
bpPtr->lastBlock = cBlockHeader & 1;
bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3);
bpPtr->origSize = cSize; /* only useful for RLE */
if (bpPtr->blockType == bt_end) return 0;
if (bpPtr->blockType == bt_rle) return 1;
if (bpPtr->blockType == bt_reserved) return ERROR(corruption_detected);
return cSize;
}
}
@@ -345,6 +349,14 @@ static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity, const void* src,
}
static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize, size_t regenSize)
{
if (srcSize != 1) return ERROR(srcSize_wrong);
if (regenSize > dstCapacity) return ERROR(dstSize_tooSmall);
memset(dst, *(const BYTE*)src, regenSize);
return regenSize;
}
/*! ZSTD_decodeLiteralsBlock() :
@return : nb of bytes read from src (< srcSize ) */
size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
@@ -889,29 +901,29 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
case bt_rle :
decodedSize = ZSTD_generateNxBytes(op, oend-op, *ip, blockProperties.origSize);
break;
case bt_end :
/* end of frame */
if (remainingSize) return ERROR(srcSize_wrong);
if (dctx->fParams.checksumFlag) {
U64 const h64 = XXH64_digest(&dctx->xxhState);
U32 const h32 = (U32)(h64>>11) & ((1<<22)-1);
U32 const check32 = MEM_readLE24(src) >> 2;
if (check32 != h32) return ERROR(checksum_wrong);
}
decodedSize = 0;
break;
case bt_reserved :
default:
return ERROR(GENERIC); /* impossible */
return ERROR(corruption_detected);
}
if (blockProperties.blockType == bt_end) break; /* bt_end */
if (ZSTD_isError(decodedSize)) return decodedSize;
if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, op, decodedSize);
op += decodedSize;
ip += cBlockSize;
remainingSize -= cBlockSize;
if (blockProperties.lastBlock) break;
}
if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */
U32 const checkCalc = (U32)XXH64_digest(&dctx->xxhState);
U32 checkRead;
if (remainingSize<4) return ERROR(checksum_wrong);
checkRead = MEM_readLE32(ip);
if (checkRead != checkCalc) return ERROR(checksum_wrong);
remainingSize -= 4;
}
if (remainingSize) return ERROR(srcSize_wrong);
return op-ostart;
}
@@ -1022,22 +1034,29 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
{ blockProperties_t bp;
size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
if (ZSTD_isError(cBlockSize)) return cBlockSize;
if (bp.blockType == bt_end) {
dctx->expected = cBlockSize;
dctx->bType = bp.blockType;
dctx->rleSize = bp.origSize;
if (cBlockSize) {
dctx->stage = bp.lastBlock ? ZSTDds_decompressLastBlock : ZSTDds_decompressBlock;
return 0;
}
/* empty block */
if (bp.lastBlock) {
if (dctx->fParams.checksumFlag) {
U64 const h64 = XXH64_digest(&dctx->xxhState);
U32 const h32 = (U32)(h64>>11) & ((1<<22)-1);
U32 const check32 = MEM_readLE24(src) >> 2;
if (check32 != h32) return ERROR(checksum_wrong);
dctx->expected = 4;
dctx->stage = ZSTDds_checkChecksum;
} else {
dctx->expected = 0; /* end of frame */
dctx->stage = ZSTDds_getFrameHeaderSize;
}
dctx->expected = 0;
dctx->stage = ZSTDds_getFrameHeaderSize;
} else {
dctx->expected = cBlockSize;
dctx->bType = bp.blockType;
dctx->stage = ZSTDds_decompressBlock;
dctx->expected = 3; /* go directly to next header */
dctx->stage = ZSTDds_decodeBlockHeader;
}
return 0;
}
case ZSTDds_decompressLastBlock:
case ZSTDds_decompressBlock:
{ size_t rSize;
switch(dctx->bType)
@@ -1049,21 +1068,37 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize);
break;
case bt_rle :
return ERROR(GENERIC); /* not yet handled */
break;
case bt_end : /* should never happen (filtered at phase 1) */
rSize = 0;
rSize = ZSTD_setRleBlock(dst, dstCapacity, src, srcSize, dctx->rleSize);
break;
case bt_reserved : /* should never happen */
default:
return ERROR(GENERIC); /* impossible */
return ERROR(corruption_detected);
}
dctx->stage = ZSTDds_decodeBlockHeader;
dctx->expected = ZSTD_blockHeaderSize;
dctx->previousDstEnd = (char*)dst + rSize;
if (ZSTD_isError(rSize)) return rSize;
if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, dst, rSize);
if (dctx->stage == ZSTDds_decompressLastBlock) { /* end of frame */
if (dctx->fParams.checksumFlag) { /* another round for frame checksum */
dctx->expected = 0;
dctx->stage = ZSTDds_checkChecksum;
}
dctx->expected = 0; /* ends here */
dctx->stage = ZSTDds_getFrameHeaderSize;
} else {
dctx->stage = ZSTDds_decodeBlockHeader;
dctx->expected = ZSTD_blockHeaderSize;
dctx->previousDstEnd = (char*)dst + rSize;
}
return rSize;
}
case ZSTDds_checkChecksum:
{ U32 const h32 = (U32)XXH64_digest(&dctx->xxhState);
U32 const check32 = MEM_readLE32(src); /* srcSize == 4, guaranteed by dctx->expected */
if (check32 != h32) return ERROR(checksum_wrong);
dctx->expected = 0;
dctx->stage = ZSTDds_getFrameHeaderSize;
return 0;
}
case ZSTDds_decodeSkippableHeader:
{ memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_min, src, dctx->expected);
dctx->expected = MEM_readLE32(dctx->headerBuffer + 4);