ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #151 from Cyan4973/framesize

Browse Source 
Framesize
This commit is contained in:
Yann Collet
2016-03-15 14:31:31 +01:00
parent 5b31fa6f16 09b21ee2f6
commit 135f319bb2
4 changed files with 160 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/zbuff.c
+38 -77
View File
@@ -40,7 +40,7 @@
***************************************/
#include <stdlib.h>
#include "error_private.h"
#include "zstd_static.h"
#include "zstd_internal.h"
#include "zbuff_static.h"
@@ -50,7 +50,7 @@
static size_t ZBUFF_blockHeaderSize = 3;
static size_t ZBUFF_endFrameSize = 3;
/** ************************************************
/*_**************************************************
* Streaming compression
*
* A ZBUFF_CCtx object is required to track streaming operation.
@@ -88,13 +88,13 @@ typedef enum { ZBUFFcs_init, ZBUFFcs_load, ZBUFFcs_flush } ZBUFF_cStage;
/* *** Ressources *** */
struct ZBUFF_CCtx_s {
ZSTD_CCtx* zc;
char* inBuff;
char* inBuff;
size_t inBuffSize;
size_t inToCompress;
size_t inBuffPos;
size_t inBuffTarget;
size_t blockSize;
char* outBuff;
char* outBuff;
size_t outBuffSize;
size_t outBuffContentSize;
size_t outBuffFlushedSize;
@@ -123,8 +123,6 @@ size_t ZBUFF_freeCCtx(ZBUFF_CCtx* zbc)
/* *** Initialization *** */
#define MIN(a,b) ( ((a)<(b)) ? (a) : (b) )
#define BLOCKSIZE (128 * 1024) /* a bit too "magic", should come from reference */
size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, ZSTD_parameters params)
{
size_t neededInBuffSize;
@@ -172,9 +170,9 @@ ZSTDLIB_API size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* zbc, const void* dic
/* *** Compression *** */
static size_t ZBUFF_limitCopy(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
static size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{
size_t length = MIN(maxDstSize, srcSize);
size_t length = MIN(dstCapacity, srcSize);
memcpy(dst, src, length);
return length;
}
@@ -322,12 +320,10 @@ size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* maxDstSizePtr)
* just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
* *******************************************************************************/
typedef enum { ZBUFFds_init, ZBUFFds_readHeader, ZBUFFds_loadHeader, ZBUFFds_decodeHeader,
typedef enum { ZBUFFds_init, ZBUFFds_readHeader,
ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFF_dStage;
/* *** Resource management *** */
#define ZSTD_frameHeaderSize_max 5 /* too magical, should come from reference */
struct ZBUFF_DCtx_s {
ZSTD_DCtx* zc;
ZSTD_frameParams fParams;
@@ -338,9 +334,7 @@ struct ZBUFF_DCtx_s {
size_t outBuffSize;
size_t outStart;
size_t outEnd;
size_t hPos;
ZBUFF_dStage stage;
unsigned char headerBuffer[ZSTD_frameHeaderSize_max];
}; /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */
@@ -370,7 +364,7 @@ size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbc)
size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* zbc, const void* dict, size_t dictSize)
{
zbc->stage = ZBUFFds_readHeader;
zbc->hPos = zbc->inPos = zbc->outStart = zbc->outEnd = 0;
zbc->inPos = zbc->outStart = zbc->outEnd = 0;
return ZSTD_decompressBegin_usingDict(zbc->zc, dict, dictSize);
}
@@ -402,69 +396,36 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc,
case ZBUFFds_readHeader :
/* read header from src */
{
size_t headerSize = ZSTD_getFrameParams(&(zbc->fParams), src, *srcSizePtr);
{ size_t const headerSize = ZSTD_getFrameParams(&(zbc->fParams), src, *srcSizePtr);
if (ZSTD_isError(headerSize)) return headerSize;
if (headerSize) {
/* not enough input to decode header : tell how many bytes would be necessary */
memcpy(zbc->headerBuffer+zbc->hPos, src, *srcSizePtr);
zbc->hPos += *srcSizePtr;
/* not enough input to decode header : needs headerSize > *srcSizePtr */
*dstCapacityPtr = 0;
zbc->stage = ZBUFFds_loadHeader;
return headerSize - zbc->hPos;
}
zbc->stage = ZBUFFds_decodeHeader;
break;
}
*srcSizePtr = 0;
return headerSize;
} }
case ZBUFFds_loadHeader:
/* complete header from src */
/* Frame header provides buffer sizes */
{ size_t const neededInSize = BLOCKSIZE; /* a block is never > BLOCKSIZE */
if (zbc->inBuffSize < neededInSize) {
free(zbc->inBuff);
zbc->inBuffSize = neededInSize;
zbc->inBuff = (char*)malloc(neededInSize);
if (zbc->inBuff == NULL) return ERROR(memory_allocation);
} }
{
size_t headerSize = ZBUFF_limitCopy(
zbc->headerBuffer + zbc->hPos, ZSTD_frameHeaderSize_max - zbc->hPos,
src, *srcSizePtr);
zbc->hPos += headerSize;
ip += headerSize;
headerSize = ZSTD_getFrameParams(&(zbc->fParams), zbc->headerBuffer, zbc->hPos);
if (ZSTD_isError(headerSize)) return headerSize;
if (headerSize) {
/* not enough input to decode header : tell how many bytes would be necessary */
*dstCapacityPtr = 0;
return headerSize - zbc->hPos;
}
// zbc->stage = ZBUFFds_decodeHeader; break; /* useless : stage follows */
}
case ZBUFFds_decodeHeader:
/* apply header to create / resize buffers */
{
size_t neededOutSize = (size_t)1 << zbc->fParams.windowLog;
size_t neededInSize = BLOCKSIZE; /* a block is never > BLOCKSIZE */
if (zbc->inBuffSize < neededInSize) {
free(zbc->inBuff);
zbc->inBuffSize = neededInSize;
zbc->inBuff = (char*)malloc(neededInSize);
if (zbc->inBuff == NULL) return ERROR(memory_allocation);
}
if (zbc->outBuffSize < neededOutSize) {
free(zbc->outBuff);
zbc->outBuffSize = neededOutSize;
zbc->outBuff = (char*)malloc(neededOutSize);
if (zbc->outBuff == NULL) return ERROR(memory_allocation);
} }
if (zbc->hPos) {
/* some data already loaded into headerBuffer : transfer into inBuff */
memcpy(zbc->inBuff, zbc->headerBuffer, zbc->hPos);
zbc->inPos = zbc->hPos;
zbc->hPos = 0;
zbc->stage = ZBUFFds_load;
break;
}
zbc->stage = ZBUFFds_read;
size_t const neededOutSize = (size_t)1 << zbc->fParams.windowLog;
if (zbc->outBuffSize < neededOutSize) {
free(zbc->outBuff);
zbc->outBuffSize = neededOutSize;
zbc->outBuff = (char*)malloc(neededOutSize);
if (zbc->outBuff == NULL) return ERROR(memory_allocation);
} }
zbc->stage = ZBUFFds_read;
case ZBUFFds_read:
{
size_t neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc);
size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc);
if (neededInSize==0) { /* end of frame */
zbc->stage = ZBUFFds_init;
notDone = 0;
@@ -472,7 +433,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc,
}
if ((size_t)(iend-ip) >= neededInSize) {
/* directly decode from src */
size_t decodedSize = ZSTD_decompressContinue(zbc->zc,
size_t const decodedSize = ZSTD_decompressContinue(zbc->zc,
zbc->outBuff + zbc->outStart, zbc->outBuffSize - zbc->outStart,
ip, neededInSize);
if (ZSTD_isError(decodedSize)) return decodedSize;
@@ -488,8 +449,8 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc,
case ZBUFFds_load:
{
size_t neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc);
size_t toLoad = neededInSize - zbc->inPos; /* should always be <= remaining space within inBuff */
size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc);
size_t const toLoad = neededInSize - zbc->inPos; /* should always be <= remaining space within inBuff */
size_t loadedSize;
if (toLoad > zbc->inBuffSize - zbc->inPos) return ERROR(corruption_detected); /* should never happen */
loadedSize = ZBUFF_limitCopy(zbc->inBuff + zbc->inPos, toLoad, ip, iend-ip);
@@ -497,7 +458,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc,
zbc->inPos += loadedSize;
if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */
{
size_t decodedSize = ZSTD_decompressContinue(zbc->zc,
size_t const decodedSize = ZSTD_decompressContinue(zbc->zc,
zbc->outBuff + zbc->outStart, zbc->outBuffSize - zbc->outStart,
zbc->inBuff, neededInSize);
if (ZSTD_isError(decodedSize)) return decodedSize;
@@ -509,8 +470,8 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc,
} }
case ZBUFFds_flush:
{
size_t toFlushSize = zbc->outEnd - zbc->outStart;
size_t flushedSize = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outStart, toFlushSize);
size_t const toFlushSize = zbc->outEnd - zbc->outStart;
size_t const flushedSize = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outStart, toFlushSize);
op += flushedSize;
zbc->outStart += flushedSize;
if (flushedSize == toFlushSize) {
@@ -526,12 +487,12 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc,
default: return ERROR(GENERIC); /* impossible */
} }
/* result */
*srcSizePtr = ip-istart;
*dstCapacityPtr = op-ostart;
{
size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zbc->zc);
if (nextSrcSizeHint > ZBUFF_blockHeaderSize) nextSrcSizeHint+= ZBUFF_blockHeaderSize; /* get next block header too */
if (nextSrcSizeHint > ZBUFF_blockHeaderSize) nextSrcSizeHint+= ZBUFF_blockHeaderSize; /* get following block header too */
nextSrcSizeHint -= zbc->inPos; /* already loaded*/
return nextSrcSizeHint;
}
lib/zstd_compress.c
+82 -67
View File
@@ -425,11 +425,11 @@ static void ZSTD_reduceIndex (ZSTD_CCtx* zc,
FSE_ENCODING_DYNAMIC : read NCount
*/
size_t ZSTD_noCompressBlock (void* dst, size_t maxDstSize, const void* src, size_t srcSize)
size_t ZSTD_noCompressBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{
BYTE* const ostart = (BYTE* const)dst;
if (srcSize + ZSTD_blockHeaderSize > maxDstSize) return ERROR(dstSize_tooSmall);
if (srcSize + ZSTD_blockHeaderSize > dstCapacity) return ERROR(dstSize_tooSmall);
memcpy(ostart + ZSTD_blockHeaderSize, src, srcSize);
/* Build header */
@@ -442,12 +442,12 @@ size_t ZSTD_noCompressBlock (void* dst, size_t maxDstSize, const void* src, size
}
static size_t ZSTD_noCompressLiterals (void* dst, size_t maxDstSize, const void* src, size_t srcSize)
static size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{
BYTE* const ostart = (BYTE* const)dst;
const U32 flSize = 1 + (srcSize>31) + (srcSize>4095);
if (srcSize + flSize > maxDstSize) return ERROR(dstSize_tooSmall);
if (srcSize + flSize > dstCapacity) return ERROR(dstSize_tooSmall);
switch(flSize)
{
@@ -470,12 +470,12 @@ static size_t ZSTD_noCompressLiterals (void* dst, size_t maxDstSize, const void*
return srcSize + flSize;
}
static size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t maxDstSize, const void* src, size_t srcSize)
static size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{
BYTE* const ostart = (BYTE* const)dst;
U32 flSize = 1 + (srcSize>31) + (srcSize>4095);
(void)maxDstSize; /* maxDstSize guaranteed to be >=4, hence large enough */
(void)dstCapacity; /* dstCapacity guaranteed to be >=4, hence large enough */
switch(flSize)
{
@@ -502,7 +502,7 @@ static size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t maxDstSize, const
size_t ZSTD_minGain(size_t srcSize) { return (srcSize >> 6) + 2; }
static size_t ZSTD_compressLiterals (ZSTD_CCtx* zc,
void* dst, size_t maxDstSize,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
{
const size_t minGain = ZSTD_minGain(srcSize);
@@ -512,19 +512,19 @@ static size_t ZSTD_compressLiterals (ZSTD_CCtx* zc,
U32 hType = IS_HUF;
size_t clitSize;
if (maxDstSize < lhSize+1) return ERROR(dstSize_tooSmall); /* not enough space for compression */
if (dstCapacity < lhSize+1) return ERROR(dstSize_tooSmall); /* not enough space for compression */
if (zc->flagStaticTables && (lhSize==3)) {
hType = IS_PCH;
singleStream = 1;
clitSize = HUF_compress1X_usingCTable(ostart+lhSize, maxDstSize-lhSize, src, srcSize, zc->hufTable);
clitSize = HUF_compress1X_usingCTable(ostart+lhSize, dstCapacity-lhSize, src, srcSize, zc->hufTable);
} else {
clitSize = singleStream ? HUF_compress1X(ostart+lhSize, maxDstSize-lhSize, src, srcSize, 255, 12)
: HUF_compress2 (ostart+lhSize, maxDstSize-lhSize, src, srcSize, 255, 12);
clitSize = singleStream ? HUF_compress1X(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 12)
: HUF_compress2 (ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 12);
}
if ((clitSize==0) || (clitSize >= srcSize - minGain)) return ZSTD_noCompressLiterals(dst, maxDstSize, src, srcSize);
if (clitSize==1) return ZSTD_compressRleLiteralsBlock(dst, maxDstSize, src, srcSize);
if ((clitSize==0) || (clitSize >= srcSize - minGain)) return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
if (clitSize==1) return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize);
/* Build header */
switch(lhSize)
@@ -557,7 +557,7 @@ static size_t ZSTD_compressLiterals (ZSTD_CCtx* zc,
#define LITERAL_NOENTROPY 63 /* don't even attempt to compress literals below this threshold (cheap heuristic) */
size_t ZSTD_compressSequences(ZSTD_CCtx* zc,
void* dst, size_t maxDstSize,
void* dst, size_t dstCapacity,
size_t srcSize)
{
const seqStore_t* seqStorePtr = &(zc->seqStore);
@@ -577,7 +577,7 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* zc,
BYTE* const offCodeTable = seqStorePtr->offCodeStart;
BYTE* const ostart = (BYTE*)dst;
BYTE* op = ostart;
BYTE* const oend = ostart + maxDstSize;
BYTE* const oend = ostart + dstCapacity;
const size_t nbSeq = llPtr - llTable;
const size_t minGain = ZSTD_minGain(srcSize);
const size_t maxCSize = srcSize - minGain;
@@ -590,9 +590,9 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* zc,
const size_t minLitSize = zc->flagStaticTables ? 6 : LITERAL_NOENTROPY;
if (litSize <= minLitSize)
cSize = ZSTD_noCompressLiterals(op, maxDstSize, op_lit_start, litSize);
cSize = ZSTD_noCompressLiterals(op, dstCapacity, op_lit_start, litSize);
else
cSize = ZSTD_compressLiterals(zc, op, maxDstSize, op_lit_start, litSize);
cSize = ZSTD_compressLiterals(zc, op, dstCapacity, op_lit_start, litSize);
if (ZSTD_isError(cSize)) return cSize;
op += cSize;
}
@@ -1940,17 +1940,17 @@ static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int
}
static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{
ZSTD_blockCompressor blockCompressor = ZSTD_selectBlockCompressor(zc->params.strategy, zc->lowLimit < zc->dictLimit);
if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) return 0; /* don't even attempt compression below a certain srcSize */
blockCompressor(zc, src, srcSize);
return ZSTD_compressSequences(zc, dst, maxDstSize, srcSize);
return ZSTD_compressSequences(zc, dst, dstCapacity, srcSize);
}
static size_t ZSTD_compress_generic (ZSTD_CCtx* zc,
void* dst, size_t maxDstSize,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
{
size_t blockSize = zc->blockSize;
@@ -1969,7 +1969,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc,
while (remaining) {
size_t cSize;
if (maxDstSize < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */
if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */
if (remaining < blockSize) blockSize = remaining;
if ((U32)(ip+blockSize - zc->base) > zc->loadedDictEnd + maxDist) { /* enforce maxDist */
@@ -1978,11 +1978,11 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc,
if (zc->dictLimit < zc->lowLimit) zc->dictLimit = zc->lowLimit;
}
cSize = ZSTD_compressBlock_internal(zc, op+ZSTD_blockHeaderSize, maxDstSize-ZSTD_blockHeaderSize, ip, blockSize);
cSize = ZSTD_compressBlock_internal(zc, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, ip, blockSize);
if (ZSTD_isError(cSize)) return cSize;
if (cSize == 0) { /* block is not compressible */
cSize = ZSTD_noCompressBlock(op, maxDstSize, ip, blockSize);
cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize);
if (ZSTD_isError(cSize)) return cSize;
} else {
op[0] = (BYTE)(cSize>>16);
@@ -1993,7 +1993,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc,
}
remaining -= blockSize;
maxDstSize -= cSize;
dstCapacity -= cSize;
ip += blockSize;
op += cSize;
}
@@ -2077,12 +2077,12 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* zc,
}
size_t ZSTD_compressBlock(ZSTD_CCtx* zc, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
size_t ZSTD_compressBlock(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{
if (srcSize > BLOCKSIZE) return ERROR(srcSize_wrong);
zc->params.searchLength = MINMATCH; /* force ZSTD_btopt to MINMATCH in block mode */
ZSTD_LOG_BLOCK("%p: ZSTD_compressBlock searchLength=%d\n", zc->base, zc->params.searchLength);
return ZSTD_compressContinue_internal(zc, dst, maxDstSize, src, srcSize, 0);
return ZSTD_compressContinue_internal(zc, dst, dstCapacity, src, srcSize, 0);
}
@@ -2174,20 +2174,20 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* zc, const void* dict, size_t
return hufHeaderSize + offcodeHeaderSize + matchlengthHeaderSize + litlengthHeaderSize;
}
/** ZSTD_compress_insertDictionary() :
* @return : 0, or an error code */
static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, size_t dictSize)
{
if (dict && (dictSize>4)) {
U32 magic = MEM_readLE32(dict);
size_t eSize;
if (magic != ZSTD_DICT_MAGIC)
return ZSTD_loadDictionaryContent(zc, dict, dictSize);
if ((dict==NULL) || (dictSize<=4)) return 0;
eSize = ZSTD_loadDictEntropyStats(zc, (const char*)dict+4, dictSize-4) + 4;
if (ZSTD_isError(eSize)) return eSize;
return ZSTD_loadDictionaryContent(zc, (const char*)dict+eSize, dictSize-eSize);
/* default : dict is pure content */
if (MEM_readLE32(dict) != ZSTD_DICT_MAGIC) return ZSTD_loadDictionaryContent(zc, dict, dictSize);
/* known magic number : dict is parsed for entropy stats and content */
{ size_t const eSize = ZSTD_loadDictEntropyStats(zc, (const char*)dict+4 /* skip magic */, dictSize-4) + 4;
if (ZSTD_isError(eSize)) return eSize;
return ZSTD_loadDictionaryContent(zc, (const char*)dict+eSize, dictSize-eSize);
}
return 0;
}
@@ -2197,27 +2197,43 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* zc,
const void* dict, size_t dictSize,
ZSTD_parameters params)
{
size_t errorCode;
ZSTD_validateParams(&params);
errorCode = ZSTD_resetCCtx_advanced(zc, params);
if (ZSTD_isError(errorCode)) return errorCode;
{ size_t const errorCode = ZSTD_resetCCtx_advanced(zc, params);
if (ZSTD_isError(errorCode)) return errorCode; }
/* Write Frame Header into ctx headerBuffer */
MEM_writeLE32(zc->headerBuffer, ZSTD_MAGICNUMBER);
{
BYTE* const op = (BYTE*)zc->headerBuffer;
U32 const fcsSize[4] = { 0, 1, 2, 8 };
U32 const fcsId = (params.srcSize>0) + (params.srcSize>=256) + (params.srcSize>=65536+256); /* 0-3 */
BYTE fdescriptor = (BYTE)(params.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN); /* windowLog : 4 KB - 128 MB */
fdescriptor |= (BYTE)((params.searchLength==3)<<4); /* mml : 3-4 */
fdescriptor |= (BYTE)(fcsId << 6);
op[4] = fdescriptor;
switch(fcsId)
{
default: /* impossible */
case 0 : break;
case 1 : op[5] = (BYTE)(params.srcSize); break;
case 2 : MEM_writeLE16(op+5, (U16)(params.srcSize-256)); break;
case 3 : MEM_writeLE64(op+5, (U64)(params.srcSize)); break;
}
zc->hbSize = ZSTD_frameHeaderSize_min + fcsSize[fcsId];
}
/* Write Frame Header */
MEM_writeLE32(zc->headerBuffer, ZSTD_MAGICNUMBER); /* Write Header */
((BYTE*)zc->headerBuffer)[4] = (BYTE)(params.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN + ((params.searchLength==3)<<4));
zc->hbSize = ZSTD_frameHeaderSize_min;
zc->stage = 0;
return ZSTD_compress_insertDictionary(zc, dict, dictSize);
}
size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* zc, const void* dict, size_t dictSize, int compressionLevel)
{
ZSTD_parameters params = ZSTD_getParams(compressionLevel, dictSize);
params.srcSize = 0;
ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin_usingDict compressionLevel=%d\n", zc->base, compressionLevel);
return ZSTD_compressBegin_advanced(zc, dict, dictSize, ZSTD_getParams(compressionLevel, MAX(128 KB, dictSize)));
return ZSTD_compressBegin_advanced(zc, dict, dictSize, params);
}
size_t ZSTD_compressBegin(ZSTD_CCtx* zc, int compressionLevel)
@@ -2230,7 +2246,7 @@ size_t ZSTD_compressBegin(ZSTD_CCtx* zc, int compressionLevel)
/*! ZSTD_compressEnd() :
* Write frame epilogue.
* @return : nb of bytes written into dst (or an error code) */
size_t ZSTD_compressEnd(ZSTD_CCtx* zc, void* dst, size_t maxDstSize)
size_t ZSTD_compressEnd(ZSTD_CCtx* zc, void* dst, size_t dstCapacity)
{
BYTE* op = (BYTE*)dst;
size_t hbSize = 0;
@@ -2238,15 +2254,15 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* zc, void* dst, size_t maxDstSize)
/* empty frame */
if (zc->stage==0) {
hbSize = zc->hbSize;
if (maxDstSize <= hbSize) return ERROR(dstSize_tooSmall);
if (dstCapacity <= hbSize) return ERROR(dstSize_tooSmall);
zc->stage = 1;
memcpy(dst, zc->headerBuffer, hbSize);
maxDstSize -= hbSize;
dstCapacity -= hbSize;
op += hbSize;
}
/* frame epilogue */
if (maxDstSize < 3) return ERROR(dstSize_tooSmall);
if (dstCapacity < 3) return ERROR(dstSize_tooSmall);
op[0] = (BYTE)(bt_end << 6);
op[1] = 0;
op[2] = 0;
@@ -2256,16 +2272,16 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* zc, void* dst, size_t maxDstSize)
size_t ZSTD_compress_usingPreparedCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx,
void* dst, size_t maxDstSize,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
{
size_t outSize;
size_t errorCode = ZSTD_copyCCtx(cctx, preparedCCtx);
if (ZSTD_isError(errorCode)) return errorCode;
errorCode = ZSTD_compressContinue(cctx, dst, maxDstSize, src, srcSize);
errorCode = ZSTD_compressContinue(cctx, dst, dstCapacity, src, srcSize);
if (ZSTD_isError(errorCode)) return errorCode;
outSize = errorCode;
errorCode = ZSTD_compressEnd(cctx, (char*)dst+outSize, maxDstSize-outSize);
errorCode = ZSTD_compressEnd(cctx, (char*)dst+outSize, dstCapacity-outSize);
if (ZSTD_isError(errorCode)) return errorCode;
outSize += errorCode;
return outSize;
@@ -2273,51 +2289,50 @@ size_t ZSTD_compress_usingPreparedCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* prepare
size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
void* dst, size_t maxDstSize,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const void* dict,size_t dictSize,
ZSTD_parameters params)
{
BYTE* const ostart = (BYTE*)dst;
BYTE* op = ostart;
size_t oSize;
/* Init */
oSize = ZSTD_compressBegin_advanced(ctx, dict, dictSize, params);
if(ZSTD_isError(oSize)) return oSize;
{ size_t const errorCode = ZSTD_compressBegin_advanced(ctx, dict, dictSize, params);
if(ZSTD_isError(errorCode)) return errorCode; }
/* body (compression) */
oSize = ZSTD_compressContinue (ctx, op, maxDstSize, src, srcSize);
{ size_t const oSize = ZSTD_compressContinue (ctx, op, dstCapacity, src, srcSize);
if(ZSTD_isError(oSize)) return oSize;
op += oSize;
maxDstSize -= oSize;
dstCapacity -= oSize; }
/* Close frame */
oSize = ZSTD_compressEnd(ctx, op, maxDstSize);
{ size_t const oSize = ZSTD_compressEnd(ctx, op, dstCapacity);
if(ZSTD_isError(oSize)) return oSize;
op += oSize;
op += oSize; }
return (op - ostart);
}
size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, const void* dict, size_t dictSize, int compressionLevel)
size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict, size_t dictSize, int compressionLevel)
{
ZSTD_LOG_BLOCK("%p: ZSTD_compress_usingDict srcSize=%d dictSize=%d compressionLevel=%d\n", ctx->base, (int)srcSize, (int)dictSize, compressionLevel);
return ZSTD_compress_advanced(ctx, dst, maxDstSize, src, srcSize, dict, dictSize, ZSTD_getParams(compressionLevel, srcSize));
return ZSTD_compress_advanced(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, ZSTD_getParams(compressionLevel, srcSize));
}
size_t ZSTD_compressCCtx (ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel)
size_t ZSTD_compressCCtx (ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel)
{
ZSTD_LOG_BLOCK("%p: ZSTD_compressCCtx srcSize=%d compressionLevel=%d\n", ctx->base, (int)srcSize, compressionLevel);
return ZSTD_compress_advanced(ctx, dst, maxDstSize, src, srcSize, NULL, 0, ZSTD_getParams(compressionLevel, srcSize));
return ZSTD_compress_advanced(ctx, dst, dstCapacity, src, srcSize, NULL, 0, ZSTD_getParams(compressionLevel, srcSize));
}
size_t ZSTD_compress(void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel)
size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel)
{
size_t result;
ZSTD_CCtx ctxBody;
memset(&ctxBody, 0, sizeof(ctxBody));
result = ZSTD_compressCCtx(&ctxBody, dst, maxDstSize, src, srcSize, compressionLevel);
result = ZSTD_compressCCtx(&ctxBody, dst, dstCapacity, src, srcSize, compressionLevel);
free(ctxBody.workSpace); /* can't free ctxBody, since it's on stack; just free heap content */
return result;
}
lib/zstd_decompress.c
+17 -21
View File
@@ -303,42 +303,38 @@ static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize)
/** ZSTD_getFrameParams() :
* decode Frame Header, or provide expected `srcSize`.
* @return : 0, `fparamsPtr` is correctly filled,
* >0, not enough srcSize, provide expected `srcSize`,
* >0, `srcSize` is too small, result is expected `srcSize`,
* or an error code, which can be tested using ZSTD_isError() */
size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize)
{
U32 magicNumber, fcsId;
const BYTE* ip = (const BYTE*)src;
BYTE frameDesc;
if (srcSize < ZSTD_frameHeaderSize_min) return ZSTD_frameHeaderSize_min;
magicNumber = MEM_readLE32(src);
if (magicNumber != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown);
if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown);
{ size_t fhsize = ZSTD_frameHeaderSize(src, srcSize);
/* ensure there is enough `srcSize` to fully read/decode frame header */
{ size_t const fhsize = ZSTD_frameHeaderSize(src, srcSize);
if (srcSize < fhsize) return fhsize; }
memset(fparamsPtr, 0, sizeof(*fparamsPtr));
frameDesc = ip[4];
fparamsPtr->windowLog = (frameDesc & 0xF) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
fparamsPtr->mml = (frameDesc & 0x10) ? MINMATCH-1 : MINMATCH;
if ((frameDesc & 0x20) != 0) return ERROR(frameParameter_unsupported); /* reserved 1 bit */
fcsId = frameDesc >> 6;
switch(fcsId)
{
default: /* impossible */
case 0 : fparamsPtr->frameContentSize = 0; break;
case 1 : fparamsPtr->frameContentSize = ip[5]; break;
case 2 : fparamsPtr->frameContentSize = MEM_readLE16(ip+5); break;
case 3 : fparamsPtr->frameContentSize = MEM_readLE64(ip+5); break;
}
{ BYTE const frameDesc = ip[4];
fparamsPtr->windowLog = (frameDesc & 0xF) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
fparamsPtr->mml = (frameDesc & 0x10) ? MINMATCH-1 : MINMATCH;
if ((frameDesc & 0x20) != 0) return ERROR(frameParameter_unsupported); /* reserved 1 bit */
switch(frameDesc >> 6) /* fcsId */
{
default: /* impossible */
case 0 : fparamsPtr->frameContentSize = 0; break;
case 1 : fparamsPtr->frameContentSize = ip[5]; break;
case 2 : fparamsPtr->frameContentSize = MEM_readLE16(ip+5)+256; break;
case 3 : fparamsPtr->frameContentSize = MEM_readLE64(ip+5); break;
} }
return 0;
}
/** ZSTD_decodeFrameHeader() :
* decode Frame Header.
* srcSize must be the size provided by ZSTD_decodeFrameHeader_Part1().
* `srcSize` must be the size provided by ZSTD_frameHeaderSize().
* @return : 0, or an error code, which can be tested using ZSTD_isError() */
static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* zc, const void* src, size_t srcSize)
{
programs/fileio.c
+23 -14
View File
@@ -64,7 +64,7 @@
#include <sys/stat.h> /* stat64 */
#include "mem.h"
#include "fileio.h"
#include "zstd_static.h" /* ZSTD_magicNumber */
#include "zstd_static.h" /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */
#include "zbuff_static.h"
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
@@ -242,7 +242,7 @@ static FILE* FIO_openDstFile(const char* dstFileName)
/*! FIO_loadFile() :
* creates a buffer, pointed by *bufferPtr,
* creates a buffer, pointed by `*bufferPtr`,
* loads `filename` content into it,
* up to MAX_DICT_SIZE bytes
*/
@@ -342,6 +342,7 @@ static int FIO_compressFilename_internal(cRess_t ress,
/* init */
filesize = MAX(FIO_getFileSize(srcFileName),dictSize);
params = ZSTD_getParams(cLevel, filesize);
params.srcSize = filesize;
if (g_maxWLog) if (params.windowLog > g_maxWLog) params.windowLog = g_maxWLog;
errorCode = ZBUFF_compressInit_advanced(ress.ctx, ress.dictBuffer, ress.dictBufferSize, params);
if (ZBUFF_isError(errorCode)) EXM_THROW(21, "Error initializing compression : %s", ZBUFF_getErrorName(errorCode));
@@ -394,7 +395,7 @@ static int FIO_compressFilename_internal(cRess_t ress,
/*! FIO_compressFilename_internal() :
* same as FIO_compressFilename_extRess(), with ress.desFile already opened
* same as FIO_compressFilename_extRess(), with ress.destFile already opened (typically stdout)
* @return : 0 : compression completed correctly,
* 1 : missing or pb opening srcFileName
*/
@@ -432,10 +433,7 @@ static int FIO_compressFilename_extRess(cRess_t ress,
if (ress.dstFile==0) { fclose(ress.srcFile); return 1; }
result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, cLevel);
if (result != 0) {
remove(dstFileName);
}
if (result!=0) remove(dstFileName); /* remove operation artefact */
fclose(ress.srcFile); /* no pb to expect : only reading */
if (fclose(ress.dstFile)) EXM_THROW(28, "Write error : cannot properly close %s", dstFileName);
@@ -556,35 +554,46 @@ static void FIO_freeDResources(dRess_t ress)
}
/** FIO_decompressFrame() :
@return : size of decoded frame
*/
unsigned long long FIO_decompressFrame(dRess_t ress,
FILE* foutput, FILE* finput, size_t alreadyLoaded)
{
U64 frameSize = 0;
size_t readSize=alreadyLoaded;
size_t readSize;
ZBUFF_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
/* Complete Header loading */
{ size_t const toLoad = ZSTD_frameHeaderSize_max - alreadyLoaded; /* assumption : alreadyLoaded <= ZSTD_frameHeaderSize_max */
size_t const checkSize = fread(((char*)ress.srcBuffer) + alreadyLoaded, 1, toLoad, finput);
if (checkSize != toLoad) EXM_THROW(32, "Read error");
}
readSize = ZSTD_frameHeaderSize_max;
/* Main decompression Loop */
ZBUFF_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
while (1) {
/* Decode */
size_t sizeCheck;
size_t inSize=readSize, decodedSize=ress.dstBufferSize;
size_t toRead = ZBUFF_decompressContinue(ress.dctx, ress.dstBuffer, &decodedSize, ress.srcBuffer, &inSize);
if (ZBUFF_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFF_getErrorName(toRead));
readSize -= inSize;
/* Write block */
sizeCheck = fwrite(ress.dstBuffer, 1, decodedSize, foutput);
if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block to destination file");
{ size_t const sizeCheck = fwrite(ress.dstBuffer, 1, decodedSize, foutput);
if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block into destination"); }
frameSize += decodedSize;
DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(frameSize>>20) );
if (toRead == 0) break;
if (toRead == 0) break; /* end of frame */
if (readSize) EXM_THROW(38, "Decoding error : should consume entire input");
/* Fill input buffer */
if (toRead > ress.srcBufferSize) EXM_THROW(34, "too large block");
readSize = fread(ress.srcBuffer, 1, toRead, finput);
if (readSize != toRead) EXM_THROW(35, "Read error");
if (readSize != toRead)
EXM_THROW(35, "Read error");
}
return frameSize;