ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

strengthened streaming bufferless compression

Browse Source
This commit is contained in:
Yann Collet
2015-12-04 17:16:37 +01:00
parent 28e7cefea9
commit 417890cec2
7 changed files with 147 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/zstd.h
+1 -1
View File
@@ -48,7 +48,7 @@ extern "C" {
***************************************/
#define ZSTD_VERSION_MAJOR 0 /* for breaking interface changes */
#define ZSTD_VERSION_MINOR 4 /* for new (non-breaking) interface capabilities */
#define ZSTD_VERSION_RELEASE 2 /* for tweaks, bug-fixes, or development */
#define ZSTD_VERSION_RELEASE 3 /* for tweaks, bug-fixes, or development */
#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
unsigned ZSTD_versionNumber (void);
lib/zstd_compress.c
+73 -15
View File
@@ -488,7 +488,7 @@ size_t ZSTD_compressSequences(void* dst, size_t maxDstSize,
BYTE litLength = llTable[i]; /* (7)*/ /* (7)*/
FSE_encodeSymbol(&blockStream, &stateMatchLength, matchLength); /* 17 */ /* 17 */
if (MEM_32bits()) BIT_flushBits(&blockStream); /* 7 */
BIT_addBits(&blockStream, offset, nbBits); /* 32 */ /* 42 */
BIT_addBits(&blockStream, offset, nbBits); /* 31 */ /* 42 */ /* 24 bits max in 32-bits mode */
if (MEM_32bits()) BIT_flushBits(&blockStream); /* 7 */
FSE_encodeSymbol(&blockStream, &stateOffsetBits, offCode); /* 16 */ /* 51 */
FSE_encodeSymbol(&blockStream, &stateLitLength, litLength); /* 26 */ /* 61 */
@@ -730,13 +730,30 @@ static size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls)
* Fast Scan
***************************************/
#define FILLHASHSTEP 3
static void ZSTD_fillHashTable (ZSTD_CCtx* zc, const void* end, const U32 mls)
{
U32* const hashTable = zc->hashTable;
const U32 hBits = zc->params.hashLog;
const BYTE* const base = zc->base;
const BYTE* ip = base + zc->nextToUpdate;
const BYTE* const iend = (const BYTE*) end;
while(ip <= iend)
{
hashTable[ZSTD_hashPtr(ip, hBits, mls)] = (U32)(ip - base);
ip += FILLHASHSTEP;
}
}
FORCE_INLINE
size_t ZSTD_compressBlock_fast_generic(ZSTD_CCtx* zc,
void* dst, size_t maxDstSize,
const void* src, size_t srcSize,
const U32 mls)
{
U32* hashTable = zc->hashTable;
U32* const hashTable = zc->hashTable;
const U32 hBits = zc->params.hashLog;
seqStore_t* seqStorePtr = &(zc->seqStore);
const BYTE* const base = zc->base;
@@ -1973,10 +1990,24 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* zc,
void* dst, size_t dstSize,
const void* src, size_t srcSize)
{
U32 adressOverflow = 0;
const BYTE* const ip = (const BYTE*) src;
/* Check if blocks follow each other */
if (src != zc->nextSrc)
{
/* not contiguous */
size_t delta = zc->nextSrc - ip;
zc->lowLimit = zc->dictLimit;
zc->dictLimit = (U32)(zc->nextSrc - zc->base);
zc->dictBase = zc->base;
if ((size_t)zc->base < delta) adressOverflow = zc->lowLimit;
zc->base -= delta;
zc->nextToUpdate = zc->dictLimit;
}
/* preemptive overflow correction */
if ((zc->base > (const BYTE*)src) || (zc->lowLimit > (1<<30) ))
if (adressOverflow || (zc->lowLimit > (1<<30) ))
{
U32 correction = zc->lowLimit-1;
ZSTD_reduceIndex(zc, correction);
@@ -1988,17 +2019,6 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* zc,
else zc->nextToUpdate -= correction;
}
/* Check if blocks follow each other */
if (src != zc->nextSrc)
{
/* not contiguous */
zc->lowLimit = zc->dictLimit;
zc->dictLimit = (U32)(zc->nextSrc - zc->base);
zc->dictBase = zc->base;
zc->base += ip - zc->nextSrc;
zc->nextToUpdate = zc->dictLimit;
}
/* input-dictionary overlap */
if ((ip+srcSize > zc->dictBase + zc->lowLimit) && (ip < zc->dictBase + zc->dictLimit))
{
@@ -2011,8 +2031,46 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* zc,
return ZSTD_compress_generic (zc, dst, dstSize, src, srcSize);
}
size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* src, size_t srcSize)
{
const BYTE* const ip = (const BYTE*) src;
const BYTE* const iend = ip + srcSize;
/** ZSTD_compressBegin_advanced
/* input becomes current prefix */
zc->lowLimit = zc->dictLimit;
zc->dictLimit = (U32)(zc->nextSrc - zc->base);
zc->dictBase = zc->base;
zc->base += ip - zc->nextSrc;
zc->nextToUpdate = zc->dictLimit;
zc->nextSrc = iend;
if (srcSize <= 8) return 0;
switch(zc->params.strategy)
{
case ZSTD_fast:
ZSTD_fillHashTable (zc, iend-8, zc->params.searchLength);
break;
case ZSTD_greedy:
case ZSTD_lazy:
case ZSTD_lazy2:
ZSTD_insertAndFindFirstIndex (zc, iend-8, zc->params.searchLength);
break;
case ZSTD_btlazy2:
ZSTD_updateTree(zc, iend-8, iend, 1 << zc->params.searchLog, zc->params.searchLength);
break;
default:
return ERROR(GENERIC); /* strategy doesn't exist; impossible */
}
return 0;
}
/*! ZSTD_compressBegin_advanced
* Write frame header, according to params
* @return : nb of bytes written */
size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* ctx,
lib/zstd_decompress.c
+19 -13
View File
@@ -127,10 +127,10 @@ struct ZSTD_DCtx_s
U32 LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
U32 OffTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
U32 MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
void* previousDstEnd;
void* base;
void* vBase;
void* dictEnd;
const void* previousDstEnd;
const void* base;
const void* vBase;
const void* dictEnd;
size_t expected;
size_t headerSize;
ZSTD_parameters params;
@@ -141,7 +141,7 @@ struct ZSTD_DCtx_s
size_t litSize;
BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */];
BYTE headerBuffer[ZSTD_frameHeaderSize_max];
}; /* typedef'd to ZSTD_Dctx within "zstd_static.h" */
}; /* typedef'd to ZSTD_DCtx within "zstd_static.h" */
size_t ZSTD_resetDCtx(ZSTD_DCtx* dctx)
{
@@ -505,7 +505,7 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
FORCE_INLINE size_t ZSTD_execSequence(BYTE* op,
BYTE* const oend, seq_t sequence,
const BYTE** litPtr, const BYTE* const litLimit_8,
BYTE* const base, BYTE* const vBase, BYTE* const dictEnd)
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
{
static const int dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* substracted */
@@ -607,9 +607,9 @@ static size_t ZSTD_decompressSequences(
U32* DTableLL = dctx->LLTable;
U32* DTableML = dctx->MLTable;
U32* DTableOffb = dctx->OffTable;
BYTE* const base = (BYTE*) (dctx->base);
BYTE* const vBase = (BYTE*) (dctx->vBase);
BYTE* const dictEnd = (BYTE*) (dctx->dictEnd);
const BYTE* const base = (const BYTE*) (dctx->base);
const BYTE* const vBase = (const BYTE*) (dctx->vBase);
const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
/* Build Decoding Tables */
errorCode = ZSTD_decodeSeqHeaders(&nbSeq, &dumps, &dumpsLength,
@@ -691,7 +691,7 @@ size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, const v
/* init */
ctx->base = ctx->vBase = ctx->dictEnd = dst;
ctx->vBase = ctx->base = ctx->dictEnd = dst;
/* Frame Header */
{
@@ -776,7 +776,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, con
if ((dst > ctx->base) && (dst < ctx->previousDstEnd)) /* rolling buffer : new segment into dictionary */
ctx->base = (char*)dst; /* temporary affectation, for vBase calculation */
ctx->dictEnd = ctx->previousDstEnd;
ctx->vBase = (char*)dst - ((char*)(ctx->previousDstEnd) - (char*)(ctx->base));
ctx->vBase = (const char*)dst - ((const char*)(ctx->previousDstEnd) - (const char*)(ctx->base));
ctx->base = dst;
ctx->previousDstEnd = dst;
}
@@ -827,10 +827,9 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, con
ctx->bType = bp.blockType;
ctx->stage = ZSTDds_decompressBlock;
}
return 0;
}
case 3:
case ZSTDds_decompressBlock:
{
/* Decompress : block content */
size_t rSize;
@@ -862,3 +861,10 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, con
}
void ZSTD_decompress_insertDictionary(ZSTD_DCtx* ctx, const void* src, size_t srcSize)
{
ctx->dictEnd = ctx->previousDstEnd;
ctx->vBase = (const char*)src - ((const char*)(ctx->previousDstEnd) - (const char*)(ctx->base));
ctx->base = src;
ctx->previousDstEnd = (const char*)src + srcSize;
}
lib/zstd_static.h
+14 -3
View File
@@ -104,6 +104,8 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
****************************************/
size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, int compressionLevel);
size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, ZSTD_parameters params);
size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* ctx, const void* src, size_t srcSize);
size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize);
@@ -118,6 +120,10 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize);
Use ZSTD_compressBegin().
You may also prefer the advanced derivative ZSTD_compressBegin_advanced(), for finer parameter control.
It's then possible to add a dictionary with ZSTD_compressDictionary()
Note that dictionary presence is a "hidden" information,
the decoder needs to be aware that it is required for proper decoding, or decoding will fail.
Then, consume your input using ZSTD_compressContinue().
The interface is synchronous, so all input will be consumed.
You must ensure there is enough space in destination buffer to store compressed data under worst case scenario.
@@ -131,12 +137,15 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize);
typedef struct ZSTD_DCtx_s ZSTD_DCtx;
ZSTD_DCtx* ZSTD_createDCtx(void);
size_t ZSTD_resetDCtx(ZSTD_DCtx* dctx);
size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
size_t ZSTD_resetDCtx(ZSTD_DCtx* dctx);
size_t ZSTD_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcSize);
void ZSTD_decompress_insertDictionary(ZSTD_DCtx* ctx, const void* src, size_t srcSize);
size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
/**
Streaming decompression, bufferless mode
@@ -146,15 +155,17 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co
First operation is to retrieve frame parameters, using ZSTD_getFrameParams().
This function doesn't consume its input. It needs enough input data to properly decode the frame header.
The objective is to retrieve *params.windowlog, to know minimum amount of memory required during decoding.
Objective is to retrieve *params.windowlog, to know minimum amount of memory required during decoding.
Result : 0 when successful, it means the ZSTD_parameters structure has been filled.
>0 : means there is not enough data into src. Provides the expected size to successfully decode header.
errorCode, which can be tested using ZSTD_isError() (For example, if it's not a ZSTD header)
Then, you can optionally insert a dictionary. This operation must mimic the compressor behavior, otherwise decompression will fail or be corrupted.
Then it's possible to start decompression.
Use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively.
ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue().
ZSTD_decompressContinue() requires this exact amount of bytes, or just fails.
ZSTD_decompressContinue() requires this exact amount of bytes, or it will fail.
ZSTD_decompressContinue() needs previous data blocks during decompression, up to (1 << windowlog).
They should preferably be located contiguously, prior to current block. Alternatively, a round buffer is also possible.