ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

modified buffer management rules

Browse Source
This commit is contained in:
Yann Collet
2016-02-02 14:36:49 +01:00
parent 82d71fe89f
commit 2ce4923238
4 changed files with 63 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/zstd_compress.c
+27 -35
View File
@@ -114,6 +114,7 @@ struct ZSTD_CCtx_s
U32 dictLimit; /* below that point, need extDict */
U32 lowLimit; /* below that point, no more data */
U32 nextToUpdate; /* index from which to continue dictionary update */
U32 loadedDictEnd;
U32 stage;
ZSTD_parameters params;
void* workSpace;
@@ -217,6 +218,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
zc->seqStore.dumpsStart = zc->seqStore.matchLengthStart + (blockSize>>2);
zc->hbSize = 0;
zc->stage = 0;
zc->loadedDictEnd = 0;
return 0;
}
@@ -906,7 +908,7 @@ static size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls)
}
/* *************************************
/*-*************************************
* Fast Scan
***************************************/
#define FILLHASHSTEP 3
@@ -916,7 +918,7 @@ static void ZSTD_fillHashTable (ZSTD_CCtx* zc, const void* end, const U32 mls)
const U32 hBits = zc->params.hashLog;
const BYTE* const base = zc->base;
const BYTE* ip = base + zc->nextToUpdate;
const BYTE* const iend = (const BYTE*) end;
const BYTE* const iend = ((const BYTE*)end) - 8;
while(ip <= iend) {
hashTable[ZSTD_hashPtr(ip, hBits, mls)] = (U32)(ip - base);
@@ -1885,47 +1887,41 @@ static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int
static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
{
ZSTD_blockCompressor blockCompressor = ZSTD_selectBlockCompressor(zc->params.strategy, zc->lowLimit < zc->dictLimit);
if (srcSize < MIN_CBLOCK_SIZE+3) return 0; /* don't even attempt compression below a certain srcSize */
if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) return 0; /* don't even attempt compression below a certain srcSize */
blockCompressor(zc, src, srcSize);
return ZSTD_compressSequences(zc, dst, maxDstSize, srcSize);
}
static size_t ZSTD_compress_generic (ZSTD_CCtx* ctxPtr,
static size_t ZSTD_compress_generic (ZSTD_CCtx* zc,
void* dst, size_t maxDstSize,
const void* src, size_t srcSize)
{
size_t blockSize = ctxPtr->blockSize;
size_t blockSize = zc->blockSize;
size_t remaining = srcSize;
const BYTE* ip = (const BYTE*)src;
BYTE* const ostart = (BYTE*)dst;
BYTE* op = ostart;
const U32 maxDist = 1 << ctxPtr->params.windowLog;
const U32 maxDist = 1 << zc->params.windowLog;
while (remaining)
{
while (remaining) {
size_t cSize;
if (maxDstSize < 3 + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */
if (maxDstSize < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */
if (remaining < blockSize) blockSize = remaining;
if ((U32)(ip+blockSize - (ctxPtr->base + ctxPtr->lowLimit)) > maxDist)
{
/* respect windowLog contract */
ctxPtr->lowLimit = (U32)(ip+blockSize - ctxPtr->base) - maxDist;
if (ctxPtr->dictLimit < ctxPtr->lowLimit) ctxPtr->dictLimit = ctxPtr->lowLimit;
if ((U32)(ip+blockSize - zc->base) > zc->loadedDictEnd + maxDist) { /* enforce maxDist */
zc->lowLimit = (U32)(ip+blockSize - zc->base) - maxDist;
if (zc->dictLimit < zc->lowLimit) zc->dictLimit = zc->lowLimit;
}
cSize = ZSTD_compressBlock_internal(ctxPtr, op+3, maxDstSize-3, ip, blockSize);
cSize = ZSTD_compressBlock_internal(zc, op+ZSTD_blockHeaderSize, maxDstSize-ZSTD_blockHeaderSize, ip, blockSize);
if (ZSTD_isError(cSize)) return cSize;
if (cSize == 0)
{
cSize = ZSTD_noCompressBlock(op, maxDstSize, ip, blockSize); /* block is not compressible */
if (cSize == 0) { /* block is not compressible */
cSize = ZSTD_noCompressBlock(op, maxDstSize, ip, blockSize);
if (ZSTD_isError(cSize)) return cSize;
}
else
{
} else {
op[0] = (BYTE)(cSize>>16);
op[1] = (BYTE)(cSize>>8);
op[2] = (BYTE)cSize;
@@ -1951,8 +1947,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* zc,
const BYTE* const ip = (const BYTE*) src;
size_t hbSize = 0;
if (frame && (zc->stage==0))
{
if (frame && (zc->stage==0)) {
hbSize = zc->hbSize;
if (dstSize <= hbSize) return ERROR(dstSize_tooSmall);
zc->stage = 1;
@@ -1962,8 +1957,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* zc,
}
/* Check if blocks follow each other */
if (src != zc->nextSrc)
{
if (src != zc->nextSrc) {
/* not contiguous */
size_t delta = zc->nextSrc - ip;
zc->lowLimit = zc->dictLimit;
@@ -1975,8 +1969,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* zc,
}
/* preemptive overflow correction */
if (zc->lowLimit > (1<<30))
{
if (zc->lowLimit > (1<<30)) {
U32 btplus = (zc->params.strategy == ZSTD_btlazy2);
U32 contentMask = (1 << (zc->params.contentLog - btplus)) - 1;
U32 newLowLimit = zc->lowLimit & contentMask; /* preserve position % contentSize */
@@ -1991,8 +1984,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* zc,
}
/* if input and dictionary overlap : reduce dictionary (presumed modified by input) */
if ((ip+srcSize > zc->dictBase + zc->lowLimit) && (ip < zc->dictBase + zc->dictLimit))
{
if ((ip+srcSize > zc->dictBase + zc->lowLimit) && (ip < zc->dictBase + zc->dictLimit)) {
zc->lowLimit = (U32)(ip + srcSize - zc->dictBase);
if (zc->lowLimit > zc->dictLimit) zc->lowLimit = zc->dictLimit;
}
@@ -2034,6 +2026,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t
zc->dictBase = zc->base;
zc->base += ip - zc->nextSrc;
zc->nextToUpdate = zc->dictLimit;
zc->loadedDictEnd = (U32)(iend - zc->base);
zc->nextSrc = iend;
if (srcSize <= 8) return 0;
@@ -2041,7 +2034,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t
switch(zc->params.strategy)
{
case ZSTD_fast:
ZSTD_fillHashTable (zc, iend-8, zc->params.searchLength);
ZSTD_fillHashTable (zc, iend, zc->params.searchLength);
break;
case ZSTD_greedy:
@@ -2052,13 +2045,13 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t
case ZSTD_btlazy2:
ZSTD_updateTree(zc, iend-8, iend, 1 << zc->params.searchLog, zc->params.searchLength);
zc->nextToUpdate = (U32)(iend - zc->base);
break;
default:
return ERROR(GENERIC); /* strategy doesn't exist; impossible */
}
zc->nextToUpdate = zc->loadedDictEnd;
return 0;
}
@@ -2112,8 +2105,7 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* zc, const void* dict, size_t
static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, size_t dictSize)
{
if (dict && dictSize)
{
if (dict && (dictSize>4)) {
U32 magic = MEM_readLE32(dict);
size_t eSize;
if (magic != ZSTD_DICT_MAGIC)
@@ -2151,7 +2143,7 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* zc,
size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* zc, const void* dict, size_t dictSize, int compressionLevel)
{
return ZSTD_compressBegin_advanced(zc, dict, dictSize, ZSTD_getParams(compressionLevel, 0));
return ZSTD_compressBegin_advanced(zc, dict, dictSize, ZSTD_getParams(compressionLevel, 128 KB));
}
size_t ZSTD_compressBegin(ZSTD_CCtx* zc, int compressionLevel)
@@ -2235,7 +2227,7 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, const void* dict, size_t dictSize, int compressionLevel)
{
return ZSTD_compress_advanced(ctx, dst, maxDstSize, src, srcSize, dict, dictSize, ZSTD_getParams(compressionLevel, srcSize+dictSize));
return ZSTD_compress_advanced(ctx, dst, maxDstSize, src, srcSize, dict, dictSize, ZSTD_getParams(compressionLevel, srcSize));
}
size_t ZSTD_compressCCtx (ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel)