ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'refs/remotes/Cyan4973/dev060' into dev

Browse Source 
# Conflicts:
#	lib/zstd_compress.c
#	lib/zstd_static.h
This commit is contained in:
inikep
2016-04-04 14:54:53 +02:00
parent 227dd4bc74 8a57b928ba
commit 0c7456c5b7
4 changed files with 46 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/zstd_compress.c
+31 -27
View File
@@ -108,7 +108,7 @@ struct ZSTD_CCtx_s
seqStore_t seqStore; /* sequences storage ptrs */
U32* hashTable;
U32* hashTable3;
U32* contentTable;
U32* chainTable;
HUF_CElt* hufTable;
U32 flagStaticTables;
FSE_CTable offcodeCTable [FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)];
@@ -143,7 +143,7 @@ const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) /* hidden interface *
size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams)
{
CLAMPCHECK(cParams.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX);
CLAMPCHECK(cParams.contentLog, ZSTD_CONTENTLOG_MIN, ZSTD_CONTENTLOG_MAX);
CLAMPCHECK(cParams.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX);
CLAMPCHECK(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
CLAMPCHECK(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX);
{ U32 const searchLengthMin = (cParams.strategy == ZSTD_btopt) ? ZSTD_SEARCHLENGTH_MIN : ZSTD_SEARCHLENGTH_MIN+1;
@@ -163,8 +163,8 @@ size_t ZSTD_checkCParams_advanced(ZSTD_compressionParameters cParams, U64 srcSiz
{
if (srcSize > (1ULL << ZSTD_WINDOWLOG_MIN)) return ZSTD_checkCParams(cParams);
if (cParams.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) return ERROR(compressionParameter_unsupported);
if (srcSize <= (1ULL << cParams.windowLog)) cParams.windowLog = ZSTD_WINDOWLOG_MIN; /* fake value - temporary work around */
if (srcSize <= (1ULL << cParams.contentLog)) cParams.contentLog = ZSTD_CONTENTLOG_MIN; /* fake value - temporary work around */
if (srcSize <= (1ULL << cParams.windowLog)) cParams.windowLog = ZSTD_WINDOWLOG_MIN; /* fake value - temporary work around */
if (srcSize <= (1ULL << cParams.chainLog)) cParams.chainLog = ZSTD_CHAINLOG_MIN; /* fake value - temporary work around */
if ((srcSize <= (1ULL << cParams.hashLog)) && ((U32)cParams.strategy < (U32)ZSTD_btlazy2)) cParams.hashLog = ZSTD_HASHLOG_MIN; /* fake value - temporary work around */
return ZSTD_checkCParams(cParams);
}
@@ -189,8 +189,8 @@ void ZSTD_adjustCParams(ZSTD_compressionParameters* params, U64 srcSize, size_t
} }
if (params->hashLog > params->windowLog) params->hashLog = params->windowLog;
{ U32 const btPlus = (params->strategy == ZSTD_btlazy2) || (params->strategy == ZSTD_btopt);
U32 const maxContentLog = params->windowLog+btPlus;
if (params->contentLog > maxContentLog) params->contentLog = maxContentLog; } /* <= ZSTD_CONTENTLOG_MAX */
U32 const maxChainLog = params->windowLog+btPlus;
if (params->chainLog > maxChainLog) params->chainLog = maxChainLog; } /* <= ZSTD_CHAINLOG_MAX */
if (params->windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) params->windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */
if ((params->hashLog < ZSTD_HASHLOG_MIN) && ((U32)params->strategy >= (U32)ZSTD_btlazy2)) params->hashLog = ZSTD_HASHLOG_MIN; /* required to ensure collision resistance in bt */
@@ -218,16 +218,16 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
const U32 divider = (params.cParams.searchLength==3) ? 3 : 4;
const size_t maxNbSeq = blockSize / divider;
const size_t tokenSpace = blockSize + 11*maxNbSeq;
const size_t contentSize = (params.cParams.strategy == ZSTD_fast) ? 0 : (1 << params.cParams.contentLog);
const size_t chainSize = (params.cParams.strategy == ZSTD_fast) ? 0 : (1 << params.cParams.chainLog);
const size_t hSize = 1 << params.cParams.hashLog;
const size_t h3Size = (zc->hashLog3) ? 1 << zc->hashLog3 : 0;
const size_t tableSpace = (contentSize + hSize + h3Size) * sizeof(U32);
const size_t tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
/* Check if workSpace is large enough, alloc a new one if needed */
{ size_t const optSpace = ((MaxML+1) + (MaxLL+1) + (1<<Offbits) + (1<<Litbits))*sizeof(U32)
+ (ZSTD_OPT_NUM+1)*(sizeof(ZSTD_match_t) + sizeof(ZSTD_optimal_t));
size_t const neededSpace = tableSpace + (256*sizeof(U32)) /* huffTable */ + tokenSpace
+ ((params.cParams.strategy == ZSTD_btopt) ? optSpace : 0);
+ ((params.cParams.strategy == ZSTD_btopt) ? optSpace : 0);
if (zc->workSpaceSize < neededSpace) {
free(zc->workSpace);
zc->workSpace = malloc(neededSpace);
@@ -238,8 +238,8 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
memset(zc->workSpace, 0, tableSpace ); /* reset only tables */
zc->hashTable3 = (U32*)(zc->workSpace);
zc->hashTable = zc->hashTable3 + h3Size;
zc->contentTable = zc->hashTable + hSize;
zc->seqStore.buffer = zc->contentTable + contentSize;
zc->chainTable = zc->hashTable + hSize;
zc->seqStore.buffer = zc->chainTable + chainSize;
zc->hufTable = (HUF_CElt*)zc->seqStore.buffer;
zc->flagStaticTables = 0;
zc->seqStore.buffer = ((U32*)(zc->seqStore.buffer)) + 256;
@@ -291,10 +291,10 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx)
ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params);
/* copy tables */
{ const size_t contentSize = (srcCCtx->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.cParams.contentLog);
{ const size_t chainSize = (srcCCtx->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.cParams.chainLog);
const size_t hSize = 1 << srcCCtx->params.cParams.hashLog;
const size_t h3Size = (srcCCtx->hashLog3) ? 1 << srcCCtx->hashLog3 : 0;
const size_t tableSpace = (contentSize + hSize + h3Size) * sizeof(U32);
const size_t tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
memcpy(dstCCtx->workSpace, srcCCtx->workSpace, tableSpace);
}
@@ -343,8 +343,8 @@ static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue)
{ const U32 hSize = 1 << zc->params.cParams.hashLog;
ZSTD_reduceTable(zc->hashTable, hSize, reducerValue); }
{ const U32 contentSize = (zc->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << zc->params.cParams.contentLog);
ZSTD_reduceTable(zc->contentTable, contentSize, reducerValue); }
{ const U32 chainSize = (zc->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << zc->params.cParams.chainLog);
ZSTD_reduceTable(zc->chainTable, chainSize, reducerValue); }
{ const U32 h3Size = (zc->hashLog3) ? 1 << zc->hashLog3 : 0;
ZSTD_reduceTable(zc->hashTable3, h3Size, reducerValue); }
@@ -1254,8 +1254,8 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co
U32* const hashTable = zc->hashTable;
const U32 hashLog = zc->params.cParams.hashLog;
const size_t h = ZSTD_hashPtr(ip, hashLog, mls);
U32* const bt = zc->contentTable;
const U32 btLog = zc->params.cParams.contentLog - 1;
U32* const bt = zc->chainTable;
const U32 btLog = zc->params.cParams.chainLog - 1;
const U32 btMask= (1 << btLog) - 1;
U32 matchIndex = hashTable[h];
size_t commonLengthSmaller=0, commonLengthLarger=0;
@@ -1356,8 +1356,8 @@ static size_t ZSTD_insertBtAndFindBestMatch (
U32* const hashTable = zc->hashTable;
const U32 hashLog = zc->params.cParams.hashLog;
const size_t h = ZSTD_hashPtr(ip, hashLog, mls);
U32* const bt = zc->contentTable;
const U32 btLog = zc->params.cParams.contentLog - 1;
U32* const bt = zc->chainTable;
const U32 btLog = zc->params.cParams.chainLog - 1;
const U32 btMask= (1 << btLog) - 1;
U32 matchIndex = hashTable[h];
size_t commonLengthSmaller=0, commonLengthLarger=0;
@@ -1517,8 +1517,8 @@ U32 ZSTD_insertAndFindFirstIndex (ZSTD_CCtx* zc, const BYTE* ip, U32 mls)
{
U32* const hashTable = zc->hashTable;
const U32 hashLog = zc->params.cParams.hashLog;
U32* const chainTable = zc->contentTable;
const U32 chainMask = (1 << zc->params.cParams.contentLog) - 1;
U32* const chainTable = zc->chainTable;
const U32 chainMask = (1 << zc->params.cParams.chainLog) - 1;
const BYTE* const base = zc->base;
const U32 target = (U32)(ip - base);
U32 idx = zc->nextToUpdate;
@@ -1542,8 +1542,8 @@ size_t ZSTD_HcFindBestMatch_generic (
size_t* offsetPtr,
const U32 maxNbAttempts, const U32 mls, const U32 extDict)
{
U32* const chainTable = zc->contentTable;
const U32 chainSize = (1 << zc->params.cParams.contentLog);
U32* const chainTable = zc->chainTable;
const U32 chainSize = (1 << zc->params.cParams.chainLog);
const U32 chainMask = chainSize-1;
const BYTE* const base = zc->base;
const BYTE* const dictBase = zc->dictBase;
@@ -2077,8 +2077,8 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* zc,
/* preemptive overflow correction */
if (zc->lowLimit > (1<<30)) {
U32 const btplus = (zc->params.cParams.strategy == ZSTD_btlazy2) || (zc->params.cParams.strategy == ZSTD_btopt);
U32 const contentMask = (1 << (zc->params.cParams.contentLog - btplus)) - 1;
U32 const newLowLimit = zc->lowLimit & contentMask; /* preserve position % contentSize */
U32 const chainMask = (1 << (zc->params.cParams.chainLog - btplus)) - 1;
U32 const newLowLimit = zc->lowLimit & chainMask; /* preserve position % chainSize */
U32 const correction = zc->lowLimit - newLowLimit;
ZSTD_reduceIndex(zc, correction);
zc->base += correction;
@@ -2533,12 +2533,16 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, size_t dictSize)
{
ZSTD_compressionParameters cp;
size_t addedSize = srcSize ? 0 : 500;
size_t const addedSize = srcSize ? 0 : 500;
U64 const rSize = srcSize+dictSize ? srcSize+dictSize+addedSize : (U64)-1;
U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); /* intentional underflow for srcSizeHint == 0 */
if (compressionLevel<=0) compressionLevel = 1;
if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL;
cp = ZSTD_defaultCParameters[tableID][compressionLevel];
if (cp.windowLog > ZSTD_WINDOWLOG_MAX) cp.windowLog = ZSTD_WINDOWLOG_MAX; /* auto-correction, for 32-bits mode */
if (MEM_32bits()) { /* auto-correction, for 32-bits mode */
if (cp.windowLog > ZSTD_WINDOWLOG_MAX) cp.windowLog = ZSTD_WINDOWLOG_MAX;
if (cp.chainLog > ZSTD_CHAINLOG_MAX) cp.chainLog = ZSTD_CHAINLOG_MAX;
if (cp.hashLog > ZSTD_HASHLOG_MAX) cp.hashLog = ZSTD_HASHLOG_MAX;
}
return cp;
}
lib/zstd_opt.h
+2 -2
View File
@@ -247,8 +247,8 @@ static U32 ZSTD_insertBtAndGetAllMatches (
const size_t h = ZSTD_hashPtr(ip, hashLog, mls);
U32* const hashTable = zc->hashTable;
U32 matchIndex = hashTable[h];
U32* const bt = zc->contentTable;
const U32 btLog = zc->params.cParams.contentLog - 1;
U32* const bt = zc->chainTable;
const U32 btLog = zc->params.cParams.chainLog - 1;
const U32 btMask= (1U << btLog) - 1;
size_t commonLengthSmaller=0, commonLengthLarger=0;
const BYTE* const dictBase = zc->dictBase;
lib/zstd_static.h
+5 -5
View File
@@ -59,13 +59,13 @@ extern "C" {
***************************************/
#define ZSTD_WINDOWLOG_MAX (MEM_32bits() ? 25 : 27)
#define ZSTD_WINDOWLOG_MIN 18
#define ZSTD_CONTENTLOG_MAX (ZSTD_WINDOWLOG_MAX+1)
#define ZSTD_CONTENTLOG_MIN 4
#define ZSTD_HASHLOG_MAX 28
#define ZSTD_CHAINLOG_MAX (ZSTD_WINDOWLOG_MAX+1)
#define ZSTD_CHAINLOG_MIN 4
#define ZSTD_HASHLOG_MAX ZSTD_WINDOWLOG_MAX
#define ZSTD_HASHLOG_MIN 12
#define ZSTD_HASHLOG3_MAX 17
#define ZSTD_HASHLOG3_MIN 15
#define ZSTD_SEARCHLOG_MAX (ZSTD_CONTENTLOG_MAX-1)
#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1)
#define ZSTD_SEARCHLOG_MIN 1
#define ZSTD_SEARCHLENGTH_MAX 7
#define ZSTD_SEARCHLENGTH_MIN 3
@@ -77,7 +77,7 @@ typedef enum { ZSTD_fast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD
typedef struct {
U32 windowLog; /* largest match distance : larger == more compression, more memory needed during decompression */
U32 contentLog; /* full search segment : larger == more compression, slower, more memory (useless for fast) */
U32 chainLog; /* fully searched segment : larger == more compression, slower, more memory (useless for fast) */
U32 hashLog; /* dispatch table : larger == faster, more memory */
U32 searchLog; /* nb of searches : larger == more compression, slower */
U32 searchLength; /* match length searched : larger == faster decompression, sometimes less compression */