ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #161 from inikep/dev

Browse Source 
Dev
This commit is contained in:
Yann Collet
2016-04-05 09:59:59 +02:00
parent b59bf963a8 19140cd80e
commit 46bdb6267e
13 changed files with 458 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/.debug/zstd_stats.h
+164
View File
@@ -0,0 +1,164 @@
/*
zstd - standard compression library
Header File for static linking only
Copyright (C) 2014-2016, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- zstd homepage : http://www.zstd.net
*/
#ifndef ZSTD_STATS_H
#define ZSTD_STATS_H
#if defined (__cplusplus)
extern "C" {
#endif
/*-*************************************
* Dependencies
***************************************/
//#include "zstd.h"
//#include "mem.h"
/*-*************************************
* Constants
***************************************/
//#define ZSTD_MAGICNUMBER 0xFD2FB526 /* v0.6 */
/*-*************************************
* Types
***************************************/
typedef struct {
U32 priceOffset, priceOffCode, priceMatchLength, priceLiteral, priceLitLength, priceDumpsLength;
U32 totalMatchSum, totalLitSum, totalSeqSum, totalRepSum;
U32 litSum, matchLengthSum, litLengthSum, offCodeSum;
U32 matchLengthFreq[1<<MLbits];
U32 litLengthFreq[1<<LLbits];
U32 litFreq[1<<Litbits];
U32 offCodeFreq[1<<Offbits];
} ZSTD_stats_t;
/*-*************************************
* Advanced functions
***************************************/
MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength)
{
stats->totalMatchSum += stats->totalSeqSum * ((searchLength == 3) ? 3 : 4);
printf("avgMatchL=%.2f avgLitL=%.2f match=%.1f%% lit=%.1f%% reps=%d seq=%d\n", (float)stats->totalMatchSum/stats->totalSeqSum, (float)stats->totalLitSum/stats->totalSeqSum, 100.0*stats->totalMatchSum/(stats->totalMatchSum+stats->totalLitSum), 100.0*stats->totalLitSum/(stats->totalMatchSum+stats->totalLitSum), stats->totalRepSum, stats->totalSeqSum);
printf("SumBytes=%d Offset=%d OffCode=%d Match=%d Literal=%d LitLength=%d DumpsLength=%d\n", (stats->priceOffset+stats->priceOffCode+stats->priceMatchLength+stats->priceLiteral+stats->priceLitLength+stats->priceDumpsLength)/8, stats->priceOffset/8, stats->priceOffCode/8, stats->priceMatchLength/8, stats->priceLiteral/8, stats->priceLitLength/8, stats->priceDumpsLength/8);
}
MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats)
{
stats->totalLitSum = stats->totalMatchSum = stats->totalSeqSum = stats->totalRepSum = 1;
stats->priceOffset = stats->priceOffCode = stats->priceMatchLength = stats->priceLiteral = stats->priceLitLength = stats->priceDumpsLength = 0;
}
MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats)
{
unsigned u;
stats->litSum = (1<<Litbits);
stats->litLengthSum = (1<<LLbits);
stats->matchLengthSum = (1<<MLbits);
stats->offCodeSum = (1<<Offbits);
for (u=0; u<=MaxLit; u++)
stats->litFreq[u] = 1;
for (u=0; u<=MaxLL; u++)
stats->litLengthFreq[u] = 1;
for (u=0; u<=MaxML; u++)
stats->matchLengthFreq[u] = 1;
for (u=0; u<=MaxOff; u++)
stats->offCodeFreq[u] = 1;
}
MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength)
{
/* offset */
BYTE offCode = offset ? (BYTE)ZSTD_highbit(offset+1) + 1 : 0;
stats->priceOffCode += ZSTD_highbit(stats->offCodeSum+1) - ZSTD_highbit(stats->offCodeFreq[offCode]+1);
stats->priceOffset += (offCode-1) + (!offCode);
/* match Length */
stats->priceDumpsLength += ((matchLength >= MaxML)<<3) + ((matchLength >= 255+MaxML)<<4) + ((matchLength>=(1<<15))<<3);
stats->priceMatchLength += ZSTD_highbit(stats->matchLengthSum+1) - ZSTD_highbit(stats->matchLengthFreq[(matchLength >= MaxML) ? MaxML : matchLength]+1);
if (litLength) {
/* literals */
U32 u;
stats->priceLiteral += litLength * ZSTD_highbit(stats->litSum+1);
for (u=0; u < litLength; u++)
stats->priceLiteral -= ZSTD_highbit(stats->litFreq[literals[u]]+1);
/* literal Length */
stats->priceDumpsLength += ((litLength >= MaxLL)<<3) + ((litLength >= 255+MaxLL)<<4) + ((litLength>=(1<<15))<<3);
stats->priceLitLength += ZSTD_highbit(stats->litLengthSum+1) - ZSTD_highbit(stats->litLengthFreq[(litLength >= MaxLL) ? MaxLL : litLength]+1);
} else {
stats->priceLitLength += ZSTD_highbit(stats->litLengthSum+1) - ZSTD_highbit(stats->litLengthFreq[0]+1);
}
if (offset == 0) stats->totalRepSum++;
stats->totalSeqSum++;
stats->totalMatchSum += matchLength;
stats->totalLitSum += litLength;
U32 u;
/* literals */
stats->litSum += litLength;
for (u=0; u < litLength; u++)
stats->litFreq[literals[u]]++;
/* literal Length */
stats->litLengthSum++;
if (litLength >= MaxLL)
stats->litLengthFreq[MaxLL]++;
else
stats->litLengthFreq[litLength]++;
/* match offset */
stats->offCodeSum++;
stats->offCodeFreq[offCode]++;
/* match Length */
stats->matchLengthSum++;
if (matchLength >= MaxML)
stats->matchLengthFreq[MaxML]++;
else
stats->matchLengthFreq[matchLength]++;
}
#if defined (__cplusplus)
}
#endif
#endif /* ZSTD_STATIC_H */
lib/zstd_compress.c
+32 -27
View File
@@ -95,6 +95,7 @@ struct ZSTD_CCtx_s
U32 lowLimit; /* below that point, no more data */
U32 nextToUpdate; /* index from which to continue dictionary update */
U32 nextToUpdate3; /* index from which to continue dictionary update */
U32 hashLog3; /* dispatch table : larger == faster, more memory */
U32 loadedDictEnd;
U32 stage;
ZSTD_parameters params;
@@ -219,7 +220,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
const size_t tokenSpace = blockSize + 11*maxNbSeq;
const size_t chainSize = (params.cParams.strategy == ZSTD_fast) ? 0 : (1 << params.cParams.chainLog);
const size_t hSize = 1 << params.cParams.hashLog;
const size_t h3Size = (params.cParams.searchLength==3) ? (1 << HASHLOG3) : 0;
const size_t h3Size = (zc->hashLog3) ? 1 << zc->hashLog3 : 0;
const size_t tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
/* Check if workSpace is large enough, alloc a new one if needed */
@@ -286,13 +287,14 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx)
{
if (srcCCtx->stage!=0) return ERROR(stage_wrong);
dstCCtx->hashLog3 = srcCCtx->hashLog3; /* must be before ZSTD_resetCCtx_advanced */
ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params);
/* copy tables */
{ const size_t chaineSize = (srcCCtx->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.cParams.chainLog);
{ const size_t chainSize = (srcCCtx->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.cParams.chainLog);
const size_t hSize = 1 << srcCCtx->params.cParams.hashLog;
const size_t h3Size = (srcCCtx->params.cParams.searchLength == 3) ? (1 << HASHLOG3) : 0;
const size_t tableSpace = (chaineSize + hSize + h3Size) * sizeof(U32);
const size_t h3Size = (srcCCtx->hashLog3) ? 1 << srcCCtx->hashLog3 : 0;
const size_t tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
memcpy(dstCCtx->workSpace, srcCCtx->workSpace, tableSpace);
}
@@ -344,7 +346,7 @@ static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue)
{ const U32 chainSize = (zc->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << zc->params.cParams.chainLog);
ZSTD_reduceTable(zc->chainTable, chainSize, reducerValue); }
{ const U32 h3Size = (zc->params.cParams.searchLength == 3) ? (1 << HASHLOG3) : 0;
{ const U32 h3Size = (zc->hashLog3) ? 1 << zc->hashLog3 : 0;
ZSTD_reduceTable(zc->hashTable3, h3Size, reducerValue); }
}
@@ -855,12 +857,7 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const B
printf("Cpos %6u :%5u literals & match %3u bytes at distance %6u \n",
pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode);
#endif
#if ZSTD_OPT_DEBUG == 3
if (offsetCode == 0) seqStorePtr->realRepSum++;
seqStorePtr->realSeqSum++;
seqStorePtr->realMatchSum += matchCode;
seqStorePtr->realLitSum += litLength;
#endif
ZSTD_statsUpdatePrices(&seqStorePtr->stats, litLength, literals, offsetCode, matchCode);
/* copy Literals */
ZSTD_wildcopy(seqStorePtr->lit, literals, litLength);
@@ -1746,6 +1743,7 @@ _storeSequence:
{ size_t const lastLLSize = iend - anchor;
memcpy(seqStorePtr->lit, anchor, lastLLSize);
seqStorePtr->lit += lastLLSize;
ZSTD_statsUpdatePrices(&seqStorePtr->stats, lastLLSize, anchor, 0, 0);
}
}
@@ -1992,6 +1990,8 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCa
}
static size_t ZSTD_compress_generic (ZSTD_CCtx* zc,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
@@ -2002,15 +2002,13 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc,
BYTE* const ostart = (BYTE*)dst;
BYTE* op = ostart;
const U32 maxDist = 1 << zc->params.cParams.windowLog;
#if ZSTD_OPT_DEBUG == 3
seqStore_t* ssPtr = &zc->seqStore;
static U32 priceFunc = 0;
ssPtr->realMatchSum = ssPtr->realLitSum = ssPtr->realSeqSum = ssPtr->realRepSum = 1;
ssPtr->priceFunc = priceFunc;
#endif
ZSTD_stats_t* stats = &zc->seqStore.stats;
ZSTD_statsInit(stats);
while (remaining) {
size_t cSize;
ZSTD_statsResetFreqs(stats);
if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */
if (remaining < blockSize) blockSize = remaining;
@@ -2042,12 +2040,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc,
op += cSize;
}
#if ZSTD_OPT_DEBUG == 3
ssPtr->realMatchSum += ssPtr->realSeqSum * ((zc->params.searchLength == 3) ? 3 : 4);
printf("avgMatchL=%.2f avgLitL=%.2f match=%.1f%% lit=%.1f%% reps=%d seq=%d priceFunc=%d\n", (float)ssPtr->realMatchSum/ssPtr->realSeqSum, (float)ssPtr->realLitSum/ssPtr->realSeqSum, 100.0*ssPtr->realMatchSum/(ssPtr->realMatchSum+ssPtr->realLitSum), 100.0*ssPtr->realLitSum/(ssPtr->realMatchSum+ssPtr->realLitSum), ssPtr->realRepSum, ssPtr->realSeqSum, ssPtr->priceFunc);
priceFunc++;
#endif
ZSTD_statsPrint(stats, zc->params.cParams.searchLength);
return op-ostart;
}
@@ -2233,13 +2226,16 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, si
}
}
/*! ZSTD_compressBegin_internal() :
* @return : 0, or an error code */
static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* zc,
const void* dict, size_t dictSize,
ZSTD_parameters params, U64 pledgedSrcSize)
{
U32 hashLog3 = (pledgedSrcSize || pledgedSrcSize >= 8192) ? ZSTD_HASHLOG3_MAX : ((pledgedSrcSize >= 2048) ? ZSTD_HASHLOG3_MIN + 1 : ZSTD_HASHLOG3_MIN);
zc->hashLog3 = (params.cParams.searchLength==3) ? hashLog3 : 0;
// printf("windowLog=%d hashLog=%d hashLog3=%d \n", params.windowLog, params.hashLog, zc->hashLog3);
{ size_t const errorCode = ZSTD_resetCCtx_advanced(zc, params);
if (ZSTD_isError(errorCode)) return errorCode; }
@@ -2291,6 +2287,18 @@ size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* zc, const void* dict, size_t dict
return ZSTD_compressBegin_internal(zc, dict, dictSize, params, 0);
}
size_t ZSTD_compressBegin_targetSrcSize(ZSTD_CCtx* zc, const void* dict, size_t dictSize, size_t targetSrcSize, int compressionLevel)
{
ZSTD_parameters params;
params.cParams = ZSTD_getCParams(compressionLevel, targetSrcSize, dictSize);
params.fParams.contentSizeFlag = 1;
ZSTD_adjustCParams(&params.cParams, targetSrcSize, dictSize);
ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin_targetSrcSize compressionLevel=%d\n", zc->base, compressionLevel);
return ZSTD_compressBegin_internal(zc, dict, dictSize, params, targetSrcSize);
}
size_t ZSTD_compressBegin(ZSTD_CCtx* zc, int compressionLevel)
{
ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin compressionLevel=%d\n", zc->base, compressionLevel);
@@ -2530,9 +2538,6 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, si
U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); /* intentional underflow for srcSizeHint == 0 */
if (compressionLevel<=0) compressionLevel = 1;
if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL;
#if ZSTD_OPT_DEBUG >= 1
tableID=0;
#endif
cp = ZSTD_defaultCParameters[tableID][compressionLevel];
if (MEM_32bits()) { /* auto-correction, for 32-bits mode */
if (cp.windowLog > ZSTD_WINDOWLOG_MAX) cp.windowLog = ZSTD_WINDOWLOG_MAX;
lib/zstd_decompress.c
+3 -3
View File
@@ -149,7 +149,7 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
dctx->hufTableX4[0] = HufLog;
dctx->flagStaticTables = 0;
dctx->fParams.mml = MINMATCH; /* overwritten by frame but forces ZSTD_btopt to MINMATCH in block mode */
ZSTD_LOG_BLOCK("%p: ZSTD_decompressBegin searchLength=%d\n", dctx->base, dctx->params.searchLength);
ZSTD_LOG_BLOCK("%p: ZSTD_decompressBegin searchLength=%d\n", dctx->base, dctx->fParams.mml);
return 0;
}
@@ -843,7 +843,7 @@ static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
if (srcSize >= ZSTD_BLOCKSIZE_MAX) return ERROR(srcSize_wrong);
ZSTD_LOG_BLOCK("%p: ZSTD_decompressBlock_internal searchLength=%d\n", dctx->base, dctx->params.searchLength);
ZSTD_LOG_BLOCK("%p: ZSTD_decompressBlock_internal searchLength=%d\n", dctx->base, dctx->fParams.mml);
/* Decode literals sub-block */
{ size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize);
@@ -951,7 +951,7 @@ size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
const void* dict, size_t dictSize)
{
ZSTD_decompressBegin_usingDict(dctx, dict, dictSize);
ZSTD_LOG_BLOCK("%p: ZSTD_decompressBegin_usingDict searchLength=%d\n", dctx->base, dctx->params.searchLength);
ZSTD_LOG_BLOCK("%p: ZSTD_decompressBegin_usingDict searchLength=%d\n", dctx->base, dctx->fParams.mml);
ZSTD_checkContinuity(dctx, dst);
return ZSTD_decompressFrame(dctx, dst, dstCapacity, src, srcSize);
}
lib/zstd_internal.h
+14 -12
View File
@@ -50,10 +50,8 @@
/*-*************************************
* Common constants
***************************************/
#define ZSTD_OPT_DEBUG 0 // 1 = tableID=0; 3 = price func tests; 5 = check encoded sequences; 9 = full logs
#if defined(ZSTD_OPT_DEBUG) && ZSTD_OPT_DEBUG>0
#include <stdio.h>
#endif
#define ZSTD_OPT_DEBUG 0 // 3 = compression stats; 5 = check encoded sequences; 9 = full logs
#include <stdio.h>
#if defined(ZSTD_OPT_DEBUG) && ZSTD_OPT_DEBUG>=9
#define ZSTD_LOG_PARSER(...) printf(__VA_ARGS__)
#define ZSTD_LOG_ENCODE(...) printf(__VA_ARGS__)
@@ -99,7 +97,6 @@ typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
#define MINMATCH 4
#define REPCODE_STARTVALUE 1
#define HASHLOG3 17
#define Litbits 8
#define Offbits 5
@@ -195,6 +192,16 @@ typedef struct {
U32 rep2;
} ZSTD_optimal_t;
#if ZSTD_OPT_DEBUG == 3
#include ".debug/zstd_stats.h"
#else
typedef struct { U32 unused; } ZSTD_stats_t;
MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; };
MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { (void)stats; };
MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { (void)stats; };
MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; };
#endif
typedef struct {
void* buffer;
U32* offsetStart;
@@ -227,17 +234,12 @@ typedef struct {
U32 log2litSum;
U32 log2offCodeSum;
U32 factor;
#if ZSTD_OPT_DEBUG == 3
U32 realMatchSum;
U32 realLitSum;
U32 realSeqSum;
U32 realRepSum;
U32 priceFunc;
#endif
ZSTD_stats_t stats;
} seqStore_t;
const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx);
void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq);
size_t ZSTD_compressBegin_targetSrcSize(ZSTD_CCtx* zc, const void* dict, size_t dictSize, size_t targetSrcSize, int compressionLevel);
#endif /* ZSTD_CCOMMON_H_MODULE */
lib/zstd_opt.h
+57 -32
View File
@@ -111,9 +111,18 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* seqStorePtr, U32 litLength, co
price -= ZSTD_highbit(seqStorePtr->litFreq[literals[u]]+1);
/* literal Length */
price += ((litLength >= MaxLL)<<3) + ((litLength >= 255+MaxLL)<<4) + ((litLength>=(1<<15))<<3);
if (litLength >= MaxLL) litLength = MaxLL;
price += seqStorePtr->log2litLengthSum - ZSTD_highbit(seqStorePtr->litLengthFreq[litLength]+1);
{ static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7,
8, 9, 10, 11, 12, 13, 14, 15,
16, 16, 17, 17, 18, 18, 19, 19,
20, 20, 20, 20, 21, 21, 21, 21,
22, 22, 22, 22, 22, 22, 22, 22,
23, 23, 23, 23, 23, 23, 23, 23,
24, 24, 24, 24, 24, 24, 24, 24,
24, 24, 24, 24, 24, 24, 24, 24 };
const BYTE LL_deltaCode = 19;
const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit(litLength) + LL_deltaCode : LL_Code[litLength];
price += LL_bits[llCode] + seqStorePtr->log2litLengthSum - ZSTD_highbit(seqStorePtr->litLengthFreq[llCode]+1);
}
return price;
}
@@ -122,25 +131,24 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* seqStorePtr, U32 litLength, co
FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength)
{
/* offset */
BYTE offCode = offset ? (BYTE)ZSTD_highbit(offset+1) + 1 : 0;
U32 price = (offCode-1) + (!offCode) + seqStorePtr->log2offCodeSum - ZSTD_highbit(seqStorePtr->offCodeFreq[offCode]+1);
BYTE offCode = (BYTE)ZSTD_highbit(offset+1);
U32 price = offCode + seqStorePtr->log2offCodeSum - ZSTD_highbit(seqStorePtr->offCodeFreq[offCode]+1);
/* match Length */
price += ((matchLength >= MaxML)<<3) + ((matchLength >= 255+MaxML)<<4) + ((matchLength>=(1<<15))<<3);
if (matchLength >= MaxML) matchLength = MaxML;
price += ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + seqStorePtr->log2matchLengthSum - ZSTD_highbit(seqStorePtr->matchLengthFreq[matchLength]+1);
#if ZSTD_OPT_DEBUG == 3
switch (seqStorePtr->priceFunc) {
default:
case 0:
return 1 + price + ((seqStorePtr->litSum>>5) / seqStorePtr->litLengthSum) + ((seqStorePtr->litSum<<1) / (seqStorePtr->litSum + seqStorePtr->matchSum));
case 1:
return 1 + price;
{ static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37,
38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39,
40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41,
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 };
const BYTE ML_deltaCode = 36;
const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit(matchLength) + ML_deltaCode : ML_Code[matchLength];
price += ML_bits[mlCode] + seqStorePtr->log2matchLengthSum - ZSTD_highbit(seqStorePtr->matchLengthFreq[mlCode]+1);
}
#else
return price + seqStorePtr->factor;
#endif
return price + ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + seqStorePtr->factor;
}
@@ -154,23 +162,39 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B
seqStorePtr->litFreq[literals[u]]++;
/* literal Length */
seqStorePtr->litLengthSum++;
if (litLength >= MaxLL)
seqStorePtr->litLengthFreq[MaxLL]++;
else
seqStorePtr->litLengthFreq[litLength]++;
{ static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7,
8, 9, 10, 11, 12, 13, 14, 15,
16, 16, 17, 17, 18, 18, 19, 19,
20, 20, 20, 20, 21, 21, 21, 21,
22, 22, 22, 22, 22, 22, 22, 22,
23, 23, 23, 23, 23, 23, 23, 23,
24, 24, 24, 24, 24, 24, 24, 24,
24, 24, 24, 24, 24, 24, 24, 24 };
const BYTE LL_deltaCode = 19;
const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit(litLength) + LL_deltaCode : LL_Code[litLength];
seqStorePtr->litLengthFreq[llCode]++;
seqStorePtr->litLengthSum++;
}
/* match offset */
seqStorePtr->offCodeSum++;
BYTE offCode = offset ? (BYTE)ZSTD_highbit(offset+1) + 1 : 0;
BYTE offCode = (BYTE)ZSTD_highbit(offset+1);
seqStorePtr->offCodeFreq[offCode]++;
/* match Length */
seqStorePtr->matchLengthSum++;
if (matchLength >= MaxML)
seqStorePtr->matchLengthFreq[MaxML]++;
else
seqStorePtr->matchLengthFreq[matchLength]++;
{ static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37,
38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39,
40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41,
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 };
const BYTE ML_deltaCode = 36;
const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit(matchLength) + ML_deltaCode : ML_Code[matchLength];
seqStorePtr->matchLengthFreq[mlCode]++;
seqStorePtr->matchLengthSum++;
}
ZSTD_setLog2Prices(seqStorePtr);
}
@@ -196,17 +220,18 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B
static U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_CCtx* zc, const BYTE* ip)
{
U32* const hashTable3 = zc->hashTable3;
U32 const hashLog3 = zc->hashLog3;
const BYTE* const base = zc->base;
const U32 target = (U32)(ip - base);
U32 idx = zc->nextToUpdate3;
while(idx < target) {
hashTable3[ZSTD_hash3Ptr(base+idx, HASHLOG3)] = idx;
hashTable3[ZSTD_hash3Ptr(base+idx, hashLog3)] = idx;
idx++;
}
zc->nextToUpdate3 = target;
return hashTable3[ZSTD_hash3Ptr(ip, HASHLOG3)];
return hashTable3[ZSTD_hash3Ptr(ip, hashLog3)];
}
lib/zstd_static.h
+2
View File
@@ -63,6 +63,8 @@ extern "C" {
#define ZSTD_CHAINLOG_MIN 4
#define ZSTD_HASHLOG_MAX ZSTD_WINDOWLOG_MAX
#define ZSTD_HASHLOG_MIN 12
#define ZSTD_HASHLOG3_MAX 17
#define ZSTD_HASHLOG3_MIN 15
#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1)
#define ZSTD_SEARCHLOG_MIN 1
#define ZSTD_SEARCHLENGTH_MAX 7