ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix and level tuning

Browse Source
This commit is contained in:
Yann Collet
2015-11-04 12:05:27 +01:00
parent 96b9f0ba4d
commit 59d7063fbc
4 changed files with 116 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/zstd_internal.h
+1 -1
View File
@@ -154,7 +154,7 @@ MEM_STATIC size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pI
return (size_t)(pIn - pStart); return (size_t)(pIn - pStart);
} }
if (MEM_32bits()) if ((pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; } if (MEM_64bits()) if ((pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; }
if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; } if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; }
if ((pIn<pInLimit) && (*pMatch == *pIn)) pIn++; if ((pIn<pInLimit) && (*pMatch == *pIn)) pIn++;
return (size_t)(pIn - pStart); return (size_t)(pIn - pStart);
lib/zstdhc.c
+56 -44
View File
@@ -107,20 +107,42 @@ size_t ZSTD_HC_freeCCtx(ZSTD_HC_CCtx* cctx)
return 0; return 0;
} }
/** ZSTD_HC_validateParams
correct params value to remain within authorized range
optimize for srcSize if srcSize > 0 */
void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, size_t srcSize)
{
const U32 chainplus = (params->strategy == ZSTD_HC_btlazy2);
/* validate params */
if (params->windowLog > ZSTD_HC_WINDOWLOG_MAX) params->windowLog = ZSTD_HC_WINDOWLOG_MAX;
if (params->windowLog < ZSTD_HC_WINDOWLOG_MIN) params->windowLog = ZSTD_HC_WINDOWLOG_MIN;
/* correct params, to use less memory */
if (srcSize > 0)
{
U32 srcLog = ZSTD_highbit((U32)srcSize-1) + 1;
if (params->windowLog > srcLog) params->windowLog = srcLog;
}
if (params->chainLog > params->windowLog + chainplus) params->chainLog = params->windowLog+chainplus; /* <= ZSTD_HC_CHAINLOG_MAX */
if (params->chainLog < ZSTD_HC_CHAINLOG_MIN) params->chainLog = ZSTD_HC_CHAINLOG_MIN;
if (params->hashLog > ZSTD_HC_HASHLOG_MAX) params->hashLog = ZSTD_HC_HASHLOG_MAX;
if (params->hashLog < ZSTD_HC_HASHLOG_MIN) params->hashLog = ZSTD_HC_HASHLOG_MIN;
if (params->searchLog > ZSTD_HC_SEARCHLOG_MAX) params->searchLog = ZSTD_HC_SEARCHLOG_MAX;
if (params->searchLog < ZSTD_HC_SEARCHLOG_MIN) params->searchLog = ZSTD_HC_SEARCHLOG_MIN;
if (params->searchLength> ZSTD_HC_SEARCHLENGTH_MAX) params->searchLength = ZSTD_HC_SEARCHLENGTH_MAX;
if (params->searchLength< ZSTD_HC_SEARCHLENGTH_MIN) params->searchLength = ZSTD_HC_SEARCHLENGTH_MIN;
if ((U32)params->strategy>(U32)ZSTD_HC_btlazy2) params->strategy = ZSTD_HC_btlazy2;
if ((int)params->strategy<(int)ZSTD_HC_greedy) params->strategy = ZSTD_HC_greedy;
}
static size_t ZSTD_HC_resetCCtx_advanced (ZSTD_HC_CCtx* zc, static size_t ZSTD_HC_resetCCtx_advanced (ZSTD_HC_CCtx* zc,
ZSTD_HC_parameters params) ZSTD_HC_parameters params)
{ {
/* validate params */ ZSTD_HC_validateParams(&params, 0);
if (params.windowLog > ZSTD_HC_WINDOWLOG_MAX) params.windowLog = ZSTD_HC_WINDOWLOG_MAX;
if (params.windowLog < ZSTD_HC_WINDOWLOG_MIN) params.windowLog = ZSTD_HC_WINDOWLOG_MIN;
if (params.chainLog > params.windowLog) params.chainLog = params.windowLog; /* <= ZSTD_HC_CHAINLOG_MAX */
if (params.chainLog < ZSTD_HC_CHAINLOG_MIN) params.chainLog = ZSTD_HC_CHAINLOG_MIN;
if (params.hashLog > ZSTD_HC_HASHLOG_MAX) params.hashLog = ZSTD_HC_HASHLOG_MAX;
if (params.hashLog < ZSTD_HC_HASHLOG_MIN) params.hashLog = ZSTD_HC_HASHLOG_MIN;
if (params.searchLog > ZSTD_HC_SEARCHLOG_MAX) params.searchLog = ZSTD_HC_SEARCHLOG_MAX;
if (params.searchLog < ZSTD_HC_SEARCHLOG_MIN) params.searchLog = ZSTD_HC_SEARCHLOG_MIN;
if (params.searchLength> ZSTD_HC_SEARCHLENGTH_MAX) params.searchLength = ZSTD_HC_SEARCHLENGTH_MAX;
if (params.searchLength< ZSTD_HC_SEARCHLENGTH_MIN) params.searchLength = ZSTD_HC_SEARCHLENGTH_MIN;
/* reserve table memory */ /* reserve table memory */
{ {
@@ -190,8 +212,6 @@ static size_t ZSTD_HC_hashPtr(const void* p, U32 hBits, U32 mls)
/* ************************************* /* *************************************
* Binary Tree search * Binary Tree search
***************************************/ ***************************************/
#define BT_SHORTCUT 256
/** ZSTD_HC_insertBt1 : add one ptr to tree /** ZSTD_HC_insertBt1 : add one ptr to tree
@ip : assumed <= iend-8 */ @ip : assumed <= iend-8 */
static void ZSTD_HC_insertBt1(ZSTD_HC_CCtx* zc, const BYTE* const ip, const U32 mls, const BYTE* const iend, U32 nbCompares) static void ZSTD_HC_insertBt1(ZSTD_HC_CCtx* zc, const BYTE* const ip, const U32 mls, const BYTE* const iend, U32 nbCompares)
@@ -891,21 +911,30 @@ size_t ZSTD_HC_compressBlock_greedy(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstS
} }
typedef size_t (*ZSTD_HC_blockCompressor) (ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
static ZSTD_HC_blockCompressor ZSTD_HC_selectBlockCompressor(ZSTD_HC_strategy strat)
{
switch(strat)
{
default :
case ZSTD_HC_greedy:
return ZSTD_HC_compressBlock_greedy;
case ZSTD_HC_lazy:
return ZSTD_HC_compressBlock_lazy;
case ZSTD_HC_lazydeep:
return ZSTD_HC_compressBlock_lazydeep;
case ZSTD_HC_btlazy2:
return ZSTD_HC_compressBlock_btLazy2;
}
}
size_t ZSTD_HC_compressBlock(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize) size_t ZSTD_HC_compressBlock(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
{ {
switch(ctx->params.strategy) ZSTD_HC_blockCompressor blockCompressor = ZSTD_HC_selectBlockCompressor(ctx->params.strategy);
{ return blockCompressor(ctx, dst, maxDstSize, src, srcSize);
case ZSTD_HC_greedy:
return ZSTD_HC_compressBlock_greedy(ctx, dst, maxDstSize, src, srcSize);
case ZSTD_HC_lazy:
return ZSTD_HC_compressBlock_lazy(ctx, dst, maxDstSize, src, srcSize);
case ZSTD_HC_lazydeep:
return ZSTD_HC_compressBlock_lazydeep(ctx, dst, maxDstSize, src, srcSize);
case ZSTD_HC_btlazy2:
return ZSTD_HC_compressBlock_btLazy2(ctx, dst, maxDstSize, src, srcSize);
default :
return ERROR(GENERIC); /* unknown block compressor */
}
} }
@@ -919,25 +948,8 @@ static size_t ZSTD_HC_compress_generic (ZSTD_HC_CCtx* ctxPtr,
BYTE* const ostart = (BYTE*)dst; BYTE* const ostart = (BYTE*)dst;
BYTE* op = ostart; BYTE* op = ostart;
BYTE* const oend = op + maxDstSize; BYTE* const oend = op + maxDstSize;
size_t (*blockCompressor) (ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); const ZSTD_HC_blockCompressor blockCompressor = ZSTD_HC_selectBlockCompressor(ctxPtr->params.strategy);
switch(ctxPtr->params.strategy)
{
case ZSTD_HC_greedy:
blockCompressor = ZSTD_HC_compressBlock_greedy;
break;
case ZSTD_HC_lazy:
blockCompressor = ZSTD_HC_compressBlock_lazy;
break;
case ZSTD_HC_lazydeep:
blockCompressor = ZSTD_HC_compressBlock_lazydeep;
break;
case ZSTD_HC_btlazy2:
blockCompressor = ZSTD_HC_compressBlock_btLazy2;
break;
default :
return ERROR(GENERIC); /* unknown block compressor */
}
while (remaining > blockSize) while (remaining > blockSize)
{ {
lib/zstdhc_static.h
+22 -18
View File
@@ -59,7 +59,7 @@ typedef struct
/* parameters boundaries */ /* parameters boundaries */
#define ZSTD_HC_WINDOWLOG_MAX 26 #define ZSTD_HC_WINDOWLOG_MAX 26
#define ZSTD_HC_WINDOWLOG_MIN 18 #define ZSTD_HC_WINDOWLOG_MIN 18
#define ZSTD_HC_CHAINLOG_MAX ZSTD_HC_WINDOWLOG_MAX #define ZSTD_HC_CHAINLOG_MAX (ZSTD_HC_WINDOWLOG_MAX+1)
#define ZSTD_HC_CHAINLOG_MIN 4 #define ZSTD_HC_CHAINLOG_MIN 4
#define ZSTD_HC_HASHLOG_MAX 28 #define ZSTD_HC_HASHLOG_MAX 28
#define ZSTD_HC_HASHLOG_MIN 4 #define ZSTD_HC_HASHLOG_MIN 4
@@ -79,6 +79,11 @@ size_t ZSTD_HC_compress_advanced (ZSTD_HC_CCtx* ctx,
const void* src, size_t srcSize, const void* src, size_t srcSize,
ZSTD_HC_parameters params); ZSTD_HC_parameters params);
/** ZSTD_HC_validateParams
correct params value to remain within authorized range
optimize for srcSize if srcSize > 0 */
void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, size_t srcSize);
/* ************************************* /* *************************************
* Streaming functions * Streaming functions
@@ -97,33 +102,32 @@ static const ZSTD_HC_parameters ZSTD_HC_defaultParameters[ZSTD_HC_MAX_CLEVEL+1]
{ 18, 12, 14, 1, 4, ZSTD_HC_greedy }, /* level 0 - never used */ { 18, 12, 14, 1, 4, ZSTD_HC_greedy }, /* level 0 - never used */
{ 18, 12, 14, 1, 4, ZSTD_HC_greedy }, /* level 1 - in fact redirected towards zstd fast */ { 18, 12, 14, 1, 4, ZSTD_HC_greedy }, /* level 1 - in fact redirected towards zstd fast */
{ 18, 12, 15, 2, 4, ZSTD_HC_greedy }, /* level 2 */ { 18, 12, 15, 2, 4, ZSTD_HC_greedy }, /* level 2 */
{ 19, 13, 17, 3, 5, ZSTD_HC_greedy }, /* level 3 */ { 19, 14, 18, 2, 5, ZSTD_HC_greedy }, /* level 3 */
{ 20, 18, 19, 2, 5, ZSTD_HC_greedy }, /* level 4 */ { 20, 17, 19, 3, 5, ZSTD_HC_greedy }, /* level 4 */
{ 20, 18, 19, 2, 5, ZSTD_HC_lazy }, /* level 5 */ { 20, 18, 19, 2, 5, ZSTD_HC_lazy }, /* level 5 */
{ 20, 18, 19, 2, 5, ZSTD_HC_lazydeep }, //{ 20, 18, 20, 3, 5, ZSTD_HC_lazy }, /* level 6 */ { 21, 18, 20, 3, 5, ZSTD_HC_lazy }, /* level 6 */
{ 20, 18, 19, 2, 5, ZSTD_HC_btlazy2 }, //{ 20, 18, 20, 4, 5, ZSTD_HC_lazy }, /* level 7 */ { 21, 18, 20, 4, 5, ZSTD_HC_lazy }, /* level 7 */
{ 21, 19, 20, 4, 5, ZSTD_HC_lazy }, /* level 8 */ { 21, 19, 20, 4, 5, ZSTD_HC_lazy }, /* level 8 */
{ 21, 19, 20, 5, 5, ZSTD_HC_lazy }, /* level 9 */ { 21, 19, 20, 5, 5, ZSTD_HC_lazy }, /* level 9 */
{ 21, 20, 20, 5, 5, ZSTD_HC_lazy }, /* level 10 */ { 21, 20, 20, 5, 5, ZSTD_HC_lazy }, /* level 10 */
{ 21, 20, 20, 5, 5, ZSTD_HC_lazydeep }, /* level 11 */ { 21, 20, 20, 5, 5, ZSTD_HC_lazydeep }, /* level 11 */
{ 21, 20, 20, 5, 5, ZSTD_HC_btlazy2 }, //{ 22, 20, 22, 5, 5, ZSTD_HC_lazydeep }, /* level 12 */ { 22, 20, 22, 5, 5, ZSTD_HC_lazydeep }, /* level 12 */
{ 22, 20, 22, 6, 5, ZSTD_HC_lazydeep }, /* level 13 */ { 22, 20, 22, 6, 5, ZSTD_HC_lazydeep }, /* level 13 */
{ 21, 21, 22, 6, 5, ZSTD_HC_lazydeep }, /* level 14 */ { 21, 21, 22, 6, 5, ZSTD_HC_lazydeep }, /* level 14 */
{ 22, 21, 22, 6, 5, ZSTD_HC_lazydeep }, /* level 15 */ { 22, 21, 22, 6, 5, ZSTD_HC_lazydeep }, /* level 15 */
{ 22, 21, 23, 7, 5, ZSTD_HC_lazydeep }, /* level 16 */ { 22, 21, 22, 5, 5, ZSTD_HC_btlazy2 }, /* level 16 */
{ 23, 21, 23, 7, 5, ZSTD_HC_lazydeep }, /* level 17 */ { 22, 22, 23, 5, 5, ZSTD_HC_btlazy2 }, /* level 17 */
{ 23, 22, 23, 7, 5, ZSTD_HC_lazydeep }, /* level 18 */ { 22, 22, 23, 7, 5, ZSTD_HC_btlazy2 }, /* level 18 */
{ 23, 22, 23, 7, 5, ZSTD_HC_lazydeep }, /* level 19 */ { 24, 24, 22, 7, 5, ZSTD_HC_btlazy2 }, /* level 19 */
{ 23, 22, 23, 8, 5, ZSTD_HC_lazydeep }, /* level 20 */ { 25, 25, 23, 8, 5, ZSTD_HC_btlazy2 }, /* level 20 */
{ 23, 22, 23, 8, 5, ZSTD_HC_lazydeep }, /* level 21 */ { 25, 25, 23, 8, 5, ZSTD_HC_btlazy2 }, /* level 21 */
{ 23, 23, 24, 8, 5, ZSTD_HC_lazydeep }, /* level 22 */ { 25, 25, 23, 8, 5, ZSTD_HC_btlazy2 }, /* level 22 */
{ 24, 24, 24, 8, 5, ZSTD_HC_lazydeep }, /* level 23 */ { 25, 25, 23, 8, 5, ZSTD_HC_btlazy2 }, /* level 23 */
{ 23, 23, 23, 9, 5, ZSTD_HC_lazydeep }, /* level 24 */ { 25, 25, 23, 8, 5, ZSTD_HC_btlazy2 }, /* level 24 */
{ 24, 23, 23, 9, 5, ZSTD_HC_lazydeep }, /* level 25 */ { 25, 25, 23, 8, 5, ZSTD_HC_btlazy2 }, /* level 25 */
{ 24, 24, 24, 9, 5, ZSTD_HC_lazydeep }, /* level 26 */ { 25, 25, 24, 9, 5, ZSTD_HC_btlazy2 }, /* level 26 */
}; };
#if defined (__cplusplus) #if defined (__cplusplus)
} }
#endif #endif
programs/paramgrill.c
+12 -17
View File
@@ -348,7 +348,7 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
if (totalTime > g_maxParamTime) break; if (totalTime > g_maxParamTime) break;
/* Compression */ /* Compression */
DISPLAY("%1u-%s : %9u ->\r", loopNb, name, (U32)srcSize); DISPLAY("\r%1u-%s : %9u ->", loopNb, name, (U32)srcSize);
memset(compressedBuffer, 0xE5, maxCompressedSize); memset(compressedBuffer, 0xE5, maxCompressedSize);
nbLoops = 0; nbLoops = 0;
@@ -371,8 +371,9 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
cSize += blockTable[blockNb].cSize; cSize += blockTable[blockNb].cSize;
if ((double)milliTime < fastestC*nbLoops) fastestC = (double)milliTime / nbLoops; if ((double)milliTime < fastestC*nbLoops) fastestC = (double)milliTime / nbLoops;
ratio = (double)srcSize / (double)cSize; ratio = (double)srcSize / (double)cSize;
DISPLAY("\r");
DISPLAY("%1u-%s : %9u ->", loopNb, name, (U32)srcSize); DISPLAY("%1u-%s : %9u ->", loopNb, name, (U32)srcSize);
DISPLAY(" %9u (%4.3f),%7.1f MB/s\r", (U32)cSize, ratio, (double)srcSize / fastestC / 1000.); DISPLAY(" %9u (%4.3f),%7.1f MB/s", (U32)cSize, ratio, (double)srcSize / fastestC / 1000.);
resultPtr->cSize = cSize; resultPtr->cSize = cSize;
resultPtr->cSpeed = (U32)((double)srcSize / fastestC); resultPtr->cSpeed = (U32)((double)srcSize / fastestC);
@@ -393,9 +394,10 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
milliTime = BMK_GetMilliSpan(milliTime); milliTime = BMK_GetMilliSpan(milliTime);
if ((double)milliTime < fastestD*nbLoops) fastestD = (double)milliTime / nbLoops; if ((double)milliTime < fastestD*nbLoops) fastestD = (double)milliTime / nbLoops;
DISPLAY("\r");
DISPLAY("%1u-%s : %9u -> ", loopNb, name, (U32)srcSize); DISPLAY("%1u-%s : %9u -> ", loopNb, name, (U32)srcSize);
DISPLAY("%9u (%4.3f),%7.1f MB/s, ", (U32)cSize, ratio, (double)srcSize / fastestC / 1000.); DISPLAY("%9u (%4.3f),%7.1f MB/s, ", (U32)cSize, ratio, (double)srcSize / fastestC / 1000.);
DISPLAY("%7.1f MB/s\r", (double)srcSize / fastestD / 1000.); DISPLAY("%7.1f MB/s", (double)srcSize / fastestD / 1000.);
resultPtr->dSpeed = (U32)((double)srcSize / fastestD); resultPtr->dSpeed = (U32)((double)srcSize / fastestD);
/* CRC Checking */ /* CRC Checking */
@@ -420,11 +422,13 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
} }
/* End cleaning */ /* End cleaning */
DISPLAY("\r");
free(compressedBuffer); free(compressedBuffer);
free(resultBuffer); free(resultBuffer);
return 0; return 0;
} }
const char* g_stratName[] = { "ZSTD_HC_greedy ", "ZSTD_HC_lazy ", "ZSTD_HC_lazydeep", "ZSTD_HC_btlazy2 " }; const char* g_stratName[] = { "ZSTD_HC_greedy ", "ZSTD_HC_lazy ", "ZSTD_HC_lazydeep", "ZSTD_HC_btlazy2 " };
static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_HC_parameters params, size_t srcSize) static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_HC_parameters params, size_t srcSize)
@@ -592,7 +596,6 @@ static void playAround(FILE* f, winnerInfo_t* winners,
const void* srcBuffer, size_t srcSize, const void* srcBuffer, size_t srcSize,
ZSTD_HC_CCtx* ctx) ZSTD_HC_CCtx* ctx)
{ {
const U32 srcLog = BMK_highbit((U32)( (g_blockSize ? g_blockSize : srcSize) -1))+1;
int nbVariations = 0; int nbVariations = 0;
const int startTime = BMK_GetMilliStart(); const int startTime = BMK_GetMilliStart();
@@ -635,19 +638,11 @@ static void playAround(FILE* f, winnerInfo_t* winners,
} }
/* validate new conf */ /* validate new conf */
if (p.windowLog > srcLog) continue; {
if (p.windowLog > ZSTD_HC_WINDOWLOG_MAX) continue; ZSTD_HC_parameters saved = p;
if (p.windowLog < MAX(ZSTD_HC_WINDOWLOG_MIN, p.chainLog)) continue; ZSTD_HC_validateParams(&p, g_blockSize ? g_blockSize : srcSize);
if (p.chainLog > p.windowLog) continue; if (memcmp(&p, &saved, sizeof(p))) continue; /* p was invalid */
if (p.chainLog < ZSTD_HC_CHAINLOG_MIN) continue; }
if (p.hashLog > ZSTD_HC_HASHLOG_MAX) continue;
if (p.hashLog < ZSTD_HC_HASHLOG_MIN) continue;
if (p.searchLog > p.chainLog) continue;
if (p.searchLog < ZSTD_HC_SEARCHLOG_MIN) continue;
if (p.searchLength > ZSTD_HC_SEARCHLENGTH_MAX) continue;
if (p.searchLength < ZSTD_HC_SEARCHLENGTH_MIN) continue;
if (p.strategy < ZSTD_HC_greedy) continue;
if (p.strategy > ZSTD_HC_btlazy2) continue;
/* exclude faster if already played params */ /* exclude faster if already played params */
if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(p))-1)) if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(p))-1))