ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

forceAttachDict

Browse Source
This commit is contained in:
George Lu
2018-08-14 14:24:41 -07:00
parent e3c679484a
commit 3f2d024dca
1 changed files with 397 additions and 349 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/paramgrill.c
+397 -349
View File
@@ -66,22 +66,10 @@ static const int g_maxNbVariations = 64;
#define CUSTOM_LEVEL 99 #define CUSTOM_LEVEL 99
#define BASE_CLEVEL 1 #define BASE_CLEVEL 1
/* indices for each of the variables */
typedef enum {
wlog_ind = 0,
clog_ind = 1,
hlog_ind = 2,
slog_ind = 3,
slen_ind = 4,
tlen_ind = 5,
strt_ind = 6
} varInds_t;
#define NUM_PARAMS 7
/* just don't use strategy as a param. */
#undef ZSTD_WINDOWLOG_MAX #undef ZSTD_WINDOWLOG_MAX
#define ZSTD_WINDOWLOG_MAX 27 //no long range stuff for now. #define ZSTD_WINDOWLOG_MAX 27 //no long range stuff for now.
#define FADT_MIN 0
#define FADT_MAX ((U32)-1)
#define ZSTD_TARGETLENGTH_MIN 0 #define ZSTD_TARGETLENGTH_MIN 0
#define ZSTD_TARGETLENGTH_MAX 999 #define ZSTD_TARGETLENGTH_MAX 999
@@ -93,13 +81,146 @@ typedef enum {
#define SLEN_RANGE (ZSTD_SEARCHLENGTH_MAX - ZSTD_SEARCHLENGTH_MIN + 1) #define SLEN_RANGE (ZSTD_SEARCHLENGTH_MAX - ZSTD_SEARCHLENGTH_MIN + 1)
#define TLEN_RANGE 17 #define TLEN_RANGE 17
#define STRT_RANGE (ZSTD_btultra - ZSTD_fast + 1) #define STRT_RANGE (ZSTD_btultra - ZSTD_fast + 1)
/* TLEN_RANGE picked manually */ #define FADT_RANGE 3
#define CHECKTIME(r) { if(BMK_timeSpan(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); return r; } } #define CHECKTIME(r) { if(BMK_timeSpan(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); return r; } }
#define CHECKTIMEGT(ret, val, _gototag) {if(BMK_timeSpan(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); ret = val; goto _gototag; } } #define CHECKTIMEGT(ret, val, _gototag) {if(BMK_timeSpan(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); ret = val; goto _gototag; } }
static const int rangetable[NUM_PARAMS] = { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE, STRT_RANGE }; #define PARAM_UNSET ((U32)-2) /* can't be -1 b/c fadt */
/*-************************************
* Setup for Adding new params
**************************************/
/* indices for each of the variables */
typedef enum {
wlog_ind = 0,
clog_ind = 1,
hlog_ind = 2,
slog_ind = 3,
slen_ind = 4,
tlen_ind = 5,
strt_ind = 6,
fadt_ind = 7, /* forceAttachDict */
NUM_PARAMS = 8
} varInds_t;
/* maximum value of parameters */
static const U32 mintable[NUM_PARAMS] =
{ ZSTD_WINDOWLOG_MIN, ZSTD_CHAINLOG_MIN, ZSTD_HASHLOG_MIN, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLENGTH_MIN, ZSTD_TARGETLENGTH_MIN, ZSTD_fast, FADT_MIN };
/* minimum value of parameters */
static const U32 maxtable[NUM_PARAMS] =
{ ZSTD_WINDOWLOG_MAX, ZSTD_CHAINLOG_MAX, ZSTD_HASHLOG_MAX, ZSTD_SEARCHLOG_MAX, ZSTD_SEARCHLENGTH_MAX, ZSTD_TARGETLENGTH_MAX, ZSTD_btultra, FADT_MAX };
/* # of values parameters can take on */
static const U32 rangetable[NUM_PARAMS] =
{ WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE, STRT_RANGE, FADT_RANGE };
static const ZSTD_cParameter cctxSetParamTable[NUM_PARAMS] =
{ ZSTD_p_windowLog, ZSTD_p_chainLog, ZSTD_p_hashLog, ZSTD_p_searchLog, ZSTD_p_minMatch, ZSTD_p_targetLength, ZSTD_p_compressionStrategy, ZSTD_p_forceAttachDict };
static const char* g_paramNames[NUM_PARAMS] =
{ "windowLog", "chainLog", "hashLog","searchLog", "searchLength", "targetLength", "strategy", "forceAttachDict"};
static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 }; static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 };
/* maps value from 0 to rangetable[param] - 1 to valid paramvalue */
static U32 rangeMap(varInds_t param, U32 ind) {
ind = MIN(ind, rangetable[param] - 1);
switch(param) {
case tlen_ind:
return tlen_table[ind];
case fadt_ind: /* 0, 1, 2 -> -1, 0, 1 */
return ind - 1;
case wlog_ind: /* using default: triggers -Wswitch-enum */
case clog_ind:
case hlog_ind:
case slog_ind:
case slen_ind:
case strt_ind:
return mintable[param] + ind;
case NUM_PARAMS:
return (U32)-1;
}
return 0; /* should never happen, stop compiler warnings */
}
/* inverse of rangeMap */
static U32 invRangeMap(varInds_t param, U32 value) {
value = MIN(MAX(mintable[param], value), maxtable[param]);
switch(param) {
case tlen_ind: /* bin search */
{
int lo = 0;
int hi = TLEN_RANGE;
while(lo < hi) {
int mid = (lo + hi) / 2;
if(tlen_table[mid] < value) {
lo = mid + 1;
} if(tlen_table[mid] == value) {
return mid;
} else {
hi = mid;
}
}
return lo;
}
case fadt_ind:
return value + 1;
case wlog_ind:
case clog_ind:
case hlog_ind:
case slog_ind:
case slen_ind:
case strt_ind:
return value - mintable[param];
case NUM_PARAMS:
return (U32)-1;
}
return 0; /* should never happen, stop compiler warnings */
}
typedef struct {
U32 vals[NUM_PARAMS];
} paramValues_t;
//TODO: unset -> 0?
static ZSTD_compressionParameters pvalsToCParams(paramValues_t p) {
ZSTD_compressionParameters c;
c.windowLog = p.vals[wlog_ind];
c.chainLog = p.vals[clog_ind];
c.hashLog = p.vals[hlog_ind];
c.searchLog = p.vals[slog_ind];
c.searchLength = p.vals[slen_ind];
c.targetLength = p.vals[tlen_ind];
c.strategy = p.vals[strt_ind];
/* no forceAttachDict */
return c;
}
/* 0 = auto for fadt */
static paramValues_t cParamsToPVals(ZSTD_compressionParameters c) {
paramValues_t p;
p.vals[wlog_ind] = c.windowLog;
p.vals[clog_ind] = c.chainLog;
p.vals[hlog_ind] = c.hashLog;
p.vals[slog_ind] = c.searchLog;
p.vals[slen_ind] = c.searchLength;
p.vals[tlen_ind] = c.targetLength;
p.vals[strt_ind] = c.strategy;
p.vals[fadt_ind] = 0;
return p;
}
/* equivalent of ZSTD_adjustCParams for paramValues_t */
static paramValues_t adjustParams(paramValues_t p, size_t maxBlockSize, size_t dictSize) {
U32 fval = p.vals[fadt_ind];
p = cParamsToPVals(ZSTD_adjustCParams(pvalsToCParams(p), maxBlockSize, dictSize));
p.vals[fadt_ind] = fval;
return p;
}
/*-************************************ /*-************************************
* Benchmark Parameters/Global Variables * Benchmark Parameters/Global Variables
**************************************/ **************************************/
@@ -115,15 +236,20 @@ static U32 g_singleRun = 0;
static U32 g_optimizer = 0; static U32 g_optimizer = 0;
static U32 g_target = 0; static U32 g_target = 0;
static U32 g_noSeed = 0; static U32 g_noSeed = 0;
static ZSTD_compressionParameters g_params; /* Initialized at the beginning of main w/ emptyParams() function */ static paramValues_t g_params; /* Initialized at the beginning of main w/ emptyParams() function */
static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */ static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */
typedef struct { typedef struct {
BMK_result_t result; BMK_result_t result;
ZSTD_compressionParameters params; paramValues_t params;
} winnerInfo_t; } winnerInfo_t;
typedef struct {
BMK_result_t result;
ZSTD_compressionParameters params;
} oldWinnerInfo_t;
typedef struct { typedef struct {
U32 cSpeed; /* bytes / sec */ U32 cSpeed; /* bytes / sec */
U32 dSpeed; U32 dSpeed;
@@ -242,8 +368,9 @@ static void findClockGranularity(void) {
DEBUGOUTPUT("Granularity: %llu\n", (unsigned long long)g_clockGranularity); DEBUGOUTPUT("Granularity: %llu\n", (unsigned long long)g_clockGranularity);
} }
/* allows zeros */
#define CLAMPCHECK(val,min,max) { \ #define CLAMPCHECK(val,min,max) { \
if (val && (((val)<(min)) | ((val)>(max)))) { \ if (((val)<(min)) | ((val)>(max))) { \
DISPLAY("INVALID PARAMETER CONSTRAINTS\n"); \ DISPLAY("INVALID PARAMETER CONSTRAINTS\n"); \
return 0; \ return 0; \
} } } }
@@ -251,36 +378,53 @@ static void findClockGranularity(void) {
/* Like ZSTD_checkCParams() but allows 0's */ /* Like ZSTD_checkCParams() but allows 0's */
/* no check on targetLen? */ /* no check on targetLen? */
static int cParamValid(ZSTD_compressionParameters paramTarget) { static int paramValid(paramValues_t paramTarget) {
CLAMPCHECK(paramTarget.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); U32 i;
CLAMPCHECK(paramTarget.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX); for(i = 0; i < NUM_PARAMS; i++) {
CLAMPCHECK(paramTarget.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX); CLAMPCHECK(paramTarget.vals[i], mintable[i], maxtable[i]);
CLAMPCHECK(paramTarget.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX);
CLAMPCHECK(paramTarget.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX);
if(paramTarget.targetLength > ZSTD_TARGETLENGTH_MAX) {
DISPLAY("INVALID PARAMETER CONSTRAINTS\n");
return 0;
}
if(paramTarget.strategy > ZSTD_btultra) {
DISPLAY("INVALID PARAMETER CONSTRAINTS\n");
return 0;
} }
//TODO: Strategy could be valid at 0 before, is that right?
return 1; return 1;
} }
static void cParamZeroMin(ZSTD_compressionParameters* paramTarget) { //TODO: doesn't affect strategy?
paramTarget->windowLog = paramTarget->windowLog ? paramTarget->windowLog : ZSTD_WINDOWLOG_MIN; static paramValues_t cParamUnsetMin(paramValues_t paramTarget) {
paramTarget->searchLog = paramTarget->searchLog ? paramTarget->searchLog : ZSTD_SEARCHLOG_MIN; varInds_t i;
paramTarget->chainLog = paramTarget->chainLog ? paramTarget->chainLog : ZSTD_CHAINLOG_MIN; for(i = 0; i < NUM_PARAMS; i++) {
paramTarget->hashLog = paramTarget->hashLog ? paramTarget->hashLog : ZSTD_HASHLOG_MIN; if(paramTarget.vals[i] == PARAM_UNSET) {
paramTarget->searchLength = paramTarget->searchLength ? paramTarget->searchLength : ZSTD_SEARCHLENGTH_MIN; paramTarget.vals[i] = mintable[i];
paramTarget->targetLength = paramTarget->targetLength ? paramTarget->targetLength : 0; }
}
return paramTarget;
} }
static void BMK_translateAdvancedParams(const ZSTD_compressionParameters params) static void BMK_translateAdvancedParams(FILE* f, const paramValues_t params) {
{ U32 i;
DISPLAY("--zstd=windowLog=%u,chainLog=%u,hashLog=%u,searchLog=%u,searchLength=%u,targetLength=%u,strategy=%u \n", fprintf(f,"--zstd=");
params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, params.targetLength, (U32)(params.strategy)); for(i = 0; i < NUM_PARAMS; i++) {
fprintf(f,"%s", g_paramNames[i]);
fprintf(f,"=%u", params.vals[i]);
if(i != NUM_PARAMS - 1) {
fprintf(f, ",");
}
}
fprintf(f, "\n");
}
static const char* g_stratName[ZSTD_btultra+1] = {
"(none) ", "ZSTD_fast ", "ZSTD_dfast ",
"ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ",
"ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "};
static void BMK_displayOneResult(FILE* f, winnerInfo_t res, size_t srcSize) {
res.params = cParamUnsetMin(res.params);
fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u,%3d, %s}, ",
res.params.vals[wlog_ind], res.params.vals[clog_ind], res.params.vals[hlog_ind], res.params.vals[slog_ind], res.params.vals[slen_ind],
res.params.vals[tlen_ind], (int)res.params.vals[fadt_ind], g_stratName[res.params.vals[strt_ind]]);
fprintf(f,
" /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
(double)srcSize / res.result.cSize, (double)res.result.cSpeed / (1 MB), (double)res.result.dSpeed / (1 MB));
} }
/* checks results are feasible */ /* checks results are feasible */
@@ -343,17 +487,16 @@ static constraint_t relaxTarget(constraint_t target) {
* Bench functions * Bench functions
*********************************************************/ *********************************************************/
const char* g_stratName[ZSTD_btultra+1] = { static paramValues_t emptyParams(void) {
"(none) ", "ZSTD_fast ", "ZSTD_dfast ", U32 i;
"ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ", paramValues_t p;
"ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "}; for(i = 0; i < NUM_PARAMS; i++) {
p.vals[i] = PARAM_UNSET;
static ZSTD_compressionParameters emptyParams(void) { }
ZSTD_compressionParameters p = { 0, 0, 0, 0, 0, 0, (ZSTD_strategy)0 };
return p; return p;
} }
static winnerInfo_t initWinnerInfo(ZSTD_compressionParameters p) { static winnerInfo_t initWinnerInfo(paramValues_t p) {
winnerInfo_t w1; winnerInfo_t w1;
w1.result.cSpeed = 0.; w1.result.cSpeed = 0.;
w1.result.dSpeed = 0.; w1.result.dSpeed = 0.;
@@ -389,27 +532,16 @@ typedef struct {
static void BMK_initCCtx(ZSTD_CCtx* ctx, static void BMK_initCCtx(ZSTD_CCtx* ctx,
const void* dictBuffer, const size_t dictBufferSize, const int cLevel, const void* dictBuffer, const size_t dictBufferSize, const int cLevel,
const ZSTD_compressionParameters* comprParams, const BMK_advancedParams_t* adv) { const paramValues_t* comprParams) {
varInds_t i;
ZSTD_CCtx_reset(ctx); ZSTD_CCtx_reset(ctx);
ZSTD_CCtx_resetParameters(ctx); ZSTD_CCtx_resetParameters(ctx);
if (adv->nbWorkers==1) {
ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, 0);
} else {
ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, adv->nbWorkers);
}
ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel); ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_enableLongDistanceMatching, adv->ldmFlag);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmMinMatch, adv->ldmMinMatch); for(i = 0; i < NUM_PARAMS; i++) {
ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashLog, adv->ldmHashLog); if(comprParams->vals[i] != PARAM_UNSET)
ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmBucketSizeLog, adv->ldmBucketSizeLog); ZSTD_CCtx_setParameter(ctx, cctxSetParamTable[i], comprParams->vals[i]);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashEveryLog, adv->ldmHashEveryLog); }
ZSTD_CCtx_setParameter(ctx, ZSTD_p_windowLog, comprParams->windowLog);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_hashLog, comprParams->hashLog);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_chainLog, comprParams->chainLog);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_searchLog, comprParams->searchLog);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_minMatch, comprParams->searchLength);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_targetLength, comprParams->targetLength);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy, comprParams->strategy);
ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize); ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize);
} }
@@ -424,13 +556,12 @@ typedef struct {
const void* dictBuffer; const void* dictBuffer;
size_t dictBufferSize; size_t dictBufferSize;
int cLevel; int cLevel;
const ZSTD_compressionParameters* comprParams; const paramValues_t* comprParams;
const BMK_advancedParams_t* adv;
} BMK_initCCtxArgs; } BMK_initCCtxArgs;
static size_t local_initCCtx(void* payload) { static size_t local_initCCtx(void* payload) {
const BMK_initCCtxArgs* ag = (const BMK_initCCtxArgs*)payload; const BMK_initCCtxArgs* ag = (const BMK_initCCtxArgs*)payload;
BMK_initCCtx(ag->ctx, ag->dictBuffer, ag->dictBufferSize, ag->cLevel, ag->comprParams, ag->adv); BMK_initCCtx(ag->ctx, ag->dictBuffer, ag->dictBufferSize, ag->cLevel, ag->comprParams);
return 0; return 0;
} }
@@ -507,10 +638,7 @@ static size_t local_defaultDecompress(
* From bench.c End * From bench.c End
*********************************************************/ *********************************************************/
static void freeBuffers(const buffers_t b) { static void freeNonSrcBuffers(const buffers_t b) {
if(b.srcPtrs != NULL) {
free(b.srcBuffer);
}
free(b.srcPtrs); free(b.srcPtrs);
free(b.srcSizes); free(b.srcSizes);
@@ -528,6 +656,13 @@ static void freeBuffers(const buffers_t b) {
free(b.resSizes); free(b.resSizes);
} }
static void freeBuffers(const buffers_t b) {
if(b.srcPtrs != NULL) {
free(b.srcBuffer);
}
freeNonSrcBuffers(b);
}
/* srcBuffer will be freed by freeBuffers now */ /* srcBuffer will be freed by freeBuffers now */
static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, size_t nbFiles, static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, size_t nbFiles,
const size_t* fileSizes) const size_t* fileSizes)
@@ -559,7 +694,7 @@ static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, size_t nbF
if(!buff->srcPtrs || !buff->srcSizes || !buff->dstPtrs || !buff->dstCapacities || !buff->dstSizes || !buff->resPtrs || !buff->resSizes) { if(!buff->srcPtrs || !buff->srcSizes || !buff->dstPtrs || !buff->dstCapacities || !buff->dstSizes || !buff->resPtrs || !buff->resSizes) {
DISPLAY("alloc error\n"); DISPLAY("alloc error\n");
freeBuffers(*buff); freeNonSrcBuffers(*buff);
return 1; return 1;
} }
@@ -570,7 +705,7 @@ static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, size_t nbF
if(!buff->dstPtrs[0] || !buff->resPtrs[0]) { if(!buff->dstPtrs[0] || !buff->resPtrs[0]) {
DISPLAY("alloc error\n"); DISPLAY("alloc error\n");
freeBuffers(*buff); freeNonSrcBuffers(*buff);
return 1; return 1;
} }
@@ -611,12 +746,20 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab
size_t totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles); size_t totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles);
size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad); size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad);
size_t* fileSizes = calloc(sizeof(size_t), nbFiles); size_t* fileSizes = calloc(sizeof(size_t), nbFiles);
void* srcBuffer = malloc(benchedSize); void* srcBuffer = NULL;
int ret = 0; int ret = 0;
if(!totalSizeToLoad || !benchedSize) {
ret = 1;
DISPLAY("Nothing to Bench\n");
goto _cleanUp;
}
srcBuffer = malloc(benchedSize);
if(!fileSizes || !srcBuffer) { if(!fileSizes || !srcBuffer) {
return 1; ret = 1;
goto _cleanUp;
} }
for(n = 0; n < nbFiles; n++) { for(n = 0; n < nbFiles; n++) {
@@ -633,10 +776,9 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab
f = fopen(fileNamesTable[n], "rb"); f = fopen(fileNamesTable[n], "rb");
if (f==NULL) { if (f==NULL) {
DISPLAY("impossible to open file %s\n", fileNamesTable[n]); DISPLAY("impossible to open file %s\n", fileNamesTable[n]);
free(fileSizes);
free(srcBuffer);
fclose(f); fclose(f);
return 10; ret = 10;
goto _cleanUp;
} }
DISPLAY("Loading %s... \r", fileNamesTable[n]); DISPLAY("Loading %s... \r", fileNamesTable[n]);
@@ -645,21 +787,22 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab
{ {
char* buffer = (char*)(srcBuffer); char* buffer = (char*)(srcBuffer);
size_t const readSize = fread((buffer)+pos, 1, (size_t)fileSize, f); size_t const readSize = fread((buffer)+pos, 1, (size_t)fileSize, f);
fclose(f);
if (readSize != (size_t)fileSize) { /* should we accept partial read? */ if (readSize != (size_t)fileSize) { /* should we accept partial read? */
DISPLAY("could not read %s", fileNamesTable[n]); DISPLAY("could not read %s", fileNamesTable[n]);
free(fileSizes); ret = 1;
free(srcBuffer); goto _cleanUp;
return 1;
} }
fileSizes[n] = readSize; fileSizes[n] = readSize;
pos += readSize; pos += readSize;
} }
fclose(f);
} }
ret = createBuffersFromMemory(buff, srcBuffer, nbFiles, fileSizes); ret = createBuffersFromMemory(buff, srcBuffer, nbFiles, fileSizes);
_cleanUp:
if(ret) { free(srcBuffer); }
free(fileSizes); free(fileSizes);
return ret; return ret;
} }
@@ -717,7 +860,7 @@ static int createContexts(contexts_t* ctx, const char* dictFileName) {
/* if in decodeOnly, then srcPtr's will be compressed blocks, and uncompressedBlocks will be written to dstPtrs? */ /* if in decodeOnly, then srcPtr's will be compressed blocks, and uncompressedBlocks will be written to dstPtrs? */
/* dictionary nullable, nothing else though. */ /* dictionary nullable, nothing else though. */
static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t ctx, static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t ctx,
const int cLevel, const ZSTD_compressionParameters* comprParams, const int cLevel, const paramValues_t* comprParams,
const BMK_mode_t mode, const BMK_loopMode_t loopMode, const unsigned nbSeconds) { const BMK_mode_t mode, const BMK_loopMode_t loopMode, const unsigned nbSeconds) {
U32 i; U32 i;
@@ -736,11 +879,6 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t
ZSTD_CCtx* cctx = ctx.cctx; ZSTD_CCtx* cctx = ctx.cctx;
ZSTD_DCtx* dctx = ctx.dctx; ZSTD_DCtx* dctx = ctx.dctx;
BMK_advancedParams_t adv = BMK_initAdvancedParams();
adv.mode = mode;
adv.loopMode = loopMode;
adv.nbSeconds = nbSeconds;
/* warmimg up memory */ /* warmimg up memory */
/* can't do this if decode only */ /* can't do this if decode only */
for(i = 0; i < buf.nbBlocks; i++) { for(i = 0; i < buf.nbBlocks; i++) {
@@ -762,7 +900,6 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t
cctxprep.dictBufferSize = dictBufferSize; cctxprep.dictBufferSize = dictBufferSize;
cctxprep.cLevel = cLevel; cctxprep.cLevel = cLevel;
cctxprep.comprParams = comprParams; cctxprep.comprParams = comprParams;
cctxprep.adv = &adv;
dctxprep.dctx = dctx; dctxprep.dctx = dctx;
dctxprep.dictBuffer = dictBuffer; dctxprep.dictBuffer = dictBuffer;
dctxprep.dictBufferSize = dictBufferSize; dctxprep.dictBufferSize = dictBufferSize;
@@ -852,13 +989,13 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t
} }
} }
/* Bench */ /* Bench */
results.result.cMem = (1 << (comprParams->windowLog)) + ZSTD_sizeof_CCtx(cctx); results.result.cMem = (1 << (comprParams->vals[wlog_ind])) + ZSTD_sizeof_CCtx(cctx);
return results; return results;
} }
static int BMK_benchParam(BMK_result_t* resultPtr, static int BMK_benchParam(BMK_result_t* resultPtr,
buffers_t buf, contexts_t ctx, buffers_t buf, contexts_t ctx,
const ZSTD_compressionParameters cParams) { const paramValues_t cParams) {
BMK_return_t res = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_both, BMK_timeMode, 3); BMK_return_t res = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_both, BMK_timeMode, 3);
*resultPtr = res.result; *resultPtr = res.result;
return res.error; return res.error;
@@ -986,23 +1123,21 @@ static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) {
/* Writes to f the results of a parameter benchmark */ /* Writes to f the results of a parameter benchmark */
/* when used with --optimize, will only print results better than previously discovered */ /* when used with --optimize, will only print results better than previously discovered */
static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const size_t srcSize) static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const paramValues_t params, const size_t srcSize)
{ {
char lvlstr[15] = "Custom Level"; char lvlstr[15] = "Custom Level";
winnerInfo_t w;
w.params = params;
w.result = result;
fprintf(f, "\r%79s\r", ""); fprintf(f, "\r%79s\r", "");
fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ",
params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength,
params.targetLength, g_stratName[(U32)(params.strategy)]);
if(cLevel != CUSTOM_LEVEL) { if(cLevel != CUSTOM_LEVEL) {
snprintf(lvlstr, 15, " Level %2u ", cLevel); snprintf(lvlstr, 15, " Level %2u ", cLevel);
} }
fprintf(f, fprintf(f, "/* %s */ ", lvlstr);
"/* %s */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */", BMK_displayOneResult(f, w, srcSize);
lvlstr, (double)srcSize / result.cSize, (double)result.cSpeed / (1 MB), (double)result.dSpeed / (1 MB));
if(TIMED) { if(TIMED) {
const U64 time = UTIL_clockSpanNano(g_time); const U64 time = UTIL_clockSpanNano(g_time);
@@ -1012,11 +1147,10 @@ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result
fprintf(f, "\n"); fprintf(f, "\n");
} }
static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const constraint_t targetConstraints, const size_t srcSize) static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t result, const paramValues_t params, const constraint_t targetConstraints, const size_t srcSize)
{ {
/* global winner used for constraints */ /* global winner used for constraints */
static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; static winnerInfo_t g_winner = { { (size_t)-1LL, 0, 0, (size_t)-1LL }, { { PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET } } };
if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize)) {
if(DEBUG && compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { if(DEBUG && compareResultLT(g_winner.result, result, targetConstraints, srcSize)) {
DISPLAY("New Winner: \n"); DISPLAY("New Winner: \n");
@@ -1025,7 +1159,7 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
BMK_printWinner(f, cLevel, result, params, srcSize); BMK_printWinner(f, cLevel, result, params, srcSize);
if(compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { if(compareResultLT(g_winner.result, result, targetConstraints, srcSize)) {
BMK_translateAdvancedParams(params); BMK_translateAdvancedParams(f, params);
g_winner.result = result; g_winner.result = result;
g_winner.params = params; g_winner.params = params;
} }
@@ -1046,13 +1180,7 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
fprintf(f, "================================\n"); fprintf(f, "================================\n");
for(n = g_winners; n != NULL; n = n->next) { for(n = g_winners; n != NULL; n = n->next) {
fprintf(f, "\r%79s\r", ""); fprintf(f, "\r%79s\r", "");
BMK_displayOneResult(f, n->res, srcSize);
fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ",
n->res.params.windowLog, n->res.params.chainLog, n->res.params.hashLog, n->res.params.searchLog, n->res.params.searchLength,
n->res.params.targetLength, g_stratName[(U32)(n->res.params.strategy)]);
fprintf(f,
" /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
(double)srcSize / n->res.result.cSize, (double)n->res.result.cSpeed / (1 MB), (double)n->res.result.dSpeed / (1 MB));
} }
fprintf(f, "================================\n"); fprintf(f, "================================\n");
fprintf(f, "Level Bounds: R: > %.3f AND C: < %.1f MB/s \n\n", fprintf(f, "Level Bounds: R: > %.3f AND C: < %.1f MB/s \n\n",
@@ -1060,27 +1188,15 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
fprintf(f, "Overall Winner: \n"); fprintf(f, "Overall Winner: \n");
fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", BMK_displayOneResult(f, g_winner, srcSize);
g_winner.params.windowLog, g_winner.params.chainLog, g_winner.params.hashLog, g_winner.params.searchLog, g_winner.params.searchLength, BMK_translateAdvancedParams(f, g_winner.params);
g_winner.params.targetLength, g_stratName[(U32)(g_winner.params.strategy)]);
fprintf(f,
" /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
(double)srcSize / g_winner.result.cSize, (double)g_winner.result.cSpeed / (1 MB), (double)g_winner.result.dSpeed / (1 MB));
BMK_translateAdvancedParams(g_winner.params);
fprintf(f, "Latest BMK: \n");
fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ",
params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength,
params.targetLength, g_stratName[(U32)(params.strategy)]);
fprintf(f,
" /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
(double)srcSize / result.cSize, (double)result.cSpeed / (1 MB), (double)result.dSpeed / (1 MB));
fprintf(f, "Latest BMK: \n");\
BMK_displayOneResult(f, w, srcSize);
} }
} }
static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize) static void BMK_printWinners2(FILE* f, const oldWinnerInfo_t* winners, size_t srcSize)
{ {
int cLevel; int cLevel;
@@ -1088,11 +1204,11 @@ static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSi
fprintf(f, " /* W, C, H, S, L, T, strat */ \n"); fprintf(f, " /* W, C, H, S, L, T, strat */ \n");
for (cLevel=0; cLevel <= NB_LEVELS_TRACKED; cLevel++) for (cLevel=0; cLevel <= NB_LEVELS_TRACKED; cLevel++)
BMK_printWinner(f, cLevel, winners[cLevel].result, winners[cLevel].params, srcSize); BMK_printWinner(f, cLevel, winners[cLevel].result, cParamsToPVals(winners[cLevel].params), srcSize);
} }
static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, size_t srcSize) static void BMK_printWinners(FILE* f, const oldWinnerInfo_t* winners, size_t srcSize)
{ {
fseek(f, 0, SEEK_SET); fseek(f, 0, SEEK_SET);
BMK_printWinners2(f, winners, srcSize); BMK_printWinners2(f, winners, srcSize);
@@ -1129,14 +1245,14 @@ static void BMK_init_level_constraints(int bytePerSec_level1)
} } } }
} }
static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters params, static int BMK_seed(oldWinnerInfo_t* winners, const ZSTD_compressionParameters params,
buffers_t buf, contexts_t ctx) buffers_t buf, contexts_t ctx)
{ {
BMK_result_t testResult; BMK_result_t testResult;
int better = 0; int better = 0;
int cLevel; int cLevel;
BMK_benchParam(&testResult, buf, ctx, params); BMK_benchParam(&testResult, buf, ctx, cParamsToPVals(params));
for (cLevel = 1; cLevel <= NB_LEVELS_TRACKED; cLevel++) { for (cLevel = 1; cLevel <= NB_LEVELS_TRACKED; cLevel++) {
@@ -1152,7 +1268,7 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para
/* first solution for this cLevel */ /* first solution for this cLevel */
winners[cLevel].result = testResult; winners[cLevel].result = testResult;
winners[cLevel].params = params; winners[cLevel].params = params;
BMK_printWinner(stdout, cLevel, testResult, params, buf.srcSize); BMK_printWinner(stdout, cLevel, testResult, cParamsToPVals(params), buf.srcSize);
better = 1; better = 1;
continue; continue;
} }
@@ -1217,7 +1333,7 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para
winners[cLevel].result = testResult; winners[cLevel].result = testResult;
winners[cLevel].params = params; winners[cLevel].params = params;
BMK_printWinner(stdout, cLevel, testResult, params, buf.srcSize); BMK_printWinner(stdout, cLevel, testResult, cParamsToPVals(params), buf.srcSize);
better = 1; better = 1;
} } } }
@@ -1234,20 +1350,21 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para
/* no point in windowLog < chainLog (no point 2x chainLog for bt) */ /* no point in windowLog < chainLog (no point 2x chainLog for bt) */
/* now with built in bounds-checking */ /* now with built in bounds-checking */
/* no longer does anything with sanitizeVarArray + clampcheck */ /* no longer does anything with sanitizeVarArray + clampcheck */
static ZSTD_compressionParameters sanitizeParams(ZSTD_compressionParameters params) static paramValues_t sanitizeParams(paramValues_t params)
{ {
if (params.strategy == ZSTD_fast) if (params.vals[strt_ind] == ZSTD_fast)
params.chainLog = 0, params.searchLog = 0; params.vals[clog_ind] = 0, params.vals[slog_ind] = 0;
if (params.strategy == ZSTD_dfast) if (params.vals[strt_ind] == ZSTD_dfast)
params.searchLog = 0; params.vals[slog_ind] = 0;
if (params.strategy != ZSTD_btopt && params.strategy != ZSTD_btultra && params.strategy != ZSTD_fast) if (params.vals[strt_ind] != ZSTD_btopt && params.vals[strt_ind] != ZSTD_btultra && params.vals[strt_ind] != ZSTD_fast)
params.targetLength = 0; params.vals[tlen_ind] = 0;
return params; return params;
} }
/* new length */ /* return: new length */
/* keep old array, will need if iter over strategy. */ /* keep old array, will need if iter over strategy. */
/* prunes useless params */
static int sanitizeVarArray(varInds_t* varNew, const int varLength, const varInds_t* varArray, const ZSTD_strategy strat) { static int sanitizeVarArray(varInds_t* varNew, const int varLength, const varInds_t* varArray, const ZSTD_strategy strat) {
int i, j = 0; int i, j = 0;
for(i = 0; i < varLength; i++) { for(i = 0; i < varLength; i++) {
@@ -1264,80 +1381,31 @@ static int sanitizeVarArray(varInds_t* varNew, const int varLength, const varInd
} }
/* res should be NUM_PARAMS size */ /* res should be NUM_PARAMS size */
/* constructs varArray from ZSTD_compressionParameters style parameter */ /* constructs varArray from paramValues_t style parameter */
static int variableParams(const ZSTD_compressionParameters paramConstraints, varInds_t* res) { /* pass in using dict. */
static int variableParams(const paramValues_t paramConstraints, varInds_t* res, const int usingDictionary) {
varInds_t i;
int j = 0; int j = 0;
if(!paramConstraints.windowLog) { for(i = 0; i < NUM_PARAMS; i++) {
res[j] = wlog_ind; if(paramConstraints.vals[i] == PARAM_UNSET) {
j++; if(i == fadt_ind && !usingDictionary) continue; /* don't use fadt if no dictionary */
} res[j] = i; j++;
if(!paramConstraints.chainLog) {
res[j] = clog_ind;
j++;
}
if(!paramConstraints.hashLog) {
res[j] = hlog_ind;
j++;
}
if(!paramConstraints.searchLog) {
res[j] = slog_ind;
j++;
}
if(!paramConstraints.searchLength) {
res[j] = slen_ind;
j++;
}
if(!paramConstraints.targetLength) {
res[j] = tlen_ind;
j++;
}
if(!paramConstraints.strategy) {
res[j] = strt_ind;
j++;
}
return j;
}
/* bin-search on tlen_table for correct index. */
static int tlen_inv(U32 x) {
int lo = 0;
int hi = TLEN_RANGE;
while(lo < hi) {
int mid = (lo + hi) / 2;
if(tlen_table[mid] < x) {
lo = mid + 1;
} if(tlen_table[mid] == x) {
return mid;
} else {
hi = mid;
} }
} }
return lo; return j;
} }
/* amt will probably always be \pm 1? */ /* amt will probably always be \pm 1? */
/* slight change from old paramVariation, targetLength can only take on powers of 2 now (999 ~= 1024?) */ /* slight change from old paramVariation, targetLength can only take on powers of 2 now (999 ~= 1024?) */
/* take max/min bounds into account as well? */ /* take max/min bounds into account as well? */
static void paramVaryOnce(const varInds_t paramIndex, const int amt, ZSTD_compressionParameters* ptr) { static void paramVaryOnce(const varInds_t paramIndex, const int amt, paramValues_t* ptr) {
switch(paramIndex) ptr->vals[paramIndex] = rangeMap(paramIndex, invRangeMap(paramIndex, ptr->vals[paramIndex]) + amt); //TODO: bounds check.
{
case wlog_ind: ptr->windowLog += amt; break;
case clog_ind: ptr->chainLog += amt; break;
case hlog_ind: ptr->hashLog += amt; break;
case slog_ind: ptr->searchLog += amt; break;
case slen_ind: ptr->searchLength += amt; break;
case tlen_ind:
ptr->targetLength = tlen_table[MAX(0, MIN(TLEN_RANGE - 1, tlen_inv(ptr->targetLength) + amt))];
break;
case strt_ind: ptr->strategy += amt; break;
default: break;
}
} }
/* varies ptr by nbChanges respecting varyParams*/ /* varies ptr by nbChanges respecting varyParams*/
static void paramVariation(ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen, const U32 nbChanges) static void paramVariation(paramValues_t* ptr, const varInds_t* varyParams, const int varyLen, const U32 nbChanges)
{ {
ZSTD_compressionParameters p; paramValues_t p;
U32 validated = 0; U32 validated = 0;
while (!validated) { while (!validated) {
U32 i; U32 i;
@@ -1346,7 +1414,7 @@ static void paramVariation(ZSTD_compressionParameters* ptr, const varInds_t* var
const U32 changeID = FUZ_rand(&g_rand) % (varyLen << 1); const U32 changeID = FUZ_rand(&g_rand) % (varyLen << 1);
paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p); paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p);
} }
validated = !ZSTD_isError(ZSTD_checkCParams(p)) && p.strategy > 0; validated = paramValid(p);
} }
*ptr = p; *ptr = p;
} }
@@ -1364,48 +1432,25 @@ static size_t memoTableLen(const varInds_t* varyParams, const int varyLen) {
} }
/* returns unique index in memotable of compression parameters */ /* returns unique index in memotable of compression parameters */
static unsigned memoTableInd(const ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen) { static unsigned memoTableInd(const paramValues_t* ptr, const varInds_t* varyParams, const int varyLen) {
int i; int i;
unsigned ind = 0; unsigned ind = 0;
for(i = 0; i < varyLen; i++) { for(i = 0; i < varyLen; i++) {
switch(varyParams[i]) { varInds_t v = varyParams[i];
case wlog_ind: ind *= WLOG_RANGE; ind += ptr->windowLog if(v == strt_ind) continue; /* exclude strategy from memotable */
- ZSTD_WINDOWLOG_MIN ; break; ind *= rangetable[v]; ind += invRangeMap(v, ptr->vals[v]);
case clog_ind: ind *= CLOG_RANGE; ind += ptr->chainLog
- ZSTD_CHAINLOG_MIN ; break;
case hlog_ind: ind *= HLOG_RANGE; ind += ptr->hashLog
- ZSTD_HASHLOG_MIN ; break;
case slog_ind: ind *= SLOG_RANGE; ind += ptr->searchLog
- ZSTD_SEARCHLOG_MIN ; break;
case slen_ind: ind *= SLEN_RANGE; ind += ptr->searchLength
- ZSTD_SEARCHLENGTH_MIN; break;
case tlen_ind: ind *= TLEN_RANGE; ind += tlen_inv(ptr->targetLength)
- ZSTD_TARGETLENGTH_MIN; break;
case strt_ind: break;
}
} }
return ind; return ind;
} }
/* inverse of above function (from index to parameters) */ /* inverse of above function (from index to parameters) */
static void memoTableIndInv(ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen, size_t ind) { static void memoTableIndInv(paramValues_t* ptr, const varInds_t* varyParams, const int varyLen, size_t ind) {
int i; int i;
for(i = varyLen - 1; i >= 0; i--) { for(i = varyLen - 1; i >= 0; i--) {
switch(varyParams[i]) { varInds_t v = varyParams[i];
case wlog_ind: ptr->windowLog = ind % WLOG_RANGE + ZSTD_WINDOWLOG_MIN; if(v == strt_ind) continue;
ind /= WLOG_RANGE; break; ptr->vals[v] = rangeMap(v, ind % rangetable[v]);
case clog_ind: ptr->chainLog = ind % CLOG_RANGE + ZSTD_CHAINLOG_MIN; ind /= rangetable[v];
ind /= CLOG_RANGE; break;
case hlog_ind: ptr->hashLog = ind % HLOG_RANGE + ZSTD_HASHLOG_MIN;
ind /= HLOG_RANGE; break;
case slog_ind: ptr->searchLog = ind % SLOG_RANGE + ZSTD_SEARCHLOG_MIN;
ind /= SLOG_RANGE; break;
case slen_ind: ptr->searchLength = ind % SLEN_RANGE + ZSTD_SEARCHLENGTH_MIN;
ind /= SLEN_RANGE; break;
case tlen_ind: ptr->targetLength = tlen_table[(ind % TLEN_RANGE)];
ind /= TLEN_RANGE; break;
case strt_ind: break;
}
} }
} }
@@ -1414,36 +1459,36 @@ static void memoTableIndInv(ZSTD_compressionParameters* ptr, const varInds_t* va
* redundant / obviously non-optimal parameter configurations (e.g. wlog - 1 larger) * redundant / obviously non-optimal parameter configurations (e.g. wlog - 1 larger)
* than srcSize, clog > wlog, ... * than srcSize, clog > wlog, ...
*/ */
static void initMemoTable(U8* memoTable, ZSTD_compressionParameters paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { static void initMemoTable(U8* memoTable, paramValues_t paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) {
size_t i; size_t i;
size_t arrayLen = memoTableLen(varyParams, varyLen); size_t arrayLen = memoTableLen(varyParams, varyLen);
int cwFixed = !paramConstraints.chainLog || !paramConstraints.windowLog; int cwFixed = paramConstraints.vals[clog_ind] == PARAM_UNSET || paramConstraints.vals[wlog_ind] == PARAM_UNSET;
int scFixed = !paramConstraints.searchLog || !paramConstraints.chainLog; int scFixed = paramConstraints.vals[slog_ind] == PARAM_UNSET || paramConstraints.vals[clog_ind] == PARAM_UNSET;
int whFixed = !paramConstraints.windowLog || !paramConstraints.hashLog; int whFixed = paramConstraints.vals[wlog_ind] == PARAM_UNSET || paramConstraints.vals[hlog_ind] == PARAM_UNSET;
int wFixed = !paramConstraints.windowLog; int wFixed = paramConstraints.vals[wlog_ind] == PARAM_UNSET;
int j = 0; int j = 0;
assert(memoTable != NULL); assert(memoTable != NULL);
memset(memoTable, 0, arrayLen); memset(memoTable, 0, arrayLen);
cParamZeroMin(&paramConstraints); paramConstraints = cParamUnsetMin(paramConstraints);
for(i = 0; i < arrayLen; i++) { for(i = 0; i < arrayLen; i++) {
memoTableIndInv(&paramConstraints, varyParams, varyLen, i); memoTableIndInv(&paramConstraints, varyParams, varyLen, i);
if(ZSTD_estimateCStreamSize_usingCParams(paramConstraints) > (size_t)target.cMem) { if(ZSTD_estimateCStreamSize_usingCParams(pvalsToCParams(paramConstraints)) > (size_t)target.cMem) {
memoTable[i] = 255; memoTable[i] = 255;
j++; j++;
} }
if(wFixed && (1ULL << (paramConstraints.windowLog - 1)) > srcSize) { if(wFixed && (1ULL << (paramConstraints.vals[wlog_ind] - 1)) >= srcSize && paramConstraints.vals[wlog_ind] != mintable[wlog_ind]) {
memoTable[i] = 255; memoTable[i] = 255;
} }
/* nil out parameter sets equivalent to others. */ /* nil out parameter sets equivalent to others. */
if(cwFixed/* at most least 1 param fixed. */) { if(cwFixed) {
if(paramConstraints.strategy == ZSTD_btlazy2 || paramConstraints.strategy == ZSTD_btopt || paramConstraints.strategy == ZSTD_btultra) { if(paramConstraints.vals[strt_ind] == ZSTD_btlazy2 || paramConstraints.vals[strt_ind] == ZSTD_btopt || paramConstraints.vals[strt_ind] == ZSTD_btultra) {
if(paramConstraints.chainLog > paramConstraints.windowLog + 1) { if(paramConstraints.vals[clog_ind] > paramConstraints.vals[wlog_ind]+ 1) {
if(memoTable[i] != 255) { j++; } if(memoTable[i] != 255) { j++; }
memoTable[i] = 255; memoTable[i] = 255;
} }
} else { } else {
if(paramConstraints.chainLog > paramConstraints.windowLog) { if(paramConstraints.vals[clog_ind] > paramConstraints.vals[wlog_ind]) {
if(memoTable[i] != 255) { j++; } if(memoTable[i] != 255) { j++; }
memoTable[i] = 255; memoTable[i] = 255;
} }
@@ -1451,14 +1496,14 @@ static void initMemoTable(U8* memoTable, ZSTD_compressionParameters paramConstra
} }
if(scFixed) { if(scFixed) {
if(paramConstraints.searchLog > paramConstraints.chainLog) { if(paramConstraints.vals[slog_ind] > paramConstraints.vals[clog_ind]) {
if(memoTable[i] != 255) { j++; } if(memoTable[i] != 255) { j++; }
memoTable[i] = 255; memoTable[i] = 255;
} }
} }
if(whFixed) { if(whFixed) {
if(paramConstraints.hashLog > paramConstraints.windowLog + 1) { if(paramConstraints.vals[hlog_ind] > paramConstraints.vals[wlog_ind] + 1) {
if(memoTable[i] != 255) { j++; } if(memoTable[i] != 255) { j++; }
memoTable[i] = 255; memoTable[i] = 255;
} }
@@ -1466,7 +1511,7 @@ static void initMemoTable(U8* memoTable, ZSTD_compressionParameters paramConstra
} }
DEBUGOUTPUT("%d / %d Invalid\n", j, (int)i); DEBUGOUTPUT("%d / %d Invalid\n", j, (int)i);
if((int)i == j) { if((int)i == j) {
DEBUGOUTPUT("!!!Strategy %d totally infeasible\n", (int)paramConstraints.strategy) DEBUGOUTPUT("!!!Strategy %d totally infeasible\n", (int)paramConstraints.vals[strt_ind]);
} }
} }
@@ -1482,7 +1527,7 @@ static void freeMemoTableArray(U8** mtAll) {
/* inits memotables for all (including mallocs), all strategies */ /* inits memotables for all (including mallocs), all strategies */
/* takes unsanitized varyParams */ /* takes unsanitized varyParams */
static U8** createMemoTableArray(ZSTD_compressionParameters paramConstraints, constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { static U8** createMemoTableArray(paramValues_t paramConstraints, constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) {
varInds_t varNew[NUM_PARAMS]; varInds_t varNew[NUM_PARAMS];
U8** mtAll = (U8**)calloc(sizeof(U8*),(ZSTD_btultra + 1)); U8** mtAll = (U8**)calloc(sizeof(U8*),(ZSTD_btultra + 1));
int i; int i;
@@ -1503,14 +1548,13 @@ static U8** createMemoTableArray(ZSTD_compressionParameters paramConstraints, co
return mtAll; return mtAll;
} }
static ZSTD_compressionParameters overwriteParams(ZSTD_compressionParameters base, ZSTD_compressionParameters mask) { static paramValues_t overwriteParams(paramValues_t base, paramValues_t mask) {
base.windowLog = mask.windowLog ? mask.windowLog : base.windowLog; U32 i;
base.chainLog = mask.chainLog ? mask.chainLog : base.chainLog; for(i = 0; i < NUM_PARAMS; i++) {
base.hashLog = mask.hashLog ? mask.hashLog : base.hashLog; if(mask.vals[i] != PARAM_UNSET) {
base.searchLog = mask.searchLog ? mask.searchLog : base.searchLog; base.vals[i] = mask.vals[i];
base.searchLength = mask.searchLength ? mask.searchLength : base.searchLength; }
base.targetLength = mask.targetLength ? mask.targetLength : base.targetLength; }
base.strategy = mask.strategy ? mask.strategy : base.strategy;
return base; return base;
} }
@@ -1524,38 +1568,41 @@ static BYTE g_alreadyTested[PARAMTABLESIZE] = {0}; /* init to zero */
g_alreadyTested[(XXH64(((void*)&sanitizeParams(p), sizeof(p), 0) >> 3) & PARAMTABLEMASK] */ g_alreadyTested[(XXH64(((void*)&sanitizeParams(p), sizeof(p), 0) >> 3) & PARAMTABLEMASK] */
static BYTE* NB_TESTS_PLAYED(ZSTD_compressionParameters p) { static BYTE* NB_TESTS_PLAYED(ZSTD_compressionParameters p) {
ZSTD_compressionParameters p2 = sanitizeParams(p); ZSTD_compressionParameters p2 = pvalsToCParams(sanitizeParams(cParamsToPVals(p)));
return &g_alreadyTested[(XXH64((void*)&p2, sizeof(p2), 0) >> 3) & PARAMTABLEMASK]; return &g_alreadyTested[(XXH64((void*)&p2, sizeof(p2), 0) >> 3) & PARAMTABLEMASK];
} }
static void playAround(FILE* f, winnerInfo_t* winners, static void playAround(FILE* f, oldWinnerInfo_t* winners,
ZSTD_compressionParameters params, ZSTD_compressionParameters params,
buffers_t buf, contexts_t ctx) buffers_t buf, contexts_t ctx)
{ {
int nbVariations = 0; int nbVariations = 0;
UTIL_time_t const clockStart = UTIL_getTime(); UTIL_time_t const clockStart = UTIL_getTime();
const U32 unconstrained[NUM_PARAMS] = { 0, 1, 2, 3, 4, 5, 6 }; const U32 unconstrained[NUM_PARAMS] = { 0, 1, 2, 3, 4, 5, 6 }; /* no fadt */
while (UTIL_clockSpanMicro(clockStart) < g_maxVariationTime) { while (UTIL_clockSpanMicro(clockStart) < g_maxVariationTime) {
ZSTD_compressionParameters p = params; paramValues_t p = cParamsToPVals(params);
ZSTD_compressionParameters p2;
BYTE* b; BYTE* b;
if (nbVariations++ > g_maxNbVariations) break; if (nbVariations++ > g_maxNbVariations) break;
paramVariation(&p, unconstrained, NUM_PARAMS, 4); paramVariation(&p, unconstrained, NUM_PARAMS, 4);
p2 = pvalsToCParams(p);
/* exclude faster if already played params */ /* exclude faster if already played params */
if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(p))-1)) if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(p2))-1))
continue; continue;
/* test */ /* test */
b = NB_TESTS_PLAYED(p); b = NB_TESTS_PLAYED(p2);
(*b)++; (*b)++;
if (!BMK_seed(winners, p, buf, ctx)) continue; if (!BMK_seed(winners, p2, buf, ctx)) continue;
/* improvement found => search more */ /* improvement found => search more */
BMK_printWinners(f, winners, buf.srcSize); BMK_printWinners(f, winners, buf.srcSize);
playAround(f, winners, p, buf, ctx); playAround(f, winners, p2, buf, ctx);
} }
} }
@@ -1587,7 +1634,7 @@ static ZSTD_compressionParameters randomParams(void)
} }
/* Sets pc to random unmeasured set of parameters */ /* Sets pc to random unmeasured set of parameters */
static void randomConstrainedParams(ZSTD_compressionParameters* pc, varInds_t* varArray, int varLen, U8* memoTable) static void randomConstrainedParams(paramValues_t* pc, varInds_t* varArray, int varLen, U8* memoTable)
{ {
size_t tries = memoTableLen(varArray, varLen); size_t tries = memoTableLen(varArray, varLen);
const size_t maxSize = memoTableLen(varArray, varLen); const size_t maxSize = memoTableLen(varArray, varLen);
@@ -1601,7 +1648,7 @@ static void randomConstrainedParams(ZSTD_compressionParameters* pc, varInds_t* v
} }
static void BMK_selectRandomStart( static void BMK_selectRandomStart(
FILE* f, winnerInfo_t* winners, FILE* f, oldWinnerInfo_t* winners,
buffers_t buf, contexts_t ctx) buffers_t buf, contexts_t ctx)
{ {
U32 const id = FUZ_rand(&g_rand) % (NB_LEVELS_TRACKED+1); U32 const id = FUZ_rand(&g_rand) % (NB_LEVELS_TRACKED+1);
@@ -1617,7 +1664,7 @@ static void BMK_selectRandomStart(
static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBlockSize) static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBlockSize)
{ {
ZSTD_compressionParameters params; ZSTD_compressionParameters params;
winnerInfo_t winners[NB_LEVELS_TRACKED+1]; oldWinnerInfo_t winners[NB_LEVELS_TRACKED+1];
const char* const rfName = "grillResults.txt"; const char* const rfName = "grillResults.txt";
FILE* const f = fopen(rfName, "w"); FILE* const f = fopen(rfName, "w");
@@ -1632,7 +1679,7 @@ static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBl
/* baseline config for level 1 */ /* baseline config for level 1 */
ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, maxBlockSize, ctx.dictSize); ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, maxBlockSize, ctx.dictSize);
BMK_result_t testResult; BMK_result_t testResult;
BMK_benchParam(&testResult, buf, ctx, l1params); BMK_benchParam(&testResult, buf, ctx, cParamsToPVals(l1params));
BMK_init_level_constraints((int)((testResult.cSpeed * 31) / 32)); BMK_init_level_constraints((int)((testResult.cSpeed * 31) / 32));
} }
@@ -1751,7 +1798,7 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam
DISPLAY("using %d Files : \n", nbFiles); DISPLAY("using %d Files : \n", nbFiles);
} }
g_params = ZSTD_adjustCParams(overwriteParams(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize), g_params), maxBlockSize, ctx.dictSize); g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize)), g_params), maxBlockSize, ctx.dictSize);
if(g_singleRun) { if(g_singleRun) {
ret = benchOnce(buf, ctx); ret = benchOnce(buf, ctx);
@@ -1769,7 +1816,7 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam
#define HIGH_VARIANCE 100.0 #define HIGH_VARIANCE 100.0
static int allBench(BMK_result_t* resultPtr, static int allBench(BMK_result_t* resultPtr,
const buffers_t buf, const contexts_t ctx, const buffers_t buf, const contexts_t ctx,
const ZSTD_compressionParameters cParams, const paramValues_t cParams,
const constraint_t target, const constraint_t target,
BMK_result_t* winnerResult, int feas) { BMK_result_t* winnerResult, int feas) {
BMK_return_t benchres; BMK_return_t benchres;
@@ -1870,7 +1917,7 @@ static int allBench(BMK_result_t* resultPtr,
/* Memoized benchmarking, won't benchmark anything which has already been benchmarked before. */ /* Memoized benchmarking, won't benchmark anything which has already been benchmarked before. */
static int benchMemo(BMK_result_t* resultPtr, static int benchMemo(BMK_result_t* resultPtr,
const buffers_t buf, const contexts_t ctx, const buffers_t buf, const contexts_t ctx,
const ZSTD_compressionParameters cParams, const paramValues_t cParams,
const constraint_t target, const constraint_t target,
BMK_result_t* winnerResult, U8* const memoTable, BMK_result_t* winnerResult, U8* const memoTable,
const varInds_t* varyParams, const int varyLen, const int feas) { const varInds_t* varyParams, const int varyLen, const int feas) {
@@ -1914,13 +1961,13 @@ static winnerInfo_t climbOnce(const constraint_t target,
const varInds_t* varArray, const int varLen, ZSTD_strategy strat, const varInds_t* varArray, const int varLen, ZSTD_strategy strat,
U8** memoTableArray, U8** memoTableArray,
buffers_t buf, contexts_t ctx, buffers_t buf, contexts_t ctx,
const ZSTD_compressionParameters init) { const paramValues_t init) {
/* /*
* cparam - currently considered 'center' * cparam - currently considered 'center'
* candidate - params to benchmark/results * candidate - params to benchmark/results
* winner - best option found so far. * winner - best option found so far.
*/ */
ZSTD_compressionParameters cparam = init; paramValues_t cparam = init;
winnerInfo_t candidateInfo, winnerInfo; winnerInfo_t candidateInfo, winnerInfo;
int better = 1; int better = 1;
int feas = 0; int feas = 0;
@@ -1948,10 +1995,10 @@ static winnerInfo_t climbOnce(const constraint_t target,
candidateInfo.params = cparam; candidateInfo.params = cparam;
paramVaryOnce(varArray[i], offset, &candidateInfo.params); paramVaryOnce(varArray[i], offset, &candidateInfo.params);
if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params)) && candidateInfo.params.strategy > 0) { if(paramValid(candidateInfo.params)) {
int res; int res;
if(strat != candidateInfo.params.strategy) { /* maybe only try strategy switching after exhausting non-switching solutions? */ if(strat != candidateInfo.params.vals[strt_ind]) { /* maybe only try strategy switching after exhausting non-switching solutions? */
strat = candidateInfo.params.strategy; strat = candidateInfo.params.vals[strt_ind];
varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat); varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat);
} }
res = benchMemo(&candidateInfo.result, buf, ctx, res = benchMemo(&candidateInfo.result, buf, ctx,
@@ -1981,8 +2028,8 @@ static winnerInfo_t climbOnce(const constraint_t target,
/* param error checking already done here */ /* param error checking already done here */
paramVariation(&candidateInfo.params, varArray, varLen, dist); paramVariation(&candidateInfo.params, varArray, varLen, dist);
if(strat != candidateInfo.params.strategy) { if(strat != candidateInfo.params.vals[strt_ind]) {
strat = candidateInfo.params.strategy; strat = candidateInfo.params.vals[strt_ind];
varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat); varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat);
} }
@@ -2030,7 +2077,7 @@ static winnerInfo_t climbOnce(const constraint_t target,
*/ */
static winnerInfo_t optimizeFixedStrategy( static winnerInfo_t optimizeFixedStrategy(
const buffers_t buf, const contexts_t ctx, const buffers_t buf, const contexts_t ctx,
const constraint_t target, ZSTD_compressionParameters paramTarget, const constraint_t target, paramValues_t paramTarget,
const ZSTD_strategy strat, const ZSTD_strategy strat,
const varInds_t* varArray, const int varLen, const varInds_t* varArray, const int varLen,
U8** memoTableArray, const int tries) { U8** memoTableArray, const int tries) {
@@ -2038,14 +2085,14 @@ static winnerInfo_t optimizeFixedStrategy(
varInds_t varNew[NUM_PARAMS]; varInds_t varNew[NUM_PARAMS];
int varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat); int varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat);
ZSTD_compressionParameters init; paramValues_t init;
winnerInfo_t winnerInfo, candidateInfo; winnerInfo_t winnerInfo, candidateInfo;
winnerInfo = initWinnerInfo(emptyParams()); winnerInfo = initWinnerInfo(emptyParams());
/* so climb is given the right fixed strategy */ /* so climb is given the right fixed strategy */
paramTarget.strategy = strat; paramTarget.vals[strt_ind] = strat;
/* to pass ZSTD_checkCParams */ /* to pass ZSTD_checkCParams */
cParamZeroMin(&paramTarget); paramTarget = cParamUnsetMin(paramTarget);
init = paramTarget; init = paramTarget;
@@ -2112,25 +2159,23 @@ static int nextStrategy(const int currentStrategy, const int bestStrategy) {
static int g_maxTries = 3; static int g_maxTries = 3;
#define TRY_DECAY 1 #define TRY_DECAY 1
static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel) static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, paramValues_t paramTarget, int cLevelOpt, int cLevelRun)
{ {
varInds_t varArray [NUM_PARAMS]; varInds_t varArray [NUM_PARAMS];
int ret = 0; int ret = 0;
const int varLen = variableParams(paramTarget, varArray); const int varLen = variableParams(paramTarget, varArray, dictFileName != NULL);
winnerInfo_t winner = initWinnerInfo(emptyParams()); winnerInfo_t winner = initWinnerInfo(emptyParams());
U8** allMT = NULL; U8** allMT = NULL;
size_t k; paramValues_t paramBase = cParamUnsetMin(paramTarget);
size_t maxBlockSize = 0; size_t k, maxBlockSize = 0;
contexts_t ctx; contexts_t ctx;
buffers_t buf; buffers_t buf;
g_time = UTIL_getTime(); g_time = UTIL_getTime();
/* Init */ if(!paramValid(paramBase)) {
if(!cParamValid(paramTarget)) {
return 1; return 1;
} }
/* load dictionary*/
if(createBuffers(&buf, fileNamesTable, nbFiles)) { if(createBuffers(&buf, fileNamesTable, nbFiles)) {
DISPLAY("unable to load files\n"); DISPLAY("unable to load files\n");
return 1; return 1;
@@ -2154,23 +2199,23 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
} }
/* if strategy is fixed, only init that part of memotable */ /* if strategy is fixed, only init that part of memotable */
if(paramTarget.strategy) { if(paramTarget.vals[strt_ind] != PARAM_UNSET) {
varInds_t varNew[NUM_PARAMS]; varInds_t varNew[NUM_PARAMS];
int varLenNew = sanitizeVarArray(varNew, varLen, varArray, paramTarget.strategy); int varLenNew = sanitizeVarArray(varNew, varLen, varArray, paramTarget.vals[strt_ind]);
allMT = (U8**)calloc(sizeof(U8*), (ZSTD_btultra + 1)); allMT = (U8**)calloc(sizeof(U8*), (ZSTD_btultra + 1));
if(allMT == NULL) { if(allMT == NULL) {
ret = 57; ret = 57;
goto _cleanUp; goto _cleanUp;
} }
allMT[paramTarget.strategy] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew)); allMT[paramTarget.vals[strt_ind]] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew));
if(allMT[paramTarget.strategy] == NULL) { if(allMT[paramTarget.vals[strt_ind]] == NULL) {
ret = 58; ret = 58;
goto _cleanUp; goto _cleanUp;
} }
initMemoTable(allMT[paramTarget.strategy], paramTarget, target, varNew, varLenNew, maxBlockSize); initMemoTable(allMT[paramTarget.vals[strt_ind]], paramTarget, target, varNew, varLenNew, maxBlockSize);
} else { } else {
allMT = createMemoTableArray(paramTarget, target, varArray, varLen, maxBlockSize); allMT = createMemoTableArray(paramTarget, target, varArray, varLen, maxBlockSize);
} }
@@ -2180,7 +2225,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
ret = 2; ret = 2;
goto _cleanUp; goto _cleanUp;
} }
/* default strictness = Maximum for */ /* default strictness = Maximum for */
if(g_strictness == DEFAULT_STRICTNESS) { if(g_strictness == DEFAULT_STRICTNESS) {
if(g_optmode) { if(g_optmode) {
@@ -2199,7 +2244,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
/* use level'ing mode instead of normal target mode */ /* use level'ing mode instead of normal target mode */
/* Should lvl be parameter-masked here? */ /* Should lvl be parameter-masked here? */
if(g_optmode) { if(g_optmode) {
winner.params = ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize); winner.params = cParamsToPVals(ZSTD_getCParams(cLevelOpt, maxBlockSize, ctx.dictSize));
if(BMK_benchParam(&winner.result, buf, ctx, winner.params)) { if(BMK_benchParam(&winner.result, buf, ctx, winner.params)) {
ret = 3; ret = 3;
goto _cleanUp; goto _cleanUp;
@@ -2213,13 +2258,13 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
target.cSpeed = (U32)g_lvltarget.cSpeed; target.cSpeed = (U32)g_lvltarget.cSpeed;
target.dSpeed = (U32)g_lvltarget.dSpeed; //See if this is reasonable. target.dSpeed = (U32)g_lvltarget.dSpeed; //See if this is reasonable.
BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize); BMK_printWinnerOpt(stdout, cLevelOpt, winner.result, winner.params, target, buf.srcSize);
} }
/* Don't want it to return anything worse than the best known result */ /* Don't want it to return anything worse than the best known result */
if(g_singleRun) { if(g_singleRun) {
BMK_result_t res; BMK_result_t res;
g_params = ZSTD_adjustCParams(maskParams(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize), g_params), maxBlockSize, ctx.dictSize); g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevelRun, maxBlockSize, ctx.dictSize)), g_params), maxBlockSize, ctx.dictSize);
if(BMK_benchParam(&res, buf, ctx, g_params)) { if(BMK_benchParam(&res, buf, ctx, g_params)) {
ret = 45; ret = 45;
goto _cleanUp; goto _cleanUp;
@@ -2246,19 +2291,19 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
findClockGranularity(); findClockGranularity();
{ {
ZSTD_compressionParameters CParams; paramValues_t CParams;
/* find best solution from default params */ /* find best solution from default params */
{ {
/* strategy selection */ /* strategy selection */
const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel();
DEBUGOUTPUT("Strategy Selection\n"); DEBUGOUTPUT("Strategy Selection\n");
if(paramTarget.strategy == 0) { if(paramTarget.vals[strt_ind] == PARAM_UNSET) {
BMK_result_t candidate; BMK_result_t candidate;
int i; int i;
for (i=1; i<=maxSeeds; i++) { for (i=1; i<=maxSeeds; i++) {
int ec; int ec;
CParams = overwriteParams(ZSTD_getCParams(i, maxBlockSize, ctx.dictSize), paramTarget); CParams = overwriteParams(cParamsToPVals(ZSTD_getCParams(i, maxBlockSize, ctx.dictSize)), paramTarget);
ec = BMK_benchParam(&candidate, buf, ctx, CParams); ec = BMK_benchParam(&candidate, buf, ctx, CParams);
BMK_printWinnerOpt(stdout, i, candidate, CParams, target, buf.srcSize); BMK_printWinnerOpt(stdout, i, candidate, CParams, target, buf.srcSize);
@@ -2271,17 +2316,18 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
/* if the current params are too slow, just stop. */ /* if the current params are too slow, just stop. */
if(target.cSpeed > candidate.cSpeed * 3 / 2) { break; } if(target.cSpeed > candidate.cSpeed * 3 / 2) { break; }
} }
BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winner.result, winner.params, target, buf.srcSize);
BMK_translateAdvancedParams(stdout, winner.params);
} }
} }
BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winner.result, winner.params, target, buf.srcSize);
BMK_translateAdvancedParams(winner.params);
DEBUGOUTPUT("Real Opt\n"); DEBUGOUTPUT("Real Opt\n");
/* start 'real' tests */ /* start 'real' tests */
{ {
int bestStrategy = (int)winner.params.strategy; int bestStrategy = (int)winner.params.vals[strt_ind];
if(paramTarget.strategy == 0) { if(paramTarget.vals[strt_ind] == PARAM_UNSET) {
int st = (int)winner.params.strategy; int st = bestStrategy;
int tries = g_maxTries; int tries = g_maxTries;
{ {
@@ -2298,7 +2344,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
winnerInfo_t wc; winnerInfo_t wc;
DEBUGOUTPUT("StrategySwitch: %s\n", g_stratName[st]); DEBUGOUTPUT("StrategySwitch: %s\n", g_stratName[st]);
wc = optimizeFixedStrategy(buf, ctx, target, paramTarget, wc = optimizeFixedStrategy(buf, ctx, target, paramBase,
st, varArray, varLen, allMT, tries); st, varArray, varLen, allMT, tries);
if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) { if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) {
@@ -2312,7 +2358,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
CHECKTIMEGT(ret, 0, _cleanUp); CHECKTIMEGT(ret, 0, _cleanUp);
} }
} else { } else {
winner = optimizeFixedStrategy(buf, ctx, target, paramTarget, paramTarget.strategy, winner = optimizeFixedStrategy(buf, ctx, target, paramBase, paramTarget.vals[strt_ind],
varArray, varLen, allMT, g_maxTries); varArray, varLen, allMT, g_maxTries);
} }
@@ -2326,7 +2372,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
} }
/* end summary */ /* end summary */
BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winner.result, winner.params, target, buf.srcSize); BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winner.result, winner.params, target, buf.srcSize);
BMK_translateAdvancedParams(winner.params); BMK_translateAdvancedParams(stdout, winner.params);
DISPLAY("grillParams size - optimizer completed \n"); DISPLAY("grillParams size - optimizer completed \n");
} }
@@ -2350,7 +2396,9 @@ static void errorOut(const char* msg)
static unsigned readU32FromChar(const char** stringPtr) static unsigned readU32FromChar(const char** stringPtr)
{ {
const char errorMsg[] = "error: numeric value too large"; const char errorMsg[] = "error: numeric value too large";
unsigned sign = 1;
unsigned result = 0; unsigned result = 0;
if(**stringPtr == '-') { sign = (unsigned)-1; (*stringPtr)++; }
while ((**stringPtr >='0') && (**stringPtr <='9')) { while ((**stringPtr >='0') && (**stringPtr <='9')) {
unsigned const max = (((unsigned)(-1)) / 10) - 1; unsigned const max = (((unsigned)(-1)) / 10) - 1;
if (result > max) errorOut(errorMsg); if (result > max) errorOut(errorMsg);
@@ -2368,7 +2416,7 @@ static unsigned readU32FromChar(const char** stringPtr)
if (**stringPtr=='i') (*stringPtr)++; if (**stringPtr=='i') (*stringPtr)++;
if (**stringPtr=='B') (*stringPtr)++; if (**stringPtr=='B') (*stringPtr)++;
} }
return result; return result * sign;
} }
static int usage(const char* exename) static int usage(const char* exename)
@@ -2408,13 +2456,14 @@ static int badusage(const char* exename)
#define PARSE_SUB_ARGS(stringLong, stringShort, variable) { if (longCommandWArg(&argument, stringLong) || longCommandWArg(&argument, stringShort)) { variable = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } } #define PARSE_SUB_ARGS(stringLong, stringShort, variable) { if (longCommandWArg(&argument, stringLong) || longCommandWArg(&argument, stringShort)) { variable = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } }
#define PARSE_CPARAMS(variable) \ #define PARSE_CPARAMS(variable) \
{ \ { \
PARSE_SUB_ARGS("windowLog=", "wlog=", variable.windowLog); \ PARSE_SUB_ARGS("windowLog=", "wlog=", variable.vals[wlog_ind]); \
PARSE_SUB_ARGS("chainLog=" , "clog=", variable.chainLog); \ PARSE_SUB_ARGS("chainLog=" , "clog=", variable.vals[clog_ind]); \
PARSE_SUB_ARGS("hashLog=", "hlog=", variable.hashLog); \ PARSE_SUB_ARGS("hashLog=", "hlog=", variable.vals[hlog_ind]); \
PARSE_SUB_ARGS("searchLog=" , "slog=", variable.searchLog); \ PARSE_SUB_ARGS("searchLog=" , "slog=", variable.vals[slog_ind]); \
PARSE_SUB_ARGS("searchLength=", "slen=", variable.searchLength); \ PARSE_SUB_ARGS("searchLength=", "slen=", variable.vals[slen_ind]); \
PARSE_SUB_ARGS("targetLength=" , "tlen=", variable.targetLength); \ PARSE_SUB_ARGS("targetLength=" , "tlen=", variable.vals[tlen_ind]); \
PARSE_SUB_ARGS("strategy=", "strat=", variable.strategy); \ PARSE_SUB_ARGS("strategy=", "strat=", variable.vals[strt_ind]); \
PARSE_SUB_ARGS("forceAttachDict=", "fad=" , variable.vals[fadt_ind]); \
} }
int main(int argc, const char** argv) int main(int argc, const char** argv)
@@ -2426,12 +2475,11 @@ int main(int argc, const char** argv)
const char* input_filename = NULL; const char* input_filename = NULL;
const char* dictFileName = NULL; const char* dictFileName = NULL;
U32 main_pause = 0; U32 main_pause = 0;
int cLevel = 0; int cLevelOpt = 0, cLevelRun = 0;
int seperateFiles = 0; int seperateFiles = 0;
constraint_t target = { 0, 0, (U32)-1 }; constraint_t target = { 0, 0, (U32)-1 };
ZSTD_compressionParameters paramTarget = emptyParams(); paramValues_t paramTarget = emptyParams();
g_params = emptyParams(); g_params = emptyParams();
assert(argc>=1); /* for exename */ assert(argc>=1); /* for exename */
@@ -2441,9 +2489,7 @@ int main(int argc, const char** argv)
for(i=1; i<argc; i++) { for(i=1; i<argc; i++) {
const char* argument = argv[i]; const char* argument = argv[i];
DEBUGOUTPUT("%d: ", i); DEBUGOUTPUT("%d: %s\n", i, argument);
DEBUGOUTPUT("%s\n", argument);
assert(argument != NULL); assert(argument != NULL);
if(!strcmp(argument,"--no-seed")) { g_noSeed = 1; continue; } if(!strcmp(argument,"--no-seed")) { g_noSeed = 1; continue; }
@@ -2459,7 +2505,7 @@ int main(int argc, const char** argv)
PARSE_SUB_ARGS("preferSpeed=", "prfSpd=", g_speedMultiplier); PARSE_SUB_ARGS("preferSpeed=", "prfSpd=", g_speedMultiplier);
PARSE_SUB_ARGS("preferRatio=", "prfRto=", g_ratioMultiplier); PARSE_SUB_ARGS("preferRatio=", "prfRto=", g_ratioMultiplier);
PARSE_SUB_ARGS("maxTries=", "tries=", g_maxTries); PARSE_SUB_ARGS("maxTries=", "tries=", g_maxTries);
if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevel = readU32FromChar(&argument); g_optmode = 1; if (argument[0]==',') { argument++; continue; } else break; } if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevelOpt = readU32FromChar(&argument); g_optmode = 1; if (argument[0]==',') { argument++; continue; } else break; }
DISPLAY("invalid optimization parameter \n"); DISPLAY("invalid optimization parameter \n");
return 1; return 1;
@@ -2473,10 +2519,9 @@ int main(int argc, const char** argv)
} else if (longCommandWArg(&argument, "--zstd=")) { } else if (longCommandWArg(&argument, "--zstd=")) {
/* Decode command (note : aggregated commands are allowed) */ /* Decode command (note : aggregated commands are allowed) */
g_singleRun = 1; g_singleRun = 1;
cLevel = 2;
for ( ; ;) { for ( ; ;) {
PARSE_CPARAMS(g_params) PARSE_CPARAMS(g_params)
if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevel = readU32FromChar(&argument); g_params = emptyParams(); if (argument[0]==',') { argument++; continue; } else break; } if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevelRun = readU32FromChar(&argument); g_params = emptyParams(); if (argument[0]==',') { argument++; continue; } else break; }
DISPLAY("invalid compression parameter \n"); DISPLAY("invalid compression parameter \n");
return 1; return 1;
@@ -2520,41 +2565,44 @@ int main(int argc, const char** argv)
case 'S': case 'S':
g_singleRun = 1; g_singleRun = 1;
argument++; argument++;
g_params = ZSTD_getCParams(2, g_blockSize, 0);
for ( ; ; ) { for ( ; ; ) {
switch(*argument) switch(*argument)
{ {
case 'w': case 'w':
argument++; argument++;
g_params.windowLog = readU32FromChar(&argument); g_params.vals[wlog_ind] = readU32FromChar(&argument);
continue; continue;
case 'c': case 'c':
argument++; argument++;
g_params.chainLog = readU32FromChar(&argument); g_params.vals[clog_ind] = readU32FromChar(&argument);
continue; continue;
case 'h': case 'h':
argument++; argument++;
g_params.hashLog = readU32FromChar(&argument); g_params.vals[hlog_ind] = readU32FromChar(&argument);
continue; continue;
case 's': case 's':
argument++; argument++;
g_params.searchLog = readU32FromChar(&argument); g_params.vals[slog_ind] = readU32FromChar(&argument);
continue; continue;
case 'l': /* search length */ case 'l': /* search length */
argument++; argument++;
g_params.searchLength = readU32FromChar(&argument); g_params.vals[slen_ind] = readU32FromChar(&argument);
continue; continue;
case 't': /* target length */ case 't': /* target length */
argument++; argument++;
g_params.targetLength = readU32FromChar(&argument); g_params.vals[tlen_ind] = readU32FromChar(&argument);
continue; continue;
case 'S': /* strategy */ case 'S': /* strategy */
argument++; argument++;
g_params.strategy = (ZSTD_strategy)readU32FromChar(&argument); g_params.vals[strt_ind] = readU32FromChar(&argument);
continue;
case 'f': /* forceAttachDict */
argument++;
g_params.vals[fadt_ind] = readU32FromChar(&argument);
continue; continue;
case 'L': case 'L':
{ argument++; { argument++;
cLevel = readU32FromChar(&argument); cLevelRun = readU32FromChar(&argument);
g_params = emptyParams(); g_params = emptyParams();
continue; continue;
} }
@@ -2622,18 +2670,18 @@ int main(int argc, const char** argv)
if(seperateFiles) { if(seperateFiles) {
for(i = 0; i < argc - filenamesStart; i++) { for(i = 0; i < argc - filenamesStart; i++) {
if (g_optimizer) { if (g_optimizer) {
result = optimizeForSize(argv+filenamesStart + i, 1, dictFileName, target, paramTarget, cLevel); result = optimizeForSize(argv+filenamesStart + i, 1, dictFileName, target, paramTarget, cLevelOpt, cLevelRun);
if(result) { DISPLAY("Error on File %d", i); return result; } if(result) { DISPLAY("Error on File %d", i); return result; }
} else { } else {
result = benchFiles(argv+filenamesStart + i, 1, dictFileName, cLevel); result = benchFiles(argv+filenamesStart + i, 1, dictFileName, cLevelRun);
if(result) { DISPLAY("Error on File %d", i); return result; } if(result) { DISPLAY("Error on File %d", i); return result; }
} }
} }
} else { } else {
if (g_optimizer) { if (g_optimizer) {
result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget, cLevel); result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget, cLevelOpt, cLevelRun);
} else { } else {
result = benchFiles(argv+filenamesStart, argc-filenamesStart, dictFileName, cLevel); result = benchFiles(argv+filenamesStart, argc-filenamesStart, dictFileName, cLevelRun);
} }
} }
} }