ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1277 from GeorgeLu97/mtoptions

Browse Source 
memoTableOptions
This commit is contained in:
Yann Collet
2018-08-15 13:45:17 -07:00
committed by GitHub
parent 90a3901d1e b1d9ca737a
commit a01b964207
2 changed files with 429 additions and 411 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/README.md
+5 -4
View File
@@ -113,14 +113,15 @@ Full list of arguments
dSpeed= : Minimum decompression speed
cMem= : Maximum compression memory
lvl= : Searches for solutions which are strictly better than that compression lvl in ratio and cSpeed,
stc= : When invoked with lvl=, represents percentage slack in ratio/cSpeed allowed for a solution to be considered (Default 99%)
stc= : When invoked with lvl=, represents percentage slack in ratio/cSpeed allowed for a solution to be considered (Default 100%)
: In normal operation, represents percentage slack in choosing viable starting strategy selection in choosing the default parameters
(Lower value will begin with stronger strategies) (Default 90%)
preferSpeed= / preferRatio=
: Only affects lvl = invocations. Defines value placed on compression speed or ratio
when determining overall winner (default speed = 1, ratio = 5 for both, higher = more valued).
speedRatio= (accepts decimals)
: determines value of gains in speed vs gains in ratio
when determining overall winner (default 5 (1% ratio = 5% speed)).
tries= : Maximum number of random restarts on a single strategy before switching (Default 3)
Higher values will make optimizer run longer, more chances to find better solution.
memLog : Limits the log of the size of each memotable (1 per strategy). Setting memLog = 0 turns off memoization
-P# : generated sample compressibility
-t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours ))
-v : Prints Benchmarking output
tests/paramgrill.c
+424 -407
View File
@@ -86,7 +86,15 @@ static const int g_maxNbVariations = 64;
#define CHECKTIME(r) { if(BMK_timeSpan(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); return r; } }
#define CHECKTIMEGT(ret, val, _gototag) {if(BMK_timeSpan(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); ret = val; goto _gototag; } }
#define PARAM_UNSET ((U32)-2) /* can't be -1 b/c fadt */
#define PARAM_UNSET ((U32)-2) /* can't be -1 b/c fadt uses -1 */
static const char* g_stratName[ZSTD_btultra+1] = {
"(none) ", "ZSTD_fast ", "ZSTD_dfast ",
"ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ",
"ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "};
static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 };
/*-************************************
* Setup for Adding new params
@@ -105,6 +113,10 @@ typedef enum {
NUM_PARAMS = 8
} varInds_t;
typedef struct {
U32 vals[NUM_PARAMS];
} paramValues_t;
/* maximum value of parameters */
static const U32 mintable[NUM_PARAMS] =
{ ZSTD_WINDOWLOG_MIN, ZSTD_CHAINLOG_MIN, ZSTD_HASHLOG_MIN, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLENGTH_MIN, ZSTD_TARGETLENGTH_MIN, ZSTD_fast, FADT_MIN };
@@ -117,15 +129,19 @@ static const U32 maxtable[NUM_PARAMS] =
static const U32 rangetable[NUM_PARAMS] =
{ WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE, STRT_RANGE, FADT_RANGE };
/* ZSTD_cctxSetParameter() index to set */
static const ZSTD_cParameter cctxSetParamTable[NUM_PARAMS] =
{ ZSTD_p_windowLog, ZSTD_p_chainLog, ZSTD_p_hashLog, ZSTD_p_searchLog, ZSTD_p_minMatch, ZSTD_p_targetLength, ZSTD_p_compressionStrategy, ZSTD_p_forceAttachDict };
/* names of parameters */
static const char* g_paramNames[NUM_PARAMS] =
{ "windowLog", "chainLog", "hashLog","searchLog", "searchLength", "targetLength", "strategy", "forceAttachDict"};
{ "windowLog", "chainLog", "hashLog","searchLog", "searchLength", "targetLength", "strategy", "forceAttachDict" };
/* shortened names of parameters */
static const char* g_shortParamNames[NUM_PARAMS] =
{ "wlog", "clog", "hlog","slog", "slen", "tlen", "strt", "fadt" };
static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 };
/* maps value from 0 to rangetable[param] - 1 to valid paramvalue */
/* maps value from { 0 to rangetable[param] - 1 } to valid paramvalues */
static U32 rangeMap(varInds_t param, U32 ind) {
ind = MIN(ind, rangetable[param] - 1);
switch(param) {
@@ -141,6 +157,7 @@ static U32 rangeMap(varInds_t param, U32 ind) {
case strt_ind:
return mintable[param] + ind;
case NUM_PARAMS:
DISPLAY("Error, not a valid param\n ");
return (U32)-1;
}
return 0; /* should never happen, stop compiler warnings */
@@ -176,49 +193,26 @@ static U32 invRangeMap(varInds_t param, U32 value) {
case strt_ind:
return value - mintable[param];
case NUM_PARAMS:
DISPLAY("Error, not a valid param\n ");
return (U32)-1;
}
return 0; /* should never happen, stop compiler warnings */
}
typedef struct {
U32 vals[NUM_PARAMS];
} paramValues_t;
//TODO: unset -> 0?
static ZSTD_compressionParameters pvalsToCParams(paramValues_t p) {
ZSTD_compressionParameters c;
c.windowLog = p.vals[wlog_ind];
c.chainLog = p.vals[clog_ind];
c.hashLog = p.vals[hlog_ind];
c.searchLog = p.vals[slog_ind];
c.searchLength = p.vals[slen_ind];
c.targetLength = p.vals[tlen_ind];
c.strategy = p.vals[strt_ind];
/* no forceAttachDict */
return c;
}
/* 0 = auto for fadt */
static paramValues_t cParamsToPVals(ZSTD_compressionParameters c) {
paramValues_t p;
p.vals[wlog_ind] = c.windowLog;
p.vals[clog_ind] = c.chainLog;
p.vals[hlog_ind] = c.hashLog;
p.vals[slog_ind] = c.searchLog;
p.vals[slen_ind] = c.searchLength;
p.vals[tlen_ind] = c.targetLength;
p.vals[strt_ind] = c.strategy;
p.vals[fadt_ind] = 0;
return p;
}
/* equivalent of ZSTD_adjustCParams for paramValues_t */
static paramValues_t adjustParams(paramValues_t p, size_t maxBlockSize, size_t dictSize) {
U32 fval = p.vals[fadt_ind];
p = cParamsToPVals(ZSTD_adjustCParams(pvalsToCParams(p), maxBlockSize, dictSize));
p.vals[fadt_ind] = fval;
return p;
/* display of params */
static void displayParamVal(FILE* f, varInds_t param, U32 value, int width) {
switch(param) {
case fadt_ind: if(width) { fprintf(f, "%*d", width, (int)value); } else { fprintf(f, "%d", (int)value); } break;
case strt_ind: if(width) { fprintf(f, "%*s", width, g_stratName[value]); } else { fprintf(f, "%s", g_stratName[value]); } break;
case wlog_ind:
case clog_ind:
case hlog_ind:
case slog_ind:
case slen_ind:
case tlen_ind: if(width) { fprintf(f, "%*u", width, value); } else { fprintf(f, "%u", value); } break;
case NUM_PARAMS:
DISPLAY("Error, not a valid param\n "); break;
}
}
/*-************************************
@@ -238,7 +232,21 @@ static U32 g_target = 0;
static U32 g_noSeed = 0;
static paramValues_t g_params; /* Initialized at the beginning of main w/ emptyParams() function */
static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */
static U32 g_memoTableLog = PARAM_UNSET;
typedef enum {
directMap,
xxhashMap,
noMemo
} memoTableType_t;
typedef struct {
memoTableType_t tableType;
BYTE* table;
size_t tableLen;
varInds_t varArray[NUM_PARAMS];
size_t varLen;
} memoTable_t;
typedef struct {
BMK_result_t result;
@@ -266,14 +274,12 @@ static winner_ll_node* g_winners; /* linked list sorted ascending by cSize & cSp
static BMK_result_t g_lvltarget;
static int g_optmode = 0;
static U32 g_speedMultiplier = 1;
static U32 g_ratioMultiplier = 5;
static double g_ratioMultiplier = 5.;
/* g_mode? */
/* range 0 - 99, measure of how strict */
#define DEFAULT_STRICTNESS 99999
static U32 g_strictness = DEFAULT_STRICTNESS;
static U32 g_strictness = PARAM_UNSET;
void BMK_SetNbIterations(int nbLoops)
{
@@ -283,7 +289,6 @@ void BMK_SetNbIterations(int nbLoops)
/*
* Additional Global Variables (Defined Above Use)
* g_stratName
* g_level_constraint
* g_alreadyTested
* g_maxTries
@@ -293,8 +298,53 @@ void BMK_SetNbIterations(int nbLoops)
* Private functions
*********************************************************/
static ZSTD_compressionParameters pvalsToCParams(paramValues_t p) {
ZSTD_compressionParameters c;
memset(&c, 0, sizeof(ZSTD_compressionParameters));
c.windowLog = p.vals[wlog_ind];
c.chainLog = p.vals[clog_ind];
c.hashLog = p.vals[hlog_ind];
c.searchLog = p.vals[slog_ind];
c.searchLength = p.vals[slen_ind];
c.targetLength = p.vals[tlen_ind];
c.strategy = p.vals[strt_ind];
/* no forceAttachDict */
return c;
}
static paramValues_t cParamsToPVals(ZSTD_compressionParameters c) {
paramValues_t p;
varInds_t i;
p.vals[wlog_ind] = c.windowLog;
p.vals[clog_ind] = c.chainLog;
p.vals[hlog_ind] = c.hashLog;
p.vals[slog_ind] = c.searchLog;
p.vals[slen_ind] = c.searchLength;
p.vals[tlen_ind] = c.targetLength;
p.vals[strt_ind] = c.strategy;
/* set all other params to their minimum value */
for(i = strt_ind + 1; i < NUM_PARAMS; i++) {
p.vals[i] = mintable[i];
}
return p;
}
/* equivalent of ZSTD_adjustCParams for paramValues_t */
static paramValues_t adjustParams(paramValues_t p, const size_t maxBlockSize, const size_t dictSize) {
paramValues_t ot = p;
varInds_t i;
p = cParamsToPVals(ZSTD_adjustCParams(pvalsToCParams(p), maxBlockSize, dictSize));
/* retain value of all other parameters */
for(i = strt_ind + 1; i < NUM_PARAMS; i++) {
p.vals[i] = ot.vals[i];
}
return p;
}
/* accuracy in seconds only, span can be multiple years */
static U32 BMK_timeSpan(UTIL_time_t tStart) { return (U32)(UTIL_clockSpanMicro(tStart) / 1000000ULL); }
static U32 BMK_timeSpan(const UTIL_time_t tStart) { return (U32)(UTIL_clockSpanMicro(tStart) / 1000000ULL); }
static size_t BMK_findMaxMem(U64 requiredMem)
{
@@ -375,19 +425,14 @@ static void findClockGranularity(void) {
return 0; \
} }
/* Like ZSTD_checkCParams() but allows 0's */
/* no check on targetLen? */
static int paramValid(paramValues_t paramTarget) {
static int paramValid(const paramValues_t paramTarget) {
U32 i;
for(i = 0; i < NUM_PARAMS; i++) {
CLAMPCHECK(paramTarget.vals[i], mintable[i], maxtable[i]);
}
//TODO: Strategy could be valid at 0 before, is that right?
return 1;
}
//TODO: doesn't affect strategy?
static paramValues_t cParamUnsetMin(paramValues_t paramTarget) {
varInds_t i;
for(i = 0; i < NUM_PARAMS; i++) {
@@ -402,8 +447,11 @@ static void BMK_translateAdvancedParams(FILE* f, const paramValues_t params) {
U32 i;
fprintf(f,"--zstd=");
for(i = 0; i < NUM_PARAMS; i++) {
fprintf(f,"%s", g_paramNames[i]);
fprintf(f,"=%u", params.vals[i]);
fprintf(f,"%s=", g_paramNames[i]);
if(i == strt_ind) { fprintf(f,"%u", params.vals[i]); }
else { displayParamVal(f, i, params.vals[i], 0); }
if(i != NUM_PARAMS - 1) {
fprintf(f, ",");
}
@@ -411,19 +459,16 @@ static void BMK_translateAdvancedParams(FILE* f, const paramValues_t params) {
fprintf(f, "\n");
}
static const char* g_stratName[ZSTD_btultra+1] = {
"(none) ", "ZSTD_fast ", "ZSTD_dfast ",
"ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ",
"ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "};
static void BMK_displayOneResult(FILE* f, winnerInfo_t res, size_t srcSize) {
static void BMK_displayOneResult(FILE* f, winnerInfo_t res, const size_t srcSize) {
varInds_t v;
res.params = cParamUnsetMin(res.params);
fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u,%3d, %s}, ",
res.params.vals[wlog_ind], res.params.vals[clog_ind], res.params.vals[hlog_ind], res.params.vals[slog_ind], res.params.vals[slen_ind],
res.params.vals[tlen_ind], (int)res.params.vals[fadt_ind], g_stratName[res.params.vals[strt_ind]]);
fprintf(f,
" /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
fprintf(f," {");
for(v = 0; v < NUM_PARAMS; v++) {
if(v != 0) { fprintf(f, ","); }
displayParamVal(f, v, res.params.vals[v], 3);
}
fprintf(f, "}, /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
(double)srcSize / res.result.cSize, (double)res.result.cSpeed / (1 MB), (double)res.result.dSpeed / (1 MB));
}
@@ -460,7 +505,7 @@ static double resultDistLvl(const BMK_result_t result1, const BMK_result_t lvlRe
if(normalizedRatioGain1 < 0 || normalizedCSpeedGain1 < 0) {
return 0.0;
}
return normalizedRatioGain1 * g_ratioMultiplier + normalizedCSpeedGain1 * g_speedMultiplier;
return normalizedRatioGain1 * g_ratioMultiplier + normalizedCSpeedGain1;
}
/* return true if r2 strictly better than r1 */
@@ -496,7 +541,7 @@ static paramValues_t emptyParams(void) {
return p;
}
static winnerInfo_t initWinnerInfo(paramValues_t p) {
static winnerInfo_t initWinnerInfo(const paramValues_t p) {
winnerInfo_t w1;
w1.result.cSpeed = 0.;
w1.result.dSpeed = 0.;
@@ -517,6 +562,7 @@ typedef struct {
void** resPtrs;
size_t* resSizes;
size_t nbBlocks;
size_t maxBlockSize;
} buffers_t;
typedef struct {
@@ -530,27 +576,6 @@ typedef struct {
* From bench.c
*********************************************************/
static void BMK_initCCtx(ZSTD_CCtx* ctx,
const void* dictBuffer, const size_t dictBufferSize, const int cLevel,
const paramValues_t* comprParams) {
varInds_t i;
ZSTD_CCtx_reset(ctx);
ZSTD_CCtx_resetParameters(ctx);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel);
for(i = 0; i < NUM_PARAMS; i++) {
if(comprParams->vals[i] != PARAM_UNSET)
ZSTD_CCtx_setParameter(ctx, cctxSetParamTable[i], comprParams->vals[i]);
}
ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize);
}
static void BMK_initDCtx(ZSTD_DCtx* dctx,
const void* dictBuffer, const size_t dictBufferSize) {
ZSTD_DCtx_reset(dctx);
ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize);
}
typedef struct {
ZSTD_CCtx* ctx;
const void* dictBuffer;
@@ -561,7 +586,17 @@ typedef struct {
static size_t local_initCCtx(void* payload) {
const BMK_initCCtxArgs* ag = (const BMK_initCCtxArgs*)payload;
BMK_initCCtx(ag->ctx, ag->dictBuffer, ag->dictBufferSize, ag->cLevel, ag->comprParams);
varInds_t i;
ZSTD_CCtx_reset(ag->ctx);
ZSTD_CCtx_resetParameters(ag->ctx);
ZSTD_CCtx_setParameter(ag->ctx, ZSTD_p_compressionLevel, ag->cLevel);
for(i = 0; i < NUM_PARAMS; i++) {
if(ag->comprParams->vals[i] != PARAM_UNSET)
ZSTD_CCtx_setParameter(ag->ctx, cctxSetParamTable[i], ag->comprParams->vals[i]);
}
ZSTD_CCtx_loadDictionary(ag->ctx, ag->dictBuffer, ag->dictBufferSize);
return 0;
}
@@ -573,7 +608,8 @@ typedef struct {
static size_t local_initDCtx(void* payload) {
const BMK_initDCtxArgs* ag = (const BMK_initDCtxArgs*)payload;
BMK_initDCtx(ag->dctx, ag->dictBuffer, ag->dictBufferSize);
ZSTD_DCtx_reset(ag->dctx);
ZSTD_DCtx_loadDictionary(ag->dctx, ag->dictBuffer, ag->dictBufferSize);
return 0;
}
@@ -638,6 +674,72 @@ static size_t local_defaultDecompress(
* From bench.c End
*********************************************************/
static void optimizerAdjustInput(paramValues_t* pc, const size_t maxBlockSize) {
varInds_t v;
for(v = 0; v < NUM_PARAMS; v++) {
if(pc->vals[v] != PARAM_UNSET) {
U32 newval = MIN(MAX(pc->vals[v], mintable[v]), maxtable[v]);
if(newval != pc->vals[v]) {
pc->vals[v] = newval;
DISPLAY("Warning: parameter %s not in valid range, adjusting to ", g_paramNames[v]); displayParamVal(stderr, v, newval, 0); DISPLAY("\n");
}
}
}
if(pc->vals[wlog_ind] != PARAM_UNSET) {
U32 sshb = maxBlockSize > 1 ? ZSTD_highbit32((U32)(maxBlockSize-1)) + 1 : 1;
/* edge case of highBit not working for 0 */
if(maxBlockSize < (1ULL << 31) && sshb + 1 < pc->vals[wlog_ind]) {
U32 adjust = MAX(mintable[wlog_ind], sshb);
if(adjust != pc->vals[wlog_ind]) {
pc->vals[wlog_ind] = adjust;
DISPLAY("Warning: windowLog larger than src/block size, adjusted to %u\n", pc->vals[wlog_ind]);
}
}
}
if(pc->vals[wlog_ind] != PARAM_UNSET && pc->vals[clog_ind] != PARAM_UNSET) {
U32 maxclog;
if(pc->vals[strt_ind] == PARAM_UNSET || pc->vals[strt_ind] >= (U32)ZSTD_btlazy2) {
maxclog = pc->vals[wlog_ind] + 1;
} else {
maxclog = pc->vals[wlog_ind];
}
if(pc->vals[clog_ind] > maxclog) {
pc->vals[clog_ind] = maxclog;
DISPLAY("Warning: chainlog too much larger than windowLog size, adjusted to %u\n", pc->vals[clog_ind]);
}
}
if(pc->vals[wlog_ind] != PARAM_UNSET && pc->vals[hlog_ind] != PARAM_UNSET) {
if(pc->vals[wlog_ind] + 1 < pc->vals[hlog_ind]) {
pc->vals[hlog_ind] = pc->vals[wlog_ind] + 1;
DISPLAY("Warning: hashlog too much larger than windowLog size, adjusted to %u\n", pc->vals[hlog_ind]);
}
}
if(pc->vals[slog_ind] != PARAM_UNSET && pc->vals[clog_ind] != PARAM_UNSET) {
if(pc->vals[slog_ind] > pc->vals[clog_ind]) {
pc->vals[clog_ind] = pc->vals[slog_ind];
DISPLAY("Warning: searchLog larger than chainLog, adjusted to %u\n", pc->vals[slog_ind]);
}
}
}
/* what about low something like clog vs hlog in lvl 1? */
static int redundantParams(const paramValues_t paramValues, const constraint_t target, const size_t maxBlockSize) {
return
(ZSTD_estimateCStreamSize_usingCParams(pvalsToCParams(paramValues)) > (size_t)target.cMem) /* Uses too much memory */
|| ((1ULL << (paramValues.vals[wlog_ind] - 1)) >= maxBlockSize && paramValues.vals[wlog_ind] != mintable[wlog_ind]) /* wlog too much bigger than src size */
|| (paramValues.vals[clog_ind] > (paramValues.vals[wlog_ind] + (paramValues.vals[strt_ind] > ZSTD_btlazy2))) /* chainLog larger than windowLog*/
|| (paramValues.vals[slog_ind] > paramValues.vals[clog_ind]) /* searchLog larger than chainLog */
|| (paramValues.vals[hlog_ind] > paramValues.vals[wlog_ind] + 1); /* hashLog larger than windowLog + 1 */
}
static void freeNonSrcBuffers(const buffers_t b) {
free(b.srcPtrs);
free(b.srcSizes);
@@ -664,7 +766,7 @@ static void freeBuffers(const buffers_t b) {
}
/* srcBuffer will be freed by freeBuffers now */
static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, size_t nbFiles,
static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, const size_t nbFiles,
const size_t* fileSizes)
{
size_t pos = 0, n, blockSize;
@@ -724,6 +826,7 @@ static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, size_t nbF
buff->dstCapacities[0] = ZSTD_compressBound(buff->srcSizes[0]);
buff->dstSizes[0] = buff->dstCapacities[0];
buff->resSizes[0] = buff->srcSizes[0];
buff->maxBlockSize = buff->srcSizes[0];
for(n = 1; n < blockNb; n++) {
buff->dstPtrs[n] = ((char*)buff->dstPtrs[n-1]) + buff->dstCapacities[n-1];
@@ -731,6 +834,8 @@ static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, size_t nbF
buff->dstCapacities[n] = ZSTD_compressBound(buff->srcSizes[n]);
buff->dstSizes[n] = buff->dstCapacities[n];
buff->resSizes[n] = buff->srcSizes[n];
buff->maxBlockSize = MAX(buff->maxBlockSize, buff->srcSizes[n]);
}
buff->nbBlocks = blockNb;
@@ -788,7 +893,7 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab
char* buffer = (char*)(srcBuffer);
size_t const readSize = fread((buffer)+pos, 1, (size_t)fileSize, f);
fclose(f);
if (readSize != (size_t)fileSize) { /* should we accept partial read? */
if (readSize != (size_t)fileSize) {
DISPLAY("could not read %s", fileNamesTable[n]);
ret = 1;
goto _cleanUp;
@@ -880,7 +985,6 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t
ZSTD_DCtx* dctx = ctx.dctx;
/* warmimg up memory */
/* can't do this if decode only */
for(i = 0; i < buf.nbBlocks; i++) {
if(mode != BMK_decodeOnly) {
RDG_genBuffer(dstPtrs[i], dstCapacities[i], 0.10, 0.50, 1);
@@ -890,7 +994,6 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t
}
/* Bench */
{
/* init args */
BMK_initCCtxArgs cctxprep;
@@ -994,7 +1097,7 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t
}
static int BMK_benchParam(BMK_result_t* resultPtr,
buffers_t buf, contexts_t ctx,
const buffers_t buf, const contexts_t ctx,
const paramValues_t cParams) {
BMK_return_t res = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_both, BMK_timeMode, 3);
*resultPtr = res.result;
@@ -1011,7 +1114,7 @@ static int BMK_benchParam(BMK_result_t* resultPtr,
#define SPEED_RESULT 4
#define SIZE_RESULT 5
/* maybe have epsilon-eq to limit table size? */
static int speedSizeCompare(BMK_result_t r1, BMK_result_t r2) {
static int speedSizeCompare(const BMK_result_t r1, const BMK_result_t r2) {
if(r1.cSpeed < r2.cSpeed) {
if(r1.cSize >= r2.cSize) {
return BETTER_RESULT;
@@ -1027,7 +1130,7 @@ static int speedSizeCompare(BMK_result_t r1, BMK_result_t r2) {
/* 0 for insertion, 1 for no insert */
/* maintain invariant speedSizeCompare(n, n->next) = SPEED_RESULT */
static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) {
static int insertWinner(const winnerInfo_t w, const constraint_t targetConstraints) {
BMK_result_t r = w.result;
winner_ll_node* cur_node = g_winners;
/* first node to insert */
@@ -1136,21 +1239,21 @@ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result
snprintf(lvlstr, 15, " Level %2u ", cLevel);
}
fprintf(f, "/* %s */ ", lvlstr);
BMK_displayOneResult(f, w, srcSize);
if(TIMED) {
if(TIMED) {
const U64 time = UTIL_clockSpanNano(g_time);
const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC);
fprintf(f, " - %1lu:%2lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC);
fprintf(f, "%1lu:%2lu:%05.2f - ", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC);
}
fprintf(f, "\n");
fprintf(f, "/* %s */ ", lvlstr);
BMK_displayOneResult(f, w, srcSize);
}
static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t result, const paramValues_t params, const constraint_t targetConstraints, const size_t srcSize)
{
/* global winner used for constraints */
static winnerInfo_t g_winner = { { (size_t)-1LL, 0, 0, (size_t)-1LL }, { { PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET } } };
/* cSize, cSpeed, dSpeed, cMem */
static winnerInfo_t g_winner = { { (size_t)-1LL, 0, 0, (size_t)-1LL }, { { PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET } } };
if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize)) {
if(DEBUG && compareResultLT(g_winner.result, result, targetConstraints, srcSize)) {
DISPLAY("New Winner: \n");
@@ -1196,7 +1299,7 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
}
}
static void BMK_printWinners2(FILE* f, const oldWinnerInfo_t* winners, size_t srcSize)
static void BMK_printWinners2(FILE* f, const oldWinnerInfo_t* winners, const size_t srcSize)
{
int cLevel;
@@ -1208,7 +1311,7 @@ static void BMK_printWinners2(FILE* f, const oldWinnerInfo_t* winners, size_t sr
}
static void BMK_printWinners(FILE* f, const oldWinnerInfo_t* winners, size_t srcSize)
static void BMK_printWinners(FILE* f, const oldWinnerInfo_t* winners, const size_t srcSize)
{
fseek(f, 0, SEEK_SET);
BMK_printWinners2(f, winners, srcSize);
@@ -1246,7 +1349,7 @@ static void BMK_init_level_constraints(int bytePerSec_level1)
}
static int BMK_seed(oldWinnerInfo_t* winners, const ZSTD_compressionParameters params,
buffers_t buf, contexts_t ctx)
const buffers_t buf, const contexts_t ctx)
{
BMK_result_t testResult;
int better = 0;
@@ -1347,9 +1450,7 @@ static int BMK_seed(oldWinnerInfo_t* winners, const ZSTD_compressionParameters p
}
/* nullified useless params, to ensure count stats */
/* no point in windowLog < chainLog (no point 2x chainLog for bt) */
/* now with built in bounds-checking */
/* no longer does anything with sanitizeVarArray + clampcheck */
/* cleans up params for memoizing / display */
static paramValues_t sanitizeParams(paramValues_t params)
{
if (params.vals[strt_ind] == ZSTD_fast)
@@ -1365,8 +1466,8 @@ static paramValues_t sanitizeParams(paramValues_t params)
/* return: new length */
/* keep old array, will need if iter over strategy. */
/* prunes useless params */
static int sanitizeVarArray(varInds_t* varNew, const int varLength, const varInds_t* varArray, const ZSTD_strategy strat) {
int i, j = 0;
static size_t sanitizeVarArray(varInds_t* varNew, const size_t varLength, const varInds_t* varArray, const ZSTD_strategy strat) {
size_t i, j = 0;
for(i = 0; i < varLength; i++) {
if( !((varArray[i] == clog_ind && strat == ZSTD_fast)
|| (varArray[i] == slog_ind && strat == ZSTD_fast)
@@ -1383,9 +1484,9 @@ static int sanitizeVarArray(varInds_t* varNew, const int varLength, const varInd
/* res should be NUM_PARAMS size */
/* constructs varArray from paramValues_t style parameter */
/* pass in using dict. */
static int variableParams(const paramValues_t paramConstraints, varInds_t* res, const int usingDictionary) {
static size_t variableParams(const paramValues_t paramConstraints, varInds_t* res, const int usingDictionary) {
varInds_t i;
int j = 0;
size_t j = 0;
for(i = 0; i < NUM_PARAMS; i++) {
if(paramConstraints.vals[i] == PARAM_UNSET) {
if(i == fadt_ind && !usingDictionary) continue; /* don't use fadt if no dictionary */
@@ -1399,11 +1500,11 @@ static int variableParams(const paramValues_t paramConstraints, varInds_t* res,
/* slight change from old paramVariation, targetLength can only take on powers of 2 now (999 ~= 1024?) */
/* take max/min bounds into account as well? */
static void paramVaryOnce(const varInds_t paramIndex, const int amt, paramValues_t* ptr) {
ptr->vals[paramIndex] = rangeMap(paramIndex, invRangeMap(paramIndex, ptr->vals[paramIndex]) + amt); //TODO: bounds check.
ptr->vals[paramIndex] = rangeMap(paramIndex, invRangeMap(paramIndex, ptr->vals[paramIndex]) + amt);
}
/* varies ptr by nbChanges respecting varyParams*/
static void paramVariation(paramValues_t* ptr, const varInds_t* varyParams, const int varyLen, const U32 nbChanges)
static void paramVariation(paramValues_t* ptr, memoTable_t* mtAll, const U32 nbChanges)
{
paramValues_t p;
U32 validated = 0;
@@ -1411,8 +1512,8 @@ static void paramVariation(paramValues_t* ptr, const varInds_t* varyParams, cons
U32 i;
p = *ptr;
for (i = 0 ; i < nbChanges ; i++) {
const U32 changeID = FUZ_rand(&g_rand) % (varyLen << 1);
paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p);
const U32 changeID = (U32)FUZ_rand(&g_rand) % (mtAll[p.vals[strt_ind]].varLen << 1);
paramVaryOnce(mtAll[p.vals[strt_ind]].varArray[changeID >> 1], ((changeID & 1) << 1) - 1, &p);
}
validated = paramValid(p);
}
@@ -1420,20 +1521,19 @@ static void paramVariation(paramValues_t* ptr, const varInds_t* varyParams, cons
}
/* length of memo table given free variables */
static size_t memoTableLen(const varInds_t* varyParams, const int varyLen) {
static size_t memoTableLen(const varInds_t* varyParams, const size_t varyLen) {
size_t arrayLen = 1;
int i;
size_t i;
for(i = 0; i < varyLen; i++) {
if(varyParams[i] != strt_ind) {
arrayLen *= rangetable[varyParams[i]];
}
if(varyParams[i] == strt_ind) continue; /* strategy separated by table */
arrayLen *= rangetable[varyParams[i]];
}
return arrayLen;
}
/* returns unique index in memotable of compression parameters */
static unsigned memoTableInd(const paramValues_t* ptr, const varInds_t* varyParams, const int varyLen) {
int i;
static unsigned memoTableIndDirect(const paramValues_t* ptr, const varInds_t* varyParams, const size_t varyLen) {
size_t i;
unsigned ind = 0;
for(i = 0; i < varyLen; i++) {
varInds_t v = varyParams[i];
@@ -1443,112 +1543,88 @@ static unsigned memoTableInd(const paramValues_t* ptr, const varInds_t* varyPara
return ind;
}
/* inverse of above function (from index to parameters) */
static void memoTableIndInv(paramValues_t* ptr, const varInds_t* varyParams, const int varyLen, size_t ind) {
int i;
for(i = varyLen - 1; i >= 0; i--) {
varInds_t v = varyParams[i];
if(v == strt_ind) continue;
ptr->vals[v] = rangeMap(v, ind % rangetable[v]);
ind /= rangetable[v];
static size_t memoTableGet(const memoTable_t* memoTableArray, const paramValues_t p) {
const memoTable_t mt = memoTableArray[p.vals[strt_ind]];
switch(mt.tableType) {
case directMap:
return mt.table[memoTableIndDirect(&p, mt.varArray, mt.varLen)];
case xxhashMap:
return mt.table[(XXH64(&p.vals, sizeof(U32) * NUM_PARAMS, 0) >> 3) % mt.tableLen];
case noMemo:
return 0;
}
return 0; /* should never happen, stop compiler warnings */
}
/* Initialize memoization table, which tracks and prevents repeated benchmarking
* of the same set of parameters. In addition, it is also used to immediately mark
* redundant / obviously non-optimal parameter configurations (e.g. wlog - 1 larger)
* than srcSize, clog > wlog, ...
*/
static void initMemoTable(U8* memoTable, paramValues_t paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) {
size_t i;
size_t arrayLen = memoTableLen(varyParams, varyLen);
int cwFixed = paramConstraints.vals[clog_ind] == PARAM_UNSET || paramConstraints.vals[wlog_ind] == PARAM_UNSET;
int scFixed = paramConstraints.vals[slog_ind] == PARAM_UNSET || paramConstraints.vals[clog_ind] == PARAM_UNSET;
int whFixed = paramConstraints.vals[wlog_ind] == PARAM_UNSET || paramConstraints.vals[hlog_ind] == PARAM_UNSET;
int wFixed = paramConstraints.vals[wlog_ind] == PARAM_UNSET;
int j = 0;
assert(memoTable != NULL);
memset(memoTable, 0, arrayLen);
paramConstraints = cParamUnsetMin(paramConstraints);
for(i = 0; i < arrayLen; i++) {
memoTableIndInv(&paramConstraints, varyParams, varyLen, i);
if(ZSTD_estimateCStreamSize_usingCParams(pvalsToCParams(paramConstraints)) > (size_t)target.cMem) {
memoTable[i] = 255;
j++;
}
if(wFixed && (1ULL << (paramConstraints.vals[wlog_ind] - 1)) >= srcSize && paramConstraints.vals[wlog_ind] != mintable[wlog_ind]) {
memoTable[i] = 255;
}
/* nil out parameter sets equivalent to others. */
if(cwFixed) {
if(paramConstraints.vals[strt_ind] == ZSTD_btlazy2 || paramConstraints.vals[strt_ind] == ZSTD_btopt || paramConstraints.vals[strt_ind] == ZSTD_btultra) {
if(paramConstraints.vals[clog_ind] > paramConstraints.vals[wlog_ind]+ 1) {
if(memoTable[i] != 255) { j++; }
memoTable[i] = 255;
}
} else {
if(paramConstraints.vals[clog_ind] > paramConstraints.vals[wlog_ind]) {
if(memoTable[i] != 255) { j++; }
memoTable[i] = 255;
}
}
}
if(scFixed) {
if(paramConstraints.vals[slog_ind] > paramConstraints.vals[clog_ind]) {
if(memoTable[i] != 255) { j++; }
memoTable[i] = 255;
}
}
if(whFixed) {
if(paramConstraints.vals[hlog_ind] > paramConstraints.vals[wlog_ind] + 1) {
if(memoTable[i] != 255) { j++; }
memoTable[i] = 255;
}
}
}
DEBUGOUTPUT("%d / %d Invalid\n", j, (int)i);
if((int)i == j) {
DEBUGOUTPUT("!!!Strategy %d totally infeasible\n", (int)paramConstraints.vals[strt_ind]);
static void memoTableSet(const memoTable_t* memoTableArray, const paramValues_t p, const BYTE value) {
const memoTable_t mt = memoTableArray[p.vals[strt_ind]];
switch(mt.tableType) {
case directMap:
mt.table[memoTableIndDirect(&p, mt.varArray, mt.varLen)] = value; break;
case xxhashMap:
mt.table[(XXH64(&p.vals, sizeof(U32) * NUM_PARAMS, 0) >> 3) % mt.tableLen] = value; break;
case noMemo:
break;
}
}
/* frees all allocated memotables */
static void freeMemoTableArray(U8** mtAll) {
static void freeMemoTableArray(memoTable_t* const mtAll) {
int i;
if(mtAll == NULL) { return; }
for(i = 1; i <= (int)ZSTD_btultra; i++) {
free(mtAll[i]);
free(mtAll[i].table);
}
free(mtAll);
}
/* inits memotables for all (including mallocs), all strategies */
/* takes unsanitized varyParams */
static U8** createMemoTableArray(paramValues_t paramConstraints, constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) {
varInds_t varNew[NUM_PARAMS];
U8** mtAll = (U8**)calloc(sizeof(U8*),(ZSTD_btultra + 1));
static memoTable_t* createMemoTableArray(const varInds_t* const varyParams, const size_t varyLen) {
memoTable_t* mtAll = (memoTable_t*)calloc(sizeof(memoTable_t),(ZSTD_btultra + 1));
int i;
if(mtAll == NULL) {
return NULL;
}
for(i = 1; i <= (int)ZSTD_btultra; i++) {
const int varLenNew = sanitizeVarArray(varNew, varyLen, varyParams, i);
mtAll[i] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew));
if(mtAll[i] == NULL) {
mtAll[i].varLen = sanitizeVarArray(mtAll[i].varArray, varyLen, varyParams, i);
}
/* no memoization */
if(g_memoTableLog == 0) {
for(i = 1; i <= (int)ZSTD_btultra; i++) {
mtAll[i].tableType = noMemo;
mtAll[i].table = NULL;
mtAll[i].tableLen = 0;
}
return mtAll;
}
/* hash table if normal table is too big */
for(i = 1; i <= (int)ZSTD_btultra; i++) {
size_t mtl = memoTableLen(mtAll[i].varArray, mtAll[i].varLen);
mtAll[i].tableType = directMap;
if(g_memoTableLog != PARAM_UNSET && mtl > (1ULL << g_memoTableLog)) { /* use hash table */ /* provide some option to only use hash tables? */
mtAll[i].tableType = xxhashMap;
mtl = (1ULL << g_memoTableLog);
}
mtAll[i].table = (BYTE*)calloc(sizeof(BYTE), mtl);
mtAll[i].tableLen = mtl;
if(mtAll[i].table == NULL) {
freeMemoTableArray(mtAll);
return NULL;
}
initMemoTable(mtAll[i], paramConstraints, target, varNew, varLenNew, srcSize);
}
return mtAll;
}
static paramValues_t overwriteParams(paramValues_t base, paramValues_t mask) {
static paramValues_t overwriteParams(paramValues_t base, const paramValues_t mask) {
U32 i;
for(i = 0; i < NUM_PARAMS; i++) {
if(mask.vals[i] != PARAM_UNSET) {
@@ -1563,10 +1639,6 @@ static paramValues_t overwriteParams(paramValues_t base, paramValues_t mask) {
#define PARAMTABLEMASK (PARAMTABLESIZE-1)
static BYTE g_alreadyTested[PARAMTABLESIZE] = {0}; /* init to zero */
/*
#define NB_TESTS_PLAYED(p) \
g_alreadyTested[(XXH64(((void*)&sanitizeParams(p), sizeof(p), 0) >> 3) & PARAMTABLEMASK] */
static BYTE* NB_TESTS_PLAYED(ZSTD_compressionParameters p) {
ZSTD_compressionParameters p2 = pvalsToCParams(sanitizeParams(cParamsToPVals(p)));
return &g_alreadyTested[(XXH64((void*)&p2, sizeof(p2), 0) >> 3) & PARAMTABLEMASK];
@@ -1574,12 +1646,10 @@ static BYTE* NB_TESTS_PLAYED(ZSTD_compressionParameters p) {
static void playAround(FILE* f, oldWinnerInfo_t* winners,
ZSTD_compressionParameters params,
buffers_t buf, contexts_t ctx)
const buffers_t buf, const contexts_t ctx)
{
int nbVariations = 0;
int nbVariations = 0, i;
UTIL_time_t const clockStart = UTIL_getTime();
const U32 unconstrained[NUM_PARAMS] = { 0, 1, 2, 3, 4, 5, 6 }; /* no fadt */
while (UTIL_clockSpanMicro(clockStart) < g_maxVariationTime) {
paramValues_t p = cParamsToPVals(params);
@@ -1587,7 +1657,9 @@ static void playAround(FILE* f, oldWinnerInfo_t* winners,
BYTE* b;
if (nbVariations++ > g_maxNbVariations) break;
paramVariation(&p, unconstrained, NUM_PARAMS, 4);
do { for(i = 0; i < 4; i++) { paramVaryOnce(FUZ_rand(&g_rand) % (strt_ind + 1), ((FUZ_rand(&g_rand) & 1) << 1) - 1, &p); } }
while(!paramValid(p));
p2 = pvalsToCParams(p);
@@ -1607,49 +1679,38 @@ static void playAround(FILE* f, oldWinnerInfo_t* winners,
}
/* Sets pc to random unmeasured set of parameters */
/* specifiy strategy */
static void randomConstrainedParams(paramValues_t* pc, const memoTable_t* memoTableArray, const ZSTD_strategy st)
{
size_t j;
const memoTable_t mt = memoTableArray[st];
pc->vals[strt_ind] = st;
for(j = 0; j < MIN(1ULL << g_memoTableLog, memoTableLen(mt.varArray, mt.varLen)); j++) {
int i;
for(i = 0; i < NUM_PARAMS; i++) {
varInds_t v = mt.varArray[i];
if(v == strt_ind) continue; //skip, already specified
pc->vals[v] = rangeMap(v, FUZ_rand(&g_rand) % rangetable[v]);
}
if(!(memoTableGet(memoTableArray, *pc))) break; //only pick unpicked params.
}
}
/* Completely random parameter selection */
static ZSTD_compressionParameters randomParams(void)
{
ZSTD_compressionParameters p;
U32 validated = 0;
while (!validated) {
/* totally random entry */
p.chainLog = (FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN))
+ ZSTD_CHAINLOG_MIN;
p.hashLog = (FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN))
+ ZSTD_HASHLOG_MIN;
p.searchLog = (FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN))
+ ZSTD_SEARCHLOG_MIN;
p.windowLog = (FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN))
+ ZSTD_WINDOWLOG_MIN;
p.searchLength=(FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN))
+ ZSTD_SEARCHLENGTH_MIN;
p.targetLength=(FUZ_rand(&g_rand) % (512));
p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btultra +1));
validated = !ZSTD_isError(ZSTD_checkCParams(p));
paramValues_t p; varInds_t v;
for(v = 0; v < NUM_PARAMS; v++) {
p.vals[v] = rangeMap(v, FUZ_rand(&g_rand) % rangetable[v]);
}
return p;
}
/* Sets pc to random unmeasured set of parameters */
static void randomConstrainedParams(paramValues_t* pc, varInds_t* varArray, int varLen, U8* memoTable)
{
size_t tries = memoTableLen(varArray, varLen);
const size_t maxSize = memoTableLen(varArray, varLen);
size_t ind;
do {
ind = (FUZ_rand(&g_rand)) % maxSize;
tries--;
} while(memoTable[ind] > 0 && tries > 0);
memoTableIndInv(pc, varArray, varLen, (unsigned)ind);
return pvalsToCParams(p);
}
static void BMK_selectRandomStart(
FILE* f, oldWinnerInfo_t* winners,
buffers_t buf, contexts_t ctx)
const buffers_t buf, const contexts_t ctx)
{
U32 const id = FUZ_rand(&g_rand) % (NB_LEVELS_TRACKED+1);
if ((id==0) || (winners[id].params.windowLog==0)) {
@@ -1661,7 +1722,7 @@ static void BMK_selectRandomStart(
}
}
static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBlockSize)
static void BMK_benchFullTable(const buffers_t buf, const contexts_t ctx)
{
ZSTD_compressionParameters params;
oldWinnerInfo_t winners[NB_LEVELS_TRACKED+1];
@@ -1677,7 +1738,7 @@ static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBl
BMK_init_level_constraints(g_target * (1 MB));
} else {
/* baseline config for level 1 */
ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, maxBlockSize, ctx.dictSize);
ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, buf.maxBlockSize, ctx.dictSize);
BMK_result_t testResult;
BMK_benchParam(&testResult, buf, ctx, cParamsToPVals(l1params));
BMK_init_level_constraints((int)((testResult.cSpeed * 31) / 32));
@@ -1687,7 +1748,7 @@ static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBl
{ const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel();
int i;
for (i=0; i<=maxSeeds; i++) {
params = ZSTD_getCParams(i, maxBlockSize, 0);
params = ZSTD_getCParams(i, buf.maxBlockSize, 0);
BMK_seed(winners, params, buf, ctx);
} }
BMK_printWinners(f, winners, buf.srcSize);
@@ -1707,7 +1768,7 @@ static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBl
fclose(f);
}
static int benchOnce(buffers_t buf, contexts_t ctx) {
static int benchOnce(const buffers_t buf, const contexts_t ctx) {
BMK_result_t testResult;
if(BMK_benchParam(&testResult, buf, ctx, g_params)) {
@@ -1724,7 +1785,6 @@ static int benchSample(void)
{
const char* const name = "Sample 10MB";
size_t const benchedSize = 10 MB;
U32 blockSize = g_blockSize ? g_blockSize : benchedSize;
void* srcBuffer = malloc(benchedSize);
int ret = 0;
@@ -1757,7 +1817,7 @@ static int benchSample(void)
if(g_singleRun) {
ret = benchOnce(buf, ctx);
} else {
BMK_benchFullTable(buf, ctx, MIN(blockSize, benchedSize));
BMK_benchFullTable(buf, ctx);
}
freeBuffers(buf);
@@ -1773,7 +1833,6 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam
{
buffers_t buf;
contexts_t ctx;
size_t maxBlockSize = 0, i;
int ret = 0;
if(createBuffers(&buf, fileNamesTable, nbFiles)) {
@@ -1787,10 +1846,6 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam
return 2;
}
for(i = 0; i < buf.nbBlocks; i++) {
maxBlockSize = MAX(maxBlockSize, buf.srcSizes[i]);
}
DISPLAY("\r%79s\r", "");
if(nbFiles == 1) {
DISPLAY("using %s : \n", fileNamesTable[0]);
@@ -1798,12 +1853,12 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam
DISPLAY("using %d Files : \n", nbFiles);
}
g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize)), g_params), maxBlockSize, ctx.dictSize);
g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevel, buf.maxBlockSize, ctx.dictSize)), g_params), buf.maxBlockSize, ctx.dictSize);
if(g_singleRun) {
ret = benchOnce(buf, ctx);
} else {
BMK_benchFullTable(buf, ctx, maxBlockSize);
BMK_benchFullTable(buf, ctx);
}
freeBuffers(buf);
@@ -1811,68 +1866,62 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam
return ret;
}
#define CBENCHMARK(conditional, resultvar, tmpret, mode, loopmode, sec) { \
if(conditional) { \
BMK_return_t tmpret = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, mode, loopmode, sec); \
if(tmpret.error) { DEBUGOUTPUT("Benchmarking failed\n"); return ERROR_RESULT; } \
if(mode != BMK_decodeOnly) { \
resultvar.cSpeed = tmpret.result.cSpeed; \
resultvar.cSize = tmpret.result.cSize; \
resultvar.cMem = tmpret.result.cMem; \
} \
if(mode != BMK_compressOnly) { resultvar.dSpeed = tmpret.result.dSpeed; } \
} \
}
/* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */
#define VARIANCE 1.2
#define HIGH_VARIANCE 100.0
static int allBench(BMK_result_t* resultPtr,
const buffers_t buf, const contexts_t ctx,
const paramValues_t cParams,
const constraint_t target,
BMK_result_t* winnerResult, int feas) {
BMK_return_t benchres;
BMK_result_t resultMax;
BMK_result_t resultMax, benchres;
U64 loopDurationC = 0, loopDurationD = 0;
double uncertaintyConstantC = 3., uncertaintyConstantD = 3.;
double winnerRS;
/* initial benchmarking, gives exact ratio and memory, warms up future runs */
benchres = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_both, BMK_iterMode, 1);
CBENCHMARK(1, benchres, tmp, BMK_both, BMK_iterMode, 1);
winnerRS = resultScore(*winnerResult, buf.srcSize, target);
DEBUGOUTPUT("WinnerScore: %f\n ", winnerRS);
if(benchres.error) {
DEBUGOUTPUT("Benchmarking failed\n");
return ERROR_RESULT;
}
*resultPtr = benchres.result;
*resultPtr = benchres;
/* calculate uncertainty in compression / decompression runs */
if(benchres.result.cSpeed) {
loopDurationC = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed);
if(benchres.cSpeed) {
loopDurationC = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.cSpeed);
uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC);
}
if(benchres.result.dSpeed) {
loopDurationD = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed);
if(benchres.dSpeed) {
loopDurationD = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.dSpeed);
uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD);
}
/* anything with worse ratio in feas is definitely worse, discard */
if(feas && benchres.result.cSize < winnerResult->cSize && !g_optmode) {
if(feas && benchres.cSize < winnerResult->cSize && !g_optmode) {
return WORSE_RESULT;
}
/* second run, if first run is too short, gives approximate cSpeed + dSpeed */
if(loopDurationC < TIMELOOP_NANOSEC / 10) {
BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_compressOnly, BMK_iterMode, 1);
if(benchres2.error) {
return ERROR_RESULT;
}
benchres = benchres2;
}
if(loopDurationD < TIMELOOP_NANOSEC / 10) {
BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_decodeOnly, BMK_iterMode, 1);
if(benchres2.error) {
return ERROR_RESULT;
}
benchres.result.dSpeed = benchres2.result.dSpeed;
}
CBENCHMARK(loopDurationC < TIMELOOP_NANOSEC / 10, benchres, tmp, BMK_compressOnly, BMK_iterMode, 1);
CBENCHMARK(loopDurationD < TIMELOOP_NANOSEC / 10, benchres, tmp, BMK_decodeOnly, BMK_iterMode, 1);
*resultPtr = benchres.result;
*resultPtr = benchres;
/* optimistic assumption of benchres.result */
resultMax = benchres.result;
/* optimistic assumption of benchres */
resultMax = benchres;
resultMax.cSpeed *= uncertaintyConstantC * VARIANCE;
resultMax.dSpeed *= uncertaintyConstantD * VARIANCE;
@@ -1883,29 +1932,15 @@ static int allBench(BMK_result_t* resultPtr,
return WORSE_RESULT;
}
/* Final full run if estimates are unclear */
if(loopDurationC < TIMELOOP_NANOSEC) {
BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_compressOnly, BMK_timeMode, 1);
if(benchres2.error) {
return ERROR_RESULT;
}
benchres.result.cSpeed = benchres2.result.cSpeed;
}
CBENCHMARK(loopDurationC < TIMELOOP_NANOSEC, benchres, tmp, BMK_compressOnly, BMK_timeMode, 1);
CBENCHMARK(loopDurationD < TIMELOOP_NANOSEC, benchres, tmp, BMK_decodeOnly, BMK_timeMode, 1);
if(loopDurationD < TIMELOOP_NANOSEC) {
BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_decodeOnly, BMK_timeMode, 1);
if(benchres2.error) {
return ERROR_RESULT;
}
benchres.result.dSpeed = benchres2.result.dSpeed;
}
*resultPtr = benchres.result;
*resultPtr = benchres;
/* compare by resultScore when in infeas */
/* compare by compareResultLT when in feas */
if((!feas && (resultScore(benchres.result, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target))) ||
(feas && (compareResultLT(*winnerResult, benchres.result, target, buf.srcSize))) ) {
if((!feas && (resultScore(benchres, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target))) ||
(feas && (compareResultLT(*winnerResult, benchres, target, buf.srcSize))) ) {
return BETTER_RESULT;
} else {
return WORSE_RESULT;
@@ -1913,19 +1948,17 @@ static int allBench(BMK_result_t* resultPtr,
}
#define INFEASIBLE_THRESHOLD 200
/* Memoized benchmarking, won't benchmark anything which has already been benchmarked before. */
static int benchMemo(BMK_result_t* resultPtr,
const buffers_t buf, const contexts_t ctx,
const paramValues_t cParams,
const constraint_t target,
BMK_result_t* winnerResult, U8* const memoTable,
const varInds_t* varyParams, const int varyLen, const int feas) {
BMK_result_t* winnerResult, memoTable_t* const memoTableArray,
const int feas) {
static int bmcount = 0;
size_t memind = memoTableInd(&cParams, varyParams, varyLen);
int res;
if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { return WORSE_RESULT; }
if(memoTableGet(memoTableArray, cParams) >= INFEASIBLE_THRESHOLD || redundantParams(cParams, target, buf.maxBlockSize)) { return WORSE_RESULT; }
res = allBench(resultPtr, buf, ctx, cParams, target, winnerResult, feas);
@@ -1936,7 +1969,7 @@ static int benchMemo(BMK_result_t* resultPtr,
BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, *resultPtr, cParams, target, buf.srcSize);
if(res == BETTER_RESULT || feas) {
memoTable[memind] = 255;
memoTableSet(memoTableArray, cParams, 255); /* what happens if collisions are frequent */
}
return res;
}
@@ -1958,9 +1991,8 @@ static int benchMemo(BMK_result_t* resultPtr,
* all generation after random should be sanitized. (maybe sanitize random)
*/
static winnerInfo_t climbOnce(const constraint_t target,
const varInds_t* varArray, const int varLen, ZSTD_strategy strat,
U8** memoTableArray,
buffers_t buf, contexts_t ctx,
memoTable_t* mtAll,
const buffers_t buf, const contexts_t ctx,
const paramValues_t init) {
/*
* cparam - currently considered 'center'
@@ -1971,8 +2003,6 @@ static winnerInfo_t climbOnce(const constraint_t target,
winnerInfo_t candidateInfo, winnerInfo;
int better = 1;
int feas = 0;
varInds_t varNew[NUM_PARAMS];
int varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat);
winnerInfo = initWinnerInfo(init);
candidateInfo = winnerInfo;
@@ -1982,7 +2012,9 @@ static winnerInfo_t climbOnce(const constraint_t target,
DEBUGOUTPUT("Climb Part 1\n");
while(better) {
int i, dist, offset;
int offset;
size_t i, dist;
const size_t varLen = mtAll[cparam.vals[strt_ind]].varLen;
better = 0;
DEBUGOUTPUT("Start\n");
cparam = winnerInfo.params;
@@ -1993,17 +2025,13 @@ static winnerInfo_t climbOnce(const constraint_t target,
for(offset = -1; offset <= 1; offset += 2) {
CHECKTIME(winnerInfo);
candidateInfo.params = cparam;
paramVaryOnce(varArray[i], offset, &candidateInfo.params);
paramVaryOnce(mtAll[cparam.vals[strt_ind]].varArray[i], offset, &candidateInfo.params);
if(paramValid(candidateInfo.params)) {
int res;
if(strat != candidateInfo.params.vals[strt_ind]) { /* maybe only try strategy switching after exhausting non-switching solutions? */
strat = candidateInfo.params.vals[strt_ind];
varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat);
}
res = benchMemo(&candidateInfo.result, buf, ctx,
sanitizeParams(candidateInfo.params), target, &winnerInfo.result, memoTableArray[strat],
varNew, varLenNew, feas);
sanitizeParams(candidateInfo.params), target, &winnerInfo.result, mtAll, feas);
DEBUGOUTPUT("Res: %d\n", res);
if(res == BETTER_RESULT) { /* synonymous with better when called w/ infeasibleBM */
winnerInfo = candidateInfo;
BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, sanitizeParams(winnerInfo.params), target, buf.srcSize);
@@ -2026,16 +2054,11 @@ static winnerInfo_t climbOnce(const constraint_t target,
CHECKTIME(winnerInfo);
candidateInfo.params = cparam;
/* param error checking already done here */
paramVariation(&candidateInfo.params, varArray, varLen, dist);
if(strat != candidateInfo.params.vals[strt_ind]) {
strat = candidateInfo.params.vals[strt_ind];
varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat);
}
paramVariation(&candidateInfo.params, mtAll, (U32)dist);
res = benchMemo(&candidateInfo.result, buf, ctx,
sanitizeParams(candidateInfo.params), target, &winnerInfo.result, memoTableArray[strat],
varNew, varLenNew, feas);
sanitizeParams(candidateInfo.params), target, &winnerInfo.result, mtAll, feas);
DEBUGOUTPUT("Res: %d\n", res);
if(res == BETTER_RESULT) { /* synonymous with better in this case*/
winnerInfo = candidateInfo;
BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, sanitizeParams(winnerInfo.params), target, buf.srcSize);
@@ -2079,11 +2102,8 @@ static winnerInfo_t optimizeFixedStrategy(
const buffers_t buf, const contexts_t ctx,
const constraint_t target, paramValues_t paramTarget,
const ZSTD_strategy strat,
const varInds_t* varArray, const int varLen,
U8** memoTableArray, const int tries) {
memoTable_t* memoTableArray, const int tries) {
int i = 0;
varInds_t varNew[NUM_PARAMS];
int varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat);
paramValues_t init;
winnerInfo_t winnerInfo, candidateInfo;
@@ -2091,19 +2111,19 @@ static winnerInfo_t optimizeFixedStrategy(
/* so climb is given the right fixed strategy */
paramTarget.vals[strt_ind] = strat;
/* to pass ZSTD_checkCParams */
paramTarget = cParamUnsetMin(paramTarget);
init = paramTarget;
while(i < tries) {
for(i = 0; i < tries; i++) {
DEBUGOUTPUT("Restart\n");
randomConstrainedParams(&init, varNew, varLenNew, memoTableArray[strat]);
candidateInfo = climbOnce(target, varArray, varLen, strat, memoTableArray, buf, ctx, init);
do { randomConstrainedParams(&init, memoTableArray, strat); } while(redundantParams(init, target, buf.maxBlockSize)); //only non-redundant params
candidateInfo = climbOnce(target, memoTableArray, buf, ctx, init);
if(compareResultLT(winnerInfo.result, candidateInfo.result, target, buf.srcSize)) {
winnerInfo = candidateInfo;
BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize);
i = 0;
continue;
}
CHECKTIME(winnerInfo);
i++;
@@ -2163,19 +2183,14 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
{
varInds_t varArray [NUM_PARAMS];
int ret = 0;
const int varLen = variableParams(paramTarget, varArray, dictFileName != NULL);
const size_t varLen = variableParams(paramTarget, varArray, dictFileName != NULL);
winnerInfo_t winner = initWinnerInfo(emptyParams());
U8** allMT = NULL;
paramValues_t paramBase = cParamUnsetMin(paramTarget);
size_t k, maxBlockSize = 0;
memoTable_t* allMT = NULL;
paramValues_t paramBase;
contexts_t ctx;
buffers_t buf;
g_time = UTIL_getTime();
if(!paramValid(paramBase)) {
return 1;
}
if(createBuffers(&buf, fileNamesTable, nbFiles)) {
DISPLAY("unable to load files\n");
return 1;
@@ -2193,43 +2208,23 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
DISPLAY("Loading %lu Files... \r", (unsigned long)nbFiles);
}
/* sanitize paramTarget */
optimizerAdjustInput(&paramTarget, buf.maxBlockSize);
paramBase = cParamUnsetMin(paramTarget);
for(k = 0; k < buf.nbBlocks; k++) {
maxBlockSize = MAX(buf.srcSizes[k], maxBlockSize);
}
// TODO: if strategy is fixed, only init that row
allMT = createMemoTableArray(varArray, varLen);
/* if strategy is fixed, only init that part of memotable */
if(paramTarget.vals[strt_ind] != PARAM_UNSET) {
varInds_t varNew[NUM_PARAMS];
int varLenNew = sanitizeVarArray(varNew, varLen, varArray, paramTarget.vals[strt_ind]);
allMT = (U8**)calloc(sizeof(U8*), (ZSTD_btultra + 1));
if(allMT == NULL) {
ret = 57;
goto _cleanUp;
}
allMT[paramTarget.vals[strt_ind]] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew));
if(allMT[paramTarget.vals[strt_ind]] == NULL) {
ret = 58;
goto _cleanUp;
}
initMemoTable(allMT[paramTarget.vals[strt_ind]], paramTarget, target, varNew, varLenNew, maxBlockSize);
} else {
allMT = createMemoTableArray(paramTarget, target, varArray, varLen, maxBlockSize);
}
if(!allMT) {
DISPLAY("MemoTable Init Error\n");
ret = 2;
goto _cleanUp;
}
/* default strictness = Maximum for */
if(g_strictness == DEFAULT_STRICTNESS) {
/* default strictnesses */
if(g_strictness == PARAM_UNSET) {
if(g_optmode) {
g_strictness = 99;
g_strictness = 100;
} else {
g_strictness = 90;
}
@@ -2244,7 +2239,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
/* use level'ing mode instead of normal target mode */
/* Should lvl be parameter-masked here? */
if(g_optmode) {
winner.params = cParamsToPVals(ZSTD_getCParams(cLevelOpt, maxBlockSize, ctx.dictSize));
winner.params = cParamsToPVals(ZSTD_getCParams(cLevelOpt, buf.maxBlockSize, ctx.dictSize));
if(BMK_benchParam(&winner.result, buf, ctx, winner.params)) {
ret = 3;
goto _cleanUp;
@@ -2264,7 +2259,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
/* Don't want it to return anything worse than the best known result */
if(g_singleRun) {
BMK_result_t res;
g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevelRun, maxBlockSize, ctx.dictSize)), g_params), maxBlockSize, ctx.dictSize);
g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevelRun, buf.maxBlockSize, ctx.dictSize)), g_params), buf.maxBlockSize, ctx.dictSize);
if(BMK_benchParam(&res, buf, ctx, g_params)) {
ret = 45;
goto _cleanUp;
@@ -2303,7 +2298,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
int i;
for (i=1; i<=maxSeeds; i++) {
int ec;
CParams = overwriteParams(cParamsToPVals(ZSTD_getCParams(i, maxBlockSize, ctx.dictSize)), paramTarget);
CParams = overwriteParams(cParamsToPVals(ZSTD_getCParams(i, buf.maxBlockSize, ctx.dictSize)), paramTarget);
ec = BMK_benchParam(&candidate, buf, ctx, CParams);
BMK_printWinnerOpt(stdout, i, candidate, CParams, target, buf.srcSize);
@@ -2323,7 +2318,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
}
DEBUGOUTPUT("Real Opt\n");
/* start 'real' tests */
/* start 'real' optimization */
{
int bestStrategy = (int)winner.params.vals[strt_ind];
if(paramTarget.vals[strt_ind] == PARAM_UNSET) {
@@ -2332,8 +2327,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
{
/* one iterations of hill climbing with the level-defined parameters. */
winnerInfo_t w1 = climbOnce(target, varArray, varLen, st, allMT,
buf, ctx, winner.params);
winnerInfo_t w1 = climbOnce(target, allMT, buf, ctx, winner.params);
if(compareResultLT(winner.result, w1.result, target, buf.srcSize)) {
winner = w1;
}
@@ -2344,8 +2338,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
winnerInfo_t wc;
DEBUGOUTPUT("StrategySwitch: %s\n", g_stratName[st]);
wc = optimizeFixedStrategy(buf, ctx, target, paramBase,
st, varArray, varLen, allMT, tries);
wc = optimizeFixedStrategy(buf, ctx, target, paramBase, st, allMT, tries);
if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) {
winner = wc;
@@ -2358,8 +2351,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
CHECKTIMEGT(ret, 0, _cleanUp);
}
} else {
winner = optimizeFixedStrategy(buf, ctx, target, paramBase, paramTarget.vals[strt_ind],
varArray, varLen, allMT, g_maxTries);
winner = optimizeFixedStrategy(buf, ctx, target, paramBase, paramTarget.vals[strt_ind], allMT, g_maxTries);
}
}
@@ -2419,6 +2411,22 @@ static unsigned readU32FromChar(const char** stringPtr)
return result * sign;
}
static double readDoubleFromChar(const char** stringPtr)
{
double result = 0, divide = 10;
while ((**stringPtr >='0') && (**stringPtr <='9')) {
result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
}
if(**stringPtr!='.') {
return result;
}
(*stringPtr)++;
while ((**stringPtr >='0') && (**stringPtr <='9')) {
result += (double)(**stringPtr - '0') / divide, divide *= 10, (*stringPtr)++ ;
}
return result;
}
static int usage(const char* exename)
{
DISPLAY( "Usage :\n");
@@ -2454,16 +2462,24 @@ static int badusage(const char* exename)
}
#define PARSE_SUB_ARGS(stringLong, stringShort, variable) { if (longCommandWArg(&argument, stringLong) || longCommandWArg(&argument, stringShort)) { variable = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } }
#define PARSE_CPARAMS(variable) \
{ \
PARSE_SUB_ARGS("windowLog=", "wlog=", variable.vals[wlog_ind]); \
PARSE_SUB_ARGS("chainLog=" , "clog=", variable.vals[clog_ind]); \
PARSE_SUB_ARGS("hashLog=", "hlog=", variable.vals[hlog_ind]); \
PARSE_SUB_ARGS("searchLog=" , "slog=", variable.vals[slog_ind]); \
PARSE_SUB_ARGS("searchLength=", "slen=", variable.vals[slen_ind]); \
PARSE_SUB_ARGS("targetLength=" , "tlen=", variable.vals[tlen_ind]); \
PARSE_SUB_ARGS("strategy=", "strat=", variable.vals[strt_ind]); \
PARSE_SUB_ARGS("forceAttachDict=", "fad=" , variable.vals[fadt_ind]); \
/* 1 if successful parse, 0 otherwise */
static int parse_params(const char** argptr, paramValues_t* pv) {
int matched = 0;
const char* argOrig = *argptr;
varInds_t v;
for(v = 0; v < NUM_PARAMS; v++) {
if(longCommandWArg(argptr,g_shortParamNames[v]) || longCommandWArg(argptr, g_paramNames[v])) {
if(**argptr == '=') {
(*argptr)++;
pv->vals[v] = readU32FromChar(argptr);
matched = 1;
break;
}
}
/* reset and try again */
*argptr = argOrig;
}
return matched;
}
int main(int argc, const char** argv)
@@ -2497,15 +2513,15 @@ int main(int argc, const char** argv)
if (longCommandWArg(&argument, "--optimize=")) {
g_optimizer = 1;
for ( ; ;) {
PARSE_CPARAMS(paramTarget);
if(parse_params(&argument, &paramTarget)) { if(argument[0] == ',') { argument++; continue; } else break; }
PARSE_SUB_ARGS("compressionSpeed=" , "cSpeed=", target.cSpeed);
PARSE_SUB_ARGS("decompressionSpeed=", "dSpeed=", target.dSpeed);
PARSE_SUB_ARGS("compressionMemory=" , "cMem=", target.cMem);
PARSE_SUB_ARGS("strict=", "stc=", g_strictness);
PARSE_SUB_ARGS("preferSpeed=", "prfSpd=", g_speedMultiplier);
PARSE_SUB_ARGS("preferRatio=", "prfRto=", g_ratioMultiplier);
PARSE_SUB_ARGS("maxTries=", "tries=", g_maxTries);
PARSE_SUB_ARGS("memoLimitLog=", "memLog=", g_memoTableLog);
if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevelOpt = readU32FromChar(&argument); g_optmode = 1; if (argument[0]==',') { argument++; continue; } else break; }
if (longCommandWArg(&argument, "speedForRatio=") || longCommandWArg(&argument, "speedRatio=")) { g_ratioMultiplier = readDoubleFromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
DISPLAY("invalid optimization parameter \n");
return 1;
@@ -2520,7 +2536,7 @@ int main(int argc, const char** argv)
/* Decode command (note : aggregated commands are allowed) */
g_singleRun = 1;
for ( ; ;) {
PARSE_CPARAMS(g_params)
if(parse_params(&argument, &g_params)) { if(argument[0] == ',') { argument++; continue; } else break; }
if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevelRun = readU32FromChar(&argument); g_params = emptyParams(); if (argument[0]==',') { argument++; continue; } else break; }
DISPLAY("invalid compression parameter \n");
@@ -2633,6 +2649,7 @@ int main(int argc, const char** argv)
break;
case 's':
argument++;
seperateFiles = 1;
break;