From cffb6da339970c7a5334e52abdf96493764561f7 Mon Sep 17 00:00:00 2001 From: George Lu Date: Wed, 6 Jun 2018 16:19:09 -0700 Subject: [PATCH 01/55] Parses additional parameters Additional constraint checking Minor fixes more param parsing Add Memory Change paramVariation work on feasibility reformat bench Changed Paramgrill to use bench.c benchmarking customlevel macro Printing Flag Minor changes Explicit casting Makefile fix casting, type fix Printing Flag Minor Changes comments, helper fn's --- programs/bench.c | 2 + programs/bench.h | 1 + tests/paramgrill.c | 369 ++++++++++++++++++++++++++++++++++++++------- 3 files changed, 320 insertions(+), 52 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 76d1ff6dd..2fa9262c9 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -692,6 +692,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( } DISPLAYLEVEL(2, "%2i#\n", cLevel); } /* Bench */ + results.result.cMem = ZSTD_sizeof_CCtx(ctx); return results; } @@ -731,6 +732,7 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, srcBuffer, srcSize, fileSizes, nbFiles, cLevel, comprParams, dictBuffer, dictBufferSize, ctx, dctx, displayLevel, displayName, adv); } + /* clean up */ BMK_freeTimeState(timeStateCompress); BMK_freeTimeState(timeStateDecompress); diff --git a/programs/bench.h b/programs/bench.h index 87cf56380..f1ac255f8 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -34,6 +34,7 @@ typedef struct { size_t cSize; double cSpeed; /* bytes / sec */ double dSpeed; + size_t cMem; } BMK_result_t; ERROR_STRUCT(BMK_result_t, BMK_return_t); diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 025bc6aad..81c6dffc2 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -63,6 +63,16 @@ static const int g_maxNbVariations = 64; #define MAX(a,b) ( (a) > (b) ? (a) : (b) ) #define CUSTOM_LEVEL 99 +/* indices for each of the variables */ +#define WLOG_IND 0 +#define CLOG_IND 1 +#define HLOG_IND 2 +#define SLOG_IND 3 +#define SLEN_IND 4 +#define TLEN_IND 5 +#define STRT_IND 6 +#define NUM_PARAMS 7 + /*-************************************ * Benchmark Parameters **************************************/ @@ -140,6 +150,13 @@ static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) return result; } + +typedef struct { + U32 cSpeed; /* bytes / sec */ + U32 dSpeed; + U32 Mem; /* bytes */ +} constraint_t; + /*-******************************************************* * Bench functions *********************************************************/ @@ -340,7 +357,6 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para return better; } - /* nullified useless params, to ensure count stats */ static ZSTD_compressionParameters* sanitizeParams(ZSTD_compressionParameters params) { @@ -354,8 +370,114 @@ static ZSTD_compressionParameters* sanitizeParams(ZSTD_compressionParameters par return &g_params; } +/* res should be NUM_PARAMS size */ +static int variableParams(const ZSTD_compressionParameters paramConstraints, U32* res) { + int j = 0; + if(!paramConstraints.windowLog) { + res[j] = WLOG_IND; + j++; + } + if(!paramConstraints.chainLog) { + res[j] = CLOG_IND; + j++; + } + if(!paramConstraints.hashLog) { + res[j] = HLOG_IND; + j++; + } + if(!paramConstraints.searchLog) { + res[j] = SLOG_IND; + j++; + } + if(!paramConstraints.searchLength) { + res[j] = SLEN_IND; + j++; + } + if(!paramConstraints.targetLength) { + res[j] = TLEN_IND; + j++; + } + if(!(U32)paramConstraints.strategy) { + res[j] = STRT_IND; + j++; + } + return j; +} -static void paramVariation(ZSTD_compressionParameters* ptr) +/* computes inverse of above array, returns same number, -1 = unused ind */ +static int inverseVariableParams(const ZSTD_compressionParameters paramConstraints, U32* res) { + int j = 0; + if(!paramConstraints.windowLog) { + res[WLOG_IND] = j; + j++; + } else { + res[WLOG_IND] = -1; + } + if(!paramConstraints.chainLog) { + res[j] = CLOG_IND; + j++; + } else { + res[WLOG_IND] = -1; + } + if(!paramConstraints.hashLog) { + res[j] = HLOG_IND; + j++; + } else { + res[WLOG_IND] = -1; + } + if(!paramConstraints.searchLog) { + res[j] = SLOG_IND; + j++; + } else { + res[WLOG_IND] = -1; + } + if(!paramConstraints.searchLength) { + res[j] = SLEN_IND; + j++; + } else { + res[WLOG_IND] = -1; + } + if(!paramConstraints.targetLength) { + res[j] = TLEN_IND; + j++; + } else { + res[WLOG_IND] = -1; + } + if(!(U32)paramConstraints.strategy) { + res[j] = STRT_IND; + j++; + } else { + res[WLOG_IND] = -1; + } + return j; +} + +/* amt will probably always be \pm 1? */ +/* slight change from old paramVariation, targetLength can only take on powers of 2 now (999 ~= 1024?) */ +static void paramVaryOnce(U32 paramIndex, int amt, ZSTD_compressionParameters* ptr) { + switch(paramIndex) + { + case WLOG_IND: ptr->windowLog += amt; break; + case CLOG_IND: ptr->chainLog += amt; break; + case HLOG_IND: ptr->hashLog += amt; break; + case SLOG_IND: ptr->searchLog += amt; break; + case SLEN_IND: ptr->searchLength += amt; break; + case TLEN_IND: + if(amt > 0) { + ptr->targetLength <<= amt; + } else { + ptr->targetLength >>= -amt; + } + break; + case STRT_IND: ptr->strategy += amt; break; + default: break; + } +} + +//Don't fuzz fixed variables. +//turn pcs to pcs array with macro for params. +//pass in variation array from variableParams +static void paramVariation(ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen) { ZSTD_compressionParameters p; U32 validated = 0; @@ -363,40 +485,13 @@ static void paramVariation(ZSTD_compressionParameters* ptr) U32 nbChanges = (FUZ_rand(&g_rand) & 3) + 1; p = *ptr; for ( ; nbChanges ; nbChanges--) { - const U32 changeID = FUZ_rand(&g_rand) % 14; - switch(changeID) - { - case 0: - p.chainLog++; break; - case 1: - p.chainLog--; break; - case 2: - p.hashLog++; break; - case 3: - p.hashLog--; break; - case 4: - p.searchLog++; break; - case 5: - p.searchLog--; break; - case 6: - p.windowLog++; break; - case 7: - p.windowLog--; break; - case 8: - p.searchLength++; break; - case 9: - p.searchLength--; break; - case 10: - p.strategy = (ZSTD_strategy)(((U32)p.strategy)+1); break; - case 11: - p.strategy = (ZSTD_strategy)(((U32)p.strategy)-1); break; - case 12: - p.targetLength *= 1 + ((double)(FUZ_rand(&g_rand)&255)) / 256.; break; - case 13: - p.targetLength /= 1 + ((double)(FUZ_rand(&g_rand)&255)) / 256.; break; - } + const U32 changeID = FUZ_rand(&g_rand) % (2 * varyLen); + paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p); } validated = !ZSTD_isError(ZSTD_checkCParams(p)); + + //Make sure memory is at least close to feasible? + //ZSTD_estimateCCtxSize thing. } *ptr = p; } @@ -418,12 +513,14 @@ static void playAround(FILE* f, winnerInfo_t* winners, { int nbVariations = 0; UTIL_time_t const clockStart = UTIL_getTime(); + const U32 unconstrained[NUM_PARAMS] = { 0, 1, 2, 3, 4, 5, 6 }; + while (UTIL_clockSpanMicro(clockStart) < g_maxVariationTime) { ZSTD_compressionParameters p = params; if (nbVariations++ > g_maxNbVariations) break; - paramVariation(&p); + paramVariation(&p, unconstrained, 7); /* exclude faster if already played params */ if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(p))-1)) @@ -459,6 +556,24 @@ static ZSTD_compressionParameters randomParams(void) return p; } +static ZSTD_compressionParameters randomConstrainedParams(ZSTD_compressionParameters pc) +{ + ZSTD_compressionParameters p; + U32 validated = 0; + while (!validated) { + /* totally random entry */ + if(!pc.chainLog) p.chainLog = (FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN)) + ZSTD_CHAINLOG_MIN; + if(!pc.chainLog) p.hashLog = (FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN)) + ZSTD_HASHLOG_MIN; + if(!pc.chainLog) p.searchLog = (FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN)) + ZSTD_SEARCHLOG_MIN; + if(!pc.chainLog) p.windowLog = (FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN)) + ZSTD_WINDOWLOG_MIN; + if(!pc.chainLog) p.searchLength=(FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN)) + ZSTD_SEARCHLENGTH_MIN; + if(!pc.chainLog) p.targetLength=(FUZ_rand(&g_rand) % (512)) + 1; //ZSTD_TARGETLENGTH_MIN; //change to 2^[0,10?] + if(!pc.chainLog) p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btultra +1)); + validated = !ZSTD_isError(ZSTD_checkCParams(p)); + } + return p; +} + static void BMK_selectRandomStart( FILE* f, winnerInfo_t* winners, const void* srcBuffer, size_t srcSize, @@ -638,14 +753,86 @@ static void BMK_translateAdvancedParams(ZSTD_compressionParameters params) params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, params.targetLength, (U32)(params.strategy)); } +//Results currently don't capture memory usage or anything. +//parameter feasibility is not checked, should just be restricted from use. +static int feasible(BMK_result_t results, constraint_t target) { + return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.Mem || !target.Mem); +} + +/* returns 1 if result2 is strictly 'better' than result1 */ +static int objective_lt(BMK_result_t result1, BMK_result_t result2) { + return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed); +} + +/* res gives array dimensions, should be size NUM_PARAMS */ +static size_t computeStateSize(const ZSTD_compressionParameters paramConstraints, U32* res) { + int ind = 0; + size_t base = 1; + if(!paramConstraints.windowLog) { res[ind] = ZSTD_WINDOWLOG_MAX - ZSTD_WINDOWLOG_MIN + 1; base *= res[ind]; ind++; } + if(!paramConstraints.chainLog) { res[ind] = ZSTD_CHAINLOG_MAX - ZSTD_CHAINLOG_MIN + 1; base *= res[ind]; ind++; } + if(!paramConstraints.hashLog) { res[ind] = ZSTD_HASHLOG_MAX - ZSTD_HASHLOG_MIN + 1; base *= res[ind]; ind++; } + if(!paramConstraints.searchLog) { res[ind] = ZSTD_SEARCHLOG_MAX - ZSTD_SEARCHLOG_MIN + 1; base *= res[ind]; ind++; } + if(!paramConstraints.searchLength) { res[ind] = ZSTD_SEARCHLENGTH_MAX - ZSTD_SEARCHLENGTH_MIN + 1; base *= res[ind]; ind++; } + if(!paramConstraints.targetLength) { res[ind] = 11; base *= res[ind]; ind++; } //restricting from 2^[0,10], no such macros + if(!(U32)paramConstraints.strategy) { res[ind] = 8; base *= 8; } //not strictly true, maybe would want to case on this. + + return base; +} + + +static unsigned calcViolation(BMK_result_t results, constraint_t target) { + int diffcSpeed = MAX(target.cSpeed - results.cSpeed, 0); + int diffdSpeed = MAX(target.dSpeed - results.dSpeed, 0); + int diffcMem = MAX(results.cMem - target.Mem, 0); + return diffcSpeed + diffdSpeed + diffcMem; +} + +/* finds some set of parameters which fulfills req's + * Prioritize highest / try to locally minimize sum? + * Is it ever useful to go out of the param constraints? ? + * random / perturb when revisit? + * momentum? + */ +static ZSTD_compressionParameters findFeasible(constraint_t target, ZSTD_compressionParameters paramTarget) { + unsigned violation; + + ZSTD_compressionParameters winner = randomConstrainedParams(paramTarget); + //just use g_alreadyTested and xxhash? + BYTE* memotable; + do { + //prioritize memory + if(diffcMem >= diffcSpeed && diffcMem >= diffdSpeed) { + + //prioritize compression Speed + } else if (diffcSpeeed >= diffdSpeed && diffcSpeed >= diffcMem) { + + //prioritize decompressionSpeed + } else { + + } + violation = calcViolation(result, target); + } while(objective); + if(validate) { + DISPLAY("Feasible Point Found\n"); + return winner; + } else { + DISPLAY("No solution found\n"); + ZSTD_compressionParameters ret = { 0, 0, 0, 0, 0, 0, 0 }; + return ret; + } +} + /* optimizeForSize(): - * targetSpeed : expressed in MB/s */ -int optimizeForSize(const char* inFileName, U32 targetSpeed) + * targetSpeed : expressed in B/s */ +/* if state space is small (from paramTarget) */ +int optimizeForSize(const char* inFileName, constraint_t target, ZSTD_compressionParameters paramTarget) { FILE* const inFile = fopen( inFileName, "rb" ); U64 const inFileSize = UTIL_getFileSize(inFileName); size_t benchedSize = BMK_findMaxMem(inFileSize*3) / 3; void* origBuff; + U32 paramVarArray [NUM_PARAMS]; + int paramCount = variableParams(paramTarget, paramVarArray); /* Init */ if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; } if (inFileSize == UTIL_FILESIZE_UNKNOWN) { @@ -682,8 +869,11 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed) /* bench */ DISPLAY("\r%79s\r", ""); - DISPLAY("optimizing for %s - limit speed %u MB/s \n", inFileName, targetSpeed); - targetSpeed *= 1000000; + DISPLAY("optimizing for %s", inFileName); + if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed / 1000000); } + if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed / 1000000); } + if(target.Mem != 0) { DISPLAY(" - limit memory %u MB", target.Mem / 1000000); } + DISPLAY("\n"); { ZSTD_CCtx* const ctx = ZSTD_createCCtx(); ZSTD_DCtx* const dctx = ZSTD_createDCtx(); winnerInfo_t winner; @@ -696,23 +886,22 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed) winner.result.cSize = (size_t)(-1); /* find best solution from default params */ + //Can't do this iteration normally w/ cparameter constraints { const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); int i; for (i=1; i<=maxSeeds; i++) { ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, 0); BMK_benchParam(&candidate, origBuff, benchedSize, ctx, dctx, CParams); - if (candidate.cSpeed < (double)targetSpeed) { + if (!feasible(candidate, target) ) { break; } - if ( (candidate.cSize < winner.result.cSize) - | ((candidate.cSize == winner.result.cSize) & (candidate.cSpeed > winner.result.cSpeed)) ) + if (feasible(candidate,target) && objective_lt(winner.result, candidate)) { winner.params = CParams; winner.result = candidate; BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); } } } - BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize); BMK_translateAdvancedParams(winner.params); @@ -721,7 +910,7 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed) { time_t const grillStart = time(NULL); do { ZSTD_compressionParameters params = winner.params; - paramVariation(¶ms); + paramVariation(¶ms, paramVarArray, paramCount); if ((FUZ_rand(&g_rand) & 31) == 3) params = randomParams(); /* totally random config to improve search space */ params = ZSTD_adjustCParams(params, blockSize, 0); @@ -733,9 +922,7 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed) BMK_benchParam(&candidate, origBuff, benchedSize, ctx, dctx, params); /* improvement found => new winner */ - if ( (candidate.cSpeed > targetSpeed) - & ( (candidate.cSize < winner.result.cSize) - | ((candidate.cSize == winner.result.cSize) & (candidate.cSpeed > winner.result.cSpeed)) ) ) + if (feasible(candidate,target) && objective_lt(winner.result, candidate)) { winner.params = params; winner.result = candidate; @@ -744,8 +931,13 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed) } } while (BMK_timeSpan(grillStart) < g_grillDuration_s); } - /* end summary */ + /* no solution found */ + if(winner.result.cSize == (size_t)-1) { + DISPLAY("No feasible solution found\n"); + return 1; + } + /* end summary */ BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize); BMK_translateAdvancedParams(winner.params); DISPLAY("grillParams size - optimizer completed \n"); @@ -834,7 +1026,9 @@ int main(int argc, const char** argv) const char* input_filename=0; U32 optimizer = 0; U32 main_pause = 0; - U32 targetSpeed = 0; + + constraint_t target = { 0 , 0, 0 }; //0 for anything unset + ZSTD_compressionParameters paramTarget = { 0, 0, 0, 0, 0, 0, 0 }; assert(argc>=1); /* for exename */ @@ -847,8 +1041,31 @@ int main(int argc, const char** argv) if(!strcmp(argument,"--no-seed")) { g_noSeed = 1; continue; } + if (longCommandWArg(&argument, "--optimize=")) { + optimizer = 1; + for ( ; ;) { + if (longCommandWArg(&argument, "windowLog=") || longCommandWArg(&argument, "wlog=")) { paramTarget.windowLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "chainLog=") || longCommandWArg(&argument, "clog=")) { paramTarget.chainLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "hashLog=") || longCommandWArg(&argument, "hlog=")) { paramTarget.hashLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "searchLog=") || longCommandWArg(&argument, "slog=")) { paramTarget.searchLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "searchLength=") || longCommandWArg(&argument, "slen=")) { paramTarget.searchLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "targetLength=") || longCommandWArg(&argument, "tlen=")) { paramTarget.targetLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "strategy=") || longCommandWArg(&argument, "strat=")) { paramTarget.strategy = (ZSTD_strategy)(readU32FromChar(&argument)); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "compressionSpeed=") || longCommandWArg(&argument, "cSpeed=")) { target.cSpeed = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "decompressionSpeed=") || longCommandWArg(&argument, "dSpeed=")) { target.dSpeed = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "compressionMemory=") || longCommandWArg(&argument, "cMem=")) { target.Mem = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } + /* in MB or MB/s */ + DISPLAY("invalid optimization parameter \n"); + return 1; + } + + if (argument[0] != 0) { + DISPLAY("invalid --optimize= format\n"); + return 1; /* check the end of string */ + } + continue; + } else if (longCommandWArg(&argument, "--zstd=")) { /* Decode command (note : aggregated commands are allowed) */ - if (longCommandWArg(&argument, "--zstd=")) { g_singleRun = 1; g_params = ZSTD_getCParams(2, g_blockSize, 0); for ( ; ;) { @@ -868,6 +1085,7 @@ int main(int argc, const char** argv) DISPLAY("invalid --zstd= format\n"); return 1; /* check the end of string */ } + continue; /* if not return, success */ } else if (argument[0]=='-') { argument++; @@ -900,7 +1118,54 @@ int main(int argc, const char** argv) case 'O': argument++; optimizer = 1; - targetSpeed = readU32FromChar(&argument); + for ( ; ; ) { + switch(*argument) + { + /* Inputs in MB or MB/s */ + case 'C': + argument++; + target.cSpeed = readU32FromChar(&argument) * 1000000; + continue; + case 'D': + argument++; + target.dSpeed = readU32FromChar(&argument) * 1000000; + continue; + case 'M': + argument++; + target.Mem = readU32FromChar(&argument) * 1000000; + continue; + case 'w': + argument++; + paramTarget.windowLog = readU32FromChar(&argument); + continue; + case 'c': + argument++; + paramTarget.chainLog = readU32FromChar(&argument); + continue; + case 'h': + argument++; + paramTarget.hashLog = readU32FromChar(&argument); + continue; + case 's': + argument++; + paramTarget.searchLog = readU32FromChar(&argument); + continue; + case 'l': /* search length */ + argument++; + paramTarget.searchLength = readU32FromChar(&argument); + continue; + case 't': /* target length */ + argument++; + paramTarget.targetLength = readU32FromChar(&argument); + continue; + case 'S': /* strategy */ + argument++; + paramTarget.strategy = (ZSTD_strategy)readU32FromChar(&argument); + continue; + default : ; + } + break; + } break; /* Run Single conf */ @@ -990,7 +1255,7 @@ int main(int argc, const char** argv) } } else { if (optimizer) { - result = optimizeForSize(input_filename, targetSpeed); + result = optimizeForSize(input_filename, target, paramTarget); } else { result = benchFiles(argv+filenamesStart, argc-filenamesStart); } } From 5f49034520940af38deb5e13d9d624afc7ffe534 Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 18 Jun 2018 11:59:45 -0700 Subject: [PATCH 02/55] Working V1 --- programs/bench.c | 95 ++-- programs/bench.h | 3 + tests/Makefile | 2 +- tests/paramgrill.c | 1256 ++++++++++++++++++++++++++++++++++++++++---- 4 files changed, 1213 insertions(+), 143 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 2fa9262c9..0d35f3eb3 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -85,7 +85,7 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; * Exceptions ***************************************/ #ifndef DEBUG -# define DEBUG 0 +# define DEBUG 1 #endif #define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); } @@ -281,12 +281,14 @@ static size_t local_defaultDecompress( } +volatile char g_touched; + /* initFn will be measured once, bench fn will be measured x times */ /* benchFn should return error value or out Size */ /* takes # of blocks and list of size & stuff for each. */ /* only does looping */ /* note time per loop could be zero if interval too short */ -BMK_customReturn_t BMK_benchFunction( +BMK_customReturn_t __attribute__((optimize("O0"))) BMK_benchFunction( BMK_benchFn_t benchFn, void* benchPayload, BMK_initFn_t initFn, void* initPayload, size_t blockCount, @@ -299,16 +301,6 @@ BMK_customReturn_t BMK_benchFunction( BMK_customReturn_t retval; UTIL_time_t clockStart; - { - unsigned i; - for(i = 0; i < blockCount; i++) { - memset(dstBlockBuffers[i], 0xE5, dstBlockCapacities[i]); /* warm up and erase result buffer */ - } - - UTIL_sleepMilli(5); /* give processor time to other processes */ - UTIL_waitForNextTick(); - } - if(!nbLoops) { EXM_THROW_ND(1, BMK_customReturn_t, "nbLoops must be nonzero \n"); } @@ -317,6 +309,22 @@ BMK_customReturn_t BMK_benchFunction( srcSize += srcBlockSizes[ind]; } + { + unsigned i, j; + for(i = 0; i < blockCount; i++) { + for(j = 0; j < srcBlockSizes[i]; j++) { + g_touched = ((const char*)srcBlockBuffers[i])[j]; /* touch */ + } + } + for(i = 0; i < blockCount; i++) { + memset(dstBlockBuffers[i], 0xE5, dstBlockCapacities[i]); /* warm up and erase result buffer */ + //this is written at end proc? where did compressed data get overwritten ny this? + } + + UTIL_sleepMilli(5); /* give processor time to other processes */ + UTIL_waitForNextTick(); + } + { unsigned i, j, firstIter = 1; clockStart = UTIL_getTime(); @@ -327,9 +335,9 @@ BMK_customReturn_t BMK_benchFunction( if(ZSTD_isError(res)) { EXM_THROW_ND(2, BMK_customReturn_t, "Function benchmarking failed on block %u of size %u : %s \n", j, (U32)dstBlockCapacities[j], ZSTD_getErrorName(res)); - } else if(firstIter) { + } else if(firstIter) { dstSize += res; - } + } } firstIter = 0; } @@ -393,7 +401,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed( { U64 const loopDuration = r.result.result.nanoSecPerRun * cont->nbLoops; r.completed = (cont->timeRemaining <= loopDuration); cont->timeRemaining -= loopDuration; - if (loopDuration > 0) { + if (loopDuration > (TIMELOOP_NANOSEC / 100)) { fastest = MIN(fastest, r.result.result.nanoSecPerRun); if(loopDuration >= MINUSABLETIME) { r.result.result.nanoSecPerRun = fastest; @@ -420,7 +428,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( const void ** const srcPtrs, size_t* const srcSizes, void** const cPtrs, size_t* const cSizes, void** const resPtrs, size_t* const resSizes, - void* resultBuffer, void* compressedBuffer, + void** resultBufferPtr, void* compressedBuffer, const size_t maxCompressedSize, BMK_timedFnState_t* timeStateCompress, BMK_timedFnState_t* timeStateDecompress, @@ -432,7 +440,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( int displayLevel, const char* displayName, const BMK_advancedParams_t* adv) { size_t const blockSize = ((adv->blockSize>=32 && (adv->mode != BMK_decodeOnly)) ? adv->blockSize : srcSize) + (!srcSize); /* avoid div by 0 */ - BMK_return_t results; + BMK_return_t results = { { 0, 0., 0., 0 }, 0 } ; size_t const loadedCompressedSize = srcSize; size_t cSize = 0; double ratio = 0.; @@ -454,13 +462,14 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( srcPtr += fileSizes[fileNb]; } { size_t const decodedSize = (size_t)totalDSize64; - free(resultBuffer); - resultBuffer = malloc(decodedSize); - if (!resultBuffer) { + free(*resultBufferPtr); + //TODO: decodedSize corrupted? + *resultBufferPtr = malloc(decodedSize); + if (!(*resultBufferPtr)) { EXM_THROW(33, BMK_return_t, "not enough memory"); } if (totalDSize64 > decodedSize) { - free(resultBuffer); + free(*resultBufferPtr); EXM_THROW(32, BMK_return_t, "original size is too large"); /* size_t overflow */ } cSize = srcSize; @@ -472,7 +481,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( /* Init data blocks */ { const char* srcPtr = (const char*)srcBuffer; char* cPtr = (char*)compressedBuffer; - char* resPtr = (char*)resultBuffer; + char* resPtr = (char*)(*resultBufferPtr); U32 fileNb; for (nbBlocks=0, fileNb=0; fileNbmode != BMK_decodeOnly) { + BMK_customReturn_t compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep, - nbBlocks, srcPtrs, srcSizes, cPtrs, cSizes, adv->nbSeconds); + nbBlocks, srcPtrs, srcSizes, cPtrs, cSizes, adv->nbSeconds); if(compressionResults.error) { results.error = compressionResults.error; return results; @@ -642,7 +652,8 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( } /* CRC Checking */ - { U64 const crcCheck = XXH64(resultBuffer, srcSize, 0); + { void* resultBuffer = *resultBufferPtr; + U64 const crcCheck = XXH64(resultBuffer, srcSize, 0); /* adv->mode == 0 -> compress + decompress */ if ((adv->mode == BMK_both) && (crcOrig!=crcCheck)) { size_t u; @@ -692,11 +703,13 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( } DISPLAYLEVEL(2, "%2i#\n", cLevel); } /* Bench */ - results.result.cMem = ZSTD_sizeof_CCtx(ctx); + results.result.cMem = (1 << (comprParams->windowLog)) + ZSTD_sizeof_CCtx(ctx); + results.error = 0; return results; } BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, + void* dstBuffer, size_t dstCapacity, const size_t* fileSizes, unsigned nbFiles, const int cLevel, const ZSTD_compressionParameters* comprParams, const void* dictBuffer, size_t dictBufferSize, @@ -717,26 +730,41 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, void ** const resPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); size_t* const resSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); - const size_t maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */ - void* compressedBuffer = malloc(maxCompressedSize); - void* resultBuffer = malloc(srcSize); BMK_timedFnState_t* timeStateCompress = BMK_createTimeState(adv->nbSeconds); BMK_timedFnState_t* timeStateDecompress = BMK_createTimeState(adv->nbSeconds); + void* compressedBuffer; + const size_t maxCompressedSize = dstCapacity ? dstCapacity : ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); + void* resultBuffer = malloc(srcSize); + + BMK_return_t results; - int allocationincomplete = !compressedBuffer || !resultBuffer || + int allocationincomplete; + + if(!dstCapacity) { + compressedBuffer = malloc(maxCompressedSize); + } else { + compressedBuffer = dstBuffer; + } + + allocationincomplete = !compressedBuffer || !resultBuffer || !srcPtrs || !srcSizes || !cPtrs || !cSizes || !resPtrs || !resSizes; + if (!allocationincomplete) { results = BMK_benchMemAdvancedNoAlloc(srcPtrs, srcSizes, cPtrs, cSizes, - resPtrs, resSizes, resultBuffer, compressedBuffer, maxCompressedSize, timeStateCompress, timeStateDecompress, + resPtrs, resSizes, &resultBuffer, compressedBuffer, maxCompressedSize, timeStateCompress, timeStateDecompress, srcBuffer, srcSize, fileSizes, nbFiles, cLevel, comprParams, dictBuffer, dictBufferSize, ctx, dctx, displayLevel, displayName, adv); } + + /* clean up */ BMK_freeTimeState(timeStateCompress); BMK_freeTimeState(timeStateDecompress); - free(compressedBuffer); + if(!dstCapacity) { /* only free if not given */ + free(compressedBuffer); + } free(resultBuffer); free((void*)srcPtrs); @@ -749,7 +777,6 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, if(allocationincomplete) { EXM_THROW(31, BMK_return_t, "allocation error : not enough memory"); } - results.error = 0; return results; } @@ -762,6 +789,7 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize, const BMK_advancedParams_t adv = BMK_initAdvancedParams(); return BMK_benchMemAdvanced(srcBuffer, srcSize, + NULL, 0, fileSizes, nbFiles, cLevel, comprParams, dictBuffer, dictBufferSize, @@ -783,6 +811,7 @@ static BMK_return_t BMK_benchMemCtxless(const void* srcBuffer, size_t srcSize, EXM_THROW(12, BMK_return_t, "not enough memory for contexts"); } res = BMK_benchMemAdvanced(srcBuffer, srcSize, + NULL, 0, fileSizes, nbFiles, cLevel, comprParams, dictBuffer, dictBufferSize, diff --git a/programs/bench.h b/programs/bench.h index f1ac255f8..625b65757 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -122,6 +122,8 @@ BMK_return_t BMK_syntheticTest(int cLevel, double compressibility, * (cLevel, comprParams + adv in advanced Mode) */ /* srcBuffer - data source, expected to be valid compressed data if in Decode Only Mode * srcSize - size of data in srcBuffer + * dstBuffer - destination buffer to write compressed output in, optional (NULL) + * dstCapacity - capacity of destination buffer, give 0 if dstBuffer = NULL * cLevel - compression level * comprParams - basic compression parameters * dictBuffer - a dictionary if used, null otherwise @@ -144,6 +146,7 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize, /* See benchMem for normal parameter uses and return, see advancedParams_t for adv */ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, + void* dstBuffer, size_t dstCapacity, const size_t* fileSizes, unsigned nbFiles, const int cLevel, const ZSTD_compressionParameters* comprParams, const void* dictBuffer, size_t dictBufferSize, diff --git a/tests/Makefile b/tests/Makefile index 81e685780..ecf7fe2a7 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -200,7 +200,7 @@ zstreamtest-dll : $(ZSTDDIR)/common/xxhash.c # xxh symbols not exposed from dll zstreamtest-dll : $(ZSTREAM_LOCAL_FILES) $(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT) -paramgrill : DEBUGFLAGS = # turn off assert() for speed measurements +#paramgrill : DEBUGFLAGS = # turn off assert() for speed measurements paramgrill : $(ZSTD_FILES) $(PRGDIR)/bench.c $(PRGDIR)/datagen.c paramgrill.c $(CC) $(FLAGS) $^ -lm -o $@$(EXT) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 81c6dffc2..44bcedef8 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -41,6 +41,8 @@ #define MB *(1<<20) #define GB *(1ULL<<30) +#define TIMELOOP_NANOSEC (1*1000000000ULL) /* 1 second */ + #define NBLOOPS 2 #define TIMELOOP (2 * SEC_TO_MICRO) #define NB_LEVELS_TRACKED 22 /* ensured being >= ZSTD_maxCLevel() in BMK_init_level_constraints() */ @@ -70,13 +72,39 @@ static const int g_maxNbVariations = 64; #define SLOG_IND 3 #define SLEN_IND 4 #define TLEN_IND 5 -#define STRT_IND 6 -#define NUM_PARAMS 7 +//#define STRT_IND 6 +//#define NUM_PARAMS 7 +#define NUM_PARAMS 6 +//just don't use strategy as a param. +#undef ZSTD_WINDOWLOG_MAX +#define ZSTD_WINDOWLOG_MAX 27 //no long range stuff for now. + +//make 2^[0,10] w/ 999 +#define ZSTD_TARGETLENGTH_MIN 0 //actually targeLengthlog min +#define ZSTD_TARGETLENGTH_MAX 10 + +//#define ZSTD_TARGETLENGTH_MAX 1024 +#define WLOG_RANGE (ZSTD_WINDOWLOG_MAX - ZSTD_WINDOWLOG_MIN + 1) +#define CLOG_RANGE (ZSTD_CHAINLOG_MAX - ZSTD_CHAINLOG_MIN + 1) +#define HLOG_RANGE (ZSTD_HASHLOG_MAX - ZSTD_HASHLOG_MIN + 1) +#define SLOG_RANGE (ZSTD_SEARCHLOG_MAX - ZSTD_SEARCHLOG_MIN + 1) +#define SLEN_RANGE (ZSTD_SEARCHLENGTH_MAX - ZSTD_SEARCHLENGTH_MIN + 1) +#define TLEN_RANGE 11 +//hard coded since we only use powers of 2 (and 999 ~ 1024) + +static const int mintable[NUM_PARAMS] = { ZSTD_WINDOWLOG_MIN, ZSTD_CHAINLOG_MIN, ZSTD_HASHLOG_MIN, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLENGTH_MIN, ZSTD_TARGETLENGTH_MIN }; +static const int maxtable[NUM_PARAMS] = { ZSTD_WINDOWLOG_MAX, ZSTD_CHAINLOG_MAX, ZSTD_HASHLOG_MAX, ZSTD_SEARCHLOG_MAX, ZSTD_SEARCHLENGTH_MAX, ZSTD_TARGETLENGTH_MAX }; +static const int rangetable[NUM_PARAMS] = { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE }; + +//use grid-search or something when space is small enough? +#define SMALL_SEARCH_SPACE 1000 /*-************************************ * Benchmark Parameters **************************************/ +typedef BYTE U8; + static double g_grillDuration_s = 99999; /* about 27 hours */ static U32 g_nbIterations = NBLOOPS; static double g_compressibility = COMPRESSIBILITY_DEFAULT; @@ -150,13 +178,74 @@ static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) return result; } +//assume that clock can at least measure .01 second intervals? +//make this a settable global initialized with fn? +//#define CLOCK_GRANULARITY 100000000ULL +static U64 g_clockGranularity = 100000000ULL; + +static void findClockGranularity(void) { + UTIL_time_t clockStart = UTIL_getTime(); + U64 el1 = 0, el2 = 0; + int i = 0; + do { + el1 = el2; + el2 = UTIL_clockSpanNano(clockStart); + if(el1 < el2) { + U64 iv = el2 - el1; + if(g_clockGranularity > iv) { + g_clockGranularity = iv; + i = 0; + } else { + i++; + } + } + } while(i < 10); + DISPLAY("Granularity: %llu\n", (unsigned long long)g_clockGranularity); +} typedef struct { U32 cSpeed; /* bytes / sec */ U32 dSpeed; - U32 Mem; /* bytes */ + U32 cMem; /* bytes */ } constraint_t; +#define CLAMPCHECK(val,min,max) { \ + if (val && (((val)<(min)) | ((val)>(max)))) { \ + DISPLAY("INVALID PARAMETER CONSTRAINTS\n"); \ + return 0; \ +} } + + +/* Like ZSTD_checkCParams() but allows 0's */ +/* no check on targetLen? */ +static int cParamValid(ZSTD_compressionParameters paramTarget) { + CLAMPCHECK(paramTarget.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); + CLAMPCHECK(paramTarget.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX); + CLAMPCHECK(paramTarget.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX); + CLAMPCHECK(paramTarget.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX); + CLAMPCHECK(paramTarget.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX); + if(paramTarget.strategy > ZSTD_btultra) { + DISPLAY("INVALID PARAMETER CONSTRAINTS\n"); + return 0; + } + return 1; +} + +static void cParamZeroMin(ZSTD_compressionParameters* paramTarget) { + paramTarget->windowLog = paramTarget->windowLog ? paramTarget->windowLog : ZSTD_WINDOWLOG_MIN; + paramTarget->searchLog = paramTarget->searchLog ? paramTarget->searchLog : ZSTD_SEARCHLOG_MIN; + paramTarget->chainLog = paramTarget->chainLog ? paramTarget->chainLog : ZSTD_CHAINLOG_MIN; + paramTarget->hashLog = paramTarget->hashLog ? paramTarget->hashLog : ZSTD_HASHLOG_MIN; + paramTarget->searchLength = paramTarget->searchLength ? paramTarget->searchLength : ZSTD_SEARCHLENGTH_MIN; + paramTarget->targetLength = paramTarget->targetLength ? paramTarget->targetLength : 1; +} + +static void BMK_translateAdvancedParams(ZSTD_compressionParameters params) +{ + DISPLAY("--zstd=windowLog=%u,chainLog=%u,hashLog=%u,searchLog=%u,searchLength=%u,targetLength=%u,strategy=%u \n", + params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, params.targetLength, (U32)(params.strategy)); +} + /*-******************************************************* * Bench functions *********************************************************/ @@ -357,20 +446,45 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para return better; } +/* bounds check in sanitize too? */ +#define CLAMP(var, lo, hi) { \ + var = MAX(MIN(var, hi), lo); \ +} + /* nullified useless params, to ensure count stats */ -static ZSTD_compressionParameters* sanitizeParams(ZSTD_compressionParameters params) +/* no point in windowLog < chainLog (no point 2x chainLog for bt) */ +/* now with built in bounds-checking */ +/* no longer does anything with sanitizeVarArray + clampcheck */ +static ZSTD_compressionParameters sanitizeParams(ZSTD_compressionParameters params) { - g_params = params; if (params.strategy == ZSTD_fast) g_params.chainLog = 0, g_params.searchLog = 0; if (params.strategy == ZSTD_dfast) g_params.searchLog = 0; - if (params.strategy != ZSTD_btopt && params.strategy != ZSTD_btultra) + if (params.strategy != ZSTD_btopt && params.strategy != ZSTD_btultra && params.strategy != ZSTD_fast) g_params.targetLength = 0; - return &g_params; + + return params; +} + +/* new length */ +/* keep old array, will need if iter over strategy. */ +static int sanitizeVarArray(int varLength, U32* varArray, U32* varNew, ZSTD_strategy strat) { + int i, j = 0; + for(i = 0; i < varLength; i++) { + if( !((varArray[i] == CLOG_IND && strat == ZSTD_fast) + || (varArray[i] == SLOG_IND && strat == ZSTD_dfast) + || (varArray[i] == TLEN_IND && strat != ZSTD_btopt && strat != ZSTD_btultra && strat != ZSTD_fast))) { + varNew[j] = varArray[i]; + j++; + } + } + return j; + } /* res should be NUM_PARAMS size */ +/* constructs varArray from ZSTD_compressionParameters style parameter */ static int variableParams(const ZSTD_compressionParameters paramConstraints, U32* res) { int j = 0; if(!paramConstraints.windowLog) { @@ -397,10 +511,6 @@ static int variableParams(const ZSTD_compressionParameters paramConstraints, U32 res[j] = TLEN_IND; j++; } - if(!(U32)paramConstraints.strategy) { - res[j] = STRT_IND; - j++; - } return j; } @@ -417,43 +527,39 @@ static int inverseVariableParams(const ZSTD_compressionParameters paramConstrain res[j] = CLOG_IND; j++; } else { - res[WLOG_IND] = -1; + res[CLOG_IND] = -1; } if(!paramConstraints.hashLog) { res[j] = HLOG_IND; j++; } else { - res[WLOG_IND] = -1; + res[HLOG_IND] = -1; } if(!paramConstraints.searchLog) { res[j] = SLOG_IND; j++; } else { - res[WLOG_IND] = -1; + res[SLOG_IND] = -1; } if(!paramConstraints.searchLength) { res[j] = SLEN_IND; j++; } else { - res[WLOG_IND] = -1; + res[SLEN_IND] = -1; } if(!paramConstraints.targetLength) { res[j] = TLEN_IND; j++; } else { - res[WLOG_IND] = -1; - } - if(!(U32)paramConstraints.strategy) { - res[j] = STRT_IND; - j++; - } else { - res[WLOG_IND] = -1; + res[TLEN_IND] = -1; } + return j; } /* amt will probably always be \pm 1? */ /* slight change from old paramVariation, targetLength can only take on powers of 2 now (999 ~= 1024?) */ +/* take max/min bounds into account as well? */ static void paramVaryOnce(U32 paramIndex, int amt, ZSTD_compressionParameters* ptr) { switch(paramIndex) { @@ -463,13 +569,16 @@ static void paramVaryOnce(U32 paramIndex, int amt, ZSTD_compressionParameters* p case SLOG_IND: ptr->searchLog += amt; break; case SLEN_IND: ptr->searchLength += amt; break; case TLEN_IND: - if(amt > 0) { + if(amt >= 0) { ptr->targetLength <<= amt; + ptr->targetLength = MIN(ptr->targetLength, 999); } else { + if(ptr->targetLength == 999) { + ptr->targetLength = 1024; + } ptr->targetLength >>= -amt; } break; - case STRT_IND: ptr->strategy += amt; break; default: break; } } @@ -477,34 +586,143 @@ static void paramVaryOnce(U32 paramIndex, int amt, ZSTD_compressionParameters* p //Don't fuzz fixed variables. //turn pcs to pcs array with macro for params. //pass in variation array from variableParams -static void paramVariation(ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen) +//take nbChanges as argument? +static void paramVariation(ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen, U32 nbChanges) { ZSTD_compressionParameters p; U32 validated = 0; while (!validated) { - U32 nbChanges = (FUZ_rand(&g_rand) & 3) + 1; + U32 i; p = *ptr; - for ( ; nbChanges ; nbChanges--) { - const U32 changeID = FUZ_rand(&g_rand) % (2 * varyLen); + for (i = 0 ; i < nbChanges ; i++) { + const U32 changeID = FUZ_rand(&g_rand) % (varyLen << 1); paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p); } - validated = !ZSTD_isError(ZSTD_checkCParams(p)); - + //validated = !ZSTD_isError(ZSTD_checkCParams(p)); + validated = cParamValid(p); + //Make sure memory is at least close to feasible? //ZSTD_estimateCCtxSize thing. } - *ptr = p; + *ptr = sanitizeParams(p); } +//varyParams gives us table size? +//1 per strategy +//varyParams should always be sorted smallest to largest +//take arrayLen to allocate memotable +//should be ~10^7 unconstrained. +static size_t memoTableLen(const U32* varyParams, const int varyLen) { + size_t arrayLen = 1; + int i; + for(i = 0; i < varyLen; i++) { + arrayLen *= rangetable[varyParams[i]]; + } + return arrayLen; +} + +//sort of ~lg2 (replace 1024 w/ 999) for memoTableInd Tlen +static unsigned lg2(unsigned x) { + unsigned j = 0; + if(x == 999) { + return 10; + } + while(x >>= 1) { + j++; + } + return j; +} + +//indexes compressionParameters into memotable +//of form +static unsigned memoTableInd(const ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen) { + int i; + unsigned ind = 0; + for(i = 0; i < varyLen; i++) { + switch(varyParams[i]) { + case WLOG_IND: ind *= WLOG_RANGE; ind += ptr->windowLog - ZSTD_WINDOWLOG_MIN ; break; + case CLOG_IND: ind *= CLOG_RANGE; ind += ptr->chainLog - ZSTD_CHAINLOG_MIN ; break; + case HLOG_IND: ind *= HLOG_RANGE; ind += ptr->hashLog - ZSTD_HASHLOG_MIN ; break; + case SLOG_IND: ind *= SLOG_RANGE; ind += ptr->searchLog - ZSTD_SEARCHLOG_MIN ; break; + case SLEN_IND: ind *= SLEN_RANGE; ind += ptr->searchLength - ZSTD_SEARCHLENGTH_MIN; break; + case TLEN_IND: ind *= TLEN_RANGE; ind += lg2(ptr->targetLength) - ZSTD_TARGETLENGTH_MIN; break; + } + } + return ind; +} + +/* presumably, the unfilled parameters are already at their correct value */ +/* inverse above function for varyParams */ +static void memoTableIndInv(ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen, size_t ind) { + int i; + for(i = varyLen - 1; i >= 0; i--) { + switch(varyParams[i]) { + case WLOG_IND: ptr->windowLog = ind % WLOG_RANGE + ZSTD_WINDOWLOG_MIN; ind /= WLOG_RANGE; break; + case CLOG_IND: ptr->chainLog = ind % CLOG_RANGE + ZSTD_CHAINLOG_MIN; ind /= CLOG_RANGE; break; + case HLOG_IND: ptr->hashLog = ind % HLOG_RANGE + ZSTD_HASHLOG_MIN; ind /= HLOG_RANGE; break; + case SLOG_IND: ptr->searchLog = ind % SLOG_RANGE + ZSTD_SEARCHLOG_MIN; ind /= SLOG_RANGE; break; + case SLEN_IND: ptr->searchLength = ind % SLEN_RANGE + ZSTD_SEARCHLENGTH_MIN; ind /= SLEN_RANGE; break; + case TLEN_IND: ptr->targetLength = MIN(1 << (ind % TLEN_RANGE), 999); ind /= TLEN_RANGE; break; + } + } +} + +//initializing memoTable +/* */ +static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstraints, constraint_t target, const U32* varyParams, const int varyLen) { + size_t i; + size_t arrayLen = memoTableLen(varyParams, varyLen); + int cwFixed = !paramConstraints.chainLog || !paramConstraints.windowLog; + int scFixed = !paramConstraints.searchLog || !paramConstraints.chainLog; + int j = 0; + memset(memoTable, 0, arrayLen); + + + for(i = 0; i < arrayLen; i++) { + memoTableIndInv(¶mConstraints, varyParams, varyLen, i); + BMK_translateAdvancedParams(paramConstraints); + if(ZSTD_estimateCCtxSize_usingCParams(paramConstraints) + (1 << paramConstraints.windowLog) > target.cMem) { + //infeasible; + memoTable[i] = 255; + j++; + } + /* nil out parameter sets equivalent to others. */ + if(cwFixed/* at most least 1 param fixed. */) { + if(paramConstraints.strategy == ZSTD_btlazy2 || paramConstraints.strategy == ZSTD_btopt || paramConstraints.strategy == ZSTD_btultra) { + if(paramConstraints.chainLog > paramConstraints.windowLog + 1) { + if(memoTable[i] != 255) { j++; } + memoTable[i] = 255; + } + } else { + if(paramConstraints.chainLog > paramConstraints.windowLog) { + if(memoTable[i] != 255) { j++; } + memoTable[i] = 255; + } + } + } + if(scFixed) { + if(paramConstraints.searchLog > paramConstraints.chainLog) { + if(memoTable[i] != 255) { j++; } + memoTable[i] = 255; + } + } + } + DISPLAY("%d / %d Invalid\n", j, (int)i); +} #define PARAMTABLELOG 25 #define PARAMTABLESIZE (1<> 3) & PARAMTABLEMASK] + g_alreadyTested[(XXH64(((void*)&sanitizeParams(p), sizeof(p), 0) >> 3) & PARAMTABLEMASK] */ +static BYTE* NB_TESTS_PLAYED(ZSTD_compressionParameters p) { + ZSTD_compressionParameters p2 = sanitizeParams(p); + return &g_alreadyTested[(XXH64((void*)&p2, sizeof(p2), 0) >> 3) & PARAMTABLEMASK]; +} static void playAround(FILE* f, winnerInfo_t* winners, ZSTD_compressionParameters params, @@ -513,21 +731,23 @@ static void playAround(FILE* f, winnerInfo_t* winners, { int nbVariations = 0; UTIL_time_t const clockStart = UTIL_getTime(); - const U32 unconstrained[NUM_PARAMS] = { 0, 1, 2, 3, 4, 5, 6 }; + const U32 unconstrained[NUM_PARAMS] = { 0, 1, 2, 3, 4, 5 }; while (UTIL_clockSpanMicro(clockStart) < g_maxVariationTime) { ZSTD_compressionParameters p = params; + BYTE* b; if (nbVariations++ > g_maxNbVariations) break; - paramVariation(&p, unconstrained, 7); + paramVariation(&p, unconstrained, 7, 4); /* exclude faster if already played params */ - if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(p))-1)) + if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(p))-1)) continue; /* test */ - NB_TESTS_PLAYED(p)++; + b = NB_TESTS_PLAYED(p); + (*b)++; if (!BMK_seed(winners, p, srcBuffer, srcSize, ctx, dctx)) continue; /* improvement found => search more */ @@ -544,34 +764,37 @@ static ZSTD_compressionParameters randomParams(void) U32 validated = 0; while (!validated) { /* totally random entry */ - p.chainLog = (FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN)) + ZSTD_CHAINLOG_MIN; - p.hashLog = (FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN)) + ZSTD_HASHLOG_MIN; - p.searchLog = (FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN)) + ZSTD_SEARCHLOG_MIN; - p.windowLog = (FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN)) + ZSTD_WINDOWLOG_MIN; + p.chainLog = (FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN)) + ZSTD_CHAINLOG_MIN; + p.hashLog = (FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN)) + ZSTD_HASHLOG_MIN; + p.searchLog = (FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN)) + ZSTD_SEARCHLOG_MIN; + p.windowLog = (FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN)) + ZSTD_WINDOWLOG_MIN; p.searchLength=(FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN)) + ZSTD_SEARCHLENGTH_MIN; p.targetLength=(FUZ_rand(&g_rand) % (512)); p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btultra +1)); - validated = !ZSTD_isError(ZSTD_checkCParams(p)); + //validated = !ZSTD_isError(ZSTD_checkCParams(p)); + validated = cParamValid(p); } return p; } -static ZSTD_compressionParameters randomConstrainedParams(ZSTD_compressionParameters pc) +//destructively modifies pc. +//Maybe if memoTable[ind] > 0 too often, count zeroes and explicitly choose from free stuff? +//^ maybe this doesn't matter, with |mt| size it has \approx 1-(1/e) of finding even single free spot in |mt| tries, not too bad. +//TODO: maybe memoTable pc before sanitization too so no repeats? +static void randomConstrainedParams(ZSTD_compressionParameters* pc, U32* varArray, int varLen, U8* memoTable) { - ZSTD_compressionParameters p; - U32 validated = 0; - while (!validated) { - /* totally random entry */ - if(!pc.chainLog) p.chainLog = (FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN)) + ZSTD_CHAINLOG_MIN; - if(!pc.chainLog) p.hashLog = (FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN)) + ZSTD_HASHLOG_MIN; - if(!pc.chainLog) p.searchLog = (FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN)) + ZSTD_SEARCHLOG_MIN; - if(!pc.chainLog) p.windowLog = (FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN)) + ZSTD_WINDOWLOG_MIN; - if(!pc.chainLog) p.searchLength=(FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN)) + ZSTD_SEARCHLENGTH_MIN; - if(!pc.chainLog) p.targetLength=(FUZ_rand(&g_rand) % (512)) + 1; //ZSTD_TARGETLENGTH_MIN; //change to 2^[0,10?] - if(!pc.chainLog) p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btultra +1)); - validated = !ZSTD_isError(ZSTD_checkCParams(p)); - } - return p; + int tries = memoTableLen(varArray, varLen); //configurable, + const size_t maxSize = memoTableLen(varArray, varLen); + size_t ind; + do { + ind = (FUZ_rand(&g_rand)) % maxSize; + tries--; + } while(memoTable[ind] > 0 && tries > 0); + //&& FUZ_rand(&g_rand) % 256 > memoTable[ind]); get nd choosing? (helpful w/ distance) /* maybe > infeasible bound? */ + + /* memoTable[ind] == 0 -> unexplored */ + memoTableIndInv(pc, varArray, varLen, (unsigned)ind); + *pc = sanitizeParams(*pc); } static void BMK_selectRandomStart( @@ -746,22 +969,80 @@ int benchFiles(const char** fileNamesTable, int nbFiles) return 0; } - -static void BMK_translateAdvancedParams(ZSTD_compressionParameters params) -{ - DISPLAY("--zstd=windowLog=%u,chainLog=%u,hashLog=%u,searchLog=%u,searchLength=%u,targetLength=%u,strategy=%u \n", - params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, params.targetLength, (U32)(params.strategy)); -} - -//Results currently don't capture memory usage or anything. //parameter feasibility is not checked, should just be restricted from use. static int feasible(BMK_result_t results, constraint_t target) { - return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.Mem || !target.Mem); + return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem || !target.cMem); +} + +#define EPSILON 0.01 +static int epsilonEqual(double c1, double c2) { + return MAX(c1/c2,c2/c1) < 1 + EPSILON; +} + +//so the compiler stops warning +static int eqZero(double c1) { + return (U64)c1 == (U64)0.0 || (U64)c1 == (U64)-0.0; } /* returns 1 if result2 is strictly 'better' than result1 */ +/* strict comparison / cutoff based */ static int objective_lt(BMK_result_t result1, BMK_result_t result2) { - return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed); + return (result1.cSize > result2.cSize) || (epsilonEqual(result1.cSize, result2.cSize) && result2.cSpeed > result1.cSpeed) + || (epsilonEqual(result1.cSize,result2.cSize) && epsilonEqual(result2.cSpeed, result1.cSpeed) && result2.dSpeed > result1.dSpeed); +} + +//will probably be some linear combinartion of comp speed, decompSpeed, & ratio (maybe size), and memory? +//pretty arbitrary right now +//maybe better - higher coefficient when below threshold, lower when above +//need to normalize speed? or just use ratio speed / target? +//Maybe don't use ratio at all when looking for feasibility? + +/* maybe dynamically vary the coefficients for this around based on what's already been discovered. (maybe make a reversed ratio cutoff?) concave to pheaily penalize below ratio? */ +static double resultScore(BMK_result_t res, size_t srcSize, constraint_t target) { + double cs = 0., ds = 0., rt, cm = 0.; + const double r1 = 1, r2 = 0.1, rtr = 0.5; + double ret; + if(target.cSpeed) { cs = res.cSpeed / (double)target.cSpeed; } + if(target.dSpeed) { ds = res.dSpeed / (double)target.dSpeed; } + if(target.cMem != (U32)-1) { cm = (double)target.cMem / res.cMem; } + rt = ((double)srcSize / res.cSize); + + //(void)rt; + //(void)rtr; + + ret = (MIN(1, cs) + MIN(1, ds) + MIN(1, cm))*r1 + rt * rtr + + (MAX(0, log(cs))+ MAX(0, log(ds))+ MAX(0, log(cm))) * r2; + //DISPLAY("resultScore: %f\n", ret); + return ret; +} + +/* + double W_ratio = (double)srcSize / testResult.cSize; + double O_ratio = (double)srcSize / winners[cLevel].result.cSize; + double W_ratioNote = log (W_ratio); + double O_ratioNote = log (O_ratio); + size_t W_DMemUsed = (1 << params.windowLog) + (16 KB); + size_t O_DMemUsed = (1 << winners[cLevel].params.windowLog) + (16 KB); + double W_DMemUsed_note = W_ratioNote * ( 40 + 9*cLevel) - log((double)W_DMemUsed); + double O_DMemUsed_note = O_ratioNote * ( 40 + 9*cLevel) - log((double)O_DMemUsed); + + size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_estimateCCtxSize_usingCParams(params); + size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_estimateCCtxSize_usingCParams(winners[cLevel].params); + double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed); + double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed); + + double W_CSpeed_note = W_ratioNote * ( 30 + 10*cLevel) + log(testResult.cSpeed); + double O_CSpeed_note = O_ratioNote * ( 30 + 10*cLevel) + log(winners[cLevel].result.cSpeed); + + double W_DSpeed_note = W_ratioNote * ( 20 + 2*cLevel) + log(testResult.dSpeed); + double O_DSpeed_note = O_ratioNote * ( 20 + 2*cLevel) + log(winners[cLevel].result.dSpeed); + +*/ +//ratio tradeoffs, may be useful in guiding + +/* objective_lt, but based on scoring function */ +static int objective_lt2(BMK_result_t result1, BMK_result_t result2, size_t srcSize, constraint_t target) { + return resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target); } /* res gives array dimensions, should be size NUM_PARAMS */ @@ -783,48 +1064,800 @@ static size_t computeStateSize(const ZSTD_compressionParameters paramConstraints static unsigned calcViolation(BMK_result_t results, constraint_t target) { int diffcSpeed = MAX(target.cSpeed - results.cSpeed, 0); int diffdSpeed = MAX(target.dSpeed - results.dSpeed, 0); - int diffcMem = MAX(results.cMem - target.Mem, 0); + int diffcMem = MAX(results.cMem - target.cMem, 0); return diffcSpeed + diffdSpeed + diffcMem; } -/* finds some set of parameters which fulfills req's - * Prioritize highest / try to locally minimize sum? - * Is it ever useful to go out of the param constraints? ? - * random / perturb when revisit? - * momentum? - */ -static ZSTD_compressionParameters findFeasible(constraint_t target, ZSTD_compressionParameters paramTarget) { - unsigned violation; - - ZSTD_compressionParameters winner = randomConstrainedParams(paramTarget); - //just use g_alreadyTested and xxhash? - BYTE* memotable; - do { - //prioritize memory - if(diffcMem >= diffcSpeed && diffcMem >= diffdSpeed) { - - //prioritize compression Speed - } else if (diffcSpeeed >= diffdSpeed && diffcSpeed >= diffcMem) { - - //prioritize decompressionSpeed - } else { - - } - violation = calcViolation(result, target); - } while(objective); - if(validate) { - DISPLAY("Feasible Point Found\n"); - return winner; +/* +uncertaintyConstant >= 1 +returns -1 = 'certainly' infeasible + 0 = unceratin + 1 = 'certainly' feasible +*/ +//paramTarget misnamed, should just be target +static int uncertainFeasibility(double const uncertaintyConstantC, double const uncertaintyConstantD, const constraint_t paramTarget, const BMK_result_t* const results) { + if((paramTarget.cSpeed != 0 && results->cSpeed * uncertaintyConstantC < paramTarget.cSpeed) || + (paramTarget.dSpeed != 0 && results->dSpeed * uncertaintyConstantD < paramTarget.dSpeed) || + (paramTarget.cMem != 0 && results->cMem > paramTarget.cMem)) { + return -1; + } else if((paramTarget.cSpeed == 0 || results->cSpeed / uncertaintyConstantC > paramTarget.cSpeed) && + (paramTarget.dSpeed == 0 || results->dSpeed / uncertaintyConstantD > paramTarget.dSpeed) && + (paramTarget.cMem == 0 || results->cMem <= paramTarget.cMem)) { + return 1; } else { - DISPLAY("No solution found\n"); - ZSTD_compressionParameters ret = { 0, 0, 0, 0, 0, 0, 0 }; - return ret; + return 0; } } +/* 1 - better than prev best + 0 - uncertain + -1 - worse + assume prev_best status is run fully? + but then we'd have to rerun any winners anyway */ +//presumably memory has already been compared, mostly worried about mem, cspeed, dspeed +//uncertainty only applies to speed. +//if using objective fn, this could be much easier since we could just scale that. +//difficult to make judgements about later parameters in prioritization type when there's +//uncertainty on the first. +static int uncertainComparison(double const uncertaintyConstantC, double const uncertaintyConstantD, BMK_result_t* candidate, BMK_result_t* prevBest) { + (void)uncertaintyConstantD; //unused for now + if(candidate->cSpeed > prevBest->cSpeed * uncertaintyConstantC) { + return 1; + } else if (candidate->cSpeed * uncertaintyConstantC < prevBest->cSpeed) { + return -1; + } else { + return 0; + } +} + +/* speed in b, srcSize in b/s loopDuration in ns */ +//TODO: simplify code in feasibleBench with this instead of writing it all out. +//only applicable for single loop +static double calcUncertainty(double speed, size_t srcSize) { + U64 loopDuration; + if(eqZero(speed)) { return 2; } + loopDuration = ((srcSize * TIMELOOP_NANOSEC) / speed); + return MIN((loopDuration + (double)2 * g_clockGranularity) / loopDuration, 2); +} + +//benchmarks and tests feasibility together +//1 = true = better +//0 = false = not better +//if true then resultPtr will give results. +//2+ on error? +//alt: error = 0 / infeasible as well; +//maybe use compress_only mode for ratio-finding benchmark? +//prioritize ratio > cSpeed > dSpeed > cMem +//Misnamed - should be worse, better, error +//alternative (to make work for feasible-pt searching as well) - only compare to winner, not to target +//but then we need to judge what better means in this context, which shouldn't be the same (strict ratio improvement) +#define INFEASIBLE_RESULT 0 +#define FEASIBLE_RESULT 1 +#define ERROR_RESULT 2 +static int feasibleBench(BMK_result_t* resultPtr, + const void* srcBuffer, size_t srcSize, + void* dstBuffer, size_t dstSize, + ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, + const ZSTD_compressionParameters cParams, + const constraint_t target, + BMK_result_t* winnerResult) { + BMK_advancedParams_t adv = BMK_initAdvancedParams(); + BMK_return_t benchres; + U64 loopDurationC = 0, loopDurationD = 0; + double uncertaintyConstantC, uncertaintyConstantD; + adv.loopMode = BMK_iterMode; + adv.nbSeconds = 1; //get ratio and 2x approx speed? + + //alternative - test 1 iter for ratio, (possibility of error 3 which is fine), + //maybe iter this until 2x measurable for better guarantee? + DISPLAY("Feas:\n"); + benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + if(benchres.error) { + DISPLAY("ERROR %d !!\n", benchres.error); + } + BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); + + if(!benchres.error) { + *resultPtr = benchres.result; + /* if speed is 0 (only happens when time = 0) */ + if(eqZero(benchres.result.cSpeed)) { + loopDurationC = 0; + uncertaintyConstantC = 2; + } else { + loopDurationC = ((srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); + //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? + //possibly has to do with initCCtx? or system stuff? + //asymmetric +/- constant needed? + uncertaintyConstantC = MIN((loopDurationC + (double)(2 * g_clockGranularity)/loopDurationC), 2); //.02 seconds + } + if(eqZero(benchres.result.dSpeed)) { + loopDurationD = 0; + uncertaintyConstantD = 2; + } else { + loopDurationD = ((srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); + //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? + //possibly has to do with initCCtx? or system stuff? + //asymmetric +/- constant needed? + uncertaintyConstantD = MIN((loopDurationD + (double)(2 * g_clockGranularity)/loopDurationD), 2); //.02 seconds + } + + + if(benchres.result.cSize < winnerResult->cSize) { //better compression ratio, just needs to be feasible + //optimistic assume speed + //incoporate some sort of tradeoff comparison with the winner's results? + int feas = uncertainFeasibility(uncertaintyConstantC, uncertaintyConstantD, target, &(benchres.result)); + if(feas == 0) { // uncertain feasibility + adv.loopMode = BMK_timeMode; + if(loopDurationC < TIMELOOP_NANOSEC) { + BMK_return_t benchres2; + adv.mode = BMK_compressOnly; + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres.result.cSpeed = benchres2.result.cSpeed; + } + } + if(loopDurationD < TIMELOOP_NANOSEC) { + BMK_return_t benchres2; + adv.mode = BMK_decodeOnly; + benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres.result.dSpeed = benchres2.result.dSpeed; + } + } + *resultPtr = benchres.result; + return feasible(benchres.result, target); + } else { //feas = 1 or -1 map to 1, 0 respectively + return (feas + 1) >> 1; //relies on INFEASIBLE_RESULT == 0, FEASIBLE_RESULT == 1 + } + } else if (benchres.result.cSize == winnerResult->cSize) { //equal ratio, needs to be better than winner in cSpeed/ dSpeed / cMem + int feas = uncertainFeasibility(uncertaintyConstantC, uncertaintyConstantD, target, &(benchres.result)); + if(feas == 0) { // uncertain feasibility + adv.loopMode = BMK_timeMode; + if(loopDurationC < TIMELOOP_NANOSEC) { + BMK_return_t benchres2; + adv.mode = BMK_compressOnly; + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres.result.cSpeed = benchres2.result.cSpeed; + } + } + if(loopDurationD < TIMELOOP_NANOSEC) { + BMK_return_t benchres2; + adv.mode = BMK_decodeOnly; + benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres.result.dSpeed = benchres2.result.dSpeed; + } + } + + *resultPtr = benchres.result; + return feasible(benchres.result, target) && objective_lt(*winnerResult, benchres.result); + } else if (feas == 1) { //no need to check feasibility compares (maybe only it is chosen as a winner) + int btw = uncertainComparison(uncertaintyConstantC, uncertaintyConstantD, &(benchres.result), winnerResult); + if(btw == -1) { + return INFEASIBLE_RESULT; + } else { //possibly better, benchmark and find out + adv.loopMode = BMK_timeMode; + benchres = BMK_benchMemAdvanced(srcBuffer, srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + *resultPtr = benchres.result; + return objective_lt(*winnerResult, benchres.result); + } + } else { //feas == -1 + return INFEASIBLE_RESULT; //infeasible + } + } else { + return INFEASIBLE_RESULT; //infeasible + } + } else { + return ERROR_RESULT; //BMK error + } + +} +//sameas before, but +/-? +//alternative, just return comparison result, leave caller to worry about feasibility. +//have version of benchMemAdvanced which takes in dstBuffer/cap as well? +//(motivation: repeat tests (maybe just on decompress) don't need further compress runs) +static int infeasibleBench(BMK_result_t* resultPtr, + const void* srcBuffer, size_t srcSize, + void* dstBuffer, size_t dstSize, + ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, + const ZSTD_compressionParameters cParams, + const constraint_t target, + BMK_result_t* winnerResult) { + BMK_advancedParams_t adv = BMK_initAdvancedParams(); + BMK_return_t benchres; + BMK_result_t resultMin, resultMax; + UTIL_time_t startTime; + U64 loopDurationC = 0, loopDurationD = 0; + double uncertaintyConstantC, uncertaintyConstantD; + double winnerRS = resultScore(*winnerResult, srcSize, target); + adv.loopMode = BMK_iterMode; //can only use this for ratio measurement then, super inaccurate timing + adv.nbSeconds = 1; //get ratio and 2x approx speed? //maybe run until twice MIN(minloopinterval * clockDuration) + + (void)startTime; //TODO: actually use this to adjust timing + DISPLAY("WinnerScore: %f\n ", winnerRS); + /* + adv.loopMode = BMK_timeMode; + adv.nbSeconds = 1; */ + benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); + + if(!benchres.error) { + *resultPtr = benchres.result; + if(eqZero(benchres.result.cSpeed)) { + loopDurationC = 0; + uncertaintyConstantC = 2; + } else { + loopDurationC = ((srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); + //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? + //possibly has to do with initCCtx? or system stuff? + uncertaintyConstantC = MIN((loopDurationC + (double)(2 * g_clockGranularity)/loopDurationC), 2); //.02 seconds + } + + if(eqZero(benchres.result.dSpeed)) { + loopDurationD = 0; + uncertaintyConstantD = 2; + } else { + loopDurationD = ((srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); + //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? + //possibly has to do with initCCtx? or system stuff? + uncertaintyConstantD = MIN((loopDurationD + (double)(2 * g_clockGranularity)/loopDurationD), 2); //.02 seconds + } + + /* benchres's certainty range. */ + resultMax = benchres.result; + resultMin = benchres.result; + resultMax.cSpeed *= uncertaintyConstantC; + resultMax.dSpeed *= uncertaintyConstantD; + resultMin.cSpeed /= uncertaintyConstantC; + resultMin.dSpeed /= uncertaintyConstantD; + (void)resultMin; + //TODO: consider if resultMin is actually needed. + if (winnerRS > resultScore(resultMax, srcSize, target)) { + return INFEASIBLE_RESULT; + } else { + //do this w/o copying / stuff + adv.loopMode = BMK_timeMode; + if(loopDurationC < TIMELOOP_NANOSEC) { + BMK_return_t benchres2; + adv.mode = BMK_compressOnly; + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres.result.cSpeed = benchres2.result.cSpeed; + } + } + if(loopDurationD < TIMELOOP_NANOSEC) { + BMK_return_t benchres2; + adv.mode = BMK_decodeOnly; + //TODO: dstBuffer corrupted sometime between top and now + //probably occuring in feasible bench too. + benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres.result.dSpeed = benchres2.result.dSpeed; + } + } + *resultPtr = benchres.result; + return (resultScore(benchres.result, srcSize, target) > winnerRS); + } + + *resultPtr = benchres.result; + } else { + return ERROR_RESULT; //BMK error + } + +} + +/* wrap feasibleBench w/ memotable */ +//TODO: void sanitized and unsanitized ver's so input doesn't double-choose +#define INFEASIBLE_THRESHOLD 200 +static int feasibleBenchMemo(BMK_result_t* resultPtr, + const void* srcBuffer, size_t srcSize, + void* dstBuffer, size_t dstSize, + ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, + const ZSTD_compressionParameters cParams, + const constraint_t target, + BMK_result_t* winnerResult, U8* memoTable, + U32* varyParams, const int varyLen) { + + size_t memind = memoTableInd(&cParams, varyParams, varyLen); + //BMK_translateAdvancedParams(cParams); + if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { + return INFEASIBLE_RESULT; //probably pick a different code for already tested? + //maybe remove this if we incorporate nonrandom location picking? + //what is the intended behavior in this case? + //ignore? stop iterating completely? other? + } else { + int res = feasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, ctx, dctx, + cParams, target, winnerResult); + memoTable[memind] = 255; //tested are all infeasible (other possible values for opti) + return res; + } +} + +//should infeasible stage searching also be memo-marked in the same way? +//don't actually memoize unless result is feasible/error? +static int infeasibleBenchMemo(BMK_result_t* resultPtr, + const void* srcBuffer, size_t srcSize, + void* dstBuffer, size_t dstSize, + ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, + const ZSTD_compressionParameters cParams, + const constraint_t target, + BMK_result_t* winnerResult, U8* memoTable, + U32* varyParams, const int varyLen) { + size_t memind = memoTableInd(&cParams, varyParams, varyLen); + //BMK_translateAdvancedParams(cParams); + if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { + return INFEASIBLE_RESULT; //see feasibleBenchMemo for concerns + } else { + int res = infeasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, ctx, dctx, + cParams, target, winnerResult); + if(res == FEASIBLE_RESULT) { + memoTable[memind] = 255; //infeasible resultscores could still be normal feasible. + } + return res; + } +} + +/* specifically feasibleBenchMemo and infeasibleBenchMemo */ +//maybe not necessary +typedef int (*BMK_benchMemo_t)(BMK_result_t*, const void*, size_t, void*, size_t, ZSTD_CCtx*, ZSTD_DCtx*, + const ZSTD_compressionParameters, const constraint_t, BMK_result_t*, U8*, U32*, const int); + +//varArray should be sanitized when this is called. +//TODO: transition to simpler greedy method if evaluation time is too long? +//would it be better to start at best feasible via feasible or infeasible metric? both? +//possibility climb is infeasible, responsibility of caller to check that. but if something feasible is evaluated, it will be returned +// *actually if it performs too +//sanitize all params here. +//all generation after random should be sanitized. (maybe sanitize random) +//TODO: paramTarget uneeded at this point w/ varArray and init; +static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varLen, U8* memoTable, + const void* srcBuffer, size_t srcSize, void* dstBuffer, size_t dstSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, ZSTD_compressionParameters init) { + //pick later initializations non-randomly? high dist from explored nodes. + //how to do this efficiently? (might not be too much of a problem, happens rarely, running time probably dominated by benchmarking) + //distance maximizing selection? + //cparam - currently considered center + //candidate - params to benchmark/results + //winner - best option found so far. + ZSTD_compressionParameters cparam = init; + winnerInfo_t candidateInfo, winnerInfo; + int better = 1; + + winnerInfo.params = init; + winnerInfo.result.cSpeed = 0; + winnerInfo.result.dSpeed = 0; + winnerInfo.result.cMem = (size_t)-1; + winnerInfo.result.cSize = (size_t)-1; + + /* ineasible -> (hopefully) feasible */ + /* when nothing is found, this garbages part 2. */ + { + //TODO: initialize these values! + winnerInfo_t bestFeasible1; /* uses feasibleBench Metric */ + winnerInfo_t bestFeasible2; /* uses resultScore Metric */ + + //init these params + bestFeasible1.params = cparam; + bestFeasible2.params = cparam; + bestFeasible1.result.cSpeed = 0; + bestFeasible1.result.dSpeed = 0; + bestFeasible1.result.cMem = (size_t)-1; + bestFeasible1.result.cSize = (size_t)-1; + bestFeasible2.result.cSpeed = 0; + bestFeasible2.result.dSpeed = 0; + bestFeasible2.result.cMem = (size_t)-1; + bestFeasible2.result.cSize = (size_t)-1; + DISPLAY("Climb Part 1\n"); + while(better) { + + //UTIL_time_t timestart = UTIL_getTime(); TODO: adjust sampling based on time + int i, d; + better = 0; + DISPLAY("Start\n"); + cparam = winnerInfo.params; + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + candidateInfo.params = cparam; + //all dist-1 targets + for(i = 0; i < varLen; i++) { + paramVaryOnce(varArray[i], 1, &candidateInfo.params); /* +1 */ + candidateInfo.params = sanitizeParams(candidateInfo.params); + //evaluate + //if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { + if(cParamValid(candidateInfo.params)) { + int res = infeasibleBenchMemo(&candidateInfo.result, + srcBuffer, srcSize, + dstBuffer, dstSize, + ctx, dctx, + candidateInfo.params, target, &winnerInfo.result, memoTable, + varArray, varLen); + if(res == FEASIBLE_RESULT) { /* synonymous with better when called w/ infeasibleBM */ + winnerInfo = candidateInfo; + //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + better = 1; + if(feasible(candidateInfo.result, target)) { + bestFeasible2 = winnerInfo; + if(objective_lt(bestFeasible1.result, bestFeasible2.result)) { + bestFeasible1 = bestFeasible2; /* using feasibleBench metric */ + } + } + } + } + candidateInfo.params = cparam; + paramVaryOnce(varArray[i], -1, &candidateInfo.params); /* -1 */ + candidateInfo.params = sanitizeParams(candidateInfo.params); + //evaluate + //if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { + if(cParamValid(candidateInfo.params)) { + int res = infeasibleBenchMemo(&candidateInfo.result, + srcBuffer, srcSize, + dstBuffer, dstSize, + ctx, dctx, + candidateInfo.params, target, &winnerInfo.result, memoTable, + varArray, varLen); + if(res == FEASIBLE_RESULT) { + winnerInfo = candidateInfo; + //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + better = 1; + if(feasible(candidateInfo.result, target)) { + bestFeasible2 = winnerInfo; + if(objective_lt(bestFeasible1.result, bestFeasible2.result)) { + bestFeasible1 = bestFeasible2; + } + } + } + } + } + + if(better) { + continue; + } + //if 'better' enough, skip further parameter search, center there? + //possible improvement - guide direction here w/ knowledge rather than completely random variation. + for(d = 2; d < varLen + 2; d++) { /* varLen is # dimensions */ + for(i = 0; i < 5; i++) { //make ? relative to # of free dimensions. + int res; + candidateInfo.params = cparam; + /* param error checking already done here */ + paramVariation(&candidateInfo.params, varArray, varLen, d); + res = infeasibleBenchMemo(&candidateInfo.result, + srcBuffer, srcSize, + dstBuffer, dstSize, + ctx, dctx, + candidateInfo.params, target, &winnerInfo.result, memoTable, + varArray, varLen); + if(res == FEASIBLE_RESULT) { /* synonymous with better in this case*/ + winnerInfo = candidateInfo; + //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + better = 1; + if(feasible(candidateInfo.result, target)) { + bestFeasible2 = winnerInfo; + if(objective_lt(bestFeasible1.result, bestFeasible2.result)) { + bestFeasible1 = bestFeasible2; + } + } + } + + } + if(better) { + continue; + } + } + //bias to test previous delta? + //change cparam -> candidate before restart + } + //TODO:Consider if this is best config. idea: explore from obj best keep rbest + cparam = bestFeasible2.params; + candidateInfo = bestFeasible2; + winnerInfo = bestFeasible1; + } + + //is it better to break here instead of bumbling about? + if(winnerInfo.result.cMem == (U32)-1) { + DISPLAY("No Feasible Found\n"); + return winnerInfo; + } + DISPLAY("Climb Part 2\n"); + + better = 1; + /* feasible -> best feasible (hopefully) */ + { + while(better) { + + //UTIL_time_t timestart = UTIL_getTime(); //TODO: if benchmarking is taking too long, be more greedy. + int i, d; + better = 0; + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + //all dist-1 targets + cparam = winnerInfo.params; //TODO: this messes the taking bestFeasible1, bestFeasible2 + candidateInfo.params = cparam; + for(i = 0; i < varLen; i++) { + paramVaryOnce(varArray[i], 1, &candidateInfo.params); + candidateInfo.params = sanitizeParams(candidateInfo.params); + + //evaluate + //if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { + if(cParamValid(candidateInfo.params)) { + int res = feasibleBenchMemo(&candidateInfo.result, + srcBuffer, srcSize, + dstBuffer, dstSize, + ctx, dctx, + candidateInfo.params, target, &winnerInfo.result, memoTable, + varArray, varLen); + if(res == FEASIBLE_RESULT) { + winnerInfo = candidateInfo; + //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + better = 1; + } + } + candidateInfo.params = cparam; + paramVaryOnce(varArray[i], -1, &candidateInfo.params); + candidateInfo.params = sanitizeParams(candidateInfo.params); + //evaluate + //if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { + if(cParamValid(candidateInfo.params)) { + int res = feasibleBenchMemo(&candidateInfo.result, + srcBuffer, srcSize, + dstBuffer, dstSize, + ctx, dctx, + candidateInfo.params, target, &winnerInfo.result, memoTable, + varArray, varLen); + if(res == FEASIBLE_RESULT) { + winnerInfo = candidateInfo; + //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + better = 1; + } + } + } + //if 'better' enough, skip further parameter search, center there? + //possible improvement - guide direction here w/ knowledge rather than completely random variation. + for(d = 2; d < varLen + 2; d++) { /* varLen is # dimensions */ + for(i = 0; i < 5; i++) { //TODO: make ? relative to # of free dimensions. + int res; + candidateInfo.params = cparam; + /* param error checking already done here */ + paramVariation(&candidateInfo.params, varArray, varLen, d); //info candidateInfo.params is garbage, this is too. + res = feasibleBenchMemo(&candidateInfo.result, + srcBuffer, srcSize, + dstBuffer, dstSize, + ctx, dctx, + candidateInfo.params, target, &winnerInfo.result, memoTable, + varArray, varLen); + if(res == FEASIBLE_RESULT) { + winnerInfo = candidateInfo; + //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + better = 1; + } + } + if(better) { + continue; + } + } + //bias to test previous delta? + //change cparam -> candidate before restart + } + } + return winnerInfo; +} + +//optimizeForSize but with fixed strategy +//place to configure/filter out strategy specific parameters. +//need args for all buffers and parameter stuff +//sanitization here. + +//flexible parameters: iterations of (failed?) climbing (or if we do non-random, maybe this is when everything is close to visitied) +//weight more on visit for bad results, less on good results/more on later results / ones with more failures. +//allocate memoTable here. +//only real use for paramTarget is to get the fixed values, right? +static winnerInfo_t optimizeFixedStrategy( + const void* srcBuffer, const size_t srcSize, + void* dstBuffer, size_t dstSize, + constraint_t target, ZSTD_compressionParameters paramTarget, + ZSTD_strategy strat, U32* varArray, int varLen) { + int i = 0; //TODO: Temp fix 10 iters, check effects of changing this? + U32* varNew = malloc(sizeof(U32) * varLen); + int varLenNew = sanitizeVarArray(varLen, varArray, varNew, strat); + size_t memoLen = memoTableLen(varNew, varLenNew); + U8* memoTable = malloc(sizeof(U8) * memoLen); + ZSTD_compressionParameters init; + ZSTD_CCtx* ctx = ZSTD_createCCtx(); + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + winnerInfo_t winnerInfo, candidateInfo; + winnerInfo.result.cSpeed = 0; + winnerInfo.result.dSpeed = 0; + winnerInfo.result.cMem = (size_t)(-1); + winnerInfo.result.cSize = (size_t)(-1); + /* so climb is given the right fixed strategy */ + paramTarget.strategy = strat; + /* to pass ZSTD_checkCParams */ + cParamZeroMin(¶mTarget); + memoTableInit(memoTable, paramTarget, target, varNew, varLenNew); + + + init = paramTarget; + + + if(!ctx || !dctx || !memoTable || !varNew) { + DISPLAY("NOT ENOUGH MEMORY ! ! ! \n"); + goto _cleanUp; + } + + while(i < 10) { + DISPLAY("Restart\n"); + randomConstrainedParams(&init, varNew, varLenNew, memoTable); + candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, srcBuffer, srcSize, dstBuffer, dstSize, ctx, dctx, init); + if(objective_lt(winnerInfo.result, candidateInfo.result)) { + winnerInfo = candidateInfo; + DISPLAY("Climb Winner: "); + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + } + i++; + } + +_cleanUp: + ZSTD_freeCCtx(ctx); + ZSTD_freeDCtx(dctx); + free(memoTable); + free(varNew); + return winnerInfo; +} + +// bigger and (hopefully) better* than optimizeForSize +// TODO: Change level bm'ing to respect constraints. +static int optimizeForSize2(const char* inFileName, constraint_t target, ZSTD_compressionParameters paramTarget) +{ + FILE* const inFile = fopen( inFileName, "rb" ); + U64 const inFileSize = UTIL_getFileSize(inFileName); + size_t benchedSize = BMK_findMaxMem(inFileSize*3) / 3; + void* origBuff; + U32 varArray [NUM_PARAMS]; + int varLen = variableParams(paramTarget, varArray); + /* Init */ + + + if(!cParamValid(paramTarget)) { + return 10; + } + + if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; } + if (inFileSize == UTIL_FILESIZE_UNKNOWN) { + DISPLAY("Pb evaluatin size of %s \n", inFileName); + fclose(inFile); + return 11; + } + + /* Memory allocation & restrictions */ + if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize; + if (benchedSize < inFileSize) { + DISPLAY("Not enough memory for '%s' \n", inFileName); + fclose(inFile); + return 11; + } + + /* Alloc */ + origBuff = malloc(benchedSize); + if(!origBuff) { + DISPLAY("\nError: not enough memory!\n"); + fclose(inFile); + return 12; + } + + /* Fill input buffer */ + DISPLAY("Loading %s... \r", inFileName); + { size_t const readSize = fread(origBuff, 1, benchedSize, inFile); + fclose(inFile); + if(readSize != benchedSize) { + DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); + free(origBuff); + return 13; + } } + + /* bench */ + DISPLAY("\r%79s\r", ""); + DISPLAY("optimizing for %s", inFileName); + if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed / 1000000); } + if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed / 1000000); } + if(target.cMem != (U32)-1) { DISPLAY(" - limit memory %u MB", target.cMem / 1000000); } + DISPLAY("\n"); + findClockGranularity(); + + { ZSTD_CCtx* const ctx = ZSTD_createCCtx(); + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + winnerInfo_t winner; + //BMK_result_t candidate; + const size_t blockSize = g_blockSize ? g_blockSize : benchedSize; + U32 const maxNbBlocks = (U32) ((benchedSize + (blockSize-1)) / blockSize) + 1; + const size_t maxCompressedSize = ZSTD_compressBound(benchedSize) + (maxNbBlocks * 1024); + void* compressedBuffer = malloc(maxCompressedSize); + + /* init */ + if (ctx==NULL) { DISPLAY("\n ZSTD_createCCtx error \n"); free(origBuff); return 14;} + if(compressedBuffer==NULL) { DISPLAY("\n Allocation Error \n"); free(origBuff); free(ctx); return 15; } + + memset(&winner, 0, sizeof(winner)); + winner.result.cSize = (size_t)(-1); + + + /* find best solution from default params */ + //Can't do this w/ cparameter constraints + //still useful though? + /* + { const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); + int i; + for (i=1; i<=maxSeeds; i++) { + ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, 0); + BMK_benchParam(&candidate, origBuff, benchedSize, ctx, dctx, CParams); + if (!feasible(candidate, target) ) { + break; + } + if (feasible(candidate,target) && objective_lt(winner.result, candidate)) + { + winner.params = CParams; + winner.result = candidate; + BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); + } } + }*/ + BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize); + + BMK_translateAdvancedParams(winner.params); + + /* start real tests */ + { + if(paramTarget.strategy == 0) { + int st; + for(st = 1; st <= 8; st++) { + winnerInfo_t wc = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, + target, paramTarget, st, varArray, varLen); + DISPLAY("StratNum %d\n", st); + if(objective_lt(winner.result, wc.result)) { + winner = wc; + } + } + } else { + winner = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, + target, paramTarget, paramTarget.strategy, varArray, varLen); + } + + } + + /* no solution found */ + if(winner.result.cSize == (size_t)-1) { + DISPLAY("No feasible solution found\n"); + return 1; + } + /* end summary */ + BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize); + BMK_translateAdvancedParams(winner.params); + DISPLAY("grillParams size - optimizer completed \n"); + + /* clean up*/ + ZSTD_freeCCtx(ctx); + ZSTD_freeDCtx(dctx); + } + + free(origBuff); + return 0; +} + + /* optimizeForSize(): * targetSpeed : expressed in B/s */ -/* if state space is small (from paramTarget) */ +/* expresses targeted compression, decompression speeds and memory requirements */ +/* if state space is small (from paramTarget), exhaustive search? */ +//things to consider : if doing strategy-separate approach, what cutoffs to evaluate each strategy +//or do all? can't be absolute, should be relative after some sort of calibration +//(synthetic? test levels (we don't care about data specifics rn, scale?) ? int optimizeForSize(const char* inFileName, constraint_t target, ZSTD_compressionParameters paramTarget) { FILE* const inFile = fopen( inFileName, "rb" ); @@ -872,7 +1905,7 @@ int optimizeForSize(const char* inFileName, constraint_t target, ZSTD_compressio DISPLAY("optimizing for %s", inFileName); if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed / 1000000); } if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed / 1000000); } - if(target.Mem != 0) { DISPLAY(" - limit memory %u MB", target.Mem / 1000000); } + if(target.cMem != 0) { DISPLAY(" - limit memory %u MB", target.cMem / 1000000); } DISPLAY("\n"); { ZSTD_CCtx* const ctx = ZSTD_createCCtx(); ZSTD_DCtx* const dctx = ZSTD_createDCtx(); @@ -881,12 +1914,13 @@ int optimizeForSize(const char* inFileName, constraint_t target, ZSTD_compressio const size_t blockSize = g_blockSize ? g_blockSize : benchedSize; /* init */ - if (ctx==NULL) { DISPLAY("\n ZSTD_createCCtx error \n"); free(origBuff); return 14;} + if (ctx==NULL) { DISPLAY("\n ZSTD_createCCtx error \n"); free(origBuff); return 14; } + memset(&winner, 0, sizeof(winner)); winner.result.cSize = (size_t)(-1); /* find best solution from default params */ - //Can't do this iteration normally w/ cparameter constraints + //Can't do this w/ cparameter constraints { const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); int i; for (i=1; i<=maxSeeds; i++) { @@ -910,15 +1944,17 @@ int optimizeForSize(const char* inFileName, constraint_t target, ZSTD_compressio { time_t const grillStart = time(NULL); do { ZSTD_compressionParameters params = winner.params; - paramVariation(¶ms, paramVarArray, paramCount); + BYTE* b; + paramVariation(¶ms, paramVarArray, paramCount, 4); if ((FUZ_rand(&g_rand) & 31) == 3) params = randomParams(); /* totally random config to improve search space */ params = ZSTD_adjustCParams(params, blockSize, 0); /* exclude faster if already played set of params */ - if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(params))-1)) continue; + if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(params))-1)) continue; /* test */ - NB_TESTS_PLAYED(params)++; + b = NB_TESTS_PLAYED(params); + (*b)++; BMK_benchParam(&candidate, origBuff, benchedSize, ctx, dctx, params); /* improvement found => new winner */ @@ -1027,7 +2063,8 @@ int main(int argc, const char** argv) U32 optimizer = 0; U32 main_pause = 0; - constraint_t target = { 0 , 0, 0 }; //0 for anything unset + + constraint_t target = { 0, 0, (U32)-1 }; //0 for anything unset ZSTD_compressionParameters paramTarget = { 0, 0, 0, 0, 0, 0, 0 }; assert(argc>=1); /* for exename */ @@ -1053,7 +2090,7 @@ int main(int argc, const char** argv) if (longCommandWArg(&argument, "strategy=") || longCommandWArg(&argument, "strat=")) { paramTarget.strategy = (ZSTD_strategy)(readU32FromChar(&argument)); if (argument[0]==',') { argument++; continue; } else break; } if (longCommandWArg(&argument, "compressionSpeed=") || longCommandWArg(&argument, "cSpeed=")) { target.cSpeed = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } if (longCommandWArg(&argument, "decompressionSpeed=") || longCommandWArg(&argument, "dSpeed=")) { target.dSpeed = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "compressionMemory=") || longCommandWArg(&argument, "cMem=")) { target.Mem = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "compressionMemory=") || longCommandWArg(&argument, "cMem=")) { target.cMem = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } /* in MB or MB/s */ DISPLAY("invalid optimization parameter \n"); return 1; @@ -1132,7 +2169,7 @@ int main(int argc, const char** argv) continue; case 'M': argument++; - target.Mem = readU32FromChar(&argument) * 1000000; + target.cMem = readU32FromChar(&argument) * 1000000; continue; case 'w': argument++; @@ -1255,7 +2292,8 @@ int main(int argc, const char** argv) } } else { if (optimizer) { - result = optimizeForSize(input_filename, target, paramTarget); + result = optimizeForSize2(input_filename, target, paramTarget); + //optimizeForSize(input_filename, target, paramTarget); } else { result = benchFiles(argv+filenamesStart, argc-filenamesStart); } } From eb21b7f48224b35ceb91b31a2fe1bb764545724e Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 9 Jul 2018 13:44:01 -0700 Subject: [PATCH 03/55] Not crashing --- programs/bench.c | 10 ++++---- tests/paramgrill.c | 59 +++++++++++++++++++++++++++------------------- 2 files changed, 40 insertions(+), 29 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 0d35f3eb3..5d5b47aba 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -288,7 +288,7 @@ volatile char g_touched; /* takes # of blocks and list of size & stuff for each. */ /* only does looping */ /* note time per loop could be zero if interval too short */ -BMK_customReturn_t __attribute__((optimize("O0"))) BMK_benchFunction( +BMK_customReturn_t BMK_benchFunction( BMK_benchFn_t benchFn, void* benchPayload, BMK_initFn_t initFn, void* initPayload, size_t blockCount, @@ -310,6 +310,7 @@ BMK_customReturn_t __attribute__((optimize("O0"))) BMK_benchFunction( } { + unsigned i, j; for(i = 0; i < blockCount; i++) { for(j = 0; j < srcBlockSizes[i]; j++) { @@ -318,11 +319,11 @@ BMK_customReturn_t __attribute__((optimize("O0"))) BMK_benchFunction( } for(i = 0; i < blockCount; i++) { memset(dstBlockBuffers[i], 0xE5, dstBlockCapacities[i]); /* warm up and erase result buffer */ - //this is written at end proc? where did compressed data get overwritten ny this? } - UTIL_sleepMilli(5); /* give processor time to other processes */ - UTIL_waitForNextTick(); + //UTIL_sleepMilli(5); /* give processor time to other processes */ + //UTIL_waitForNextTick(); + } { @@ -463,7 +464,6 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( } { size_t const decodedSize = (size_t)totalDSize64; free(*resultBufferPtr); - //TODO: decodedSize corrupted? *resultBufferPtr = malloc(decodedSize); if (!(*resultBufferPtr)) { EXM_THROW(33, BMK_return_t, "not enough memory"); diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 44bcedef8..77e9f2250 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -598,8 +598,8 @@ static void paramVariation(ZSTD_compressionParameters* ptr, const U32* varyParam const U32 changeID = FUZ_rand(&g_rand) % (varyLen << 1); paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p); } - //validated = !ZSTD_isError(ZSTD_checkCParams(p)); - validated = cParamValid(p); + validated = !ZSTD_isError(ZSTD_checkCParams(p)); + //validated = cParamValid(p); //Make sure memory is at least close to feasible? //ZSTD_estimateCCtxSize thing. @@ -669,23 +669,27 @@ static void memoTableIndInv(ZSTD_compressionParameters* ptr, const U32* varyPara //initializing memoTable /* */ -static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstraints, constraint_t target, const U32* varyParams, const int varyLen) { +static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstraints, constraint_t target, const U32* varyParams, const int varyLen, const size_t srcSize) { size_t i; size_t arrayLen = memoTableLen(varyParams, varyLen); int cwFixed = !paramConstraints.chainLog || !paramConstraints.windowLog; int scFixed = !paramConstraints.searchLog || !paramConstraints.chainLog; + int wFixed = !paramConstraints.windowLog; int j = 0; memset(memoTable, 0, arrayLen); - + cParamZeroMin(¶mConstraints); for(i = 0; i < arrayLen; i++) { memoTableIndInv(¶mConstraints, varyParams, varyLen, i); - BMK_translateAdvancedParams(paramConstraints); if(ZSTD_estimateCCtxSize_usingCParams(paramConstraints) + (1 << paramConstraints.windowLog) > target.cMem) { //infeasible; memoTable[i] = 255; j++; } + //TODO: remove any where memoTable wlog is mark any where windowlog is too big for data. + if(wFixed && (1 << paramConstraints.windowLog) > (srcSize << 1)) { + memoTable[i] = 255; + } /* nil out parameter sets equivalent to others. */ if(cwFixed/* at most least 1 param fixed. */) { if(paramConstraints.strategy == ZSTD_btlazy2 || paramConstraints.strategy == ZSTD_btopt || paramConstraints.strategy == ZSTD_btultra) { @@ -771,8 +775,8 @@ static ZSTD_compressionParameters randomParams(void) p.searchLength=(FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN)) + ZSTD_SEARCHLENGTH_MIN; p.targetLength=(FUZ_rand(&g_rand) % (512)); p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btultra +1)); - //validated = !ZSTD_isError(ZSTD_checkCParams(p)); - validated = cParamValid(p); + validated = !ZSTD_isError(ZSTD_checkCParams(p)); + //validated = cParamValid(p); } return p; } @@ -1201,7 +1205,7 @@ static int feasibleBench(BMK_result_t* resultPtr, if(loopDurationD < TIMELOOP_NANOSEC) { BMK_return_t benchres2; adv.mode = BMK_decodeOnly; - benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1230,7 +1234,7 @@ static int feasibleBench(BMK_result_t* resultPtr, if(loopDurationD < TIMELOOP_NANOSEC) { BMK_return_t benchres2; adv.mode = BMK_decodeOnly; - benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1275,14 +1279,12 @@ static int infeasibleBench(BMK_result_t* resultPtr, BMK_advancedParams_t adv = BMK_initAdvancedParams(); BMK_return_t benchres; BMK_result_t resultMin, resultMax; - UTIL_time_t startTime; U64 loopDurationC = 0, loopDurationD = 0; double uncertaintyConstantC, uncertaintyConstantD; double winnerRS = resultScore(*winnerResult, srcSize, target); adv.loopMode = BMK_iterMode; //can only use this for ratio measurement then, super inaccurate timing adv.nbSeconds = 1; //get ratio and 2x approx speed? //maybe run until twice MIN(minloopinterval * clockDuration) - (void)startTime; //TODO: actually use this to adjust timing DISPLAY("WinnerScore: %f\n ", winnerRS); /* adv.loopMode = BMK_timeMode; @@ -1290,6 +1292,11 @@ static int infeasibleBench(BMK_result_t* resultPtr, benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); + adv.loopMode = BMK_timeMode; + adv.nbSeconds = 1; + benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); + if(!benchres.error) { *resultPtr = benchres.result; if(eqZero(benchres.result.cSpeed)) { @@ -1372,6 +1379,7 @@ static int feasibleBenchMemo(BMK_result_t* resultPtr, U32* varyParams, const int varyLen) { size_t memind = memoTableInd(&cParams, varyParams, varyLen); + //BMK_translateAdvancedParams(cParams); if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { return INFEASIBLE_RESULT; //probably pick a different code for already tested? @@ -1397,6 +1405,7 @@ static int infeasibleBenchMemo(BMK_result_t* resultPtr, BMK_result_t* winnerResult, U8* memoTable, U32* varyParams, const int varyLen) { size_t memind = memoTableInd(&cParams, varyParams, varyLen); + //BMK_translateAdvancedParams(cParams); if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { return INFEASIBLE_RESULT; //see feasibleBenchMemo for concerns @@ -1422,7 +1431,6 @@ typedef int (*BMK_benchMemo_t)(BMK_result_t*, const void*, size_t, void*, size_t // *actually if it performs too //sanitize all params here. //all generation after random should be sanitized. (maybe sanitize random) -//TODO: paramTarget uneeded at this point w/ varArray and init; static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varLen, U8* memoTable, const void* srcBuffer, size_t srcSize, void* dstBuffer, size_t dstSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, ZSTD_compressionParameters init) { //pick later initializations non-randomly? high dist from explored nodes. @@ -1470,12 +1478,13 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); candidateInfo.params = cparam; //all dist-1 targets + //if we early end this, we should also randomize the order these are picked. for(i = 0; i < varLen; i++) { paramVaryOnce(varArray[i], 1, &candidateInfo.params); /* +1 */ candidateInfo.params = sanitizeParams(candidateInfo.params); //evaluate - //if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { - if(cParamValid(candidateInfo.params)) { + if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { + //if(cParamValid(candidateInfo.params)) { int res = infeasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, @@ -1498,8 +1507,8 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL paramVaryOnce(varArray[i], -1, &candidateInfo.params); /* -1 */ candidateInfo.params = sanitizeParams(candidateInfo.params); //evaluate - //if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { - if(cParamValid(candidateInfo.params)) { + if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { + //if(cParamValid(candidateInfo.params)) { int res = infeasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, @@ -1587,8 +1596,8 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL candidateInfo.params = sanitizeParams(candidateInfo.params); //evaluate - //if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { - if(cParamValid(candidateInfo.params)) { + if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { + //if(cParamValid(candidateInfo.params)) { int res = feasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, @@ -1605,8 +1614,8 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL paramVaryOnce(varArray[i], -1, &candidateInfo.params); candidateInfo.params = sanitizeParams(candidateInfo.params); //evaluate - //if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { - if(cParamValid(candidateInfo.params)) { + if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { + //if(cParamValid(candidateInfo.params)) { int res = feasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, @@ -1681,9 +1690,11 @@ static winnerInfo_t optimizeFixedStrategy( /* so climb is given the right fixed strategy */ paramTarget.strategy = strat; /* to pass ZSTD_checkCParams */ - cParamZeroMin(¶mTarget); - memoTableInit(memoTable, paramTarget, target, varNew, varLenNew); + memoTableInit(memoTable, paramTarget, target, varNew, varLenNew, srcSize); + + //needs to happen after memoTableInit as that assumes 0 = undefined. + cParamZeroMin(¶mTarget); init = paramTarget; @@ -1699,7 +1710,7 @@ static winnerInfo_t optimizeFixedStrategy( candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, srcBuffer, srcSize, dstBuffer, dstSize, ctx, dctx, init); if(objective_lt(winnerInfo.result, candidateInfo.result)) { winnerInfo = candidateInfo; - DISPLAY("Climb Winner: "); + DISPLAY("New Winner: "); BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); } i++; @@ -1714,7 +1725,7 @@ _cleanUp: } // bigger and (hopefully) better* than optimizeForSize -// TODO: Change level bm'ing to respect constraints. +// TODO: allow accept multiple files like benchFiles or bench.c fn's static int optimizeForSize2(const char* inFileName, constraint_t target, ZSTD_compressionParameters paramTarget) { FILE* const inFile = fopen( inFileName, "rb" ); From fab44388014a8fbb3cf4ba9f46cfdc241807299d Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 9 Jul 2018 18:37:54 -0700 Subject: [PATCH 04/55] Dictionary + Multiple file Loading --- tests/paramgrill.c | 209 ++++++++++++++++++++++++++++++++------------- 1 file changed, 151 insertions(+), 58 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 77e9f2250..2ad8c5752 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -114,6 +114,7 @@ static U32 g_singleRun = 0; static U32 g_target = 0; static U32 g_noSeed = 0; static ZSTD_compressionParameters g_params = { 0, 0, 0, 0, 0, 0, ZSTD_greedy }; +static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */ void BMK_SetNbIterations(int nbLoops) { @@ -1141,6 +1142,7 @@ static double calcUncertainty(double speed, size_t srcSize) { static int feasibleBench(BMK_result_t* resultPtr, const void* srcBuffer, size_t srcSize, void* dstBuffer, size_t dstSize, + void* dictBuffer, size_t dictSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams, const constraint_t target, @@ -1155,7 +1157,7 @@ static int feasibleBench(BMK_result_t* resultPtr, //alternative - test 1 iter for ratio, (possibility of error 3 which is fine), //maybe iter this until 2x measurable for better guarantee? DISPLAY("Feas:\n"); - benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres.error) { DISPLAY("ERROR %d !!\n", benchres.error); } @@ -1195,7 +1197,7 @@ static int feasibleBench(BMK_result_t* resultPtr, if(loopDurationC < TIMELOOP_NANOSEC) { BMK_return_t benchres2; adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1205,7 +1207,7 @@ static int feasibleBench(BMK_result_t* resultPtr, if(loopDurationD < TIMELOOP_NANOSEC) { BMK_return_t benchres2; adv.mode = BMK_decodeOnly; - benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1224,7 +1226,7 @@ static int feasibleBench(BMK_result_t* resultPtr, if(loopDurationC < TIMELOOP_NANOSEC) { BMK_return_t benchres2; adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1234,7 +1236,7 @@ static int feasibleBench(BMK_result_t* resultPtr, if(loopDurationD < TIMELOOP_NANOSEC) { BMK_return_t benchres2; adv.mode = BMK_decodeOnly; - benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1250,7 +1252,7 @@ static int feasibleBench(BMK_result_t* resultPtr, return INFEASIBLE_RESULT; } else { //possibly better, benchmark and find out adv.loopMode = BMK_timeMode; - benchres = BMK_benchMemAdvanced(srcBuffer, srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres = BMK_benchMemAdvanced(srcBuffer, srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); *resultPtr = benchres.result; return objective_lt(*winnerResult, benchres.result); } @@ -1272,6 +1274,7 @@ static int feasibleBench(BMK_result_t* resultPtr, static int infeasibleBench(BMK_result_t* resultPtr, const void* srcBuffer, size_t srcSize, void* dstBuffer, size_t dstSize, + void* dictBuffer, size_t dictSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams, const constraint_t target, @@ -1289,12 +1292,14 @@ static int infeasibleBench(BMK_result_t* resultPtr, /* adv.loopMode = BMK_timeMode; adv.nbSeconds = 1; */ - benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); + benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); adv.loopMode = BMK_timeMode; adv.nbSeconds = 1; - benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); if(!benchres.error) { @@ -1336,7 +1341,7 @@ static int infeasibleBench(BMK_result_t* resultPtr, if(loopDurationC < TIMELOOP_NANOSEC) { BMK_return_t benchres2; adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1348,7 +1353,7 @@ static int infeasibleBench(BMK_result_t* resultPtr, adv.mode = BMK_decodeOnly; //TODO: dstBuffer corrupted sometime between top and now //probably occuring in feasible bench too. - benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1372,6 +1377,7 @@ static int infeasibleBench(BMK_result_t* resultPtr, static int feasibleBenchMemo(BMK_result_t* resultPtr, const void* srcBuffer, size_t srcSize, void* dstBuffer, size_t dstSize, + void* dictBuffer, size_t dictSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams, const constraint_t target, @@ -1387,7 +1393,7 @@ static int feasibleBenchMemo(BMK_result_t* resultPtr, //what is the intended behavior in this case? //ignore? stop iterating completely? other? } else { - int res = feasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, ctx, dctx, + int res = feasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, ctx, dctx, cParams, target, winnerResult); memoTable[memind] = 255; //tested are all infeasible (other possible values for opti) return res; @@ -1399,6 +1405,7 @@ static int feasibleBenchMemo(BMK_result_t* resultPtr, static int infeasibleBenchMemo(BMK_result_t* resultPtr, const void* srcBuffer, size_t srcSize, void* dstBuffer, size_t dstSize, + void* dictBuffer, size_t dictSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams, const constraint_t target, @@ -1410,7 +1417,7 @@ static int infeasibleBenchMemo(BMK_result_t* resultPtr, if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { return INFEASIBLE_RESULT; //see feasibleBenchMemo for concerns } else { - int res = infeasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, ctx, dctx, + int res = infeasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, ctx, dctx, cParams, target, winnerResult); if(res == FEASIBLE_RESULT) { memoTable[memind] = 255; //infeasible resultscores could still be normal feasible. @@ -1432,7 +1439,7 @@ typedef int (*BMK_benchMemo_t)(BMK_result_t*, const void*, size_t, void*, size_t //sanitize all params here. //all generation after random should be sanitized. (maybe sanitize random) static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varLen, U8* memoTable, - const void* srcBuffer, size_t srcSize, void* dstBuffer, size_t dstSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, ZSTD_compressionParameters init) { + const void* srcBuffer, size_t srcSize, void* dstBuffer, size_t dstSize, void* dictBuffer, size_t dictSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, ZSTD_compressionParameters init) { //pick later initializations non-randomly? high dist from explored nodes. //how to do this efficiently? (might not be too much of a problem, happens rarely, running time probably dominated by benchmarking) //distance maximizing selection? @@ -1488,6 +1495,7 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL int res = infeasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, + dictBuffer, dictSize, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); @@ -1512,6 +1520,7 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL int res = infeasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, + dictBuffer, dictSize, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); @@ -1519,7 +1528,7 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL winnerInfo = candidateInfo; //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); better = 1; - if(feasible(candidateInfo.result, target)) { + if(feasible(candidateInfo.result, target)) { //TODO: maybe just break here and move on to part 2? bestFeasible2 = winnerInfo; if(objective_lt(bestFeasible1.result, bestFeasible2.result)) { bestFeasible1 = bestFeasible2; @@ -1535,14 +1544,15 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL //if 'better' enough, skip further parameter search, center there? //possible improvement - guide direction here w/ knowledge rather than completely random variation. for(d = 2; d < varLen + 2; d++) { /* varLen is # dimensions */ - for(i = 0; i < 5; i++) { //make ? relative to # of free dimensions. + for(i = 0; i < 2 * varLen + 2; i++) { int res; candidateInfo.params = cparam; /* param error checking already done here */ paramVariation(&candidateInfo.params, varArray, varLen, d); res = infeasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, - dstBuffer, dstSize, + dstBuffer, dstSize, + dictBuffer, dictSize, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); @@ -1601,6 +1611,7 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL int res = feasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, + dictBuffer, dictSize, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); @@ -1619,6 +1630,7 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL int res = feasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, + dictBuffer, dictSize, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); @@ -1632,7 +1644,7 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL //if 'better' enough, skip further parameter search, center there? //possible improvement - guide direction here w/ knowledge rather than completely random variation. for(d = 2; d < varLen + 2; d++) { /* varLen is # dimensions */ - for(i = 0; i < 5; i++) { //TODO: make ? relative to # of free dimensions. + for(i = 0; i < 2 * varLen + 2; i++) { int res; candidateInfo.params = cparam; /* param error checking already done here */ @@ -1640,6 +1652,7 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL res = feasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, + dictBuffer, dictSize, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); @@ -1672,6 +1685,7 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL static winnerInfo_t optimizeFixedStrategy( const void* srcBuffer, const size_t srcSize, void* dstBuffer, size_t dstSize, + void* dictBuffer, size_t dictSize, constraint_t target, ZSTD_compressionParameters paramTarget, ZSTD_strategy strat, U32* varArray, int varLen) { int i = 0; //TODO: Temp fix 10 iters, check effects of changing this? @@ -1706,12 +1720,14 @@ static winnerInfo_t optimizeFixedStrategy( while(i < 10) { DISPLAY("Restart\n"); + //TODO: look into improving this to maximize distance from searched infeasible stuff / towards promising regions? randomConstrainedParams(&init, varNew, varLenNew, memoTable); - candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, srcBuffer, srcSize, dstBuffer, dstSize, ctx, dctx, init); + candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, ctx, dctx, init); if(objective_lt(winnerInfo.result, candidateInfo.result)) { winnerInfo = candidateInfo; DISPLAY("New Winner: "); BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + i = 0; } i++; } @@ -1724,59 +1740,125 @@ _cleanUp: return winnerInfo; } +static int BMK_loadFiles(void* buffer, size_t bufferSize, + size_t* fileSizes, const char* const * const fileNamesTable, + unsigned nbFiles) +{ + size_t pos = 0, totalSize = 0; + unsigned n; + for (n=0; n bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */ + { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f); + if (readSize != (size_t)fileSize) { + DISPLAY("could not read %s", fileNamesTable[n]); + return 11; + } + pos += readSize; } + fileSizes[n] = (size_t)fileSize; + totalSize += (size_t)fileSize; + fclose(f); + } + + if (totalSize == 0) { DISPLAY("no data to bench\n"); return 12; } + return 0; +} + // bigger and (hopefully) better* than optimizeForSize // TODO: allow accept multiple files like benchFiles or bench.c fn's -static int optimizeForSize2(const char* inFileName, constraint_t target, ZSTD_compressionParameters paramTarget) +static int optimizeForSize2(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget) { - FILE* const inFile = fopen( inFileName, "rb" ); - U64 const inFileSize = UTIL_getFileSize(inFileName); - size_t benchedSize = BMK_findMaxMem(inFileSize*3) / 3; + size_t benchedSize; void* origBuff; + void* dictBuffer; + size_t dictBufferSize; U32 varArray [NUM_PARAMS]; int varLen = variableParams(paramTarget, varArray); /* Init */ - - if(!cParamValid(paramTarget)) { return 10; } - if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; } - if (inFileSize == UTIL_FILESIZE_UNKNOWN) { - DISPLAY("Pb evaluatin size of %s \n", inFileName); - fclose(inFile); - return 11; - } - - /* Memory allocation & restrictions */ - if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize; - if (benchedSize < inFileSize) { - DISPLAY("Not enough memory for '%s' \n", inFileName); - fclose(inFile); - return 11; - } - - /* Alloc */ - origBuff = malloc(benchedSize); - if(!origBuff) { - DISPLAY("\nError: not enough memory!\n"); - fclose(inFile); - return 12; + /* load dictionary*/ + if (dictFileName != NULL) { + U64 const dictFileSize = UTIL_getFileSize(dictFileName); + if (dictFileSize > 64 MB) { + DISPLAY("dictionary file %s too large", dictFileName); + return 10; + } + dictBufferSize = (size_t)dictFileSize; + dictBuffer = malloc(dictBufferSize); + if (dictBuffer==NULL) { + DISPLAY("not enough memory for dictionary (%u bytes)", + (U32)dictBufferSize); + return 11; + } + { + int errorCode = BMK_loadFiles(dictBuffer, dictBufferSize, &dictBufferSize, &dictFileName, 1); + if(errorCode) { + free(dictBuffer); + return errorCode; + } + } } /* Fill input buffer */ - DISPLAY("Loading %s... \r", inFileName); - { size_t const readSize = fread(origBuff, 1, benchedSize, inFile); - fclose(inFile); - if(readSize != benchedSize) { - DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); + if(nbFiles == 1) { + DISPLAY("Loading %s... \r", fileNamesTable[0]); + } else { + DISPLAY("Loading %zd Files... \r", nbFiles); + } + + { + U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles); + int ec; + size_t* fileSizes = calloc(sizeof(size_t),nbFiles); + benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3; + origBuff = malloc(benchedSize); + if(!origBuff || !fileSizes) { + DISPLAY("Not enough memory for stuff\n"); free(origBuff); - return 13; - } } + free(fileSizes); + free(dictBuffer); + return 1; + } + ec = BMK_loadFiles(origBuff, benchedSize, fileSizes, fileNamesTable, nbFiles); + if(ec) { + DISPLAY("Error Loading Files"); + free(origBuff); + free(fileSizes); + free(dictBuffer); + return ec; + } + free(fileSizes); + } + + /* bench */ DISPLAY("\r%79s\r", ""); - DISPLAY("optimizing for %s", inFileName); + if(nbFiles == 1) { + DISPLAY("optimizing for %s", fileNamesTable[0]); + } else { + DISPLAY("optimizing for %zd Files", nbFiles); + } if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed / 1000000); } if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed / 1000000); } if(target.cMem != (U32)-1) { DISPLAY(" - limit memory %u MB", target.cMem / 1000000); } @@ -1828,7 +1910,7 @@ static int optimizeForSize2(const char* inFileName, constraint_t target, ZSTD_co if(paramTarget.strategy == 0) { int st; for(st = 1; st <= 8; st++) { - winnerInfo_t wc = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, + winnerInfo_t wc = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, dictBuffer, dictBufferSize, target, paramTarget, st, varArray, varLen); DISPLAY("StratNum %d\n", st); if(objective_lt(winner.result, wc.result)) { @@ -1836,7 +1918,7 @@ static int optimizeForSize2(const char* inFileName, constraint_t target, ZSTD_co } } } else { - winner = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, + winner = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, dictBuffer, dictBufferSize, target, paramTarget, paramTarget.strategy, varArray, varLen); } @@ -2070,7 +2152,8 @@ int main(int argc, const char** argv) filenamesStart=0, result; const char* exename=argv[0]; - const char* input_filename=0; + const char* input_filename = 0; + const char* dictFileName = 0; U32 optimizer = 0; U32 main_pause = 0; @@ -2283,6 +2366,17 @@ int main(int argc, const char** argv) g_grillDuration_s = (double)readU32FromChar(&argument); break; + /* load dictionary file (only applicable for optimizer rn) */ + case 'D': + if(i == argc - 1) { //last argument, return error. + DISPLAY("Dictionary file expected but not given\n"); + return 1; + } else { + i++; + dictFileName = argv[i]; + } + break; + /* Unknown command */ default : return badusage(exename); } @@ -2303,8 +2397,7 @@ int main(int argc, const char** argv) } } else { if (optimizer) { - result = optimizeForSize2(input_filename, target, paramTarget); - //optimizeForSize(input_filename, target, paramTarget); + result = optimizeForSize2(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget); } else { result = benchFiles(argv+filenamesStart, argc-filenamesStart); } } From 3adc217ea47f93f11e32ca6d3bf2f103ff3dbbd0 Mon Sep 17 00:00:00 2001 From: George Lu Date: Thu, 12 Jul 2018 17:30:39 -0700 Subject: [PATCH 05/55] Total Changes: Add different constraint types (decompression speed, compression memory, parameter constraints) Separate search space by strategy + strategy selection Memoize results Real random restarts Support multiple files Support Dictionary inputs Debug Macro for extra printing --- programs/bench.c | 14 +- programs/bench.h | 4 +- tests/Makefile | 2 +- tests/fullbench.c | 4 +- tests/paramgrill.c | 1090 ++++++++++++++++++++++---------------------- 5 files changed, 548 insertions(+), 566 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 5d5b47aba..f5184fa8f 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -85,7 +85,7 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; * Exceptions ***************************************/ #ifndef DEBUG -# define DEBUG 1 +# define DEBUG 0 #endif #define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); } @@ -188,7 +188,7 @@ static void BMK_initCCtx(ZSTD_CCtx* ctx, ZSTD_CCtx_setParameter(ctx, ZSTD_p_searchLog, comprParams->searchLog); ZSTD_CCtx_setParameter(ctx, ZSTD_p_minMatch, comprParams->searchLength); ZSTD_CCtx_setParameter(ctx, ZSTD_p_targetLength, comprParams->targetLength); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy, comprParams->strategy); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy , comprParams->strategy); ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize); } @@ -293,7 +293,7 @@ BMK_customReturn_t BMK_benchFunction( BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBlockBuffers, const size_t* srcBlockSizes, - void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, + void** const dstBlockBuffers, size_t* dstBlockCapacities, unsigned nbLoops) { size_t srcSize = 0, dstSize = 0, ind = 0; U64 totalTime; @@ -338,6 +338,11 @@ BMK_customReturn_t BMK_benchFunction( j, (U32)dstBlockCapacities[j], ZSTD_getErrorName(res)); } else if(firstIter) { dstSize += res; + //Make compressed blocks continuous + if(j != blockCount - 1) { + dstBlockBuffers[j+1] = (void*)((char*)dstBlockBuffers[j] + res); + dstBlockCapacities[j] = res; + } } } firstIter = 0; @@ -370,13 +375,14 @@ void BMK_freeTimeState(BMK_timedFnState_t* state) { free(state); } +/* make option for dstBlocks to be */ BMK_customTimedReturn_t BMK_benchFunctionTimed( BMK_timedFnState_t* cont, BMK_benchFn_t benchFn, void* benchPayload, BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const* const srcBlockBuffers, const size_t* srcBlockSizes, - void* const* const dstBlockBuffers, const size_t* dstBlockCapacities) + void** const dstBlockBuffers, size_t* dstBlockCapacities) { U64 fastest = cont->fastestTime; int completed = 0; diff --git a/programs/bench.h b/programs/bench.h index 625b65757..1a298cc48 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -191,7 +191,7 @@ BMK_customReturn_t BMK_benchFunction( BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBuffers, const size_t* srcSizes, - void* const * const dstBuffers, const size_t* dstCapacities, + void** const dstBuffers, size_t* dstCapacities, unsigned nbLoops); @@ -220,7 +220,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed(BMK_timedFnState_t* cont, BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBlockBuffers, const size_t* srcBlockSizes, - void* const * const dstBlockBuffers, const size_t* dstBlockCapacities); + void** const dstBlockBuffers, size_t* dstBlockCapacities); #endif /* BENCH_H_121279284357 */ diff --git a/tests/Makefile b/tests/Makefile index ecf7fe2a7..81e685780 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -200,7 +200,7 @@ zstreamtest-dll : $(ZSTDDIR)/common/xxhash.c # xxh symbols not exposed from dll zstreamtest-dll : $(ZSTREAM_LOCAL_FILES) $(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT) -#paramgrill : DEBUGFLAGS = # turn off assert() for speed measurements +paramgrill : DEBUGFLAGS = # turn off assert() for speed measurements paramgrill : $(ZSTD_FILES) $(PRGDIR)/bench.c $(PRGDIR)/datagen.c paramgrill.c $(CC) $(FLAGS) $^ -lm -o $@$(EXT) diff --git a/tests/fullbench.c b/tests/fullbench.c index 7859745a3..9e7639f92 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -336,7 +336,7 @@ size_t local_ZSTD_decompressContinue(const void* src, size_t srcSize, void* dst, static size_t benchMem(const void* src, size_t srcSize, U32 benchNb, int cLevel, ZSTD_compressionParameters* cparams) { BYTE* dstBuff; - size_t const dstBuffSize = ZSTD_compressBound(srcSize); + size_t dstBuffSize = ZSTD_compressBound(srcSize); void* buff2, *buff1; const char* benchName; BMK_benchFn_t benchFunction; @@ -516,7 +516,7 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb, int cLevel, { r = BMK_benchFunction(benchFunction, buff2, NULL, NULL, 1, &src, &srcSize, - (void * const * const)&dstBuff, &dstBuffSize, g_nbIterations); + (void **)&dstBuff, &dstBuffSize, g_nbIterations); if(r.error) { DISPLAY("ERROR %d ! ! \n", r.error); errorcode = r.error; diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 2ad8c5752..9af60a740 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -58,6 +58,11 @@ static const int g_maxNbVariations = 64; * Macros **************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define TIMED 0 +#ifndef DEBUG +# define DEBUG 0 +#endif +#define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); } #undef MIN #undef MAX @@ -81,8 +86,8 @@ static const int g_maxNbVariations = 64; #define ZSTD_WINDOWLOG_MAX 27 //no long range stuff for now. //make 2^[0,10] w/ 999 -#define ZSTD_TARGETLENGTH_MIN 0 //actually targeLengthlog min -#define ZSTD_TARGETLENGTH_MAX 10 +#define ZSTD_TARGETLENGTH_MIN 0 +#define ZSTD_TARGETLENGTH_MAX 999 //#define ZSTD_TARGETLENGTH_MAX 1024 #define WLOG_RANGE (ZSTD_WINDOWLOG_MAX - ZSTD_WINDOWLOG_MIN + 1) @@ -90,15 +95,14 @@ static const int g_maxNbVariations = 64; #define HLOG_RANGE (ZSTD_HASHLOG_MAX - ZSTD_HASHLOG_MIN + 1) #define SLOG_RANGE (ZSTD_SEARCHLOG_MAX - ZSTD_SEARCHLOG_MIN + 1) #define SLEN_RANGE (ZSTD_SEARCHLENGTH_MAX - ZSTD_SEARCHLENGTH_MIN + 1) -#define TLEN_RANGE 11 +#define TLEN_RANGE 12 +//TLEN_RANGE = 0, 2^0 to 2^10; //hard coded since we only use powers of 2 (and 999 ~ 1024) -static const int mintable[NUM_PARAMS] = { ZSTD_WINDOWLOG_MIN, ZSTD_CHAINLOG_MIN, ZSTD_HASHLOG_MIN, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLENGTH_MIN, ZSTD_TARGETLENGTH_MIN }; -static const int maxtable[NUM_PARAMS] = { ZSTD_WINDOWLOG_MAX, ZSTD_CHAINLOG_MAX, ZSTD_HASHLOG_MAX, ZSTD_SEARCHLOG_MAX, ZSTD_SEARCHLENGTH_MAX, ZSTD_TARGETLENGTH_MAX }; +//static const int mintable[NUM_PARAMS] = { ZSTD_WINDOWLOG_MIN, ZSTD_CHAINLOG_MIN, ZSTD_HASHLOG_MIN, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLENGTH_MIN, ZSTD_TARGETLENGTH_MIN }; +//static const int maxtable[NUM_PARAMS] = { ZSTD_WINDOWLOG_MAX, ZSTD_CHAINLOG_MAX, ZSTD_HASHLOG_MAX, ZSTD_SEARCHLOG_MAX, ZSTD_SEARCHLENGTH_MAX, ZSTD_TARGETLENGTH_MAX }; static const int rangetable[NUM_PARAMS] = { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE }; -//use grid-search or something when space is small enough? -#define SMALL_SEARCH_SPACE 1000 /*-************************************ * Benchmark Parameters **************************************/ @@ -201,7 +205,7 @@ static void findClockGranularity(void) { } } } while(i < 10); - DISPLAY("Granularity: %llu\n", (unsigned long long)g_clockGranularity); + DEBUGOUTPUT("Granularity: %llu\n", (unsigned long long)g_clockGranularity); } typedef struct { @@ -225,6 +229,10 @@ static int cParamValid(ZSTD_compressionParameters paramTarget) { CLAMPCHECK(paramTarget.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX); CLAMPCHECK(paramTarget.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX); CLAMPCHECK(paramTarget.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX); + if(paramTarget.targetLength > ZSTD_TARGETLENGTH_MAX) { + DISPLAY("INVALID PARAMETER CONSTRAINTS\n"); + return 0; + } if(paramTarget.strategy > ZSTD_btultra) { DISPLAY("INVALID PARAMETER CONSTRAINTS\n"); return 0; @@ -232,21 +240,68 @@ static int cParamValid(ZSTD_compressionParameters paramTarget) { return 1; } +//TODO: let targetLength = 0; static void cParamZeroMin(ZSTD_compressionParameters* paramTarget) { paramTarget->windowLog = paramTarget->windowLog ? paramTarget->windowLog : ZSTD_WINDOWLOG_MIN; paramTarget->searchLog = paramTarget->searchLog ? paramTarget->searchLog : ZSTD_SEARCHLOG_MIN; paramTarget->chainLog = paramTarget->chainLog ? paramTarget->chainLog : ZSTD_CHAINLOG_MIN; paramTarget->hashLog = paramTarget->hashLog ? paramTarget->hashLog : ZSTD_HASHLOG_MIN; paramTarget->searchLength = paramTarget->searchLength ? paramTarget->searchLength : ZSTD_SEARCHLENGTH_MIN; - paramTarget->targetLength = paramTarget->targetLength ? paramTarget->targetLength : 1; + paramTarget->targetLength = paramTarget->targetLength ? paramTarget->targetLength : 0; } -static void BMK_translateAdvancedParams(ZSTD_compressionParameters params) +static void BMK_translateAdvancedParams(const ZSTD_compressionParameters params) { DISPLAY("--zstd=windowLog=%u,chainLog=%u,hashLog=%u,searchLog=%u,searchLength=%u,targetLength=%u,strategy=%u \n", params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, params.targetLength, (U32)(params.strategy)); } +/* checks results are feasible */ +static int feasible(const BMK_result_t results, const constraint_t target) { + return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem || !target.cMem); +} + +#define EPSILON 0.01 +static int epsilonEqual(const double c1, const double c2) { + return MAX(c1/c2,c2/c1) < 1 + EPSILON; +} + +/* checks exact equivalence to 0, to stop compiler complaining fpeq */ +static int eqZero(const double c1) { + return (U64)c1 == (U64)0.0 || (U64)c1 == (U64)-0.0; +} + +/* returns 1 if result2 is strictly 'better' than result1 */ +/* strict comparison / cutoff based */ +static int objective_lt(const BMK_result_t result1, const BMK_result_t result2) { + return (result1.cSize > result2.cSize) || (epsilonEqual(result1.cSize, result2.cSize) && result2.cSpeed > result1.cSpeed) + || (epsilonEqual(result1.cSize,result2.cSize) && epsilonEqual(result2.cSpeed, result1.cSpeed) && result2.dSpeed > result1.dSpeed); +} + +/* hill climbing value for part 1 */ +static double resultScore(const BMK_result_t res, const size_t srcSize, const constraint_t target) { + double cs = 0., ds = 0., rt, cm = 0.; + const double r1 = 1, r2 = 0.1, rtr = 0.5; + double ret; + if(target.cSpeed) { cs = res.cSpeed / (double)target.cSpeed; } + if(target.dSpeed) { ds = res.dSpeed / (double)target.dSpeed; } + if(target.cMem != (U32)-1) { cm = (double)target.cMem / res.cMem; } + rt = ((double)srcSize / res.cSize); + + ret = (MIN(1, cs) + MIN(1, ds) + MIN(1, cm))*r1 + rt * rtr + + (MAX(0, log(cs))+ MAX(0, log(ds))+ MAX(0, log(cm))) * r2; + //DISPLAY("resultScore: %f\n", ret); + return ret; +} + +/* factor sort of arbitrary */ +static constraint_t relaxTarget(constraint_t target) { + target.cMem = (U32)-1; + target.cSpeed *= 0.9; + target.dSpeed *= 0.9; + return target; +} + /*-******************************************************* * Bench functions *********************************************************/ @@ -268,9 +323,21 @@ const char* g_stratName[ZSTD_btultra+1] = { "ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ", "ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "}; -/* TODO: support additional parameters (more files, fileSizes) */ static size_t BMK_benchParam(BMK_result_t* resultPtr, + const void* srcBuffer, const size_t srcSize, + const size_t* fileSizes, const unsigned nbFiles, + ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, + const ZSTD_compressionParameters cParams) { + + BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, fileSizes, nbFiles, 0, &cParams, NULL, 0, ctx, dctx, 0, "File"); + *resultPtr = res.result; + return res.error; +} + +/* benchParam but only takes in one file. */ +static size_t +BMK_benchParam1(BMK_result_t* resultPtr, const void* srcBuffer, size_t srcSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams) { @@ -280,27 +347,54 @@ BMK_benchParam(BMK_result_t* resultPtr, return res.error; } -static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_compressionParameters params, size_t srcSize) -{ - char lvlstr[15] = "Custom Level"; - DISPLAY("\r%79s\r", ""); - fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", - params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, - params.targetLength, g_stratName[(U32)(params.strategy)]); - if(cLevel != CUSTOM_LEVEL) { - snprintf(lvlstr, 15, " Level %2u ", cLevel); - } - fprintf(f, - "/* %s */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", - lvlstr, (double)srcSize / result.cSize, result.cSpeed / 1000000., result.dSpeed / 1000000.); -} - - typedef struct { BMK_result_t result; ZSTD_compressionParameters params; } winnerInfo_t; +/* global winner used for display. */ +//Should be totally 0 initialized? +static winnerInfo_t g_winner; //TODO: ratio is infinite at initialization, instead of 0 +static constraint_t g_targetConstraints; + +static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const size_t srcSize) +{ + if(DEBUG || (objective_lt(g_winner.result, result) && feasible(result, g_targetConstraints))) { + char lvlstr[15] = "Custom Level"; + const U64 time = UTIL_clockSpanNano(g_time); + const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC); + if(DEBUG && (objective_lt(g_winner.result, result) && feasible(result, g_targetConstraints))) { + DISPLAY("New Winner: \n"); + } + + DISPLAY("\r%79s\r", ""); + + fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", + params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, + params.targetLength, g_stratName[(U32)(params.strategy)]); + if(cLevel != CUSTOM_LEVEL) { + snprintf(lvlstr, 15, " Level %2u ", cLevel); + } + fprintf(f, + "/* %s */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */", + lvlstr, (double)srcSize / result.cSize, result.cSpeed / 1000000., result.dSpeed / 1000000.); + + if(TIMED) { fprintf(f, " - %lu:%lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); } + fprintf(f, "\n"); + if(objective_lt(g_winner.result, result) && feasible(result, g_targetConstraints)) { + BMK_translateAdvancedParams(params); + g_winner.result = result; + g_winner.params = params; + } + } + //else { + // DISPLAY("G_WINNER: "); + // DISPLAY("/* R:%5.3f at %5.1f MB/s - %5.1f MB/s */ \n",(double)srcSize / g_winner.result.cSize , g_winner.result.cSpeed / 1000000 , g_winner.result.dSpeed / 1000000); + // DISPLAY("LOSER : "); + // DISPLAY("/* R:%5.3f at %5.1f MB/s - %5.1f MB/s */ \n",(double)srcSize / result.cSize, result.cSpeed / 1000000 , result.dSpeed / 1000000); + //} +} + static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize) { int cLevel; @@ -358,7 +452,7 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para int better = 0; int cLevel; - BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, dctx, params); + BMK_benchParam1(&testResult, srcBuffer, srcSize, ctx, dctx, params); for (cLevel = 1; cLevel <= NB_LEVELS_TRACKED; cLevel++) { @@ -470,7 +564,7 @@ static ZSTD_compressionParameters sanitizeParams(ZSTD_compressionParameters para /* new length */ /* keep old array, will need if iter over strategy. */ -static int sanitizeVarArray(int varLength, U32* varArray, U32* varNew, ZSTD_strategy strat) { +static int sanitizeVarArray(const int varLength, const U32* varArray, U32* varNew, const ZSTD_strategy strat) { int i, j = 0; for(i = 0; i < varLength; i++) { if( !((varArray[i] == CLOG_IND && strat == ZSTD_fast) @@ -515,53 +609,10 @@ static int variableParams(const ZSTD_compressionParameters paramConstraints, U32 return j; } -/* computes inverse of above array, returns same number, -1 = unused ind */ -static int inverseVariableParams(const ZSTD_compressionParameters paramConstraints, U32* res) { - int j = 0; - if(!paramConstraints.windowLog) { - res[WLOG_IND] = j; - j++; - } else { - res[WLOG_IND] = -1; - } - if(!paramConstraints.chainLog) { - res[j] = CLOG_IND; - j++; - } else { - res[CLOG_IND] = -1; - } - if(!paramConstraints.hashLog) { - res[j] = HLOG_IND; - j++; - } else { - res[HLOG_IND] = -1; - } - if(!paramConstraints.searchLog) { - res[j] = SLOG_IND; - j++; - } else { - res[SLOG_IND] = -1; - } - if(!paramConstraints.searchLength) { - res[j] = SLEN_IND; - j++; - } else { - res[SLEN_IND] = -1; - } - if(!paramConstraints.targetLength) { - res[j] = TLEN_IND; - j++; - } else { - res[TLEN_IND] = -1; - } - - return j; -} - /* amt will probably always be \pm 1? */ /* slight change from old paramVariation, targetLength can only take on powers of 2 now (999 ~= 1024?) */ /* take max/min bounds into account as well? */ -static void paramVaryOnce(U32 paramIndex, int amt, ZSTD_compressionParameters* ptr) { +static void paramVaryOnce(const U32 paramIndex, const int amt, ZSTD_compressionParameters* ptr) { switch(paramIndex) { case WLOG_IND: ptr->windowLog += amt; break; @@ -571,8 +622,14 @@ static void paramVaryOnce(U32 paramIndex, int amt, ZSTD_compressionParameters* p case SLEN_IND: ptr->searchLength += amt; break; case TLEN_IND: if(amt >= 0) { - ptr->targetLength <<= amt; - ptr->targetLength = MIN(ptr->targetLength, 999); + if(ptr->targetLength == 0) { + if(amt > 0) { + ptr->targetLength = MIN(1 << (amt - 1), 999); + } + } else { + ptr->targetLength <<= amt; + ptr->targetLength = MIN(ptr->targetLength, 999); + } } else { if(ptr->targetLength == 999) { ptr->targetLength = 1024; @@ -584,11 +641,8 @@ static void paramVaryOnce(U32 paramIndex, int amt, ZSTD_compressionParameters* p } } -//Don't fuzz fixed variables. -//turn pcs to pcs array with macro for params. -//pass in variation array from variableParams -//take nbChanges as argument? -static void paramVariation(ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen, U32 nbChanges) +/* varies ptr by nbChanges respecting varyParams*/ +static void paramVariation(ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen, const U32 nbChanges) { ZSTD_compressionParameters p; U32 validated = 0; @@ -600,19 +654,11 @@ static void paramVariation(ZSTD_compressionParameters* ptr, const U32* varyParam paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p); } validated = !ZSTD_isError(ZSTD_checkCParams(p)); - //validated = cParamValid(p); - - //Make sure memory is at least close to feasible? - //ZSTD_estimateCCtxSize thing. } - *ptr = sanitizeParams(p); + *ptr = p;//sanitizeParams(p); } -//varyParams gives us table size? -//1 per strategy -//varyParams should always be sorted smallest to largest -//take arrayLen to allocate memotable -//should be ~10^7 unconstrained. +/* length of memo table given free variables */ static size_t memoTableLen(const U32* varyParams, const int varyLen) { size_t arrayLen = 1; int i; @@ -622,11 +668,14 @@ static size_t memoTableLen(const U32* varyParams, const int varyLen) { return arrayLen; } -//sort of ~lg2 (replace 1024 w/ 999) for memoTableInd Tlen +//sort of ~lg2 (replace 1024 w/ 999, and add 0 at lower end of range) for memoTableInd Tlen static unsigned lg2(unsigned x) { - unsigned j = 0; + unsigned j = 1; if(x == 999) { - return 10; + return 11; + } + if(!x) { + return 0; } while(x >>= 1) { j++; @@ -634,8 +683,7 @@ static unsigned lg2(unsigned x) { return j; } -//indexes compressionParameters into memotable -//of form +/* returns unique index of compression parameters */ static unsigned memoTableInd(const ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen) { int i; unsigned ind = 0; @@ -652,25 +700,24 @@ static unsigned memoTableInd(const ZSTD_compressionParameters* ptr, const U32* v return ind; } -/* presumably, the unfilled parameters are already at their correct value */ -/* inverse above function for varyParams */ +/* inverse of above function (from index to parameters) */ static void memoTableIndInv(ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen, size_t ind) { int i; for(i = varyLen - 1; i >= 0; i--) { switch(varyParams[i]) { - case WLOG_IND: ptr->windowLog = ind % WLOG_RANGE + ZSTD_WINDOWLOG_MIN; ind /= WLOG_RANGE; break; - case CLOG_IND: ptr->chainLog = ind % CLOG_RANGE + ZSTD_CHAINLOG_MIN; ind /= CLOG_RANGE; break; - case HLOG_IND: ptr->hashLog = ind % HLOG_RANGE + ZSTD_HASHLOG_MIN; ind /= HLOG_RANGE; break; - case SLOG_IND: ptr->searchLog = ind % SLOG_RANGE + ZSTD_SEARCHLOG_MIN; ind /= SLOG_RANGE; break; - case SLEN_IND: ptr->searchLength = ind % SLEN_RANGE + ZSTD_SEARCHLENGTH_MIN; ind /= SLEN_RANGE; break; - case TLEN_IND: ptr->targetLength = MIN(1 << (ind % TLEN_RANGE), 999); ind /= TLEN_RANGE; break; + case WLOG_IND: ptr->windowLog = ind % WLOG_RANGE + ZSTD_WINDOWLOG_MIN; ind /= WLOG_RANGE; break; + case CLOG_IND: ptr->chainLog = ind % CLOG_RANGE + ZSTD_CHAINLOG_MIN; ind /= CLOG_RANGE; break; + case HLOG_IND: ptr->hashLog = ind % HLOG_RANGE + ZSTD_HASHLOG_MIN; ind /= HLOG_RANGE; break; + case SLOG_IND: ptr->searchLog = ind % SLOG_RANGE + ZSTD_SEARCHLOG_MIN; ind /= SLOG_RANGE; break; + case SLEN_IND: ptr->searchLength = ind % SLEN_RANGE + ZSTD_SEARCHLENGTH_MIN; ind /= SLEN_RANGE; break; + case TLEN_IND: ptr->targetLength = (ind % TLEN_RANGE) ? MIN(1 << ((ind % TLEN_RANGE) - 1), 999) : 0; ind /= TLEN_RANGE; break; } } } -//initializing memoTable -/* */ -static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstraints, constraint_t target, const U32* varyParams, const int varyLen, const size_t srcSize) { + +/* Initialize memotable, immediately mark redundant / obviously infeasible params as */ +static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstraints, const constraint_t target, const U32* varyParams, const int varyLen, const size_t srcSize) { size_t i; size_t arrayLen = memoTableLen(varyParams, varyLen); int cwFixed = !paramConstraints.chainLog || !paramConstraints.windowLog; @@ -682,13 +729,11 @@ static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstra for(i = 0; i < arrayLen; i++) { memoTableIndInv(¶mConstraints, varyParams, varyLen, i); - if(ZSTD_estimateCCtxSize_usingCParams(paramConstraints) + (1 << paramConstraints.windowLog) > target.cMem) { - //infeasible; + if((ZSTD_estimateCCtxSize_usingCParams(paramConstraints) + (1ULL << paramConstraints.windowLog)) > (size_t)target.cMem + (size_t)(target.cMem / 10)) { memoTable[i] = 255; j++; } - //TODO: remove any where memoTable wlog is mark any where windowlog is too big for data. - if(wFixed && (1 << paramConstraints.windowLog) > (srcSize << 1)) { + if(wFixed && (1ULL << paramConstraints.windowLog) > (srcSize << 1)) { memoTable[i] = 255; } /* nil out parameter sets equivalent to others. */ @@ -712,7 +757,42 @@ static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstra } } } - DISPLAY("%d / %d Invalid\n", j, (int)i); + DEBUGOUTPUT("%d / %d Invalid\n", j, (int)i); + if((int)i == j) { + DEBUGOUTPUT("!!!Strategy %d totally infeasible\n", (int)paramConstraints.strategy) + } +} + +/* inits memotables for all (including mallocs), all strategies */ +/* takes unsanitized varyParams */ + +//TODO: check for errors/nulls +static U8** memoTableInitAll(ZSTD_compressionParameters paramConstraints, constraint_t target, const U32* varyParams, const int varyLen, const size_t srcSize) { + U32 varNew[NUM_PARAMS]; + int varLenNew; + U8** mtAll = malloc(sizeof(U8*) * (ZSTD_btultra + 1)); + int i; + if(mtAll == NULL) { + return NULL; + } + for(i = 1; i <= (int)ZSTD_btultra; i++) { + varLenNew = sanitizeVarArray(varyLen, varyParams, varNew, i); + mtAll[i] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew)); + if(mtAll[i] == NULL) { + return NULL; + } + memoTableInit(mtAll[i], paramConstraints, target, varNew, varLenNew, srcSize); + } + return mtAll; +} + +static void memoTableFreeAll(U8** mtAll) { + int i; + if(mtAll == NULL) { return; } + for(i = 1; i <= (int)ZSTD_btultra; i++) { + free(mtAll[i]); + } + free(mtAll); } #define PARAMTABLELOG 25 @@ -782,10 +862,7 @@ static ZSTD_compressionParameters randomParams(void) return p; } -//destructively modifies pc. -//Maybe if memoTable[ind] > 0 too often, count zeroes and explicitly choose from free stuff? -//^ maybe this doesn't matter, with |mt| size it has \approx 1-(1/e) of finding even single free spot in |mt| tries, not too bad. -//TODO: maybe memoTable pc before sanitization too so no repeats? +/* Sets pc to random unmeasured set of parameters */ static void randomConstrainedParams(ZSTD_compressionParameters* pc, U32* varArray, int varLen, U8* memoTable) { int tries = memoTableLen(varArray, varLen); //configurable, @@ -795,9 +872,7 @@ static void randomConstrainedParams(ZSTD_compressionParameters* pc, U32* varArra ind = (FUZ_rand(&g_rand)) % maxSize; tries--; } while(memoTable[ind] > 0 && tries > 0); - //&& FUZ_rand(&g_rand) % 256 > memoTable[ind]); get nd choosing? (helpful w/ distance) /* maybe > infeasible bound? */ - /* memoTable[ind] == 0 -> unexplored */ memoTableIndInv(pc, varArray, varLen, (unsigned)ind); *pc = sanitizeParams(*pc); } @@ -822,7 +897,7 @@ static void BMK_benchOnce(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* srcBuffe { BMK_result_t testResult; g_params = ZSTD_adjustCParams(g_params, srcSize, 0); - BMK_benchParam(&testResult, srcBuffer, srcSize, cctx, dctx, g_params); + BMK_benchParam1(&testResult, srcBuffer, srcSize, cctx, dctx, g_params); DISPLAY("Compression Ratio: %.3f Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)srcSize / testResult.cSize, testResult.cSpeed / 1000000, testResult.dSpeed / 1000000); return; @@ -847,7 +922,7 @@ static void BMK_benchFullTable(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* src /* baseline config for level 1 */ ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, blockSize, 0); BMK_result_t testResult; - BMK_benchParam(&testResult, srcBuffer, srcSize, cctx, dctx, l1params); + BMK_benchParam1(&testResult, srcBuffer, srcSize, cctx, dctx, l1params); BMK_init_level_constraints((int)((testResult.cSpeed * 31) / 32)); } @@ -974,112 +1049,14 @@ int benchFiles(const char** fileNamesTable, int nbFiles) return 0; } -//parameter feasibility is not checked, should just be restricted from use. -static int feasible(BMK_result_t results, constraint_t target) { - return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem || !target.cMem); -} -#define EPSILON 0.01 -static int epsilonEqual(double c1, double c2) { - return MAX(c1/c2,c2/c1) < 1 + EPSILON; -} - -//so the compiler stops warning -static int eqZero(double c1) { - return (U64)c1 == (U64)0.0 || (U64)c1 == (U64)-0.0; -} - -/* returns 1 if result2 is strictly 'better' than result1 */ -/* strict comparison / cutoff based */ -static int objective_lt(BMK_result_t result1, BMK_result_t result2) { - return (result1.cSize > result2.cSize) || (epsilonEqual(result1.cSize, result2.cSize) && result2.cSpeed > result1.cSpeed) - || (epsilonEqual(result1.cSize,result2.cSize) && epsilonEqual(result2.cSpeed, result1.cSpeed) && result2.dSpeed > result1.dSpeed); -} - -//will probably be some linear combinartion of comp speed, decompSpeed, & ratio (maybe size), and memory? -//pretty arbitrary right now -//maybe better - higher coefficient when below threshold, lower when above -//need to normalize speed? or just use ratio speed / target? -//Maybe don't use ratio at all when looking for feasibility? - -/* maybe dynamically vary the coefficients for this around based on what's already been discovered. (maybe make a reversed ratio cutoff?) concave to pheaily penalize below ratio? */ -static double resultScore(BMK_result_t res, size_t srcSize, constraint_t target) { - double cs = 0., ds = 0., rt, cm = 0.; - const double r1 = 1, r2 = 0.1, rtr = 0.5; - double ret; - if(target.cSpeed) { cs = res.cSpeed / (double)target.cSpeed; } - if(target.dSpeed) { ds = res.dSpeed / (double)target.dSpeed; } - if(target.cMem != (U32)-1) { cm = (double)target.cMem / res.cMem; } - rt = ((double)srcSize / res.cSize); - - //(void)rt; - //(void)rtr; - - ret = (MIN(1, cs) + MIN(1, ds) + MIN(1, cm))*r1 + rt * rtr + - (MAX(0, log(cs))+ MAX(0, log(ds))+ MAX(0, log(cm))) * r2; - //DISPLAY("resultScore: %f\n", ret); - return ret; -} - -/* - double W_ratio = (double)srcSize / testResult.cSize; - double O_ratio = (double)srcSize / winners[cLevel].result.cSize; - double W_ratioNote = log (W_ratio); - double O_ratioNote = log (O_ratio); - size_t W_DMemUsed = (1 << params.windowLog) + (16 KB); - size_t O_DMemUsed = (1 << winners[cLevel].params.windowLog) + (16 KB); - double W_DMemUsed_note = W_ratioNote * ( 40 + 9*cLevel) - log((double)W_DMemUsed); - double O_DMemUsed_note = O_ratioNote * ( 40 + 9*cLevel) - log((double)O_DMemUsed); - - size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_estimateCCtxSize_usingCParams(params); - size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_estimateCCtxSize_usingCParams(winners[cLevel].params); - double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed); - double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed); - - double W_CSpeed_note = W_ratioNote * ( 30 + 10*cLevel) + log(testResult.cSpeed); - double O_CSpeed_note = O_ratioNote * ( 30 + 10*cLevel) + log(winners[cLevel].result.cSpeed); - - double W_DSpeed_note = W_ratioNote * ( 20 + 2*cLevel) + log(testResult.dSpeed); - double O_DSpeed_note = O_ratioNote * ( 20 + 2*cLevel) + log(winners[cLevel].result.dSpeed); - -*/ -//ratio tradeoffs, may be useful in guiding - -/* objective_lt, but based on scoring function */ -static int objective_lt2(BMK_result_t result1, BMK_result_t result2, size_t srcSize, constraint_t target) { - return resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target); -} - -/* res gives array dimensions, should be size NUM_PARAMS */ -static size_t computeStateSize(const ZSTD_compressionParameters paramConstraints, U32* res) { - int ind = 0; - size_t base = 1; - if(!paramConstraints.windowLog) { res[ind] = ZSTD_WINDOWLOG_MAX - ZSTD_WINDOWLOG_MIN + 1; base *= res[ind]; ind++; } - if(!paramConstraints.chainLog) { res[ind] = ZSTD_CHAINLOG_MAX - ZSTD_CHAINLOG_MIN + 1; base *= res[ind]; ind++; } - if(!paramConstraints.hashLog) { res[ind] = ZSTD_HASHLOG_MAX - ZSTD_HASHLOG_MIN + 1; base *= res[ind]; ind++; } - if(!paramConstraints.searchLog) { res[ind] = ZSTD_SEARCHLOG_MAX - ZSTD_SEARCHLOG_MIN + 1; base *= res[ind]; ind++; } - if(!paramConstraints.searchLength) { res[ind] = ZSTD_SEARCHLENGTH_MAX - ZSTD_SEARCHLENGTH_MIN + 1; base *= res[ind]; ind++; } - if(!paramConstraints.targetLength) { res[ind] = 11; base *= res[ind]; ind++; } //restricting from 2^[0,10], no such macros - if(!(U32)paramConstraints.strategy) { res[ind] = 8; base *= 8; } //not strictly true, maybe would want to case on this. - - return base; -} - - -static unsigned calcViolation(BMK_result_t results, constraint_t target) { - int diffcSpeed = MAX(target.cSpeed - results.cSpeed, 0); - int diffdSpeed = MAX(target.dSpeed - results.dSpeed, 0); - int diffcMem = MAX(results.cMem - target.cMem, 0); - return diffcSpeed + diffdSpeed + diffcMem; -} /* -uncertaintyConstant >= 1 -returns -1 = 'certainly' infeasible - 0 = unceratin - 1 = 'certainly' feasible +checks feasibility with uncertainty. +-1 : certainly infeasible + 0 : uncertain + 1 : certainly feasible */ -//paramTarget misnamed, should just be target static int uncertainFeasibility(double const uncertaintyConstantC, double const uncertaintyConstantD, const constraint_t paramTarget, const BMK_result_t* const results) { if((paramTarget.cSpeed != 0 && results->cSpeed * uncertaintyConstantC < paramTarget.cSpeed) || (paramTarget.dSpeed != 0 && results->dSpeed * uncertaintyConstantD < paramTarget.dSpeed) || @@ -1099,12 +1076,8 @@ static int uncertainFeasibility(double const uncertaintyConstantC, double const -1 - worse assume prev_best status is run fully? but then we'd have to rerun any winners anyway */ -//presumably memory has already been compared, mostly worried about mem, cspeed, dspeed -//uncertainty only applies to speed. -//if using objective fn, this could be much easier since we could just scale that. -//difficult to make judgements about later parameters in prioritization type when there's -//uncertainty on the first. -static int uncertainComparison(double const uncertaintyConstantC, double const uncertaintyConstantD, BMK_result_t* candidate, BMK_result_t* prevBest) { +/* not as useful as initially believed */ +static int uncertainComparison(double const uncertaintyConstantC, double const uncertaintyConstantD, const BMK_result_t* candidate, const BMK_result_t* prevBest) { (void)uncertaintyConstantD; //unused for now if(candidate->cSpeed > prevBest->cSpeed * uncertaintyConstantC) { return 1; @@ -1115,34 +1088,21 @@ static int uncertainComparison(double const uncertaintyConstantC, double const u } } -/* speed in b, srcSize in b/s loopDuration in ns */ -//TODO: simplify code in feasibleBench with this instead of writing it all out. -//only applicable for single loop -static double calcUncertainty(double speed, size_t srcSize) { - U64 loopDuration; - if(eqZero(speed)) { return 2; } - loopDuration = ((srcSize * TIMELOOP_NANOSEC) / speed); - return MIN((loopDuration + (double)2 * g_clockGranularity) / loopDuration, 2); -} +/*benchmarks and tests feasibility together + 1 = true = better + 0 = false = not better + if true then resultPtr will give results. + 2+ on error? */ -//benchmarks and tests feasibility together -//1 = true = better -//0 = false = not better -//if true then resultPtr will give results. -//2+ on error? -//alt: error = 0 / infeasible as well; -//maybe use compress_only mode for ratio-finding benchmark? -//prioritize ratio > cSpeed > dSpeed > cMem -//Misnamed - should be worse, better, error -//alternative (to make work for feasible-pt searching as well) - only compare to winner, not to target -//but then we need to judge what better means in this context, which shouldn't be the same (strict ratio improvement) +//Maybe use compress_only for benchmark #define INFEASIBLE_RESULT 0 #define FEASIBLE_RESULT 1 #define ERROR_RESULT 2 static int feasibleBench(BMK_result_t* resultPtr, - const void* srcBuffer, size_t srcSize, - void* dstBuffer, size_t dstSize, - void* dictBuffer, size_t dictSize, + const void* srcBuffer, const size_t srcSize, + void* dstBuffer, const size_t dstSize, + void* dictBuffer, const size_t dictSize, + const size_t* fileSizes, const size_t nbFiles, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams, const constraint_t target, @@ -1156,8 +1116,8 @@ static int feasibleBench(BMK_result_t* resultPtr, //alternative - test 1 iter for ratio, (possibility of error 3 which is fine), //maybe iter this until 2x measurable for better guarantee? - DISPLAY("Feas:\n"); - benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + DEBUGOUTPUT("Feas:\n"); + benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres.error) { DISPLAY("ERROR %d !!\n", benchres.error); } @@ -1174,7 +1134,7 @@ static int feasibleBench(BMK_result_t* resultPtr, //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? //possibly has to do with initCCtx? or system stuff? //asymmetric +/- constant needed? - uncertaintyConstantC = MIN((loopDurationC + (double)(2 * g_clockGranularity)/loopDurationC), 2); //.02 seconds + uncertaintyConstantC = MIN((loopDurationC + (double)(2 * g_clockGranularity)/loopDurationC) * 1.1, 3); //.02 seconds } if(eqZero(benchres.result.dSpeed)) { loopDurationD = 0; @@ -1184,20 +1144,41 @@ static int feasibleBench(BMK_result_t* resultPtr, //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? //possibly has to do with initCCtx? or system stuff? //asymmetric +/- constant needed? - uncertaintyConstantD = MIN((loopDurationD + (double)(2 * g_clockGranularity)/loopDurationD), 2); //.02 seconds + uncertaintyConstantD = MIN((loopDurationD + (double)(2 * g_clockGranularity)/loopDurationD) * 1.1, 3); //.02 seconds } if(benchres.result.cSize < winnerResult->cSize) { //better compression ratio, just needs to be feasible - //optimistic assume speed - //incoporate some sort of tradeoff comparison with the winner's results? - int feas = uncertainFeasibility(uncertaintyConstantC, uncertaintyConstantD, target, &(benchres.result)); + int feas; + if(loopDurationC < TIMELOOP_NANOSEC / 10) { + BMK_return_t benchres2; + adv.mode = BMK_compressOnly; + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres = benchres2; + } + } + if(loopDurationD < TIMELOOP_NANOSEC / 10) { + BMK_return_t benchres2; + adv.mode = BMK_decodeOnly; + benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres.result.dSpeed = benchres2.result.dSpeed; + } + } + *resultPtr = benchres.result; + + feas = uncertainFeasibility(uncertaintyConstantC, uncertaintyConstantD, target, &(benchres.result)); if(feas == 0) { // uncertain feasibility adv.loopMode = BMK_timeMode; if(loopDurationC < TIMELOOP_NANOSEC) { BMK_return_t benchres2; adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1220,13 +1201,34 @@ static int feasibleBench(BMK_result_t* resultPtr, return (feas + 1) >> 1; //relies on INFEASIBLE_RESULT == 0, FEASIBLE_RESULT == 1 } } else if (benchres.result.cSize == winnerResult->cSize) { //equal ratio, needs to be better than winner in cSpeed/ dSpeed / cMem - int feas = uncertainFeasibility(uncertaintyConstantC, uncertaintyConstantD, target, &(benchres.result)); + int feas; + if(loopDurationC < TIMELOOP_NANOSEC / 10) { + BMK_return_t benchres2; + adv.mode = BMK_compressOnly; + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres = benchres2; + } + } + if(loopDurationD < TIMELOOP_NANOSEC / 10) { + BMK_return_t benchres2; + adv.mode = BMK_decodeOnly; + benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres.result.dSpeed = benchres2.result.dSpeed; + } + } + feas = uncertainFeasibility(uncertaintyConstantC, uncertaintyConstantD, target, &(benchres.result)); if(feas == 0) { // uncertain feasibility adv.loopMode = BMK_timeMode; if(loopDurationC < TIMELOOP_NANOSEC) { BMK_return_t benchres2; adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1252,7 +1254,7 @@ static int feasibleBench(BMK_result_t* resultPtr, return INFEASIBLE_RESULT; } else { //possibly better, benchmark and find out adv.loopMode = BMK_timeMode; - benchres = BMK_benchMemAdvanced(srcBuffer, srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + benchres = BMK_benchMemAdvanced(srcBuffer, srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); *resultPtr = benchres.result; return objective_lt(*winnerResult, benchres.result); } @@ -1265,16 +1267,17 @@ static int feasibleBench(BMK_result_t* resultPtr, } else { return ERROR_RESULT; //BMK error } - } -//sameas before, but +/-? + +//same as before, but +/-? //alternative, just return comparison result, leave caller to worry about feasibility. //have version of benchMemAdvanced which takes in dstBuffer/cap as well? //(motivation: repeat tests (maybe just on decompress) don't need further compress runs) static int infeasibleBench(BMK_result_t* resultPtr, - const void* srcBuffer, size_t srcSize, - void* dstBuffer, size_t dstSize, - void* dictBuffer, size_t dictSize, + const void* srcBuffer, const size_t srcSize, + void* dstBuffer, const size_t dstSize, + void* dictBuffer, const size_t dictSize, + const size_t* fileSizes, const size_t nbFiles, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams, const constraint_t target, @@ -1288,19 +1291,10 @@ static int infeasibleBench(BMK_result_t* resultPtr, adv.loopMode = BMK_iterMode; //can only use this for ratio measurement then, super inaccurate timing adv.nbSeconds = 1; //get ratio and 2x approx speed? //maybe run until twice MIN(minloopinterval * clockDuration) - DISPLAY("WinnerScore: %f\n ", winnerRS); - /* - adv.loopMode = BMK_timeMode; - adv.nbSeconds = 1; */ - benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); - BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); - benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); - BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); + DEBUGOUTPUT("WinnerScore: %f\n ", winnerRS); - adv.loopMode = BMK_timeMode; - adv.nbSeconds = 1; - benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); - BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); + benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); if(!benchres.error) { *resultPtr = benchres.result; @@ -1309,9 +1303,7 @@ static int infeasibleBench(BMK_result_t* resultPtr, uncertaintyConstantC = 2; } else { loopDurationC = ((srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); - //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? - //possibly has to do with initCCtx? or system stuff? - uncertaintyConstantC = MIN((loopDurationC + (double)(2 * g_clockGranularity)/loopDurationC), 2); //.02 seconds + uncertaintyConstantC = MIN((loopDurationC + (double)(2 * g_clockGranularity)/loopDurationC * 1.1), 3); //.02 seconds } if(eqZero(benchres.result.dSpeed)) { @@ -1319,11 +1311,32 @@ static int infeasibleBench(BMK_result_t* resultPtr, uncertaintyConstantD = 2; } else { loopDurationD = ((srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); - //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? - //possibly has to do with initCCtx? or system stuff? - uncertaintyConstantD = MIN((loopDurationD + (double)(2 * g_clockGranularity)/loopDurationD), 2); //.02 seconds + uncertaintyConstantD = MIN((loopDurationD + (double)(2 * g_clockGranularity)/loopDurationD) * 1.1 , 3); //.02 seconds } + + if(loopDurationC < TIMELOOP_NANOSEC / 10) { + BMK_return_t benchres2; + adv.mode = BMK_compressOnly; + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres = benchres2; + } + } + if(loopDurationD < TIMELOOP_NANOSEC / 10) { + BMK_return_t benchres2; + adv.mode = BMK_decodeOnly; + benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres.result.dSpeed = benchres2.result.dSpeed; + } + } + *resultPtr = benchres.result; + /* benchres's certainty range. */ resultMax = benchres.result; resultMin = benchres.result; @@ -1331,8 +1344,6 @@ static int infeasibleBench(BMK_result_t* resultPtr, resultMax.dSpeed *= uncertaintyConstantD; resultMin.cSpeed /= uncertaintyConstantC; resultMin.dSpeed /= uncertaintyConstantD; - (void)resultMin; - //TODO: consider if resultMin is actually needed. if (winnerRS > resultScore(resultMax, srcSize, target)) { return INFEASIBLE_RESULT; } else { @@ -1341,7 +1352,7 @@ static int infeasibleBench(BMK_result_t* resultPtr, if(loopDurationC < TIMELOOP_NANOSEC) { BMK_return_t benchres2; adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1350,9 +1361,7 @@ static int infeasibleBench(BMK_result_t* resultPtr, } if(loopDurationD < TIMELOOP_NANOSEC) { BMK_return_t benchres2; - adv.mode = BMK_decodeOnly; - //TODO: dstBuffer corrupted sometime between top and now - //probably occuring in feasible bench too. + adv.mode = BMK_decodeOnly; benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; @@ -1372,28 +1381,27 @@ static int infeasibleBench(BMK_result_t* resultPtr, } /* wrap feasibleBench w/ memotable */ -//TODO: void sanitized and unsanitized ver's so input doesn't double-choose #define INFEASIBLE_THRESHOLD 200 static int feasibleBenchMemo(BMK_result_t* resultPtr, - const void* srcBuffer, size_t srcSize, - void* dstBuffer, size_t dstSize, - void* dictBuffer, size_t dictSize, + const void* srcBuffer, const size_t srcSize, + void* dstBuffer, const size_t dstSize, + void* dictBuffer, const size_t dictSize, + const size_t* fileSizes, const size_t nbFiles, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams, const constraint_t target, BMK_result_t* winnerResult, U8* memoTable, - U32* varyParams, const int varyLen) { + const U32* varyParams, const int varyLen) { - size_t memind = memoTableInd(&cParams, varyParams, varyLen); + const size_t memind = memoTableInd(&cParams, varyParams, varyLen); - //BMK_translateAdvancedParams(cParams); if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { return INFEASIBLE_RESULT; //probably pick a different code for already tested? //maybe remove this if we incorporate nonrandom location picking? //what is the intended behavior in this case? //ignore? stop iterating completely? other? } else { - int res = feasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, ctx, dctx, + int res = feasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, fileSizes, nbFiles, ctx, dctx, cParams, target, winnerResult); memoTable[memind] = 255; //tested are all infeasible (other possible values for opti) return res; @@ -1403,21 +1411,21 @@ static int feasibleBenchMemo(BMK_result_t* resultPtr, //should infeasible stage searching also be memo-marked in the same way? //don't actually memoize unless result is feasible/error? static int infeasibleBenchMemo(BMK_result_t* resultPtr, - const void* srcBuffer, size_t srcSize, - void* dstBuffer, size_t dstSize, - void* dictBuffer, size_t dictSize, + const void* srcBuffer, const size_t srcSize, + void* dstBuffer, const size_t dstSize, + void* dictBuffer, const size_t dictSize, + const size_t* fileSizes, const size_t nbFiles, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams, const constraint_t target, BMK_result_t* winnerResult, U8* memoTable, - U32* varyParams, const int varyLen) { + const U32* varyParams, const int varyLen) { size_t memind = memoTableInd(&cParams, varyParams, varyLen); - //BMK_translateAdvancedParams(cParams); if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { return INFEASIBLE_RESULT; //see feasibleBenchMemo for concerns } else { - int res = infeasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, ctx, dctx, + int res = infeasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, fileSizes, nbFiles, ctx, dctx, cParams, target, winnerResult); if(res == FEASIBLE_RESULT) { memoTable[memind] = 255; //infeasible resultscores could still be normal feasible. @@ -1432,14 +1440,18 @@ typedef int (*BMK_benchMemo_t)(BMK_result_t*, const void*, size_t, void*, size_t const ZSTD_compressionParameters, const constraint_t, BMK_result_t*, U8*, U32*, const int); //varArray should be sanitized when this is called. -//TODO: transition to simpler greedy method if evaluation time is too long? -//would it be better to start at best feasible via feasible or infeasible metric? both? //possibility climb is infeasible, responsibility of caller to check that. but if something feasible is evaluated, it will be returned // *actually if it performs too //sanitize all params here. //all generation after random should be sanitized. (maybe sanitize random) -static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varLen, U8* memoTable, - const void* srcBuffer, size_t srcSize, void* dstBuffer, size_t dstSize, void* dictBuffer, size_t dictSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, ZSTD_compressionParameters init) { +static winnerInfo_t climbOnce(const constraint_t target, + const U32* varArray, const int varLen, U8* memoTable, + const void* srcBuffer, size_t srcSize, + void* dstBuffer, const size_t dstSize, + void* dictBuffer, const size_t dictSize, + const size_t* fileSizes, const size_t nbFiles, + ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, + const ZSTD_compressionParameters init) { //pick later initializations non-randomly? high dist from explored nodes. //how to do this efficiently? (might not be too much of a problem, happens rarely, running time probably dominated by benchmarking) //distance maximizing selection? @@ -1459,28 +1471,20 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL /* ineasible -> (hopefully) feasible */ /* when nothing is found, this garbages part 2. */ { - //TODO: initialize these values! winnerInfo_t bestFeasible1; /* uses feasibleBench Metric */ - winnerInfo_t bestFeasible2; /* uses resultScore Metric */ //init these params bestFeasible1.params = cparam; - bestFeasible2.params = cparam; bestFeasible1.result.cSpeed = 0; bestFeasible1.result.dSpeed = 0; bestFeasible1.result.cMem = (size_t)-1; bestFeasible1.result.cSize = (size_t)-1; - bestFeasible2.result.cSpeed = 0; - bestFeasible2.result.dSpeed = 0; - bestFeasible2.result.cMem = (size_t)-1; - bestFeasible2.result.cSize = (size_t)-1; DISPLAY("Climb Part 1\n"); while(better) { - //UTIL_time_t timestart = UTIL_getTime(); TODO: adjust sampling based on time int i, d; better = 0; - DISPLAY("Start\n"); + DEBUGOUTPUT("Start\n"); cparam = winnerInfo.params; BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); candidateInfo.params = cparam; @@ -1496,18 +1500,16 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, + fileSizes, nbFiles, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); if(res == FEASIBLE_RESULT) { /* synonymous with better when called w/ infeasibleBM */ winnerInfo = candidateInfo; - //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); better = 1; - if(feasible(candidateInfo.result, target)) { - bestFeasible2 = winnerInfo; - if(objective_lt(bestFeasible1.result, bestFeasible2.result)) { - bestFeasible1 = bestFeasible2; /* using feasibleBench metric */ - } + if(feasible(candidateInfo.result, target) && objective_lt(bestFeasible1.result, winnerInfo.result)) { + bestFeasible1 = winnerInfo; } } } @@ -1521,18 +1523,16 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, + fileSizes, nbFiles, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); if(res == FEASIBLE_RESULT) { winnerInfo = candidateInfo; - //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); better = 1; - if(feasible(candidateInfo.result, target)) { //TODO: maybe just break here and move on to part 2? - bestFeasible2 = winnerInfo; - if(objective_lt(bestFeasible1.result, bestFeasible2.result)) { - bestFeasible1 = bestFeasible2; - } + if(feasible(candidateInfo.result, target) && objective_lt(bestFeasible1.result, winnerInfo.result)) { + bestFeasible1 = winnerInfo; } } } @@ -1553,18 +1553,16 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, + fileSizes, nbFiles, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); if(res == FEASIBLE_RESULT) { /* synonymous with better in this case*/ winnerInfo = candidateInfo; - //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); better = 1; - if(feasible(candidateInfo.result, target)) { - bestFeasible2 = winnerInfo; - if(objective_lt(bestFeasible1.result, bestFeasible2.result)) { - bestFeasible1 = bestFeasible2; - } + if(feasible(candidateInfo.result, target) && objective_lt(bestFeasible1.result, winnerInfo.result)) { + bestFeasible1 = winnerInfo; } } @@ -1576,15 +1574,12 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL //bias to test previous delta? //change cparam -> candidate before restart } - //TODO:Consider if this is best config. idea: explore from obj best keep rbest - cparam = bestFeasible2.params; - candidateInfo = bestFeasible2; winnerInfo = bestFeasible1; } - //is it better to break here instead of bumbling about? + //break out if no feasible. if(winnerInfo.result.cMem == (U32)-1) { - DISPLAY("No Feasible Found\n"); + DEBUGOUTPUT("No Feasible Found\n"); return winnerInfo; } DISPLAY("Climb Part 2\n"); @@ -1592,14 +1587,12 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL better = 1; /* feasible -> best feasible (hopefully) */ { - while(better) { - - //UTIL_time_t timestart = UTIL_getTime(); //TODO: if benchmarking is taking too long, be more greedy. + while(better) { int i, d; better = 0; BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); //all dist-1 targets - cparam = winnerInfo.params; //TODO: this messes the taking bestFeasible1, bestFeasible2 + cparam = winnerInfo.params; candidateInfo.params = cparam; for(i = 0; i < varLen; i++) { paramVaryOnce(varArray[i], 1, &candidateInfo.params); @@ -1612,12 +1605,13 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, + fileSizes, nbFiles, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); if(res == FEASIBLE_RESULT) { winnerInfo = candidateInfo; - //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); better = 1; } } @@ -1626,17 +1620,17 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL candidateInfo.params = sanitizeParams(candidateInfo.params); //evaluate if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { - //if(cParamValid(candidateInfo.params)) { int res = feasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, + fileSizes, nbFiles, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); if(res == FEASIBLE_RESULT) { winnerInfo = candidateInfo; - //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); better = 1; } } @@ -1653,12 +1647,13 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, + fileSizes, nbFiles, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); if(res == FEASIBLE_RESULT) { winnerInfo = candidateInfo; - //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); better = 1; } } @@ -1684,29 +1679,26 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL //only real use for paramTarget is to get the fixed values, right? static winnerInfo_t optimizeFixedStrategy( const void* srcBuffer, const size_t srcSize, - void* dstBuffer, size_t dstSize, - void* dictBuffer, size_t dictSize, - constraint_t target, ZSTD_compressionParameters paramTarget, - ZSTD_strategy strat, U32* varArray, int varLen) { - int i = 0; //TODO: Temp fix 10 iters, check effects of changing this? + void* dstBuffer, const size_t dstSize, + void* dictBuffer, const size_t dictSize, + const size_t* fileSizes, const size_t nbFiles, + const constraint_t target, ZSTD_compressionParameters paramTarget, + const ZSTD_strategy strat, const U32* varArray, const int varLen, U8* memoTable) { + int i = 0; U32* varNew = malloc(sizeof(U32) * varLen); int varLenNew = sanitizeVarArray(varLen, varArray, varNew, strat); - size_t memoLen = memoTableLen(varNew, varLenNew); - U8* memoTable = malloc(sizeof(U8) * memoLen); ZSTD_compressionParameters init; ZSTD_CCtx* ctx = ZSTD_createCCtx(); ZSTD_DCtx* dctx = ZSTD_createDCtx(); winnerInfo_t winnerInfo, candidateInfo; winnerInfo.result.cSpeed = 0; winnerInfo.result.dSpeed = 0; - winnerInfo.result.cMem = (size_t)(-1); - winnerInfo.result.cSize = (size_t)(-1); + winnerInfo.result.cMem = (size_t)(-1LL); + winnerInfo.result.cSize = (size_t)(-1LL); /* so climb is given the right fixed strategy */ paramTarget.strategy = strat; /* to pass ZSTD_checkCParams */ - memoTableInit(memoTable, paramTarget, target, varNew, varLenNew, srcSize); - //needs to happen after memoTableInit as that assumes 0 = undefined. cParamZeroMin(¶mTarget); @@ -1718,11 +1710,11 @@ static winnerInfo_t optimizeFixedStrategy( goto _cleanUp; } - while(i < 10) { - DISPLAY("Restart\n"); - //TODO: look into improving this to maximize distance from searched infeasible stuff / towards promising regions? + while(i < 10) { //make i adjustable (user input?) depending on how much time they have. + DISPLAY("Restart\n"); //TODO: make better printing across restarts + //look into improving this to maximize distance from searched infeasible stuff / towards promising regions? randomConstrainedParams(&init, varNew, varLenNew, memoTable); - candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, ctx, dctx, init); + candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, fileSizes, nbFiles, ctx, dctx, init); if(objective_lt(winnerInfo.result, candidateInfo.result)) { winnerInfo = candidateInfo; DISPLAY("New Winner: "); @@ -1735,7 +1727,6 @@ static winnerInfo_t optimizeFixedStrategy( _cleanUp: ZSTD_freeCCtx(ctx); ZSTD_freeDCtx(dctx); - free(memoTable); free(varNew); return winnerInfo; } @@ -1777,20 +1768,54 @@ static int BMK_loadFiles(void* buffer, size_t bufferSize, fclose(f); } - if (totalSize == 0) { DISPLAY("no data to bench\n"); return 12; } + if (totalSize == 0) { DISPLAY("\nno data to bench\n"); return 12; } return 0; } -// bigger and (hopefully) better* than optimizeForSize -// TODO: allow accept multiple files like benchFiles or bench.c fn's -static int optimizeForSize2(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget) +//goes best, best-1, best+1, best-2, ... +//return 0 if nothing remaining +static int nextStrategy(const int currentStrategy, const int bestStrategy) { + if(bestStrategy <= currentStrategy) { + int candidate = 2 * bestStrategy - currentStrategy - 1; + if(candidate < 1) { + candidate = currentStrategy + 1; + if(candidate > (int)ZSTD_btultra) { + return 0; + } else { + return candidate; + } + } else { + return candidate; + } + } else { /* bestStrategy >= currentStrategy */ + int candidate = 2 * bestStrategy - currentStrategy; + if(candidate > (int)ZSTD_btultra) { + candidate = currentStrategy - 1; + if(candidate < 1) { + return 0; + } else { + return candidate; + } + } else { + return candidate; + } + } +} + +//optimize fixed strategy. +static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget) { size_t benchedSize; - void* origBuff; - void* dictBuffer; - size_t dictBufferSize; + void* origBuff = NULL; + void* dictBuffer = NULL; + size_t dictBufferSize = 0; U32 varArray [NUM_PARAMS]; - int varLen = variableParams(paramTarget, varArray); + int ret = 0; + size_t* fileSizes = calloc(sizeof(size_t),nbFiles); + const int varLen = variableParams(paramTarget, varArray); + U8** allMT = NULL; + g_targetConstraints = target; + g_winner.result.cSize = (size_t)-1; /* Init */ if(!cParamValid(paramTarget)) { return 10; @@ -1801,20 +1826,24 @@ static int optimizeForSize2(const char* const * const fileNamesTable, const size U64 const dictFileSize = UTIL_getFileSize(dictFileName); if (dictFileSize > 64 MB) { DISPLAY("dictionary file %s too large", dictFileName); - return 10; + ret = 10; + goto _cleanUp; } dictBufferSize = (size_t)dictFileSize; dictBuffer = malloc(dictBufferSize); if (dictBuffer==NULL) { DISPLAY("not enough memory for dictionary (%u bytes)", (U32)dictBufferSize); - return 11; + ret = 11; + goto _cleanUp; + } + { int errorCode = BMK_loadFiles(dictBuffer, dictBufferSize, &dictBufferSize, &dictFileName, 1); if(errorCode) { - free(dictBuffer); - return errorCode; + ret = errorCode; + goto _cleanUp; } } } @@ -1823,41 +1852,46 @@ static int optimizeForSize2(const char* const * const fileNamesTable, const size if(nbFiles == 1) { DISPLAY("Loading %s... \r", fileNamesTable[0]); } else { - DISPLAY("Loading %zd Files... \r", nbFiles); + DISPLAY("Loading %lu Files... \r", (unsigned long)nbFiles); } { U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles); int ec; - size_t* fileSizes = calloc(sizeof(size_t),nbFiles); + unsigned i; benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3; + origBuff = malloc(benchedSize); if(!origBuff || !fileSizes) { DISPLAY("Not enough memory for stuff\n"); - free(origBuff); - free(fileSizes); - free(dictBuffer); - return 1; + ret = 1; + goto _cleanUp; } ec = BMK_loadFiles(origBuff, benchedSize, fileSizes, fileNamesTable, nbFiles); if(ec) { DISPLAY("Error Loading Files"); - free(origBuff); - free(fileSizes); - free(dictBuffer); - return ec; + ret = ec; + goto _cleanUp; } - free(fileSizes); + benchedSize = 0; + for(i = 0; i < nbFiles; i++) { + benchedSize += fileSizes[i]; + } + origBuff = realloc(origBuff, benchedSize); } - + allMT = memoTableInitAll(paramTarget, target, varArray, varLen, benchedSize); + if(!allMT) { + ret = 2; + goto _cleanUp; + } /* bench */ DISPLAY("\r%79s\r", ""); if(nbFiles == 1) { DISPLAY("optimizing for %s", fileNamesTable[0]); } else { - DISPLAY("optimizing for %zd Files", nbFiles); + DISPLAY("optimizing for %lu Files", (unsigned long)nbFiles); } if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed / 1000000); } if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed / 1000000); } @@ -1868,7 +1902,7 @@ static int optimizeForSize2(const char* const * const fileNamesTable, const size { ZSTD_CCtx* const ctx = ZSTD_createCCtx(); ZSTD_DCtx* const dctx = ZSTD_createDCtx(); winnerInfo_t winner; - //BMK_result_t candidate; + U32 varNew[NUM_PARAMS]; const size_t blockSize = g_blockSize ? g_blockSize : benchedSize; U32 const maxNbBlocks = (U32) ((benchedSize + (blockSize-1)) / blockSize) + 1; const size_t maxCompressedSize = ZSTD_compressBound(benchedSize) + (maxNbBlocks * 1024); @@ -1877,49 +1911,120 @@ static int optimizeForSize2(const char* const * const fileNamesTable, const size /* init */ if (ctx==NULL) { DISPLAY("\n ZSTD_createCCtx error \n"); free(origBuff); return 14;} if(compressedBuffer==NULL) { DISPLAY("\n Allocation Error \n"); free(origBuff); free(ctx); return 15; } - memset(&winner, 0, sizeof(winner)); winner.result.cSize = (size_t)(-1); /* find best solution from default params */ - //Can't do this w/ cparameter constraints - //still useful though? - /* - { const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); - int i; - for (i=1; i<=maxSeeds; i++) { - ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, 0); - BMK_benchParam(&candidate, origBuff, benchedSize, ctx, dctx, CParams); - if (!feasible(candidate, target) ) { - break; + { + /* strategy selection */ + //TODO: don't factor memory into strategy selection in constraint_t + const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); + DEBUGOUTPUT("Strategy Selection\n"); + if(varLen == NUM_PARAMS && paramTarget.strategy == 0) { /* no variable based constraints */ + BMK_result_t candidate; + int feas = 0, i; + for (i=1; i<=maxSeeds; i++) { + ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, 0); + int ec = BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, nbFiles, ctx, dctx, CParams); + BMK_printWinner(stdout, i, candidate, CParams, benchedSize); + + if(!ec) { + if(feas) { + if(feasible(candidate, relaxTarget(target)) && objective_lt(winner.result, candidate)) { + winner.result = candidate; + winner.params = CParams; + } + } else { + if(feasible(candidate, relaxTarget(target))) { + feas = 1; + winner.result = candidate; + winner.params = CParams; + + } else { + if(resultScore(candidate, benchedSize, target) > resultScore(winner.result, benchedSize, target)) { + winner.result = candidate; + winner.params = CParams; + } + } + } + } + } //best, -1, +1, ..., + + } else if (paramTarget.strategy == 0) { //constrained + int feas = 0, i, j; + for(j = 1; j < 10; j++) { + for(i = 1; i <= maxSeeds; i++) { + int varLenNew = sanitizeVarArray(varLen, varArray, varNew, i); + ZSTD_compressionParameters candidateParams = paramTarget; + BMK_result_t candidate; + int ec; + randomConstrainedParams(&candidateParams, varNew, varLenNew, allMT[i]); + cParamZeroMin(&candidateParams); + candidateParams = sanitizeParams(candidateParams); + ec = BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, nbFiles, ctx, dctx, candidateParams); + + if(!ec) { + if(feas) { + if(feasible(candidate, relaxTarget(target)) && objective_lt(winner.result, candidate)) { + winner.result = candidate; + winner.params = candidateParams; + BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); + } + } else { + if(feasible(candidate, relaxTarget(target))) { + feas = 1; + winner.result = candidate; + winner.params = candidateParams; + BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); + + } else { + if(resultScore(candidate, benchedSize, target) > resultScore(winner.result, benchedSize, target)) { + winner.result = candidate; + winner.params = candidateParams; + BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); + } + } + } + } + + } } - if (feasible(candidate,target) && objective_lt(winner.result, candidate)) - { - winner.params = CParams; - winner.result = candidate; - BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); - } } - }*/ + } + } + BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize); - BMK_translateAdvancedParams(winner.params); - - /* start real tests */ + DEBUGOUTPUT("Real Opt\n"); + /* start 'real' tests */ { + int bestStrategy = (int)winner.params.strategy; if(paramTarget.strategy == 0) { - int st; - for(st = 1; st <= 8; st++) { - winnerInfo_t wc = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, dictBuffer, dictBufferSize, - target, paramTarget, st, varArray, varLen); - DISPLAY("StratNum %d\n", st); + int st = (int)winner.params.strategy; + + { + int varLenNew = sanitizeVarArray(varLen, varArray, varNew, st); + winnerInfo_t w1 = climbOnce(target, varNew, varLenNew, allMT[st], + origBuff, benchedSize, compressedBuffer, maxCompressedSize, dictBuffer, dictBufferSize, + fileSizes, nbFiles, ctx, dctx, winner.params); + if(objective_lt(winner.result, w1.result)) { + winner = w1; + } + } + + while(st) { + winnerInfo_t wc = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, dictBuffer, dictBufferSize, fileSizes, nbFiles, + target, paramTarget, st, varArray, varLen, allMT[st]); + DEBUGOUTPUT("StratNum %d\n", st); if(objective_lt(winner.result, wc.result)) { winner = wc; } + //TODO: we could double back to increase search of 'better' strategies + st = nextStrategy(st, bestStrategy); } } else { - winner = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, dictBuffer, dictBufferSize, - target, paramTarget, paramTarget.strategy, varArray, varLen); + winner = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, dictBuffer, dictBufferSize, fileSizes, nbFiles, + target, paramTarget, paramTarget.strategy, varArray, varLen, allMT[paramTarget.strategy]); } } @@ -1934,150 +2039,18 @@ static int optimizeForSize2(const char* const * const fileNamesTable, const size BMK_translateAdvancedParams(winner.params); DISPLAY("grillParams size - optimizer completed \n"); - /* clean up*/ - ZSTD_freeCCtx(ctx); - ZSTD_freeDCtx(dctx); - } - - free(origBuff); - return 0; -} - - -/* optimizeForSize(): - * targetSpeed : expressed in B/s */ -/* expresses targeted compression, decompression speeds and memory requirements */ -/* if state space is small (from paramTarget), exhaustive search? */ -//things to consider : if doing strategy-separate approach, what cutoffs to evaluate each strategy -//or do all? can't be absolute, should be relative after some sort of calibration -//(synthetic? test levels (we don't care about data specifics rn, scale?) ? -int optimizeForSize(const char* inFileName, constraint_t target, ZSTD_compressionParameters paramTarget) -{ - FILE* const inFile = fopen( inFileName, "rb" ); - U64 const inFileSize = UTIL_getFileSize(inFileName); - size_t benchedSize = BMK_findMaxMem(inFileSize*3) / 3; - void* origBuff; - U32 paramVarArray [NUM_PARAMS]; - int paramCount = variableParams(paramTarget, paramVarArray); - /* Init */ - if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; } - if (inFileSize == UTIL_FILESIZE_UNKNOWN) { - DISPLAY("Pb evaluatin size of %s \n", inFileName); - fclose(inFile); - return 11; - } - - /* Memory allocation & restrictions */ - if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize; - if (benchedSize < inFileSize) { - DISPLAY("Not enough memory for '%s' \n", inFileName); - fclose(inFile); - return 11; - } - - /* Alloc */ - origBuff = malloc(benchedSize); - if(!origBuff) { - DISPLAY("\nError: not enough memory!\n"); - fclose(inFile); - return 12; - } - - /* Fill input buffer */ - DISPLAY("Loading %s... \r", inFileName); - { size_t const readSize = fread(origBuff, 1, benchedSize, inFile); - fclose(inFile); - if(readSize != benchedSize) { - DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); - free(origBuff); - return 13; - } } - - /* bench */ - DISPLAY("\r%79s\r", ""); - DISPLAY("optimizing for %s", inFileName); - if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed / 1000000); } - if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed / 1000000); } - if(target.cMem != 0) { DISPLAY(" - limit memory %u MB", target.cMem / 1000000); } - DISPLAY("\n"); - { ZSTD_CCtx* const ctx = ZSTD_createCCtx(); - ZSTD_DCtx* const dctx = ZSTD_createDCtx(); - winnerInfo_t winner; - BMK_result_t candidate; - const size_t blockSize = g_blockSize ? g_blockSize : benchedSize; - - /* init */ - if (ctx==NULL) { DISPLAY("\n ZSTD_createCCtx error \n"); free(origBuff); return 14; } - - memset(&winner, 0, sizeof(winner)); - winner.result.cSize = (size_t)(-1); - - /* find best solution from default params */ - //Can't do this w/ cparameter constraints - { const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); - int i; - for (i=1; i<=maxSeeds; i++) { - ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, 0); - BMK_benchParam(&candidate, origBuff, benchedSize, ctx, dctx, CParams); - if (!feasible(candidate, target) ) { - break; - } - if (feasible(candidate,target) && objective_lt(winner.result, candidate)) - { - winner.params = CParams; - winner.result = candidate; - BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); - } } - } - BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize); - - BMK_translateAdvancedParams(winner.params); - - /* start tests */ - { time_t const grillStart = time(NULL); - do { - ZSTD_compressionParameters params = winner.params; - BYTE* b; - paramVariation(¶ms, paramVarArray, paramCount, 4); - if ((FUZ_rand(&g_rand) & 31) == 3) params = randomParams(); /* totally random config to improve search space */ - params = ZSTD_adjustCParams(params, blockSize, 0); - - /* exclude faster if already played set of params */ - if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(params))-1)) continue; - - /* test */ - b = NB_TESTS_PLAYED(params); - (*b)++; - BMK_benchParam(&candidate, origBuff, benchedSize, ctx, dctx, params); - - /* improvement found => new winner */ - if (feasible(candidate,target) && objective_lt(winner.result, candidate)) - { - winner.params = params; - winner.result = candidate; - BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize); - BMK_translateAdvancedParams(winner.params); - } - } while (BMK_timeSpan(grillStart) < g_grillDuration_s); - } - - /* no solution found */ - if(winner.result.cSize == (size_t)-1) { - DISPLAY("No feasible solution found\n"); - return 1; - } - /* end summary */ - BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize); - BMK_translateAdvancedParams(winner.params); - DISPLAY("grillParams size - optimizer completed \n"); /* clean up*/ ZSTD_freeCCtx(ctx); ZSTD_freeDCtx(dctx); - } + } +_cleanUp: + free(fileSizes); + free(dictBuffer); + memoTableFreeAll(allMT); free(origBuff); - return 0; + return ret; } static void errorOut(const char* msg) @@ -2136,6 +2109,7 @@ static int usage_advanced(void) DISPLAY( " -P# : generated sample compressibility (default : %.1f%%) \n", COMPRESSIBILITY_DEFAULT * 100); DISPLAY( " -t# : Caps runtime of operation in seconds (default : %u seconds (%.1f hours)) \n", (U32)g_grillDuration_s, g_grillDuration_s / 3600); DISPLAY( " -v : Prints Benchmarking output\n"); + DISPLAY( " -D : Next argument dictionary file\n"); return 0; } @@ -2163,6 +2137,8 @@ int main(int argc, const char** argv) assert(argc>=1); /* for exename */ + g_time = UTIL_getTime(); + /* Welcome message */ DISPLAY(WELCOME_MESSAGE); @@ -2397,7 +2373,7 @@ int main(int argc, const char** argv) } } else { if (optimizer) { - result = optimizeForSize2(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget); + result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget); } else { result = benchFiles(argv+filenamesStart, argc-filenamesStart); } } From 7b5b3d7ae383f4efe53a95d68a4a1f1760f78650 Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 16 Jul 2018 16:16:31 -0700 Subject: [PATCH 06/55] BenchMem with block compressed sizes passed back up --- programs/bench.c | 31 +-- programs/bench.h | 13 +- tests/paramgrill.c | 625 +++++++++++++++++++++++++++++++++++++-------- 3 files changed, 540 insertions(+), 129 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index f5184fa8f..8e57db4c9 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -188,7 +188,7 @@ static void BMK_initCCtx(ZSTD_CCtx* ctx, ZSTD_CCtx_setParameter(ctx, ZSTD_p_searchLog, comprParams->searchLog); ZSTD_CCtx_setParameter(ctx, ZSTD_p_minMatch, comprParams->searchLength); ZSTD_CCtx_setParameter(ctx, ZSTD_p_targetLength, comprParams->targetLength); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy , comprParams->strategy); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy, comprParams->strategy); ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize); } @@ -281,8 +281,6 @@ static size_t local_defaultDecompress( } -volatile char g_touched; - /* initFn will be measured once, bench fn will be measured x times */ /* benchFn should return error value or out Size */ /* takes # of blocks and list of size & stuff for each. */ @@ -293,7 +291,7 @@ BMK_customReturn_t BMK_benchFunction( BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBlockBuffers, const size_t* srcBlockSizes, - void** const dstBlockBuffers, size_t* dstBlockCapacities, + void* const * const dstBlockBuffers, size_t* dstBlockCapacitiesToSizes, unsigned nbLoops) { size_t srcSize = 0, dstSize = 0, ind = 0; U64 totalTime; @@ -310,20 +308,13 @@ BMK_customReturn_t BMK_benchFunction( } { - - unsigned i, j; + size_t i; for(i = 0; i < blockCount; i++) { - for(j = 0; j < srcBlockSizes[i]; j++) { - g_touched = ((const char*)srcBlockBuffers[i])[j]; /* touch */ - } - } - for(i = 0; i < blockCount; i++) { - memset(dstBlockBuffers[i], 0xE5, dstBlockCapacities[i]); /* warm up and erase result buffer */ + memset(dstBlockBuffers[i], 0xE5, dstBlockCapacitiesToSizes[i]); /* warm up and erase result buffer */ } //UTIL_sleepMilli(5); /* give processor time to other processes */ //UTIL_waitForNextTick(); - } { @@ -332,17 +323,13 @@ BMK_customReturn_t BMK_benchFunction( if(initFn != NULL) { initFn(initPayload); } for(i = 0; i < nbLoops; i++) { for(j = 0; j < blockCount; j++) { - size_t res = benchFn(srcBlockBuffers[j], srcBlockSizes[j], dstBlockBuffers[j], dstBlockCapacities[j], benchPayload); + size_t res = benchFn(srcBlockBuffers[j], srcBlockSizes[j], dstBlockBuffers[j], dstBlockCapacitiesToSizes[j], benchPayload); if(ZSTD_isError(res)) { EXM_THROW_ND(2, BMK_customReturn_t, "Function benchmarking failed on block %u of size %u : %s \n", - j, (U32)dstBlockCapacities[j], ZSTD_getErrorName(res)); + j, (U32)dstBlockCapacitiesToSizes[j], ZSTD_getErrorName(res)); } else if(firstIter) { dstSize += res; - //Make compressed blocks continuous - if(j != blockCount - 1) { - dstBlockBuffers[j+1] = (void*)((char*)dstBlockBuffers[j] + res); - dstBlockCapacities[j] = res; - } + dstBlockCapacitiesToSizes[j] = res; } } firstIter = 0; @@ -382,7 +369,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed( BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const* const srcBlockBuffers, const size_t* srcBlockSizes, - void** const dstBlockBuffers, size_t* dstBlockCapacities) + void * const * const dstBlockBuffers, size_t * dstBlockCapacitiesToSizes) { U64 fastest = cont->fastestTime; int completed = 0; @@ -399,7 +386,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed( } r.result = BMK_benchFunction(benchFn, benchPayload, initFn, initPayload, - blockCount, srcBlockBuffers, srcBlockSizes, dstBlockBuffers, dstBlockCapacities, cont->nbLoops); + blockCount, srcBlockBuffers, srcBlockSizes, dstBlockBuffers, dstBlockCapacitiesToSizes, cont->nbLoops); if(r.result.error) { /* completed w/ error */ r.completed = 1; return r; diff --git a/programs/bench.h b/programs/bench.h index 1a298cc48..25d9f24a7 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -122,8 +122,6 @@ BMK_return_t BMK_syntheticTest(int cLevel, double compressibility, * (cLevel, comprParams + adv in advanced Mode) */ /* srcBuffer - data source, expected to be valid compressed data if in Decode Only Mode * srcSize - size of data in srcBuffer - * dstBuffer - destination buffer to write compressed output in, optional (NULL) - * dstCapacity - capacity of destination buffer, give 0 if dstBuffer = NULL * cLevel - compression level * comprParams - basic compression parameters * dictBuffer - a dictionary if used, null otherwise @@ -144,7 +142,10 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, int displayLevel, const char* displayName); -/* See benchMem for normal parameter uses and return, see advancedParams_t for adv */ +/* See benchMem for normal parameter uses and return, see advancedParams_t for adv + * dstBuffer - destination buffer to write compressed output in, NULL if none provided. + * dstCapacity - capacity of destination buffer, give 0 if dstBuffer = NULL + */ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, void* dstBuffer, size_t dstCapacity, const size_t* fileSizes, unsigned nbFiles, @@ -174,7 +175,7 @@ typedef size_t (*BMK_initFn_t)(void*); * srcBuffers - an array of buffers to be operated on by benchFn * srcSizes - an array of the sizes of above buffers * dstBuffers - an array of buffers to be written into by benchFn - * dstCapacities - an array of the capacities of above buffers. + * dstCapacitiesToSizes - an array of the capacities of above buffers. Output modified to compressed sizes of those blocks. * nbLoops - defines number of times benchFn is run. * return * .error will give a nonzero value if ZSTD_isError() is nonzero for any of the return @@ -191,7 +192,7 @@ BMK_customReturn_t BMK_benchFunction( BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBuffers, const size_t* srcSizes, - void** const dstBuffers, size_t* dstCapacities, + void * const * const dstBuffers, size_t* dstCapacitiesToSizes, unsigned nbLoops); @@ -220,7 +221,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed(BMK_timedFnState_t* cont, BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBlockBuffers, const size_t* srcBlockSizes, - void** const dstBlockBuffers, size_t* dstBlockCapacities); + void* const * const dstBlockBuffers, size_t* dstBlockCapacitiesToSizes); #endif /* BENCH_H_121279284357 */ diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 9af60a740..e4d468846 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -27,7 +27,8 @@ #include "xxhash.h" #include "util.h" #include "bench.h" - +#include "zstd_errors.h" +#include "zstd_internal.h" /*-************************************ * Constants @@ -36,11 +37,6 @@ #define AUTHOR "Yann Collet" #define WELCOME_MESSAGE "*** %s %s %i-bits, by %s ***\n", PROGRAM_DESCRIPTION, ZSTD_VERSION_STRING, (int)(sizeof(void*)*8), AUTHOR - -#define KB *(1<<10) -#define MB *(1<<20) -#define GB *(1ULL<<30) - #define TIMELOOP_NANOSEC (1*1000000000ULL) /* 1 second */ #define NBLOOPS 2 @@ -240,7 +236,6 @@ static int cParamValid(ZSTD_compressionParameters paramTarget) { return 1; } -//TODO: let targetLength = 0; static void cParamZeroMin(ZSTD_compressionParameters* paramTarget) { paramTarget->windowLog = paramTarget->windowLog ? paramTarget->windowLog : ZSTD_WINDOWLOG_MIN; paramTarget->searchLog = paramTarget->searchLog ? paramTarget->searchLog : ZSTD_SEARCHLOG_MIN; @@ -261,7 +256,7 @@ static int feasible(const BMK_result_t results, const constraint_t target) { return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem || !target.cMem); } -#define EPSILON 0.01 +#define EPSILON 0.001 static int epsilonEqual(const double c1, const double c2) { return MAX(c1/c2,c2/c1) < 1 + EPSILON; } @@ -274,8 +269,8 @@ static int eqZero(const double c1) { /* returns 1 if result2 is strictly 'better' than result1 */ /* strict comparison / cutoff based */ static int objective_lt(const BMK_result_t result1, const BMK_result_t result2) { - return (result1.cSize > result2.cSize) || (epsilonEqual(result1.cSize, result2.cSize) && result2.cSpeed > result1.cSpeed) - || (epsilonEqual(result1.cSize,result2.cSize) && epsilonEqual(result2.cSpeed, result1.cSpeed) && result2.dSpeed > result1.dSpeed); + return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed) + || (result1.cSize == result2.cSize && epsilonEqual(result2.cSpeed, result1.cSpeed) && result2.dSpeed > result1.dSpeed); } /* hill climbing value for part 1 */ @@ -352,9 +347,362 @@ typedef struct { ZSTD_compressionParameters params; } winnerInfo_t; +/*-******************************************************* +* From Paramgrill +*********************************************************/ + +static void BMK_initCCtx(ZSTD_CCtx* ctx, + const void* dictBuffer, size_t dictBufferSize, int cLevel, + const ZSTD_compressionParameters* comprParams, const BMK_advancedParams_t* adv) { + if (adv->nbWorkers==1) { + ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, 0); + } else { + ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, adv->nbWorkers); + } + ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_enableLongDistanceMatching, adv->ldmFlag); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmMinMatch, adv->ldmMinMatch); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashLog, adv->ldmHashLog); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmBucketSizeLog, adv->ldmBucketSizeLog); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashEveryLog, adv->ldmHashEveryLog); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_windowLog, comprParams->windowLog); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_hashLog, comprParams->hashLog); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_chainLog, comprParams->chainLog); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_searchLog, comprParams->searchLog); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_minMatch, comprParams->searchLength); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_targetLength, comprParams->targetLength); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy, comprParams->strategy); + ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize); +} + + +static void BMK_initDCtx(ZSTD_DCtx* dctx, + const void* dictBuffer, size_t dictBufferSize) { + ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize); +} + +typedef struct { + ZSTD_CCtx* ctx; + const void* dictBuffer; + size_t dictBufferSize; + int cLevel; + const ZSTD_compressionParameters* comprParams; + const BMK_advancedParams_t* adv; +} BMK_initCCtxArgs; + +static size_t local_initCCtx(void* payload) { + BMK_initCCtxArgs* ag = (BMK_initCCtxArgs*)payload; + BMK_initCCtx(ag->ctx, ag->dictBuffer, ag->dictBufferSize, ag->cLevel, ag->comprParams, ag->adv); + return 0; +} + +typedef struct { + ZSTD_DCtx* dctx; + const void* dictBuffer; + size_t dictBufferSize; +} BMK_initDCtxArgs; + +static size_t local_initDCtx(void* payload) { + BMK_initDCtxArgs* ag = (BMK_initDCtxArgs*)payload; + BMK_initDCtx(ag->dctx, ag->dictBuffer, ag->dictBufferSize); + return 0; +} + +/* additional argument is just the context */ +static size_t local_defaultCompress( + const void* srcBuffer, size_t srcSize, + void* dstBuffer, size_t dstSize, + void* addArgs) { + size_t moreToFlush = 1; + ZSTD_CCtx* ctx = (ZSTD_CCtx*)addArgs; + ZSTD_inBuffer in; + ZSTD_outBuffer out; + in.src = srcBuffer; + in.size = srcSize; + in.pos = 0; + out.dst = dstBuffer; + out.size = dstSize; + out.pos = 0; + while (moreToFlush) { + if(out.pos == out.size) { + return (size_t)-ZSTD_error_dstSize_tooSmall; + } + moreToFlush = ZSTD_compress_generic(ctx, &out, &in, ZSTD_e_end); + if (ZSTD_isError(moreToFlush)) { + return moreToFlush; + } + } + return out.pos; +} + +/* additional argument is just the context */ +static size_t local_defaultDecompress( + const void* srcBuffer, size_t srcSize, + void* dstBuffer, size_t dstSize, + void* addArgs) { + size_t moreToFlush = 1; + ZSTD_DCtx* dctx = (ZSTD_DCtx*)addArgs; + ZSTD_inBuffer in; + ZSTD_outBuffer out; + in.src = srcBuffer; + in.size = srcSize; + in.pos = 0; + out.dst = dstBuffer; + out.size = dstSize; + out.pos = 0; + while (moreToFlush) { + if(out.pos == out.size) { + return (size_t)-ZSTD_error_dstSize_tooSmall; + } + moreToFlush = ZSTD_decompress_generic(dctx, + &out, &in); + if (ZSTD_isError(moreToFlush)) { + return moreToFlush; + } + } + return out.pos; + +} + +/*-******************************************************* +* From Paramgrill End +*********************************************************/ + +/* Replicate function of benchMemAdvanced, but with pre-split src / dst buffers, with relevant info to invert it (compressedSizes) passed out. */ +/*BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); */ +/* nbSeconds used in same way as in BMK_advancedParams_t, as nbIters when in iterMode */ + +/* if in decodeOnly, then srcPtr's will be compressed blocks, and uncompressedBlocks will be written to dstPtrs? */ +/* dictionary nullable, nothing else though. */ +static BMK_return_t BMK_benchMemInvertible(const void * const * const srcPtrs, size_t const * const srcSizes, + void** dstPtrs, size_t* dstCapacityToSizes, U32 const nbBlocks, + const int cLevel, const ZSTD_compressionParameters* comprParams, + const void* dictBuffer, const size_t dictBufferSize, + ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, + const BMK_mode_t mode, const BMK_loopMode_t loopMode, const unsigned nbSeconds) { + U32 i; + BMK_return_t results = { { 0, 0., 0., 0 }, 0 } ; + size_t srcSize = 0; + void** const resPtrs = malloc(sizeof(void*) * nbBlocks); /* only really needed in both mode. */ + size_t* const resSizes = malloc(sizeof(size_t) * nbBlocks); + int freeDST = 0; + + BMK_advancedParams_t adv = BMK_initAdvancedParams(); + adv.mode = mode; + adv.loopMode = loopMode; + adv.nbSeconds = nbSeconds; + + /* resSizes == srcSizes, but modifiable */ + memcpy(resSizes, srcSizes, sizeof(size_t) * nbBlocks); + + for(i = 0; i < nbBlocks; i++) { + srcSize += srcSizes[i]; + } + + if(!ctx || !dctx || !srcPtrs || ! srcSizes) + { + results.error = 31; + DISPLAY("error: passed in null argument\n"); + free(resPtrs); + free(resSizes); + return results; + } + if(!resPtrs || !resSizes) { + results.error = 32; + DISPLAY("error: allocation failed\n"); + free(resPtrs); + free(resSizes); + return results; + } + + /* so resPtr is continuous */ + resPtrs[0] = malloc(srcSize); + + if(!(resPtrs[0])) { + results.error = 32; + DISPLAY("error: allocation failed\n"); + free(resPtrs); + free(resSizes); + return results; + } + + for(i = 1; i < nbBlocks; i++) { + resPtrs[i] = (void*)(((char*)resPtrs[i-1]) + srcSizes[i-1]); + } + + /* allocate own dst if NULL */ + if(dstPtrs == NULL) { + freeDST = 1; + dstPtrs = malloc(nbBlocks * sizeof(void*)); + dstCapacityToSizes = malloc(nbBlocks * sizeof(size_t)); + if(dstPtrs == NULL) { + results.error = 33; + DISPLAY("error: allocation failed\n"); + free(resPtrs); + free(resSizes); + return results; + } + + if(mode == BMK_decodeOnly) { //dst is src + size_t dstSize = 0; + for(i = 0; i < nbBlocks; i++) { + dstCapacityToSizes[i] = ZSTD_getDecompressedSize(srcPtrs[i], srcSizes[i]); + dstSize += dstCapacityToSizes[i]; + } + dstPtrs[0] = malloc(dstSize); + if(dstPtrs[0] == NULL) { + results.error = 34; + DISPLAY("error: allocation failed\n"); + goto _cleanUp; + } + for(i = 1; i < nbBlocks; i++) { + dstPtrs[i] = (void*)(((char*)dstPtrs[i-1]) + ZSTD_getDecompressedSize(srcPtrs[i-1], srcSizes[i-1])); + } + } else { + dstPtrs[0] = malloc(ZSTD_compressBound(srcSize) + (nbBlocks * 1024)); + if(dstPtrs[0] == NULL) { + results.error = 35; + DISPLAY("error: allocation failed\n"); + goto _cleanUp; + } + dstCapacityToSizes[0] = ZSTD_compressBound(srcSizes[0]); + for(i = 1; i < nbBlocks; i++) { + dstPtrs[i] = (void*)(((char*)dstPtrs[i-1]) + dstCapacityToSizes[i-1]); + dstCapacityToSizes[i] = ZSTD_compressBound(srcSizes[i]); + } + } + } + + /* warmimg up memory */ + for(i = 0; i < nbBlocks; i++) { + RDG_genBuffer(dstPtrs[i], dstCapacityToSizes[i], 0.10, 0.50, 1); + } + + /* Bench */ + { + { + BMK_initCCtxArgs cctxprep; + BMK_initDCtxArgs dctxprep; + cctxprep.ctx = ctx; + cctxprep.dictBuffer = dictBuffer; + cctxprep.dictBufferSize = dictBufferSize; + cctxprep.cLevel = cLevel; + cctxprep.comprParams = comprParams; + cctxprep.adv = &adv; + dctxprep.dctx = dctx; + dctxprep.dictBuffer = dictBuffer; + dctxprep.dictBufferSize = dictBufferSize; + if(loopMode == BMK_timeMode) { + BMK_customTimedReturn_t intermediateResultCompress; + BMK_customTimedReturn_t intermediateResultDecompress; + BMK_timedFnState_t* timeStateCompress = BMK_createTimeState(nbSeconds); + BMK_timedFnState_t* timeStateDecompress = BMK_createTimeState(nbSeconds); + if(mode == BMK_compressOnly) { + intermediateResultCompress.completed = 0; + intermediateResultDecompress.completed = 1; + } else if (mode == BMK_decodeOnly) { + intermediateResultCompress.completed = 1; + intermediateResultDecompress.completed = 0; + } else { /* both */ + intermediateResultCompress.completed = 0; + intermediateResultDecompress.completed = 0; + } + while(!(intermediateResultCompress.completed && intermediateResultDecompress.completed)) { + if(!intermediateResultCompress.completed) { + intermediateResultCompress = BMK_benchFunctionTimed(timeStateCompress, &local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep, + nbBlocks, srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes); + if(intermediateResultCompress.result.error) { + results.error = intermediateResultCompress.result.error; + BMK_freeTimeState(timeStateCompress); + BMK_freeTimeState(timeStateDecompress); + goto _cleanUp; + } + results.result.cSpeed = ((double)srcSize / intermediateResultCompress.result.result.nanoSecPerRun) * 1000000000; + results.result.cSize = intermediateResultCompress.result.result.sumOfReturn; + } + + if(!intermediateResultDecompress.completed) { + if(mode == BMK_decodeOnly) { + intermediateResultDecompress = BMK_benchFunctionTimed(timeStateDecompress, &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep, + nbBlocks, (const void* const*)srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes); + } else { /* both, decompressed result already written to dstPtr */ + intermediateResultDecompress = BMK_benchFunctionTimed(timeStateDecompress, &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep, + nbBlocks, (const void* const*)dstPtrs, dstCapacityToSizes, resPtrs, resSizes); + } + + if(intermediateResultDecompress.result.error) { + results.error = intermediateResultDecompress.result.error; + BMK_freeTimeState(timeStateCompress); + BMK_freeTimeState(timeStateDecompress); + goto _cleanUp; + } + results.result.dSpeed = ((double)srcSize / intermediateResultDecompress.result.result.nanoSecPerRun) * 1000000000; + } + } + BMK_freeTimeState(timeStateCompress); + BMK_freeTimeState(timeStateDecompress); + } else { //iterMode; + if(mode != BMK_decodeOnly) { + + BMK_customReturn_t compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep, + nbBlocks, srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbSeconds); + if(compressionResults.error) { + results.error = compressionResults.error; + goto _cleanUp; + } + if(compressionResults.result.nanoSecPerRun == 0) { + results.result.cSpeed = 0; + } else { + results.result.cSpeed = (double)srcSize / compressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; + } + results.result.cSize = compressionResults.result.sumOfReturn; + } + if(mode != BMK_compressOnly) { + BMK_customReturn_t decompressionResults; + if(mode == BMK_decodeOnly) { + decompressionResults = BMK_benchFunction( + &local_defaultDecompress, (void*)(dctx), + &local_initDCtx, (void*)&dctxprep, nbBlocks, + (const void* const*)srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, + nbSeconds); + } else { + decompressionResults = BMK_benchFunction( + &local_defaultDecompress, (void*)(dctx), + &local_initDCtx, (void*)&dctxprep, nbBlocks, + (const void* const*)dstPtrs, dstCapacityToSizes, resPtrs, resSizes, + nbSeconds); + } + + if(decompressionResults.error) { + results.error = decompressionResults.error; + goto _cleanUp; + } + + if(decompressionResults.result.nanoSecPerRun == 0) { + results.result.dSpeed = 0; + } else { + results.result.dSpeed = (double)srcSize / decompressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; + } + } + } + } + } /* Bench */ + results.result.cMem = (1 << (comprParams->windowLog)) + ZSTD_sizeof_CCtx(ctx); + +_cleanUp: + free(resPtrs[0]); + free(resPtrs); + free(resSizes); + if(freeDST) { + free(dstPtrs[0]); + free(dstPtrs); + } + return results; +} + /* global winner used for display. */ //Should be totally 0 initialized? -static winnerInfo_t g_winner; //TODO: ratio is infinite at initialization, instead of 0 +static winnerInfo_t g_winner; static constraint_t g_targetConstraints; static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const size_t srcSize) @@ -379,7 +727,7 @@ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result "/* %s */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */", lvlstr, (double)srcSize / result.cSize, result.cSpeed / 1000000., result.dSpeed / 1000000.); - if(TIMED) { fprintf(f, " - %lu:%lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); } + if(TIMED) { fprintf(f, " - %1lu:%2lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); } fprintf(f, "\n"); if(objective_lt(g_winner.result, result) && feasible(result, g_targetConstraints)) { BMK_translateAdvancedParams(params); @@ -670,17 +1018,10 @@ static size_t memoTableLen(const U32* varyParams, const int varyLen) { //sort of ~lg2 (replace 1024 w/ 999, and add 0 at lower end of range) for memoTableInd Tlen static unsigned lg2(unsigned x) { - unsigned j = 1; if(x == 999) { return 11; } - if(!x) { - return 0; - } - while(x >>= 1) { - j++; - } - return j; + return x ? ZSTD_highbit32(x) + 1 : 0; } /* returns unique index of compression parameters */ @@ -729,7 +1070,7 @@ static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstra for(i = 0; i < arrayLen; i++) { memoTableIndInv(¶mConstraints, varyParams, varyLen, i); - if((ZSTD_estimateCCtxSize_usingCParams(paramConstraints) + (1ULL << paramConstraints.windowLog)) > (size_t)target.cMem + (size_t)(target.cMem / 10)) { + if(ZSTD_estimateCStreamSize_usingCParams(paramConstraints) > (size_t)target.cMem) { memoTable[i] = 255; j++; } @@ -765,8 +1106,6 @@ static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstra /* inits memotables for all (including mallocs), all strategies */ /* takes unsanitized varyParams */ - -//TODO: check for errors/nulls static U8** memoTableInitAll(ZSTD_compressionParameters paramConstraints, constraint_t target, const U32* varyParams, const int varyLen, const size_t srcSize) { U32 varNew[NUM_PARAMS]; int varLenNew; @@ -1099,28 +1438,29 @@ static int uncertainComparison(double const uncertaintyConstantC, double const u #define FEASIBLE_RESULT 1 #define ERROR_RESULT 2 static int feasibleBench(BMK_result_t* resultPtr, - const void* srcBuffer, const size_t srcSize, - void* dstBuffer, const size_t dstSize, - void* dictBuffer, const size_t dictSize, - const size_t* fileSizes, const size_t nbFiles, + const void* const * const srcPtrs, size_t const * const srcSizes, + void** const dstPtrs, size_t* dstCapacityToSizes, U32 const nbBlocks, + void* dictBuffer, const size_t dictBufferSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams, const constraint_t target, BMK_result_t* winnerResult) { - BMK_advancedParams_t adv = BMK_initAdvancedParams(); BMK_return_t benchres; U64 loopDurationC = 0, loopDurationD = 0; double uncertaintyConstantC, uncertaintyConstantD; - adv.loopMode = BMK_iterMode; - adv.nbSeconds = 1; //get ratio and 2x approx speed? - + size_t srcSize = 0; + U32 i; //alternative - test 1 iter for ratio, (possibility of error 3 which is fine), //maybe iter this until 2x measurable for better guarantee? DEBUGOUTPUT("Feas:\n"); - benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + benchres = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_both, BMK_iterMode, 1); if(benchres.error) { DISPLAY("ERROR %d !!\n", benchres.error); } + for(i = 0; i < nbBlocks; i++) { + srcSize += srcSizes[i]; + } BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); if(!benchres.error) { @@ -1151,9 +1491,8 @@ static int feasibleBench(BMK_result_t* resultPtr, if(benchres.result.cSize < winnerResult->cSize) { //better compression ratio, just needs to be feasible int feas; if(loopDurationC < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2; - adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_compressOnly, BMK_iterMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1161,9 +1500,8 @@ static int feasibleBench(BMK_result_t* resultPtr, } } if(loopDurationD < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2; - adv.mode = BMK_decodeOnly; - benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_decodeOnly, BMK_iterMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1174,11 +1512,9 @@ static int feasibleBench(BMK_result_t* resultPtr, feas = uncertainFeasibility(uncertaintyConstantC, uncertaintyConstantD, target, &(benchres.result)); if(feas == 0) { // uncertain feasibility - adv.loopMode = BMK_timeMode; if(loopDurationC < TIMELOOP_NANOSEC) { - BMK_return_t benchres2; - adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_compressOnly, BMK_timeMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1186,9 +1522,8 @@ static int feasibleBench(BMK_result_t* resultPtr, } } if(loopDurationD < TIMELOOP_NANOSEC) { - BMK_return_t benchres2; - adv.mode = BMK_decodeOnly; - benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_decodeOnly, BMK_timeMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1203,9 +1538,8 @@ static int feasibleBench(BMK_result_t* resultPtr, } else if (benchres.result.cSize == winnerResult->cSize) { //equal ratio, needs to be better than winner in cSpeed/ dSpeed / cMem int feas; if(loopDurationC < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2; - adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_compressOnly, BMK_iterMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1213,9 +1547,8 @@ static int feasibleBench(BMK_result_t* resultPtr, } } if(loopDurationD < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2; - adv.mode = BMK_decodeOnly; - benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_decodeOnly, BMK_iterMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1224,11 +1557,9 @@ static int feasibleBench(BMK_result_t* resultPtr, } feas = uncertainFeasibility(uncertaintyConstantC, uncertaintyConstantD, target, &(benchres.result)); if(feas == 0) { // uncertain feasibility - adv.loopMode = BMK_timeMode; if(loopDurationC < TIMELOOP_NANOSEC) { - BMK_return_t benchres2; - adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_compressOnly, BMK_timeMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1236,9 +1567,8 @@ static int feasibleBench(BMK_result_t* resultPtr, } } if(loopDurationD < TIMELOOP_NANOSEC) { - BMK_return_t benchres2; - adv.mode = BMK_decodeOnly; - benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_decodeOnly, BMK_timeMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1253,8 +1583,8 @@ static int feasibleBench(BMK_result_t* resultPtr, if(btw == -1) { return INFEASIBLE_RESULT; } else { //possibly better, benchmark and find out - adv.loopMode = BMK_timeMode; - benchres = BMK_benchMemAdvanced(srcBuffer, srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + benchres = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_both, BMK_timeMode, 1); *resultPtr = benchres.result; return objective_lt(*winnerResult, benchres.result); } @@ -1274,28 +1604,31 @@ static int feasibleBench(BMK_result_t* resultPtr, //have version of benchMemAdvanced which takes in dstBuffer/cap as well? //(motivation: repeat tests (maybe just on decompress) don't need further compress runs) static int infeasibleBench(BMK_result_t* resultPtr, - const void* srcBuffer, const size_t srcSize, - void* dstBuffer, const size_t dstSize, - void* dictBuffer, const size_t dictSize, - const size_t* fileSizes, const size_t nbFiles, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, - const ZSTD_compressionParameters cParams, - const constraint_t target, - BMK_result_t* winnerResult) { - BMK_advancedParams_t adv = BMK_initAdvancedParams(); + const void* const * const srcPtrs, size_t const * const srcSizes, + void** const dstPtrs, size_t* dstCapacityToSizes, U32 const nbBlocks, + void* dictBuffer, const size_t dictBufferSize, + ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, + const ZSTD_compressionParameters cParams, + const constraint_t target, + BMK_result_t* winnerResult) { BMK_return_t benchres; BMK_result_t resultMin, resultMax; U64 loopDurationC = 0, loopDurationD = 0; double uncertaintyConstantC, uncertaintyConstantD; - double winnerRS = resultScore(*winnerResult, srcSize, target); - adv.loopMode = BMK_iterMode; //can only use this for ratio measurement then, super inaccurate timing - adv.nbSeconds = 1; //get ratio and 2x approx speed? //maybe run until twice MIN(minloopinterval * clockDuration) + double winnerRS; + size_t srcSize = 0; + U32 i; + + benchres = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_both, BMK_iterMode, 1); + for(i = 0; i < nbBlocks; i++) { + srcSize += srcSizes[i]; + } + BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); + winnerRS = resultScore(*winnerResult, srcSize, target); DEBUGOUTPUT("WinnerScore: %f\n ", winnerRS); - benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); - BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); - if(!benchres.error) { *resultPtr = benchres.result; if(eqZero(benchres.result.cSpeed)) { @@ -1316,9 +1649,8 @@ static int infeasibleBench(BMK_result_t* resultPtr, if(loopDurationC < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2; - adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_compressOnly, BMK_iterMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1326,9 +1658,8 @@ static int infeasibleBench(BMK_result_t* resultPtr, } } if(loopDurationD < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2; - adv.mode = BMK_decodeOnly; - benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_decodeOnly, BMK_iterMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1347,12 +1678,9 @@ static int infeasibleBench(BMK_result_t* resultPtr, if (winnerRS > resultScore(resultMax, srcSize, target)) { return INFEASIBLE_RESULT; } else { - //do this w/o copying / stuff - adv.loopMode = BMK_timeMode; if(loopDurationC < TIMELOOP_NANOSEC) { - BMK_return_t benchres2; - adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_compressOnly, BMK_timeMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1360,9 +1688,8 @@ static int infeasibleBench(BMK_result_t* resultPtr, } } if(loopDurationD < TIMELOOP_NANOSEC) { - BMK_return_t benchres2; - adv.mode = BMK_decodeOnly; - benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_decodeOnly, BMK_timeMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1394,16 +1721,62 @@ static int feasibleBenchMemo(BMK_result_t* resultPtr, const U32* varyParams, const int varyLen) { const size_t memind = memoTableInd(&cParams, varyParams, varyLen); - if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { - return INFEASIBLE_RESULT; //probably pick a different code for already tested? - //maybe remove this if we incorporate nonrandom location picking? - //what is the intended behavior in this case? - //ignore? stop iterating completely? other? + return INFEASIBLE_RESULT; } else { - int res = feasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, fileSizes, nbFiles, ctx, dctx, + const size_t blockSize = g_blockSize ? g_blockSize : srcSize; + U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; + const void ** const srcPtrs = (const void** const)malloc(maxNbBlocks * sizeof(void*)); + size_t* const srcSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); + void ** const dstPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); + size_t* const dstCapacities = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); + U32 nbBlocks; + int res; + + if(!srcPtrs || !srcSizes || !dstPtrs || !dstCapacities) { + free(srcPtrs); + free(srcSizes); + free(dstPtrs); + free(dstCapacities); + DISPLAY("Allocation Error\n"); + return ERROR_RESULT; + } + + { + const char* srcPtr = (const char*)srcBuffer; + char* dstPtr = (char*)dstBuffer; + size_t dstSizeRemaining = dstSize; + U32 fileNb; + for (nbBlocks=0, fileNb=0; fileNb dstSizeRemaining) { + DEBUGOUTPUT("Warning: dstSize too small to benchmark completely \n"); + remaining = dstSizeRemaining; + dstSizeRemaining = 0; + } else { + dstSizeRemaining -= remaining; + } + for ( ; nbBlocks= INFEASIBLE_THRESHOLD) { return INFEASIBLE_RESULT; //see feasibleBenchMemo for concerns } else { - int res = infeasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, fileSizes, nbFiles, ctx, dctx, + const size_t blockSize = g_blockSize ? g_blockSize : srcSize; + U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; + const void ** const srcPtrs = (const void** const)malloc(maxNbBlocks * sizeof(void*)); + size_t* const srcSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); + void ** const dstPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); + size_t* const dstCapacities = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); + U32 nbBlocks; + int res; + + if(!srcPtrs || !srcSizes || !dstPtrs || !dstCapacities) { + free(srcPtrs); + free(srcSizes); + free(dstPtrs); + free(dstCapacities); + DISPLAY("Allocation Error\n"); + return ERROR_RESULT; + } + + { + const char* srcPtr = (const char*)srcBuffer; + char* dstPtr = (char*)dstBuffer; + size_t dstSizeRemaining = dstSize; + U32 fileNb; + for (nbBlocks=0, fileNb=0; fileNb dstSizeRemaining) { + DEBUGOUTPUT("Warning: dstSize too small to benchmark completely \n"); + remaining = dstSizeRemaining; + dstSizeRemaining = 0; + } else { + dstSizeRemaining -= remaining; + } + for ( ; nbBlocks candidate before restart } } + return winnerInfo; } @@ -1711,7 +2135,7 @@ static winnerInfo_t optimizeFixedStrategy( } while(i < 10) { //make i adjustable (user input?) depending on how much time they have. - DISPLAY("Restart\n"); //TODO: make better printing across restarts + DEBUGOUTPUT("Restart\n"); //look into improving this to maximize distance from searched infeasible stuff / towards promising regions? randomConstrainedParams(&init, varNew, varLenNew, memoTable); candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, fileSizes, nbFiles, ctx, dctx, init); @@ -1918,7 +2342,6 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ /* find best solution from default params */ { /* strategy selection */ - //TODO: don't factor memory into strategy selection in constraint_t const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); DEBUGOUTPUT("Strategy Selection\n"); if(varLen == NUM_PARAMS && paramTarget.strategy == 0) { /* no variable based constraints */ @@ -2019,7 +2442,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ if(objective_lt(winner.result, wc.result)) { winner = wc; } - //TODO: we could double back to increase search of 'better' strategies + //We could double back to increase search of 'better' strategies st = nextStrategy(st, bestStrategy); } } else { From 0f91b039ff934c4fd42d292a90ea62b52cb2d696 Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 16 Jul 2018 18:04:57 -0700 Subject: [PATCH 07/55] Add Levels --- tests/paramgrill.c | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) mode change 100644 => 100755 tests/paramgrill.c diff --git a/tests/paramgrill.c b/tests/paramgrill.c old mode 100644 new mode 100755 index e4d468846..086931b3d --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -56,7 +56,7 @@ static const int g_maxNbVariations = 64; #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define TIMED 0 #ifndef DEBUG -# define DEBUG 0 +# define DEBUG 1 #endif #define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); } @@ -2227,7 +2227,7 @@ static int nextStrategy(const int currentStrategy, const int bestStrategy) { } //optimize fixed strategy. -static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget) +static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel) { size_t benchedSize; void* origBuff = NULL; @@ -2238,7 +2238,6 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ size_t* fileSizes = calloc(sizeof(size_t),nbFiles); const int varLen = variableParams(paramTarget, varArray); U8** allMT = NULL; - g_targetConstraints = target; g_winner.result.cSize = (size_t)-1; /* Init */ if(!cParamValid(paramTarget)) { @@ -2310,6 +2309,30 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ goto _cleanUp; } + //TODO: cLevel Stuff. + if(cLevel) { + BMK_result_t candidate; + const size_t blockSize = g_blockSize ? g_blockSize : benchedSize; + ZSTD_CCtx* const ctx = ZSTD_createCCtx(); + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + ZSTD_compressionParameters const CParams = ZSTD_getCParams(cLevel, blockSize, dictBufferSize); + if(BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, nbFiles, ctx, dctx, CParams)) { + ZSTD_freeCCtx(ctx); + ZSTD_freeDCtx(dctx); + ret = 3; + goto _cleanUp; + } + + target.cSpeed = candidate.cSpeed; //TODO: maybe have a small bit of slack here, like x.99? + target.dSpeed = candidate.dSpeed; + BMK_printWinner(stdout, cLevel, candidate, CParams, benchedSize); + + ZSTD_freeCCtx(ctx); + ZSTD_freeDCtx(dctx); + } + + g_targetConstraints = target; + /* bench */ DISPLAY("\r%79s\r", ""); if(nbFiles == 1) { @@ -2348,7 +2371,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ BMK_result_t candidate; int feas = 0, i; for (i=1; i<=maxSeeds; i++) { - ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, 0); + ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, dictBufferSize); int ec = BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, nbFiles, ctx, dctx, CParams); BMK_printWinner(stdout, i, candidate, CParams, benchedSize); @@ -2553,6 +2576,7 @@ int main(int argc, const char** argv) const char* dictFileName = 0; U32 optimizer = 0; U32 main_pause = 0; + int optimizerCLevel = 0; constraint_t target = { 0, 0, (U32)-1 }; //0 for anything unset @@ -2584,6 +2608,7 @@ int main(int argc, const char** argv) if (longCommandWArg(&argument, "compressionSpeed=") || longCommandWArg(&argument, "cSpeed=")) { target.cSpeed = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } if (longCommandWArg(&argument, "decompressionSpeed=") || longCommandWArg(&argument, "dSpeed=")) { target.dSpeed = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } if (longCommandWArg(&argument, "compressionMemory=") || longCommandWArg(&argument, "cMem=")) { target.cMem = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } + //TODO: add Level; /* in MB or MB/s */ DISPLAY("invalid optimization parameter \n"); return 1; @@ -2692,6 +2717,10 @@ int main(int argc, const char** argv) argument++; paramTarget.strategy = (ZSTD_strategy)readU32FromChar(&argument); continue; + case 'L': /* level centers around a level */ + argument++; + optimizerCLevel = (int)readU32FromChar(&argument); + continue; default : ; } break; @@ -2796,7 +2825,7 @@ int main(int argc, const char** argv) } } else { if (optimizer) { - result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget); + result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget, optimizerCLevel); } else { result = benchFiles(argv+filenamesStart, argc-filenamesStart); } } From df026e159f7e5ff57a0caa5817176b00c059bc49 Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 16 Jul 2018 18:22:04 -0700 Subject: [PATCH 08/55] Fix windows implicit casting bugs --- programs/bench.c | 4 ++-- tests/paramgrill.c | 40 ++++++++++++++++++++-------------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 8e57db4c9..a34b74cf1 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -696,7 +696,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( } DISPLAYLEVEL(2, "%2i#\n", cLevel); } /* Bench */ - results.result.cMem = (1 << (comprParams->windowLog)) + ZSTD_sizeof_CCtx(ctx); + results.result.cMem = (1ULL << (comprParams->windowLog)) + ZSTD_sizeof_CCtx(ctx); results.error = 0; return results; } @@ -731,7 +731,7 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, void* resultBuffer = malloc(srcSize); - BMK_return_t results; + BMK_return_t results = { { 0, 0, 0, 0 }, 0 }; int allocationincomplete; if(!dstCapacity) { diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 086931b3d..02811b9c1 100755 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -263,7 +263,9 @@ static int epsilonEqual(const double c1, const double c2) { /* checks exact equivalence to 0, to stop compiler complaining fpeq */ static int eqZero(const double c1) { - return (U64)c1 == (U64)0.0 || (U64)c1 == (U64)-0.0; + const double z1 = 0.0; + const double z2 = -0.0; + return !(memcmp(&c1, &z1, sizeof(double))) || !(memcmp(&c1, &z2, sizeof(double))); } /* returns 1 if result2 is strictly 'better' than result1 */ @@ -318,7 +320,7 @@ const char* g_stratName[ZSTD_btultra+1] = { "ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ", "ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "}; -static size_t +static int BMK_benchParam(BMK_result_t* resultPtr, const void* srcBuffer, const size_t srcSize, const size_t* fileSizes, const unsigned nbFiles, @@ -331,7 +333,7 @@ BMK_benchParam(BMK_result_t* resultPtr, } /* benchParam but only takes in one file. */ -static size_t +static int BMK_benchParam1(BMK_result_t* resultPtr, const void* srcBuffer, size_t srcSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, @@ -1204,7 +1206,7 @@ static ZSTD_compressionParameters randomParams(void) /* Sets pc to random unmeasured set of parameters */ static void randomConstrainedParams(ZSTD_compressionParameters* pc, U32* varArray, int varLen, U8* memoTable) { - int tries = memoTableLen(varArray, varLen); //configurable, + size_t tries = memoTableLen(varArray, varLen); //configurable, const size_t maxSize = memoTableLen(varArray, varLen); size_t ind; do { @@ -1470,7 +1472,7 @@ static int feasibleBench(BMK_result_t* resultPtr, loopDurationC = 0; uncertaintyConstantC = 2; } else { - loopDurationC = ((srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); + loopDurationC = (U64)((double)(srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? //possibly has to do with initCCtx? or system stuff? //asymmetric +/- constant needed? @@ -1480,7 +1482,7 @@ static int feasibleBench(BMK_result_t* resultPtr, loopDurationD = 0; uncertaintyConstantD = 2; } else { - loopDurationD = ((srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); + loopDurationD = (U64)((double)(srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? //possibly has to do with initCCtx? or system stuff? //asymmetric +/- constant needed? @@ -1635,7 +1637,7 @@ static int infeasibleBench(BMK_result_t* resultPtr, loopDurationC = 0; uncertaintyConstantC = 2; } else { - loopDurationC = ((srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); + loopDurationC = (U64)((double)(srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); uncertaintyConstantC = MIN((loopDurationC + (double)(2 * g_clockGranularity)/loopDurationC * 1.1), 3); //.02 seconds } @@ -1643,7 +1645,7 @@ static int infeasibleBench(BMK_result_t* resultPtr, loopDurationD = 0; uncertaintyConstantD = 2; } else { - loopDurationD = ((srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); + loopDurationD = (U64)((double)(srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); uncertaintyConstantD = MIN((loopDurationD + (double)(2 * g_clockGranularity)/loopDurationD) * 1.1 , 3); //.02 seconds } @@ -1725,7 +1727,7 @@ static int feasibleBenchMemo(BMK_result_t* resultPtr, return INFEASIBLE_RESULT; } else { const size_t blockSize = g_blockSize ? g_blockSize : srcSize; - U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; + U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + (U32)nbFiles; const void ** const srcPtrs = (const void** const)malloc(maxNbBlocks * sizeof(void*)); size_t* const srcSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); void ** const dstPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); @@ -1798,7 +1800,7 @@ static int infeasibleBenchMemo(BMK_result_t* resultPtr, return INFEASIBLE_RESULT; //see feasibleBenchMemo for concerns } else { const size_t blockSize = g_blockSize ? g_blockSize : srcSize; - U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; + U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + (U32)nbFiles; const void ** const srcPtrs = (const void** const)malloc(maxNbBlocks * sizeof(void*)); size_t* const srcSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); void ** const dstPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); @@ -2279,7 +2281,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } { - U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles); + U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles); int ec; unsigned i; benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3; @@ -2290,7 +2292,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ ret = 1; goto _cleanUp; } - ec = BMK_loadFiles(origBuff, benchedSize, fileSizes, fileNamesTable, nbFiles); + ec = BMK_loadFiles(origBuff, benchedSize, fileSizes, fileNamesTable, (U32)nbFiles); if(ec) { DISPLAY("Error Loading Files"); ret = ec; @@ -2308,23 +2310,21 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ ret = 2; goto _cleanUp; } - - //TODO: cLevel Stuff. + if(cLevel) { BMK_result_t candidate; const size_t blockSize = g_blockSize ? g_blockSize : benchedSize; ZSTD_CCtx* const ctx = ZSTD_createCCtx(); ZSTD_DCtx* const dctx = ZSTD_createDCtx(); ZSTD_compressionParameters const CParams = ZSTD_getCParams(cLevel, blockSize, dictBufferSize); - if(BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, nbFiles, ctx, dctx, CParams)) { + if(BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, (U32)nbFiles, ctx, dctx, CParams)) { ZSTD_freeCCtx(ctx); ZSTD_freeDCtx(dctx); ret = 3; goto _cleanUp; } - target.cSpeed = candidate.cSpeed; //TODO: maybe have a small bit of slack here, like x.99? - target.dSpeed = candidate.dSpeed; + target.cSpeed = (U32)candidate.cSpeed; //Maybe have a small bit of slack here, like x.99? BMK_printWinner(stdout, cLevel, candidate, CParams, benchedSize); ZSTD_freeCCtx(ctx); @@ -2372,7 +2372,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ int feas = 0, i; for (i=1; i<=maxSeeds; i++) { ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, dictBufferSize); - int ec = BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, nbFiles, ctx, dctx, CParams); + int ec = BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, (U32)nbFiles, ctx, dctx, CParams); BMK_printWinner(stdout, i, candidate, CParams, benchedSize); if(!ec) { @@ -2408,7 +2408,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ randomConstrainedParams(&candidateParams, varNew, varLenNew, allMT[i]); cParamZeroMin(&candidateParams); candidateParams = sanitizeParams(candidateParams); - ec = BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, nbFiles, ctx, dctx, candidateParams); + ec = BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, (U32)nbFiles, ctx, dctx, candidateParams); if(!ec) { if(feas) { @@ -2608,7 +2608,7 @@ int main(int argc, const char** argv) if (longCommandWArg(&argument, "compressionSpeed=") || longCommandWArg(&argument, "cSpeed=")) { target.cSpeed = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } if (longCommandWArg(&argument, "decompressionSpeed=") || longCommandWArg(&argument, "dSpeed=")) { target.dSpeed = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } if (longCommandWArg(&argument, "compressionMemory=") || longCommandWArg(&argument, "cMem=")) { target.cMem = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } - //TODO: add Level; + if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { optimizerCLevel = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } /* in MB or MB/s */ DISPLAY("invalid optimization parameter \n"); return 1; From e148db366ea424bbc2dc7269c566adb97cc11ec6 Mon Sep 17 00:00:00 2001 From: George Lu Date: Fri, 20 Jul 2018 14:35:09 -0700 Subject: [PATCH 09/55] Separate capacity vs size Also: Make suggested fixes -varInds_t -reorder some arguments -remove code duplication -update README / -h -Fix memory leaks --- programs/bench.c | 63 +- programs/bench.h | 8 +- tests/README.md | 32 + tests/fullbench.c | 2 +- tests/paramgrill.c | 1963 ++++++++++++++++---------------------------- 5 files changed, 779 insertions(+), 1289 deletions(-) mode change 100755 => 100644 tests/paramgrill.c diff --git a/programs/bench.c b/programs/bench.c index a34b74cf1..177dbe0e3 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -171,6 +171,8 @@ struct BMK_timeState_t{ static void BMK_initCCtx(ZSTD_CCtx* ctx, const void* dictBuffer, size_t dictBufferSize, int cLevel, const ZSTD_compressionParameters* comprParams, const BMK_advancedParams_t* adv) { + ZSTD_CCtx_reset(ctx); + ZSTD_CCtx_resetParameters(ctx); if (adv->nbWorkers==1) { ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, 0); } else { @@ -195,6 +197,7 @@ static void BMK_initCCtx(ZSTD_CCtx* ctx, static void BMK_initDCtx(ZSTD_DCtx* dctx, const void* dictBuffer, size_t dictBufferSize) { + ZSTD_DCtx_reset(dctx); ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize); } @@ -291,9 +294,9 @@ BMK_customReturn_t BMK_benchFunction( BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBlockBuffers, const size_t* srcBlockSizes, - void* const * const dstBlockBuffers, size_t* dstBlockCapacitiesToSizes, + void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, size_t* cSizes, unsigned nbLoops) { - size_t srcSize = 0, dstSize = 0, ind = 0; + size_t dstSize = 0; U64 totalTime; BMK_customReturn_t retval; @@ -303,36 +306,37 @@ BMK_customReturn_t BMK_benchFunction( EXM_THROW_ND(1, BMK_customReturn_t, "nbLoops must be nonzero \n"); } - for(ind = 0; ind < blockCount; ind++) { - srcSize += srcBlockSizes[ind]; - } - { size_t i; for(i = 0; i < blockCount; i++) { - memset(dstBlockBuffers[i], 0xE5, dstBlockCapacitiesToSizes[i]); /* warm up and erase result buffer */ + memset(dstBlockBuffers[i], 0xE5, dstBlockCapacities[i]); /* warm up and erase result buffer */ } - - //UTIL_sleepMilli(5); /* give processor time to other processes */ - //UTIL_waitForNextTick(); +#if 0 + /* based on testing these seem to lower accuracy of multiple calls of 1 nbLoops vs 1 call of multiple nbLoops + * (Makes former slower) + */ + UTIL_sleepMilli(5); /* give processor time to other processes */ + UTIL_waitForNextTick(); +#endif } { - unsigned i, j, firstIter = 1; + unsigned i, j; clockStart = UTIL_getTime(); if(initFn != NULL) { initFn(initPayload); } for(i = 0; i < nbLoops; i++) { for(j = 0; j < blockCount; j++) { - size_t res = benchFn(srcBlockBuffers[j], srcBlockSizes[j], dstBlockBuffers[j], dstBlockCapacitiesToSizes[j], benchPayload); + size_t res = benchFn(srcBlockBuffers[j], srcBlockSizes[j], dstBlockBuffers[j], dstBlockCapacities[j], benchPayload); if(ZSTD_isError(res)) { EXM_THROW_ND(2, BMK_customReturn_t, "Function benchmarking failed on block %u of size %u : %s \n", - j, (U32)dstBlockCapacitiesToSizes[j], ZSTD_getErrorName(res)); - } else if(firstIter) { + j, (U32)dstBlockCapacities[j], ZSTD_getErrorName(res)); + } else if(i == nbLoops - 1) { dstSize += res; - dstBlockCapacitiesToSizes[j] = res; + if(cSizes != NULL) { + cSizes[j] = res; + } } } - firstIter = 0; } totalTime = UTIL_clockSpanNano(clockStart); } @@ -369,7 +373,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed( BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const* const srcBlockBuffers, const size_t* srcBlockSizes, - void * const * const dstBlockBuffers, size_t * dstBlockCapacitiesToSizes) + void * const * const dstBlockBuffers, const size_t * dstBlockCapacities, size_t* dstSizes) { U64 fastest = cont->fastestTime; int completed = 0; @@ -384,9 +388,9 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed( UTIL_sleep(COOLPERIOD_SEC); cont->coolTime = UTIL_getTime(); } - + /* reinitialize capacity */ r.result = BMK_benchFunction(benchFn, benchPayload, initFn, initPayload, - blockCount, srcBlockBuffers, srcBlockSizes, dstBlockBuffers, dstBlockCapacitiesToSizes, cont->nbLoops); + blockCount, srcBlockBuffers, srcBlockSizes, dstBlockBuffers, dstBlockCapacities, dstSizes, cont->nbLoops); if(r.result.error) { /* completed w/ error */ r.completed = 1; return r; @@ -420,7 +424,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed( /* benchMem with no allocation */ static BMK_return_t BMK_benchMemAdvancedNoAlloc( const void ** const srcPtrs, size_t* const srcSizes, - void** const cPtrs, size_t* const cSizes, + void** const cPtrs, size_t* const cCapacities, size_t* const cSizes, void** const resPtrs, size_t* const resSizes, void** resultBufferPtr, void* compressedBuffer, const size_t maxCompressedSize, @@ -485,11 +489,11 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( srcPtrs[nbBlocks] = (const void*)srcPtr; srcSizes[nbBlocks] = thisBlockSize; cPtrs[nbBlocks] = (void*)cPtr; - cSizes[nbBlocks] = (adv->mode == BMK_decodeOnly) ? thisBlockSize : ZSTD_compressBound(thisBlockSize); + cCapacities[nbBlocks] = (adv->mode == BMK_decodeOnly) ? thisBlockSize : ZSTD_compressBound(thisBlockSize); resPtrs[nbBlocks] = (void*)resPtr; resSizes[nbBlocks] = (adv->mode == BMK_decodeOnly) ? (size_t) ZSTD_findDecompressedSize(srcPtr, thisBlockSize) : thisBlockSize; srcPtr += thisBlockSize; - cPtr += cSizes[nbBlocks]; + cPtr += cCapacities[nbBlocks]; resPtr += thisBlockSize; remaining -= thisBlockSize; } @@ -540,7 +544,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( while(!(intermediateResultCompress.completed && intermediateResultDecompress.completed)) { if(!intermediateResultCompress.completed) { intermediateResultCompress = BMK_benchFunctionTimed(timeStateCompress, &local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep, - nbBlocks, srcPtrs, srcSizes, cPtrs, cSizes); + nbBlocks, srcPtrs, srcSizes, cPtrs, cCapacities, cSizes); if(intermediateResultCompress.result.error) { results.error = intermediateResultCompress.result.error; return results; @@ -564,7 +568,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( if(!intermediateResultDecompress.completed) { intermediateResultDecompress = BMK_benchFunctionTimed(timeStateDecompress, &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep, - nbBlocks, (const void* const*)cPtrs, cSizes, resPtrs, resSizes); + nbBlocks, (const void* const*)cPtrs, cSizes, resPtrs, resSizes, NULL); if(intermediateResultDecompress.result.error) { results.error = intermediateResultDecompress.result.error; return results; @@ -590,7 +594,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( if(adv->mode != BMK_decodeOnly) { BMK_customReturn_t compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep, - nbBlocks, srcPtrs, srcSizes, cPtrs, cSizes, adv->nbSeconds); + nbBlocks, srcPtrs, srcSizes, cPtrs, cCapacities, cSizes, adv->nbSeconds); if(compressionResults.error) { results.error = compressionResults.error; return results; @@ -617,7 +621,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( BMK_customReturn_t decompressionResults = BMK_benchFunction( &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep, nbBlocks, - (const void* const*)cPtrs, cSizes, resPtrs, resSizes, + (const void* const*)cPtrs, cSizes, resPtrs, resSizes, NULL, adv->nbSeconds); if(decompressionResults.error) { results.error = decompressionResults.error; @@ -717,8 +721,10 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, const void ** const srcPtrs = (const void** const)malloc(maxNbBlocks * sizeof(void*)); size_t* const srcSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); + void ** const cPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); size_t* const cSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); + size_t* const cCapacities = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); void ** const resPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); size_t* const resSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); @@ -744,13 +750,11 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, !srcPtrs || !srcSizes || !cPtrs || !cSizes || !resPtrs || !resSizes; if (!allocationincomplete) { - results = BMK_benchMemAdvancedNoAlloc(srcPtrs, srcSizes, cPtrs, cSizes, + results = BMK_benchMemAdvancedNoAlloc(srcPtrs, srcSizes, cPtrs, cCapacities, cSizes, resPtrs, resSizes, &resultBuffer, compressedBuffer, maxCompressedSize, timeStateCompress, timeStateDecompress, srcBuffer, srcSize, fileSizes, nbFiles, cLevel, comprParams, dictBuffer, dictBufferSize, ctx, dctx, displayLevel, displayName, adv); } - - /* clean up */ BMK_freeTimeState(timeStateCompress); @@ -764,6 +768,7 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, free(srcSizes); free(cPtrs); free(cSizes); + free(cCapacities); free(resPtrs); free(resSizes); diff --git a/programs/bench.h b/programs/bench.h index 25d9f24a7..2a9945ac8 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -177,6 +177,7 @@ typedef size_t (*BMK_initFn_t)(void*); * dstBuffers - an array of buffers to be written into by benchFn * dstCapacitiesToSizes - an array of the capacities of above buffers. Output modified to compressed sizes of those blocks. * nbLoops - defines number of times benchFn is run. + * assumed array of size blockCount, will have compressed size of each block written to it. * return * .error will give a nonzero value if ZSTD_isError() is nonzero for any of the return * of the calls to initFn and benchFn, or if benchFunction errors internally @@ -187,12 +188,11 @@ typedef size_t (*BMK_initFn_t)(void*); * into dstBuffer, hence this value will be the total amount of bytes written to * dstBuffer. */ -BMK_customReturn_t BMK_benchFunction( - BMK_benchFn_t benchFn, void* benchPayload, +BMK_customReturn_t BMK_benchFunction(BMK_benchFn_t benchFn, void* benchPayload, BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBuffers, const size_t* srcSizes, - void * const * const dstBuffers, size_t* dstCapacitiesToSizes, + void * const * const dstBuffers, const size_t* dstCapacities, size_t* cSizes, unsigned nbLoops); @@ -221,7 +221,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed(BMK_timedFnState_t* cont, BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBlockBuffers, const size_t* srcBlockSizes, - void* const * const dstBlockBuffers, size_t* dstBlockCapacitiesToSizes); + void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, size_t* cSizes); #endif /* BENCH_H_121279284357 */ diff --git a/tests/README.md b/tests/README.md index 24a28ab7b..8bedd0a3c 100644 --- a/tests/README.md +++ b/tests/README.md @@ -88,3 +88,35 @@ as well as the 10,000 original files for more detailed comparison of decompressi will choose a random seed, and for 1 minute, generate random test frames and ensure that the zstd library correctly decompresses them in both simple and streaming modes. + +#### `paramgrill` - tool for generating compression table parameters and optimizing parameters on file given constraints + +Full list of arguments +``` + -T# : set level 1 speed objective + -B# : cut input into blocks of size # (default : single block) + -i# : iteration loops + -S : benchmarks a single run (example command: -Sl3w10h12) + w# - windowLog + h# - hashLog + c# - chainLog + s# - searchLog + l# - searchLength + t# - targetLength + S# - strategy + L# - level + --zstd= : Single run, parameter selection syntax same as zstdcli + --optimize= : find parameters to maximize compression ratio given parameters + Can use all --zstd= commands to constrain the type of solution found in addition to the following constraints + cSpeed= - Minimum compression speed + dSpeed= - Minimum decompression speed + cMem= - compression memory + lvl= - Automatically sets compression speed constraint to the speed of that level + --optimize= : same as -O with more verbose syntax + -P# : generated sample compressibility + -t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) + -v : Prints Benchmarking output + -D : Next argument dictionary file + +``` + Any inputs afterwards are treated as files to benchmark. diff --git a/tests/fullbench.c b/tests/fullbench.c index 9e7639f92..270cac86a 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -516,7 +516,7 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb, int cLevel, { r = BMK_benchFunction(benchFunction, buff2, NULL, NULL, 1, &src, &srcSize, - (void **)&dstBuff, &dstBuffSize, g_nbIterations); + (void **)&dstBuff, &dstBuffSize, NULL, g_nbIterations); if(r.error) { DISPLAY("ERROR %d ! ! \n", r.error); errorcode = r.error; diff --git a/tests/paramgrill.c b/tests/paramgrill.c old mode 100755 new mode 100644 index 02811b9c1..099be3688 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -56,7 +56,7 @@ static const int g_maxNbVariations = 64; #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define TIMED 0 #ifndef DEBUG -# define DEBUG 1 +# define DEBUG 0 #endif #define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); } @@ -67,14 +67,15 @@ static const int g_maxNbVariations = 64; #define CUSTOM_LEVEL 99 /* indices for each of the variables */ -#define WLOG_IND 0 -#define CLOG_IND 1 -#define HLOG_IND 2 -#define SLOG_IND 3 -#define SLEN_IND 4 -#define TLEN_IND 5 -//#define STRT_IND 6 -//#define NUM_PARAMS 7 +typedef enum { + wlog_ind = 0, + clog_ind = 1, + hlog_ind = 2, + slog_ind = 3, + slen_ind = 4, + tlen_ind = 5 +} varInds_t; + #define NUM_PARAMS 6 //just don't use strategy as a param. @@ -85,20 +86,16 @@ static const int g_maxNbVariations = 64; #define ZSTD_TARGETLENGTH_MIN 0 #define ZSTD_TARGETLENGTH_MAX 999 -//#define ZSTD_TARGETLENGTH_MAX 1024 #define WLOG_RANGE (ZSTD_WINDOWLOG_MAX - ZSTD_WINDOWLOG_MIN + 1) #define CLOG_RANGE (ZSTD_CHAINLOG_MAX - ZSTD_CHAINLOG_MIN + 1) #define HLOG_RANGE (ZSTD_HASHLOG_MAX - ZSTD_HASHLOG_MIN + 1) #define SLOG_RANGE (ZSTD_SEARCHLOG_MAX - ZSTD_SEARCHLOG_MIN + 1) #define SLEN_RANGE (ZSTD_SEARCHLENGTH_MAX - ZSTD_SEARCHLENGTH_MIN + 1) -#define TLEN_RANGE 12 -//TLEN_RANGE = 0, 2^0 to 2^10; -//hard coded since we only use powers of 2 (and 999 ~ 1024) +#define TLEN_RANGE 17 +/* TLEN_RANGE picked manually */ -//static const int mintable[NUM_PARAMS] = { ZSTD_WINDOWLOG_MIN, ZSTD_CHAINLOG_MIN, ZSTD_HASHLOG_MIN, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLENGTH_MIN, ZSTD_TARGETLENGTH_MIN }; -//static const int maxtable[NUM_PARAMS] = { ZSTD_WINDOWLOG_MAX, ZSTD_CHAINLOG_MAX, ZSTD_HASHLOG_MAX, ZSTD_SEARCHLOG_MAX, ZSTD_SEARCHLENGTH_MAX, ZSTD_TARGETLENGTH_MAX }; static const int rangetable[NUM_PARAMS] = { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE }; - +static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 }; /*-************************************ * Benchmark Parameters **************************************/ @@ -179,9 +176,6 @@ static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) return result; } -//assume that clock can at least measure .01 second intervals? -//make this a settable global initialized with fn? -//#define CLOCK_GRANULARITY 100000000ULL static U64 g_clockGranularity = 100000000ULL; static void findClockGranularity(void) { @@ -253,7 +247,7 @@ static void BMK_translateAdvancedParams(const ZSTD_compressionParameters params) /* checks results are feasible */ static int feasible(const BMK_result_t results, const constraint_t target) { - return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem || !target.cMem); + return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem); } #define EPSILON 0.001 @@ -268,13 +262,6 @@ static int eqZero(const double c1) { return !(memcmp(&c1, &z1, sizeof(double))) || !(memcmp(&c1, &z2, sizeof(double))); } -/* returns 1 if result2 is strictly 'better' than result1 */ -/* strict comparison / cutoff based */ -static int objective_lt(const BMK_result_t result1, const BMK_result_t result2) { - return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed) - || (result1.cSize == result2.cSize && epsilonEqual(result2.cSpeed, result1.cSpeed) && result2.dSpeed > result1.dSpeed); -} - /* hill climbing value for part 1 */ static double resultScore(const BMK_result_t res, const size_t srcSize, const constraint_t target) { double cs = 0., ds = 0., rt, cm = 0.; @@ -291,6 +278,16 @@ static double resultScore(const BMK_result_t res, const size_t srcSize, const co return ret; } +/* return true if r2 strictly better than r1 */ +static int compareResultLT(const BMK_result_t result1, const BMK_result_t result2, const constraint_t target, size_t srcSize) { + if(feasible(result1, target) && feasible(result2, target)) { + return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed) + || (result1.cSize == result2.cSize && epsilonEqual(result2.cSpeed, result1.cSpeed) && result2.dSpeed > result1.dSpeed); + } + return feasible(result2, target) || (!feasible(result1, target) && (resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target))); + +} + /* factor sort of arbitrary */ static constraint_t relaxTarget(constraint_t target) { target.cMem = (U32)-1; @@ -303,35 +300,11 @@ static constraint_t relaxTarget(constraint_t target) { * Bench functions *********************************************************/ -typedef struct -{ - const char* srcPtr; - size_t srcSize; - char* cPtr; - size_t cRoom; - size_t cSize; - char* resPtr; - size_t resSize; -} blockParam_t; - - const char* g_stratName[ZSTD_btultra+1] = { "(none) ", "ZSTD_fast ", "ZSTD_dfast ", "ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ", "ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "}; -static int -BMK_benchParam(BMK_result_t* resultPtr, - const void* srcBuffer, const size_t srcSize, - const size_t* fileSizes, const unsigned nbFiles, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, - const ZSTD_compressionParameters cParams) { - - BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, fileSizes, nbFiles, 0, &cParams, NULL, 0, ctx, dctx, 0, "File"); - *resultPtr = res.result; - return res.error; -} - /* benchParam but only takes in one file. */ static int BMK_benchParam1(BMK_result_t* resultPtr, @@ -349,6 +322,49 @@ typedef struct { ZSTD_compressionParameters params; } winnerInfo_t; +static ZSTD_compressionParameters emptyParams(void) { + ZSTD_compressionParameters p = { 0, 0, 0, 0, 0, 0, (ZSTD_strategy)0 }; + return p; +} + +static winnerInfo_t initWinnerInfo(ZSTD_compressionParameters p) { + winnerInfo_t w1; + w1.result.cSpeed = 0.; + w1.result.dSpeed = 0.; + w1.result.cMem = (size_t)-1; + w1.result.cSize = (size_t)-1; + w1.params = p; + return w1; +} + +typedef struct { + size_t srcSize; + void** srcPtrs; + size_t* srcSizes; + void** dstPtrs; + size_t* dstCapacities; + size_t* dstSizes; + void** resPtrs; + size_t* resSizes; + size_t nbBlocks; +} buffers_t; + +typedef struct { + size_t dictSize; + void* dictBuffer; + ZSTD_CCtx* cctx; + ZSTD_DCtx* dctx; +} contexts_t; + +static int +BMK_benchParam(BMK_result_t* resultPtr, + buffers_t buf, contexts_t ctx, + const ZSTD_compressionParameters cParams) { + BMK_return_t res = BMK_benchMem(buf.srcPtrs[0], buf.srcSize, buf.srcSizes, (unsigned)buf.nbBlocks, 0, &cParams, ctx.dictBuffer, ctx.dictSize, ctx.cctx, ctx.dctx, 0, "Files"); + *resultPtr = res.result; + return res.error; +} + /*-******************************************************* * From Paramgrill *********************************************************/ @@ -356,6 +372,8 @@ typedef struct { static void BMK_initCCtx(ZSTD_CCtx* ctx, const void* dictBuffer, size_t dictBufferSize, int cLevel, const ZSTD_compressionParameters* comprParams, const BMK_advancedParams_t* adv) { + ZSTD_CCtx_reset(ctx); + ZSTD_CCtx_resetParameters(ctx); if (adv->nbWorkers==1) { ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, 0); } else { @@ -380,6 +398,7 @@ static void BMK_initCCtx(ZSTD_CCtx* ctx, static void BMK_initDCtx(ZSTD_DCtx* dctx, const void* dictBuffer, size_t dictBufferSize) { + ZSTD_DCtx_reset(dctx); ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize); } @@ -425,6 +444,7 @@ static size_t local_defaultCompress( out.dst = dstBuffer; out.size = dstSize; out.pos = 0; + assert(dstSize == ZSTD_compressBound(srcSize)); /* specific to this version, which is only used in paramgrill */ while (moreToFlush) { if(out.pos == out.size) { return (size_t)-ZSTD_error_dstSize_tooSmall; @@ -471,249 +491,164 @@ static size_t local_defaultDecompress( *********************************************************/ /* Replicate function of benchMemAdvanced, but with pre-split src / dst buffers, with relevant info to invert it (compressedSizes) passed out. */ -/*BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); */ +/* BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); */ /* nbSeconds used in same way as in BMK_advancedParams_t, as nbIters when in iterMode */ /* if in decodeOnly, then srcPtr's will be compressed blocks, and uncompressedBlocks will be written to dstPtrs? */ /* dictionary nullable, nothing else though. */ -static BMK_return_t BMK_benchMemInvertible(const void * const * const srcPtrs, size_t const * const srcSizes, - void** dstPtrs, size_t* dstCapacityToSizes, U32 const nbBlocks, +static BMK_return_t BMK_benchMemInvertible(buffers_t buf, contexts_t ctx, const int cLevel, const ZSTD_compressionParameters* comprParams, - const void* dictBuffer, const size_t dictBufferSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const BMK_mode_t mode, const BMK_loopMode_t loopMode, const unsigned nbSeconds) { + U32 i; BMK_return_t results = { { 0, 0., 0., 0 }, 0 } ; - size_t srcSize = 0; - void** const resPtrs = malloc(sizeof(void*) * nbBlocks); /* only really needed in both mode. */ - size_t* const resSizes = malloc(sizeof(size_t) * nbBlocks); - int freeDST = 0; + const void *const *const srcPtrs = (const void *const *const)buf.srcPtrs; + size_t const *const srcSizes = buf.srcSizes; + void** dstPtrs = buf.dstPtrs; + size_t* dstCapacities = buf.dstCapacities; + size_t* dstSizes = buf.dstSizes; + void** resPtrs = buf.resPtrs; + size_t* resSizes = buf.resSizes; + const void* dictBuffer = ctx.dictBuffer; + const size_t dictBufferSize = ctx.dictSize; + const size_t nbBlocks = buf.nbBlocks; + const size_t srcSize = buf.srcSize; + ZSTD_CCtx* cctx = ctx.cctx; + ZSTD_DCtx* dctx = ctx.dctx; BMK_advancedParams_t adv = BMK_initAdvancedParams(); adv.mode = mode; adv.loopMode = loopMode; adv.nbSeconds = nbSeconds; - /* resSizes == srcSizes, but modifiable */ - memcpy(resSizes, srcSizes, sizeof(size_t) * nbBlocks); - - for(i = 0; i < nbBlocks; i++) { - srcSize += srcSizes[i]; - } - - if(!ctx || !dctx || !srcPtrs || ! srcSizes) - { - results.error = 31; - DISPLAY("error: passed in null argument\n"); - free(resPtrs); - free(resSizes); - return results; - } - if(!resPtrs || !resSizes) { - results.error = 32; - DISPLAY("error: allocation failed\n"); - free(resPtrs); - free(resSizes); - return results; - } - - /* so resPtr is continuous */ - resPtrs[0] = malloc(srcSize); - - if(!(resPtrs[0])) { - results.error = 32; - DISPLAY("error: allocation failed\n"); - free(resPtrs); - free(resSizes); - return results; - } - - for(i = 1; i < nbBlocks; i++) { - resPtrs[i] = (void*)(((char*)resPtrs[i-1]) + srcSizes[i-1]); - } - - /* allocate own dst if NULL */ - if(dstPtrs == NULL) { - freeDST = 1; - dstPtrs = malloc(nbBlocks * sizeof(void*)); - dstCapacityToSizes = malloc(nbBlocks * sizeof(size_t)); - if(dstPtrs == NULL) { - results.error = 33; - DISPLAY("error: allocation failed\n"); - free(resPtrs); - free(resSizes); - return results; - } - - if(mode == BMK_decodeOnly) { //dst is src - size_t dstSize = 0; - for(i = 0; i < nbBlocks; i++) { - dstCapacityToSizes[i] = ZSTD_getDecompressedSize(srcPtrs[i], srcSizes[i]); - dstSize += dstCapacityToSizes[i]; - } - dstPtrs[0] = malloc(dstSize); - if(dstPtrs[0] == NULL) { - results.error = 34; - DISPLAY("error: allocation failed\n"); - goto _cleanUp; - } - for(i = 1; i < nbBlocks; i++) { - dstPtrs[i] = (void*)(((char*)dstPtrs[i-1]) + ZSTD_getDecompressedSize(srcPtrs[i-1], srcSizes[i-1])); - } - } else { - dstPtrs[0] = malloc(ZSTD_compressBound(srcSize) + (nbBlocks * 1024)); - if(dstPtrs[0] == NULL) { - results.error = 35; - DISPLAY("error: allocation failed\n"); - goto _cleanUp; - } - dstCapacityToSizes[0] = ZSTD_compressBound(srcSizes[0]); - for(i = 1; i < nbBlocks; i++) { - dstPtrs[i] = (void*)(((char*)dstPtrs[i-1]) + dstCapacityToSizes[i-1]); - dstCapacityToSizes[i] = ZSTD_compressBound(srcSizes[i]); - } - } - } - /* warmimg up memory */ - for(i = 0; i < nbBlocks; i++) { - RDG_genBuffer(dstPtrs[i], dstCapacityToSizes[i], 0.10, 0.50, 1); + /* can't do this if decode only */ + for(i = 0; i < buf.nbBlocks; i++) { + if(mode != BMK_decodeOnly) { + RDG_genBuffer(dstPtrs[i], dstCapacities[i], 0.10, 0.50, 1); + } else { + RDG_genBuffer(resPtrs[i], resSizes[i], 0.10, 0.50, 1); + } } /* Bench */ - { - { - BMK_initCCtxArgs cctxprep; - BMK_initDCtxArgs dctxprep; - cctxprep.ctx = ctx; - cctxprep.dictBuffer = dictBuffer; - cctxprep.dictBufferSize = dictBufferSize; - cctxprep.cLevel = cLevel; - cctxprep.comprParams = comprParams; - cctxprep.adv = &adv; - dctxprep.dctx = dctx; - dctxprep.dictBuffer = dictBuffer; - dctxprep.dictBufferSize = dictBufferSize; - if(loopMode == BMK_timeMode) { - BMK_customTimedReturn_t intermediateResultCompress; - BMK_customTimedReturn_t intermediateResultDecompress; - BMK_timedFnState_t* timeStateCompress = BMK_createTimeState(nbSeconds); - BMK_timedFnState_t* timeStateDecompress = BMK_createTimeState(nbSeconds); - if(mode == BMK_compressOnly) { - intermediateResultCompress.completed = 0; - intermediateResultDecompress.completed = 1; - } else if (mode == BMK_decodeOnly) { - intermediateResultCompress.completed = 1; - intermediateResultDecompress.completed = 0; - } else { /* both */ - intermediateResultCompress.completed = 0; - intermediateResultDecompress.completed = 0; + + { + /* init args */ + BMK_initCCtxArgs cctxprep; + BMK_initDCtxArgs dctxprep; + cctxprep.ctx = cctx; + cctxprep.dictBuffer = dictBuffer; + cctxprep.dictBufferSize = dictBufferSize; + cctxprep.cLevel = cLevel; + cctxprep.comprParams = comprParams; + cctxprep.adv = &adv; + dctxprep.dctx = dctx; + dctxprep.dictBuffer = dictBuffer; + dctxprep.dictBufferSize = dictBufferSize; + + if(loopMode == BMK_timeMode) { + BMK_customTimedReturn_t intermediateResultCompress; + BMK_customTimedReturn_t intermediateResultDecompress; + BMK_timedFnState_t* timeStateCompress = BMK_createTimeState(nbSeconds); + BMK_timedFnState_t* timeStateDecompress = BMK_createTimeState(nbSeconds); + if(mode == BMK_compressOnly) { + intermediateResultCompress.completed = 0; + intermediateResultDecompress.completed = 1; + } else if (mode == BMK_decodeOnly) { + intermediateResultCompress.completed = 1; + intermediateResultDecompress.completed = 0; + } else { /* both */ + intermediateResultCompress.completed = 0; + intermediateResultDecompress.completed = 0; + } + + while(!intermediateResultCompress.completed) { + intermediateResultCompress = BMK_benchFunctionTimed(timeStateCompress, &local_defaultCompress, (void*)cctx, &local_initCCtx, (void*)&cctxprep, + nbBlocks, srcPtrs, srcSizes, dstPtrs, dstCapacities, dstSizes); + + if(intermediateResultCompress.result.error) { + results.error = intermediateResultCompress.result.error; + BMK_freeTimeState(timeStateCompress); + BMK_freeTimeState(timeStateDecompress); + return results; } - while(!(intermediateResultCompress.completed && intermediateResultDecompress.completed)) { - if(!intermediateResultCompress.completed) { - intermediateResultCompress = BMK_benchFunctionTimed(timeStateCompress, &local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep, - nbBlocks, srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes); - if(intermediateResultCompress.result.error) { - results.error = intermediateResultCompress.result.error; - BMK_freeTimeState(timeStateCompress); - BMK_freeTimeState(timeStateDecompress); - goto _cleanUp; - } - results.result.cSpeed = ((double)srcSize / intermediateResultCompress.result.result.nanoSecPerRun) * 1000000000; - results.result.cSize = intermediateResultCompress.result.result.sumOfReturn; - } + results.result.cSpeed = ((double)srcSize / intermediateResultCompress.result.result.nanoSecPerRun) * TIMELOOP_NANOSEC; + results.result.cSize = intermediateResultCompress.result.result.sumOfReturn; + } - if(!intermediateResultDecompress.completed) { - if(mode == BMK_decodeOnly) { - intermediateResultDecompress = BMK_benchFunctionTimed(timeStateDecompress, &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep, - nbBlocks, (const void* const*)srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes); - } else { /* both, decompressed result already written to dstPtr */ - intermediateResultDecompress = BMK_benchFunctionTimed(timeStateDecompress, &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep, - nbBlocks, (const void* const*)dstPtrs, dstCapacityToSizes, resPtrs, resSizes); - } + while(!intermediateResultDecompress.completed) { + intermediateResultDecompress = BMK_benchFunctionTimed(timeStateDecompress, &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep, + nbBlocks, (const void* const*)dstPtrs, dstSizes, resPtrs, resSizes, NULL); - if(intermediateResultDecompress.result.error) { - results.error = intermediateResultDecompress.result.error; - BMK_freeTimeState(timeStateCompress); - BMK_freeTimeState(timeStateDecompress); - goto _cleanUp; - } - results.result.dSpeed = ((double)srcSize / intermediateResultDecompress.result.result.nanoSecPerRun) * 1000000000; - } + if(intermediateResultDecompress.result.error) { + results.error = intermediateResultDecompress.result.error; + BMK_freeTimeState(timeStateCompress); + BMK_freeTimeState(timeStateDecompress); + return results; } - BMK_freeTimeState(timeStateCompress); - BMK_freeTimeState(timeStateDecompress); - } else { //iterMode; - if(mode != BMK_decodeOnly) { + results.result.dSpeed = ((double)srcSize / intermediateResultDecompress.result.result.nanoSecPerRun) * TIMELOOP_NANOSEC; + } - BMK_customReturn_t compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep, - nbBlocks, srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbSeconds); - if(compressionResults.error) { - results.error = compressionResults.error; - goto _cleanUp; - } - if(compressionResults.result.nanoSecPerRun == 0) { - results.result.cSpeed = 0; - } else { - results.result.cSpeed = (double)srcSize / compressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; - } - results.result.cSize = compressionResults.result.sumOfReturn; + BMK_freeTimeState(timeStateCompress); + BMK_freeTimeState(timeStateDecompress); + + } else { //iterMode; + if(mode != BMK_decodeOnly) { + + BMK_customReturn_t compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)cctx, &local_initCCtx, (void*)&cctxprep, + nbBlocks, srcPtrs, srcSizes, dstPtrs, dstCapacities, dstSizes, nbSeconds); + if(compressionResults.error) { + results.error = compressionResults.error; + return results; } - if(mode != BMK_compressOnly) { - BMK_customReturn_t decompressionResults; - if(mode == BMK_decodeOnly) { - decompressionResults = BMK_benchFunction( - &local_defaultDecompress, (void*)(dctx), - &local_initDCtx, (void*)&dctxprep, nbBlocks, - (const void* const*)srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, - nbSeconds); - } else { - decompressionResults = BMK_benchFunction( - &local_defaultDecompress, (void*)(dctx), - &local_initDCtx, (void*)&dctxprep, nbBlocks, - (const void* const*)dstPtrs, dstCapacityToSizes, resPtrs, resSizes, - nbSeconds); - } + if(compressionResults.result.nanoSecPerRun == 0) { + results.result.cSpeed = 0; + } else { + results.result.cSpeed = (double)srcSize / compressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; + } + results.result.cSize = compressionResults.result.sumOfReturn; + } - if(decompressionResults.error) { - results.error = decompressionResults.error; - goto _cleanUp; - } + if(mode != BMK_compressOnly) { + BMK_customReturn_t decompressionResults; + decompressionResults = BMK_benchFunction( + &local_defaultDecompress, (void*)(dctx), + &local_initDCtx, (void*)&dctxprep, nbBlocks, + (const void* const*)dstPtrs, dstSizes, resPtrs, resSizes, NULL, + nbSeconds); - if(decompressionResults.result.nanoSecPerRun == 0) { - results.result.dSpeed = 0; - } else { - results.result.dSpeed = (double)srcSize / decompressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; - } + if(decompressionResults.error) { + results.error = decompressionResults.error; + return results; + } + + if(decompressionResults.result.nanoSecPerRun == 0) { + results.result.dSpeed = 0; + } else { + results.result.dSpeed = (double)srcSize / decompressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; } } } - } /* Bench */ - results.result.cMem = (1 << (comprParams->windowLog)) + ZSTD_sizeof_CCtx(ctx); - -_cleanUp: - free(resPtrs[0]); - free(resPtrs); - free(resSizes); - if(freeDST) { - free(dstPtrs[0]); - free(dstPtrs); } + /* Bench */ + results.result.cMem = (1 << (comprParams->windowLog)) + ZSTD_sizeof_CCtx(cctx); return results; } /* global winner used for display. */ //Should be totally 0 initialized? -static winnerInfo_t g_winner; +static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; static constraint_t g_targetConstraints; static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const size_t srcSize) { - if(DEBUG || (objective_lt(g_winner.result, result) && feasible(result, g_targetConstraints))) { + if(DEBUG || compareResultLT(g_winner.result, result, g_targetConstraints, srcSize)) { char lvlstr[15] = "Custom Level"; const U64 time = UTIL_clockSpanNano(g_time); const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC); - if(DEBUG && (objective_lt(g_winner.result, result) && feasible(result, g_targetConstraints))) { + + if(DEBUG && compareResultLT(g_winner.result, result, g_targetConstraints, srcSize)) { DISPLAY("New Winner: \n"); } @@ -722,27 +657,24 @@ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, params.targetLength, g_stratName[(U32)(params.strategy)]); + if(cLevel != CUSTOM_LEVEL) { snprintf(lvlstr, 15, " Level %2u ", cLevel); } + fprintf(f, "/* %s */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */", - lvlstr, (double)srcSize / result.cSize, result.cSpeed / 1000000., result.dSpeed / 1000000.); + lvlstr, (double)srcSize / result.cSize, result.cSpeed / (1 << 20), result.dSpeed / (1 << 20)); if(TIMED) { fprintf(f, " - %1lu:%2lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); } fprintf(f, "\n"); - if(objective_lt(g_winner.result, result) && feasible(result, g_targetConstraints)) { + + if(compareResultLT(g_winner.result, result, g_targetConstraints, srcSize)) { BMK_translateAdvancedParams(params); g_winner.result = result; g_winner.params = params; } } - //else { - // DISPLAY("G_WINNER: "); - // DISPLAY("/* R:%5.3f at %5.1f MB/s - %5.1f MB/s */ \n",(double)srcSize / g_winner.result.cSize , g_winner.result.cSpeed / 1000000 , g_winner.result.dSpeed / 1000000); - // DISPLAY("LOSER : "); - // DISPLAY("/* R:%5.3f at %5.1f MB/s - %5.1f MB/s */ \n",(double)srcSize / result.cSize, result.cSpeed / 1000000 , result.dSpeed / 1000000); - //} } static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize) @@ -903,23 +835,23 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para static ZSTD_compressionParameters sanitizeParams(ZSTD_compressionParameters params) { if (params.strategy == ZSTD_fast) - g_params.chainLog = 0, g_params.searchLog = 0; + params.chainLog = 0, params.searchLog = 0; if (params.strategy == ZSTD_dfast) - g_params.searchLog = 0; + params.searchLog = 0; if (params.strategy != ZSTD_btopt && params.strategy != ZSTD_btultra && params.strategy != ZSTD_fast) - g_params.targetLength = 0; + params.targetLength = 0; return params; } /* new length */ /* keep old array, will need if iter over strategy. */ -static int sanitizeVarArray(const int varLength, const U32* varArray, U32* varNew, const ZSTD_strategy strat) { +static int sanitizeVarArray(varInds_t* varNew, const int varLength, const varInds_t* varArray, const ZSTD_strategy strat) { int i, j = 0; for(i = 0; i < varLength; i++) { - if( !((varArray[i] == CLOG_IND && strat == ZSTD_fast) - || (varArray[i] == SLOG_IND && strat == ZSTD_dfast) - || (varArray[i] == TLEN_IND && strat != ZSTD_btopt && strat != ZSTD_btultra && strat != ZSTD_fast))) { + if( !((varArray[i] == clog_ind && strat == ZSTD_fast) + || (varArray[i] == slog_ind && strat == ZSTD_dfast) + || (varArray[i] == tlen_ind && strat != ZSTD_btopt && strat != ZSTD_btultra && strat != ZSTD_fast))) { varNew[j] = varArray[i]; j++; } @@ -930,69 +862,72 @@ static int sanitizeVarArray(const int varLength, const U32* varArray, U32* varNe /* res should be NUM_PARAMS size */ /* constructs varArray from ZSTD_compressionParameters style parameter */ -static int variableParams(const ZSTD_compressionParameters paramConstraints, U32* res) { +static int variableParams(const ZSTD_compressionParameters paramConstraints, varInds_t* res) { int j = 0; if(!paramConstraints.windowLog) { - res[j] = WLOG_IND; + res[j] = wlog_ind; j++; } if(!paramConstraints.chainLog) { - res[j] = CLOG_IND; + res[j] = clog_ind; j++; } if(!paramConstraints.hashLog) { - res[j] = HLOG_IND; + res[j] = hlog_ind; j++; } if(!paramConstraints.searchLog) { - res[j] = SLOG_IND; + res[j] = slog_ind; j++; } if(!paramConstraints.searchLength) { - res[j] = SLEN_IND; + res[j] = slen_ind; j++; } if(!paramConstraints.targetLength) { - res[j] = TLEN_IND; + res[j] = tlen_ind; j++; } return j; } +/* bin-search on tlen_table for correct index. */ +static int tlen_inv(U32 x) { + int lo = 0; + int hi = TLEN_RANGE; + while(lo < hi) { + int mid = (lo + hi) / 2; + if(tlen_table[mid] < x) { + lo = mid + 1; + } if(tlen_table[mid] == x) { + return mid; + } else { + hi = mid; + } + } + return lo; +} + /* amt will probably always be \pm 1? */ /* slight change from old paramVariation, targetLength can only take on powers of 2 now (999 ~= 1024?) */ /* take max/min bounds into account as well? */ -static void paramVaryOnce(const U32 paramIndex, const int amt, ZSTD_compressionParameters* ptr) { +static void paramVaryOnce(const varInds_t paramIndex, const int amt, ZSTD_compressionParameters* ptr) { switch(paramIndex) { - case WLOG_IND: ptr->windowLog += amt; break; - case CLOG_IND: ptr->chainLog += amt; break; - case HLOG_IND: ptr->hashLog += amt; break; - case SLOG_IND: ptr->searchLog += amt; break; - case SLEN_IND: ptr->searchLength += amt; break; - case TLEN_IND: - if(amt >= 0) { - if(ptr->targetLength == 0) { - if(amt > 0) { - ptr->targetLength = MIN(1 << (amt - 1), 999); - } - } else { - ptr->targetLength <<= amt; - ptr->targetLength = MIN(ptr->targetLength, 999); - } - } else { - if(ptr->targetLength == 999) { - ptr->targetLength = 1024; - } - ptr->targetLength >>= -amt; - } + case wlog_ind: ptr->windowLog += amt; break; + case clog_ind: ptr->chainLog += amt; break; + case hlog_ind: ptr->hashLog += amt; break; + case slog_ind: ptr->searchLog += amt; break; + case slen_ind: ptr->searchLength += amt; break; + case tlen_ind: + ptr->targetLength = tlen_table[MAX(0, MIN(TLEN_RANGE - 1, tlen_inv(ptr->targetLength) + amt))]; break; default: break; } } /* varies ptr by nbChanges respecting varyParams*/ -static void paramVariation(ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen, const U32 nbChanges) +static void paramVariation(ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen, const U32 nbChanges) { ZSTD_compressionParameters p; U32 validated = 0; @@ -1009,7 +944,7 @@ static void paramVariation(ZSTD_compressionParameters* ptr, const U32* varyParam } /* length of memo table given free variables */ -static size_t memoTableLen(const U32* varyParams, const int varyLen) { +static size_t memoTableLen(const varInds_t* varyParams, const int varyLen) { size_t arrayLen = 1; int i; for(i = 0; i < varyLen; i++) { @@ -1018,53 +953,45 @@ static size_t memoTableLen(const U32* varyParams, const int varyLen) { return arrayLen; } -//sort of ~lg2 (replace 1024 w/ 999, and add 0 at lower end of range) for memoTableInd Tlen -static unsigned lg2(unsigned x) { - if(x == 999) { - return 11; - } - return x ? ZSTD_highbit32(x) + 1 : 0; -} - /* returns unique index of compression parameters */ -static unsigned memoTableInd(const ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen) { +static unsigned memoTableInd(const ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen) { int i; unsigned ind = 0; for(i = 0; i < varyLen; i++) { switch(varyParams[i]) { - case WLOG_IND: ind *= WLOG_RANGE; ind += ptr->windowLog - ZSTD_WINDOWLOG_MIN ; break; - case CLOG_IND: ind *= CLOG_RANGE; ind += ptr->chainLog - ZSTD_CHAINLOG_MIN ; break; - case HLOG_IND: ind *= HLOG_RANGE; ind += ptr->hashLog - ZSTD_HASHLOG_MIN ; break; - case SLOG_IND: ind *= SLOG_RANGE; ind += ptr->searchLog - ZSTD_SEARCHLOG_MIN ; break; - case SLEN_IND: ind *= SLEN_RANGE; ind += ptr->searchLength - ZSTD_SEARCHLENGTH_MIN; break; - case TLEN_IND: ind *= TLEN_RANGE; ind += lg2(ptr->targetLength) - ZSTD_TARGETLENGTH_MIN; break; + case wlog_ind: ind *= WLOG_RANGE; ind += ptr->windowLog - ZSTD_WINDOWLOG_MIN ; break; + case clog_ind: ind *= CLOG_RANGE; ind += ptr->chainLog - ZSTD_CHAINLOG_MIN ; break; + case hlog_ind: ind *= HLOG_RANGE; ind += ptr->hashLog - ZSTD_HASHLOG_MIN ; break; + case slog_ind: ind *= SLOG_RANGE; ind += ptr->searchLog - ZSTD_SEARCHLOG_MIN ; break; + case slen_ind: ind *= SLEN_RANGE; ind += ptr->searchLength - ZSTD_SEARCHLENGTH_MIN; break; + case tlen_ind: ind *= TLEN_RANGE; ind += tlen_inv(ptr->targetLength) - ZSTD_TARGETLENGTH_MIN; break; } } return ind; } /* inverse of above function (from index to parameters) */ -static void memoTableIndInv(ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen, size_t ind) { +static void memoTableIndInv(ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen, size_t ind) { int i; for(i = varyLen - 1; i >= 0; i--) { switch(varyParams[i]) { - case WLOG_IND: ptr->windowLog = ind % WLOG_RANGE + ZSTD_WINDOWLOG_MIN; ind /= WLOG_RANGE; break; - case CLOG_IND: ptr->chainLog = ind % CLOG_RANGE + ZSTD_CHAINLOG_MIN; ind /= CLOG_RANGE; break; - case HLOG_IND: ptr->hashLog = ind % HLOG_RANGE + ZSTD_HASHLOG_MIN; ind /= HLOG_RANGE; break; - case SLOG_IND: ptr->searchLog = ind % SLOG_RANGE + ZSTD_SEARCHLOG_MIN; ind /= SLOG_RANGE; break; - case SLEN_IND: ptr->searchLength = ind % SLEN_RANGE + ZSTD_SEARCHLENGTH_MIN; ind /= SLEN_RANGE; break; - case TLEN_IND: ptr->targetLength = (ind % TLEN_RANGE) ? MIN(1 << ((ind % TLEN_RANGE) - 1), 999) : 0; ind /= TLEN_RANGE; break; + case wlog_ind: ptr->windowLog = ind % WLOG_RANGE + ZSTD_WINDOWLOG_MIN; ind /= WLOG_RANGE; break; + case clog_ind: ptr->chainLog = ind % CLOG_RANGE + ZSTD_CHAINLOG_MIN; ind /= CLOG_RANGE; break; + case hlog_ind: ptr->hashLog = ind % HLOG_RANGE + ZSTD_HASHLOG_MIN; ind /= HLOG_RANGE; break; + case slog_ind: ptr->searchLog = ind % SLOG_RANGE + ZSTD_SEARCHLOG_MIN; ind /= SLOG_RANGE; break; + case slen_ind: ptr->searchLength = ind % SLEN_RANGE + ZSTD_SEARCHLENGTH_MIN; ind /= SLEN_RANGE; break; + case tlen_ind: ptr->targetLength = tlen_table[(ind % TLEN_RANGE)]; ind /= TLEN_RANGE; break; } } } - /* Initialize memotable, immediately mark redundant / obviously infeasible params as */ -static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstraints, const constraint_t target, const U32* varyParams, const int varyLen, const size_t srcSize) { +static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { size_t i; size_t arrayLen = memoTableLen(varyParams, varyLen); int cwFixed = !paramConstraints.chainLog || !paramConstraints.windowLog; int scFixed = !paramConstraints.searchLog || !paramConstraints.chainLog; + int whFixed = !paramConstraints.windowLog || !paramConstraints.hashLog; int wFixed = !paramConstraints.windowLog; int j = 0; memset(memoTable, 0, arrayLen); @@ -1076,7 +1003,7 @@ static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstra memoTable[i] = 255; j++; } - if(wFixed && (1ULL << paramConstraints.windowLog) > (srcSize << 1)) { + if(wFixed && (1ULL << paramConstraints.windowLog) > (srcSize << 2)) { memoTable[i] = 255; } /* nil out parameter sets equivalent to others. */ @@ -1093,12 +1020,20 @@ static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstra } } } + if(scFixed) { if(paramConstraints.searchLog > paramConstraints.chainLog) { if(memoTable[i] != 255) { j++; } memoTable[i] = 255; } } + + if(whFixed) { + if(paramConstraints.hashLog > paramConstraints.windowLog + 1) { + if(memoTable[i] != 255) { j++; } + memoTable[i] = 255; + } + } } DEBUGOUTPUT("%d / %d Invalid\n", j, (int)i); if((int)i == j) { @@ -1106,27 +1041,7 @@ static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstra } } -/* inits memotables for all (including mallocs), all strategies */ -/* takes unsanitized varyParams */ -static U8** memoTableInitAll(ZSTD_compressionParameters paramConstraints, constraint_t target, const U32* varyParams, const int varyLen, const size_t srcSize) { - U32 varNew[NUM_PARAMS]; - int varLenNew; - U8** mtAll = malloc(sizeof(U8*) * (ZSTD_btultra + 1)); - int i; - if(mtAll == NULL) { - return NULL; - } - for(i = 1; i <= (int)ZSTD_btultra; i++) { - varLenNew = sanitizeVarArray(varyLen, varyParams, varNew, i); - mtAll[i] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew)); - if(mtAll[i] == NULL) { - return NULL; - } - memoTableInit(mtAll[i], paramConstraints, target, varNew, varLenNew, srcSize); - } - return mtAll; -} - +/* frees all allocated memotables */ static void memoTableFreeAll(U8** mtAll) { int i; if(mtAll == NULL) { return; } @@ -1136,6 +1051,28 @@ static void memoTableFreeAll(U8** mtAll) { free(mtAll); } +/* inits memotables for all (including mallocs), all strategies */ +/* takes unsanitized varyParams */ +static U8** memoTableInitAll(ZSTD_compressionParameters paramConstraints, constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { + varInds_t varNew[NUM_PARAMS]; + int varLenNew; + U8** mtAll = calloc(sizeof(U8*),(ZSTD_btultra + 1)); + int i; + if(mtAll == NULL) { + return NULL; + } + for(i = 1; i <= (int)ZSTD_btultra; i++) { + varLenNew = sanitizeVarArray(varNew, varyLen, varyParams, i); + mtAll[i] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew)); + if(mtAll[i] == NULL) { + memoTableFreeAll(mtAll); + return NULL; + } + memoTableInit(mtAll[i], paramConstraints, target, varNew, varLenNew, srcSize); + } + return mtAll; +} + #define PARAMTABLELOG 25 #define PARAMTABLESIZE (1< 0 && tries > 0); memoTableIndInv(pc, varArray, varLen, (unsigned)ind); - *pc = sanitizeParams(*pc); + //*pc = sanitizeParams(*pc); } static void BMK_selectRandomStart( @@ -1250,7 +1187,7 @@ static void BMK_benchFullTable(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* src winnerInfo_t winners[NB_LEVELS_TRACKED+1]; const char* const rfName = "grillResults.txt"; FILE* const f = fopen(rfName, "w"); - const size_t blockSize = g_blockSize ? g_blockSize : srcSize; /* cut by block or not ? */ + const size_t blockSize = g_blockSize ? g_blockSize : ZSTD_BLOCKSIZE_MAX; /* cut by block or not ? */ /* init */ assert(g_singleRun==0); @@ -1390,574 +1327,204 @@ int benchFiles(const char** fileNamesTable, int nbFiles) return 0; } - - -/* -checks feasibility with uncertainty. --1 : certainly infeasible - 0 : uncertain - 1 : certainly feasible -*/ -static int uncertainFeasibility(double const uncertaintyConstantC, double const uncertaintyConstantD, const constraint_t paramTarget, const BMK_result_t* const results) { - if((paramTarget.cSpeed != 0 && results->cSpeed * uncertaintyConstantC < paramTarget.cSpeed) || - (paramTarget.dSpeed != 0 && results->dSpeed * uncertaintyConstantD < paramTarget.dSpeed) || - (paramTarget.cMem != 0 && results->cMem > paramTarget.cMem)) { - return -1; - } else if((paramTarget.cSpeed == 0 || results->cSpeed / uncertaintyConstantC > paramTarget.cSpeed) && - (paramTarget.dSpeed == 0 || results->dSpeed / uncertaintyConstantD > paramTarget.dSpeed) && - (paramTarget.cMem == 0 || results->cMem <= paramTarget.cMem)) { - return 1; - } else { - return 0; - } -} - -/* 1 - better than prev best - 0 - uncertain - -1 - worse - assume prev_best status is run fully? - but then we'd have to rerun any winners anyway */ -/* not as useful as initially believed */ -static int uncertainComparison(double const uncertaintyConstantC, double const uncertaintyConstantD, const BMK_result_t* candidate, const BMK_result_t* prevBest) { - (void)uncertaintyConstantD; //unused for now - if(candidate->cSpeed > prevBest->cSpeed * uncertaintyConstantC) { - return 1; - } else if (candidate->cSpeed * uncertaintyConstantC < prevBest->cSpeed) { - return -1; - } else { - return 0; - } -} - /*benchmarks and tests feasibility together 1 = true = better 0 = false = not better if true then resultPtr will give results. 2+ on error? */ -//Maybe use compress_only for benchmark -#define INFEASIBLE_RESULT 0 -#define FEASIBLE_RESULT 1 +//Maybe use compress_only for benchmark first run? +#define WORSE_RESULT 0 +#define BETTER_RESULT 1 #define ERROR_RESULT 2 -static int feasibleBench(BMK_result_t* resultPtr, - const void* const * const srcPtrs, size_t const * const srcSizes, - void** const dstPtrs, size_t* dstCapacityToSizes, U32 const nbBlocks, - void* dictBuffer, const size_t dictBufferSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, - const ZSTD_compressionParameters cParams, - const constraint_t target, - BMK_result_t* winnerResult) { - BMK_return_t benchres; - U64 loopDurationC = 0, loopDurationD = 0; - double uncertaintyConstantC, uncertaintyConstantD; - size_t srcSize = 0; - U32 i; - //alternative - test 1 iter for ratio, (possibility of error 3 which is fine), - //maybe iter this until 2x measurable for better guarantee? - DEBUGOUTPUT("Feas:\n"); - benchres = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_both, BMK_iterMode, 1); - if(benchres.error) { - DISPLAY("ERROR %d !!\n", benchres.error); - } - for(i = 0; i < nbBlocks; i++) { - srcSize += srcSizes[i]; - } - BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); +//add worse result complete for worse results of length > 1 sec? - if(!benchres.error) { - *resultPtr = benchres.result; - /* if speed is 0 (only happens when time = 0) */ - if(eqZero(benchres.result.cSpeed)) { - loopDurationC = 0; - uncertaintyConstantC = 2; - } else { - loopDurationC = (U64)((double)(srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); - //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? - //possibly has to do with initCCtx? or system stuff? - //asymmetric +/- constant needed? - uncertaintyConstantC = MIN((loopDurationC + (double)(2 * g_clockGranularity)/loopDurationC) * 1.1, 3); //.02 seconds - } - if(eqZero(benchres.result.dSpeed)) { - loopDurationD = 0; - uncertaintyConstantD = 2; - } else { - loopDurationD = (U64)((double)(srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); - //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? - //possibly has to do with initCCtx? or system stuff? - //asymmetric +/- constant needed? - uncertaintyConstantD = MIN((loopDurationD + (double)(2 * g_clockGranularity)/loopDurationD) * 1.1, 3); //.02 seconds - } - - - if(benchres.result.cSize < winnerResult->cSize) { //better compression ratio, just needs to be feasible - int feas; - if(loopDurationC < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_compressOnly, BMK_iterMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres = benchres2; - } - } - if(loopDurationD < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_decodeOnly, BMK_iterMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres.result.dSpeed = benchres2.result.dSpeed; - } - } - *resultPtr = benchres.result; - - feas = uncertainFeasibility(uncertaintyConstantC, uncertaintyConstantD, target, &(benchres.result)); - if(feas == 0) { // uncertain feasibility - if(loopDurationC < TIMELOOP_NANOSEC) { - BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_compressOnly, BMK_timeMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres.result.cSpeed = benchres2.result.cSpeed; - } - } - if(loopDurationD < TIMELOOP_NANOSEC) { - BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_decodeOnly, BMK_timeMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres.result.dSpeed = benchres2.result.dSpeed; - } - } - *resultPtr = benchres.result; - return feasible(benchres.result, target); - } else { //feas = 1 or -1 map to 1, 0 respectively - return (feas + 1) >> 1; //relies on INFEASIBLE_RESULT == 0, FEASIBLE_RESULT == 1 - } - } else if (benchres.result.cSize == winnerResult->cSize) { //equal ratio, needs to be better than winner in cSpeed/ dSpeed / cMem - int feas; - if(loopDurationC < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_compressOnly, BMK_iterMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres = benchres2; - } - } - if(loopDurationD < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_decodeOnly, BMK_iterMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres.result.dSpeed = benchres2.result.dSpeed; - } - } - feas = uncertainFeasibility(uncertaintyConstantC, uncertaintyConstantD, target, &(benchres.result)); - if(feas == 0) { // uncertain feasibility - if(loopDurationC < TIMELOOP_NANOSEC) { - BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_compressOnly, BMK_timeMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres.result.cSpeed = benchres2.result.cSpeed; - } - } - if(loopDurationD < TIMELOOP_NANOSEC) { - BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_decodeOnly, BMK_timeMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres.result.dSpeed = benchres2.result.dSpeed; - } - } - - *resultPtr = benchres.result; - return feasible(benchres.result, target) && objective_lt(*winnerResult, benchres.result); - } else if (feas == 1) { //no need to check feasibility compares (maybe only it is chosen as a winner) - int btw = uncertainComparison(uncertaintyConstantC, uncertaintyConstantD, &(benchres.result), winnerResult); - if(btw == -1) { - return INFEASIBLE_RESULT; - } else { //possibly better, benchmark and find out - benchres = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_both, BMK_timeMode, 1); - *resultPtr = benchres.result; - return objective_lt(*winnerResult, benchres.result); - } - } else { //feas == -1 - return INFEASIBLE_RESULT; //infeasible - } - } else { - return INFEASIBLE_RESULT; //infeasible - } - } else { - return ERROR_RESULT; //BMK error - } -} - -//same as before, but +/-? -//alternative, just return comparison result, leave caller to worry about feasibility. -//have version of benchMemAdvanced which takes in dstBuffer/cap as well? -//(motivation: repeat tests (maybe just on decompress) don't need further compress runs) -static int infeasibleBench(BMK_result_t* resultPtr, - const void* const * const srcPtrs, size_t const * const srcSizes, - void** const dstPtrs, size_t* dstCapacityToSizes, U32 const nbBlocks, - void* dictBuffer, const size_t dictBufferSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, +/* variation between 2nd run and full second bmk */ +#define VARIANCE 1.1 +static int allBench(BMK_result_t* resultPtr, + buffers_t buf, contexts_t ctx, const ZSTD_compressionParameters cParams, const constraint_t target, - BMK_result_t* winnerResult) { + BMK_result_t* winnerResult, int feas) { BMK_return_t benchres; - BMK_result_t resultMin, resultMax; + BMK_result_t resultMax; U64 loopDurationC = 0, loopDurationD = 0; double uncertaintyConstantC, uncertaintyConstantD; double winnerRS; - size_t srcSize = 0; - U32 i; - benchres = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_both, BMK_iterMode, 1); - for(i = 0; i < nbBlocks; i++) { - srcSize += srcSizes[i]; - } - BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); - winnerRS = resultScore(*winnerResult, srcSize, target); + /* initial benchmarking, gives exact ratio and memory, warms up future runs */ + benchres = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_both, BMK_iterMode, 1); + winnerRS = resultScore(*winnerResult, buf.srcSize, target); DEBUGOUTPUT("WinnerScore: %f\n ", winnerRS); - if(!benchres.error) { - *resultPtr = benchres.result; - if(eqZero(benchres.result.cSpeed)) { - loopDurationC = 0; - uncertaintyConstantC = 2; - } else { - loopDurationC = (U64)((double)(srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); - uncertaintyConstantC = MIN((loopDurationC + (double)(2 * g_clockGranularity)/loopDurationC * 1.1), 3); //.02 seconds - } + if(benchres.error) { + DEBUGOUTPUT("Benchmarking failed\n"); + return ERROR_RESULT; + } + *resultPtr = benchres.result; - if(eqZero(benchres.result.dSpeed)) { - loopDurationD = 0; - uncertaintyConstantD = 2; - } else { - loopDurationD = (U64)((double)(srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); - uncertaintyConstantD = MIN((loopDurationD + (double)(2 * g_clockGranularity)/loopDurationD) * 1.1 , 3); //.02 seconds - } + /* calculate uncertainty in compression / decompression runs */ + if(eqZero(benchres.result.cSpeed)) { + loopDurationC = 0; + uncertaintyConstantC = 3; + } else { + loopDurationC = (U64)((double)(buf.srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); + uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC) * VARIANCE; + } + if(eqZero(benchres.result.dSpeed)) { + loopDurationD = 0; + uncertaintyConstantD = 3; + } else { + loopDurationD = (U64)((double)(buf.srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); + uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD) * VARIANCE; + } - if(loopDurationC < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_compressOnly, BMK_iterMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres = benchres2; - } - } - if(loopDurationD < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_decodeOnly, BMK_iterMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres.result.dSpeed = benchres2.result.dSpeed; - } - } - *resultPtr = benchres.result; + /* anything with worse ratio in feas is definitely worse, discard */ + if(feas && benchres.result.cSize < winnerResult->cSize) { + return WORSE_RESULT; + } - /* benchres's certainty range. */ - resultMax = benchres.result; - resultMin = benchres.result; - resultMax.cSpeed *= uncertaintyConstantC; - resultMax.dSpeed *= uncertaintyConstantD; - resultMin.cSpeed /= uncertaintyConstantC; - resultMin.dSpeed /= uncertaintyConstantD; - if (winnerRS > resultScore(resultMax, srcSize, target)) { - return INFEASIBLE_RESULT; - } else { - if(loopDurationC < TIMELOOP_NANOSEC) { - BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_compressOnly, BMK_timeMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres.result.cSpeed = benchres2.result.cSpeed; - } - } - if(loopDurationD < TIMELOOP_NANOSEC) { - BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_decodeOnly, BMK_timeMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres.result.dSpeed = benchres2.result.dSpeed; - } - } - *resultPtr = benchres.result; - return (resultScore(benchres.result, srcSize, target) > winnerRS); + /* second run, if first run is too short, gives approximate cSpeed + dSpeed */ + if(loopDurationC < TIMELOOP_NANOSEC / 10) { + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_compressOnly, BMK_iterMode, 1); + if(benchres2.error) { + return ERROR_RESULT; } + benchres = benchres2; + } + if(loopDurationD < TIMELOOP_NANOSEC / 10) { + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_decodeOnly, BMK_iterMode, 1); + if(benchres2.error) { + return ERROR_RESULT; + } + benchres.result.dSpeed = benchres2.result.dSpeed; + } *resultPtr = benchres.result; - } else { - return ERROR_RESULT; //BMK error + + /* optimistic assumption of benchres.result */ + resultMax = benchres.result; + resultMax.cSpeed *= uncertaintyConstantC; + resultMax.dSpeed *= uncertaintyConstantD; + + /* disregard infeasible results in feas mode */ + /* disregard if resultMax < winner in infeas mode */ + if((feas && !feasible(resultMax, target)) || + (!feas && (winnerRS > resultScore(resultMax, buf.srcSize, target)))) { + return WORSE_RESULT; + } + + /* Final full run if estimates are unclear */ + if(loopDurationC < TIMELOOP_NANOSEC) { + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_compressOnly, BMK_timeMode, 1); + if(benchres2.error) { + return ERROR_RESULT; + } + benchres.result.cSpeed = benchres2.result.cSpeed; + } + + if(loopDurationD < TIMELOOP_NANOSEC) { + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_decodeOnly, BMK_timeMode, 1); + if(benchres2.error) { + return ERROR_RESULT; + } + benchres.result.dSpeed = benchres2.result.dSpeed; + } + + *resultPtr = benchres.result; + + /* compare by resultScore when in infeas */ + /* compare by compareResultLT when in feas */ + if((!feas && (resultScore(benchres.result, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target))) || + (feas && (compareResultLT(*winnerResult, benchres.result, target, buf.srcSize))) ) { + return BETTER_RESULT; + } else { + return WORSE_RESULT; } } /* wrap feasibleBench w/ memotable */ #define INFEASIBLE_THRESHOLD 200 -static int feasibleBenchMemo(BMK_result_t* resultPtr, - const void* srcBuffer, const size_t srcSize, - void* dstBuffer, const size_t dstSize, - void* dictBuffer, const size_t dictSize, - const size_t* fileSizes, const size_t nbFiles, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, - const ZSTD_compressionParameters cParams, - const constraint_t target, - BMK_result_t* winnerResult, U8* memoTable, - const U32* varyParams, const int varyLen) { - - const size_t memind = memoTableInd(&cParams, varyParams, varyLen); - if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { - return INFEASIBLE_RESULT; - } else { - const size_t blockSize = g_blockSize ? g_blockSize : srcSize; - U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + (U32)nbFiles; - const void ** const srcPtrs = (const void** const)malloc(maxNbBlocks * sizeof(void*)); - size_t* const srcSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); - void ** const dstPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); - size_t* const dstCapacities = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); - U32 nbBlocks; - int res; - - if(!srcPtrs || !srcSizes || !dstPtrs || !dstCapacities) { - free(srcPtrs); - free(srcSizes); - free(dstPtrs); - free(dstCapacities); - DISPLAY("Allocation Error\n"); - return ERROR_RESULT; - } - - { - const char* srcPtr = (const char*)srcBuffer; - char* dstPtr = (char*)dstBuffer; - size_t dstSizeRemaining = dstSize; - U32 fileNb; - for (nbBlocks=0, fileNb=0; fileNb dstSizeRemaining) { - DEBUGOUTPUT("Warning: dstSize too small to benchmark completely \n"); - remaining = dstSizeRemaining; - dstSizeRemaining = 0; - } else { - dstSizeRemaining -= remaining; - } - for ( ; nbBlocks= INFEASIBLE_THRESHOLD) { - return INFEASIBLE_RESULT; //see feasibleBenchMemo for concerns - } else { - const size_t blockSize = g_blockSize ? g_blockSize : srcSize; - U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + (U32)nbFiles; - const void ** const srcPtrs = (const void** const)malloc(maxNbBlocks * sizeof(void*)); - size_t* const srcSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); - void ** const dstPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); - size_t* const dstCapacities = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); - U32 nbBlocks; - int res; + int res; - if(!srcPtrs || !srcSizes || !dstPtrs || !dstCapacities) { - free(srcPtrs); - free(srcSizes); - free(dstPtrs); - free(dstCapacities); - DISPLAY("Allocation Error\n"); - return ERROR_RESULT; - } + if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { return WORSE_RESULT; } - { - const char* srcPtr = (const char*)srcBuffer; - char* dstPtr = (char*)dstBuffer; - size_t dstSizeRemaining = dstSize; - U32 fileNb; - for (nbBlocks=0, fileNb=0; fileNb dstSizeRemaining) { - DEBUGOUTPUT("Warning: dstSize too small to benchmark completely \n"); - remaining = dstSizeRemaining; - dstSizeRemaining = 0; - } else { - dstSizeRemaining -= remaining; - } - for ( ; nbBlocks (hopefully) feasible */ - /* when nothing is found, this garbages part 2. */ { - winnerInfo_t bestFeasible1; /* uses feasibleBench Metric */ - - //init these params - bestFeasible1.params = cparam; - bestFeasible1.result.cSpeed = 0; - bestFeasible1.result.dSpeed = 0; - bestFeasible1.result.cMem = (size_t)-1; - bestFeasible1.result.cSize = (size_t)-1; + winnerInfo_t bestFeasible1 = initWinnerInfo(cparam); DISPLAY("Climb Part 1\n"); while(better) { - int i, d; + int i, dist, offset; better = 0; DEBUGOUTPUT("Start\n"); cparam = winnerInfo.params; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, buf.srcSize); candidateInfo.params = cparam; //all dist-1 targets //if we early end this, we should also randomize the order these are picked. for(i = 0; i < varLen; i++) { - paramVaryOnce(varArray[i], 1, &candidateInfo.params); /* +1 */ - candidateInfo.params = sanitizeParams(candidateInfo.params); - //evaluate - if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { - //if(cParamValid(candidateInfo.params)) { - int res = infeasibleBenchMemo(&candidateInfo.result, - srcBuffer, srcSize, - dstBuffer, dstSize, - dictBuffer, dictSize, - fileSizes, nbFiles, - ctx, dctx, - candidateInfo.params, target, &winnerInfo.result, memoTable, - varArray, varLen); - if(res == FEASIBLE_RESULT) { /* synonymous with better when called w/ infeasibleBM */ - winnerInfo = candidateInfo; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); - better = 1; - if(feasible(candidateInfo.result, target) && objective_lt(bestFeasible1.result, winnerInfo.result)) { - bestFeasible1 = winnerInfo; - } - } - } - candidateInfo.params = cparam; - paramVaryOnce(varArray[i], -1, &candidateInfo.params); /* -1 */ - candidateInfo.params = sanitizeParams(candidateInfo.params); - //evaluate - if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { - //if(cParamValid(candidateInfo.params)) { - int res = infeasibleBenchMemo(&candidateInfo.result, - srcBuffer, srcSize, - dstBuffer, dstSize, - dictBuffer, dictSize, - fileSizes, nbFiles, - ctx, dctx, - candidateInfo.params, target, &winnerInfo.result, memoTable, - varArray, varLen); - if(res == FEASIBLE_RESULT) { - winnerInfo = candidateInfo; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); - better = 1; - if(feasible(candidateInfo.result, target) && objective_lt(bestFeasible1.result, winnerInfo.result)) { - bestFeasible1 = winnerInfo; + for(offset = -1; offset <= 1; offset += 2) { + candidateInfo.params = cparam; + paramVaryOnce(varArray[i], offset, &candidateInfo.params); /* +1 */ + candidateInfo.params = sanitizeParams(candidateInfo.params); + //evaluate + if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { + int res = benchMemo(&candidateInfo.result, + buf, ctx, + candidateInfo.params, target, &winnerInfo.result, memoTable, + varArray, varLen, feas); + if(res == BETTER_RESULT) { /* synonymous with better when called w/ infeasibleBM */ + winnerInfo = candidateInfo; + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, buf.srcSize); + better = 1; + if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) { + bestFeasible1 = winnerInfo; + } } } } @@ -1966,235 +1533,255 @@ static winnerInfo_t climbOnce(const constraint_t target, if(better) { continue; } - //if 'better' enough, skip further parameter search, center there? - //possible improvement - guide direction here w/ knowledge rather than completely random variation. - for(d = 2; d < varLen + 2; d++) { /* varLen is # dimensions */ + + for(dist = 2; dist < varLen + 2; dist++) { /* varLen is # dimensions */ for(i = 0; i < 2 * varLen + 2; i++) { int res; candidateInfo.params = cparam; /* param error checking already done here */ - paramVariation(&candidateInfo.params, varArray, varLen, d); - res = infeasibleBenchMemo(&candidateInfo.result, - srcBuffer, srcSize, - dstBuffer, dstSize, - dictBuffer, dictSize, - fileSizes, nbFiles, - ctx, dctx, - candidateInfo.params, target, &winnerInfo.result, memoTable, - varArray, varLen); - if(res == FEASIBLE_RESULT) { /* synonymous with better in this case*/ + paramVariation(&candidateInfo.params, varArray, varLen, dist); + res = benchMemo(&candidateInfo.result, + buf, ctx, + candidateInfo.params, target, &winnerInfo.result, memoTable, + varArray, varLen, feas); + if(res == BETTER_RESULT) { /* synonymous with better in this case*/ winnerInfo = candidateInfo; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, buf.srcSize); better = 1; - if(feasible(candidateInfo.result, target) && objective_lt(bestFeasible1.result, winnerInfo.result)) { + if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) { bestFeasible1 = winnerInfo; } } } if(better) { - continue; + break; } } - //bias to test previous delta? - //change cparam -> candidate before restart + + if(!better) { //infeas -> feas -> stop. + if(feas) { return winnerInfo; } + + feas = 1; + better = 1; + winnerInfo = bestFeasible1; /* note with change, bestFeasible may not necessarily be feasible, but if one has been benchmarked, it will be. */ + DISPLAY("Climb Part 2\n"); + } } winnerInfo = bestFeasible1; } - //break out if no feasible. - if(winnerInfo.result.cMem == (U32)-1) { - DEBUGOUTPUT("No Feasible Found\n"); - return winnerInfo; - } - DISPLAY("Climb Part 2\n"); - - better = 1; - /* feasible -> best feasible (hopefully) */ - { - while(better) { - int i, d; - better = 0; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); - //all dist-1 targets - cparam = winnerInfo.params; - candidateInfo.params = cparam; - for(i = 0; i < varLen; i++) { - paramVaryOnce(varArray[i], 1, &candidateInfo.params); - candidateInfo.params = sanitizeParams(candidateInfo.params); - - //evaluate - if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { - //if(cParamValid(candidateInfo.params)) { - int res = feasibleBenchMemo(&candidateInfo.result, - srcBuffer, srcSize, - dstBuffer, dstSize, - dictBuffer, dictSize, - fileSizes, nbFiles, - ctx, dctx, - candidateInfo.params, target, &winnerInfo.result, memoTable, - varArray, varLen); - if(res == FEASIBLE_RESULT) { - winnerInfo = candidateInfo; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); - better = 1; - } - } - candidateInfo.params = cparam; - paramVaryOnce(varArray[i], -1, &candidateInfo.params); - candidateInfo.params = sanitizeParams(candidateInfo.params); - //evaluate - if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { - int res = feasibleBenchMemo(&candidateInfo.result, - srcBuffer, srcSize, - dstBuffer, dstSize, - dictBuffer, dictSize, - fileSizes, nbFiles, - ctx, dctx, - candidateInfo.params, target, &winnerInfo.result, memoTable, - varArray, varLen); - if(res == FEASIBLE_RESULT) { - winnerInfo = candidateInfo; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); - better = 1; - } - } - } - //if 'better' enough, skip further parameter search, center there? - //possible improvement - guide direction here w/ knowledge rather than completely random variation. - for(d = 2; d < varLen + 2; d++) { /* varLen is # dimensions */ - for(i = 0; i < 2 * varLen + 2; i++) { - int res; - candidateInfo.params = cparam; - /* param error checking already done here */ - paramVariation(&candidateInfo.params, varArray, varLen, d); //info candidateInfo.params is garbage, this is too. - res = feasibleBenchMemo(&candidateInfo.result, - srcBuffer, srcSize, - dstBuffer, dstSize, - dictBuffer, dictSize, - fileSizes, nbFiles, - ctx, dctx, - candidateInfo.params, target, &winnerInfo.result, memoTable, - varArray, varLen); - if(res == FEASIBLE_RESULT) { - winnerInfo = candidateInfo; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); - better = 1; - } - } - if(better) { - continue; - } - } - //bias to test previous delta? - //change cparam -> candidate before restart - } - } - return winnerInfo; } //optimizeForSize but with fixed strategy //place to configure/filter out strategy specific parameters. -//need args for all buffers and parameter stuff -//sanitization here. //flexible parameters: iterations of (failed?) climbing (or if we do non-random, maybe this is when everything is close to visitied) //weight more on visit for bad results, less on good results/more on later results / ones with more failures. //allocate memoTable here. //only real use for paramTarget is to get the fixed values, right? +//maybe allow giving it a first init? static winnerInfo_t optimizeFixedStrategy( - const void* srcBuffer, const size_t srcSize, - void* dstBuffer, const size_t dstSize, - void* dictBuffer, const size_t dictSize, - const size_t* fileSizes, const size_t nbFiles, - const constraint_t target, ZSTD_compressionParameters paramTarget, - const ZSTD_strategy strat, const U32* varArray, const int varLen, U8* memoTable) { + buffers_t buf, contexts_t ctx, + const constraint_t target, ZSTD_compressionParameters paramTarget, + const ZSTD_strategy strat, + const varInds_t* varArray, const int varLen, + U8* memoTable, const int tries) { int i = 0; - U32* varNew = malloc(sizeof(U32) * varLen); - int varLenNew = sanitizeVarArray(varLen, varArray, varNew, strat); + varInds_t varNew[NUM_PARAMS]; + int varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat); ZSTD_compressionParameters init; - ZSTD_CCtx* ctx = ZSTD_createCCtx(); - ZSTD_DCtx* dctx = ZSTD_createDCtx(); winnerInfo_t winnerInfo, candidateInfo; - winnerInfo.result.cSpeed = 0; - winnerInfo.result.dSpeed = 0; - winnerInfo.result.cMem = (size_t)(-1LL); - winnerInfo.result.cSize = (size_t)(-1LL); + winnerInfo = initWinnerInfo(emptyParams()); /* so climb is given the right fixed strategy */ paramTarget.strategy = strat; /* to pass ZSTD_checkCParams */ - //needs to happen after memoTableInit as that assumes 0 = undefined. cParamZeroMin(¶mTarget); init = paramTarget; - - if(!ctx || !dctx || !memoTable || !varNew) { - DISPLAY("NOT ENOUGH MEMORY ! ! ! \n"); - goto _cleanUp; - } - - while(i < 10) { //make i adjustable (user input?) depending on how much time they have. + while(i < tries) { //make i adjustable (user input?) depending on how much time they have. DEBUGOUTPUT("Restart\n"); //look into improving this to maximize distance from searched infeasible stuff / towards promising regions? randomConstrainedParams(&init, varNew, varLenNew, memoTable); - candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, fileSizes, nbFiles, ctx, dctx, init); - if(objective_lt(winnerInfo.result, candidateInfo.result)) { + candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, buf, ctx, init); + if(compareResultLT(winnerInfo.result, candidateInfo.result, target, buf.srcSize)) { winnerInfo = candidateInfo; - DISPLAY("New Winner: "); - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, buf.srcSize); i = 0; } i++; } - -_cleanUp: - ZSTD_freeCCtx(ctx); - ZSTD_freeDCtx(dctx); - free(varNew); return winnerInfo; } -static int BMK_loadFiles(void* buffer, size_t bufferSize, - size_t* fileSizes, const char* const * const fileNamesTable, - unsigned nbFiles) +static void freeBuffers(buffers_t b) { + if(b.srcPtrs != NULL) { + free(b.srcPtrs[0]); + } + free(b.srcPtrs); + free(b.srcSizes); + + if(b.dstPtrs != NULL) { + free(b.dstPtrs[0]); + } + free(b.dstPtrs); + free(b.dstCapacities); + free(b.dstSizes); + + if(b.resPtrs != NULL) { + free(b.resPtrs[0]); + } + free(b.resPtrs); +} + +/* allocates buffer's arguments. returns success / failuere */ +static int initBuffers(buffers_t* buff, const char* const * const fileNamesTable, + size_t nbFiles) { - size_t pos = 0, totalSize = 0; - unsigned n; - for (n=0; nsrcPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); + buff->srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + + buff->dstPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); + buff->dstCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + buff->dstSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + + buff->resPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); + buff->resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + + if(!buff->srcPtrs || !buff->srcSizes || !buff->dstPtrs || !buff->dstCapacities || !buff->dstCapacities || !buff->resPtrs || !buff->resSizes) { + DISPLAY("alloc error\n"); + freeBuffers(*buff); + return 1; + } + + buff->srcPtrs[0] = malloc(benchedSize); + buff->dstPtrs[0] = malloc(ZSTD_compressBound(benchedSize) + (maxNbBlocks * 1024)); + buff->resPtrs[0] = malloc(benchedSize); + + if(!buff->srcPtrs[0] || !buff->dstPtrs[0] || !buff->resPtrs[0]) { + DISPLAY("alloc error\n"); + freeBuffers(*buff); + return 1; + } + + for(n = 0; n < nbFiles; n++) { FILE* f; U64 fileSize = UTIL_getFileSize(fileNamesTable[n]); if (UTIL_isDirectory(fileNamesTable[n])) { DISPLAY("Ignoring %s directory... \n", fileNamesTable[n]); - fileSizes[n] = 0; continue; } if (fileSize == UTIL_FILESIZE_UNKNOWN) { DISPLAY("Cannot evaluate size of %s, ignoring ... \n", fileNamesTable[n]); - fileSizes[n] = 0; continue; } f = fopen(fileNamesTable[n], "rb"); if (f==NULL) { - DISPLAY("impossible to open file %s", fileNamesTable[n]); + DISPLAY("impossible to open file %s\n", fileNamesTable[n]); + freeBuffers(*buff); + fclose(f); return 10; } + DISPLAY("Loading %s... \r", fileNamesTable[n]); - if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */ - { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f); + + if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, nbFiles=n; /* buffer too small - stop after this file */ + { + char* buffer = (char*)(buff->srcPtrs[0]); + size_t const readSize = fread((buffer)+pos, 1, (size_t)fileSize, f); + size_t blocked = 0; + while(blocked < readSize) { + buff->srcPtrs[blockNb] = (buffer) + (pos + blocked); + buff->srcSizes[blockNb] = blockSize; + blocked += blockSize; + blockNb++; + } + if(readSize > 0) { buff->srcSizes[blockNb - 1] = ((readSize - 1) % blockSize) + 1; } + if (readSize != (size_t)fileSize) { DISPLAY("could not read %s", fileNamesTable[n]); - return 11; + freeBuffers(*buff); + fclose(f); + return 1; } - pos += readSize; } - fileSizes[n] = (size_t)fileSize; - totalSize += (size_t)fileSize; + + pos += readSize; + + } fclose(f); } - if (totalSize == 0) { DISPLAY("\nno data to bench\n"); return 12; } + buff->dstCapacities[0] = ZSTD_compressBound(buff->srcSizes[0]); + buff->dstSizes[0] = buff->dstCapacities[0]; + buff->resSizes[0] = buff->srcSizes[0]; + + for(n = 1; n < blockNb; n++) { + buff->dstPtrs[n] = ((char*)buff->dstPtrs[n-1]) + buff->dstCapacities[n-1]; + buff->resPtrs[n] = ((char*)buff->resPtrs[n-1]) + buff->resSizes[n-1]; + buff->dstCapacities[n] = ZSTD_compressBound(buff->srcSizes[n]); + buff->dstSizes[n] = buff->dstCapacities[n]; + buff->resSizes[n] = buff->srcSizes[n]; + } + buff->srcSize = pos; + buff->nbBlocks = blockNb; + + if (pos == 0) { DISPLAY("\nno data to bench\n"); return 1; } + + return 0; +} + +static void freeContexts(contexts_t ctx) { + free(ctx.dictBuffer); + ZSTD_freeCCtx(ctx.cctx); + ZSTD_freeDCtx(ctx.dctx); +} + +static int initContexts(contexts_t* ctx, const char* dictFileName) { + FILE* f; + size_t readSize; + ctx->cctx = ZSTD_createCCtx(); + ctx->dctx = ZSTD_createDCtx(); + if(dictFileName == NULL) { + ctx->dictSize = 0; + ctx->dictBuffer = NULL; + return 0; + } + ctx->dictSize = UTIL_getFileSize(dictFileName); + ctx->dictBuffer = malloc(ctx->dictSize); + + f = fopen(dictFileName, "rb"); + + if(!f) { + DISPLAY("unable to open file\n"); + fclose(f); + freeContexts(*ctx); + return 1; + } + + if(ctx->dictSize > 64 MB || !(ctx->dictBuffer)) { + DISPLAY("dictionary too large\n"); + fclose(f); + freeContexts(*ctx); + return 1; + } + readSize = fread(ctx->dictBuffer, 1, ctx->dictSize, f); + if(readSize != ctx->dictSize) { + DISPLAY("unable to read file\n"); + fclose(f); + freeContexts(*ctx); + return 1; + } return 0; } @@ -2228,107 +1815,98 @@ static int nextStrategy(const int currentStrategy, const int bestStrategy) { } } +static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZSTD_compressionParameters mask) { + base.windowLog = mask.windowLog ? mask.windowLog : base.windowLog; + base.chainLog = mask.chainLog ? mask.chainLog : base.chainLog; + base.hashLog = mask.hashLog ? mask.hashLog : base.hashLog; + base.searchLog = mask.searchLog ? mask.searchLog : base.searchLog; + base.searchLength = mask.searchLength ? mask.searchLength : base.searchLength; + base.targetLength = mask.targetLength ? mask.targetLength : base.targetLength; + base.strategy = mask.strategy ? mask.strategy : base.strategy; + return base; +} + +#define MAX_TRIES 8 //optimize fixed strategy. static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel) { - size_t benchedSize; - void* origBuff = NULL; - void* dictBuffer = NULL; - size_t dictBufferSize = 0; - U32 varArray [NUM_PARAMS]; + varInds_t varArray [NUM_PARAMS]; int ret = 0; - size_t* fileSizes = calloc(sizeof(size_t),nbFiles); const int varLen = variableParams(paramTarget, varArray); + winnerInfo_t winner = initWinnerInfo(emptyParams()); U8** allMT = NULL; - g_winner.result.cSize = (size_t)-1; + size_t k; + size_t maxBlockSize = 0; + contexts_t ctx; + buffers_t buf; + /* Init */ if(!cParamValid(paramTarget)) { - return 10; + return 1; } /* load dictionary*/ - if (dictFileName != NULL) { - U64 const dictFileSize = UTIL_getFileSize(dictFileName); - if (dictFileSize > 64 MB) { - DISPLAY("dictionary file %s too large", dictFileName); - ret = 10; - goto _cleanUp; - } - dictBufferSize = (size_t)dictFileSize; - dictBuffer = malloc(dictBufferSize); - if (dictBuffer==NULL) { - DISPLAY("not enough memory for dictionary (%u bytes)", - (U32)dictBufferSize); - ret = 11; - goto _cleanUp; + if(initBuffers(&buf, fileNamesTable, nbFiles)) { + DISPLAY("unable to load files\n"); + return 1; + } - } - - { - int errorCode = BMK_loadFiles(dictBuffer, dictBufferSize, &dictBufferSize, &dictFileName, 1); - if(errorCode) { - ret = errorCode; - goto _cleanUp; - } - } + if(initContexts(&ctx, dictFileName)) { + DISPLAY("unable to load dictionary\n"); + freeBuffers(buf); + return 2; } - /* Fill input buffer */ if(nbFiles == 1) { DISPLAY("Loading %s... \r", fileNamesTable[0]); } else { DISPLAY("Loading %lu Files... \r", (unsigned long)nbFiles); } - { - U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles); - int ec; - unsigned i; - benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3; - origBuff = malloc(benchedSize); - if(!origBuff || !fileSizes) { - DISPLAY("Not enough memory for stuff\n"); - ret = 1; - goto _cleanUp; - } - ec = BMK_loadFiles(origBuff, benchedSize, fileSizes, fileNamesTable, (U32)nbFiles); - if(ec) { - DISPLAY("Error Loading Files"); - ret = ec; - goto _cleanUp; - } - benchedSize = 0; - for(i = 0; i < nbFiles; i++) { - benchedSize += fileSizes[i]; - } - origBuff = realloc(origBuff, benchedSize); + for(k = 0; k < buf.nbBlocks; k++) { + maxBlockSize = MAX(buf.srcSizes[k], maxBlockSize); } - allMT = memoTableInitAll(paramTarget, target, varArray, varLen, benchedSize); + /* if strategy is fixed, only init that part of memotable */ + if(paramTarget.strategy) { + varInds_t varNew[NUM_PARAMS]; + int varLenNew = sanitizeVarArray(varNew, varLen, varArray, paramTarget.strategy); + allMT = calloc(sizeof(U8), (ZSTD_btultra + 1)); + if(allMT == NULL) { + ret = 57; + goto _cleanUp; + } + + allMT[paramTarget.strategy] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew)); + + if(allMT[paramTarget.strategy] == NULL) { + ret = 58; + goto _cleanUp; + } + + memoTableInit(allMT[paramTarget.strategy], paramTarget, target, varNew, varLenNew, maxBlockSize); + } else { + allMT = memoTableInitAll(paramTarget, target, varArray, varLen, maxBlockSize); + } + + if(!allMT) { + DISPLAY("MemoTable Init Error\n"); ret = 2; goto _cleanUp; } if(cLevel) { - BMK_result_t candidate; - const size_t blockSize = g_blockSize ? g_blockSize : benchedSize; - ZSTD_CCtx* const ctx = ZSTD_createCCtx(); - ZSTD_DCtx* const dctx = ZSTD_createDCtx(); - ZSTD_compressionParameters const CParams = ZSTD_getCParams(cLevel, blockSize, dictBufferSize); - if(BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, (U32)nbFiles, ctx, dctx, CParams)) { - ZSTD_freeCCtx(ctx); - ZSTD_freeDCtx(dctx); + winner.params = ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize); + if(BMK_benchParam(&winner.result, buf, ctx, winner.params)) { ret = 3; goto _cleanUp; } - target.cSpeed = (U32)candidate.cSpeed; //Maybe have a small bit of slack here, like x.99? - BMK_printWinner(stdout, cLevel, candidate, CParams, benchedSize); - - ZSTD_freeCCtx(ctx); - ZSTD_freeDCtx(dctx); + target.cSpeed = (U32)winner.result.cSpeed; //Maybe have a small bit of slack here, like x.99? + g_targetConstraints = target; + BMK_printWinner(stdout, cLevel, winner.result, winner.params, buf.srcSize); } g_targetConstraints = target; @@ -2340,106 +1918,40 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } else { DISPLAY("optimizing for %lu Files", (unsigned long)nbFiles); } - if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed / 1000000); } - if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed / 1000000); } - if(target.cMem != (U32)-1) { DISPLAY(" - limit memory %u MB", target.cMem / 1000000); } + if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed >> 20); } + if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed >> 20); } + if(target.cMem != (U32)-1) { DISPLAY(" - limit memory %u MB", target.cMem >> 20); } + DISPLAY("\n"); findClockGranularity(); - { ZSTD_CCtx* const ctx = ZSTD_createCCtx(); - ZSTD_DCtx* const dctx = ZSTD_createDCtx(); - winnerInfo_t winner; - U32 varNew[NUM_PARAMS]; - const size_t blockSize = g_blockSize ? g_blockSize : benchedSize; - U32 const maxNbBlocks = (U32) ((benchedSize + (blockSize-1)) / blockSize) + 1; - const size_t maxCompressedSize = ZSTD_compressBound(benchedSize) + (maxNbBlocks * 1024); - void* compressedBuffer = malloc(maxCompressedSize); - - /* init */ - if (ctx==NULL) { DISPLAY("\n ZSTD_createCCtx error \n"); free(origBuff); return 14;} - if(compressedBuffer==NULL) { DISPLAY("\n Allocation Error \n"); free(origBuff); free(ctx); return 15; } - memset(&winner, 0, sizeof(winner)); - winner.result.cSize = (size_t)(-1); - + { + varInds_t varNew[NUM_PARAMS]; /* find best solution from default params */ { /* strategy selection */ const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); DEBUGOUTPUT("Strategy Selection\n"); - if(varLen == NUM_PARAMS && paramTarget.strategy == 0) { /* no variable based constraints */ + if(paramTarget.strategy == 0) { /* no variable based constraints */ BMK_result_t candidate; - int feas = 0, i; + int i; for (i=1; i<=maxSeeds; i++) { - ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, dictBufferSize); - int ec = BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, (U32)nbFiles, ctx, dctx, CParams); - BMK_printWinner(stdout, i, candidate, CParams, benchedSize); - - if(!ec) { - if(feas) { - if(feasible(candidate, relaxTarget(target)) && objective_lt(winner.result, candidate)) { - winner.result = candidate; - winner.params = CParams; - } - } else { - if(feasible(candidate, relaxTarget(target))) { - feas = 1; - winner.result = candidate; - winner.params = CParams; - - } else { - if(resultScore(candidate, benchedSize, target) > resultScore(winner.result, benchedSize, target)) { - winner.result = candidate; - winner.params = CParams; - } - } - } - } - } //best, -1, +1, ..., - - } else if (paramTarget.strategy == 0) { //constrained - int feas = 0, i, j; - for(j = 1; j < 10; j++) { - for(i = 1; i <= maxSeeds; i++) { - int varLenNew = sanitizeVarArray(varLen, varArray, varNew, i); - ZSTD_compressionParameters candidateParams = paramTarget; - BMK_result_t candidate; - int ec; - randomConstrainedParams(&candidateParams, varNew, varLenNew, allMT[i]); - cParamZeroMin(&candidateParams); - candidateParams = sanitizeParams(candidateParams); - ec = BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, (U32)nbFiles, ctx, dctx, candidateParams); - - if(!ec) { - if(feas) { - if(feasible(candidate, relaxTarget(target)) && objective_lt(winner.result, candidate)) { - winner.result = candidate; - winner.params = candidateParams; - BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); - } - } else { - if(feasible(candidate, relaxTarget(target))) { - feas = 1; - winner.result = candidate; - winner.params = candidateParams; - BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); - - } else { - if(resultScore(candidate, benchedSize, target) > resultScore(winner.result, benchedSize, target)) { - winner.result = candidate; - winner.params = candidateParams; - BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); - } - } - } - } + int ec; + ZSTD_compressionParameters CParams = ZSTD_getCParams(i, maxBlockSize, ctx.dictSize); + CParams = maskParams(CParams, paramTarget); + ec = BMK_benchParam(&candidate, buf, ctx, CParams); + BMK_printWinner(stdout, i, candidate, CParams, buf.srcSize); + if(!ec && compareResultLT(winner.result, candidate, relaxTarget(target), buf.srcSize)) { + winner.result = candidate; + winner.params = CParams; } } } } - BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, buf.srcSize); BMK_translateAdvancedParams(winner.params); DEBUGOUTPUT("Real Opt\n"); /* start 'real' tests */ @@ -2447,55 +1959,51 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ int bestStrategy = (int)winner.params.strategy; if(paramTarget.strategy == 0) { int st = (int)winner.params.strategy; + int tries = MAX_TRIES; { - int varLenNew = sanitizeVarArray(varLen, varArray, varNew, st); + int varLenNew = sanitizeVarArray(varNew, varLen, varArray, st); winnerInfo_t w1 = climbOnce(target, varNew, varLenNew, allMT[st], - origBuff, benchedSize, compressedBuffer, maxCompressedSize, dictBuffer, dictBufferSize, - fileSizes, nbFiles, ctx, dctx, winner.params); - if(objective_lt(winner.result, w1.result)) { + buf, ctx, winner.params); + if(compareResultLT(winner.result, w1.result, target, buf.srcSize)) { winner = w1; } } - while(st) { - winnerInfo_t wc = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, dictBuffer, dictBufferSize, fileSizes, nbFiles, - target, paramTarget, st, varArray, varLen, allMT[st]); + while(st && tries) { + winnerInfo_t wc = optimizeFixedStrategy(buf, ctx, target, paramTarget, + st, varArray, varLen, allMT[st], tries); DEBUGOUTPUT("StratNum %d\n", st); - if(objective_lt(winner.result, wc.result)) { + if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) { winner = wc; } //We could double back to increase search of 'better' strategies st = nextStrategy(st, bestStrategy); + tries--; } } else { - winner = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, dictBuffer, dictBufferSize, fileSizes, nbFiles, - target, paramTarget, paramTarget.strategy, varArray, varLen, allMT[paramTarget.strategy]); + winner = optimizeFixedStrategy(buf, ctx, target, paramTarget, paramTarget.strategy, + varArray, varLen, allMT[paramTarget.strategy], 10); } } /* no solution found */ if(winner.result.cSize == (size_t)-1) { + ret = 1; DISPLAY("No feasible solution found\n"); - return 1; + goto _cleanUp; } /* end summary */ - BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, buf.srcSize); BMK_translateAdvancedParams(winner.params); DISPLAY("grillParams size - optimizer completed \n"); - - /* clean up*/ - ZSTD_freeCCtx(ctx); - ZSTD_freeDCtx(dctx); - } _cleanUp: - free(fileSizes); - free(dictBuffer); + freeContexts(ctx); + freeBuffers(buf); memoTableFreeAll(allMT); - free(origBuff); return ret; } @@ -2546,16 +2054,16 @@ static int usage(const char* exename) static int usage_advanced(void) { DISPLAY( "\nAdvanced options :\n"); - DISPLAY( " -T# : set level 1 speed objective \n"); - DISPLAY( " -B# : cut input into blocks of size # (default : single block) \n"); - DISPLAY( " -i# : iteration loops [1-9](default : %i) \n", NBLOOPS); - DISPLAY( " -O# : find Optimized parameters for # MB/s compression speed (default : 0) \n"); - DISPLAY( " -S : Single run \n"); - DISPLAY( " --zstd : Single run, parameter selection same as zstdcli \n"); - DISPLAY( " -P# : generated sample compressibility (default : %.1f%%) \n", COMPRESSIBILITY_DEFAULT * 100); - DISPLAY( " -t# : Caps runtime of operation in seconds (default : %u seconds (%.1f hours)) \n", (U32)g_grillDuration_s, g_grillDuration_s / 3600); - DISPLAY( " -v : Prints Benchmarking output\n"); - DISPLAY( " -D : Next argument dictionary file\n"); + DISPLAY( " -T# : set level 1 speed objective \n"); + DISPLAY( " -B# : cut input into blocks of size # (default : single block) \n"); + DISPLAY( " -i# : iteration loops (default : %i) \n", NBLOOPS); + DISPLAY( " --optimize= : same as -O with more verbose syntax (see README.md)\n"); + DISPLAY( " -S : Single run \n"); + DISPLAY( " --zstd : Single run, parameter selection same as zstdcli \n"); + DISPLAY( " -P# : generated sample compressibility (default : %.1f%%) \n", COMPRESSIBILITY_DEFAULT * 100); + DISPLAY( " -t# : Caps runtime of operation in seconds (default : %u seconds (%.1f hours)) \n", (U32)g_grillDuration_s, g_grillDuration_s / 3600); + DISPLAY( " -v : Prints Benchmarking output\n"); + DISPLAY( " -D : Next argument dictionary file\n"); return 0; } @@ -2572,8 +2080,8 @@ int main(int argc, const char** argv) filenamesStart=0, result; const char* exename=argv[0]; - const char* input_filename = 0; - const char* dictFileName = 0; + const char* input_filename = NULL; + const char* dictFileName = NULL; U32 optimizer = 0; U32 main_pause = 0; int optimizerCLevel = 0; @@ -2591,6 +2099,9 @@ int main(int argc, const char** argv) for(i=1; i Date: Fri, 27 Jul 2018 08:49:25 -0700 Subject: [PATCH 10/55] Renaming / Style fixes --- programs/bench.c | 56 +++++++++++++++++++++++++--------------------- programs/bench.h | 7 +++--- tests/README.md | 1 - tests/paramgrill.c | 27 +++++++++++----------- 4 files changed, 48 insertions(+), 43 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 177dbe0e3..b496caf27 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -294,7 +294,7 @@ BMK_customReturn_t BMK_benchFunction( BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBlockBuffers, const size_t* srcBlockSizes, - void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, size_t* cSizes, + void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, size_t* blockResult, unsigned nbLoops) { size_t dstSize = 0; U64 totalTime; @@ -332,8 +332,8 @@ BMK_customReturn_t BMK_benchFunction( j, (U32)dstBlockCapacities[j], ZSTD_getErrorName(res)); } else if(i == nbLoops - 1) { dstSize += res; - if(cSizes != NULL) { - cSizes[j] = res; + if(blockResult != NULL) { + blockResult[j] = res; } } } @@ -358,6 +358,9 @@ void BMK_resetTimeState(BMK_timedFnState_t* r, unsigned nbSeconds) { BMK_timedFnState_t* BMK_createTimeState(unsigned nbSeconds) { BMK_timedFnState_t* r = (BMK_timedFnState_t*)malloc(sizeof(struct BMK_timeState_t)); + if(r == NULL) { + return r; + } BMK_resetTimeState(r, nbSeconds); return r; } @@ -373,7 +376,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed( BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const* const srcBlockBuffers, const size_t* srcBlockSizes, - void * const * const dstBlockBuffers, const size_t * dstBlockCapacities, size_t* dstSizes) + void * const * const dstBlockBuffers, const size_t * dstBlockCapacities, size_t* blockResults) { U64 fastest = cont->fastestTime; int completed = 0; @@ -390,7 +393,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed( } /* reinitialize capacity */ r.result = BMK_benchFunction(benchFn, benchPayload, initFn, initPayload, - blockCount, srcBlockBuffers, srcBlockSizes, dstBlockBuffers, dstBlockCapacities, dstSizes, cont->nbLoops); + blockCount, srcBlockBuffers, srcBlockSizes, dstBlockBuffers, dstBlockCapacities, blockResults, cont->nbLoops); if(r.result.error) { /* completed w/ error */ r.completed = 1; return r; @@ -718,38 +721,36 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; /* these are the blockTable parameters, just split up */ - const void ** const srcPtrs = (const void** const)malloc(maxNbBlocks * sizeof(void*)); - size_t* const srcSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); + const void ** const srcPtrs = (const void**)malloc(maxNbBlocks * sizeof(void*)); + size_t* const srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); - void ** const cPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); - size_t* const cSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); - size_t* const cCapacities = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); + void ** const cPtrs = (void**)malloc(maxNbBlocks * sizeof(void*)); + size_t* const cSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + size_t* const cCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); - void ** const resPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); - size_t* const resSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); + void ** const resPtrs = (void**)malloc(maxNbBlocks * sizeof(void*)); + size_t* const resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); BMK_timedFnState_t* timeStateCompress = BMK_createTimeState(adv->nbSeconds); BMK_timedFnState_t* timeStateDecompress = BMK_createTimeState(adv->nbSeconds); - void* compressedBuffer; const size_t maxCompressedSize = dstCapacity ? dstCapacity : ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); + + void* const internalDstBuffer = dstBuffer ? NULL : malloc(maxCompressedSize); + void* const compressedBuffer = dstBuffer ? dstBuffer : internalDstBuffer; + void* resultBuffer = malloc(srcSize); - BMK_return_t results = { { 0, 0, 0, 0 }, 0 }; - int allocationincomplete; + int allocationincomplete = !srcPtrs || !srcSizes || !cPtrs || + !cSizes || !cCapacities || !resPtrs || !resSizes || + !timeStateCompress || !timeStateDecompress || !compressedBuffer || !resultBuffer; - if(!dstCapacity) { - compressedBuffer = malloc(maxCompressedSize); - } else { - compressedBuffer = dstBuffer; - } + int parametersConflict = !dstBuffer ^ !dstCapacity; - allocationincomplete = !compressedBuffer || !resultBuffer || - !srcPtrs || !srcSizes || !cPtrs || !cSizes || !resPtrs || !resSizes; - if (!allocationincomplete) { + if (!allocationincomplete && !parametersConflict) { results = BMK_benchMemAdvancedNoAlloc(srcPtrs, srcSizes, cPtrs, cCapacities, cSizes, resPtrs, resSizes, &resultBuffer, compressedBuffer, maxCompressedSize, timeStateCompress, timeStateDecompress, srcBuffer, srcSize, fileSizes, nbFiles, cLevel, comprParams, @@ -759,9 +760,8 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, /* clean up */ BMK_freeTimeState(timeStateCompress); BMK_freeTimeState(timeStateDecompress); - if(!dstCapacity) { /* only free if not given */ - free(compressedBuffer); - } + + free(internalDstBuffer); free(resultBuffer); free((void*)srcPtrs); @@ -775,6 +775,10 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, if(allocationincomplete) { EXM_THROW(31, BMK_return_t, "allocation error : not enough memory"); } + + if(parametersConflict) { + EXM_THROW(32, BMK_return_t, "Conflicting input results"); + } return results; } diff --git a/programs/bench.h b/programs/bench.h index 2a9945ac8..8baf33a0a 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -175,7 +175,8 @@ typedef size_t (*BMK_initFn_t)(void*); * srcBuffers - an array of buffers to be operated on by benchFn * srcSizes - an array of the sizes of above buffers * dstBuffers - an array of buffers to be written into by benchFn - * dstCapacitiesToSizes - an array of the capacities of above buffers. Output modified to compressed sizes of those blocks. + * dstCapacities - an array of the capacities of above buffers + * blockResults - the return value of benchFn called on each block. * nbLoops - defines number of times benchFn is run. * assumed array of size blockCount, will have compressed size of each block written to it. * return @@ -192,7 +193,7 @@ BMK_customReturn_t BMK_benchFunction(BMK_benchFn_t benchFn, void* benchPayload, BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBuffers, const size_t* srcSizes, - void * const * const dstBuffers, const size_t* dstCapacities, size_t* cSizes, + void * const * const dstBuffers, const size_t* dstCapacities, size_t* blockResults, unsigned nbLoops); @@ -221,7 +222,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed(BMK_timedFnState_t* cont, BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBlockBuffers, const size_t* srcBlockSizes, - void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, size_t* cSizes); + void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, size_t* blockResults); #endif /* BENCH_H_121279284357 */ diff --git a/tests/README.md b/tests/README.md index 8bedd0a3c..2f0026fda 100644 --- a/tests/README.md +++ b/tests/README.md @@ -112,7 +112,6 @@ Full list of arguments dSpeed= - Minimum decompression speed cMem= - compression memory lvl= - Automatically sets compression speed constraint to the speed of that level - --optimize= : same as -O with more verbose syntax -P# : generated sample compressibility -t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) -v : Prints Benchmarking output diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 099be3688..a4f4a5f15 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -986,7 +986,7 @@ static void memoTableIndInv(ZSTD_compressionParameters* ptr, const varInds_t* va } /* Initialize memotable, immediately mark redundant / obviously infeasible params as */ -static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { +static void createMemoTable(U8* memoTable, ZSTD_compressionParameters paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { size_t i; size_t arrayLen = memoTableLen(varyParams, varyLen); int cwFixed = !paramConstraints.chainLog || !paramConstraints.windowLog; @@ -1053,23 +1053,24 @@ static void memoTableFreeAll(U8** mtAll) { /* inits memotables for all (including mallocs), all strategies */ /* takes unsanitized varyParams */ -static U8** memoTableInitAll(ZSTD_compressionParameters paramConstraints, constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { +static U8** createMemoTableArray(ZSTD_compressionParameters paramConstraints, constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { varInds_t varNew[NUM_PARAMS]; - int varLenNew; U8** mtAll = calloc(sizeof(U8*),(ZSTD_btultra + 1)); int i; if(mtAll == NULL) { return NULL; } + for(i = 1; i <= (int)ZSTD_btultra; i++) { - varLenNew = sanitizeVarArray(varNew, varyLen, varyParams, i); + const int varLenNew = sanitizeVarArray(varNew, varyLen, varyParams, i); mtAll[i] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew)); if(mtAll[i] == NULL) { memoTableFreeAll(mtAll); return NULL; } - memoTableInit(mtAll[i], paramConstraints, target, varNew, varLenNew, srcSize); + createMemoTable(mtAll[i], paramConstraints, target, varNew, varLenNew, srcSize); } + return mtAll; } @@ -1187,7 +1188,7 @@ static void BMK_benchFullTable(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* src winnerInfo_t winners[NB_LEVELS_TRACKED+1]; const char* const rfName = "grillResults.txt"; FILE* const f = fopen(rfName, "w"); - const size_t blockSize = g_blockSize ? g_blockSize : ZSTD_BLOCKSIZE_MAX; /* cut by block or not ? */ + const size_t blockSize = g_blockSize ? g_blockSize : srcSize; /* cut by block or not ? */ /* init */ assert(g_singleRun==0); @@ -1638,7 +1639,7 @@ static void freeBuffers(buffers_t b) { } /* allocates buffer's arguments. returns success / failuere */ -static int initBuffers(buffers_t* buff, const char* const * const fileNamesTable, +static int createBuffers(buffers_t* buff, const char* const * const fileNamesTable, size_t nbFiles) { size_t pos = 0; @@ -1659,7 +1660,7 @@ static int initBuffers(buffers_t* buff, const char* const * const fileNamesTable buff->resPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); buff->resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); - if(!buff->srcPtrs || !buff->srcSizes || !buff->dstPtrs || !buff->dstCapacities || !buff->dstCapacities || !buff->resPtrs || !buff->resSizes) { + if(!buff->srcPtrs || !buff->srcSizes || !buff->dstPtrs || !buff->dstCapacities || !buff->dstSizes || !buff->resPtrs || !buff->resSizes) { DISPLAY("alloc error\n"); freeBuffers(*buff); return 1; @@ -1747,7 +1748,7 @@ static void freeContexts(contexts_t ctx) { ZSTD_freeDCtx(ctx.dctx); } -static int initContexts(contexts_t* ctx, const char* dictFileName) { +static int createContexts(contexts_t* ctx, const char* dictFileName) { FILE* f; size_t readSize; ctx->cctx = ZSTD_createCCtx(); @@ -1846,12 +1847,12 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } /* load dictionary*/ - if(initBuffers(&buf, fileNamesTable, nbFiles)) { + if(createBuffers(&buf, fileNamesTable, nbFiles)) { DISPLAY("unable to load files\n"); return 1; } - if(initContexts(&ctx, dictFileName)) { + if(createContexts(&ctx, dictFileName)) { DISPLAY("unable to load dictionary\n"); freeBuffers(buf); return 2; @@ -1885,9 +1886,9 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ goto _cleanUp; } - memoTableInit(allMT[paramTarget.strategy], paramTarget, target, varNew, varLenNew, maxBlockSize); + createMemoTable(allMT[paramTarget.strategy], paramTarget, target, varNew, varLenNew, maxBlockSize); } else { - allMT = memoTableInitAll(paramTarget, target, varArray, varLen, maxBlockSize); + allMT = createMemoTableArray(paramTarget, target, varArray, varLen, maxBlockSize); } From 8faeb41679a5ef379cfb7893825e6ed98e571eb0 Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 30 Jul 2018 11:30:38 -0700 Subject: [PATCH 11/55] Update Documentation Change comment // to /* */ Add more description of what functions do Remove outdated comments --- tests/paramgrill.c | 117 ++++++++++++++++++++++++--------------------- 1 file changed, 63 insertions(+), 54 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index a4f4a5f15..b8396c72e 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -77,12 +77,11 @@ typedef enum { } varInds_t; #define NUM_PARAMS 6 -//just don't use strategy as a param. +/* just don't use strategy as a param. */ #undef ZSTD_WINDOWLOG_MAX #define ZSTD_WINDOWLOG_MAX 27 //no long range stuff for now. -//make 2^[0,10] w/ 999 #define ZSTD_TARGETLENGTH_MIN 0 #define ZSTD_TARGETLENGTH_MAX 999 @@ -274,7 +273,6 @@ static double resultScore(const BMK_result_t res, const size_t srcSize, const co ret = (MIN(1, cs) + MIN(1, ds) + MIN(1, cm))*r1 + rt * rtr + (MAX(0, log(cs))+ MAX(0, log(ds))+ MAX(0, log(cm))) * r2; - //DISPLAY("resultScore: %f\n", ret); return ret; } @@ -593,7 +591,7 @@ static BMK_return_t BMK_benchMemInvertible(buffers_t buf, contexts_t ctx, BMK_freeTimeState(timeStateCompress); BMK_freeTimeState(timeStateDecompress); - } else { //iterMode; + } else { /* iterMode; */ if(mode != BMK_decodeOnly) { BMK_customReturn_t compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)cctx, &local_initCCtx, (void*)&cctxprep, @@ -636,8 +634,7 @@ static BMK_return_t BMK_benchMemInvertible(buffers_t buf, contexts_t ctx, return results; } -/* global winner used for display. */ -//Should be totally 0 initialized? +/* global winner used for display. */ static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; static constraint_t g_targetConstraints; @@ -940,7 +937,7 @@ static void paramVariation(ZSTD_compressionParameters* ptr, const varInds_t* var } validated = !ZSTD_isError(ZSTD_checkCParams(p)); } - *ptr = p;//sanitizeParams(p); + *ptr = p; } /* length of memo table given free variables */ @@ -953,7 +950,7 @@ static size_t memoTableLen(const varInds_t* varyParams, const int varyLen) { return arrayLen; } -/* returns unique index of compression parameters */ +/* returns unique index in memotable of compression parameters */ static unsigned memoTableInd(const ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen) { int i; unsigned ind = 0; @@ -985,7 +982,7 @@ static void memoTableIndInv(ZSTD_compressionParameters* ptr, const varInds_t* va } } -/* Initialize memotable, immediately mark redundant / obviously infeasible params as */ +/* Initialize memotable, immediately mark redundant / obviously infeasible params as such */ static void createMemoTable(U8* memoTable, ZSTD_compressionParameters paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { size_t i; size_t arrayLen = memoTableLen(varyParams, varyLen); @@ -1003,7 +1000,7 @@ static void createMemoTable(U8* memoTable, ZSTD_compressionParameters paramConst memoTable[i] = 255; j++; } - if(wFixed && (1ULL << paramConstraints.windowLog) > (srcSize << 2)) { + if(wFixed && (1ULL << paramConstraints.windowLog) > srcSize) { memoTable[i] = 255; } /* nil out parameter sets equivalent to others. */ @@ -1121,7 +1118,7 @@ static void playAround(FILE* f, winnerInfo_t* winners, } - +/* Completely random parameter selection */ static ZSTD_compressionParameters randomParams(void) { ZSTD_compressionParameters p; @@ -1136,7 +1133,6 @@ static ZSTD_compressionParameters randomParams(void) p.targetLength=(FUZ_rand(&g_rand) % (512)); p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btultra +1)); validated = !ZSTD_isError(ZSTD_checkCParams(p)); - //validated = cParamValid(p); } return p; } @@ -1144,7 +1140,7 @@ static ZSTD_compressionParameters randomParams(void) /* Sets pc to random unmeasured set of parameters */ static void randomConstrainedParams(ZSTD_compressionParameters* pc, varInds_t* varArray, int varLen, U8* memoTable) { - size_t tries = memoTableLen(varArray, varLen); //configurable, + size_t tries = memoTableLen(varArray, varLen); const size_t maxSize = memoTableLen(varArray, varLen); size_t ind; do { @@ -1153,7 +1149,6 @@ static void randomConstrainedParams(ZSTD_compressionParameters* pc, varInds_t* v } while(memoTable[ind] > 0 && tries > 0); memoTableIndInv(pc, varArray, varLen, (unsigned)ind); - //*pc = sanitizeParams(*pc); } static void BMK_selectRandomStart( @@ -1328,19 +1323,13 @@ int benchFiles(const char** fileNamesTable, int nbFiles) return 0; } -/*benchmarks and tests feasibility together - 1 = true = better - 0 = false = not better - if true then resultPtr will give results. - 2+ on error? */ -//Maybe use compress_only for benchmark first run? + #define WORSE_RESULT 0 #define BETTER_RESULT 1 #define ERROR_RESULT 2 -//add worse result complete for worse results of length > 1 sec? -/* variation between 2nd run and full second bmk */ +/* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */ #define VARIANCE 1.1 static int allBench(BMK_result_t* resultPtr, buffers_t buf, contexts_t ctx, @@ -1447,8 +1436,9 @@ static int allBench(BMK_result_t* resultPtr, } -/* wrap feasibleBench w/ memotable */ #define INFEASIBLE_THRESHOLD 200 + +/* Memoized benchmarking, won't benchmark anything which has already been benchmarked before. */ static int benchMemo(BMK_result_t* resultPtr, buffers_t buf, contexts_t ctx, const ZSTD_compressionParameters cParams, @@ -1475,18 +1465,28 @@ static int benchMemo(BMK_result_t* resultPtr, return res; } - -//sanitize all params here. -//all generation after random should be sanitized. (maybe sanitize random) +/* One iteration of hill climbing. Specifically, it first tries all + * valid parameter configurations w/ manhattan distance 1 and picks the best one + * failing that, it progressively tries candidates further and further away (up to #dim + 2) + * if it finds a candidate exceeding winnerInfo, it will repeat. Otherwise, it will stop the + * current stage of hill climbing. + * Each iteration of hill climbing proceeds in 2 'phases'. Phase 1 climbs according to + * the resultScore function, which is effectively a linear increase in reward until it reaches + * the constraint-satisfying value, it which point any excess results in only logarithmic reward. + * This aims to find some constraint-satisfying point. + * Phase 2 optimizes in accordance with what the original function sets out to maximize, with + * all feasible solutions valued over all infeasible solutions. + */ static winnerInfo_t climbOnce(const constraint_t target, const varInds_t* varArray, const int varLen, U8* memoTable, buffers_t buf, contexts_t ctx, const ZSTD_compressionParameters init) { - //distance maximizing selection? - //cparam - currently considered 'center' - //candidate - params to benchmark/results - //winner - best option found so far. + /* + * cparam - currently considered 'center' + * candidate - params to benchmark/results + * winner - best option found so far. + */ ZSTD_compressionParameters cparam = init; winnerInfo_t candidateInfo, winnerInfo; int better = 1; @@ -1506,14 +1506,12 @@ static winnerInfo_t climbOnce(const constraint_t target, cparam = winnerInfo.params; BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, buf.srcSize); candidateInfo.params = cparam; - //all dist-1 targets - //if we early end this, we should also randomize the order these are picked. + /* all dist-1 candidates */ for(i = 0; i < varLen; i++) { for(offset = -1; offset <= 1; offset += 2) { candidateInfo.params = cparam; - paramVaryOnce(varArray[i], offset, &candidateInfo.params); /* +1 */ + paramVaryOnce(varArray[i], offset, &candidateInfo.params); candidateInfo.params = sanitizeParams(candidateInfo.params); - //evaluate if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { int res = benchMemo(&candidateInfo.result, buf, ctx, @@ -1560,7 +1558,7 @@ static winnerInfo_t climbOnce(const constraint_t target, } } - if(!better) { //infeas -> feas -> stop. + if(!better) { /* infeas -> feas -> stop */ if(feas) { return winnerInfo; } feas = 1; @@ -1575,14 +1573,14 @@ static winnerInfo_t climbOnce(const constraint_t target, return winnerInfo; } -//optimizeForSize but with fixed strategy -//place to configure/filter out strategy specific parameters. +/* Optimizes for a fixed strategy */ -//flexible parameters: iterations of (failed?) climbing (or if we do non-random, maybe this is when everything is close to visitied) -//weight more on visit for bad results, less on good results/more on later results / ones with more failures. -//allocate memoTable here. -//only real use for paramTarget is to get the fixed values, right? -//maybe allow giving it a first init? +/* flexible parameters: iterations of (failed?) climbing (or if we do non-random, maybe this is when everything is close to visitied) + weight more on visit for bad results, less on good results/more on later results / ones with more failures. + allocate memoTable here. + only real use for paramTarget is to get the fixed values, right? + maybe allow giving it a first init? + */ static winnerInfo_t optimizeFixedStrategy( buffers_t buf, contexts_t ctx, const constraint_t target, ZSTD_compressionParameters paramTarget, @@ -1603,9 +1601,8 @@ static winnerInfo_t optimizeFixedStrategy( init = paramTarget; - while(i < tries) { //make i adjustable (user input?) depending on how much time they have. + while(i < tries) { DEBUGOUTPUT("Restart\n"); - //look into improving this to maximize distance from searched infeasible stuff / towards promising regions? randomConstrainedParams(&init, varNew, varLenNew, memoTable); candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, buf, ctx, init); if(compareResultLT(winnerInfo.result, candidateInfo.result, target, buf.srcSize)) { @@ -1638,7 +1635,7 @@ static void freeBuffers(buffers_t b) { free(b.resPtrs); } -/* allocates buffer's arguments. returns success / failuere */ +/* allocates buffer's arguments. returns 0 = success / 1 = failuere */ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTable, size_t nbFiles) { @@ -1748,16 +1745,25 @@ static void freeContexts(contexts_t ctx) { ZSTD_freeDCtx(ctx.dctx); } +/* Creates struct holding contexts and dictionary buffers. returns 0 on success, 1 on failure. */ static int createContexts(contexts_t* ctx, const char* dictFileName) { FILE* f; size_t readSize; ctx->cctx = ZSTD_createCCtx(); ctx->dctx = ZSTD_createDCtx(); + ctx->dictSize = 0; + ctx->dictBuffer = NULL; + + if(!ctx->cctx || !ctx->dctx) { + DISPLAY("context allocation error\n"); + freeContexts(*ctx); + return 1; + } + if(dictFileName == NULL) { - ctx->dictSize = 0; - ctx->dictBuffer = NULL; return 0; } + ctx->dictSize = UTIL_getFileSize(dictFileName); ctx->dictBuffer = malloc(ctx->dictSize); @@ -1786,8 +1792,8 @@ static int createContexts(contexts_t* ctx, const char* dictFileName) { return 0; } -//goes best, best-1, best+1, best-2, ... -//return 0 if nothing remaining +/* goes best, best-1, best+1, best-2, ... */ +/* return 0 if nothing remaining */ static int nextStrategy(const int currentStrategy, const int bestStrategy) { if(bestStrategy <= currentStrategy) { int candidate = 2 * bestStrategy - currentStrategy - 1; @@ -1828,7 +1834,10 @@ static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZS } #define MAX_TRIES 8 -//optimize fixed strategy. +/* main fn called when using --optimize */ +/* Does strategy selection by benchmarking default compression levels + * then optimizes by strategy, starting with the best one and moving + * progressively moving further away by number */ static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel) { varInds_t varArray [NUM_PARAMS]; @@ -1905,7 +1914,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ goto _cleanUp; } - target.cSpeed = (U32)winner.result.cSpeed; //Maybe have a small bit of slack here, like x.99? + target.cSpeed = (U32)winner.result.cSpeed; g_targetConstraints = target; BMK_printWinner(stdout, cLevel, winner.result, winner.params, buf.srcSize); } @@ -1978,7 +1987,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) { winner = wc; } - //We could double back to increase search of 'better' strategies + st = nextStrategy(st, bestStrategy); tries--; } @@ -2088,7 +2097,7 @@ int main(int argc, const char** argv) int optimizerCLevel = 0; - constraint_t target = { 0, 0, (U32)-1 }; //0 for anything unset + constraint_t target = { 0, 0, (U32)-1 }; ZSTD_compressionParameters paramTarget = { 0, 0, 0, 0, 0, 0, 0 }; assert(argc>=1); /* for exename */ @@ -2250,7 +2259,7 @@ int main(int argc, const char** argv) /* load dictionary file (only applicable for optimizer rn) */ case 'D': - if(i == argc - 1) { //last argument, return error. + if(i == argc - 1) { /* last argument, return error. */ DISPLAY("Dictionary file expected but not given : %d\n", i); return 1; } else { From 3d230db85391aa5810abc6301035e9741c986765 Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 6 Aug 2018 17:13:36 -0700 Subject: [PATCH 12/55] Change speed representation from floating point to integral --- programs/bench.c | 43 +++++++++++++++++++------------------------ programs/bench.h | 4 ++-- tests/paramgrill.c | 46 +++++++++++++++++----------------------------- 3 files changed, 38 insertions(+), 55 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index b496caf27..49b317870 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -555,9 +555,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( ratio = (double)(srcSize / intermediateResultCompress.result.result.sumOfReturn); { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - double const compressionSpeed = ((double)srcSize / intermediateResultCompress.result.result.nanoSecPerRun) * 1000; - int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; - results.result.cSpeed = compressionSpeed * 1000000; + results.result.cSpeed = (srcSize * TIMELOOP_NANOSEC / intermediateResultCompress.result.result.nanoSecPerRun); cSize = intermediateResultCompress.result.result.sumOfReturn; results.result.cSize = cSize; ratio = (double)srcSize / results.result.cSize; @@ -565,7 +563,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r", marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize, ratioAccuracy, ratio, - cSpeedAccuracy, compressionSpeed); + results.result.cSpeed < (10 MB) ? 2 : 1, (double)results.result.cSpeed / (1 MB)); } } @@ -579,16 +577,13 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - double const compressionSpeed = results.result.cSpeed / 1000000; - int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; - double const decompressionSpeed = ((double)srcSize / intermediateResultDecompress.result.result.nanoSecPerRun) * 1000; - results.result.dSpeed = decompressionSpeed * 1000000; + results.result.dSpeed = (srcSize * TIMELOOP_NANOSEC/ intermediateResultDecompress.result.result.nanoSecPerRun); markNb = (markNb+1) % NB_MARKS; DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r", marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize, ratioAccuracy, ratio, - cSpeedAccuracy, compressionSpeed, - decompressionSpeed); + results.result.cSpeed < (10 MB) ? 2 : 1, (double)results.result.cSpeed / (1 MB), + (double)results.result.dSpeed / (1 MB)); } } } @@ -605,19 +600,20 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( if(compressionResults.result.nanoSecPerRun == 0) { results.result.cSpeed = 0; } else { - results.result.cSpeed = (double)srcSize / compressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; + results.result.cSpeed = srcSize * TIMELOOP_NANOSEC / compressionResults.result.nanoSecPerRun; } results.result.cSize = compressionResults.result.sumOfReturn; { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - double const compressionSpeed = results.result.cSpeed / 1000000; - int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; + results.result.cSpeed = (srcSize * TIMELOOP_NANOSEC / compressionResults.result.nanoSecPerRun); + cSize = compressionResults.result.sumOfReturn; + results.result.cSize = cSize; ratio = (double)srcSize / results.result.cSize; markNb = (markNb+1) % NB_MARKS; DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r", marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize, ratioAccuracy, ratio, - cSpeedAccuracy, compressionSpeed); + results.result.cSpeed < (10 MB) ? 2 : 1, (double)results.result.cSpeed / (1 MB)); } } if(adv->mode != BMK_compressOnly) { @@ -633,19 +629,18 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( if(decompressionResults.result.nanoSecPerRun == 0) { results.result.dSpeed = 0; } else { - results.result.dSpeed = (double)srcSize / decompressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; + results.result.dSpeed = srcSize * TIMELOOP_NANOSEC / decompressionResults.result.nanoSecPerRun; } - { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - double const compressionSpeed = results.result.cSpeed / 1000000; - int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; - double const decompressionSpeed = ((double)srcSize / decompressionResults.result.nanoSecPerRun) * 1000; - results.result.dSpeed = decompressionSpeed * 1000000; + + { + int const ratioAccuracy = (ratio < 10.) ? 3 : 2; + results.result.dSpeed = (srcSize * TIMELOOP_NANOSEC/ decompressionResults.result.nanoSecPerRun); markNb = (markNb+1) % NB_MARKS; DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r", marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize, ratioAccuracy, ratio, - cSpeedAccuracy, compressionSpeed, - decompressionSpeed); + results.result.cSpeed < (10 MB) ? 2 : 1, (double)results.result.cSpeed / (1 MB), + (double)results.result.dSpeed / (1 MB)); } } } @@ -693,8 +688,8 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( } /* CRC Checking */ if (displayLevel == 1) { /* hidden display mode -q, used by python speed benchmark */ - double const cSpeed = results.result.cSpeed / 1000000; - double const dSpeed = results.result.dSpeed / 1000000; + double const cSpeed = (double)results.result.cSpeed / (1 MB); + double const dSpeed = (double)results.result.dSpeed / (1 MB); if (adv->additionalParam) { DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, adv->additionalParam); } else { diff --git a/programs/bench.h b/programs/bench.h index 8baf33a0a..6247fa596 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -32,8 +32,8 @@ extern "C" { typedef struct { size_t cSize; - double cSpeed; /* bytes / sec */ - double dSpeed; + U64 cSpeed; /* bytes / sec */ + U64 dSpeed; size_t cMem; } BMK_result_t; diff --git a/tests/paramgrill.c b/tests/paramgrill.c index b8396c72e..5af3d88fa 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -249,18 +249,6 @@ static int feasible(const BMK_result_t results, const constraint_t target) { return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem); } -#define EPSILON 0.001 -static int epsilonEqual(const double c1, const double c2) { - return MAX(c1/c2,c2/c1) < 1 + EPSILON; -} - -/* checks exact equivalence to 0, to stop compiler complaining fpeq */ -static int eqZero(const double c1) { - const double z1 = 0.0; - const double z2 = -0.0; - return !(memcmp(&c1, &z1, sizeof(double))) || !(memcmp(&c1, &z2, sizeof(double))); -} - /* hill climbing value for part 1 */ static double resultScore(const BMK_result_t res, const size_t srcSize, const constraint_t target) { double cs = 0., ds = 0., rt, cm = 0.; @@ -280,7 +268,7 @@ static double resultScore(const BMK_result_t res, const size_t srcSize, const co static int compareResultLT(const BMK_result_t result1, const BMK_result_t result2, const constraint_t target, size_t srcSize) { if(feasible(result1, target) && feasible(result2, target)) { return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed) - || (result1.cSize == result2.cSize && epsilonEqual(result2.cSpeed, result1.cSpeed) && result2.dSpeed > result1.dSpeed); + || (result1.cSize == result2.cSize && result2.cSpeed == result1.cSpeed && result2.dSpeed > result1.dSpeed); } return feasible(result2, target) || (!feasible(result1, target) && (resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target))); @@ -661,7 +649,7 @@ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result fprintf(f, "/* %s */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */", - lvlstr, (double)srcSize / result.cSize, result.cSpeed / (1 << 20), result.dSpeed / (1 << 20)); + lvlstr, (double)srcSize / result.cSize, (double)result.cSpeed / (1 << 20), (double)result.dSpeed / (1 << 20)); if(TIMED) { fprintf(f, " - %1lu:%2lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); } fprintf(f, "\n"); @@ -696,8 +684,8 @@ static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, size_t srcSiz typedef struct { - double cSpeed_min; - double dSpeed_min; + U64 cSpeed_min; + U64 dSpeed_min; U32 windowLog_max; ZSTD_strategy strategy_max; } level_constraints_t; @@ -794,16 +782,16 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para /* too large compression speed difference for the compression benefit */ if (W_ratio > O_ratio) DISPLAY ("Compression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n", - W_ratio, testResult.cSpeed / 1000000, - O_ratio, winners[cLevel].result.cSpeed / 1000000., cLevel); + W_ratio, (double)testResult.cSpeed / 1000000, + O_ratio, (double)winners[cLevel].result.cSpeed / 1000000., cLevel); continue; } if (W_DSpeed_note < O_DSpeed_note ) { /* too large decompression speed difference for the compression benefit */ if (W_ratio > O_ratio) DISPLAY ("Decompression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n", - W_ratio, testResult.dSpeed / 1000000., - O_ratio, winners[cLevel].result.dSpeed / 1000000., cLevel); + W_ratio, (double)testResult.dSpeed / 1000000., + O_ratio, (double)winners[cLevel].result.dSpeed / 1000000., cLevel); continue; } @@ -1173,7 +1161,7 @@ static void BMK_benchOnce(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* srcBuffe g_params = ZSTD_adjustCParams(g_params, srcSize, 0); BMK_benchParam1(&testResult, srcBuffer, srcSize, cctx, dctx, g_params); DISPLAY("Compression Ratio: %.3f Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)srcSize / testResult.cSize, - testResult.cSpeed / 1000000, testResult.dSpeed / 1000000); + (double)testResult.cSpeed / 1000000, (double)testResult.dSpeed / 1000000); return; } @@ -1355,20 +1343,20 @@ static int allBench(BMK_result_t* resultPtr, *resultPtr = benchres.result; /* calculate uncertainty in compression / decompression runs */ - if(eqZero(benchres.result.cSpeed)) { + if(benchres.result.cSpeed) { + loopDurationC = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); + uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC) * VARIANCE; + } else { loopDurationC = 0; uncertaintyConstantC = 3; - } else { - loopDurationC = (U64)((double)(buf.srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); - uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC) * VARIANCE; } - if(eqZero(benchres.result.dSpeed)) { + if(benchres.result.dSpeed) { + loopDurationD = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); + uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD) * VARIANCE; + } else { loopDurationD = 0; uncertaintyConstantD = 3; - } else { - loopDurationD = (U64)((double)(buf.srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); - uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD) * VARIANCE; } /* anything with worse ratio in feas is definitely worse, discard */ From 8278a49cb608af483b70f301742899f5b0b8ab7d Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 6 Aug 2018 18:00:36 -0700 Subject: [PATCH 13/55] const srcPtrs --- programs/bench.c | 2 +- tests/paramgrill.c | 23 +++++++++++++++-------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 49b317870..874710c02 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -441,7 +441,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( int displayLevel, const char* displayName, const BMK_advancedParams_t* adv) { size_t const blockSize = ((adv->blockSize>=32 && (adv->mode != BMK_decodeOnly)) ? adv->blockSize : srcSize) + (!srcSize); /* avoid div by 0 */ - BMK_return_t results = { { 0, 0., 0., 0 }, 0 } ; + BMK_return_t results = { { 0, 0, 0, 0 }, 0 } ; size_t const loadedCompressedSize = srcSize; size_t cSize = 0; double ratio = 0.; diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 5af3d88fa..5d1a73081 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -250,6 +250,11 @@ static int feasible(const BMK_result_t results, const constraint_t target) { } /* hill climbing value for part 1 */ +/* Scoring here is a linear reward for all set constraints normalized between 0 to 1 + * (with 0 at 0 and 1 being fully fulfilling the constraint), summed with a logarithmic + * bonus to exceeding the constraint value. We also give linear ratio for compression ratio. + * The constant factors are experimental. + */ static double resultScore(const BMK_result_t res, const size_t srcSize, const constraint_t target) { double cs = 0., ds = 0., rt, cm = 0.; const double r1 = 1, r2 = 0.1, rtr = 0.5; @@ -291,7 +296,7 @@ const char* g_stratName[ZSTD_btultra+1] = { "ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ", "ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "}; -/* benchParam but only takes in one file. */ +/* benchParam but only takes in one input buffer. */ static int BMK_benchParam1(BMK_result_t* resultPtr, const void* srcBuffer, size_t srcSize, @@ -324,8 +329,9 @@ static winnerInfo_t initWinnerInfo(ZSTD_compressionParameters p) { } typedef struct { + void* srcBuffer; size_t srcSize; - void** srcPtrs; + const void** srcPtrs; size_t* srcSizes; void** dstPtrs; size_t* dstCapacities; @@ -1605,7 +1611,7 @@ static winnerInfo_t optimizeFixedStrategy( static void freeBuffers(buffers_t b) { if(b.srcPtrs != NULL) { - free(b.srcPtrs[0]); + free(b.srcBuffer); } free(b.srcPtrs); free(b.srcSizes); @@ -1635,7 +1641,7 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab U32 const maxNbBlocks = (U32) ((totalSizeToLoad + (blockSize-1)) / blockSize) + (U32)nbFiles; U32 blockNb = 0; - buff->srcPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); + buff->srcPtrs = (const void**)calloc(maxNbBlocks, sizeof(void*)); buff->srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); buff->dstPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); @@ -1651,7 +1657,8 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab return 1; } - buff->srcPtrs[0] = malloc(benchedSize); + buff->srcBuffer = malloc(benchedSize); + buff->srcPtrs[0] = (const void*)buff->srcBuffer; buff->dstPtrs[0] = malloc(ZSTD_compressBound(benchedSize) + (maxNbBlocks * 1024)); buff->resPtrs[0] = malloc(benchedSize); @@ -1684,11 +1691,11 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, nbFiles=n; /* buffer too small - stop after this file */ { - char* buffer = (char*)(buff->srcPtrs[0]); - size_t const readSize = fread((buffer)+pos, 1, (size_t)fileSize, f); + char* buffer = (char*)(buff->srcBuffer); + size_t const readSize = fread(((buffer)+pos), 1, (size_t)fileSize, f); size_t blocked = 0; while(blocked < readSize) { - buff->srcPtrs[blockNb] = (buffer) + (pos + blocked); + buff->srcPtrs[blockNb] = (const void*)((buffer) + (pos + blocked)); buff->srcSizes[blockNb] = blockSize; blocked += blockSize; blockNb++; From ad16a69408139d87cf44eb830b6749bf6066bee1 Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 6 Aug 2018 18:37:55 -0700 Subject: [PATCH 14/55] Readability improvements, renaming --- tests/paramgrill.c | 131 +++++++++++++++++++++++++-------------------- 1 file changed, 72 insertions(+), 59 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 5d1a73081..c0679e88e 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -362,7 +362,7 @@ BMK_benchParam(BMK_result_t* resultPtr, *********************************************************/ static void BMK_initCCtx(ZSTD_CCtx* ctx, - const void* dictBuffer, size_t dictBufferSize, int cLevel, + const void* dictBuffer, const size_t dictBufferSize, const int cLevel, const ZSTD_compressionParameters* comprParams, const BMK_advancedParams_t* adv) { ZSTD_CCtx_reset(ctx); ZSTD_CCtx_resetParameters(ctx); @@ -389,7 +389,7 @@ static void BMK_initCCtx(ZSTD_CCtx* ctx, static void BMK_initDCtx(ZSTD_DCtx* dctx, - const void* dictBuffer, size_t dictBufferSize) { + const void* dictBuffer, const size_t dictBufferSize) { ZSTD_DCtx_reset(dctx); ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize); } @@ -404,7 +404,7 @@ typedef struct { } BMK_initCCtxArgs; static size_t local_initCCtx(void* payload) { - BMK_initCCtxArgs* ag = (BMK_initCCtxArgs*)payload; + const BMK_initCCtxArgs* ag = (const BMK_initCCtxArgs*)payload; BMK_initCCtx(ag->ctx, ag->dictBuffer, ag->dictBufferSize, ag->cLevel, ag->comprParams, ag->adv); return 0; } @@ -416,7 +416,7 @@ typedef struct { } BMK_initDCtxArgs; static size_t local_initDCtx(void* payload) { - BMK_initDCtxArgs* ag = (BMK_initDCtxArgs*)payload; + const BMK_initDCtxArgs* ag = (const BMK_initDCtxArgs*)payload; BMK_initDCtx(ag->dctx, ag->dictBuffer, ag->dictBufferSize); return 0; } @@ -565,7 +565,7 @@ static BMK_return_t BMK_benchMemInvertible(buffers_t buf, contexts_t ctx, BMK_freeTimeState(timeStateDecompress); return results; } - results.result.cSpeed = ((double)srcSize / intermediateResultCompress.result.result.nanoSecPerRun) * TIMELOOP_NANOSEC; + results.result.cSpeed = (srcSize * TIMELOOP_NANOSEC) / intermediateResultCompress.result.result.nanoSecPerRun; results.result.cSize = intermediateResultCompress.result.result.sumOfReturn; } @@ -579,7 +579,7 @@ static BMK_return_t BMK_benchMemInvertible(buffers_t buf, contexts_t ctx, BMK_freeTimeState(timeStateDecompress); return results; } - results.result.dSpeed = ((double)srcSize / intermediateResultDecompress.result.result.nanoSecPerRun) * TIMELOOP_NANOSEC; + results.result.dSpeed = (srcSize * TIMELOOP_NANOSEC) / intermediateResultDecompress.result.result.nanoSecPerRun; } BMK_freeTimeState(timeStateCompress); @@ -597,7 +597,7 @@ static BMK_return_t BMK_benchMemInvertible(buffers_t buf, contexts_t ctx, if(compressionResults.result.nanoSecPerRun == 0) { results.result.cSpeed = 0; } else { - results.result.cSpeed = (double)srcSize / compressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; + results.result.cSpeed = srcSize * TIMELOOP_NANOSEC / compressionResults.result.nanoSecPerRun; } results.result.cSize = compressionResults.result.sumOfReturn; } @@ -618,7 +618,7 @@ static BMK_return_t BMK_benchMemInvertible(buffers_t buf, contexts_t ctx, if(decompressionResults.result.nanoSecPerRun == 0) { results.result.dSpeed = 0; } else { - results.result.dSpeed = (double)srcSize / decompressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; + results.result.dSpeed = srcSize * TIMELOOP_NANOSEC / decompressionResults.result.nanoSecPerRun; } } } @@ -628,39 +628,44 @@ static BMK_return_t BMK_benchMemInvertible(buffers_t buf, contexts_t ctx, return results; } -/* global winner used for display. */ -static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; -static constraint_t g_targetConstraints; static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const size_t srcSize) { - if(DEBUG || compareResultLT(g_winner.result, result, g_targetConstraints, srcSize)) { - char lvlstr[15] = "Custom Level"; - const U64 time = UTIL_clockSpanNano(g_time); - const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC); + char lvlstr[15] = "Custom Level"; + const U64 time = UTIL_clockSpanNano(g_time); + const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC); - if(DEBUG && compareResultLT(g_winner.result, result, g_targetConstraints, srcSize)) { + DISPLAY("\r%79s\r", ""); + + fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", + params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, + params.targetLength, g_stratName[(U32)(params.strategy)]); + + if(cLevel != CUSTOM_LEVEL) { + snprintf(lvlstr, 15, " Level %2u ", cLevel); + } + + fprintf(f, + "/* %s */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */", + lvlstr, (double)srcSize / result.cSize, (double)result.cSpeed / (1 << 20), (double)result.dSpeed / (1 << 20)); + + if(TIMED) { fprintf(f, " - %1lu:%2lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); } + fprintf(f, "\n"); +} + +static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const constraint_t targetConstraints, const size_t srcSize) +{ + /* global winner used for constraints */ + static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; + + if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { + if(DEBUG && compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { DISPLAY("New Winner: \n"); } - DISPLAY("\r%79s\r", ""); + BMK_printWinner(f, cLevel, result, params, srcSize); - fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", - params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, - params.targetLength, g_stratName[(U32)(params.strategy)]); - - if(cLevel != CUSTOM_LEVEL) { - snprintf(lvlstr, 15, " Level %2u ", cLevel); - } - - fprintf(f, - "/* %s */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */", - lvlstr, (double)srcSize / result.cSize, (double)result.cSpeed / (1 << 20), (double)result.dSpeed / (1 << 20)); - - if(TIMED) { fprintf(f, " - %1lu:%2lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); } - fprintf(f, "\n"); - - if(compareResultLT(g_winner.result, result, g_targetConstraints, srcSize)) { + if(compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { BMK_translateAdvancedParams(params); g_winner.result = result; g_winner.params = params; @@ -950,12 +955,18 @@ static unsigned memoTableInd(const ZSTD_compressionParameters* ptr, const varInd unsigned ind = 0; for(i = 0; i < varyLen; i++) { switch(varyParams[i]) { - case wlog_ind: ind *= WLOG_RANGE; ind += ptr->windowLog - ZSTD_WINDOWLOG_MIN ; break; - case clog_ind: ind *= CLOG_RANGE; ind += ptr->chainLog - ZSTD_CHAINLOG_MIN ; break; - case hlog_ind: ind *= HLOG_RANGE; ind += ptr->hashLog - ZSTD_HASHLOG_MIN ; break; - case slog_ind: ind *= SLOG_RANGE; ind += ptr->searchLog - ZSTD_SEARCHLOG_MIN ; break; - case slen_ind: ind *= SLEN_RANGE; ind += ptr->searchLength - ZSTD_SEARCHLENGTH_MIN; break; - case tlen_ind: ind *= TLEN_RANGE; ind += tlen_inv(ptr->targetLength) - ZSTD_TARGETLENGTH_MIN; break; + case wlog_ind: ind *= WLOG_RANGE; ind += ptr->windowLog + - ZSTD_WINDOWLOG_MIN ; break; + case clog_ind: ind *= CLOG_RANGE; ind += ptr->chainLog + - ZSTD_CHAINLOG_MIN ; break; + case hlog_ind: ind *= HLOG_RANGE; ind += ptr->hashLog + - ZSTD_HASHLOG_MIN ; break; + case slog_ind: ind *= SLOG_RANGE; ind += ptr->searchLog + - ZSTD_SEARCHLOG_MIN ; break; + case slen_ind: ind *= SLEN_RANGE; ind += ptr->searchLength + - ZSTD_SEARCHLENGTH_MIN; break; + case tlen_ind: ind *= TLEN_RANGE; ind += tlen_inv(ptr->targetLength) + - ZSTD_TARGETLENGTH_MIN; break; } } return ind; @@ -976,8 +987,12 @@ static void memoTableIndInv(ZSTD_compressionParameters* ptr, const varInds_t* va } } -/* Initialize memotable, immediately mark redundant / obviously infeasible params as such */ -static void createMemoTable(U8* memoTable, ZSTD_compressionParameters paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { +/* Initialize memoization table, which tracks and prevents repeated benchmarking + * of the same set of parameters. In addition, it is also used to immediately mark + * redundant / obviously non-optimal parameter configurations (e.g. wlog - 1 larger) + * than srcSize, clog > wlog, ... + */ +static void initMemoTable(U8* memoTable, ZSTD_compressionParameters paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { size_t i; size_t arrayLen = memoTableLen(varyParams, varyLen); int cwFixed = !paramConstraints.chainLog || !paramConstraints.windowLog; @@ -985,6 +1000,7 @@ static void createMemoTable(U8* memoTable, ZSTD_compressionParameters paramConst int whFixed = !paramConstraints.windowLog || !paramConstraints.hashLog; int wFixed = !paramConstraints.windowLog; int j = 0; + assert(memoTable != NULL); memset(memoTable, 0, arrayLen); cParamZeroMin(¶mConstraints); @@ -994,7 +1010,7 @@ static void createMemoTable(U8* memoTable, ZSTD_compressionParameters paramConst memoTable[i] = 255; j++; } - if(wFixed && (1ULL << paramConstraints.windowLog) > srcSize) { + if(wFixed && (1ULL << (paramConstraints.windowLog - 1)) > srcSize) { memoTable[i] = 255; } /* nil out parameter sets equivalent to others. */ @@ -1033,7 +1049,7 @@ static void createMemoTable(U8* memoTable, ZSTD_compressionParameters paramConst } /* frees all allocated memotables */ -static void memoTableFreeAll(U8** mtAll) { +static void freeMemoTableArray(U8** mtAll) { int i; if(mtAll == NULL) { return; } for(i = 1; i <= (int)ZSTD_btultra; i++) { @@ -1056,10 +1072,10 @@ static U8** createMemoTableArray(ZSTD_compressionParameters paramConstraints, co const int varLenNew = sanitizeVarArray(varNew, varyLen, varyParams, i); mtAll[i] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew)); if(mtAll[i] == NULL) { - memoTableFreeAll(mtAll); + freeMemoTableArray(mtAll); return NULL; } - createMemoTable(mtAll[i], paramConstraints, target, varNew, varLenNew, srcSize); + initMemoTable(mtAll[i], paramConstraints, target, varNew, varLenNew, srcSize); } return mtAll; @@ -1451,7 +1467,7 @@ static int benchMemo(BMK_result_t* resultPtr, DISPLAY("Count: %d\n", bmcount); bmcount++; } - BMK_printWinner(stdout, CUSTOM_LEVEL, *resultPtr, cParams, buf.srcSize); + BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, *resultPtr, cParams, target, buf.srcSize); if(res == BETTER_RESULT || feas) { memoTable[memind] = 255; @@ -1498,7 +1514,7 @@ static winnerInfo_t climbOnce(const constraint_t target, better = 0; DEBUGOUTPUT("Start\n"); cparam = winnerInfo.params; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, buf.srcSize); + BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize); candidateInfo.params = cparam; /* all dist-1 candidates */ for(i = 0; i < varLen; i++) { @@ -1513,7 +1529,7 @@ static winnerInfo_t climbOnce(const constraint_t target, varArray, varLen, feas); if(res == BETTER_RESULT) { /* synonymous with better when called w/ infeasibleBM */ winnerInfo = candidateInfo; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, buf.srcSize); + BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize); better = 1; if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) { bestFeasible1 = winnerInfo; @@ -1539,7 +1555,7 @@ static winnerInfo_t climbOnce(const constraint_t target, varArray, varLen, feas); if(res == BETTER_RESULT) { /* synonymous with better in this case*/ winnerInfo = candidateInfo; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, buf.srcSize); + BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize); better = 1; if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) { bestFeasible1 = winnerInfo; @@ -1601,7 +1617,7 @@ static winnerInfo_t optimizeFixedStrategy( candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, buf, ctx, init); if(compareResultLT(winnerInfo.result, candidateInfo.result, target, buf.srcSize)) { winnerInfo = candidateInfo; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, buf.srcSize); + BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize); i = 0; } i++; @@ -1890,7 +1906,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ goto _cleanUp; } - createMemoTable(allMT[paramTarget.strategy], paramTarget, target, varNew, varLenNew, maxBlockSize); + initMemoTable(allMT[paramTarget.strategy], paramTarget, target, varNew, varLenNew, maxBlockSize); } else { allMT = createMemoTableArray(paramTarget, target, varArray, varLen, maxBlockSize); } @@ -1910,12 +1926,9 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } target.cSpeed = (U32)winner.result.cSpeed; - g_targetConstraints = target; - BMK_printWinner(stdout, cLevel, winner.result, winner.params, buf.srcSize); + BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize); } - g_targetConstraints = target; - /* bench */ DISPLAY("\r%79s\r", ""); if(nbFiles == 1) { @@ -1946,7 +1959,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ ZSTD_compressionParameters CParams = ZSTD_getCParams(i, maxBlockSize, ctx.dictSize); CParams = maskParams(CParams, paramTarget); ec = BMK_benchParam(&candidate, buf, ctx, CParams); - BMK_printWinner(stdout, i, candidate, CParams, buf.srcSize); + BMK_printWinnerOpt(stdout, i, candidate, CParams, target, buf.srcSize); if(!ec && compareResultLT(winner.result, candidate, relaxTarget(target), buf.srcSize)) { winner.result = candidate; @@ -1956,7 +1969,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } } - BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, buf.srcSize); + BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winner.result, winner.params, target, buf.srcSize); BMK_translateAdvancedParams(winner.params); DEBUGOUTPUT("Real Opt\n"); /* start 'real' tests */ @@ -2000,7 +2013,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ goto _cleanUp; } /* end summary */ - BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, buf.srcSize); + BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winner.result, winner.params, target, buf.srcSize); BMK_translateAdvancedParams(winner.params); DISPLAY("grillParams size - optimizer completed \n"); @@ -2008,7 +2021,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ _cleanUp: freeContexts(ctx); freeBuffers(buf); - memoTableFreeAll(allMT); + freeMemoTableArray(allMT); return ret; } From 6f480927af465d34ac9914789291094477c619bb Mon Sep 17 00:00:00 2001 From: George Lu Date: Tue, 7 Aug 2018 11:56:14 -0700 Subject: [PATCH 15/55] argument parsing cleanup + clarifying comment --- tests/paramgrill.c | 95 +++++++++++++++++++++++++++++++--------------- 1 file changed, 64 insertions(+), 31 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index c0679e88e..a1d409d67 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -977,12 +977,18 @@ static void memoTableIndInv(ZSTD_compressionParameters* ptr, const varInds_t* va int i; for(i = varyLen - 1; i >= 0; i--) { switch(varyParams[i]) { - case wlog_ind: ptr->windowLog = ind % WLOG_RANGE + ZSTD_WINDOWLOG_MIN; ind /= WLOG_RANGE; break; - case clog_ind: ptr->chainLog = ind % CLOG_RANGE + ZSTD_CHAINLOG_MIN; ind /= CLOG_RANGE; break; - case hlog_ind: ptr->hashLog = ind % HLOG_RANGE + ZSTD_HASHLOG_MIN; ind /= HLOG_RANGE; break; - case slog_ind: ptr->searchLog = ind % SLOG_RANGE + ZSTD_SEARCHLOG_MIN; ind /= SLOG_RANGE; break; - case slen_ind: ptr->searchLength = ind % SLEN_RANGE + ZSTD_SEARCHLENGTH_MIN; ind /= SLEN_RANGE; break; - case tlen_ind: ptr->targetLength = tlen_table[(ind % TLEN_RANGE)]; ind /= TLEN_RANGE; break; + case wlog_ind: ptr->windowLog = ind % WLOG_RANGE + ZSTD_WINDOWLOG_MIN; + ind /= WLOG_RANGE; break; + case clog_ind: ptr->chainLog = ind % CLOG_RANGE + ZSTD_CHAINLOG_MIN; + ind /= CLOG_RANGE; break; + case hlog_ind: ptr->hashLog = ind % HLOG_RANGE + ZSTD_HASHLOG_MIN; + ind /= HLOG_RANGE; break; + case slog_ind: ptr->searchLog = ind % SLOG_RANGE + ZSTD_SEARCHLOG_MIN; + ind /= SLOG_RANGE; break; + case slen_ind: ptr->searchLength = ind % SLEN_RANGE + ZSTD_SEARCHLENGTH_MIN; + ind /= SLEN_RANGE; break; + case tlen_ind: ptr->targetLength = tlen_table[(ind % TLEN_RANGE)]; + ind /= TLEN_RANGE; break; } } } @@ -1135,13 +1141,20 @@ static ZSTD_compressionParameters randomParams(void) U32 validated = 0; while (!validated) { /* totally random entry */ - p.chainLog = (FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN)) + ZSTD_CHAINLOG_MIN; - p.hashLog = (FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN)) + ZSTD_HASHLOG_MIN; - p.searchLog = (FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN)) + ZSTD_SEARCHLOG_MIN; - p.windowLog = (FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN)) + ZSTD_WINDOWLOG_MIN; - p.searchLength=(FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN)) + ZSTD_SEARCHLENGTH_MIN; + p.chainLog = (FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN)) + + ZSTD_CHAINLOG_MIN; + p.hashLog = (FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN)) + + ZSTD_HASHLOG_MIN; + p.searchLog = (FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN)) + + ZSTD_SEARCHLOG_MIN; + p.windowLog = (FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN)) + + ZSTD_WINDOWLOG_MIN; + p.searchLength=(FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN)) + + ZSTD_SEARCHLENGTH_MIN; p.targetLength=(FUZ_rand(&g_rand) % (512)); + p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btultra +1)); + validated = !ZSTD_isError(ZSTD_checkCParams(p)); } return p; @@ -1657,6 +1670,8 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab U32 const maxNbBlocks = (U32) ((totalSizeToLoad + (blockSize-1)) / blockSize) + (U32)nbFiles; U32 blockNb = 0; + memset(buff, 0, sizeof(buffers_t)); + buff->srcPtrs = (const void**)calloc(maxNbBlocks, sizeof(void*)); buff->srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); @@ -1760,6 +1775,7 @@ static void freeContexts(contexts_t ctx) { static int createContexts(contexts_t* ctx, const char* dictFileName) { FILE* f; size_t readSize; + U64 dictSize; ctx->cctx = ZSTD_createCCtx(); ctx->dctx = ZSTD_createDCtx(); ctx->dictSize = 0; @@ -1775,7 +1791,16 @@ static int createContexts(contexts_t* ctx, const char* dictFileName) { return 0; } - ctx->dictSize = UTIL_getFileSize(dictFileName); + dictSize = UTIL_getFileSize(dictFileName); + + if(dictSize == UTIL_FILESIZE_UNKNOWN) { + DISPLAY("Unable to get dictionary size\n"); + freeContexts(*ctx); + return 1; + } else { + ctx->dictSize = (size_t)dictSize; + } + ctx->dictBuffer = malloc(ctx->dictSize); f = fopen(dictFileName, "rb"); @@ -1848,7 +1873,15 @@ static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZS /* main fn called when using --optimize */ /* Does strategy selection by benchmarking default compression levels * then optimizes by strategy, starting with the best one and moving - * progressively moving further away by number */ + * progressively moving further away by number + * args: + * fileNamesTable - list of files to benchmark + * nbFiles - length of fileNamesTable + * dictFileName - name of dictionary file if one, else NULL + * target - performance constraints (cSpeed, dSpeed, cMem) + * paramTarget - parameter constraints (i.e. restriction search space to where strategy = ZSTD_fast) + * cLevel - compression level to exceed (all solutions must be > lvl in cSpeed + ratio) + */ static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel) { varInds_t varArray [NUM_PARAMS]; @@ -2092,6 +2125,18 @@ static int badusage(const char* exename) return 1; } +#define PARSE_SUB_ARGS(stringLong, stringShort, variable) { if (longCommandWArg(&argument, stringLong) || longCommandWArg(&argument, stringShort)) { variable = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } } +#define PARSE_CPARAMS(variable) \ +{ \ + PARSE_SUB_ARGS("windowLog=", "wlog=", variable.windowLog); \ + PARSE_SUB_ARGS("chainLog=" , "clog=", variable.chainLog); \ + PARSE_SUB_ARGS("hashLog=", "hlog=", variable.hashLog); \ + PARSE_SUB_ARGS("searchLog=" , "slog=", variable.searchLog); \ + PARSE_SUB_ARGS("searchLength=", "slen=", variable.searchLength); \ + PARSE_SUB_ARGS("targetLength=" , "tlen=", variable.targetLength); \ + PARSE_SUB_ARGS("strategy=", "strat=", variable.strategy); \ +} + int main(int argc, const char** argv) { int i, @@ -2127,17 +2172,11 @@ int main(int argc, const char** argv) if (longCommandWArg(&argument, "--optimize=")) { optimizer = 1; for ( ; ;) { - if (longCommandWArg(&argument, "windowLog=") || longCommandWArg(&argument, "wlog=")) { paramTarget.windowLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "chainLog=") || longCommandWArg(&argument, "clog=")) { paramTarget.chainLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "hashLog=") || longCommandWArg(&argument, "hlog=")) { paramTarget.hashLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "searchLog=") || longCommandWArg(&argument, "slog=")) { paramTarget.searchLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "searchLength=") || longCommandWArg(&argument, "slen=")) { paramTarget.searchLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "targetLength=") || longCommandWArg(&argument, "tlen=")) { paramTarget.targetLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "strategy=") || longCommandWArg(&argument, "strat=")) { paramTarget.strategy = (ZSTD_strategy)(readU32FromChar(&argument)); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "compressionSpeed=") || longCommandWArg(&argument, "cSpeed=")) { target.cSpeed = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "decompressionSpeed=") || longCommandWArg(&argument, "dSpeed=")) { target.dSpeed = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "compressionMemory=") || longCommandWArg(&argument, "cMem=")) { target.cMem = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { optimizerCLevel = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } + PARSE_CPARAMS(paramTarget); + PARSE_SUB_ARGS("compressionSpeed=" , "cSpeed=", target.cSpeed); + PARSE_SUB_ARGS("decompressionSpeed=", "dSpeed=", target.dSpeed); + PARSE_SUB_ARGS("compressionMemory=" , "cMem=", target.cMem); + PARSE_SUB_ARGS("level=", "lvl=", optimizerCLevel); DISPLAY("invalid optimization parameter \n"); return 1; } @@ -2152,13 +2191,7 @@ int main(int argc, const char** argv) g_singleRun = 1; g_params = ZSTD_getCParams(2, g_blockSize, 0); for ( ; ;) { - if (longCommandWArg(&argument, "windowLog=") || longCommandWArg(&argument, "wlog=")) { g_params.windowLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "chainLog=") || longCommandWArg(&argument, "clog=")) { g_params.chainLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "hashLog=") || longCommandWArg(&argument, "hlog=")) { g_params.hashLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "searchLog=") || longCommandWArg(&argument, "slog=")) { g_params.searchLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "searchLength=") || longCommandWArg(&argument, "slen=")) { g_params.searchLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "targetLength=") || longCommandWArg(&argument, "tlen=")) { g_params.targetLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "strategy=") || longCommandWArg(&argument, "strat=")) { g_params.strategy = (ZSTD_strategy)(readU32FromChar(&argument)); if (argument[0]==',') { argument++; continue; } else break; } + PARSE_CPARAMS(g_params) if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { g_params = ZSTD_getCParams(readU32FromChar(&argument), g_blockSize, 0); if (argument[0]==',') { argument++; continue; } else break; } DISPLAY("invalid compression parameter \n"); return 1; From 0ece2e5cdc33d9e1499125820a85cae684c6deaa Mon Sep 17 00:00:00 2001 From: George Lu Date: Thu, 9 Aug 2018 11:38:09 -0700 Subject: [PATCH 16/55] Add consts + fix gcc-8 warnings --- tests/fullbench.c | 3 ++- tests/paramgrill.c | 48 ++++++++++++++++++++++++---------------------- 2 files changed, 27 insertions(+), 24 deletions(-) diff --git a/tests/fullbench.c b/tests/fullbench.c index 270cac86a..12c1e1ae4 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -514,9 +514,10 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb, int cLevel, /* benchmark loop */ { + void* dstBuffv = (void*)dstBuff; r = BMK_benchFunction(benchFunction, buff2, NULL, NULL, 1, &src, &srcSize, - (void **)&dstBuff, &dstBuffSize, NULL, g_nbIterations); + &dstBuffv, &dstBuffSize, NULL, g_nbIterations); if(r.error) { DISPLAY("ERROR %d ! ! \n", r.error); errorcode = r.error; diff --git a/tests/paramgrill.c b/tests/paramgrill.c index a1d409d67..eeafc8302 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -350,7 +350,7 @@ typedef struct { static int BMK_benchParam(BMK_result_t* resultPtr, - buffers_t buf, contexts_t ctx, + const buffers_t buf, const contexts_t ctx, const ZSTD_compressionParameters cParams) { BMK_return_t res = BMK_benchMem(buf.srcPtrs[0], buf.srcSize, buf.srcSizes, (unsigned)buf.nbBlocks, 0, &cParams, ctx.dictBuffer, ctx.dictSize, ctx.cctx, ctx.dctx, 0, "Files"); *resultPtr = res.result; @@ -482,13 +482,14 @@ static size_t local_defaultDecompress( * From Paramgrill End *********************************************************/ -/* Replicate function of benchMemAdvanced, but with pre-split src / dst buffers, with relevant info to invert it (compressedSizes) passed out. */ +/* Replicate functionality of benchMemAdvanced, but with pre-split src / dst buffers */ +/* The purpose is so that sufficient information is returned so that a decompression call to benchMemInvertible is possible */ /* BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); */ /* nbSeconds used in same way as in BMK_advancedParams_t, as nbIters when in iterMode */ /* if in decodeOnly, then srcPtr's will be compressed blocks, and uncompressedBlocks will be written to dstPtrs? */ /* dictionary nullable, nothing else though. */ -static BMK_return_t BMK_benchMemInvertible(buffers_t buf, contexts_t ctx, +static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t ctx, const int cLevel, const ZSTD_compressionParameters* comprParams, const BMK_mode_t mode, const BMK_loopMode_t loopMode, const unsigned nbSeconds) { @@ -496,11 +497,11 @@ static BMK_return_t BMK_benchMemInvertible(buffers_t buf, contexts_t ctx, BMK_return_t results = { { 0, 0., 0., 0 }, 0 } ; const void *const *const srcPtrs = (const void *const *const)buf.srcPtrs; size_t const *const srcSizes = buf.srcSizes; - void** dstPtrs = buf.dstPtrs; - size_t* dstCapacities = buf.dstCapacities; - size_t* dstSizes = buf.dstSizes; - void** resPtrs = buf.resPtrs; - size_t* resSizes = buf.resSizes; + void** const dstPtrs = buf.dstPtrs; + size_t const *const dstCapacities = buf.dstCapacities; + size_t* const dstSizes = buf.dstSizes; + void** const resPtrs = buf.resPtrs; + size_t const *const resSizes = buf.resSizes; const void* dictBuffer = ctx.dictBuffer; const size_t dictBufferSize = ctx.dictSize; const size_t nbBlocks = buf.nbBlocks; @@ -1355,7 +1356,7 @@ int benchFiles(const char** fileNamesTable, int nbFiles) /* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */ #define VARIANCE 1.1 static int allBench(BMK_result_t* resultPtr, - buffers_t buf, contexts_t ctx, + const buffers_t buf, const contexts_t ctx, const ZSTD_compressionParameters cParams, const constraint_t target, BMK_result_t* winnerResult, int feas) { @@ -1463,11 +1464,11 @@ static int allBench(BMK_result_t* resultPtr, /* Memoized benchmarking, won't benchmark anything which has already been benchmarked before. */ static int benchMemo(BMK_result_t* resultPtr, - buffers_t buf, contexts_t ctx, + const buffers_t buf, const contexts_t ctx, const ZSTD_compressionParameters cParams, const constraint_t target, - BMK_result_t* winnerResult, U8* memoTable, - const varInds_t* varyParams, const int varyLen, int feas) { + BMK_result_t* winnerResult, U8* const memoTable, + const varInds_t* varyParams, const int varyLen, const int feas) { static int bmcount = 0; size_t memind = memoTableInd(&cParams, varyParams, varyLen); int res; @@ -1502,8 +1503,8 @@ static int benchMemo(BMK_result_t* resultPtr, */ static winnerInfo_t climbOnce(const constraint_t target, const varInds_t* varArray, const int varLen, - U8* memoTable, - buffers_t buf, contexts_t ctx, + U8* const memoTable, + const buffers_t buf, const contexts_t ctx, const ZSTD_compressionParameters init) { /* * cparam - currently considered 'center' @@ -1605,11 +1606,11 @@ static winnerInfo_t climbOnce(const constraint_t target, maybe allow giving it a first init? */ static winnerInfo_t optimizeFixedStrategy( - buffers_t buf, contexts_t ctx, + const buffers_t buf, const contexts_t ctx, const constraint_t target, ZSTD_compressionParameters paramTarget, const ZSTD_strategy strat, const varInds_t* varArray, const int varLen, - U8* memoTable, const int tries) { + U8* const memoTable, const int tries) { int i = 0; varInds_t varNew[NUM_PARAMS]; int varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat); @@ -1638,7 +1639,7 @@ static winnerInfo_t optimizeFixedStrategy( return winnerInfo; } -static void freeBuffers(buffers_t b) { +static void freeBuffers(const buffers_t b) { if(b.srcPtrs != NULL) { free(b.srcBuffer); } @@ -1659,8 +1660,8 @@ static void freeBuffers(buffers_t b) { } /* allocates buffer's arguments. returns 0 = success / 1 = failuere */ -static int createBuffers(buffers_t* buff, const char* const * const fileNamesTable, - size_t nbFiles) +static int createBuffers(buffers_t* const buff, const char* const * const fileNamesTable, + const size_t nbFiles) { size_t pos = 0; size_t n; @@ -1720,7 +1721,7 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab DISPLAY("Loading %s... \r", fileNamesTable[n]); - if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, nbFiles=n; /* buffer too small - stop after this file */ + if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, n = nbFiles; /* buffer too small - stop after this file */ { char* buffer = (char*)(buff->srcBuffer); size_t const readSize = fread(((buffer)+pos), 1, (size_t)fileSize, f); @@ -1765,14 +1766,14 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab return 0; } -static void freeContexts(contexts_t ctx) { +static void freeContexts(const contexts_t ctx) { free(ctx.dictBuffer); ZSTD_freeCCtx(ctx.cctx); ZSTD_freeDCtx(ctx.dctx); } /* Creates struct holding contexts and dictionary buffers. returns 0 on success, 1 on failure. */ -static int createContexts(contexts_t* ctx, const char* dictFileName) { +static int createContexts(contexts_t* const ctx, const char* dictFileName) { FILE* f; size_t readSize; U64 dictSize; @@ -2268,7 +2269,8 @@ int main(int argc, const char** argv) g_params.strategy = (ZSTD_strategy)readU32FromChar(&argument); continue; case 'L': - { int const cLevel = readU32FromChar(&argument); + { argument++; + int const cLevel = readU32FromChar(&argument); g_params = ZSTD_getCParams(cLevel, g_blockSize, 0); continue; } From bfe8392e23cbb561963451c154bde2a368fde926 Mon Sep 17 00:00:00 2001 From: George Lu Date: Thu, 9 Aug 2018 12:07:57 -0700 Subject: [PATCH 17/55] Remove ctx from benchMem --- programs/bench.c | 46 ++++++++++---------------------------- programs/bench.h | 4 ---- tests/paramgrill.c | 55 ++++++++++++++++++---------------------------- 3 files changed, 33 insertions(+), 72 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 874710c02..7662678ab 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -708,7 +708,6 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, const size_t* fileSizes, unsigned nbFiles, const int cLevel, const ZSTD_compressionParameters* comprParams, const void* dictBuffer, size_t dictBufferSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, int displayLevel, const char* displayName, const BMK_advancedParams_t* adv) { @@ -730,6 +729,9 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, BMK_timedFnState_t* timeStateCompress = BMK_createTimeState(adv->nbSeconds); BMK_timedFnState_t* timeStateDecompress = BMK_createTimeState(adv->nbSeconds); + ZSTD_CCtx* ctx = ZSTD_createCCtx(); + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + const size_t maxCompressedSize = dstCapacity ? dstCapacity : ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); void* const internalDstBuffer = dstBuffer ? NULL : malloc(maxCompressedSize); @@ -756,6 +758,9 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, BMK_freeTimeState(timeStateCompress); BMK_freeTimeState(timeStateDecompress); + ZSTD_freeCCtx(ctx); + ZSTD_freeDCtx(dctx); + free(internalDstBuffer); free(resultBuffer); @@ -781,7 +786,6 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize, const size_t* fileSizes, unsigned nbFiles, const int cLevel, const ZSTD_compressionParameters* comprParams, const void* dictBuffer, size_t dictBufferSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, int displayLevel, const char* displayName) { const BMK_advancedParams_t adv = BMK_initAdvancedParams(); @@ -790,35 +794,9 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize, fileSizes, nbFiles, cLevel, comprParams, dictBuffer, dictBufferSize, - ctx, dctx, displayLevel, displayName, &adv); } -static BMK_return_t BMK_benchMemCtxless(const void* srcBuffer, size_t srcSize, - const size_t* fileSizes, unsigned nbFiles, - int cLevel, const ZSTD_compressionParameters* const comprParams, - const void* dictBuffer, size_t dictBufferSize, - int displayLevel, const char* displayName, - const BMK_advancedParams_t* const adv) -{ - BMK_return_t res; - ZSTD_CCtx* ctx = ZSTD_createCCtx(); - ZSTD_DCtx* dctx = ZSTD_createDCtx(); - if(ctx == NULL || dctx == NULL) { - EXM_THROW(12, BMK_return_t, "not enough memory for contexts"); - } - res = BMK_benchMemAdvanced(srcBuffer, srcSize, - NULL, 0, - fileSizes, nbFiles, - cLevel, comprParams, - dictBuffer, dictBufferSize, - ctx, dctx, - displayLevel, displayName, adv); - ZSTD_freeCCtx(ctx); - ZSTD_freeDCtx(dctx); - return res; -} - static size_t BMK_findMaxMem(U64 requiredMem) { size_t const step = 64 MB; @@ -859,12 +837,12 @@ static BMK_return_t BMK_benchCLevel(const void* srcBuffer, size_t benchedSize, if (displayLevel == 1 && !adv->additionalParam) DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, adv->nbSeconds, (U32)(adv->blockSize>>10)); - res = BMK_benchMemCtxless(srcBuffer, benchedSize, - fileSizes, nbFiles, - cLevel, comprParams, - dictBuffer, dictBufferSize, - displayLevel, displayName, - adv); + res = BMK_benchMemAdvanced(srcBuffer, benchedSize, + NULL, 0, + fileSizes, nbFiles, + cLevel, comprParams, + dictBuffer, dictBufferSize, + displayLevel, displayName, adv); return res; } diff --git a/programs/bench.h b/programs/bench.h index 6247fa596..ad4ede1f0 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -126,8 +126,6 @@ BMK_return_t BMK_syntheticTest(int cLevel, double compressibility, * comprParams - basic compression parameters * dictBuffer - a dictionary if used, null otherwise * dictBufferSize - size of dictBuffer, 0 otherwise - * ctx - Compression Context (must be provided) - * dctx - Decompression Context (must be provided) * diplayLevel - see BMK_benchFiles * displayName - name used by display * return @@ -139,7 +137,6 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize, const size_t* fileSizes, unsigned nbFiles, const int cLevel, const ZSTD_compressionParameters* comprParams, const void* dictBuffer, size_t dictBufferSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, int displayLevel, const char* displayName); /* See benchMem for normal parameter uses and return, see advancedParams_t for adv @@ -151,7 +148,6 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, const size_t* fileSizes, unsigned nbFiles, const int cLevel, const ZSTD_compressionParameters* comprParams, const void* dictBuffer, size_t dictBufferSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, int displayLevel, const char* displayName, const BMK_advancedParams_t* adv); diff --git a/tests/paramgrill.c b/tests/paramgrill.c index eeafc8302..e530af735 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -300,10 +300,9 @@ const char* g_stratName[ZSTD_btultra+1] = { static int BMK_benchParam1(BMK_result_t* resultPtr, const void* srcBuffer, size_t srcSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams) { - BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File"); + BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, &srcSize, 1, 0, &cParams, NULL, 0, 0, "File"); *resultPtr = res.result; return res.error; } @@ -352,7 +351,7 @@ static int BMK_benchParam(BMK_result_t* resultPtr, const buffers_t buf, const contexts_t ctx, const ZSTD_compressionParameters cParams) { - BMK_return_t res = BMK_benchMem(buf.srcPtrs[0], buf.srcSize, buf.srcSizes, (unsigned)buf.nbBlocks, 0, &cParams, ctx.dictBuffer, ctx.dictSize, ctx.cctx, ctx.dctx, 0, "Files"); + BMK_return_t res = BMK_benchMem(buf.srcPtrs[0], buf.srcSize, buf.srcSizes, (unsigned)buf.nbBlocks, 0, &cParams, ctx.dictBuffer, ctx.dictSize, 0, "Files"); *resultPtr = res.result; return res.error; } @@ -724,14 +723,13 @@ static void BMK_init_level_constraints(int bytePerSec_level1) } static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters params, - const void* srcBuffer, size_t srcSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx) + const void* srcBuffer, size_t srcSize) { BMK_result_t testResult; int better = 0; int cLevel; - BMK_benchParam1(&testResult, srcBuffer, srcSize, ctx, dctx, params); + BMK_benchParam1(&testResult, srcBuffer, srcSize, params); for (cLevel = 1; cLevel <= NB_LEVELS_TRACKED; cLevel++) { @@ -1104,8 +1102,7 @@ static BYTE* NB_TESTS_PLAYED(ZSTD_compressionParameters p) { static void playAround(FILE* f, winnerInfo_t* winners, ZSTD_compressionParameters params, - const void* srcBuffer, size_t srcSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx) + const void* srcBuffer, size_t srcSize) { int nbVariations = 0; UTIL_time_t const clockStart = UTIL_getTime(); @@ -1126,11 +1123,11 @@ static void playAround(FILE* f, winnerInfo_t* winners, /* test */ b = NB_TESTS_PLAYED(p); (*b)++; - if (!BMK_seed(winners, p, srcBuffer, srcSize, ctx, dctx)) continue; + if (!BMK_seed(winners, p, srcBuffer, srcSize)) continue; /* improvement found => search more */ BMK_printWinners(f, winners, srcSize); - playAround(f, winners, p, srcBuffer, srcSize, ctx, dctx); + playAround(f, winners, p, srcBuffer, srcSize); } } @@ -1177,31 +1174,30 @@ static void randomConstrainedParams(ZSTD_compressionParameters* pc, varInds_t* v static void BMK_selectRandomStart( FILE* f, winnerInfo_t* winners, - const void* srcBuffer, size_t srcSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx) + const void* srcBuffer, size_t srcSize) { U32 const id = FUZ_rand(&g_rand) % (NB_LEVELS_TRACKED+1); if ((id==0) || (winners[id].params.windowLog==0)) { /* use some random entry */ ZSTD_compressionParameters const p = ZSTD_adjustCParams(randomParams(), srcSize, 0); - playAround(f, winners, p, srcBuffer, srcSize, ctx, dctx); + playAround(f, winners, p, srcBuffer, srcSize); } else { - playAround(f, winners, winners[id].params, srcBuffer, srcSize, ctx, dctx); + playAround(f, winners, winners[id].params, srcBuffer, srcSize); } } -static void BMK_benchOnce(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* srcBuffer, size_t srcSize) +static void BMK_benchOnce(const void* srcBuffer, size_t srcSize) { BMK_result_t testResult; g_params = ZSTD_adjustCParams(g_params, srcSize, 0); - BMK_benchParam1(&testResult, srcBuffer, srcSize, cctx, dctx, g_params); + BMK_benchParam1(&testResult, srcBuffer, srcSize, g_params); DISPLAY("Compression Ratio: %.3f Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)srcSize / testResult.cSize, (double)testResult.cSpeed / 1000000, (double)testResult.dSpeed / 1000000); return; } -static void BMK_benchFullTable(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* srcBuffer, size_t srcSize) +static void BMK_benchFullTable(const void* srcBuffer, size_t srcSize) { ZSTD_compressionParameters params; winnerInfo_t winners[NB_LEVELS_TRACKED+1]; @@ -1220,7 +1216,7 @@ static void BMK_benchFullTable(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* src /* baseline config for level 1 */ ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, blockSize, 0); BMK_result_t testResult; - BMK_benchParam1(&testResult, srcBuffer, srcSize, cctx, dctx, l1params); + BMK_benchParam1(&testResult, srcBuffer, srcSize, l1params); BMK_init_level_constraints((int)((testResult.cSpeed * 31) / 32)); } @@ -1229,14 +1225,14 @@ static void BMK_benchFullTable(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* src int i; for (i=0; i<=maxSeeds; i++) { params = ZSTD_getCParams(i, blockSize, 0); - BMK_seed(winners, params, srcBuffer, srcSize, cctx, dctx); + BMK_seed(winners, params, srcBuffer, srcSize); } } BMK_printWinners(f, winners, srcSize); /* start tests */ { const time_t grillStart = time(NULL); do { - BMK_selectRandomStart(f, winners, srcBuffer, srcSize, cctx, dctx); + BMK_selectRandomStart(f, winners, srcBuffer, srcSize); } while (BMK_timeSpan(grillStart) < g_grillDuration_s); } @@ -1248,21 +1244,12 @@ static void BMK_benchFullTable(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* src fclose(f); } -static void BMK_benchMem_usingCCtx(ZSTD_CCtx* const cctx, ZSTD_DCtx* const dctx, const void* srcBuffer, size_t srcSize) +static void BMK_benchMemInit(const void* srcBuffer, size_t srcSize) { if (g_singleRun) - return BMK_benchOnce(cctx, dctx, srcBuffer, srcSize); + return BMK_benchOnce(srcBuffer, srcSize); else - return BMK_benchFullTable(cctx, dctx, srcBuffer, srcSize); -} - -static void BMK_benchMemCCtxInit(const void* srcBuffer, size_t srcSize) -{ - ZSTD_CCtx* const cctx = ZSTD_createCCtx(); - ZSTD_DCtx* const dctx = ZSTD_createDCtx(); - if (cctx==NULL || dctx==NULL) { DISPLAY("Context Creation failed \n"); exit(1); } - BMK_benchMem_usingCCtx(cctx, dctx, srcBuffer, srcSize); - ZSTD_freeCCtx(cctx); + return BMK_benchFullTable(srcBuffer, srcSize); } @@ -1280,7 +1267,7 @@ static int benchSample(void) /* bench */ DISPLAY("\r%79s\r", ""); DISPLAY("using %s %i%%: \n", name, (int)(g_compressibility*100)); - BMK_benchMemCCtxInit(origBuff, benchedSize); + BMK_benchMemInit(origBuff, benchedSize); free(origBuff); return 0; @@ -1338,7 +1325,7 @@ int benchFiles(const char** fileNamesTable, int nbFiles) /* bench */ DISPLAY("\r%79s\r", ""); DISPLAY("using %s : \n", inFileName); - BMK_benchMemCCtxInit(origBuff, benchedSize); + BMK_benchMemInit(origBuff, benchedSize); /* clean */ free(origBuff); From 3ac2c22485ab5508f47e3eab642b787af0e68b5f Mon Sep 17 00:00:00 2001 From: George Lu Date: Thu, 9 Aug 2018 16:38:32 -0700 Subject: [PATCH 18/55] Reorder declaration --- tests/paramgrill.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index e530af735..bcb6bcae3 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -2256,8 +2256,9 @@ int main(int argc, const char** argv) g_params.strategy = (ZSTD_strategy)readU32FromChar(&argument); continue; case 'L': - { argument++; - int const cLevel = readU32FromChar(&argument); + { int cLevel; + argument++; + cLevel = readU32FromChar(&argument); g_params = ZSTD_getCParams(cLevel, g_blockSize, 0); continue; } From 0cc75d6ee02efe9da7901848ce588a7f9f61b4f9 Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 13 Aug 2018 13:56:18 -0700 Subject: [PATCH 19/55] Default lvl 1 MB to 2^20 --- tests/paramgrill.c | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index bcb6bcae3..d7165021d 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -65,6 +65,7 @@ static const int g_maxNbVariations = 64; #define MIN(a,b) ( (a) < (b) ? (a) : (b) ) #define MAX(a,b) ( (a) > (b) ? (a) : (b) ) #define CUSTOM_LEVEL 99 +#define BASE_CLEVEL 1 /* indices for each of the variables */ typedef enum { @@ -302,7 +303,7 @@ BMK_benchParam1(BMK_result_t* resultPtr, const void* srcBuffer, size_t srcSize, const ZSTD_compressionParameters cParams) { - BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, &srcSize, 1, 0, &cParams, NULL, 0, 0, "File"); + BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, &srcSize, 1, BASE_CLEVEL, &cParams, NULL, 0, 0, "File"); *resultPtr = res.result; return res.error; } @@ -792,16 +793,16 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para /* too large compression speed difference for the compression benefit */ if (W_ratio > O_ratio) DISPLAY ("Compression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n", - W_ratio, (double)testResult.cSpeed / 1000000, - O_ratio, (double)winners[cLevel].result.cSpeed / 1000000., cLevel); + W_ratio, (double)testResult.cSpeed / (1 MB), + O_ratio, (double)winners[cLevel].result.cSpeed / (1 MB), cLevel); continue; } if (W_DSpeed_note < O_DSpeed_note ) { /* too large decompression speed difference for the compression benefit */ if (W_ratio > O_ratio) DISPLAY ("Decompression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n", - W_ratio, (double)testResult.dSpeed / 1000000., - O_ratio, (double)winners[cLevel].result.dSpeed / 1000000., cLevel); + W_ratio, (double)testResult.dSpeed / (1 MB), + O_ratio, (double)winners[cLevel].result.dSpeed / (1 MB), cLevel); continue; } @@ -1193,7 +1194,7 @@ static void BMK_benchOnce(const void* srcBuffer, size_t srcSize) g_params = ZSTD_adjustCParams(g_params, srcSize, 0); BMK_benchParam1(&testResult, srcBuffer, srcSize, g_params); DISPLAY("Compression Ratio: %.3f Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)srcSize / testResult.cSize, - (double)testResult.cSpeed / 1000000, (double)testResult.dSpeed / 1000000); + (double)testResult.cSpeed / (1 MB), (double)testResult.dSpeed / (1 MB)); return; } @@ -1211,7 +1212,7 @@ static void BMK_benchFullTable(const void* srcBuffer, size_t srcSize) if (f==NULL) { DISPLAY("error opening %s \n", rfName); exit(1); } if (g_target) { - BMK_init_level_constraints(g_target*1000000); + BMK_init_level_constraints(g_target*(1 MB)); } else { /* baseline config for level 1 */ ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, blockSize, 0); @@ -1256,7 +1257,7 @@ static void BMK_benchMemInit(const void* srcBuffer, size_t srcSize) static int benchSample(void) { const char* const name = "Sample 10MB"; - size_t const benchedSize = 10000000; + size_t const benchedSize = (10 MB); void* origBuff = malloc(benchedSize); if (!origBuff) { perror("not enough memory"); return 12; } @@ -1354,7 +1355,7 @@ static int allBench(BMK_result_t* resultPtr, double winnerRS; /* initial benchmarking, gives exact ratio and memory, warms up future runs */ - benchres = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_both, BMK_iterMode, 1); + benchres = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_both, BMK_iterMode, 1); winnerRS = resultScore(*winnerResult, buf.srcSize, target); DEBUGOUTPUT("WinnerScore: %f\n ", winnerRS); @@ -1389,14 +1390,14 @@ static int allBench(BMK_result_t* resultPtr, /* second run, if first run is too short, gives approximate cSpeed + dSpeed */ if(loopDurationC < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_compressOnly, BMK_iterMode, 1); + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_compressOnly, BMK_iterMode, 1); if(benchres2.error) { return ERROR_RESULT; } benchres = benchres2; } if(loopDurationD < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_decodeOnly, BMK_iterMode, 1); + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_decodeOnly, BMK_iterMode, 1); if(benchres2.error) { return ERROR_RESULT; } @@ -1419,7 +1420,7 @@ static int allBench(BMK_result_t* resultPtr, /* Final full run if estimates are unclear */ if(loopDurationC < TIMELOOP_NANOSEC) { - BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_compressOnly, BMK_timeMode, 1); + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_compressOnly, BMK_timeMode, 1); if(benchres2.error) { return ERROR_RESULT; } @@ -1427,7 +1428,7 @@ static int allBench(BMK_result_t* resultPtr, } if(loopDurationD < TIMELOOP_NANOSEC) { - BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_decodeOnly, BMK_timeMode, 1); + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_decodeOnly, BMK_timeMode, 1); if(benchres2.error) { return ERROR_RESULT; } From 486e586eed5d30f6c4ec079bb403d6c15004bb9d Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 13 Aug 2018 16:13:46 -0700 Subject: [PATCH 20/55] Revert "Default lvl 1" This reverts commit 0cc75d6ee02efe9da7901848ce588a7f9f61b4f9. --- tests/paramgrill.c | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index d7165021d..bcb6bcae3 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -65,7 +65,6 @@ static const int g_maxNbVariations = 64; #define MIN(a,b) ( (a) < (b) ? (a) : (b) ) #define MAX(a,b) ( (a) > (b) ? (a) : (b) ) #define CUSTOM_LEVEL 99 -#define BASE_CLEVEL 1 /* indices for each of the variables */ typedef enum { @@ -303,7 +302,7 @@ BMK_benchParam1(BMK_result_t* resultPtr, const void* srcBuffer, size_t srcSize, const ZSTD_compressionParameters cParams) { - BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, &srcSize, 1, BASE_CLEVEL, &cParams, NULL, 0, 0, "File"); + BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, &srcSize, 1, 0, &cParams, NULL, 0, 0, "File"); *resultPtr = res.result; return res.error; } @@ -793,16 +792,16 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para /* too large compression speed difference for the compression benefit */ if (W_ratio > O_ratio) DISPLAY ("Compression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n", - W_ratio, (double)testResult.cSpeed / (1 MB), - O_ratio, (double)winners[cLevel].result.cSpeed / (1 MB), cLevel); + W_ratio, (double)testResult.cSpeed / 1000000, + O_ratio, (double)winners[cLevel].result.cSpeed / 1000000., cLevel); continue; } if (W_DSpeed_note < O_DSpeed_note ) { /* too large decompression speed difference for the compression benefit */ if (W_ratio > O_ratio) DISPLAY ("Decompression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n", - W_ratio, (double)testResult.dSpeed / (1 MB), - O_ratio, (double)winners[cLevel].result.dSpeed / (1 MB), cLevel); + W_ratio, (double)testResult.dSpeed / 1000000., + O_ratio, (double)winners[cLevel].result.dSpeed / 1000000., cLevel); continue; } @@ -1194,7 +1193,7 @@ static void BMK_benchOnce(const void* srcBuffer, size_t srcSize) g_params = ZSTD_adjustCParams(g_params, srcSize, 0); BMK_benchParam1(&testResult, srcBuffer, srcSize, g_params); DISPLAY("Compression Ratio: %.3f Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)srcSize / testResult.cSize, - (double)testResult.cSpeed / (1 MB), (double)testResult.dSpeed / (1 MB)); + (double)testResult.cSpeed / 1000000, (double)testResult.dSpeed / 1000000); return; } @@ -1212,7 +1211,7 @@ static void BMK_benchFullTable(const void* srcBuffer, size_t srcSize) if (f==NULL) { DISPLAY("error opening %s \n", rfName); exit(1); } if (g_target) { - BMK_init_level_constraints(g_target*(1 MB)); + BMK_init_level_constraints(g_target*1000000); } else { /* baseline config for level 1 */ ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, blockSize, 0); @@ -1257,7 +1256,7 @@ static void BMK_benchMemInit(const void* srcBuffer, size_t srcSize) static int benchSample(void) { const char* const name = "Sample 10MB"; - size_t const benchedSize = (10 MB); + size_t const benchedSize = 10000000; void* origBuff = malloc(benchedSize); if (!origBuff) { perror("not enough memory"); return 12; } @@ -1355,7 +1354,7 @@ static int allBench(BMK_result_t* resultPtr, double winnerRS; /* initial benchmarking, gives exact ratio and memory, warms up future runs */ - benchres = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_both, BMK_iterMode, 1); + benchres = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_both, BMK_iterMode, 1); winnerRS = resultScore(*winnerResult, buf.srcSize, target); DEBUGOUTPUT("WinnerScore: %f\n ", winnerRS); @@ -1390,14 +1389,14 @@ static int allBench(BMK_result_t* resultPtr, /* second run, if first run is too short, gives approximate cSpeed + dSpeed */ if(loopDurationC < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_compressOnly, BMK_iterMode, 1); + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_compressOnly, BMK_iterMode, 1); if(benchres2.error) { return ERROR_RESULT; } benchres = benchres2; } if(loopDurationD < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_decodeOnly, BMK_iterMode, 1); + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_decodeOnly, BMK_iterMode, 1); if(benchres2.error) { return ERROR_RESULT; } @@ -1420,7 +1419,7 @@ static int allBench(BMK_result_t* resultPtr, /* Final full run if estimates are unclear */ if(loopDurationC < TIMELOOP_NANOSEC) { - BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_compressOnly, BMK_timeMode, 1); + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_compressOnly, BMK_timeMode, 1); if(benchres2.error) { return ERROR_RESULT; } @@ -1428,7 +1427,7 @@ static int allBench(BMK_result_t* resultPtr, } if(loopDurationD < TIMELOOP_NANOSEC) { - BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_decodeOnly, BMK_timeMode, 1); + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_decodeOnly, BMK_timeMode, 1); if(benchres2.error) { return ERROR_RESULT; } From 0cea75402478ded21d2300544bcb538f4d3e3786 Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 13 Aug 2018 16:15:34 -0700 Subject: [PATCH 21/55] Revert "Reorder declaration" This reverts commit 3ac2c22485ab5508f47e3eab642b787af0e68b5f. --- tests/paramgrill.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index bcb6bcae3..e530af735 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -2256,9 +2256,8 @@ int main(int argc, const char** argv) g_params.strategy = (ZSTD_strategy)readU32FromChar(&argument); continue; case 'L': - { int cLevel; - argument++; - cLevel = readU32FromChar(&argument); + { argument++; + int const cLevel = readU32FromChar(&argument); g_params = ZSTD_getCParams(cLevel, g_blockSize, 0); continue; } From 13611249a5d1426e4902fe32b1cf913228f246fa Mon Sep 17 00:00:00 2001 From: George Lu Date: Tue, 24 Jul 2018 17:26:21 -0700 Subject: [PATCH 22/55] Table Compiling +Euclidean Metric --- tests/paramgrill.c | 186 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 179 insertions(+), 7 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index e530af735..5c654d993 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -279,6 +279,29 @@ static int compareResultLT(const BMK_result_t result1, const BMK_result_t result } +/* calculates normalized euclidean distance of result1 if it is in the first quadrant relative to lvlRes */ +static double resultDistLvl(const BMK_result_t result1, const BMK_result_t lvlRes) { + double normalizedCSpeedGain1 = result1.cSpeed / lvlRes.cSpeed - 1; + double normalizedRatioGain1 = lvlRes.cSize / result1.cSize - 1; + if(normalizedRatioGain1 < 0 || normalizedRatioGain1 < 0) { + return 0.0; + } + return normalizedRatioGain1 * normalizedRatioGain1 + normalizedCSpeedGain1 * normalizedCSpeedGain1; +} + +static int lvlFeasible(const BMK_result_t result, const BMK_result_t lvlRes) { + return lvlRes.cSpeed < result.cSpeed && lvlRes.cSize > result.cSize; +} + +/* redefines feasibility for lvl mode */ +static int compareResultLT2(const BMK_result_t result1, const BMK_result_t result2, const BMK_result_t lvltarget, size_t srcSize) { + constraint_t target = { (U32)lvltarget.cSpeed, 0, (U32)-1 }; + if(lvlFeasible(result1, lvltarget) && lvlFeasible(result2, lvltarget)) { + return resultDistLvl(result1, lvltarget) < resultDistLvl(result2, lvltarget); + } + return lvlFeasible(result2, lvltarget) || (!lvlFeasible(result1, lvltarget) && (resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target))); +} + /* factor sort of arbitrary */ static constraint_t relaxTarget(constraint_t target) { target.cMem = (U32)-1; @@ -629,6 +652,132 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t } +typedef struct ll_node ll_node; +struct ll_node { + winnerInfo_t res; + ll_node* next; +}; + +static ll_node* g_winners; /* linked list sorted ascending by cSize & cSpeed */ +static BMK_result_t g_lvltarget; + +/* comparison function: */ +/* strictly better, strictly worse, equal, speed-side adv, size-side adv */ +//Maybe use compress_only for benchmark first run? +#define WORSE_RESULT 0 +#define BETTER_RESULT 1 +#define ERROR_RESULT 2 + +#define SPEED_RESULT 4 +#define SIZE_RESULT 5 +static int speedSizeCompare(BMK_result_t r1, BMK_result_t r2) { + if(r1.cSpeed > r2.cSpeed) { + if(r1.cSize <= r2.cSize) { + return WORSE_RESULT; + } + return SIZE_RESULT; /* r2 is smaller but not faster. */ + } else { + if(r1.cSize >= r2.cSize) { + return BETTER_RESULT; + } + return SPEED_RESULT; /* r2 is faster but not smaller */ + } +} +/* assumes candidate is already strictly better than old winner. */ +/* 0 for success, 1 for no insert */ +/* indicate whether inserted as well? */ +/* maintain invariant speedSizeCompare(n, n->next) = SPEED_RESULT */ +static int insertWinner(winnerInfo_t w) { + BMK_result_t r = w.result; + ll_node* cur_node = g_winners; + /* first node to insert */ + if(!lvlFeasible(r, g_lvltarget)) { + return 1; + } + + if(g_winners == NULL) { + ll_node* first_node = malloc(sizeof(ll_node)); + if(first_node == NULL) { + return 1; + } + first_node->next = NULL; + first_node->res = w; + g_winners = first_node; + return 0; + } + + while(cur_node->next != NULL) { + switch(speedSizeCompare(r, cur_node->res.result)) { + case BETTER_RESULT: + { + return 1; /* never insert if better */ + } + case WORSE_RESULT: + { + ll_node* tmp; + cur_node->res = cur_node->next->res; + tmp = cur_node->next; + cur_node->next = cur_node->next->next; + free(tmp); + break; + } + case SPEED_RESULT: + cur_node = cur_node->next; + case SIZE_RESULT: /* insert after first size result, then return */ + { + ll_node* newnode = malloc(sizeof(ll_node)); + if(newnode == NULL) { + return 1; + } + newnode->res = cur_node->res; + cur_node->res = w; + newnode->next = cur_node->next; + cur_node->next = newnode; + return 0; + } + } + + } + + //assert(cur_node->next == NULL) + switch(speedSizeCompare(r, cur_node->res.result)) { + case BETTER_RESULT: + { + return 1; /* never insert if better */ + } + case WORSE_RESULT: + { + cur_node->res = w; + return 0; + } + case SPEED_RESULT: + { + ll_node* newnode = malloc(sizeof(ll_node)); + if(newnode == NULL) { + return 1; + } + newnode->res = w; + newnode->next = NULL; + cur_node->next = newnode; + return 0; + } + case SIZE_RESULT: /* insert before first size result, then return */ + { + ll_node* newnode = malloc(sizeof(ll_node)); + if(newnode == NULL) { + return 1; + } + newnode->res = cur_node->res; + cur_node->res = w; + newnode->next = cur_node->next; + cur_node->next = newnode; + return 0; + } + default: + return 1; + } +} + static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const size_t srcSize) { char lvlstr[15] = "Custom Level"; @@ -658,6 +807,29 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res /* global winner used for constraints */ static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; + /* print lvl if optmode */ + if(g_lvltarget.cSize != 0) { + winnerInfo_t w; + ll_node* n; + int i; + w.result = result; + w.params = params; + i = insertWinner(w); + if(i) return; + + fprintf(f, "\033c"); + for(n = g_winners; n != NULL; n = n->next) { + DISPLAY("\r%79s\r", ""); + fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", + params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, + params.targetLength, g_stratName[(U32)(params.strategy)]); + fprintf(f, + " /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", + (double)srcSize / result.cSize, result.cSpeed / (1 << 20), result.dSpeed / (1 << 20)); + } + return; + } + if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { if(DEBUG && compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { DISPLAY("New Winner: \n"); @@ -1334,12 +1506,6 @@ int benchFiles(const char** fileNamesTable, int nbFiles) return 0; } - - -#define WORSE_RESULT 0 -#define BETTER_RESULT 1 -#define ERROR_RESULT 2 - /* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */ #define VARIANCE 1.1 static int allBench(BMK_result_t* resultPtr, @@ -1939,6 +2105,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ goto _cleanUp; } + /* use level'ing mode instead of normal target mode */ if(cLevel) { winner.params = ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize); if(BMK_benchParam(&winner.result, buf, ctx, winner.params)) { @@ -1947,6 +2114,11 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } target.cSpeed = (U32)winner.result.cSpeed; + + g_targetConstraints = target; + + g_lvltarget = winner.result; + BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize); } @@ -2137,8 +2309,8 @@ int main(int argc, const char** argv) U32 main_pause = 0; int optimizerCLevel = 0; - constraint_t target = { 0, 0, (U32)-1 }; + ZSTD_compressionParameters paramTarget = { 0, 0, 0, 0, 0, 0, 0 }; assert(argc>=1); /* for exename */ From 5f4502fc0775f10175e759a7987289a80d9dbd92 Mon Sep 17 00:00:00 2001 From: George Lu Date: Tue, 24 Jul 2018 17:55:17 -0700 Subject: [PATCH 23/55] New climb feas part 2 uses euclidean metric --- tests/paramgrill.c | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 5c654d993..ffb5c1f7d 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -1518,6 +1518,7 @@ static int allBench(BMK_result_t* resultPtr, U64 loopDurationC = 0, loopDurationD = 0; double uncertaintyConstantC, uncertaintyConstantD; double winnerRS; + int lvlmode = g_lvltarget.cSize != 0; /* initial benchmarking, gives exact ratio and memory, warms up future runs */ benchres = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_both, BMK_iterMode, 1); @@ -1548,9 +1549,11 @@ static int allBench(BMK_result_t* resultPtr, uncertaintyConstantD = 3; } + if(!lvlmode) { /* anything with worse ratio in feas is definitely worse, discard */ - if(feas && benchres.result.cSize < winnerResult->cSize) { - return WORSE_RESULT; + if(feas && benchres.result.cSize < winnerResult->cSize) { + return WORSE_RESULT; + } } /* second run, if first run is too short, gives approximate cSpeed + dSpeed */ @@ -1578,8 +1581,9 @@ static int allBench(BMK_result_t* resultPtr, /* disregard infeasible results in feas mode */ /* disregard if resultMax < winner in infeas mode */ - if((feas && !feasible(resultMax, target)) || - (!feas && (winnerRS > resultScore(resultMax, buf.srcSize, target)))) { + if((feas && (!lvlmode && !feasible(resultMax, target))) || + (!feas && ((!lvlmode && winnerRS > resultScore(resultMax, buf.srcSize, target)) || + (lvlmode && resultDistLvl(*winnerResult, g_lvltarget) > resultDistLvl(resultMax, g_lvltarget))))) { return WORSE_RESULT; } @@ -1604,11 +1608,20 @@ static int allBench(BMK_result_t* resultPtr, /* compare by resultScore when in infeas */ /* compare by compareResultLT when in feas */ - if((!feas && (resultScore(benchres.result, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target))) || - (feas && (compareResultLT(*winnerResult, benchres.result, target, buf.srcSize))) ) { - return BETTER_RESULT; - } else { - return WORSE_RESULT; + if(!lvlmode) { + if((!feas && (resultScore(benchres.result, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target))) || + (feas && (compareResultLT(*winnerResult, benchres.result, target, buf.srcSize))) ) { + return BETTER_RESULT; + } else { + return WORSE_RESULT; + } + } else { + if((feas && (compareResultLT2(*winnerResult, benchres.result, g_lvltarget, buf.srcSize))) || + (!feas && (resultScore(benchres.result, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target)))) { + return BETTER_RESULT; + } else { + return WORSE_RESULT; + } } } From f67d040c39af10f68d7b6c61b06e72e1273551ff Mon Sep 17 00:00:00 2001 From: George Lu Date: Wed, 25 Jul 2018 11:37:20 -0700 Subject: [PATCH 24/55] Bugfixes, style changes Complete euclidean distance climb --- tests/paramgrill.c | 191 +++++++++++++++++++++++++-------------------- 1 file changed, 108 insertions(+), 83 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index ffb5c1f7d..0dbe66534 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -112,6 +112,36 @@ static U32 g_noSeed = 0; static ZSTD_compressionParameters g_params = { 0, 0, 0, 0, 0, 0, ZSTD_greedy }; static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */ + +typedef struct { + BMK_result_t result; + ZSTD_compressionParameters params; +} winnerInfo_t; + +/* global winner used for display. */ +//Should be totally 0 initialized? +static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; + +typedef struct { + U32 cSpeed; /* bytes / sec */ + U32 dSpeed; + U32 cMem; /* bytes */ +} constraint_t; + +static constraint_t g_targetConstraints; + +typedef struct ll_node ll_node; +struct ll_node { + winnerInfo_t res; + ll_node* next; +}; + +static ll_node* g_winners; /* linked list sorted ascending by cSize & cSpeed */ +static BMK_result_t g_lvltarget; + +/* range 0 - 99 */ +static U32 g_strictness = 99; + void BMK_SetNbIterations(int nbLoops) { g_nbIterations = nbLoops; @@ -197,12 +227,6 @@ static void findClockGranularity(void) { DEBUGOUTPUT("Granularity: %llu\n", (unsigned long long)g_clockGranularity); } -typedef struct { - U32 cSpeed; /* bytes / sec */ - U32 dSpeed; - U32 cMem; /* bytes */ -} constraint_t; - #define CLAMPCHECK(val,min,max) { \ if (val && (((val)<(min)) | ((val)>(max)))) { \ DISPLAY("INVALID PARAMETER CONSTRAINTS\n"); \ @@ -246,7 +270,7 @@ static void BMK_translateAdvancedParams(const ZSTD_compressionParameters params) /* checks results are feasible */ static int feasible(const BMK_result_t results, const constraint_t target) { - return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem); + return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem) && (!g_lvltarget.cSize || results.cSize <= g_lvltarget.cSize); } /* hill climbing value for part 1 */ @@ -269,44 +293,33 @@ static double resultScore(const BMK_result_t res, const size_t srcSize, const co return ret; } -/* return true if r2 strictly better than r1 */ -static int compareResultLT(const BMK_result_t result1, const BMK_result_t result2, const constraint_t target, size_t srcSize) { - if(feasible(result1, target) && feasible(result2, target)) { - return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed) - || (result1.cSize == result2.cSize && result2.cSpeed == result1.cSpeed && result2.dSpeed > result1.dSpeed); - } - return feasible(result2, target) || (!feasible(result1, target) && (resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target))); - -} - /* calculates normalized euclidean distance of result1 if it is in the first quadrant relative to lvlRes */ static double resultDistLvl(const BMK_result_t result1, const BMK_result_t lvlRes) { - double normalizedCSpeedGain1 = result1.cSpeed / lvlRes.cSpeed - 1; - double normalizedRatioGain1 = lvlRes.cSize / result1.cSize - 1; - if(normalizedRatioGain1 < 0 || normalizedRatioGain1 < 0) { + double normalizedCSpeedGain1 = (result1.cSpeed / lvlRes.cSpeed) - 1; + double normalizedRatioGain1 = ((double)lvlRes.cSize / result1.cSize) - 1; + if(normalizedRatioGain1 < 0 || normalizedCSpeedGain1 < 0) { return 0.0; } return normalizedRatioGain1 * normalizedRatioGain1 + normalizedCSpeedGain1 * normalizedCSpeedGain1; } -static int lvlFeasible(const BMK_result_t result, const BMK_result_t lvlRes) { - return lvlRes.cSpeed < result.cSpeed && lvlRes.cSize > result.cSize; -} - -/* redefines feasibility for lvl mode */ -static int compareResultLT2(const BMK_result_t result1, const BMK_result_t result2, const BMK_result_t lvltarget, size_t srcSize) { - constraint_t target = { (U32)lvltarget.cSpeed, 0, (U32)-1 }; - if(lvlFeasible(result1, lvltarget) && lvlFeasible(result2, lvltarget)) { - return resultDistLvl(result1, lvltarget) < resultDistLvl(result2, lvltarget); +/* return true if r2 strictly better than r1 */ +static int compareResultLT(const BMK_result_t result1, const BMK_result_t result2, const constraint_t target, size_t srcSize) { + if(feasible(result1, target) && feasible(result2, target)) { + if(g_lvltarget.cSize == 0) { + return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed) + || (result1.cSize == result2.cSize && result2.cSpeed == result1.cSpeed && result2.dSpeed > result1.dSpeed); + } else { + return resultDistLvl(result1, g_lvltarget) < resultDistLvl(result2, g_lvltarget); + } } - return lvlFeasible(result2, lvltarget) || (!lvlFeasible(result1, lvltarget) && (resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target))); + return feasible(result2, target) || (!feasible(result1, target) && (resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target))); } -/* factor sort of arbitrary */ static constraint_t relaxTarget(constraint_t target) { target.cMem = (U32)-1; - target.cSpeed *= 0.9; - target.dSpeed *= 0.9; + target.cSpeed *= ((double)(g_strictness + 1) / 100); + target.dSpeed *= ((double)(g_strictness + 1) / 100); return target; } @@ -330,11 +343,6 @@ BMK_benchParam1(BMK_result_t* resultPtr, return res.error; } -typedef struct { - BMK_result_t result; - ZSTD_compressionParameters params; -} winnerInfo_t; - static ZSTD_compressionParameters emptyParams(void) { ZSTD_compressionParameters p = { 0, 0, 0, 0, 0, 0, (ZSTD_strategy)0 }; return p; @@ -651,16 +659,6 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t return results; } - -typedef struct ll_node ll_node; -struct ll_node { - winnerInfo_t res; - ll_node* next; -}; - -static ll_node* g_winners; /* linked list sorted ascending by cSize & cSpeed */ -static BMK_result_t g_lvltarget; - /* comparison function: */ /* strictly better, strictly worse, equal, speed-side adv, size-side adv */ //Maybe use compress_only for benchmark first run? @@ -691,7 +689,7 @@ static int insertWinner(winnerInfo_t w) { BMK_result_t r = w.result; ll_node* cur_node = g_winners; /* first node to insert */ - if(!lvlFeasible(r, g_lvltarget)) { + if(!feasible(r, g_targetConstraints)) { return 1; } @@ -722,7 +720,10 @@ static int insertWinner(winnerInfo_t w) { break; } case SPEED_RESULT: + { cur_node = cur_node->next; + break; + } case SIZE_RESULT: /* insert after first size result, then return */ { ll_node* newnode = malloc(sizeof(ll_node)); @@ -843,6 +844,34 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res g_winner.params = params; } } + + //prints out tradeoff table if using lvl + if(g_lvltarget.cSize != 0) { + winnerInfo_t w; + ll_node* n; + int i; + w.result = result; + w.params = params; + i = insertWinner(w); + if(i) return; + + if(!DEBUG) { fprintf(f, "\033c"); } + fprintf(f, "\n"); + + /* the table */ + fprintf(f, "================================\n"); + for(n = g_winners; n != NULL; n = n->next) { + DISPLAY("\r%79s\r", ""); + + fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", + n->res.params.windowLog, n->res.params.chainLog, n->res.params.hashLog, n->res.params.searchLog, n->res.params.searchLength, + n->res.params.targetLength, g_stratName[(U32)(n->res.params.strategy)]); + fprintf(f, + " /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", + (double)srcSize / n->res.result.cSize, n->res.result.cSpeed / (1 << 20), n->res.result.dSpeed / (1 << 20)); + } + fprintf(f, "================================\n"); + } } static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize) @@ -1518,7 +1547,6 @@ static int allBench(BMK_result_t* resultPtr, U64 loopDurationC = 0, loopDurationD = 0; double uncertaintyConstantC, uncertaintyConstantD; double winnerRS; - int lvlmode = g_lvltarget.cSize != 0; /* initial benchmarking, gives exact ratio and memory, warms up future runs */ benchres = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_both, BMK_iterMode, 1); @@ -1549,11 +1577,9 @@ static int allBench(BMK_result_t* resultPtr, uncertaintyConstantD = 3; } - if(!lvlmode) { /* anything with worse ratio in feas is definitely worse, discard */ - if(feas && benchres.result.cSize < winnerResult->cSize) { - return WORSE_RESULT; - } + if(feas && benchres.result.cSize < winnerResult->cSize && g_lvltarget.cSize == 0) { + return WORSE_RESULT; } /* second run, if first run is too short, gives approximate cSpeed + dSpeed */ @@ -1581,9 +1607,8 @@ static int allBench(BMK_result_t* resultPtr, /* disregard infeasible results in feas mode */ /* disregard if resultMax < winner in infeas mode */ - if((feas && (!lvlmode && !feasible(resultMax, target))) || - (!feas && ((!lvlmode && winnerRS > resultScore(resultMax, buf.srcSize, target)) || - (lvlmode && resultDistLvl(*winnerResult, g_lvltarget) > resultDistLvl(resultMax, g_lvltarget))))) { + if((feas && !feasible(resultMax, target)) || + (!feas && (winnerRS > resultScore(resultMax, buf.srcSize, target)))) { return WORSE_RESULT; } @@ -1608,22 +1633,12 @@ static int allBench(BMK_result_t* resultPtr, /* compare by resultScore when in infeas */ /* compare by compareResultLT when in feas */ - if(!lvlmode) { - if((!feas && (resultScore(benchres.result, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target))) || - (feas && (compareResultLT(*winnerResult, benchres.result, target, buf.srcSize))) ) { - return BETTER_RESULT; - } else { - return WORSE_RESULT; - } - } else { - if((feas && (compareResultLT2(*winnerResult, benchres.result, g_lvltarget, buf.srcSize))) || - (!feas && (resultScore(benchres.result, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target)))) { - return BETTER_RESULT; - } else { - return WORSE_RESULT; - } + if((!feas && (resultScore(benchres.result, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target))) || + (feas && (compareResultLT(*winnerResult, benchres.result, target, buf.srcSize))) ) { + return BETTER_RESULT; + } else { + return WORSE_RESULT; } - } #define INFEASIBLE_THRESHOLD 200 @@ -1655,6 +1670,7 @@ static int benchMemo(BMK_result_t* resultPtr, return res; } + /* One iteration of hill climbing. Specifically, it first tries all * valid parameter configurations w/ manhattan distance 1 and picks the best one * failing that, it progressively tries candidates further and further away (up to #dim + 2) @@ -1667,6 +1683,10 @@ static int benchMemo(BMK_result_t* resultPtr, * Phase 2 optimizes in accordance with what the original function sets out to maximize, with * all feasible solutions valued over all infeasible solutions. */ + +/* sanitize all params here. + * all generation after random should be sanitized. (maybe sanitize random) + */ static winnerInfo_t climbOnce(const constraint_t target, const varInds_t* varArray, const int varLen, U8* const memoTable, @@ -1800,6 +1820,7 @@ static winnerInfo_t optimizeFixedStrategy( BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize); i = 0; } + i++; } return winnerInfo; @@ -2129,8 +2150,10 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ target.cSpeed = (U32)winner.result.cSpeed; g_targetConstraints = target; - - g_lvltarget = winner.result; + + g_lvltarget = winner.result; + g_lvltarget.cSpeed *= ((double)(g_strictness + 1) / 100); + g_lvltarget.cSize /= ((double)(g_strictness + 1) / 100); BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize); } @@ -2299,15 +2322,16 @@ static int badusage(const char* exename) } #define PARSE_SUB_ARGS(stringLong, stringShort, variable) { if (longCommandWArg(&argument, stringLong) || longCommandWArg(&argument, stringShort)) { variable = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } } -#define PARSE_CPARAMS(variable) \ -{ \ - PARSE_SUB_ARGS("windowLog=", "wlog=", variable.windowLog); \ - PARSE_SUB_ARGS("chainLog=" , "clog=", variable.chainLog); \ - PARSE_SUB_ARGS("hashLog=", "hlog=", variable.hashLog); \ - PARSE_SUB_ARGS("searchLog=" , "slog=", variable.searchLog); \ - PARSE_SUB_ARGS("searchLength=", "slen=", variable.searchLength); \ - PARSE_SUB_ARGS("targetLength=" , "tlen=", variable.targetLength); \ - PARSE_SUB_ARGS("strategy=", "strat=", variable.strategy); \ +#define PARSE_CPARAMS(variable) \ +{ \ + PARSE_SUB_ARGS("windowLog=", "wlog=", variable.vals[wlog_ind]); \ + PARSE_SUB_ARGS("chainLog=" , "clog=", variable.vals[clog_ind]); \ + PARSE_SUB_ARGS("hashLog=", "hlog=", variable.vals[hlog_ind]); \ + PARSE_SUB_ARGS("searchLog=" , "slog=", variable.vals[slog_ind]); \ + PARSE_SUB_ARGS("searchLength=", "slen=", variable.vals[slen_ind]); \ + PARSE_SUB_ARGS("targetLength=" , "tlen=", variable.vals[tlen_ind]); \ + PARSE_SUB_ARGS("strategy=", "strat=", variable.vals[strt_ind]); \ + PARSE_SUB_ARGS("forceAttachDict=", "fad=" , variable.vals[strt_ind]); \ } int main(int argc, const char** argv) @@ -2350,6 +2374,7 @@ int main(int argc, const char** argv) PARSE_SUB_ARGS("decompressionSpeed=", "dSpeed=", target.dSpeed); PARSE_SUB_ARGS("compressionMemory=" , "cMem=", target.cMem); PARSE_SUB_ARGS("level=", "lvl=", optimizerCLevel); + PARSE_SUB_ARGS("strict=", "stc=", g_strictness); DISPLAY("invalid optimization parameter \n"); return 1; } From 2bdfe6ca71d2171c02468214759080bc1d3e7e3a Mon Sep 17 00:00:00 2001 From: George Lu Date: Wed, 25 Jul 2018 11:55:09 -0700 Subject: [PATCH 25/55] Better Display --- tests/paramgrill.c | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 0dbe66534..fa5a5390a 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -293,7 +293,7 @@ static double resultScore(const BMK_result_t res, const size_t srcSize, const co return ret; } -/* calculates normalized euclidean distance of result1 if it is in the first quadrant relative to lvlRes */ +/* calculates normalized squared euclidean distance of result1 if it is in the first quadrant relative to lvlRes */ static double resultDistLvl(const BMK_result_t result1, const BMK_result_t lvlRes) { double normalizedCSpeedGain1 = (result1.cSpeed / lvlRes.cSpeed) - 1; double normalizedRatioGain1 = ((double)lvlRes.cSize / result1.cSize) - 1; @@ -668,6 +668,7 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t #define SPEED_RESULT 4 #define SIZE_RESULT 5 +/* maybe have epsilon-eq to limit table size? */ static int speedSizeCompare(BMK_result_t r1, BMK_result_t r2) { if(r1.cSpeed > r2.cSpeed) { if(r1.cSize <= r2.cSize) { @@ -853,11 +854,11 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res w.result = result; w.params = params; i = insertWinner(w); - if(i) return; + //if(i) return; if(!DEBUG) { fprintf(f, "\033c"); } fprintf(f, "\n"); - + /* the table */ fprintf(f, "================================\n"); for(n = g_winners; n != NULL; n = n->next) { @@ -871,6 +872,27 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res (double)srcSize / n->res.result.cSize, n->res.result.cSpeed / (1 << 20), n->res.result.dSpeed / (1 << 20)); } fprintf(f, "================================\n"); + fprintf(f, "Level Bounds: R: > %.3f AND C: < %.1f MB/s \n\n", + (double)srcSize / g_lvltarget.cSize, g_lvltarget.cSpeed / (1 << 20)); + + + fprintf(f, "Overall Winner: \n"); + fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", + g_winner.params.windowLog, g_winner.params.chainLog, g_winner.params.hashLog, g_winner.params.searchLog, g_winner.params.searchLength, + g_winner.params.targetLength, g_stratName[(U32)(g_winner.params.strategy)]); + fprintf(f, + " /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", + (double)srcSize / g_winner.result.cSize, g_winner.result.cSpeed / (1 << 20), g_winner.result.dSpeed / (1 << 20)); + + + fprintf(f, "Latest BMK: \n"); + fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", + params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, + params.targetLength, g_stratName[(U32)(params.strategy)]); + fprintf(f, + " /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", + (double)srcSize / result.cSize, result.cSpeed / (1 << 20), result.dSpeed / (1 << 20)); + } } @@ -1707,7 +1729,7 @@ static winnerInfo_t climbOnce(const constraint_t target, { winnerInfo_t bestFeasible1 = initWinnerInfo(cparam); - DISPLAY("Climb Part 1\n"); + DEBUGOUTPUT("Climb Part 1\n"); while(better) { int i, dist, offset; @@ -1774,7 +1796,7 @@ static winnerInfo_t climbOnce(const constraint_t target, feas = 1; better = 1; winnerInfo = bestFeasible1; /* note with change, bestFeasible may not necessarily be feasible, but if one has been benchmarked, it will be. */ - DISPLAY("Climb Part 2\n"); + DEBUGOUTPUT("Climb Part 2\n"); } } winnerInfo = bestFeasible1; @@ -2218,9 +2240,9 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } while(st && tries) { + DEBUGOUTPUT("StrategySwitch: %s\n", g_stratName[st]); winnerInfo_t wc = optimizeFixedStrategy(buf, ctx, target, paramTarget, st, varArray, varLen, allMT[st], tries); - DEBUGOUTPUT("StratNum %d\n", st); if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) { winner = wc; } From 3a2e95eba45a28afa57afae00dac15dce29959e9 Mon Sep 17 00:00:00 2001 From: George Lu Date: Thu, 26 Jul 2018 16:45:00 -0700 Subject: [PATCH 26/55] Perf improvements try decay strategy selection skipping --- tests/paramgrill.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index fa5a5390a..0db8510e1 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -2079,7 +2079,9 @@ static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZS return base; } +/* experiment with playing with this and decay value */ #define MAX_TRIES 8 +#define TRY_DECAY 3 /* main fn called when using --optimize */ /* Does strategy selection by benchmarking default compression levels * then optimizes by strategy, starting with the best one and moving @@ -2092,6 +2094,7 @@ static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZS * paramTarget - parameter constraints (i.e. restriction search space to where strategy = ZSTD_fast) * cLevel - compression level to exceed (all solutions must be > lvl in cSpeed + ratio) */ + static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel) { varInds_t varArray [NUM_PARAMS]; @@ -2202,7 +2205,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ /* strategy selection */ const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); DEBUGOUTPUT("Strategy Selection\n"); - if(paramTarget.strategy == 0) { /* no variable based constraints */ + if(paramTarget.strategy == 0) { BMK_result_t candidate; int i; for (i=1; i<=maxSeeds; i++) { @@ -2216,6 +2219,11 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ winner.result = candidate; winner.params = CParams; } + + /* if the current params are too slow, just stop. */ + if(target.cSpeed != 0 && target.cSpeed > winner.result.cSpeed / 2) { + break; + } } } } @@ -2248,7 +2256,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } st = nextStrategy(st, bestStrategy); - tries--; + tries -= TRY_DECAY; } } else { winner = optimizeFixedStrategy(buf, ctx, target, paramTarget, paramTarget.strategy, From 8ff0de15e4484b656dc199e118b063b3665cb2dd Mon Sep 17 00:00:00 2001 From: George Lu Date: Fri, 27 Jul 2018 08:20:31 -0700 Subject: [PATCH 27/55] Generalize, macro magic numbers --- tests/README.md | 3 +++ tests/paramgrill.c | 54 +++++++++++++++++++++++++++++++--------------- 2 files changed, 40 insertions(+), 17 deletions(-) diff --git a/tests/README.md b/tests/README.md index 2f0026fda..04eb5094e 100644 --- a/tests/README.md +++ b/tests/README.md @@ -112,6 +112,9 @@ Full list of arguments dSpeed= - Minimum decompression speed cMem= - compression memory lvl= - Automatically sets compression speed constraint to the speed of that level + stc= - In lvl mode, represents slack in ratio/cSpeed allowed for a solution to be considered + - In normal operation, represents slack in strategy selection in choosing the default parameters + --optimize= : same as -O with more verbose syntax -P# : generated sample compressibility -t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) -v : Prints Benchmarking output diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 0db8510e1..fca6b2002 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -139,8 +139,9 @@ struct ll_node { static ll_node* g_winners; /* linked list sorted ascending by cSize & cSpeed */ static BMK_result_t g_lvltarget; -/* range 0 - 99 */ -static U32 g_strictness = 99; +/* range 0 - 99, measure of how strict */ +#define DEFAULT_STRICTNESS 99999 +static U32 g_strictness = DEFAULT_STRICTNESS; void BMK_SetNbIterations(int nbLoops) { @@ -318,8 +319,8 @@ static int compareResultLT(const BMK_result_t result1, const BMK_result_t result static constraint_t relaxTarget(constraint_t target) { target.cMem = (U32)-1; - target.cSpeed *= ((double)(g_strictness + 1) / 100); - target.dSpeed *= ((double)(g_strictness + 1) / 100); + target.cSpeed *= ((double)(g_strictness) / 100); + target.dSpeed *= ((double)(g_strictness) / 100); return target; } @@ -2080,8 +2081,7 @@ static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZS } /* experiment with playing with this and decay value */ -#define MAX_TRIES 8 -#define TRY_DECAY 3 + /* main fn called when using --optimize */ /* Does strategy selection by benchmarking default compression levels * then optimizes by strategy, starting with the best one and moving @@ -2095,6 +2095,9 @@ static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZS * cLevel - compression level to exceed (all solutions must be > lvl in cSpeed + ratio) */ +#define MAX_TRIES 3 +#define TRY_DECAY 1 + static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel) { varInds_t varArray [NUM_PARAMS]; @@ -2164,6 +2167,21 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ goto _cleanUp; } + /* default strictness = Maximum for */ + if(g_strictness == DEFAULT_STRICTNESS) { + if(cLevel) { + g_strictness = 99; + } else { + g_strictness = 90; + } + } else { + if(0 >= g_strictness || g_strictness > 100) { + DISPLAY("Strictness Outside of Bounds\n"); + ret = 4; + goto _cleanUp; + } + } + /* use level'ing mode instead of normal target mode */ if(cLevel) { winner.params = ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize); @@ -2177,8 +2195,8 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ g_targetConstraints = target; g_lvltarget = winner.result; - g_lvltarget.cSpeed *= ((double)(g_strictness + 1) / 100); - g_lvltarget.cSize /= ((double)(g_strictness + 1) / 100); + g_lvltarget.cSpeed *= ((double)(g_strictness) / 100); + g_lvltarget.cSize /= ((double)(g_strictness) / 100); BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize); } @@ -2199,6 +2217,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ { varInds_t varNew[NUM_PARAMS]; + ZSTD_compressionParameters CParams; /* find best solution from default params */ { @@ -2210,8 +2229,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ int i; for (i=1; i<=maxSeeds; i++) { int ec; - ZSTD_compressionParameters CParams = ZSTD_getCParams(i, maxBlockSize, ctx.dictSize); - CParams = maskParams(CParams, paramTarget); + CParams = maskParams(ZSTD_getCParams(i, maxBlockSize, ctx.dictSize), paramTarget); ec = BMK_benchParam(&candidate, buf, ctx, CParams); BMK_printWinnerOpt(stdout, i, candidate, CParams, target, buf.srcSize); @@ -2221,9 +2239,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } /* if the current params are too slow, just stop. */ - if(target.cSpeed != 0 && target.cSpeed > winner.result.cSpeed / 2) { - break; - } + if(target.cSpeed > candidate.cSpeed * 2) { break; } } } } @@ -2239,6 +2255,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ int tries = MAX_TRIES; { + /* one iterations of hill climbing with the level-defined parameters. */ int varLenNew = sanitizeVarArray(varNew, varLen, varArray, st); winnerInfo_t w1 = climbOnce(target, varNew, varLenNew, allMT[st], buf, ctx, winner.params); @@ -2247,16 +2264,19 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } } - while(st && tries) { + while(st && tries > 0) { DEBUGOUTPUT("StrategySwitch: %s\n", g_stratName[st]); winnerInfo_t wc = optimizeFixedStrategy(buf, ctx, target, paramTarget, st, varArray, varLen, allMT[st], tries); + if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) { winner = wc; + tries = MAX_TRIES; + bestStrategy = st; + } else { + st = nextStrategy(st, bestStrategy); + tries -= TRY_DECAY; } - - st = nextStrategy(st, bestStrategy); - tries -= TRY_DECAY; } } else { winner = optimizeFixedStrategy(buf, ctx, target, paramTarget, paramTarget.strategy, From b3544217b77bcff66b7bf754a4f7b745a7d0493a Mon Sep 17 00:00:00 2001 From: George Lu Date: Fri, 27 Jul 2018 11:47:14 -0700 Subject: [PATCH 28/55] Cleanup --- tests/README.md | 6 ++++-- tests/paramgrill.c | 30 ++++++++++++++++++++---------- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/tests/README.md b/tests/README.md index 04eb5094e..1410ca979 100644 --- a/tests/README.md +++ b/tests/README.md @@ -111,9 +111,11 @@ Full list of arguments cSpeed= - Minimum compression speed dSpeed= - Minimum decompression speed cMem= - compression memory - lvl= - Automatically sets compression speed constraint to the speed of that level - stc= - In lvl mode, represents slack in ratio/cSpeed allowed for a solution to be considered + lvl= - Searches for solutions which are strictly better than that compression lvl in ratio and cSpeed, + stc= - When invoked with lvl=, represents slack in ratio/cSpeed allowed for a solution to be considered - In normal operation, represents slack in strategy selection in choosing the default parameters + prefer[Speed/Ratio]= - Only affects lvl= invocations. Defines value placed on compression speed or ratio + when determining overall winner (default 1 for both). --optimize= : same as -O with more verbose syntax -P# : generated sample compressibility -t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index fca6b2002..188b04935 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -123,7 +123,7 @@ typedef struct { static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; typedef struct { - U32 cSpeed; /* bytes / sec */ + U32 cSpeed; /* bytes / sec */ U32 dSpeed; U32 cMem; /* bytes */ } constraint_t; @@ -138,6 +138,12 @@ struct ll_node { static ll_node* g_winners; /* linked list sorted ascending by cSize & cSpeed */ static BMK_result_t g_lvltarget; +static int g_optmode = 0; + +static U32 g_speedMultiplier = 1; +static U32 g_ratioMultiplier = 1; + +/* g_mode? */ /* range 0 - 99, measure of how strict */ #define DEFAULT_STRICTNESS 99999 @@ -271,7 +277,7 @@ static void BMK_translateAdvancedParams(const ZSTD_compressionParameters params) /* checks results are feasible */ static int feasible(const BMK_result_t results, const constraint_t target) { - return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem) && (!g_lvltarget.cSize || results.cSize <= g_lvltarget.cSize); + return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem) && (!g_optmode || results.cSize <= g_lvltarget.cSize); } /* hill climbing value for part 1 */ @@ -291,6 +297,7 @@ static double resultScore(const BMK_result_t res, const size_t srcSize, const co ret = (MIN(1, cs) + MIN(1, ds) + MIN(1, cm))*r1 + rt * rtr + (MAX(0, log(cs))+ MAX(0, log(ds))+ MAX(0, log(cm))) * r2; + return ret; } @@ -301,17 +308,17 @@ static double resultDistLvl(const BMK_result_t result1, const BMK_result_t lvlRe if(normalizedRatioGain1 < 0 || normalizedCSpeedGain1 < 0) { return 0.0; } - return normalizedRatioGain1 * normalizedRatioGain1 + normalizedCSpeedGain1 * normalizedCSpeedGain1; + return normalizedRatioGain1 * g_ratioMultiplier + normalizedCSpeedGain1 * g_speedMultiplier; } /* return true if r2 strictly better than r1 */ static int compareResultLT(const BMK_result_t result1, const BMK_result_t result2, const constraint_t target, size_t srcSize) { if(feasible(result1, target) && feasible(result2, target)) { - if(g_lvltarget.cSize == 0) { + if(g_optmode) { + return resultDistLvl(result1, g_lvltarget) < resultDistLvl(result2, g_lvltarget); + } else { return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed) || (result1.cSize == result2.cSize && result2.cSpeed == result1.cSpeed && result2.dSpeed > result1.dSpeed); - } else { - return resultDistLvl(result1, g_lvltarget) < resultDistLvl(result2, g_lvltarget); } } return feasible(result2, target) || (!feasible(result1, target) && (resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target))); @@ -848,7 +855,7 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res } //prints out tradeoff table if using lvl - if(g_lvltarget.cSize != 0) { + if(g_optmode) { winnerInfo_t w; ll_node* n; int i; @@ -1601,7 +1608,7 @@ static int allBench(BMK_result_t* resultPtr, } /* anything with worse ratio in feas is definitely worse, discard */ - if(feas && benchres.result.cSize < winnerResult->cSize && g_lvltarget.cSize == 0) { + if(feas && benchres.result.cSize < winnerResult->cSize && !g_optmode) { return WORSE_RESULT; } @@ -2169,7 +2176,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ /* default strictness = Maximum for */ if(g_strictness == DEFAULT_STRICTNESS) { - if(cLevel) { + if(g_optmode) { g_strictness = 99; } else { g_strictness = 90; @@ -2183,7 +2190,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } /* use level'ing mode instead of normal target mode */ - if(cLevel) { + if(g_optmode) { winner.params = ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize); if(BMK_benchParam(&winner.result, buf, ctx, winner.params)) { ret = 3; @@ -2425,6 +2432,9 @@ int main(int argc, const char** argv) PARSE_SUB_ARGS("compressionMemory=" , "cMem=", target.cMem); PARSE_SUB_ARGS("level=", "lvl=", optimizerCLevel); PARSE_SUB_ARGS("strict=", "stc=", g_strictness); + PARSE_SUB_ARGS("preferSpeed=", "prfSpd=", g_speedMultiplier); + PARSE_SUB_ARGS("preferRatio=", "prfRto=", g_ratioMultiplier); + DISPLAY("invalid optimization parameter \n"); return 1; } From a884b76bc2fedd086c36b16e52b0dc43d075ca28 Mon Sep 17 00:00:00 2001 From: George Lu Date: Fri, 27 Jul 2018 14:19:55 -0700 Subject: [PATCH 29/55] Style Changes Add single run dictionaries Change MB to be consistent 1 << 20 rather than 1,000,000 --- tests/paramgrill.c | 481 ++++++++++++++++++++++----------------------- 1 file changed, 238 insertions(+), 243 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 188b04935..8df72eb01 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -107,6 +107,7 @@ static double g_compressibility = COMPRESSIBILITY_DEFAULT; static U32 g_blockSize = 0; static U32 g_rand = 1; static U32 g_singleRun = 0; +static U32 g_optimizer = 0; static U32 g_target = 0; static U32 g_noSeed = 0; static ZSTD_compressionParameters g_params = { 0, 0, 0, 0, 0, 0, ZSTD_greedy }; @@ -118,10 +119,6 @@ typedef struct { ZSTD_compressionParameters params; } winnerInfo_t; -/* global winner used for display. */ -//Should be totally 0 initialized? -static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; - typedef struct { U32 cSpeed; /* bytes / sec */ U32 dSpeed; @@ -386,8 +383,7 @@ typedef struct { ZSTD_DCtx* dctx; } contexts_t; -static int -BMK_benchParam(BMK_result_t* resultPtr, +static int BMK_benchParam(BMK_result_t* resultPtr, const buffers_t buf, const contexts_t ctx, const ZSTD_compressionParameters cParams) { BMK_return_t res = BMK_benchMem(buf.srcPtrs[0], buf.srcSize, buf.srcSizes, (unsigned)buf.nbBlocks, 0, &cParams, ctx.dictBuffer, ctx.dictSize, 0, "Files"); @@ -520,6 +516,175 @@ static size_t local_defaultDecompress( * From Paramgrill End *********************************************************/ +static void freeBuffers(const buffers_t b) { + if(b.srcPtrs != NULL) { + free(b.srcBuffer); + } + free(b.srcPtrs); + free(b.srcSizes); + + if(b.dstPtrs != NULL) { + free(b.dstPtrs[0]); + } + free(b.dstPtrs); + free(b.dstCapacities); + free(b.dstSizes); + + if(b.resPtrs != NULL) { + free(b.resPtrs[0]); + } + free(b.resPtrs); +} + +/* allocates buffer's arguments. returns success / failuere */ +static int createBuffers(buffers_t* buff, const char* const * const fileNamesTable, + const size_t nbFiles) +{ + size_t pos = 0; + size_t n; + U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles); + size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad); + const size_t blockSize = g_blockSize ? g_blockSize : totalSizeToLoad; //(largest fileSize or total fileSize) + U32 const maxNbBlocks = (U32) ((totalSizeToLoad + (blockSize-1)) / blockSize) + (U32)nbFiles; + U32 blockNb = 0; + + buff->srcPtrs = (const void**)calloc(maxNbBlocks, sizeof(void*)); + buff->srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + + buff->dstPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); + buff->dstCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + buff->dstSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + + buff->resPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); + buff->resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + + if(!buff->srcPtrs || !buff->srcSizes || !buff->dstPtrs || !buff->dstCapacities || !buff->dstSizes || !buff->resPtrs || !buff->resSizes) { + DISPLAY("alloc error\n"); + freeBuffers(*buff); + return 1; + } + + buff->srcBuffer = malloc(benchedSize); + buff->srcPtrs[0] = (const void*)buff->srcBuffer; + buff->dstPtrs[0] = malloc(ZSTD_compressBound(benchedSize) + (maxNbBlocks * 1024)); + buff->resPtrs[0] = malloc(benchedSize); + + if(!buff->srcPtrs[0] || !buff->dstPtrs[0] || !buff->resPtrs[0]) { + DISPLAY("alloc error\n"); + freeBuffers(*buff); + return 1; + } + + for(n = 0; n < nbFiles; n++) { + FILE* f; + U64 fileSize = UTIL_getFileSize(fileNamesTable[n]); + if (UTIL_isDirectory(fileNamesTable[n])) { + DISPLAY("Ignoring %s directory... \n", fileNamesTable[n]); + continue; + } + if (fileSize == UTIL_FILESIZE_UNKNOWN) { + DISPLAY("Cannot evaluate size of %s, ignoring ... \n", fileNamesTable[n]); + continue; + } + f = fopen(fileNamesTable[n], "rb"); + if (f==NULL) { + DISPLAY("impossible to open file %s\n", fileNamesTable[n]); + freeBuffers(*buff); + fclose(f); + return 10; + } + + DISPLAY("Loading %s... \r", fileNamesTable[n]); + + if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, n=nbFiles; /* buffer too small - stop after this file */ + { + char* buffer = (char*)(buff->srcBuffer); + size_t const readSize = fread((buffer)+pos, 1, (size_t)fileSize, f); + size_t blocked = 0; + while(blocked < readSize) { + buff->srcPtrs[blockNb] = (const void*)((buffer) + (pos + blocked)); + buff->srcSizes[blockNb] = blockSize; + blocked += blockSize; + blockNb++; + } + if(readSize > 0) { buff->srcSizes[blockNb - 1] = ((readSize - 1) % blockSize) + 1; } + + if (readSize != (size_t)fileSize) { + DISPLAY("could not read %s", fileNamesTable[n]); + freeBuffers(*buff); + fclose(f); + return 1; + } + + pos += readSize; + + } + fclose(f); + } + + buff->dstCapacities[0] = ZSTD_compressBound(buff->srcSizes[0]); + buff->dstSizes[0] = buff->dstCapacities[0]; + buff->resSizes[0] = buff->srcSizes[0]; + + for(n = 1; n < blockNb; n++) { + buff->dstPtrs[n] = ((char*)buff->dstPtrs[n-1]) + buff->dstCapacities[n-1]; + buff->resPtrs[n] = ((char*)buff->resPtrs[n-1]) + buff->resSizes[n-1]; + buff->dstCapacities[n] = ZSTD_compressBound(buff->srcSizes[n]); + buff->dstSizes[n] = buff->dstCapacities[n]; + buff->resSizes[n] = buff->srcSizes[n]; + } + buff->srcSize = pos; + buff->nbBlocks = blockNb; + + if (pos == 0) { DISPLAY("\nno data to bench\n"); return 1; } + + return 0; +} + +static void freeContexts(const contexts_t ctx) { + free(ctx.dictBuffer); + ZSTD_freeCCtx(ctx.cctx); + ZSTD_freeDCtx(ctx.dctx); +} + +static int createContexts(contexts_t* ctx, const char* dictFileName) { + FILE* f; + size_t readSize; + ctx->cctx = ZSTD_createCCtx(); + ctx->dctx = ZSTD_createDCtx(); + if(dictFileName == NULL) { + ctx->dictSize = 0; + ctx->dictBuffer = NULL; + return 0; + } + ctx->dictSize = UTIL_getFileSize(dictFileName); + ctx->dictBuffer = malloc(ctx->dictSize); + + f = fopen(dictFileName, "rb"); + + if(!f) { + DISPLAY("unable to open file\n"); + fclose(f); + freeContexts(*ctx); + return 1; + } + + if(ctx->dictSize > 64 MB || !(ctx->dictBuffer)) { + DISPLAY("dictionary too large\n"); + fclose(f); + freeContexts(*ctx); + return 1; + } + readSize = fread(ctx->dictBuffer, 1, ctx->dictSize, f); + if(readSize != ctx->dictSize) { + DISPLAY("unable to read file\n"); + fclose(f); + freeContexts(*ctx); + return 1; + } + return 0; +} + /* Replicate functionality of benchMemAdvanced, but with pre-split src / dst buffers */ /* The purpose is so that sufficient information is returned so that a decompression call to benchMemInvertible is possible */ /* BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); */ @@ -788,6 +953,8 @@ static int insertWinner(winnerInfo_t w) { } } +/* Writes to f the results of a parameter benchmark */ +/* when used with --optimize, will only print results better than previously discovered */ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const size_t srcSize) { char lvlstr[15] = "Custom Level"; @@ -806,7 +973,7 @@ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result fprintf(f, "/* %s */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */", - lvlstr, (double)srcSize / result.cSize, (double)result.cSpeed / (1 << 20), (double)result.dSpeed / (1 << 20)); + lvlstr, (double)srcSize / result.cSize, (double)result.cSpeed / (1 MB), (double)result.dSpeed / (1 MB)); if(TIMED) { fprintf(f, " - %1lu:%2lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); } fprintf(f, "\n"); @@ -816,29 +983,6 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res { /* global winner used for constraints */ static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; - - /* print lvl if optmode */ - if(g_lvltarget.cSize != 0) { - winnerInfo_t w; - ll_node* n; - int i; - w.result = result; - w.params = params; - i = insertWinner(w); - if(i) return; - - fprintf(f, "\033c"); - for(n = g_winners; n != NULL; n = n->next) { - DISPLAY("\r%79s\r", ""); - fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", - params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, - params.targetLength, g_stratName[(U32)(params.strategy)]); - fprintf(f, - " /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", - (double)srcSize / result.cSize, result.cSpeed / (1 << 20), result.dSpeed / (1 << 20)); - } - return; - } if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { if(DEBUG && compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { @@ -855,14 +999,12 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res } //prints out tradeoff table if using lvl - if(g_optmode) { + if(g_optmode && g_optimizer) { winnerInfo_t w; ll_node* n; - int i; w.result = result; w.params = params; - i = insertWinner(w); - //if(i) return; + insertWinner(w); if(!DEBUG) { fprintf(f, "\033c"); } fprintf(f, "\n"); @@ -877,11 +1019,11 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res n->res.params.targetLength, g_stratName[(U32)(n->res.params.strategy)]); fprintf(f, " /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", - (double)srcSize / n->res.result.cSize, n->res.result.cSpeed / (1 << 20), n->res.result.dSpeed / (1 << 20)); + (double)srcSize / n->res.result.cSize, (double)n->res.result.cSpeed / (1 MB), (double)n->res.result.dSpeed / (1 MB)); } fprintf(f, "================================\n"); fprintf(f, "Level Bounds: R: > %.3f AND C: < %.1f MB/s \n\n", - (double)srcSize / g_lvltarget.cSize, g_lvltarget.cSpeed / (1 << 20)); + (double)srcSize / g_lvltarget.cSize, (double)g_lvltarget.cSpeed / (1 MB)); fprintf(f, "Overall Winner: \n"); @@ -890,8 +1032,9 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res g_winner.params.targetLength, g_stratName[(U32)(g_winner.params.strategy)]); fprintf(f, " /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", - (double)srcSize / g_winner.result.cSize, g_winner.result.cSpeed / (1 << 20), g_winner.result.dSpeed / (1 << 20)); + (double)srcSize / g_winner.result.cSize, (double)g_winner.result.cSpeed / (1 MB), (double)g_winner.result.dSpeed / (1 MB)); + BMK_translateAdvancedParams(g_winner.params); fprintf(f, "Latest BMK: \n"); fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", @@ -899,7 +1042,7 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res params.targetLength, g_stratName[(U32)(params.strategy)]); fprintf(f, " /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", - (double)srcSize / result.cSize, result.cSpeed / (1 << 20), result.dSpeed / (1 << 20)); + (double)srcSize / result.cSize, (double)result.cSpeed / (1 MB), (double)result.dSpeed / (1 MB)); } } @@ -1023,16 +1166,16 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para /* too large compression speed difference for the compression benefit */ if (W_ratio > O_ratio) DISPLAY ("Compression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n", - W_ratio, (double)testResult.cSpeed / 1000000, - O_ratio, (double)winners[cLevel].result.cSpeed / 1000000., cLevel); + W_ratio, (double)testResult.cSpeed / (1 MB), + O_ratio, (double)winners[cLevel].result.cSpeed / (1 MB), cLevel); continue; } if (W_DSpeed_note < O_DSpeed_note ) { /* too large decompression speed difference for the compression benefit */ if (W_ratio > O_ratio) DISPLAY ("Decompression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n", - W_ratio, (double)testResult.dSpeed / 1000000., - O_ratio, (double)winners[cLevel].result.dSpeed / 1000000., cLevel); + W_ratio, (double)testResult.dSpeed / (1 MB), + O_ratio, (double)winners[cLevel].result.dSpeed / (1 MB), cLevel); continue; } @@ -1417,7 +1560,6 @@ static void BMK_selectRandomStart( } } - static void BMK_benchOnce(const void* srcBuffer, size_t srcSize) { BMK_result_t testResult; @@ -1442,7 +1584,7 @@ static void BMK_benchFullTable(const void* srcBuffer, size_t srcSize) if (f==NULL) { DISPLAY("error opening %s \n", rfName); exit(1); } if (g_target) { - BMK_init_level_constraints(g_target*1000000); + BMK_init_level_constraints(g_target * (1 MB)); } else { /* baseline config for level 1 */ ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, blockSize, 0); @@ -1487,7 +1629,7 @@ static void BMK_benchMemInit(const void* srcBuffer, size_t srcSize) static int benchSample(void) { const char* const name = "Sample 10MB"; - size_t const benchedSize = 10000000; + size_t const benchedSize = 10 MB; void* origBuff = malloc(benchedSize); if (!origBuff) { perror("not enough memory"); return 12; } @@ -1505,13 +1647,56 @@ static int benchSample(void) } +static int benchOnce(const char** fileNamesTable, int nbFiles, const char* dictFileName) { + buffers_t buf; + contexts_t ctx; + BMK_result_t testResult; + size_t maxBlockSize = 0, i; + + if(createBuffers(&buf, fileNamesTable, nbFiles)) { + DISPLAY("unable to load files\n"); + return 1; + } + + if(createContexts(&ctx, dictFileName)) { + DISPLAY("unable to load dictionary\n"); + freeBuffers(buf); + return 2; + } + + for(i = 0; i < buf.nbBlocks; i++) { + maxBlockSize = MAX(maxBlockSize, buf.srcSizes[i]); + } + + g_params = ZSTD_adjustCParams(g_params, maxBlockSize, 0); + + if(BMK_benchParam(&testResult, buf, ctx, g_params)) { + DISPLAY("Error during benchmarking\n"); + freeBuffers(buf); + freeContexts(ctx); + return 3; + } + + DISPLAY("Compression Ratio: %.3f Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)buf.srcSize / testResult.cSize, + (double)testResult.cSpeed / (1 MB), (double)testResult.dSpeed / (1 MB)); + + freeBuffers(buf); + freeContexts(ctx); + return 0; +} + /* benchFiles() : * note: while this function takes a table of filenames, * in practice, only the first filename will be used */ -int benchFiles(const char** fileNamesTable, int nbFiles) +//TODO: dictionaries still not supported in fullTable mode +int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileName) { int fileIdx=0; + if(g_singleRun) { + return benchOnce(fileNamesTable, nbFiles, dictFileName); + } + /* Loop for each file */ while (fileIdxsrcPtrs = (const void**)calloc(maxNbBlocks, sizeof(void*)); - buff->srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); - - buff->dstPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); - buff->dstCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); - buff->dstSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); - - buff->resPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); - buff->resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); - - if(!buff->srcPtrs || !buff->srcSizes || !buff->dstPtrs || !buff->dstCapacities || !buff->dstSizes || !buff->resPtrs || !buff->resSizes) { - DISPLAY("alloc error\n"); - freeBuffers(*buff); - return 1; - } - - buff->srcBuffer = malloc(benchedSize); - buff->srcPtrs[0] = (const void*)buff->srcBuffer; - buff->dstPtrs[0] = malloc(ZSTD_compressBound(benchedSize) + (maxNbBlocks * 1024)); - buff->resPtrs[0] = malloc(benchedSize); - - if(!buff->srcPtrs[0] || !buff->dstPtrs[0] || !buff->resPtrs[0]) { - DISPLAY("alloc error\n"); - freeBuffers(*buff); - return 1; - } - - for(n = 0; n < nbFiles; n++) { - FILE* f; - U64 fileSize = UTIL_getFileSize(fileNamesTable[n]); - if (UTIL_isDirectory(fileNamesTable[n])) { - DISPLAY("Ignoring %s directory... \n", fileNamesTable[n]); - continue; - } - if (fileSize == UTIL_FILESIZE_UNKNOWN) { - DISPLAY("Cannot evaluate size of %s, ignoring ... \n", fileNamesTable[n]); - continue; - } - f = fopen(fileNamesTable[n], "rb"); - if (f==NULL) { - DISPLAY("impossible to open file %s\n", fileNamesTable[n]); - freeBuffers(*buff); - fclose(f); - return 10; - } - - DISPLAY("Loading %s... \r", fileNamesTable[n]); - - if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, n = nbFiles; /* buffer too small - stop after this file */ - { - char* buffer = (char*)(buff->srcBuffer); - size_t const readSize = fread(((buffer)+pos), 1, (size_t)fileSize, f); - size_t blocked = 0; - while(blocked < readSize) { - buff->srcPtrs[blockNb] = (const void*)((buffer) + (pos + blocked)); - buff->srcSizes[blockNb] = blockSize; - blocked += blockSize; - blockNb++; - } - if(readSize > 0) { buff->srcSizes[blockNb - 1] = ((readSize - 1) % blockSize) + 1; } - - if (readSize != (size_t)fileSize) { - DISPLAY("could not read %s", fileNamesTable[n]); - freeBuffers(*buff); - fclose(f); - return 1; - } - - pos += readSize; - - } - fclose(f); - } - - buff->dstCapacities[0] = ZSTD_compressBound(buff->srcSizes[0]); - buff->dstSizes[0] = buff->dstCapacities[0]; - buff->resSizes[0] = buff->srcSizes[0]; - - for(n = 1; n < blockNb; n++) { - buff->dstPtrs[n] = ((char*)buff->dstPtrs[n-1]) + buff->dstCapacities[n-1]; - buff->resPtrs[n] = ((char*)buff->resPtrs[n-1]) + buff->resSizes[n-1]; - buff->dstCapacities[n] = ZSTD_compressBound(buff->srcSizes[n]); - buff->dstSizes[n] = buff->dstCapacities[n]; - buff->resSizes[n] = buff->srcSizes[n]; - } - buff->srcSize = pos; - buff->nbBlocks = blockNb; - - if (pos == 0) { DISPLAY("\nno data to bench\n"); return 1; } - - return 0; -} - -static void freeContexts(const contexts_t ctx) { - free(ctx.dictBuffer); - ZSTD_freeCCtx(ctx.cctx); - ZSTD_freeDCtx(ctx.dctx); -} - -/* Creates struct holding contexts and dictionary buffers. returns 0 on success, 1 on failure. */ -static int createContexts(contexts_t* const ctx, const char* dictFileName) { - FILE* f; - size_t readSize; - U64 dictSize; - ctx->cctx = ZSTD_createCCtx(); - ctx->dctx = ZSTD_createDCtx(); - ctx->dictSize = 0; - ctx->dictBuffer = NULL; - - if(!ctx->cctx || !ctx->dctx) { - DISPLAY("context allocation error\n"); - freeContexts(*ctx); - return 1; - } - - if(dictFileName == NULL) { - return 0; - } - - dictSize = UTIL_getFileSize(dictFileName); - - if(dictSize == UTIL_FILESIZE_UNKNOWN) { - DISPLAY("Unable to get dictionary size\n"); - freeContexts(*ctx); - return 1; - } else { - ctx->dictSize = (size_t)dictSize; - } - - ctx->dictBuffer = malloc(ctx->dictSize); - - f = fopen(dictFileName, "rb"); - - if(!f) { - DISPLAY("unable to open file\n"); - fclose(f); - freeContexts(*ctx); - return 1; - } - - if(ctx->dictSize > 64 MB || !(ctx->dictBuffer)) { - DISPLAY("dictionary too large\n"); - fclose(f); - freeContexts(*ctx); - return 1; - } - readSize = fread(ctx->dictBuffer, 1, ctx->dictSize, f); - if(readSize != ctx->dictSize) { - DISPLAY("unable to read file\n"); - fclose(f); - freeContexts(*ctx); - return 1; - } - return 0; -} - /* goes best, best-1, best+1, best-2, ... */ /* return 0 if nothing remaining */ static int nextStrategy(const int currentStrategy, const int bestStrategy) { @@ -2102,7 +2097,7 @@ static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZS * cLevel - compression level to exceed (all solutions must be > lvl in cSpeed + ratio) */ -#define MAX_TRIES 3 +#define MAX_TRIES 5 #define TRY_DECAY 1 static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel) @@ -2272,8 +2267,9 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } while(st && tries > 0) { + winnerInfo_t wc; DEBUGOUTPUT("StrategySwitch: %s\n", g_stratName[st]); - winnerInfo_t wc = optimizeFixedStrategy(buf, ctx, target, paramTarget, + wc = optimizeFixedStrategy(buf, ctx, target, paramTarget, st, varArray, varLen, allMT[st], tries); if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) { @@ -2399,7 +2395,6 @@ int main(int argc, const char** argv) const char* exename=argv[0]; const char* input_filename = NULL; const char* dictFileName = NULL; - U32 optimizer = 0; U32 main_pause = 0; int optimizerCLevel = 0; @@ -2424,7 +2419,7 @@ int main(int argc, const char** argv) if(!strcmp(argument,"--no-seed")) { g_noSeed = 1; continue; } if (longCommandWArg(&argument, "--optimize=")) { - optimizer = 1; + g_optimizer = 1; for ( ; ;) { PARSE_CPARAMS(paramTarget); PARSE_SUB_ARGS("compressionSpeed=" , "cSpeed=", target.cSpeed); @@ -2580,17 +2575,17 @@ int main(int argc, const char** argv) if (!input_filename) { input_filename=argument; filenamesStart=i; continue; } } if (filenamesStart==0) { - if (optimizer) { + if (g_optimizer) { DISPLAY("Optimizer Expects File\n"); return 1; } else { result = benchSample(); } } else { - if (optimizer) { + if (g_optimizer) { result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget, optimizerCLevel); } else { - result = benchFiles(argv+filenamesStart, argc-filenamesStart); + result = benchFiles(argv+filenamesStart, argc-filenamesStart, dictFileName); } } if (main_pause) { int unused; printf("press enter...\n"); unused = getchar(); (void)unused; } From 43b4971ca8910ccedebb6c23a6daa89069df6a17 Mon Sep 17 00:00:00 2001 From: George Lu Date: Fri, 27 Jul 2018 16:49:33 -0700 Subject: [PATCH 30/55] Renames, Documentation Updates --- tests/README.md | 22 +++++++----- tests/paramgrill.c | 84 +++++++++++++++++++++++++--------------------- 2 files changed, 58 insertions(+), 48 deletions(-) diff --git a/tests/README.md b/tests/README.md index 1410ca979..946f890b1 100644 --- a/tests/README.md +++ b/tests/README.md @@ -107,15 +107,19 @@ Full list of arguments L# - level --zstd= : Single run, parameter selection syntax same as zstdcli --optimize= : find parameters to maximize compression ratio given parameters - Can use all --zstd= commands to constrain the type of solution found in addition to the following constraints - cSpeed= - Minimum compression speed - dSpeed= - Minimum decompression speed - cMem= - compression memory - lvl= - Searches for solutions which are strictly better than that compression lvl in ratio and cSpeed, - stc= - When invoked with lvl=, represents slack in ratio/cSpeed allowed for a solution to be considered - - In normal operation, represents slack in strategy selection in choosing the default parameters - prefer[Speed/Ratio]= - Only affects lvl= invocations. Defines value placed on compression speed or ratio - when determining overall winner (default 1 for both). + Can use all --zstd= commands to constrain the type of solution found in addition to the following constraints + cSpeed= : Minimum compression speed + dSpeed= : Minimum decompression speed + cMem= : Maximum compression memory + lvl= : Searches for solutions which are strictly better than that compression lvl in ratio and cSpeed, + stc= : When invoked with lvl=, represents percentage slack in ratio/cSpeed allowed for a solution to be considered (Default 99%) + : In normal operation, represents percentage slack in choosing viable starting strategy selection in choosing the default parameters + (Lower value will begin with stronger strategies) (Default 90%) + preferSpeed= / preferRatio= + : Only affects lvl = invocations. Defines value placed on compression speed or ratio + when determining overall winner (default 1 for both, higher = more valued). + tries= : Maximum number of random restarts on a single strategy before switching (Default 5) + Higher values will make optimizer run longer, more chances to find better solution. --optimize= : same as -O with more verbose syntax -P# : generated sample compressibility -t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 8df72eb01..7e86c2e9e 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -125,15 +125,13 @@ typedef struct { U32 cMem; /* bytes */ } constraint_t; -static constraint_t g_targetConstraints; - -typedef struct ll_node ll_node; -struct ll_node { +typedef struct winner_ll_node winner_ll_node; +struct winner_ll_node { winnerInfo_t res; - ll_node* next; + winner_ll_node* next; }; -static ll_node* g_winners; /* linked list sorted ascending by cSize & cSpeed */ +static winner_ll_node* g_winners; /* linked list sorted ascending by cSize & cSpeed */ static BMK_result_t g_lvltarget; static int g_optmode = 0; @@ -323,8 +321,8 @@ static int compareResultLT(const BMK_result_t result1, const BMK_result_t result static constraint_t relaxTarget(constraint_t target) { target.cMem = (U32)-1; - target.cSpeed *= ((double)(g_strictness) / 100); - target.dSpeed *= ((double)(g_strictness) / 100); + target.cSpeed *= ((double)g_strictness) / 100; + target.dSpeed *= ((double)g_strictness) / 100; return target; } @@ -855,20 +853,19 @@ static int speedSizeCompare(BMK_result_t r1, BMK_result_t r2) { return SPEED_RESULT; /* r2 is faster but not smaller */ } } -/* assumes candidate is already strictly better than old winner. */ -/* 0 for success, 1 for no insert */ -/* indicate whether inserted as well? */ + +/* 0 for insertion, 1 for no insert */ /* maintain invariant speedSizeCompare(n, n->next) = SPEED_RESULT */ -static int insertWinner(winnerInfo_t w) { +static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) { BMK_result_t r = w.result; - ll_node* cur_node = g_winners; + winner_ll_node* cur_node = g_winners; /* first node to insert */ - if(!feasible(r, g_targetConstraints)) { + if(!feasible(r, targetConstraints)) { return 1; } if(g_winners == NULL) { - ll_node* first_node = malloc(sizeof(ll_node)); + winner_ll_node* first_node = malloc(sizeof(winner_ll_node)); if(first_node == NULL) { return 1; } @@ -886,7 +883,7 @@ static int insertWinner(winnerInfo_t w) { } case WORSE_RESULT: { - ll_node* tmp; + winner_ll_node* tmp; cur_node->res = cur_node->next->res; tmp = cur_node->next; cur_node->next = cur_node->next->next; @@ -900,7 +897,7 @@ static int insertWinner(winnerInfo_t w) { } case SIZE_RESULT: /* insert after first size result, then return */ { - ll_node* newnode = malloc(sizeof(ll_node)); + winner_ll_node* newnode = malloc(sizeof(winner_ll_node)); if(newnode == NULL) { return 1; } @@ -927,7 +924,7 @@ static int insertWinner(winnerInfo_t w) { } case SPEED_RESULT: { - ll_node* newnode = malloc(sizeof(ll_node)); + winner_ll_node* newnode = malloc(sizeof(winner_ll_node)); if(newnode == NULL) { return 1; } @@ -938,7 +935,7 @@ static int insertWinner(winnerInfo_t w) { } case SIZE_RESULT: /* insert before first size result, then return */ { - ll_node* newnode = malloc(sizeof(ll_node)); + winner_ll_node* newnode = malloc(sizeof(winner_ll_node)); if(newnode == NULL) { return 1; } @@ -961,7 +958,7 @@ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result const U64 time = UTIL_clockSpanNano(g_time); const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC); - DISPLAY("\r%79s\r", ""); + fprintf(f, "\r%79s\r", ""); fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, @@ -1001,10 +998,10 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res //prints out tradeoff table if using lvl if(g_optmode && g_optimizer) { winnerInfo_t w; - ll_node* n; + winner_ll_node* n; w.result = result; w.params = params; - insertWinner(w); + insertWinner(w, targetConstraints); if(!DEBUG) { fprintf(f, "\033c"); } fprintf(f, "\n"); @@ -1012,7 +1009,7 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res /* the table */ fprintf(f, "================================\n"); for(n = g_winners; n != NULL; n = n->next) { - DISPLAY("\r%79s\r", ""); + fprintf(f, "\r%79s\r", ""); fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", n->res.params.windowLog, n->res.params.chainLog, n->res.params.hashLog, n->res.params.searchLog, n->res.params.searchLength, @@ -1045,6 +1042,12 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res (double)srcSize / result.cSize, (double)result.cSpeed / (1 MB), (double)result.dSpeed / (1 MB)); } + +#if 0 + if(BMK_timeSpan(g_time) > g_grillDuration_s) { + exit(0); + } +#endif } static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize) @@ -2097,7 +2100,7 @@ static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZS * cLevel - compression level to exceed (all solutions must be > lvl in cSpeed + ratio) */ -#define MAX_TRIES 5 +static int g_maxTries = 5; #define TRY_DECAY 1 static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel) @@ -2185,20 +2188,21 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } /* use level'ing mode instead of normal target mode */ + /* Should lvl be parameter-masked here? */ if(g_optmode) { winner.params = ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize); if(BMK_benchParam(&winner.result, buf, ctx, winner.params)) { ret = 3; goto _cleanUp; } - - target.cSpeed = (U32)winner.result.cSpeed; - - g_targetConstraints = target; - + g_lvltarget = winner.result; - g_lvltarget.cSpeed *= ((double)(g_strictness) / 100); - g_lvltarget.cSize /= ((double)(g_strictness) / 100); + g_lvltarget.cSpeed *= ((double)g_strictness) / 100; + g_lvltarget.dSpeed *= ((double)g_strictness) / 100; + g_lvltarget.cSize /= ((double)g_strictness) / 100; + + target.cSpeed = (U32)g_lvltarget.cSpeed; + target.dSpeed = (U32)g_lvltarget.dSpeed; //See if this is worth BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize); } @@ -2241,7 +2245,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } /* if the current params are too slow, just stop. */ - if(target.cSpeed > candidate.cSpeed * 2) { break; } + if(target.cSpeed > candidate.cSpeed * 3 / 2) { break; } } } } @@ -2254,7 +2258,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ int bestStrategy = (int)winner.params.strategy; if(paramTarget.strategy == 0) { int st = (int)winner.params.strategy; - int tries = MAX_TRIES; + int tries = g_maxTries; { /* one iterations of hill climbing with the level-defined parameters. */ @@ -2269,12 +2273,13 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ while(st && tries > 0) { winnerInfo_t wc; DEBUGOUTPUT("StrategySwitch: %s\n", g_stratName[st]); + wc = optimizeFixedStrategy(buf, ctx, target, paramTarget, st, varArray, varLen, allMT[st], tries); if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) { winner = wc; - tries = MAX_TRIES; + tries = g_maxTries; bestStrategy = st; } else { st = nextStrategy(st, bestStrategy); @@ -2283,7 +2288,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } } else { winner = optimizeFixedStrategy(buf, ctx, target, paramTarget, paramTarget.strategy, - varArray, varLen, allMT[paramTarget.strategy], 10); + varArray, varLen, allMT[paramTarget.strategy], g_maxTries); } } @@ -2402,17 +2407,17 @@ int main(int argc, const char** argv) ZSTD_compressionParameters paramTarget = { 0, 0, 0, 0, 0, 0, 0 }; - assert(argc>=1); /* for exename */ - g_time = UTIL_getTime(); + assert(argc>=1); /* for exename */ + /* Welcome message */ DISPLAY(WELCOME_MESSAGE); for(i=1; i Date: Mon, 30 Jul 2018 17:42:46 -0700 Subject: [PATCH 31/55] Update fulltable to use same interface Add seperateFiles flag --- tests/README.md | 1 + tests/paramgrill.c | 389 ++++++++++++++++++++++++--------------------- 2 files changed, 207 insertions(+), 183 deletions(-) diff --git a/tests/README.md b/tests/README.md index 946f890b1..736916936 100644 --- a/tests/README.md +++ b/tests/README.md @@ -125,6 +125,7 @@ Full list of arguments -t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) -v : Prints Benchmarking output -D : Next argument dictionary file + -s : Benchmark all files separately ``` Any inputs afterwards are treated as files to benchmark. diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 7e86c2e9e..cba97ebbb 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -96,7 +96,7 @@ typedef enum { static const int rangetable[NUM_PARAMS] = { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE }; static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 }; /*-************************************ -* Benchmark Parameters +* Benchmark Parameters/Global Variables **************************************/ typedef BYTE U8; @@ -110,7 +110,7 @@ static U32 g_singleRun = 0; static U32 g_optimizer = 0; static U32 g_target = 0; static U32 g_noSeed = 0; -static ZSTD_compressionParameters g_params = { 0, 0, 0, 0, 0, 0, ZSTD_greedy }; +static ZSTD_compressionParameters g_params; static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */ @@ -150,6 +150,14 @@ void BMK_SetNbIterations(int nbLoops) DISPLAY("- %u iterations -\n", g_nbIterations); } +/* + * Additional Global Variables (Defined Above Use) + * g_stratName + * g_level_constraint + * g_alreadyTested + * g_maxTries + */ + /*-******************************************************* * Private functions *********************************************************/ @@ -335,17 +343,6 @@ const char* g_stratName[ZSTD_btultra+1] = { "ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ", "ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "}; -/* benchParam but only takes in one input buffer. */ -static int -BMK_benchParam1(BMK_result_t* resultPtr, - const void* srcBuffer, size_t srcSize, - const ZSTD_compressionParameters cParams) { - - BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, &srcSize, 1, 0, &cParams, NULL, 0, 0, "File"); - *resultPtr = res.result; - return res.error; -} - static ZSTD_compressionParameters emptyParams(void) { ZSTD_compressionParameters p = { 0, 0, 0, 0, 0, 0, (ZSTD_strategy)0 }; return p; @@ -381,14 +378,6 @@ typedef struct { ZSTD_DCtx* dctx; } contexts_t; -static int BMK_benchParam(BMK_result_t* resultPtr, - const buffers_t buf, const contexts_t ctx, - const ZSTD_compressionParameters cParams) { - BMK_return_t res = BMK_benchMem(buf.srcPtrs[0], buf.srcSize, buf.srcSizes, (unsigned)buf.nbBlocks, 0, &cParams, ctx.dictBuffer, ctx.dictSize, 0, "Files"); - *resultPtr = res.result; - return res.error; -} - /*-******************************************************* * From Paramgrill *********************************************************/ @@ -542,7 +531,7 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab size_t n; U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles); size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad); - const size_t blockSize = g_blockSize ? g_blockSize : totalSizeToLoad; //(largest fileSize or total fileSize) + const size_t blockSize = g_blockSize ? g_blockSize : totalSizeToLoad; U32 const maxNbBlocks = (U32) ((totalSizeToLoad + (blockSize-1)) / blockSize) + (U32)nbFiles; U32 blockNb = 0; @@ -830,6 +819,14 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t return results; } +static int BMK_benchParam(BMK_result_t* resultPtr, + buffers_t buf, contexts_t ctx, + const ZSTD_compressionParameters cParams) { + BMK_return_t res = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_both, BMK_timeMode, 3); + *resultPtr = res.result; + return res.error; +} + /* comparison function: */ /* strictly better, strictly worse, equal, speed-side adv, size-side adv */ //Maybe use compress_only for benchmark first run? @@ -911,7 +908,7 @@ static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) { } - //assert(cur_node->next == NULL) + assert(cur_node->next == NULL); switch(speedSizeCompare(r, cur_node->res.result)) { case BETTER_RESULT: { @@ -995,7 +992,7 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res } } - //prints out tradeoff table if using lvl + //prints out tradeoff table if using lvloptimize if(g_optmode && g_optimizer) { winnerInfo_t w; winner_ll_node* n; @@ -1042,12 +1039,6 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res (double)srcSize / result.cSize, (double)result.cSpeed / (1 MB), (double)result.dSpeed / (1 MB)); } - -#if 0 - if(BMK_timeSpan(g_time) > g_grillDuration_s) { - exit(0); - } -#endif } static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize) @@ -1099,14 +1090,14 @@ static void BMK_init_level_constraints(int bytePerSec_level1) } } } -static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters params, - const void* srcBuffer, size_t srcSize) +static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters params, + buffers_t buf, contexts_t ctx) { BMK_result_t testResult; int better = 0; int cLevel; - BMK_benchParam1(&testResult, srcBuffer, srcSize, params); + BMK_benchParam(&testResult, buf, ctx, params); for (cLevel = 1; cLevel <= NB_LEVELS_TRACKED; cLevel++) { @@ -1122,15 +1113,15 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para /* first solution for this cLevel */ winners[cLevel].result = testResult; winners[cLevel].params = params; - BMK_printWinner(stdout, cLevel, testResult, params, srcSize); + BMK_printWinner(stdout, cLevel, testResult, params, buf.srcSize); better = 1; continue; } if ((double)testResult.cSize <= ((double)winners[cLevel].result.cSize * (1. + (0.02 / cLevel))) ) { /* Validate solution is "good enough" */ - double W_ratio = (double)srcSize / testResult.cSize; - double O_ratio = (double)srcSize / winners[cLevel].result.cSize; + double W_ratio = (double)buf.srcSize / testResult.cSize; + double O_ratio = (double)buf.srcSize / winners[cLevel].result.cSize; double W_ratioNote = log (W_ratio); double O_ratioNote = log (O_ratio); size_t W_DMemUsed = (1 << params.windowLog) + (16 KB); @@ -1187,7 +1178,7 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para winners[cLevel].result = testResult; winners[cLevel].params = params; - BMK_printWinner(stdout, cLevel, testResult, params, srcSize); + BMK_printWinner(stdout, cLevel, testResult, params, buf.srcSize); better = 1; } } @@ -1315,6 +1306,26 @@ static void paramVariation(ZSTD_compressionParameters* ptr, const varInds_t* var *ptr = p; } +/* maybe put strategy back in */ +static void paramVariationWithStrategy(ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen, const U32 nbChanges) +{ + ZSTD_compressionParameters p; + U32 validated = 0; + while (!validated) { + U32 i; + p = *ptr; + for (i = 0 ; i < nbChanges ; i++) { + const U32 changeID = FUZ_rand(&g_rand) % ((varyLen + 1) << 1); + if(changeID < (U32)(varyLen << 1)) { + paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p); + } else { + p.strategy += ((FUZ_rand(&g_rand) % 2) << 1) - 1; /* +/- 1 */ + } + } + validated = !ZSTD_isError(ZSTD_checkCParams(p)); + } + *ptr = p; +} /* length of memo table given free variables */ static size_t memoTableLen(const varInds_t* varyParams, const int varyLen) { size_t arrayLen = 1; @@ -1463,6 +1474,17 @@ static U8** createMemoTableArray(ZSTD_compressionParameters paramConstraints, co return mtAll; } +static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZSTD_compressionParameters mask) { + base.windowLog = mask.windowLog ? mask.windowLog : base.windowLog; + base.chainLog = mask.chainLog ? mask.chainLog : base.chainLog; + base.hashLog = mask.hashLog ? mask.hashLog : base.hashLog; + base.searchLog = mask.searchLog ? mask.searchLog : base.searchLog; + base.searchLength = mask.searchLength ? mask.searchLength : base.searchLength; + base.targetLength = mask.targetLength ? mask.targetLength : base.targetLength; + base.strategy = mask.strategy ? mask.strategy : base.strategy; + return base; +} + #define PARAMTABLELOG 25 #define PARAMTABLESIZE (1< g_maxNbVariations) break; - paramVariation(&p, unconstrained, 7, 4); + paramVariationWithStrategy(&p, unconstrained, NUM_PARAMS, 4); /* exclude faster if already played params */ if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(p))-1)) @@ -1500,11 +1522,11 @@ static void playAround(FILE* f, winnerInfo_t* winners, /* test */ b = NB_TESTS_PLAYED(p); (*b)++; - if (!BMK_seed(winners, p, srcBuffer, srcSize)) continue; + if (!BMK_seed(winners, p, buf, ctx)) continue; /* improvement found => search more */ - BMK_printWinners(f, winners, srcSize); - playAround(f, winners, p, srcBuffer, srcSize); + BMK_printWinners(f, winners, buf.srcSize); + playAround(f, winners, p, buf, ctx); } } @@ -1551,35 +1573,24 @@ static void randomConstrainedParams(ZSTD_compressionParameters* pc, varInds_t* v static void BMK_selectRandomStart( FILE* f, winnerInfo_t* winners, - const void* srcBuffer, size_t srcSize) + buffers_t buf, contexts_t ctx) { U32 const id = FUZ_rand(&g_rand) % (NB_LEVELS_TRACKED+1); if ((id==0) || (winners[id].params.windowLog==0)) { /* use some random entry */ - ZSTD_compressionParameters const p = ZSTD_adjustCParams(randomParams(), srcSize, 0); - playAround(f, winners, p, srcBuffer, srcSize); + ZSTD_compressionParameters const p = ZSTD_adjustCParams(randomParams(), buf.srcSize, 0); + playAround(f, winners, p, buf, ctx); } else { - playAround(f, winners, winners[id].params, srcBuffer, srcSize); + playAround(f, winners, winners[id].params, buf, ctx); } } -static void BMK_benchOnce(const void* srcBuffer, size_t srcSize) -{ - BMK_result_t testResult; - g_params = ZSTD_adjustCParams(g_params, srcSize, 0); - BMK_benchParam1(&testResult, srcBuffer, srcSize, g_params); - DISPLAY("Compression Ratio: %.3f Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)srcSize / testResult.cSize, - (double)testResult.cSpeed / 1000000, (double)testResult.dSpeed / 1000000); - return; -} - -static void BMK_benchFullTable(const void* srcBuffer, size_t srcSize) +static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBlockSize) { ZSTD_compressionParameters params; winnerInfo_t winners[NB_LEVELS_TRACKED+1]; const char* const rfName = "grillResults.txt"; FILE* const f = fopen(rfName, "w"); - const size_t blockSize = g_blockSize ? g_blockSize : srcSize; /* cut by block or not ? */ /* init */ assert(g_singleRun==0); @@ -1590,9 +1601,9 @@ static void BMK_benchFullTable(const void* srcBuffer, size_t srcSize) BMK_init_level_constraints(g_target * (1 MB)); } else { /* baseline config for level 1 */ - ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, blockSize, 0); + ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, maxBlockSize, ctx.dictSize); //is dictionary ever even useful here? BMK_result_t testResult; - BMK_benchParam1(&testResult, srcBuffer, srcSize, l1params); + BMK_benchParam(&testResult, buf, ctx, l1params); BMK_init_level_constraints((int)((testResult.cSpeed * 31) / 32)); } @@ -1600,61 +1611,124 @@ static void BMK_benchFullTable(const void* srcBuffer, size_t srcSize) { const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); int i; for (i=0; i<=maxSeeds; i++) { - params = ZSTD_getCParams(i, blockSize, 0); - BMK_seed(winners, params, srcBuffer, srcSize); + params = ZSTD_getCParams(i, maxBlockSize, 0); + BMK_seed(winners, params, buf, ctx); } } - BMK_printWinners(f, winners, srcSize); + BMK_printWinners(f, winners, buf.srcSize); /* start tests */ { const time_t grillStart = time(NULL); do { - BMK_selectRandomStart(f, winners, srcBuffer, srcSize); + BMK_selectRandomStart(f, winners, buf, ctx); } while (BMK_timeSpan(grillStart) < g_grillDuration_s); } /* end summary */ - BMK_printWinners(f, winners, srcSize); + BMK_printWinners(f, winners, buf.srcSize); DISPLAY("grillParams operations completed \n"); /* clean up*/ fclose(f); } -static void BMK_benchMemInit(const void* srcBuffer, size_t srcSize) -{ - if (g_singleRun) - return BMK_benchOnce(srcBuffer, srcSize); - else - return BMK_benchFullTable(srcBuffer, srcSize); -} - - static int benchSample(void) { const char* const name = "Sample 10MB"; size_t const benchedSize = 10 MB; + U32 blockSize = g_blockSize ? g_blockSize : benchedSize; + U32 const maxNbBlocks = (U32) ((benchedSize + (blockSize-1)) / blockSize) + 1; + size_t splitSize = 0; - void* origBuff = malloc(benchedSize); - if (!origBuff) { perror("not enough memory"); return 12; } + buffers_t buf; + contexts_t ctx; - /* Fill buffer */ - RDG_genBuffer(origBuff, benchedSize, g_compressibility, 0.0, 0); + buf.srcPtrs = (const void**)calloc(maxNbBlocks, sizeof(void*)); + buf.dstPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); + buf.resPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); + buf.srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + buf.dstSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + buf.dstCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + buf.resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + buf.srcSize = benchedSize; + + if(!buf.srcPtrs || !buf.dstPtrs || !buf.resPtrs || !buf.srcSizes || !buf.dstSizes || !buf.dstCapacities || !buf.resSizes) { + DISPLAY("Allocation Error\n"); + freeBuffers(buf); + return 1; + } + + buf.srcBuffer = malloc(benchedSize); + buf.srcPtrs[0] = (const void*)buf.srcBuffer; + buf.dstPtrs[0] = malloc(ZSTD_compressBound(benchedSize) + 1024 * maxNbBlocks); + buf.resPtrs[0] = malloc(benchedSize); + + if(!buf.srcPtrs[0] || !buf.dstPtrs[0] || !buf.resPtrs[0]) { + DISPLAY("Allocation Error\n"); + freeBuffers(buf); + return 1; + } + + + splitSize = MIN(benchedSize, blockSize); + buf.srcSizes[0] = splitSize; + buf.dstCapacities[0] = ZSTD_compressBound(splitSize); + buf.resSizes[0] = splitSize; + + for(buf.nbBlocks = 1; splitSize < benchedSize; buf.nbBlocks++) { + const size_t i = buf.nbBlocks; + const size_t nextBlockSize = MIN(benchedSize - splitSize, blockSize); + buf.srcSizes[i] = nextBlockSize; + buf.dstCapacities[i] = ZSTD_compressBound(nextBlockSize); + buf.resSizes[i] = nextBlockSize; + buf.srcPtrs[i] = (const void*)(((const char*)buf.srcPtrs[i-1]) + buf.srcSizes[i-1]); + buf.dstPtrs[i] = (void*)(((char*)buf.dstPtrs[i-1]) + buf.dstSizes[i-1]); + buf.resPtrs[i] = (void*)(((char*)buf.resPtrs[i-1]) + buf.resSizes[i-1]); + splitSize += nextBlockSize; + } + + if(createContexts(&ctx, NULL)) { + DISPLAY("Context Creation Error\n"); + freeBuffers(buf); + return 1; + } + + RDG_genBuffer(buf.srcBuffer, benchedSize, g_compressibility, 0.0, 0); /* bench */ DISPLAY("\r%79s\r", ""); DISPLAY("using %s %i%%: \n", name, (int)(g_compressibility*100)); - BMK_benchMemInit(origBuff, benchedSize); - free(origBuff); + BMK_benchFullTable(buf, ctx, MIN(blockSize, benchedSize)); + + freeBuffers(buf); + freeContexts(ctx); + return 0; } -static int benchOnce(const char** fileNamesTable, int nbFiles, const char* dictFileName) { +static int benchOnce(buffers_t buf, contexts_t ctx) { + BMK_result_t testResult; + + if(BMK_benchParam(&testResult, buf, ctx, g_params)) { + DISPLAY("Error during benchmarking\n"); + return 1; + } + + DISPLAY("Compression Ratio: %.3f Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)buf.srcSize / testResult.cSize, + (double)testResult.cSpeed / (1 MB), (double)testResult.dSpeed / (1 MB)); + return 0; +} + +/* benchFiles() : + * note: while this function takes a table of filenames, + * in practice, only the first filename will be used */ +int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileName, int cLevel) +{ buffers_t buf; contexts_t ctx; - BMK_result_t testResult; size_t maxBlockSize = 0, i; + int ret = 0; if(createBuffers(&buf, fileNamesTable, nbFiles)) { DISPLAY("unable to load files\n"); @@ -1671,86 +1745,24 @@ static int benchOnce(const char** fileNamesTable, int nbFiles, const char* dictF maxBlockSize = MAX(maxBlockSize, buf.srcSizes[i]); } - g_params = ZSTD_adjustCParams(g_params, maxBlockSize, 0); - - if(BMK_benchParam(&testResult, buf, ctx, g_params)) { - DISPLAY("Error during benchmarking\n"); - freeBuffers(buf); - freeContexts(ctx); - return 3; + DISPLAY("\r%79s\r", ""); + if(nbFiles == 1) { + DISPLAY("using %s : \n", fileNamesTable[0]); + } else { + DISPLAY("using %d Files : \n", nbFiles); } - DISPLAY("Compression Ratio: %.3f Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)buf.srcSize / testResult.cSize, - (double)testResult.cSpeed / (1 MB), (double)testResult.dSpeed / (1 MB)); + g_params = ZSTD_adjustCParams(maskParams(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize), g_params), maxBlockSize, ctx.dictSize); + + if(g_singleRun) { + ret = benchOnce(buf, ctx); + } else { + BMK_benchFullTable(buf, ctx, maxBlockSize); + } freeBuffers(buf); freeContexts(ctx); - return 0; -} - -/* benchFiles() : - * note: while this function takes a table of filenames, - * in practice, only the first filename will be used */ -//TODO: dictionaries still not supported in fullTable mode -int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileName) -{ - int fileIdx=0; - - if(g_singleRun) { - return benchOnce(fileNamesTable, nbFiles, dictFileName); - } - - /* Loop for each file */ - while (fileIdx inFileSize) benchedSize = (size_t)inFileSize; - if (benchedSize < inFileSize) - DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20)); - origBuff = malloc(benchedSize); - if (origBuff==NULL) { - DISPLAY("\nError: not enough memory!\n"); - fclose(inFile); - return 12; - } - - /* Fill input buffer */ - DISPLAY("Loading %s... \r", inFileName); - { size_t const readSize = fread(origBuff, 1, benchedSize, inFile); - fclose(inFile); - if(readSize != benchedSize) { - DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); - free(origBuff); - return 13; - } } - - /* bench */ - DISPLAY("\r%79s\r", ""); - DISPLAY("using %s : \n", inFileName); - BMK_benchMemInit(origBuff, benchedSize); - - /* clean */ - free(origBuff); - } - - return 0; + return ret; } /* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */ @@ -2074,17 +2086,6 @@ static int nextStrategy(const int currentStrategy, const int bestStrategy) { } } -static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZSTD_compressionParameters mask) { - base.windowLog = mask.windowLog ? mask.windowLog : base.windowLog; - base.chainLog = mask.chainLog ? mask.chainLog : base.chainLog; - base.hashLog = mask.hashLog ? mask.hashLog : base.hashLog; - base.searchLog = mask.searchLog ? mask.searchLog : base.searchLog; - base.searchLength = mask.searchLength ? mask.searchLength : base.searchLength; - base.targetLength = mask.targetLength ? mask.targetLength : base.targetLength; - base.strategy = mask.strategy ? mask.strategy : base.strategy; - return base; -} - /* experiment with playing with this and decay value */ /* main fn called when using --optimize */ @@ -2115,6 +2116,8 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ contexts_t ctx; buffers_t buf; + g_time = UTIL_getTime(); + /* Init */ if(!cParamValid(paramTarget)) { return 1; @@ -2202,7 +2205,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ g_lvltarget.cSize /= ((double)g_strictness) / 100; target.cSpeed = (U32)g_lvltarget.cSpeed; - target.dSpeed = (U32)g_lvltarget.dSpeed; //See if this is worth + target.dSpeed = (U32)g_lvltarget.dSpeed; //See if this is reasonable. BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize); } @@ -2214,6 +2217,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } else { DISPLAY("optimizing for %lu Files", (unsigned long)nbFiles); } + if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed >> 20); } if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed >> 20); } if(target.cMem != (U32)-1) { DISPLAY(" - limit memory %u MB", target.cMem >> 20); } @@ -2369,6 +2373,7 @@ static int usage_advanced(void) DISPLAY( " -t# : Caps runtime of operation in seconds (default : %u seconds (%.1f hours)) \n", (U32)g_grillDuration_s, g_grillDuration_s / 3600); DISPLAY( " -v : Prints Benchmarking output\n"); DISPLAY( " -D : Next argument dictionary file\n"); + DISPLAY( " -s : Seperate Files\n"); return 0; } @@ -2401,13 +2406,13 @@ int main(int argc, const char** argv) const char* input_filename = NULL; const char* dictFileName = NULL; U32 main_pause = 0; - int optimizerCLevel = 0; + int cLevel = 0; + int seperateFiles = 0; constraint_t target = { 0, 0, (U32)-1 }; - ZSTD_compressionParameters paramTarget = { 0, 0, 0, 0, 0, 0, 0 }; - - g_time = UTIL_getTime(); + ZSTD_compressionParameters paramTarget = emptyParams(); + g_params = emptyParams(); assert(argc>=1); /* for exename */ @@ -2430,11 +2435,11 @@ int main(int argc, const char** argv) PARSE_SUB_ARGS("compressionSpeed=" , "cSpeed=", target.cSpeed); PARSE_SUB_ARGS("decompressionSpeed=", "dSpeed=", target.dSpeed); PARSE_SUB_ARGS("compressionMemory=" , "cMem=", target.cMem); - PARSE_SUB_ARGS("level=", "lvl=", optimizerCLevel); + PARSE_SUB_ARGS("level=", "lvl=", cLevel); PARSE_SUB_ARGS("strict=", "stc=", g_strictness); PARSE_SUB_ARGS("preferSpeed=", "prfSpd=", g_speedMultiplier); PARSE_SUB_ARGS("preferRatio=", "prfRto=", g_ratioMultiplier); - PARSE_SUB_ARGS("maxTries", "tries", g_maxTries); + PARSE_SUB_ARGS("maxTries=", "tries=", g_maxTries); DISPLAY("invalid optimization parameter \n"); return 1; @@ -2448,10 +2453,11 @@ int main(int argc, const char** argv) } else if (longCommandWArg(&argument, "--zstd=")) { /* Decode command (note : aggregated commands are allowed) */ g_singleRun = 1; - g_params = ZSTD_getCParams(2, g_blockSize, 0); + cLevel = 2; for ( ; ;) { PARSE_CPARAMS(g_params) - if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { g_params = ZSTD_getCParams(readU32FromChar(&argument), g_blockSize, 0); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevel = readU32FromChar(&argument); g_params = emptyParams(); if (argument[0]==',') { argument++; continue; } else break; } + DISPLAY("invalid compression parameter \n"); return 1; } @@ -2528,8 +2534,8 @@ int main(int argc, const char** argv) continue; case 'L': { argument++; - int const cLevel = readU32FromChar(&argument); - g_params = ZSTD_getCParams(cLevel, g_blockSize, 0); + cLevel = readU32FromChar(&argument); + g_params = emptyParams(); continue; } default : ; @@ -2558,6 +2564,10 @@ int main(int argc, const char** argv) g_grillDuration_s = (double)readU32FromChar(&argument); break; + case 's': + seperateFiles = 1; + break; + /* load dictionary file (only applicable for optimizer rn) */ case 'D': if(i == argc - 1) { /* last argument, return error. */ @@ -2588,11 +2598,24 @@ int main(int argc, const char** argv) result = benchSample(); } } else { - if (g_optimizer) { - result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget, optimizerCLevel); + if(seperateFiles) { + for(i = 0; i < argc - filenamesStart; i++) { + if (g_optimizer) { + result = optimizeForSize(argv+filenamesStart + i, 1, dictFileName, target, paramTarget, cLevel); + if(result) { DISPLAY("Error on File %d", i); return result; } + } else { + result = benchFiles(argv+filenamesStart + i, 1, dictFileName, cLevel); + if(result) { DISPLAY("Error on File %d", i); return result; } + } + } } else { - result = benchFiles(argv+filenamesStart, argc-filenamesStart, dictFileName); - } } + if (g_optimizer) { + result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget, cLevel); + } else { + result = benchFiles(argv+filenamesStart, argc-filenamesStart, dictFileName, cLevel); + } + } + } if (main_pause) { int unused; printf("press enter...\n"); unused = getchar(); (void)unused; } From 3b36fe5c68952dd03556ab04ff50dcbe0f1c9ec4 Mon Sep 17 00:00:00 2001 From: George Lu Date: Tue, 31 Jul 2018 11:13:44 -0700 Subject: [PATCH 32/55] strategy switching --- tests/paramgrill.c | 124 +++++++++++++++++++++++---------------------- 1 file changed, 64 insertions(+), 60 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index cba97ebbb..955032e84 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -73,10 +73,11 @@ typedef enum { hlog_ind = 2, slog_ind = 3, slen_ind = 4, - tlen_ind = 5 + tlen_ind = 5, + strt_ind = 6 } varInds_t; -#define NUM_PARAMS 6 +#define NUM_PARAMS 7 /* just don't use strategy as a param. */ #undef ZSTD_WINDOWLOG_MAX @@ -91,9 +92,10 @@ typedef enum { #define SLOG_RANGE (ZSTD_SEARCHLOG_MAX - ZSTD_SEARCHLOG_MIN + 1) #define SLEN_RANGE (ZSTD_SEARCHLENGTH_MAX - ZSTD_SEARCHLENGTH_MIN + 1) #define TLEN_RANGE 17 +#define STRT_RANGE (ZSTD_btultra - ZSTD_fast + 1) /* TLEN_RANGE picked manually */ -static const int rangetable[NUM_PARAMS] = { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE }; +static const int rangetable[NUM_PARAMS] = { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE, STRT_RANGE }; static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 }; /*-************************************ * Benchmark Parameters/Global Variables @@ -1213,8 +1215,10 @@ static int sanitizeVarArray(varInds_t* varNew, const int varLength, const varInd int i, j = 0; for(i = 0; i < varLength; i++) { if( !((varArray[i] == clog_ind && strat == ZSTD_fast) + || (varArray[i] == slog_ind && strat == ZSTD_fast) || (varArray[i] == slog_ind && strat == ZSTD_dfast) - || (varArray[i] == tlen_ind && strat != ZSTD_btopt && strat != ZSTD_btultra && strat != ZSTD_fast))) { + || (varArray[i] == tlen_ind && strat != ZSTD_btopt && strat != ZSTD_btultra && strat != ZSTD_fast) + /* || varArray[i] == strt_ind */ )) { varNew[j] = varArray[i]; j++; } @@ -1251,6 +1255,10 @@ static int variableParams(const ZSTD_compressionParameters paramConstraints, var res[j] = tlen_ind; j++; } + if(!paramConstraints.strategy) { + res[j] = strt_ind; + j++; + } return j; } @@ -1285,6 +1293,7 @@ static void paramVaryOnce(const varInds_t paramIndex, const int amt, ZSTD_compre case tlen_ind: ptr->targetLength = tlen_table[MAX(0, MIN(TLEN_RANGE - 1, tlen_inv(ptr->targetLength) + amt))]; break; + case strt_ind: ptr->strategy += amt; break; default: break; } } @@ -1301,37 +1310,19 @@ static void paramVariation(ZSTD_compressionParameters* ptr, const varInds_t* var const U32 changeID = FUZ_rand(&g_rand) % (varyLen << 1); paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p); } - validated = !ZSTD_isError(ZSTD_checkCParams(p)); + validated = !ZSTD_isError(ZSTD_checkCParams(p)) && p.strategy > 0; } *ptr = p; } -/* maybe put strategy back in */ -static void paramVariationWithStrategy(ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen, const U32 nbChanges) -{ - ZSTD_compressionParameters p; - U32 validated = 0; - while (!validated) { - U32 i; - p = *ptr; - for (i = 0 ; i < nbChanges ; i++) { - const U32 changeID = FUZ_rand(&g_rand) % ((varyLen + 1) << 1); - if(changeID < (U32)(varyLen << 1)) { - paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p); - } else { - p.strategy += ((FUZ_rand(&g_rand) % 2) << 1) - 1; /* +/- 1 */ - } - } - validated = !ZSTD_isError(ZSTD_checkCParams(p)); - } - *ptr = p; -} /* length of memo table given free variables */ static size_t memoTableLen(const varInds_t* varyParams, const int varyLen) { size_t arrayLen = 1; int i; for(i = 0; i < varyLen; i++) { - arrayLen *= rangetable[varyParams[i]]; + if(varyParams[i] != strt_ind) { + arrayLen *= rangetable[varyParams[i]]; + } } return arrayLen; } @@ -1354,6 +1345,7 @@ static unsigned memoTableInd(const ZSTD_compressionParameters* ptr, const varInd - ZSTD_SEARCHLENGTH_MIN; break; case tlen_ind: ind *= TLEN_RANGE; ind += tlen_inv(ptr->targetLength) - ZSTD_TARGETLENGTH_MIN; break; + case strt_ind: break; } } return ind; @@ -1376,6 +1368,7 @@ static void memoTableIndInv(ZSTD_compressionParameters* ptr, const varInds_t* va ind /= SLEN_RANGE; break; case tlen_ind: ptr->targetLength = tlen_table[(ind % TLEN_RANGE)]; ind /= TLEN_RANGE; break; + case strt_ind: break; } } } @@ -1505,7 +1498,7 @@ static void playAround(FILE* f, winnerInfo_t* winners, { int nbVariations = 0; UTIL_time_t const clockStart = UTIL_getTime(); - const U32 unconstrained[NUM_PARAMS] = { 0, 1, 2, 3, 4, 5 }; + const U32 unconstrained[NUM_PARAMS] = { 0, 1, 2, 3, 4, 5, 6 }; while (UTIL_clockSpanMicro(clockStart) < g_maxVariationTime) { @@ -1513,7 +1506,7 @@ static void playAround(FILE* f, winnerInfo_t* winners, BYTE* b; if (nbVariations++ > g_maxNbVariations) break; - paramVariationWithStrategy(&p, unconstrained, NUM_PARAMS, 4); + paramVariation(&p, unconstrained, NUM_PARAMS, 4); /* exclude faster if already played params */ if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(p))-1)) @@ -1715,8 +1708,7 @@ static int benchOnce(buffers_t buf, contexts_t ctx) { return 1; } - DISPLAY("Compression Ratio: %.3f Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)buf.srcSize / testResult.cSize, - (double)testResult.cSpeed / (1 MB), (double)testResult.dSpeed / (1 MB)); + BMK_printWinner(stdout, CUSTOM_LEVEL, testResult, g_params, buf.srcSize); return 0; } @@ -1766,7 +1758,7 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam } /* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */ -#define VARIANCE 1.1 +#define VARIANCE 1.2 static int allBench(BMK_result_t* resultPtr, const buffers_t buf, const contexts_t ctx, const ZSTD_compressionParameters cParams, @@ -1918,9 +1910,9 @@ static int benchMemo(BMK_result_t* resultPtr, * all generation after random should be sanitized. (maybe sanitize random) */ static winnerInfo_t climbOnce(const constraint_t target, - const varInds_t* varArray, const int varLen, - U8* const memoTable, - const buffers_t buf, const contexts_t ctx, + const varInds_t* varArray, const int varLen, ZSTD_strategy strat, + U8** memoTableArray, + buffers_t buf, contexts_t ctx, const ZSTD_compressionParameters init) { /* * cparam - currently considered 'center' @@ -1931,6 +1923,8 @@ static winnerInfo_t climbOnce(const constraint_t target, winnerInfo_t candidateInfo, winnerInfo; int better = 1; int feas = 0; + varInds_t varNew[NUM_PARAMS]; + int varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat); winnerInfo = initWinnerInfo(init); candidateInfo = winnerInfo; @@ -1951,15 +1945,20 @@ static winnerInfo_t climbOnce(const constraint_t target, for(offset = -1; offset <= 1; offset += 2) { candidateInfo.params = cparam; paramVaryOnce(varArray[i], offset, &candidateInfo.params); - candidateInfo.params = sanitizeParams(candidateInfo.params); - if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { - int res = benchMemo(&candidateInfo.result, + + if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params)) && candidateInfo.params.strategy > 0) { + int res; + if(strat != candidateInfo.params.strategy) { /* maybe only try strategy switching after exhausting non-switching solutions? */ + strat = candidateInfo.params.strategy; + varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat); + } + res = benchMemo(&candidateInfo.result, buf, ctx, - candidateInfo.params, target, &winnerInfo.result, memoTable, - varArray, varLen, feas); + sanitizeParams(candidateInfo.params), target, &winnerInfo.result, memoTableArray[strat], + varNew, varLenNew, feas); if(res == BETTER_RESULT) { /* synonymous with better when called w/ infeasibleBM */ winnerInfo = candidateInfo; - BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize); + BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, sanitizeParams(winnerInfo.params), target, buf.srcSize); better = 1; if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) { bestFeasible1 = winnerInfo; @@ -1974,22 +1973,29 @@ static winnerInfo_t climbOnce(const constraint_t target, } for(dist = 2; dist < varLen + 2; dist++) { /* varLen is # dimensions */ - for(i = 0; i < 2 * varLen + 2; i++) { + for(i = 0; i < (1 << varLen) / varLen + 2; i++) { int res; candidateInfo.params = cparam; /* param error checking already done here */ paramVariation(&candidateInfo.params, varArray, varLen, dist); + + if(strat != candidateInfo.params.strategy) { + strat = candidateInfo.params.strategy; + varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat); + } + res = benchMemo(&candidateInfo.result, buf, ctx, - candidateInfo.params, target, &winnerInfo.result, memoTable, - varArray, varLen, feas); + sanitizeParams(candidateInfo.params), target, &winnerInfo.result, memoTableArray[strat], + varNew, varLenNew, feas); if(res == BETTER_RESULT) { /* synonymous with better in this case*/ winnerInfo = candidateInfo; - BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize); + BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, sanitizeParams(winnerInfo.params), target, buf.srcSize); better = 1; if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) { bestFeasible1 = winnerInfo; } + break; } } @@ -2026,10 +2032,11 @@ static winnerInfo_t optimizeFixedStrategy( const constraint_t target, ZSTD_compressionParameters paramTarget, const ZSTD_strategy strat, const varInds_t* varArray, const int varLen, - U8* const memoTable, const int tries) { + U8** memoTableArray, const int tries) { int i = 0; varInds_t varNew[NUM_PARAMS]; int varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat); + ZSTD_compressionParameters init; winnerInfo_t winnerInfo, candidateInfo; winnerInfo = initWinnerInfo(emptyParams()); @@ -2043,8 +2050,8 @@ static winnerInfo_t optimizeFixedStrategy( while(i < tries) { DEBUGOUTPUT("Restart\n"); - randomConstrainedParams(&init, varNew, varLenNew, memoTable); - candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, buf, ctx, init); + randomConstrainedParams(&init, varNew, varLenNew, memoTableArray[strat]); + candidateInfo = climbOnce(target, varArray, varLen, strat, memoTableArray, buf, ctx, init); if(compareResultLT(winnerInfo.result, candidateInfo.result, target, buf.srcSize)) { winnerInfo = candidateInfo; BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize); @@ -2226,7 +2233,6 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ findClockGranularity(); { - varInds_t varNew[NUM_PARAMS]; ZSTD_compressionParameters CParams; /* find best solution from default params */ @@ -2266,8 +2272,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ { /* one iterations of hill climbing with the level-defined parameters. */ - int varLenNew = sanitizeVarArray(varNew, varLen, varArray, st); - winnerInfo_t w1 = climbOnce(target, varNew, varLenNew, allMT[st], + winnerInfo_t w1 = climbOnce(target, varArray, varLen, st, allMT, buf, ctx, winner.params); if(compareResultLT(winner.result, w1.result, target, buf.srcSize)) { winner = w1; @@ -2279,7 +2284,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ DEBUGOUTPUT("StrategySwitch: %s\n", g_stratName[st]); wc = optimizeFixedStrategy(buf, ctx, target, paramTarget, - st, varArray, varLen, allMT[st], tries); + st, varArray, varLen, allMT, tries); if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) { winner = wc; @@ -2292,7 +2297,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } } else { winner = optimizeFixedStrategy(buf, ctx, target, paramTarget, paramTarget.strategy, - varArray, varLen, allMT[paramTarget.strategy], g_maxTries); + varArray, varLen, allMT, g_maxTries); } } @@ -2387,14 +2392,13 @@ static int badusage(const char* exename) #define PARSE_SUB_ARGS(stringLong, stringShort, variable) { if (longCommandWArg(&argument, stringLong) || longCommandWArg(&argument, stringShort)) { variable = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } } #define PARSE_CPARAMS(variable) \ { \ - PARSE_SUB_ARGS("windowLog=", "wlog=", variable.vals[wlog_ind]); \ - PARSE_SUB_ARGS("chainLog=" , "clog=", variable.vals[clog_ind]); \ - PARSE_SUB_ARGS("hashLog=", "hlog=", variable.vals[hlog_ind]); \ - PARSE_SUB_ARGS("searchLog=" , "slog=", variable.vals[slog_ind]); \ - PARSE_SUB_ARGS("searchLength=", "slen=", variable.vals[slen_ind]); \ - PARSE_SUB_ARGS("targetLength=" , "tlen=", variable.vals[tlen_ind]); \ - PARSE_SUB_ARGS("strategy=", "strat=", variable.vals[strt_ind]); \ - PARSE_SUB_ARGS("forceAttachDict=", "fad=" , variable.vals[strt_ind]); \ + PARSE_SUB_ARGS("windowLog=", "wlog=", variable.windowLog); \ + PARSE_SUB_ARGS("chainLog=" , "clog=", variable.chainLog); \ + PARSE_SUB_ARGS("hashLog=", "hlog=", variable.hashLog); \ + PARSE_SUB_ARGS("searchLog=" , "slog=", variable.searchLog); \ + PARSE_SUB_ARGS("searchLength=", "slen=", variable.searchLength); \ + PARSE_SUB_ARGS("targetLength=" , "tlen=", variable.targetLength); \ + PARSE_SUB_ARGS("strategy=", "strat=", variable.strategy); \ } int main(int argc, const char** argv) From a6df961497b766546db31b7876aef5b3718c17a4 Mon Sep 17 00:00:00 2001 From: Eden Zik Date: Mon, 13 Aug 2018 20:28:52 -0400 Subject: [PATCH 33/55] Cmake now builds with CMAKE_BUILD_TYPE=Release by default, both while being invoked from the main Makefile (via cmakebuild) or directly from the build/cmake directory. Suggested by @pdknsk (#1081). --- Makefile | 2 +- README.md | 2 ++ build/.gitignore | 11 +++++++++++ build/cmake/CMakeLists.txt | 4 ++++ 4 files changed, 18 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f67791275..6c4ab9c9b 100644 --- a/Makefile +++ b/Makefile @@ -114,7 +114,7 @@ clean: ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly NetBSD MSYS_NT)) HOST_OS = POSIX -CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON +CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON -DCMAKE_BUILD_TYPE=Release .PHONY: list list: diff --git a/README.md b/README.md index 17edecb71..dc99dc0fd 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,8 @@ A `cmake` project generator is provided within `build/cmake`. It can generate Makefiles or other build scripts to create `zstd` binary, and `libzstd` dynamic and static libraries. +By default, `CMAKE_BUILD_TYPE` is set to `Release`. + #### Meson A Meson project is provided within `contrib/meson`. diff --git a/build/.gitignore b/build/.gitignore index b00e709ba..1ceb70ebc 100644 --- a/build/.gitignore +++ b/build/.gitignore @@ -18,3 +18,14 @@ Studio* # CMake cmake/build/ +CMakeCache.txt +CMakeFiles +CMakeScripts +Testing +Makefile +cmake_install.cmake +install_manifest.txt +compile_commands.json +CTestTestfile.cmake +build +lib diff --git a/build/cmake/CMakeLists.txt b/build/cmake/CMakeLists.txt index fd9bc2b1e..1e2921de8 100644 --- a/build/cmake/CMakeLists.txt +++ b/build/cmake/CMakeLists.txt @@ -10,6 +10,10 @@ PROJECT(zstd) CMAKE_MINIMUM_REQUIRED(VERSION 2.8.9) SET(ZSTD_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../..") + +# Ensure Release build even if not invoked via Makefile +SET(CMAKE_BUILD_TYPE "Release") + LIST(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules") INCLUDE(GNUInstallDirs) From 614aaa3ae152aa13c3b17ae01a39f151be6c1c56 Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 13 Aug 2018 16:38:51 -0700 Subject: [PATCH 34/55] rebase clevel --- tests/paramgrill.c | 63 ++++++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 30 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 955032e84..58f2e13bc 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -65,6 +65,7 @@ static const int g_maxNbVariations = 64; #define MIN(a,b) ( (a) < (b) ? (a) : (b) ) #define MAX(a,b) ( (a) > (b) ? (a) : (b) ) #define CUSTOM_LEVEL 99 +#define BASE_CLEVEL 1 /* indices for each of the variables */ typedef enum { @@ -112,7 +113,7 @@ static U32 g_singleRun = 0; static U32 g_optimizer = 0; static U32 g_target = 0; static U32 g_noSeed = 0; -static ZSTD_compressionParameters g_params; +static ZSTD_compressionParameters g_params; /* Initialized at the beginning of main w/ emptyParams() function */ static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */ @@ -523,6 +524,7 @@ static void freeBuffers(const buffers_t b) { free(b.resPtrs[0]); } free(b.resPtrs); + free(b.resSizes); } /* allocates buffer's arguments. returns success / failuere */ @@ -532,7 +534,7 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab size_t pos = 0; size_t n; U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles); - size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad); + const size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad); const size_t blockSize = g_blockSize ? g_blockSize : totalSizeToLoad; U32 const maxNbBlocks = (U32) ((totalSizeToLoad + (blockSize-1)) / blockSize) + (U32)nbFiles; U32 blockNb = 0; @@ -671,6 +673,7 @@ static int createContexts(contexts_t* ctx, const char* dictFileName) { freeContexts(*ctx); return 1; } + fclose(f); return 0; } @@ -824,7 +827,7 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t static int BMK_benchParam(BMK_result_t* resultPtr, buffers_t buf, contexts_t ctx, const ZSTD_compressionParameters cParams) { - BMK_return_t res = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_both, BMK_timeMode, 3); + BMK_return_t res = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_both, BMK_timeMode, 3); *resultPtr = res.result; return res.error; } @@ -840,16 +843,16 @@ static int BMK_benchParam(BMK_result_t* resultPtr, #define SIZE_RESULT 5 /* maybe have epsilon-eq to limit table size? */ static int speedSizeCompare(BMK_result_t r1, BMK_result_t r2) { - if(r1.cSpeed > r2.cSpeed) { - if(r1.cSize <= r2.cSize) { - return WORSE_RESULT; - } - return SIZE_RESULT; /* r2 is smaller but not faster. */ - } else { + if(r1.cSpeed < r2.cSpeed) { if(r1.cSize >= r2.cSize) { return BETTER_RESULT; } - return SPEED_RESULT; /* r2 is faster but not smaller */ + return SPEED_RESULT; /* r2 is smaller but not faster. */ + } else { + if(r1.cSize <= r2.cSize) { + return WORSE_RESULT; + } + return SIZE_RESULT; /* r2 is faster but not smaller */ } } @@ -875,12 +878,12 @@ static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) { } while(cur_node->next != NULL) { - switch(speedSizeCompare(r, cur_node->res.result)) { - case BETTER_RESULT: + switch(speedSizeCompare(cur_node->res.result, r)) { + case WORSE_RESULT: { return 1; /* never insert if better */ } - case WORSE_RESULT: + case BETTER_RESULT: { winner_ll_node* tmp; cur_node->res = cur_node->next->res; @@ -889,12 +892,12 @@ static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) { free(tmp); break; } - case SPEED_RESULT: + case SIZE_RESULT: { cur_node = cur_node->next; break; } - case SIZE_RESULT: /* insert after first size result, then return */ + case SPEED_RESULT: /* insert after first size result, then return */ { winner_ll_node* newnode = malloc(sizeof(winner_ll_node)); if(newnode == NULL) { @@ -911,17 +914,17 @@ static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) { } assert(cur_node->next == NULL); - switch(speedSizeCompare(r, cur_node->res.result)) { - case BETTER_RESULT: + switch(speedSizeCompare(cur_node->res.result, r)) { + case WORSE_RESULT: { return 1; /* never insert if better */ } - case WORSE_RESULT: + case BETTER_RESULT: { cur_node->res = w; return 0; } - case SPEED_RESULT: + case SIZE_RESULT: { winner_ll_node* newnode = malloc(sizeof(winner_ll_node)); if(newnode == NULL) { @@ -932,7 +935,7 @@ static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) { cur_node->next = newnode; return 0; } - case SIZE_RESULT: /* insert before first size result, then return */ + case SPEED_RESULT: /* insert before first size result, then return */ { winner_ll_node* newnode = malloc(sizeof(winner_ll_node)); if(newnode == NULL) { @@ -1448,7 +1451,7 @@ static void freeMemoTableArray(U8** mtAll) { /* takes unsanitized varyParams */ static U8** createMemoTableArray(ZSTD_compressionParameters paramConstraints, constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { varInds_t varNew[NUM_PARAMS]; - U8** mtAll = calloc(sizeof(U8*),(ZSTD_btultra + 1)); + U8** mtAll = (U8**)calloc(sizeof(U8*),(ZSTD_btultra + 1)); int i; if(mtAll == NULL) { return NULL; @@ -1467,7 +1470,7 @@ static U8** createMemoTableArray(ZSTD_compressionParameters paramConstraints, co return mtAll; } -static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZSTD_compressionParameters mask) { +static ZSTD_compressionParameters overwriteParams(ZSTD_compressionParameters base, ZSTD_compressionParameters mask) { base.windowLog = mask.windowLog ? mask.windowLog : base.windowLog; base.chainLog = mask.chainLog ? mask.chainLog : base.chainLog; base.hashLog = mask.hashLog ? mask.hashLog : base.hashLog; @@ -1744,7 +1747,7 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam DISPLAY("using %d Files : \n", nbFiles); } - g_params = ZSTD_adjustCParams(maskParams(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize), g_params), maxBlockSize, ctx.dictSize); + g_params = ZSTD_adjustCParams(overwriteParams(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize), g_params), maxBlockSize, ctx.dictSize); if(g_singleRun) { ret = benchOnce(buf, ctx); @@ -1771,7 +1774,7 @@ static int allBench(BMK_result_t* resultPtr, double winnerRS; /* initial benchmarking, gives exact ratio and memory, warms up future runs */ - benchres = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_both, BMK_iterMode, 1); + benchres = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_both, BMK_iterMode, 1); winnerRS = resultScore(*winnerResult, buf.srcSize, target); DEBUGOUTPUT("WinnerScore: %f\n ", winnerRS); @@ -1806,14 +1809,14 @@ static int allBench(BMK_result_t* resultPtr, /* second run, if first run is too short, gives approximate cSpeed + dSpeed */ if(loopDurationC < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_compressOnly, BMK_iterMode, 1); + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_compressOnly, BMK_iterMode, 1); if(benchres2.error) { return ERROR_RESULT; } benchres = benchres2; } if(loopDurationD < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_decodeOnly, BMK_iterMode, 1); + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_decodeOnly, BMK_iterMode, 1); if(benchres2.error) { return ERROR_RESULT; } @@ -1836,7 +1839,7 @@ static int allBench(BMK_result_t* resultPtr, /* Final full run if estimates are unclear */ if(loopDurationC < TIMELOOP_NANOSEC) { - BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_compressOnly, BMK_timeMode, 1); + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_compressOnly, BMK_timeMode, 1); if(benchres2.error) { return ERROR_RESULT; } @@ -1844,7 +1847,7 @@ static int allBench(BMK_result_t* resultPtr, } if(loopDurationD < TIMELOOP_NANOSEC) { - BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_decodeOnly, BMK_timeMode, 1); + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_decodeOnly, BMK_timeMode, 1); if(benchres2.error) { return ERROR_RESULT; } @@ -2245,7 +2248,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ int i; for (i=1; i<=maxSeeds; i++) { int ec; - CParams = maskParams(ZSTD_getCParams(i, maxBlockSize, ctx.dictSize), paramTarget); + CParams = overwriteParams(ZSTD_getCParams(i, maxBlockSize, ctx.dictSize), paramTarget); ec = BMK_benchParam(&candidate, buf, ctx, CParams); BMK_printWinnerOpt(stdout, i, candidate, CParams, target, buf.srcSize); @@ -2439,11 +2442,11 @@ int main(int argc, const char** argv) PARSE_SUB_ARGS("compressionSpeed=" , "cSpeed=", target.cSpeed); PARSE_SUB_ARGS("decompressionSpeed=", "dSpeed=", target.dSpeed); PARSE_SUB_ARGS("compressionMemory=" , "cMem=", target.cMem); - PARSE_SUB_ARGS("level=", "lvl=", cLevel); PARSE_SUB_ARGS("strict=", "stc=", g_strictness); PARSE_SUB_ARGS("preferSpeed=", "prfSpd=", g_speedMultiplier); PARSE_SUB_ARGS("preferRatio=", "prfRto=", g_ratioMultiplier); PARSE_SUB_ARGS("maxTries=", "tries=", g_maxTries); + if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevel = readU32FromChar(&argument); g_optmode = 1; if (argument[0]==',') { argument++; continue; } else break; } DISPLAY("invalid optimization parameter \n"); return 1; From 76acba025da5ead78532bfd0dff4de6af67a6506 Mon Sep 17 00:00:00 2001 From: George Lu Date: Tue, 14 Aug 2018 11:57:15 -0700 Subject: [PATCH 35/55] scan-build --- tests/paramgrill.c | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 58f2e13bc..7e85bf832 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -176,14 +176,14 @@ static size_t BMK_findMaxMem(U64 requiredMem) requiredMem = (((requiredMem >> 26) + 1) << 26); if (requiredMem > maxMemory) requiredMem = maxMemory; - requiredMem += 2*step; - while (!testmem) { - requiredMem -= step; + requiredMem += 2 * step; + while (!testmem && requiredMem > 0) { testmem = malloc ((size_t)requiredMem); + requiredMem -= step; } free (testmem); - return (size_t) (requiredMem - step); + return (size_t) requiredMem; } @@ -536,9 +536,14 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles); const size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad); const size_t blockSize = g_blockSize ? g_blockSize : totalSizeToLoad; - U32 const maxNbBlocks = (U32) ((totalSizeToLoad + (blockSize-1)) / blockSize) + (U32)nbFiles; + U32 const maxNbBlocks = (U32) ((totalSizeToLoad + (blockSize-1)) / MAX(blockSize, 1)) + (U32)nbFiles; U32 blockNb = 0; + if(!totalSizeToLoad || !benchedSize) { + DISPLAY("Nothing to Bench\n"); + return 1; + } + buff->srcPtrs = (const void**)calloc(maxNbBlocks, sizeof(void*)); buff->srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); @@ -555,6 +560,7 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab return 1; } + buff->srcBuffer = malloc(benchedSize); buff->srcPtrs[0] = (const void*)buff->srcBuffer; buff->dstPtrs[0] = malloc(ZSTD_compressBound(benchedSize) + (maxNbBlocks * 1024)); @@ -613,6 +619,12 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab fclose(f); } + if(!blockNb) { + DISPLAY("Failed to load any files\n"); + freeBuffers(*buff); + return 1; + } + buff->dstCapacities[0] = ZSTD_compressBound(buff->srcSizes[0]); buff->dstSizes[0] = buff->dstCapacities[0]; buff->resSizes[0] = buff->srcSizes[0]; @@ -957,8 +969,6 @@ static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) { static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const size_t srcSize) { char lvlstr[15] = "Custom Level"; - const U64 time = UTIL_clockSpanNano(g_time); - const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC); fprintf(f, "\r%79s\r", ""); @@ -974,7 +984,11 @@ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result "/* %s */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */", lvlstr, (double)srcSize / result.cSize, (double)result.cSpeed / (1 MB), (double)result.dSpeed / (1 MB)); - if(TIMED) { fprintf(f, " - %1lu:%2lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); } + if(TIMED) { + const U64 time = UTIL_clockSpanNano(g_time); + const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC); + fprintf(f, " - %1lu:%2lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); + } fprintf(f, "\n"); } @@ -2160,7 +2174,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ if(paramTarget.strategy) { varInds_t varNew[NUM_PARAMS]; int varLenNew = sanitizeVarArray(varNew, varLen, varArray, paramTarget.strategy); - allMT = calloc(sizeof(U8), (ZSTD_btultra + 1)); + allMT = (U8**)calloc(sizeof(U8*), (ZSTD_btultra + 1)); if(allMT == NULL) { ret = 57; goto _cleanUp; From f581ccd267244da3d0a4d8d6ba1cb8a1b191bfd8 Mon Sep 17 00:00:00 2001 From: George Lu Date: Tue, 31 Jul 2018 18:47:27 -0700 Subject: [PATCH 36/55] Doc Updates Add option to pass in existing parameters in use --- tests/README.md | 4 ++-- tests/paramgrill.c | 13 +++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/README.md b/tests/README.md index 736916936..4d2f314d2 100644 --- a/tests/README.md +++ b/tests/README.md @@ -105,7 +105,8 @@ Full list of arguments t# - targetLength S# - strategy L# - level - --zstd= : Single run, parameter selection syntax same as zstdcli + --zstd= : Single run, parameter selection syntax same as zstdcli. + When invoked with --optimize, this represents the sample to exceed. --optimize= : find parameters to maximize compression ratio given parameters Can use all --zstd= commands to constrain the type of solution found in addition to the following constraints cSpeed= : Minimum compression speed @@ -120,7 +121,6 @@ Full list of arguments when determining overall winner (default 1 for both, higher = more valued). tries= : Maximum number of random restarts on a single strategy before switching (Default 5) Higher values will make optimizer run longer, more chances to find better solution. - --optimize= : same as -O with more verbose syntax -P# : generated sample compressibility -t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) -v : Prints Benchmarking output diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 7e85bf832..47e0a9421 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -2234,6 +2234,19 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize); } + if(g_singleRun) { + BMK_result_t res; + g_params = ZSTD_adjustCParams(maskParams(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize), g_params), maxBlockSize, ctx.dictSize); + if(BMK_benchParam(&res, buf, ctx, g_params)) { + ret = 45; + goto _cleanUp; + } + if(compareResultLT(winner.result, res, relaxTarget(target), buf.srcSize)) { + winner.result = res; + winner.params = g_params; + } + } + /* bench */ DISPLAY("\r%79s\r", ""); if(nbFiles == 1) { From 88dda922854550959bda7640f554b5f02e5d0d4f Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 6 Aug 2018 15:08:35 -0700 Subject: [PATCH 37/55] Reduce Duplication Change Defaults Asserts actually disabled in paramgrill + fullbench --- tests/Makefile | 4 +- tests/README.md | 4 +- tests/paramgrill.c | 287 +++++++++++++++++++++------------------------ 3 files changed, 140 insertions(+), 155 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index 81e685780..88a5d763c 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -129,7 +129,7 @@ zstdmt_d_%.o : $(ZSTDDIR)/decompress/%.c fullbench32: CPPFLAGS += -m32 fullbench fullbench32 : CPPFLAGS += $(MULTITHREAD_CPP) fullbench fullbench32 : LDFLAGS += $(MULTITHREAD_LD) -fullbench fullbench32 : DEBUGFLAGS = # turn off assert() for speed measurements +fullbench fullbench32 : DEBUGFLAGS = -DNDEBUG # turn off assert() for speed measurements fullbench fullbench32 : $(ZSTD_FILES) fullbench fullbench32 : $(PRGDIR)/datagen.c $(PRGDIR)/bench.c fullbench.c $(CC) $(FLAGS) $^ -o $@$(EXT) @@ -200,7 +200,7 @@ zstreamtest-dll : $(ZSTDDIR)/common/xxhash.c # xxh symbols not exposed from dll zstreamtest-dll : $(ZSTREAM_LOCAL_FILES) $(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT) -paramgrill : DEBUGFLAGS = # turn off assert() for speed measurements +paramgrill : DEBUGFLAGS = -DNDEBUG # turn off assert() for speed measurements paramgrill : $(ZSTD_FILES) $(PRGDIR)/bench.c $(PRGDIR)/datagen.c paramgrill.c $(CC) $(FLAGS) $^ -lm -o $@$(EXT) diff --git a/tests/README.md b/tests/README.md index 4d2f314d2..3c35ecf01 100644 --- a/tests/README.md +++ b/tests/README.md @@ -118,8 +118,8 @@ Full list of arguments (Lower value will begin with stronger strategies) (Default 90%) preferSpeed= / preferRatio= : Only affects lvl = invocations. Defines value placed on compression speed or ratio - when determining overall winner (default 1 for both, higher = more valued). - tries= : Maximum number of random restarts on a single strategy before switching (Default 5) + when determining overall winner (default speed = 1, ratio = 5 for both, higher = more valued). + tries= : Maximum number of random restarts on a single strategy before switching (Default 3) Higher values will make optimizer run longer, more chances to find better solution. -P# : generated sample compressibility -t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 47e0a9421..193b2ed20 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -139,7 +139,7 @@ static BMK_result_t g_lvltarget; static int g_optmode = 0; static U32 g_speedMultiplier = 1; -static U32 g_ratioMultiplier = 1; +static U32 g_ratioMultiplier = 5; /* g_mode? */ @@ -382,7 +382,7 @@ typedef struct { } contexts_t; /*-******************************************************* -* From Paramgrill +* From bench.c *********************************************************/ static void BMK_initCCtx(ZSTD_CCtx* ctx, @@ -411,7 +411,6 @@ static void BMK_initCCtx(ZSTD_CCtx* ctx, ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize); } - static void BMK_initDCtx(ZSTD_DCtx* dctx, const void* dictBuffer, const size_t dictBufferSize) { ZSTD_DCtx_reset(dctx); @@ -503,7 +502,7 @@ static size_t local_defaultDecompress( } /*-******************************************************* -* From Paramgrill End +* From bench.c End *********************************************************/ static void freeBuffers(const buffers_t b) { @@ -527,23 +526,25 @@ static void freeBuffers(const buffers_t b) { free(b.resSizes); } -/* allocates buffer's arguments. returns success / failuere */ -static int createBuffers(buffers_t* buff, const char* const * const fileNamesTable, - const size_t nbFiles) +/* srcBuffer will be freed by freeBuffers now */ +static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, size_t nbFiles, + const size_t* fileSizes) { - size_t pos = 0; - size_t n; - U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles); - const size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad); - const size_t blockSize = g_blockSize ? g_blockSize : totalSizeToLoad; - U32 const maxNbBlocks = (U32) ((totalSizeToLoad + (blockSize-1)) / MAX(blockSize, 1)) + (U32)nbFiles; - U32 blockNb = 0; + size_t pos = 0, n, blockSize; + U32 maxNbBlocks, blockNb = 0; + buff->srcSize = 0; + for(n = 0; n < nbFiles; n++) { + buff->srcSize += fileSizes[n]; + } - if(!totalSizeToLoad || !benchedSize) { - DISPLAY("Nothing to Bench\n"); + if(buff->srcSize == 0) { + DISPLAY("No data to bench\n"); return 1; } - + + blockSize = g_blockSize ? g_blockSize : buff->srcSize; + maxNbBlocks = (U32) ((buff->srcSize + (blockSize-1)) / blockSize) + (U32)nbFiles; + buff->srcPtrs = (const void**)calloc(maxNbBlocks, sizeof(void*)); buff->srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); @@ -560,18 +561,62 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab return 1; } - - buff->srcBuffer = malloc(benchedSize); + buff->srcBuffer = srcBuffer; buff->srcPtrs[0] = (const void*)buff->srcBuffer; - buff->dstPtrs[0] = malloc(ZSTD_compressBound(benchedSize) + (maxNbBlocks * 1024)); - buff->resPtrs[0] = malloc(benchedSize); + buff->dstPtrs[0] = malloc(ZSTD_compressBound(buff->srcSize) + (maxNbBlocks * 1024)); + buff->resPtrs[0] = malloc(buff->srcSize); - if(!buff->srcPtrs[0] || !buff->dstPtrs[0] || !buff->resPtrs[0]) { + if(!buff->dstPtrs[0] || !buff->resPtrs[0]) { DISPLAY("alloc error\n"); freeBuffers(*buff); return 1; } + for(n = 0; n < nbFiles; n++) { + size_t pos_end = pos + fileSizes[n]; + for(; pos < pos_end; blockNb++) { + buff->srcPtrs[blockNb] = (const void*)((char*)srcBuffer + pos); + buff->srcSizes[blockNb] = blockSize; + pos += blockSize; + } + + if(fileSizes[n] > 0) { buff->srcSizes[blockNb - 1] = ((fileSizes[n] - 1) % blockSize) + 1; } + pos = pos_end; + } + + buff->dstCapacities[0] = ZSTD_compressBound(buff->srcSizes[0]); + buff->dstSizes[0] = buff->dstCapacities[0]; + buff->resSizes[0] = buff->srcSizes[0]; + + for(n = 1; n < blockNb; n++) { + buff->dstPtrs[n] = ((char*)buff->dstPtrs[n-1]) + buff->dstCapacities[n-1]; + buff->resPtrs[n] = ((char*)buff->resPtrs[n-1]) + buff->resSizes[n-1]; + buff->dstCapacities[n] = ZSTD_compressBound(buff->srcSizes[n]); + buff->dstSizes[n] = buff->dstCapacities[n]; + buff->resSizes[n] = buff->srcSizes[n]; + } + + buff->nbBlocks = blockNb; + + return 0; +} + +/* allocates buffer's arguments. returns success / failuere */ +static int createBuffers(buffers_t* buff, const char* const * const fileNamesTable, + size_t nbFiles) { + size_t pos = 0; + size_t n; + size_t totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles); + size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad); + size_t* fileSizes = calloc(sizeof(size_t), nbFiles); + void* srcBuffer = malloc(benchedSize); + int ret = 0; + + + if(!fileSizes || !srcBuffer) { + return 1; + } + for(n = 0; n < nbFiles; n++) { FILE* f; U64 fileSize = UTIL_getFileSize(fileNamesTable[n]); @@ -586,62 +631,35 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab f = fopen(fileNamesTable[n], "rb"); if (f==NULL) { DISPLAY("impossible to open file %s\n", fileNamesTable[n]); - freeBuffers(*buff); + free(fileSizes); + free(srcBuffer); fclose(f); return 10; } DISPLAY("Loading %s... \r", fileNamesTable[n]); - if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, n=nbFiles; /* buffer too small - stop after this file */ + if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, nbFiles=n; /* buffer too small - stop after this file */ { - char* buffer = (char*)(buff->srcBuffer); + char* buffer = (char*)(srcBuffer); size_t const readSize = fread((buffer)+pos, 1, (size_t)fileSize, f); - size_t blocked = 0; - while(blocked < readSize) { - buff->srcPtrs[blockNb] = (const void*)((buffer) + (pos + blocked)); - buff->srcSizes[blockNb] = blockSize; - blocked += blockSize; - blockNb++; - } - if(readSize > 0) { buff->srcSizes[blockNb - 1] = ((readSize - 1) % blockSize) + 1; } - if (readSize != (size_t)fileSize) { + if (readSize != (size_t)fileSize) { /* should we accept partial read? */ DISPLAY("could not read %s", fileNamesTable[n]); - freeBuffers(*buff); - fclose(f); + free(fileSizes); + free(srcBuffer); return 1; } + fileSizes[n] = readSize; pos += readSize; - } fclose(f); } - if(!blockNb) { - DISPLAY("Failed to load any files\n"); - freeBuffers(*buff); - return 1; - } - - buff->dstCapacities[0] = ZSTD_compressBound(buff->srcSizes[0]); - buff->dstSizes[0] = buff->dstCapacities[0]; - buff->resSizes[0] = buff->srcSizes[0]; - - for(n = 1; n < blockNb; n++) { - buff->dstPtrs[n] = ((char*)buff->dstPtrs[n-1]) + buff->dstCapacities[n-1]; - buff->resPtrs[n] = ((char*)buff->resPtrs[n-1]) + buff->resSizes[n-1]; - buff->dstCapacities[n] = ZSTD_compressBound(buff->srcSizes[n]); - buff->dstSizes[n] = buff->dstCapacities[n]; - buff->resSizes[n] = buff->srcSizes[n]; - } - buff->srcSize = pos; - buff->nbBlocks = blockNb; - - if (pos == 0) { DISPLAY("\nno data to bench\n"); return 1; } - - return 0; + ret = createBuffersFromMemory(buff, srcBuffer, nbFiles, fileSizes); + free(fileSizes); + return ret; } static void freeContexts(const contexts_t ctx) { @@ -1611,7 +1629,7 @@ static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBl BMK_init_level_constraints(g_target * (1 MB)); } else { /* baseline config for level 1 */ - ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, maxBlockSize, ctx.dictSize); //is dictionary ever even useful here? + ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, maxBlockSize, ctx.dictSize); BMK_result_t testResult; BMK_benchParam(&testResult, buf, ctx, l1params); BMK_init_level_constraints((int)((testResult.cSpeed * 31) / 32)); @@ -1641,82 +1659,6 @@ static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBl fclose(f); } -static int benchSample(void) -{ - const char* const name = "Sample 10MB"; - size_t const benchedSize = 10 MB; - U32 blockSize = g_blockSize ? g_blockSize : benchedSize; - U32 const maxNbBlocks = (U32) ((benchedSize + (blockSize-1)) / blockSize) + 1; - size_t splitSize = 0; - - buffers_t buf; - contexts_t ctx; - - buf.srcPtrs = (const void**)calloc(maxNbBlocks, sizeof(void*)); - buf.dstPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); - buf.resPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); - buf.srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); - buf.dstSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); - buf.dstCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); - buf.resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); - buf.srcSize = benchedSize; - - if(!buf.srcPtrs || !buf.dstPtrs || !buf.resPtrs || !buf.srcSizes || !buf.dstSizes || !buf.dstCapacities || !buf.resSizes) { - DISPLAY("Allocation Error\n"); - freeBuffers(buf); - return 1; - } - - buf.srcBuffer = malloc(benchedSize); - buf.srcPtrs[0] = (const void*)buf.srcBuffer; - buf.dstPtrs[0] = malloc(ZSTD_compressBound(benchedSize) + 1024 * maxNbBlocks); - buf.resPtrs[0] = malloc(benchedSize); - - if(!buf.srcPtrs[0] || !buf.dstPtrs[0] || !buf.resPtrs[0]) { - DISPLAY("Allocation Error\n"); - freeBuffers(buf); - return 1; - } - - - splitSize = MIN(benchedSize, blockSize); - buf.srcSizes[0] = splitSize; - buf.dstCapacities[0] = ZSTD_compressBound(splitSize); - buf.resSizes[0] = splitSize; - - for(buf.nbBlocks = 1; splitSize < benchedSize; buf.nbBlocks++) { - const size_t i = buf.nbBlocks; - const size_t nextBlockSize = MIN(benchedSize - splitSize, blockSize); - buf.srcSizes[i] = nextBlockSize; - buf.dstCapacities[i] = ZSTD_compressBound(nextBlockSize); - buf.resSizes[i] = nextBlockSize; - buf.srcPtrs[i] = (const void*)(((const char*)buf.srcPtrs[i-1]) + buf.srcSizes[i-1]); - buf.dstPtrs[i] = (void*)(((char*)buf.dstPtrs[i-1]) + buf.dstSizes[i-1]); - buf.resPtrs[i] = (void*)(((char*)buf.resPtrs[i-1]) + buf.resSizes[i-1]); - splitSize += nextBlockSize; - } - - if(createContexts(&ctx, NULL)) { - DISPLAY("Context Creation Error\n"); - freeBuffers(buf); - return 1; - } - - RDG_genBuffer(buf.srcBuffer, benchedSize, g_compressibility, 0.0, 0); - - /* bench */ - DISPLAY("\r%79s\r", ""); - DISPLAY("using %s %i%%: \n", name, (int)(g_compressibility*100)); - - BMK_benchFullTable(buf, ctx, MIN(blockSize, benchedSize)); - - freeBuffers(buf); - freeContexts(ctx); - - return 0; -} - - static int benchOnce(buffers_t buf, contexts_t ctx) { BMK_result_t testResult; @@ -1726,9 +1668,56 @@ static int benchOnce(buffers_t buf, contexts_t ctx) { } BMK_printWinner(stdout, CUSTOM_LEVEL, testResult, g_params, buf.srcSize); + return 0; } +static int benchSample(void) +{ + const char* const name = "Sample 10MB"; + size_t const benchedSize = 10 MB; + U32 blockSize = g_blockSize ? g_blockSize : benchedSize; + void* srcBuffer = malloc(benchedSize); + int ret = 0; + + buffers_t buf; + contexts_t ctx; + + if(srcBuffer == NULL) { + DISPLAY("Out of Memory\n"); + return 2; + } + + RDG_genBuffer(srcBuffer, benchedSize, g_compressibility, 0.0, 0); + + if(createBuffersFromMemory(&buf, srcBuffer, 1, &benchedSize)) { + DISPLAY("Buffer Creation Error\n"); + free(srcBuffer); + return 3; + } + + if(createContexts(&ctx, NULL)) { + DISPLAY("Context Creation Error\n"); + freeBuffers(buf); + return 1; + } + + /* bench */ + DISPLAY("\r%79s\r", ""); + DISPLAY("using %s %i%%: \n", name, (int)(g_compressibility*100)); + + if(g_singleRun) { + ret = benchOnce(buf, ctx); + } else { + BMK_benchFullTable(buf, ctx, MIN(blockSize, benchedSize)); + } + + freeBuffers(buf); + freeContexts(ctx); + + return ret; +} + /* benchFiles() : * note: while this function takes a table of filenames, * in practice, only the first filename will be used */ @@ -1776,6 +1765,7 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam /* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */ #define VARIANCE 1.2 +#define HIGH_VARIANCE 100.0 static int allBench(BMK_result_t* resultPtr, const buffers_t buf, const contexts_t ctx, const ZSTD_compressionParameters cParams, @@ -1784,7 +1774,7 @@ static int allBench(BMK_result_t* resultPtr, BMK_return_t benchres; BMK_result_t resultMax; U64 loopDurationC = 0, loopDurationD = 0; - double uncertaintyConstantC, uncertaintyConstantD; + double uncertaintyConstantC = 3., uncertaintyConstantD = 3.; double winnerRS; /* initial benchmarking, gives exact ratio and memory, warms up future runs */ @@ -1802,18 +1792,12 @@ static int allBench(BMK_result_t* resultPtr, /* calculate uncertainty in compression / decompression runs */ if(benchres.result.cSpeed) { loopDurationC = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); - uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC) * VARIANCE; - } else { - loopDurationC = 0; - uncertaintyConstantC = 3; + uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC); } if(benchres.result.dSpeed) { loopDurationD = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); - uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD) * VARIANCE; - } else { - loopDurationD = 0; - uncertaintyConstantD = 3; + uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD); } /* anything with worse ratio in feas is definitely worse, discard */ @@ -1841,8 +1825,8 @@ static int allBench(BMK_result_t* resultPtr, /* optimistic assumption of benchres.result */ resultMax = benchres.result; - resultMax.cSpeed *= uncertaintyConstantC; - resultMax.dSpeed *= uncertaintyConstantD; + resultMax.cSpeed *= uncertaintyConstantC * VARIANCE; + resultMax.dSpeed *= uncertaintyConstantD * VARIANCE; /* disregard infeasible results in feas mode */ /* disregard if resultMax < winner in infeas mode */ @@ -2125,7 +2109,7 @@ static int nextStrategy(const int currentStrategy, const int bestStrategy) { * cLevel - compression level to exceed (all solutions must be > lvl in cSpeed + ratio) */ -static int g_maxTries = 5; +static int g_maxTries = 3; #define TRY_DECAY 1 static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel) @@ -2140,8 +2124,6 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ contexts_t ctx; buffers_t buf; - g_time = UTIL_getTime(); - /* Init */ if(!cParamValid(paramTarget)) { return 1; @@ -2234,6 +2216,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize); } + /* Don't want it to return anything worse than the best known result */ if(g_singleRun) { BMK_result_t res; g_params = ZSTD_adjustCParams(maskParams(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize), g_params), maxBlockSize, ctx.dictSize); @@ -2450,6 +2433,8 @@ int main(int argc, const char** argv) assert(argc>=1); /* for exename */ + g_time = UTIL_getTime(); + /* Welcome message */ DISPLAY(WELCOME_MESSAGE); From e3c679484aed822ec5574ae426a77490966b35aa Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 6 Aug 2018 16:52:17 -0700 Subject: [PATCH 38/55] Add Time Checks Fix double -> U64 display --- tests/paramgrill.c | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 193b2ed20..af4459187 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -17,7 +17,6 @@ #include /* fprintf, fopen, ftello64 */ #include /* strcmp */ #include /* log */ -#include #include #include "mem.h" @@ -96,6 +95,9 @@ typedef enum { #define STRT_RANGE (ZSTD_btultra - ZSTD_fast + 1) /* TLEN_RANGE picked manually */ +#define CHECKTIME(r) { if(BMK_timeSpan(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); return r; } } +#define CHECKTIMEGT(ret, val, _gototag) {if(BMK_timeSpan(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); ret = val; goto _gototag; } } + static const int rangetable[NUM_PARAMS] = { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE, STRT_RANGE }; static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 }; /*-************************************ @@ -104,7 +106,7 @@ static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48 typedef BYTE U8; -static double g_grillDuration_s = 99999; /* about 27 hours */ +static U32 g_timeLimit_s = 99999; /* about 27 hours */ static U32 g_nbIterations = NBLOOPS; static double g_compressibility = COMPRESSIBILITY_DEFAULT; static U32 g_blockSize = 0; @@ -166,7 +168,7 @@ void BMK_SetNbIterations(int nbLoops) *********************************************************/ /* accuracy in seconds only, span can be multiple years */ -static double BMK_timeSpan(time_t tStart) { return difftime(time(NULL), tStart); } +static U32 BMK_timeSpan(UTIL_time_t tStart) { return (U32)(UTIL_clockSpanMicro(tStart) / 1000000ULL); } static size_t BMK_findMaxMem(U64 requiredMem) { @@ -1252,8 +1254,7 @@ static int sanitizeVarArray(varInds_t* varNew, const int varLength, const varInd if( !((varArray[i] == clog_ind && strat == ZSTD_fast) || (varArray[i] == slog_ind && strat == ZSTD_fast) || (varArray[i] == slog_ind && strat == ZSTD_dfast) - || (varArray[i] == tlen_ind && strat != ZSTD_btopt && strat != ZSTD_btultra && strat != ZSTD_fast) - /* || varArray[i] == strt_ind */ )) { + || (varArray[i] == tlen_ind && strat != ZSTD_btopt && strat != ZSTD_btultra && strat != ZSTD_fast))) { varNew[j] = varArray[i]; j++; } @@ -1645,10 +1646,10 @@ static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBl BMK_printWinners(f, winners, buf.srcSize); /* start tests */ - { const time_t grillStart = time(NULL); + { const UTIL_time_t grillStart = UTIL_getTime(); do { BMK_selectRandomStart(f, winners, buf, ctx); - } while (BMK_timeSpan(grillStart) < g_grillDuration_s); + } while (BMK_timeSpan(grillStart) < g_timeLimit_s); } /* end summary */ @@ -1893,7 +1894,6 @@ static int benchMemo(BMK_result_t* resultPtr, return res; } - /* One iteration of hill climbing. Specifically, it first tries all * valid parameter configurations w/ manhattan distance 1 and picks the best one * failing that, it progressively tries candidates further and further away (up to #dim + 2) @@ -1944,6 +1944,7 @@ static winnerInfo_t climbOnce(const constraint_t target, /* all dist-1 candidates */ for(i = 0; i < varLen; i++) { for(offset = -1; offset <= 1; offset += 2) { + CHECKTIME(winnerInfo); candidateInfo.params = cparam; paramVaryOnce(varArray[i], offset, &candidateInfo.params); @@ -1953,8 +1954,7 @@ static winnerInfo_t climbOnce(const constraint_t target, strat = candidateInfo.params.strategy; varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat); } - res = benchMemo(&candidateInfo.result, - buf, ctx, + res = benchMemo(&candidateInfo.result, buf, ctx, sanitizeParams(candidateInfo.params), target, &winnerInfo.result, memoTableArray[strat], varNew, varLenNew, feas); if(res == BETTER_RESULT) { /* synonymous with better when called w/ infeasibleBM */ @@ -1976,6 +1976,7 @@ static winnerInfo_t climbOnce(const constraint_t target, for(dist = 2; dist < varLen + 2; dist++) { /* varLen is # dimensions */ for(i = 0; i < (1 << varLen) / varLen + 2; i++) { int res; + CHECKTIME(winnerInfo); candidateInfo.params = cparam; /* param error checking already done here */ paramVariation(&candidateInfo.params, varArray, varLen, dist); @@ -1985,8 +1986,7 @@ static winnerInfo_t climbOnce(const constraint_t target, varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat); } - res = benchMemo(&candidateInfo.result, - buf, ctx, + res = benchMemo(&candidateInfo.result, buf, ctx, sanitizeParams(candidateInfo.params), target, &winnerInfo.result, memoTableArray[strat], varNew, varLenNew, feas); if(res == BETTER_RESULT) { /* synonymous with better in this case*/ @@ -2058,7 +2058,7 @@ static winnerInfo_t optimizeFixedStrategy( BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize); i = 0; } - + CHECKTIME(winnerInfo); i++; } return winnerInfo; @@ -2123,6 +2123,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ size_t maxBlockSize = 0; contexts_t ctx; buffers_t buf; + g_time = UTIL_getTime(); /* Init */ if(!cParamValid(paramTarget)) { @@ -2174,7 +2175,6 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ allMT = createMemoTableArray(paramTarget, target, varArray, varLen, maxBlockSize); } - if(!allMT) { DISPLAY("MemoTable Init Error\n"); ret = 2; @@ -2267,6 +2267,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ winner.params = CParams; } + CHECKTIMEGT(ret, 0, _cleanUp); /* if pass time limit, stop */ /* if the current params are too slow, just stop. */ if(target.cSpeed > candidate.cSpeed * 3 / 2) { break; } } @@ -2290,6 +2291,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ if(compareResultLT(winner.result, w1.result, target, buf.srcSize)) { winner = w1; } + CHECKTIMEGT(ret, 0, _cleanUp); } while(st && tries > 0) { @@ -2307,6 +2309,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ st = nextStrategy(st, bestStrategy); tries -= TRY_DECAY; } + CHECKTIMEGT(ret, 0, _cleanUp); } } else { winner = optimizeFixedStrategy(buf, ctx, target, paramTarget, paramTarget.strategy, @@ -2388,7 +2391,7 @@ static int usage_advanced(void) DISPLAY( " -S : Single run \n"); DISPLAY( " --zstd : Single run, parameter selection same as zstdcli \n"); DISPLAY( " -P# : generated sample compressibility (default : %.1f%%) \n", COMPRESSIBILITY_DEFAULT * 100); - DISPLAY( " -t# : Caps runtime of operation in seconds (default : %u seconds (%.1f hours)) \n", (U32)g_grillDuration_s, g_grillDuration_s / 3600); + DISPLAY( " -t# : Caps runtime of operation in seconds (default : %u seconds (%.1f hours)) \n", g_timeLimit_s, (double)g_timeLimit_s / 3600); DISPLAY( " -v : Prints Benchmarking output\n"); DISPLAY( " -D : Next argument dictionary file\n"); DISPLAY( " -s : Seperate Files\n"); @@ -2433,8 +2436,6 @@ int main(int argc, const char** argv) assert(argc>=1); /* for exename */ - g_time = UTIL_getTime(); - /* Welcome message */ DISPLAY(WELCOME_MESSAGE); @@ -2580,7 +2581,7 @@ int main(int argc, const char** argv) /* caps runtime (in seconds) */ case 't': argument++; - g_grillDuration_s = (double)readU32FromChar(&argument); + g_timeLimit_s = readU32FromChar(&argument); break; case 's': @@ -2609,6 +2610,7 @@ int main(int argc, const char** argv) /* first provided filename is input */ if (!input_filename) { input_filename=argument; filenamesStart=i; continue; } } + if (filenamesStart==0) { if (g_optimizer) { DISPLAY("Optimizer Expects File\n"); From 3f2d024dca7723b4235618d99ce755287e530472 Mon Sep 17 00:00:00 2001 From: George Lu Date: Thu, 9 Aug 2018 10:42:35 -0700 Subject: [PATCH 39/55] forceAttachDict --- tests/paramgrill.c | 746 ++++++++++++++++++++++++--------------------- 1 file changed, 397 insertions(+), 349 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index af4459187..aab46ede5 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -66,22 +66,10 @@ static const int g_maxNbVariations = 64; #define CUSTOM_LEVEL 99 #define BASE_CLEVEL 1 -/* indices for each of the variables */ -typedef enum { - wlog_ind = 0, - clog_ind = 1, - hlog_ind = 2, - slog_ind = 3, - slen_ind = 4, - tlen_ind = 5, - strt_ind = 6 -} varInds_t; - -#define NUM_PARAMS 7 -/* just don't use strategy as a param. */ - #undef ZSTD_WINDOWLOG_MAX #define ZSTD_WINDOWLOG_MAX 27 //no long range stuff for now. +#define FADT_MIN 0 +#define FADT_MAX ((U32)-1) #define ZSTD_TARGETLENGTH_MIN 0 #define ZSTD_TARGETLENGTH_MAX 999 @@ -93,13 +81,146 @@ typedef enum { #define SLEN_RANGE (ZSTD_SEARCHLENGTH_MAX - ZSTD_SEARCHLENGTH_MIN + 1) #define TLEN_RANGE 17 #define STRT_RANGE (ZSTD_btultra - ZSTD_fast + 1) -/* TLEN_RANGE picked manually */ +#define FADT_RANGE 3 #define CHECKTIME(r) { if(BMK_timeSpan(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); return r; } } #define CHECKTIMEGT(ret, val, _gototag) {if(BMK_timeSpan(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); ret = val; goto _gototag; } } -static const int rangetable[NUM_PARAMS] = { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE, STRT_RANGE }; +#define PARAM_UNSET ((U32)-2) /* can't be -1 b/c fadt */ + +/*-************************************ +* Setup for Adding new params +**************************************/ + +/* indices for each of the variables */ +typedef enum { + wlog_ind = 0, + clog_ind = 1, + hlog_ind = 2, + slog_ind = 3, + slen_ind = 4, + tlen_ind = 5, + strt_ind = 6, + fadt_ind = 7, /* forceAttachDict */ + NUM_PARAMS = 8 +} varInds_t; + +/* maximum value of parameters */ +static const U32 mintable[NUM_PARAMS] = + { ZSTD_WINDOWLOG_MIN, ZSTD_CHAINLOG_MIN, ZSTD_HASHLOG_MIN, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLENGTH_MIN, ZSTD_TARGETLENGTH_MIN, ZSTD_fast, FADT_MIN }; + +/* minimum value of parameters */ +static const U32 maxtable[NUM_PARAMS] = + { ZSTD_WINDOWLOG_MAX, ZSTD_CHAINLOG_MAX, ZSTD_HASHLOG_MAX, ZSTD_SEARCHLOG_MAX, ZSTD_SEARCHLENGTH_MAX, ZSTD_TARGETLENGTH_MAX, ZSTD_btultra, FADT_MAX }; + +/* # of values parameters can take on */ +static const U32 rangetable[NUM_PARAMS] = + { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE, STRT_RANGE, FADT_RANGE }; + +static const ZSTD_cParameter cctxSetParamTable[NUM_PARAMS] = + { ZSTD_p_windowLog, ZSTD_p_chainLog, ZSTD_p_hashLog, ZSTD_p_searchLog, ZSTD_p_minMatch, ZSTD_p_targetLength, ZSTD_p_compressionStrategy, ZSTD_p_forceAttachDict }; + +static const char* g_paramNames[NUM_PARAMS] = + { "windowLog", "chainLog", "hashLog","searchLog", "searchLength", "targetLength", "strategy", "forceAttachDict"}; + + static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 }; +/* maps value from 0 to rangetable[param] - 1 to valid paramvalue */ +static U32 rangeMap(varInds_t param, U32 ind) { + ind = MIN(ind, rangetable[param] - 1); + switch(param) { + case tlen_ind: + return tlen_table[ind]; + case fadt_ind: /* 0, 1, 2 -> -1, 0, 1 */ + return ind - 1; + case wlog_ind: /* using default: triggers -Wswitch-enum */ + case clog_ind: + case hlog_ind: + case slog_ind: + case slen_ind: + case strt_ind: + return mintable[param] + ind; + case NUM_PARAMS: + return (U32)-1; + } + return 0; /* should never happen, stop compiler warnings */ +} + +/* inverse of rangeMap */ +static U32 invRangeMap(varInds_t param, U32 value) { + value = MIN(MAX(mintable[param], value), maxtable[param]); + switch(param) { + case tlen_ind: /* bin search */ + { + int lo = 0; + int hi = TLEN_RANGE; + while(lo < hi) { + int mid = (lo + hi) / 2; + if(tlen_table[mid] < value) { + lo = mid + 1; + } if(tlen_table[mid] == value) { + return mid; + } else { + hi = mid; + } + } + return lo; + } + case fadt_ind: + return value + 1; + case wlog_ind: + case clog_ind: + case hlog_ind: + case slog_ind: + case slen_ind: + case strt_ind: + return value - mintable[param]; + case NUM_PARAMS: + return (U32)-1; + } + return 0; /* should never happen, stop compiler warnings */ +} + +typedef struct { + U32 vals[NUM_PARAMS]; +} paramValues_t; + +//TODO: unset -> 0? +static ZSTD_compressionParameters pvalsToCParams(paramValues_t p) { + ZSTD_compressionParameters c; + c.windowLog = p.vals[wlog_ind]; + c.chainLog = p.vals[clog_ind]; + c.hashLog = p.vals[hlog_ind]; + c.searchLog = p.vals[slog_ind]; + c.searchLength = p.vals[slen_ind]; + c.targetLength = p.vals[tlen_ind]; + c.strategy = p.vals[strt_ind]; + /* no forceAttachDict */ + return c; +} + +/* 0 = auto for fadt */ +static paramValues_t cParamsToPVals(ZSTD_compressionParameters c) { + paramValues_t p; + p.vals[wlog_ind] = c.windowLog; + p.vals[clog_ind] = c.chainLog; + p.vals[hlog_ind] = c.hashLog; + p.vals[slog_ind] = c.searchLog; + p.vals[slen_ind] = c.searchLength; + p.vals[tlen_ind] = c.targetLength; + p.vals[strt_ind] = c.strategy; + p.vals[fadt_ind] = 0; + return p; +} + +/* equivalent of ZSTD_adjustCParams for paramValues_t */ +static paramValues_t adjustParams(paramValues_t p, size_t maxBlockSize, size_t dictSize) { + U32 fval = p.vals[fadt_ind]; + p = cParamsToPVals(ZSTD_adjustCParams(pvalsToCParams(p), maxBlockSize, dictSize)); + p.vals[fadt_ind] = fval; + return p; +} + /*-************************************ * Benchmark Parameters/Global Variables **************************************/ @@ -115,15 +236,20 @@ static U32 g_singleRun = 0; static U32 g_optimizer = 0; static U32 g_target = 0; static U32 g_noSeed = 0; -static ZSTD_compressionParameters g_params; /* Initialized at the beginning of main w/ emptyParams() function */ +static paramValues_t g_params; /* Initialized at the beginning of main w/ emptyParams() function */ static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */ typedef struct { BMK_result_t result; - ZSTD_compressionParameters params; + paramValues_t params; } winnerInfo_t; +typedef struct { + BMK_result_t result; + ZSTD_compressionParameters params; +} oldWinnerInfo_t; + typedef struct { U32 cSpeed; /* bytes / sec */ U32 dSpeed; @@ -242,8 +368,9 @@ static void findClockGranularity(void) { DEBUGOUTPUT("Granularity: %llu\n", (unsigned long long)g_clockGranularity); } +/* allows zeros */ #define CLAMPCHECK(val,min,max) { \ - if (val && (((val)<(min)) | ((val)>(max)))) { \ + if (((val)<(min)) | ((val)>(max))) { \ DISPLAY("INVALID PARAMETER CONSTRAINTS\n"); \ return 0; \ } } @@ -251,36 +378,53 @@ static void findClockGranularity(void) { /* Like ZSTD_checkCParams() but allows 0's */ /* no check on targetLen? */ -static int cParamValid(ZSTD_compressionParameters paramTarget) { - CLAMPCHECK(paramTarget.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); - CLAMPCHECK(paramTarget.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX); - CLAMPCHECK(paramTarget.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX); - CLAMPCHECK(paramTarget.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX); - CLAMPCHECK(paramTarget.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX); - if(paramTarget.targetLength > ZSTD_TARGETLENGTH_MAX) { - DISPLAY("INVALID PARAMETER CONSTRAINTS\n"); - return 0; - } - if(paramTarget.strategy > ZSTD_btultra) { - DISPLAY("INVALID PARAMETER CONSTRAINTS\n"); - return 0; +static int paramValid(paramValues_t paramTarget) { + U32 i; + for(i = 0; i < NUM_PARAMS; i++) { + CLAMPCHECK(paramTarget.vals[i], mintable[i], maxtable[i]); } + //TODO: Strategy could be valid at 0 before, is that right? return 1; } -static void cParamZeroMin(ZSTD_compressionParameters* paramTarget) { - paramTarget->windowLog = paramTarget->windowLog ? paramTarget->windowLog : ZSTD_WINDOWLOG_MIN; - paramTarget->searchLog = paramTarget->searchLog ? paramTarget->searchLog : ZSTD_SEARCHLOG_MIN; - paramTarget->chainLog = paramTarget->chainLog ? paramTarget->chainLog : ZSTD_CHAINLOG_MIN; - paramTarget->hashLog = paramTarget->hashLog ? paramTarget->hashLog : ZSTD_HASHLOG_MIN; - paramTarget->searchLength = paramTarget->searchLength ? paramTarget->searchLength : ZSTD_SEARCHLENGTH_MIN; - paramTarget->targetLength = paramTarget->targetLength ? paramTarget->targetLength : 0; +//TODO: doesn't affect strategy? +static paramValues_t cParamUnsetMin(paramValues_t paramTarget) { + varInds_t i; + for(i = 0; i < NUM_PARAMS; i++) { + if(paramTarget.vals[i] == PARAM_UNSET) { + paramTarget.vals[i] = mintable[i]; + } + } + return paramTarget; } -static void BMK_translateAdvancedParams(const ZSTD_compressionParameters params) -{ - DISPLAY("--zstd=windowLog=%u,chainLog=%u,hashLog=%u,searchLog=%u,searchLength=%u,targetLength=%u,strategy=%u \n", - params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, params.targetLength, (U32)(params.strategy)); +static void BMK_translateAdvancedParams(FILE* f, const paramValues_t params) { + U32 i; + fprintf(f,"--zstd="); + for(i = 0; i < NUM_PARAMS; i++) { + fprintf(f,"%s", g_paramNames[i]); + fprintf(f,"=%u", params.vals[i]); + if(i != NUM_PARAMS - 1) { + fprintf(f, ","); + } + } + fprintf(f, "\n"); +} + + +static const char* g_stratName[ZSTD_btultra+1] = { + "(none) ", "ZSTD_fast ", "ZSTD_dfast ", + "ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ", + "ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "}; + +static void BMK_displayOneResult(FILE* f, winnerInfo_t res, size_t srcSize) { + res.params = cParamUnsetMin(res.params); + fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u,%3d, %s}, ", + res.params.vals[wlog_ind], res.params.vals[clog_ind], res.params.vals[hlog_ind], res.params.vals[slog_ind], res.params.vals[slen_ind], + res.params.vals[tlen_ind], (int)res.params.vals[fadt_ind], g_stratName[res.params.vals[strt_ind]]); + fprintf(f, + " /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", + (double)srcSize / res.result.cSize, (double)res.result.cSpeed / (1 MB), (double)res.result.dSpeed / (1 MB)); } /* checks results are feasible */ @@ -343,17 +487,16 @@ static constraint_t relaxTarget(constraint_t target) { * Bench functions *********************************************************/ -const char* g_stratName[ZSTD_btultra+1] = { - "(none) ", "ZSTD_fast ", "ZSTD_dfast ", - "ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ", - "ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "}; - -static ZSTD_compressionParameters emptyParams(void) { - ZSTD_compressionParameters p = { 0, 0, 0, 0, 0, 0, (ZSTD_strategy)0 }; +static paramValues_t emptyParams(void) { + U32 i; + paramValues_t p; + for(i = 0; i < NUM_PARAMS; i++) { + p.vals[i] = PARAM_UNSET; + } return p; } -static winnerInfo_t initWinnerInfo(ZSTD_compressionParameters p) { +static winnerInfo_t initWinnerInfo(paramValues_t p) { winnerInfo_t w1; w1.result.cSpeed = 0.; w1.result.dSpeed = 0.; @@ -389,27 +532,16 @@ typedef struct { static void BMK_initCCtx(ZSTD_CCtx* ctx, const void* dictBuffer, const size_t dictBufferSize, const int cLevel, - const ZSTD_compressionParameters* comprParams, const BMK_advancedParams_t* adv) { + const paramValues_t* comprParams) { + varInds_t i; ZSTD_CCtx_reset(ctx); ZSTD_CCtx_resetParameters(ctx); - if (adv->nbWorkers==1) { - ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, 0); - } else { - ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, adv->nbWorkers); - } ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_enableLongDistanceMatching, adv->ldmFlag); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmMinMatch, adv->ldmMinMatch); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashLog, adv->ldmHashLog); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmBucketSizeLog, adv->ldmBucketSizeLog); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashEveryLog, adv->ldmHashEveryLog); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_windowLog, comprParams->windowLog); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_hashLog, comprParams->hashLog); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_chainLog, comprParams->chainLog); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_searchLog, comprParams->searchLog); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_minMatch, comprParams->searchLength); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_targetLength, comprParams->targetLength); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy, comprParams->strategy); + + for(i = 0; i < NUM_PARAMS; i++) { + if(comprParams->vals[i] != PARAM_UNSET) + ZSTD_CCtx_setParameter(ctx, cctxSetParamTable[i], comprParams->vals[i]); + } ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize); } @@ -424,13 +556,12 @@ typedef struct { const void* dictBuffer; size_t dictBufferSize; int cLevel; - const ZSTD_compressionParameters* comprParams; - const BMK_advancedParams_t* adv; + const paramValues_t* comprParams; } BMK_initCCtxArgs; static size_t local_initCCtx(void* payload) { const BMK_initCCtxArgs* ag = (const BMK_initCCtxArgs*)payload; - BMK_initCCtx(ag->ctx, ag->dictBuffer, ag->dictBufferSize, ag->cLevel, ag->comprParams, ag->adv); + BMK_initCCtx(ag->ctx, ag->dictBuffer, ag->dictBufferSize, ag->cLevel, ag->comprParams); return 0; } @@ -507,10 +638,7 @@ static size_t local_defaultDecompress( * From bench.c End *********************************************************/ -static void freeBuffers(const buffers_t b) { - if(b.srcPtrs != NULL) { - free(b.srcBuffer); - } +static void freeNonSrcBuffers(const buffers_t b) { free(b.srcPtrs); free(b.srcSizes); @@ -528,6 +656,13 @@ static void freeBuffers(const buffers_t b) { free(b.resSizes); } +static void freeBuffers(const buffers_t b) { + if(b.srcPtrs != NULL) { + free(b.srcBuffer); + } + freeNonSrcBuffers(b); +} + /* srcBuffer will be freed by freeBuffers now */ static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, size_t nbFiles, const size_t* fileSizes) @@ -559,7 +694,7 @@ static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, size_t nbF if(!buff->srcPtrs || !buff->srcSizes || !buff->dstPtrs || !buff->dstCapacities || !buff->dstSizes || !buff->resPtrs || !buff->resSizes) { DISPLAY("alloc error\n"); - freeBuffers(*buff); + freeNonSrcBuffers(*buff); return 1; } @@ -570,7 +705,7 @@ static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, size_t nbF if(!buff->dstPtrs[0] || !buff->resPtrs[0]) { DISPLAY("alloc error\n"); - freeBuffers(*buff); + freeNonSrcBuffers(*buff); return 1; } @@ -611,12 +746,20 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab size_t totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles); size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad); size_t* fileSizes = calloc(sizeof(size_t), nbFiles); - void* srcBuffer = malloc(benchedSize); + void* srcBuffer = NULL; int ret = 0; + if(!totalSizeToLoad || !benchedSize) { + ret = 1; + DISPLAY("Nothing to Bench\n"); + goto _cleanUp; + } + + srcBuffer = malloc(benchedSize); if(!fileSizes || !srcBuffer) { - return 1; + ret = 1; + goto _cleanUp; } for(n = 0; n < nbFiles; n++) { @@ -633,10 +776,9 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab f = fopen(fileNamesTable[n], "rb"); if (f==NULL) { DISPLAY("impossible to open file %s\n", fileNamesTable[n]); - free(fileSizes); - free(srcBuffer); fclose(f); - return 10; + ret = 10; + goto _cleanUp; } DISPLAY("Loading %s... \r", fileNamesTable[n]); @@ -645,21 +787,22 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab { char* buffer = (char*)(srcBuffer); size_t const readSize = fread((buffer)+pos, 1, (size_t)fileSize, f); - + fclose(f); if (readSize != (size_t)fileSize) { /* should we accept partial read? */ DISPLAY("could not read %s", fileNamesTable[n]); - free(fileSizes); - free(srcBuffer); - return 1; + ret = 1; + goto _cleanUp; } fileSizes[n] = readSize; pos += readSize; } - fclose(f); } ret = createBuffersFromMemory(buff, srcBuffer, nbFiles, fileSizes); + +_cleanUp: + if(ret) { free(srcBuffer); } free(fileSizes); return ret; } @@ -717,7 +860,7 @@ static int createContexts(contexts_t* ctx, const char* dictFileName) { /* if in decodeOnly, then srcPtr's will be compressed blocks, and uncompressedBlocks will be written to dstPtrs? */ /* dictionary nullable, nothing else though. */ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t ctx, - const int cLevel, const ZSTD_compressionParameters* comprParams, + const int cLevel, const paramValues_t* comprParams, const BMK_mode_t mode, const BMK_loopMode_t loopMode, const unsigned nbSeconds) { U32 i; @@ -736,11 +879,6 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t ZSTD_CCtx* cctx = ctx.cctx; ZSTD_DCtx* dctx = ctx.dctx; - BMK_advancedParams_t adv = BMK_initAdvancedParams(); - adv.mode = mode; - adv.loopMode = loopMode; - adv.nbSeconds = nbSeconds; - /* warmimg up memory */ /* can't do this if decode only */ for(i = 0; i < buf.nbBlocks; i++) { @@ -762,7 +900,6 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t cctxprep.dictBufferSize = dictBufferSize; cctxprep.cLevel = cLevel; cctxprep.comprParams = comprParams; - cctxprep.adv = &adv; dctxprep.dctx = dctx; dctxprep.dictBuffer = dictBuffer; dctxprep.dictBufferSize = dictBufferSize; @@ -852,13 +989,13 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t } } /* Bench */ - results.result.cMem = (1 << (comprParams->windowLog)) + ZSTD_sizeof_CCtx(cctx); + results.result.cMem = (1 << (comprParams->vals[wlog_ind])) + ZSTD_sizeof_CCtx(cctx); return results; } static int BMK_benchParam(BMK_result_t* resultPtr, buffers_t buf, contexts_t ctx, - const ZSTD_compressionParameters cParams) { + const paramValues_t cParams) { BMK_return_t res = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_both, BMK_timeMode, 3); *resultPtr = res.result; return res.error; @@ -986,23 +1123,21 @@ static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) { /* Writes to f the results of a parameter benchmark */ /* when used with --optimize, will only print results better than previously discovered */ -static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const size_t srcSize) +static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const paramValues_t params, const size_t srcSize) { char lvlstr[15] = "Custom Level"; + winnerInfo_t w; + w.params = params; + w.result = result; fprintf(f, "\r%79s\r", ""); - fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", - params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, - params.targetLength, g_stratName[(U32)(params.strategy)]); - if(cLevel != CUSTOM_LEVEL) { snprintf(lvlstr, 15, " Level %2u ", cLevel); } - fprintf(f, - "/* %s */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */", - lvlstr, (double)srcSize / result.cSize, (double)result.cSpeed / (1 MB), (double)result.dSpeed / (1 MB)); + fprintf(f, "/* %s */ ", lvlstr); + BMK_displayOneResult(f, w, srcSize); if(TIMED) { const U64 time = UTIL_clockSpanNano(g_time); @@ -1012,11 +1147,10 @@ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result fprintf(f, "\n"); } -static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const constraint_t targetConstraints, const size_t srcSize) +static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t result, const paramValues_t params, const constraint_t targetConstraints, const size_t srcSize) { /* global winner used for constraints */ - static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; - + static winnerInfo_t g_winner = { { (size_t)-1LL, 0, 0, (size_t)-1LL }, { { PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET } } }; if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { if(DEBUG && compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { DISPLAY("New Winner: \n"); @@ -1025,7 +1159,7 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res BMK_printWinner(f, cLevel, result, params, srcSize); if(compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { - BMK_translateAdvancedParams(params); + BMK_translateAdvancedParams(f, params); g_winner.result = result; g_winner.params = params; } @@ -1046,13 +1180,7 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res fprintf(f, "================================\n"); for(n = g_winners; n != NULL; n = n->next) { fprintf(f, "\r%79s\r", ""); - - fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", - n->res.params.windowLog, n->res.params.chainLog, n->res.params.hashLog, n->res.params.searchLog, n->res.params.searchLength, - n->res.params.targetLength, g_stratName[(U32)(n->res.params.strategy)]); - fprintf(f, - " /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", - (double)srcSize / n->res.result.cSize, (double)n->res.result.cSpeed / (1 MB), (double)n->res.result.dSpeed / (1 MB)); + BMK_displayOneResult(f, n->res, srcSize); } fprintf(f, "================================\n"); fprintf(f, "Level Bounds: R: > %.3f AND C: < %.1f MB/s \n\n", @@ -1060,27 +1188,15 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res fprintf(f, "Overall Winner: \n"); - fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", - g_winner.params.windowLog, g_winner.params.chainLog, g_winner.params.hashLog, g_winner.params.searchLog, g_winner.params.searchLength, - g_winner.params.targetLength, g_stratName[(U32)(g_winner.params.strategy)]); - fprintf(f, - " /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", - (double)srcSize / g_winner.result.cSize, (double)g_winner.result.cSpeed / (1 MB), (double)g_winner.result.dSpeed / (1 MB)); - - BMK_translateAdvancedParams(g_winner.params); - - fprintf(f, "Latest BMK: \n"); - fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", - params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, - params.targetLength, g_stratName[(U32)(params.strategy)]); - fprintf(f, - " /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", - (double)srcSize / result.cSize, (double)result.cSpeed / (1 MB), (double)result.dSpeed / (1 MB)); + BMK_displayOneResult(f, g_winner, srcSize); + BMK_translateAdvancedParams(f, g_winner.params); + fprintf(f, "Latest BMK: \n");\ + BMK_displayOneResult(f, w, srcSize); } } -static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize) +static void BMK_printWinners2(FILE* f, const oldWinnerInfo_t* winners, size_t srcSize) { int cLevel; @@ -1088,11 +1204,11 @@ static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSi fprintf(f, " /* W, C, H, S, L, T, strat */ \n"); for (cLevel=0; cLevel <= NB_LEVELS_TRACKED; cLevel++) - BMK_printWinner(f, cLevel, winners[cLevel].result, winners[cLevel].params, srcSize); + BMK_printWinner(f, cLevel, winners[cLevel].result, cParamsToPVals(winners[cLevel].params), srcSize); } -static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, size_t srcSize) +static void BMK_printWinners(FILE* f, const oldWinnerInfo_t* winners, size_t srcSize) { fseek(f, 0, SEEK_SET); BMK_printWinners2(f, winners, srcSize); @@ -1129,14 +1245,14 @@ static void BMK_init_level_constraints(int bytePerSec_level1) } } } -static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters params, +static int BMK_seed(oldWinnerInfo_t* winners, const ZSTD_compressionParameters params, buffers_t buf, contexts_t ctx) { BMK_result_t testResult; int better = 0; int cLevel; - BMK_benchParam(&testResult, buf, ctx, params); + BMK_benchParam(&testResult, buf, ctx, cParamsToPVals(params)); for (cLevel = 1; cLevel <= NB_LEVELS_TRACKED; cLevel++) { @@ -1152,7 +1268,7 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para /* first solution for this cLevel */ winners[cLevel].result = testResult; winners[cLevel].params = params; - BMK_printWinner(stdout, cLevel, testResult, params, buf.srcSize); + BMK_printWinner(stdout, cLevel, testResult, cParamsToPVals(params), buf.srcSize); better = 1; continue; } @@ -1217,7 +1333,7 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para winners[cLevel].result = testResult; winners[cLevel].params = params; - BMK_printWinner(stdout, cLevel, testResult, params, buf.srcSize); + BMK_printWinner(stdout, cLevel, testResult, cParamsToPVals(params), buf.srcSize); better = 1; } } @@ -1234,20 +1350,21 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para /* no point in windowLog < chainLog (no point 2x chainLog for bt) */ /* now with built in bounds-checking */ /* no longer does anything with sanitizeVarArray + clampcheck */ -static ZSTD_compressionParameters sanitizeParams(ZSTD_compressionParameters params) +static paramValues_t sanitizeParams(paramValues_t params) { - if (params.strategy == ZSTD_fast) - params.chainLog = 0, params.searchLog = 0; - if (params.strategy == ZSTD_dfast) - params.searchLog = 0; - if (params.strategy != ZSTD_btopt && params.strategy != ZSTD_btultra && params.strategy != ZSTD_fast) - params.targetLength = 0; + if (params.vals[strt_ind] == ZSTD_fast) + params.vals[clog_ind] = 0, params.vals[slog_ind] = 0; + if (params.vals[strt_ind] == ZSTD_dfast) + params.vals[slog_ind] = 0; + if (params.vals[strt_ind] != ZSTD_btopt && params.vals[strt_ind] != ZSTD_btultra && params.vals[strt_ind] != ZSTD_fast) + params.vals[tlen_ind] = 0; return params; } -/* new length */ +/* return: new length */ /* keep old array, will need if iter over strategy. */ +/* prunes useless params */ static int sanitizeVarArray(varInds_t* varNew, const int varLength, const varInds_t* varArray, const ZSTD_strategy strat) { int i, j = 0; for(i = 0; i < varLength; i++) { @@ -1264,80 +1381,31 @@ static int sanitizeVarArray(varInds_t* varNew, const int varLength, const varInd } /* res should be NUM_PARAMS size */ -/* constructs varArray from ZSTD_compressionParameters style parameter */ -static int variableParams(const ZSTD_compressionParameters paramConstraints, varInds_t* res) { +/* constructs varArray from paramValues_t style parameter */ +/* pass in using dict. */ +static int variableParams(const paramValues_t paramConstraints, varInds_t* res, const int usingDictionary) { + varInds_t i; int j = 0; - if(!paramConstraints.windowLog) { - res[j] = wlog_ind; - j++; - } - if(!paramConstraints.chainLog) { - res[j] = clog_ind; - j++; - } - if(!paramConstraints.hashLog) { - res[j] = hlog_ind; - j++; - } - if(!paramConstraints.searchLog) { - res[j] = slog_ind; - j++; - } - if(!paramConstraints.searchLength) { - res[j] = slen_ind; - j++; - } - if(!paramConstraints.targetLength) { - res[j] = tlen_ind; - j++; - } - if(!paramConstraints.strategy) { - res[j] = strt_ind; - j++; - } - return j; -} - -/* bin-search on tlen_table for correct index. */ -static int tlen_inv(U32 x) { - int lo = 0; - int hi = TLEN_RANGE; - while(lo < hi) { - int mid = (lo + hi) / 2; - if(tlen_table[mid] < x) { - lo = mid + 1; - } if(tlen_table[mid] == x) { - return mid; - } else { - hi = mid; + for(i = 0; i < NUM_PARAMS; i++) { + if(paramConstraints.vals[i] == PARAM_UNSET) { + if(i == fadt_ind && !usingDictionary) continue; /* don't use fadt if no dictionary */ + res[j] = i; j++; } } - return lo; + return j; } /* amt will probably always be \pm 1? */ /* slight change from old paramVariation, targetLength can only take on powers of 2 now (999 ~= 1024?) */ /* take max/min bounds into account as well? */ -static void paramVaryOnce(const varInds_t paramIndex, const int amt, ZSTD_compressionParameters* ptr) { - switch(paramIndex) - { - case wlog_ind: ptr->windowLog += amt; break; - case clog_ind: ptr->chainLog += amt; break; - case hlog_ind: ptr->hashLog += amt; break; - case slog_ind: ptr->searchLog += amt; break; - case slen_ind: ptr->searchLength += amt; break; - case tlen_ind: - ptr->targetLength = tlen_table[MAX(0, MIN(TLEN_RANGE - 1, tlen_inv(ptr->targetLength) + amt))]; - break; - case strt_ind: ptr->strategy += amt; break; - default: break; - } +static void paramVaryOnce(const varInds_t paramIndex, const int amt, paramValues_t* ptr) { + ptr->vals[paramIndex] = rangeMap(paramIndex, invRangeMap(paramIndex, ptr->vals[paramIndex]) + amt); //TODO: bounds check. } /* varies ptr by nbChanges respecting varyParams*/ -static void paramVariation(ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen, const U32 nbChanges) +static void paramVariation(paramValues_t* ptr, const varInds_t* varyParams, const int varyLen, const U32 nbChanges) { - ZSTD_compressionParameters p; + paramValues_t p; U32 validated = 0; while (!validated) { U32 i; @@ -1346,7 +1414,7 @@ static void paramVariation(ZSTD_compressionParameters* ptr, const varInds_t* var const U32 changeID = FUZ_rand(&g_rand) % (varyLen << 1); paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p); } - validated = !ZSTD_isError(ZSTD_checkCParams(p)) && p.strategy > 0; + validated = paramValid(p); } *ptr = p; } @@ -1364,48 +1432,25 @@ static size_t memoTableLen(const varInds_t* varyParams, const int varyLen) { } /* returns unique index in memotable of compression parameters */ -static unsigned memoTableInd(const ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen) { +static unsigned memoTableInd(const paramValues_t* ptr, const varInds_t* varyParams, const int varyLen) { int i; unsigned ind = 0; for(i = 0; i < varyLen; i++) { - switch(varyParams[i]) { - case wlog_ind: ind *= WLOG_RANGE; ind += ptr->windowLog - - ZSTD_WINDOWLOG_MIN ; break; - case clog_ind: ind *= CLOG_RANGE; ind += ptr->chainLog - - ZSTD_CHAINLOG_MIN ; break; - case hlog_ind: ind *= HLOG_RANGE; ind += ptr->hashLog - - ZSTD_HASHLOG_MIN ; break; - case slog_ind: ind *= SLOG_RANGE; ind += ptr->searchLog - - ZSTD_SEARCHLOG_MIN ; break; - case slen_ind: ind *= SLEN_RANGE; ind += ptr->searchLength - - ZSTD_SEARCHLENGTH_MIN; break; - case tlen_ind: ind *= TLEN_RANGE; ind += tlen_inv(ptr->targetLength) - - ZSTD_TARGETLENGTH_MIN; break; - case strt_ind: break; - } + varInds_t v = varyParams[i]; + if(v == strt_ind) continue; /* exclude strategy from memotable */ + ind *= rangetable[v]; ind += invRangeMap(v, ptr->vals[v]); } return ind; } /* inverse of above function (from index to parameters) */ -static void memoTableIndInv(ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen, size_t ind) { +static void memoTableIndInv(paramValues_t* ptr, const varInds_t* varyParams, const int varyLen, size_t ind) { int i; for(i = varyLen - 1; i >= 0; i--) { - switch(varyParams[i]) { - case wlog_ind: ptr->windowLog = ind % WLOG_RANGE + ZSTD_WINDOWLOG_MIN; - ind /= WLOG_RANGE; break; - case clog_ind: ptr->chainLog = ind % CLOG_RANGE + ZSTD_CHAINLOG_MIN; - ind /= CLOG_RANGE; break; - case hlog_ind: ptr->hashLog = ind % HLOG_RANGE + ZSTD_HASHLOG_MIN; - ind /= HLOG_RANGE; break; - case slog_ind: ptr->searchLog = ind % SLOG_RANGE + ZSTD_SEARCHLOG_MIN; - ind /= SLOG_RANGE; break; - case slen_ind: ptr->searchLength = ind % SLEN_RANGE + ZSTD_SEARCHLENGTH_MIN; - ind /= SLEN_RANGE; break; - case tlen_ind: ptr->targetLength = tlen_table[(ind % TLEN_RANGE)]; - ind /= TLEN_RANGE; break; - case strt_ind: break; - } + varInds_t v = varyParams[i]; + if(v == strt_ind) continue; + ptr->vals[v] = rangeMap(v, ind % rangetable[v]); + ind /= rangetable[v]; } } @@ -1414,36 +1459,36 @@ static void memoTableIndInv(ZSTD_compressionParameters* ptr, const varInds_t* va * redundant / obviously non-optimal parameter configurations (e.g. wlog - 1 larger) * than srcSize, clog > wlog, ... */ -static void initMemoTable(U8* memoTable, ZSTD_compressionParameters paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { +static void initMemoTable(U8* memoTable, paramValues_t paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { size_t i; size_t arrayLen = memoTableLen(varyParams, varyLen); - int cwFixed = !paramConstraints.chainLog || !paramConstraints.windowLog; - int scFixed = !paramConstraints.searchLog || !paramConstraints.chainLog; - int whFixed = !paramConstraints.windowLog || !paramConstraints.hashLog; - int wFixed = !paramConstraints.windowLog; + int cwFixed = paramConstraints.vals[clog_ind] == PARAM_UNSET || paramConstraints.vals[wlog_ind] == PARAM_UNSET; + int scFixed = paramConstraints.vals[slog_ind] == PARAM_UNSET || paramConstraints.vals[clog_ind] == PARAM_UNSET; + int whFixed = paramConstraints.vals[wlog_ind] == PARAM_UNSET || paramConstraints.vals[hlog_ind] == PARAM_UNSET; + int wFixed = paramConstraints.vals[wlog_ind] == PARAM_UNSET; int j = 0; assert(memoTable != NULL); memset(memoTable, 0, arrayLen); - cParamZeroMin(¶mConstraints); + paramConstraints = cParamUnsetMin(paramConstraints); for(i = 0; i < arrayLen; i++) { memoTableIndInv(¶mConstraints, varyParams, varyLen, i); - if(ZSTD_estimateCStreamSize_usingCParams(paramConstraints) > (size_t)target.cMem) { + if(ZSTD_estimateCStreamSize_usingCParams(pvalsToCParams(paramConstraints)) > (size_t)target.cMem) { memoTable[i] = 255; j++; } - if(wFixed && (1ULL << (paramConstraints.windowLog - 1)) > srcSize) { + if(wFixed && (1ULL << (paramConstraints.vals[wlog_ind] - 1)) >= srcSize && paramConstraints.vals[wlog_ind] != mintable[wlog_ind]) { memoTable[i] = 255; } /* nil out parameter sets equivalent to others. */ - if(cwFixed/* at most least 1 param fixed. */) { - if(paramConstraints.strategy == ZSTD_btlazy2 || paramConstraints.strategy == ZSTD_btopt || paramConstraints.strategy == ZSTD_btultra) { - if(paramConstraints.chainLog > paramConstraints.windowLog + 1) { + if(cwFixed) { + if(paramConstraints.vals[strt_ind] == ZSTD_btlazy2 || paramConstraints.vals[strt_ind] == ZSTD_btopt || paramConstraints.vals[strt_ind] == ZSTD_btultra) { + if(paramConstraints.vals[clog_ind] > paramConstraints.vals[wlog_ind]+ 1) { if(memoTable[i] != 255) { j++; } memoTable[i] = 255; } } else { - if(paramConstraints.chainLog > paramConstraints.windowLog) { + if(paramConstraints.vals[clog_ind] > paramConstraints.vals[wlog_ind]) { if(memoTable[i] != 255) { j++; } memoTable[i] = 255; } @@ -1451,14 +1496,14 @@ static void initMemoTable(U8* memoTable, ZSTD_compressionParameters paramConstra } if(scFixed) { - if(paramConstraints.searchLog > paramConstraints.chainLog) { + if(paramConstraints.vals[slog_ind] > paramConstraints.vals[clog_ind]) { if(memoTable[i] != 255) { j++; } memoTable[i] = 255; } } if(whFixed) { - if(paramConstraints.hashLog > paramConstraints.windowLog + 1) { + if(paramConstraints.vals[hlog_ind] > paramConstraints.vals[wlog_ind] + 1) { if(memoTable[i] != 255) { j++; } memoTable[i] = 255; } @@ -1466,7 +1511,7 @@ static void initMemoTable(U8* memoTable, ZSTD_compressionParameters paramConstra } DEBUGOUTPUT("%d / %d Invalid\n", j, (int)i); if((int)i == j) { - DEBUGOUTPUT("!!!Strategy %d totally infeasible\n", (int)paramConstraints.strategy) + DEBUGOUTPUT("!!!Strategy %d totally infeasible\n", (int)paramConstraints.vals[strt_ind]); } } @@ -1482,7 +1527,7 @@ static void freeMemoTableArray(U8** mtAll) { /* inits memotables for all (including mallocs), all strategies */ /* takes unsanitized varyParams */ -static U8** createMemoTableArray(ZSTD_compressionParameters paramConstraints, constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { +static U8** createMemoTableArray(paramValues_t paramConstraints, constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { varInds_t varNew[NUM_PARAMS]; U8** mtAll = (U8**)calloc(sizeof(U8*),(ZSTD_btultra + 1)); int i; @@ -1503,14 +1548,13 @@ static U8** createMemoTableArray(ZSTD_compressionParameters paramConstraints, co return mtAll; } -static ZSTD_compressionParameters overwriteParams(ZSTD_compressionParameters base, ZSTD_compressionParameters mask) { - base.windowLog = mask.windowLog ? mask.windowLog : base.windowLog; - base.chainLog = mask.chainLog ? mask.chainLog : base.chainLog; - base.hashLog = mask.hashLog ? mask.hashLog : base.hashLog; - base.searchLog = mask.searchLog ? mask.searchLog : base.searchLog; - base.searchLength = mask.searchLength ? mask.searchLength : base.searchLength; - base.targetLength = mask.targetLength ? mask.targetLength : base.targetLength; - base.strategy = mask.strategy ? mask.strategy : base.strategy; +static paramValues_t overwriteParams(paramValues_t base, paramValues_t mask) { + U32 i; + for(i = 0; i < NUM_PARAMS; i++) { + if(mask.vals[i] != PARAM_UNSET) { + base.vals[i] = mask.vals[i]; + } + } return base; } @@ -1524,38 +1568,41 @@ static BYTE g_alreadyTested[PARAMTABLESIZE] = {0}; /* init to zero */ g_alreadyTested[(XXH64(((void*)&sanitizeParams(p), sizeof(p), 0) >> 3) & PARAMTABLEMASK] */ static BYTE* NB_TESTS_PLAYED(ZSTD_compressionParameters p) { - ZSTD_compressionParameters p2 = sanitizeParams(p); + ZSTD_compressionParameters p2 = pvalsToCParams(sanitizeParams(cParamsToPVals(p))); return &g_alreadyTested[(XXH64((void*)&p2, sizeof(p2), 0) >> 3) & PARAMTABLEMASK]; } -static void playAround(FILE* f, winnerInfo_t* winners, +static void playAround(FILE* f, oldWinnerInfo_t* winners, ZSTD_compressionParameters params, buffers_t buf, contexts_t ctx) { int nbVariations = 0; UTIL_time_t const clockStart = UTIL_getTime(); - const U32 unconstrained[NUM_PARAMS] = { 0, 1, 2, 3, 4, 5, 6 }; + const U32 unconstrained[NUM_PARAMS] = { 0, 1, 2, 3, 4, 5, 6 }; /* no fadt */ while (UTIL_clockSpanMicro(clockStart) < g_maxVariationTime) { - ZSTD_compressionParameters p = params; + paramValues_t p = cParamsToPVals(params); + ZSTD_compressionParameters p2; BYTE* b; if (nbVariations++ > g_maxNbVariations) break; paramVariation(&p, unconstrained, NUM_PARAMS, 4); + p2 = pvalsToCParams(p); + /* exclude faster if already played params */ - if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(p))-1)) + if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(p2))-1)) continue; /* test */ - b = NB_TESTS_PLAYED(p); + b = NB_TESTS_PLAYED(p2); (*b)++; - if (!BMK_seed(winners, p, buf, ctx)) continue; + if (!BMK_seed(winners, p2, buf, ctx)) continue; /* improvement found => search more */ BMK_printWinners(f, winners, buf.srcSize); - playAround(f, winners, p, buf, ctx); + playAround(f, winners, p2, buf, ctx); } } @@ -1587,7 +1634,7 @@ static ZSTD_compressionParameters randomParams(void) } /* Sets pc to random unmeasured set of parameters */ -static void randomConstrainedParams(ZSTD_compressionParameters* pc, varInds_t* varArray, int varLen, U8* memoTable) +static void randomConstrainedParams(paramValues_t* pc, varInds_t* varArray, int varLen, U8* memoTable) { size_t tries = memoTableLen(varArray, varLen); const size_t maxSize = memoTableLen(varArray, varLen); @@ -1601,7 +1648,7 @@ static void randomConstrainedParams(ZSTD_compressionParameters* pc, varInds_t* v } static void BMK_selectRandomStart( - FILE* f, winnerInfo_t* winners, + FILE* f, oldWinnerInfo_t* winners, buffers_t buf, contexts_t ctx) { U32 const id = FUZ_rand(&g_rand) % (NB_LEVELS_TRACKED+1); @@ -1617,7 +1664,7 @@ static void BMK_selectRandomStart( static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBlockSize) { ZSTD_compressionParameters params; - winnerInfo_t winners[NB_LEVELS_TRACKED+1]; + oldWinnerInfo_t winners[NB_LEVELS_TRACKED+1]; const char* const rfName = "grillResults.txt"; FILE* const f = fopen(rfName, "w"); @@ -1632,7 +1679,7 @@ static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBl /* baseline config for level 1 */ ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, maxBlockSize, ctx.dictSize); BMK_result_t testResult; - BMK_benchParam(&testResult, buf, ctx, l1params); + BMK_benchParam(&testResult, buf, ctx, cParamsToPVals(l1params)); BMK_init_level_constraints((int)((testResult.cSpeed * 31) / 32)); } @@ -1751,7 +1798,7 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam DISPLAY("using %d Files : \n", nbFiles); } - g_params = ZSTD_adjustCParams(overwriteParams(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize), g_params), maxBlockSize, ctx.dictSize); + g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize)), g_params), maxBlockSize, ctx.dictSize); if(g_singleRun) { ret = benchOnce(buf, ctx); @@ -1769,7 +1816,7 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam #define HIGH_VARIANCE 100.0 static int allBench(BMK_result_t* resultPtr, const buffers_t buf, const contexts_t ctx, - const ZSTD_compressionParameters cParams, + const paramValues_t cParams, const constraint_t target, BMK_result_t* winnerResult, int feas) { BMK_return_t benchres; @@ -1870,7 +1917,7 @@ static int allBench(BMK_result_t* resultPtr, /* Memoized benchmarking, won't benchmark anything which has already been benchmarked before. */ static int benchMemo(BMK_result_t* resultPtr, const buffers_t buf, const contexts_t ctx, - const ZSTD_compressionParameters cParams, + const paramValues_t cParams, const constraint_t target, BMK_result_t* winnerResult, U8* const memoTable, const varInds_t* varyParams, const int varyLen, const int feas) { @@ -1914,13 +1961,13 @@ static winnerInfo_t climbOnce(const constraint_t target, const varInds_t* varArray, const int varLen, ZSTD_strategy strat, U8** memoTableArray, buffers_t buf, contexts_t ctx, - const ZSTD_compressionParameters init) { + const paramValues_t init) { /* * cparam - currently considered 'center' * candidate - params to benchmark/results * winner - best option found so far. */ - ZSTD_compressionParameters cparam = init; + paramValues_t cparam = init; winnerInfo_t candidateInfo, winnerInfo; int better = 1; int feas = 0; @@ -1948,10 +1995,10 @@ static winnerInfo_t climbOnce(const constraint_t target, candidateInfo.params = cparam; paramVaryOnce(varArray[i], offset, &candidateInfo.params); - if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params)) && candidateInfo.params.strategy > 0) { + if(paramValid(candidateInfo.params)) { int res; - if(strat != candidateInfo.params.strategy) { /* maybe only try strategy switching after exhausting non-switching solutions? */ - strat = candidateInfo.params.strategy; + if(strat != candidateInfo.params.vals[strt_ind]) { /* maybe only try strategy switching after exhausting non-switching solutions? */ + strat = candidateInfo.params.vals[strt_ind]; varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat); } res = benchMemo(&candidateInfo.result, buf, ctx, @@ -1981,8 +2028,8 @@ static winnerInfo_t climbOnce(const constraint_t target, /* param error checking already done here */ paramVariation(&candidateInfo.params, varArray, varLen, dist); - if(strat != candidateInfo.params.strategy) { - strat = candidateInfo.params.strategy; + if(strat != candidateInfo.params.vals[strt_ind]) { + strat = candidateInfo.params.vals[strt_ind]; varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat); } @@ -2030,7 +2077,7 @@ static winnerInfo_t climbOnce(const constraint_t target, */ static winnerInfo_t optimizeFixedStrategy( const buffers_t buf, const contexts_t ctx, - const constraint_t target, ZSTD_compressionParameters paramTarget, + const constraint_t target, paramValues_t paramTarget, const ZSTD_strategy strat, const varInds_t* varArray, const int varLen, U8** memoTableArray, const int tries) { @@ -2038,14 +2085,14 @@ static winnerInfo_t optimizeFixedStrategy( varInds_t varNew[NUM_PARAMS]; int varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat); - ZSTD_compressionParameters init; + paramValues_t init; winnerInfo_t winnerInfo, candidateInfo; winnerInfo = initWinnerInfo(emptyParams()); /* so climb is given the right fixed strategy */ - paramTarget.strategy = strat; + paramTarget.vals[strt_ind] = strat; /* to pass ZSTD_checkCParams */ - cParamZeroMin(¶mTarget); + paramTarget = cParamUnsetMin(paramTarget); init = paramTarget; @@ -2112,25 +2159,23 @@ static int nextStrategy(const int currentStrategy, const int bestStrategy) { static int g_maxTries = 3; #define TRY_DECAY 1 -static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel) +static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, paramValues_t paramTarget, int cLevelOpt, int cLevelRun) { varInds_t varArray [NUM_PARAMS]; int ret = 0; - const int varLen = variableParams(paramTarget, varArray); + const int varLen = variableParams(paramTarget, varArray, dictFileName != NULL); winnerInfo_t winner = initWinnerInfo(emptyParams()); U8** allMT = NULL; - size_t k; - size_t maxBlockSize = 0; + paramValues_t paramBase = cParamUnsetMin(paramTarget); + size_t k, maxBlockSize = 0; contexts_t ctx; buffers_t buf; g_time = UTIL_getTime(); - /* Init */ - if(!cParamValid(paramTarget)) { + if(!paramValid(paramBase)) { return 1; } - /* load dictionary*/ if(createBuffers(&buf, fileNamesTable, nbFiles)) { DISPLAY("unable to load files\n"); return 1; @@ -2154,23 +2199,23 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } /* if strategy is fixed, only init that part of memotable */ - if(paramTarget.strategy) { + if(paramTarget.vals[strt_ind] != PARAM_UNSET) { varInds_t varNew[NUM_PARAMS]; - int varLenNew = sanitizeVarArray(varNew, varLen, varArray, paramTarget.strategy); + int varLenNew = sanitizeVarArray(varNew, varLen, varArray, paramTarget.vals[strt_ind]); allMT = (U8**)calloc(sizeof(U8*), (ZSTD_btultra + 1)); if(allMT == NULL) { ret = 57; goto _cleanUp; } - allMT[paramTarget.strategy] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew)); + allMT[paramTarget.vals[strt_ind]] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew)); - if(allMT[paramTarget.strategy] == NULL) { + if(allMT[paramTarget.vals[strt_ind]] == NULL) { ret = 58; goto _cleanUp; } - initMemoTable(allMT[paramTarget.strategy], paramTarget, target, varNew, varLenNew, maxBlockSize); + initMemoTable(allMT[paramTarget.vals[strt_ind]], paramTarget, target, varNew, varLenNew, maxBlockSize); } else { allMT = createMemoTableArray(paramTarget, target, varArray, varLen, maxBlockSize); } @@ -2180,7 +2225,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ ret = 2; goto _cleanUp; } - + /* default strictness = Maximum for */ if(g_strictness == DEFAULT_STRICTNESS) { if(g_optmode) { @@ -2199,7 +2244,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ /* use level'ing mode instead of normal target mode */ /* Should lvl be parameter-masked here? */ if(g_optmode) { - winner.params = ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize); + winner.params = cParamsToPVals(ZSTD_getCParams(cLevelOpt, maxBlockSize, ctx.dictSize)); if(BMK_benchParam(&winner.result, buf, ctx, winner.params)) { ret = 3; goto _cleanUp; @@ -2213,13 +2258,13 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ target.cSpeed = (U32)g_lvltarget.cSpeed; target.dSpeed = (U32)g_lvltarget.dSpeed; //See if this is reasonable. - BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize); + BMK_printWinnerOpt(stdout, cLevelOpt, winner.result, winner.params, target, buf.srcSize); } /* Don't want it to return anything worse than the best known result */ if(g_singleRun) { BMK_result_t res; - g_params = ZSTD_adjustCParams(maskParams(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize), g_params), maxBlockSize, ctx.dictSize); + g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevelRun, maxBlockSize, ctx.dictSize)), g_params), maxBlockSize, ctx.dictSize); if(BMK_benchParam(&res, buf, ctx, g_params)) { ret = 45; goto _cleanUp; @@ -2246,19 +2291,19 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ findClockGranularity(); { - ZSTD_compressionParameters CParams; + paramValues_t CParams; /* find best solution from default params */ { /* strategy selection */ const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); DEBUGOUTPUT("Strategy Selection\n"); - if(paramTarget.strategy == 0) { + if(paramTarget.vals[strt_ind] == PARAM_UNSET) { BMK_result_t candidate; int i; for (i=1; i<=maxSeeds; i++) { int ec; - CParams = overwriteParams(ZSTD_getCParams(i, maxBlockSize, ctx.dictSize), paramTarget); + CParams = overwriteParams(cParamsToPVals(ZSTD_getCParams(i, maxBlockSize, ctx.dictSize)), paramTarget); ec = BMK_benchParam(&candidate, buf, ctx, CParams); BMK_printWinnerOpt(stdout, i, candidate, CParams, target, buf.srcSize); @@ -2271,17 +2316,18 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ /* if the current params are too slow, just stop. */ if(target.cSpeed > candidate.cSpeed * 3 / 2) { break; } } + + BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winner.result, winner.params, target, buf.srcSize); + BMK_translateAdvancedParams(stdout, winner.params); } } - BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winner.result, winner.params, target, buf.srcSize); - BMK_translateAdvancedParams(winner.params); DEBUGOUTPUT("Real Opt\n"); /* start 'real' tests */ { - int bestStrategy = (int)winner.params.strategy; - if(paramTarget.strategy == 0) { - int st = (int)winner.params.strategy; + int bestStrategy = (int)winner.params.vals[strt_ind]; + if(paramTarget.vals[strt_ind] == PARAM_UNSET) { + int st = bestStrategy; int tries = g_maxTries; { @@ -2298,7 +2344,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ winnerInfo_t wc; DEBUGOUTPUT("StrategySwitch: %s\n", g_stratName[st]); - wc = optimizeFixedStrategy(buf, ctx, target, paramTarget, + wc = optimizeFixedStrategy(buf, ctx, target, paramBase, st, varArray, varLen, allMT, tries); if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) { @@ -2312,7 +2358,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ CHECKTIMEGT(ret, 0, _cleanUp); } } else { - winner = optimizeFixedStrategy(buf, ctx, target, paramTarget, paramTarget.strategy, + winner = optimizeFixedStrategy(buf, ctx, target, paramBase, paramTarget.vals[strt_ind], varArray, varLen, allMT, g_maxTries); } @@ -2326,7 +2372,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } /* end summary */ BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winner.result, winner.params, target, buf.srcSize); - BMK_translateAdvancedParams(winner.params); + BMK_translateAdvancedParams(stdout, winner.params); DISPLAY("grillParams size - optimizer completed \n"); } @@ -2350,7 +2396,9 @@ static void errorOut(const char* msg) static unsigned readU32FromChar(const char** stringPtr) { const char errorMsg[] = "error: numeric value too large"; + unsigned sign = 1; unsigned result = 0; + if(**stringPtr == '-') { sign = (unsigned)-1; (*stringPtr)++; } while ((**stringPtr >='0') && (**stringPtr <='9')) { unsigned const max = (((unsigned)(-1)) / 10) - 1; if (result > max) errorOut(errorMsg); @@ -2368,7 +2416,7 @@ static unsigned readU32FromChar(const char** stringPtr) if (**stringPtr=='i') (*stringPtr)++; if (**stringPtr=='B') (*stringPtr)++; } - return result; + return result * sign; } static int usage(const char* exename) @@ -2408,13 +2456,14 @@ static int badusage(const char* exename) #define PARSE_SUB_ARGS(stringLong, stringShort, variable) { if (longCommandWArg(&argument, stringLong) || longCommandWArg(&argument, stringShort)) { variable = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } } #define PARSE_CPARAMS(variable) \ { \ - PARSE_SUB_ARGS("windowLog=", "wlog=", variable.windowLog); \ - PARSE_SUB_ARGS("chainLog=" , "clog=", variable.chainLog); \ - PARSE_SUB_ARGS("hashLog=", "hlog=", variable.hashLog); \ - PARSE_SUB_ARGS("searchLog=" , "slog=", variable.searchLog); \ - PARSE_SUB_ARGS("searchLength=", "slen=", variable.searchLength); \ - PARSE_SUB_ARGS("targetLength=" , "tlen=", variable.targetLength); \ - PARSE_SUB_ARGS("strategy=", "strat=", variable.strategy); \ + PARSE_SUB_ARGS("windowLog=", "wlog=", variable.vals[wlog_ind]); \ + PARSE_SUB_ARGS("chainLog=" , "clog=", variable.vals[clog_ind]); \ + PARSE_SUB_ARGS("hashLog=", "hlog=", variable.vals[hlog_ind]); \ + PARSE_SUB_ARGS("searchLog=" , "slog=", variable.vals[slog_ind]); \ + PARSE_SUB_ARGS("searchLength=", "slen=", variable.vals[slen_ind]); \ + PARSE_SUB_ARGS("targetLength=" , "tlen=", variable.vals[tlen_ind]); \ + PARSE_SUB_ARGS("strategy=", "strat=", variable.vals[strt_ind]); \ + PARSE_SUB_ARGS("forceAttachDict=", "fad=" , variable.vals[fadt_ind]); \ } int main(int argc, const char** argv) @@ -2426,12 +2475,11 @@ int main(int argc, const char** argv) const char* input_filename = NULL; const char* dictFileName = NULL; U32 main_pause = 0; - int cLevel = 0; + int cLevelOpt = 0, cLevelRun = 0; int seperateFiles = 0; - constraint_t target = { 0, 0, (U32)-1 }; - ZSTD_compressionParameters paramTarget = emptyParams(); + paramValues_t paramTarget = emptyParams(); g_params = emptyParams(); assert(argc>=1); /* for exename */ @@ -2441,9 +2489,7 @@ int main(int argc, const char** argv) for(i=1; i Date: Tue, 14 Aug 2018 14:44:47 -0700 Subject: [PATCH 40/55] Fix scan-build warnings in bench.c --- programs/bench.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 7662678ab..79ef42caa 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -597,15 +597,16 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( results.error = compressionResults.error; return results; } + if(compressionResults.result.nanoSecPerRun == 0) { results.result.cSpeed = 0; } else { results.result.cSpeed = srcSize * TIMELOOP_NANOSEC / compressionResults.result.nanoSecPerRun; } + results.result.cSize = compressionResults.result.sumOfReturn; { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - results.result.cSpeed = (srcSize * TIMELOOP_NANOSEC / compressionResults.result.nanoSecPerRun); cSize = compressionResults.result.sumOfReturn; results.result.cSize = cSize; ratio = (double)srcSize / results.result.cSize; @@ -626,6 +627,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( results.error = decompressionResults.error; return results; } + if(decompressionResults.result.nanoSecPerRun == 0) { results.result.dSpeed = 0; } else { @@ -634,7 +636,6 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - results.result.dSpeed = (srcSize * TIMELOOP_NANOSEC/ decompressionResults.result.nanoSecPerRun); markNb = (markNb+1) % NB_MARKS; DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r", marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize, @@ -737,14 +738,16 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, void* const internalDstBuffer = dstBuffer ? NULL : malloc(maxCompressedSize); void* const compressedBuffer = dstBuffer ? dstBuffer : internalDstBuffer; - void* resultBuffer = malloc(srcSize); - BMK_return_t results = { { 0, 0, 0, 0 }, 0 }; + + int parametersConflict = !dstBuffer ^ !dstCapacity; + + void* resultBuffer = srcSize ? malloc(srcSize) : NULL; + int allocationincomplete = !srcPtrs || !srcSizes || !cPtrs || !cSizes || !cCapacities || !resPtrs || !resSizes || !timeStateCompress || !timeStateDecompress || !compressedBuffer || !resultBuffer; - int parametersConflict = !dstBuffer ^ !dstCapacity; if (!allocationincomplete && !parametersConflict) { @@ -809,7 +812,7 @@ static size_t BMK_findMaxMem(U64 requiredMem) do { testmem = (BYTE*)malloc((size_t)requiredMem); requiredMem -= step; - } while (!testmem); + } while (!testmem && requiredMem > 0); free(testmem); return (size_t)(requiredMem); @@ -937,7 +940,8 @@ BMK_return_t BMK_benchFilesAdvanced(const char* const * const fileNamesTable, un if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad; if (benchedSize < totalSizeToLoad) DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20)); - srcBuffer = malloc(benchedSize); + + srcBuffer = benchedSize ? malloc(benchedSize) : NULL; if (!srcBuffer) { free(dictBuffer); free(fileSizes); From 96725989ef376b4fec697958837216f66d962406 Mon Sep 17 00:00:00 2001 From: George Lu Date: Thu, 9 Aug 2018 16:36:34 -0700 Subject: [PATCH 41/55] Temp fix perf regression --- tests/paramgrill.c | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index aab46ede5..4324678ad 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -185,9 +185,9 @@ typedef struct { U32 vals[NUM_PARAMS]; } paramValues_t; -//TODO: unset -> 0? static ZSTD_compressionParameters pvalsToCParams(paramValues_t p) { ZSTD_compressionParameters c; + memset(&c, 0, sizeof(ZSTD_compressionParameters)); c.windowLog = p.vals[wlog_ind]; c.chainLog = p.vals[clog_ind]; c.hashLog = p.vals[hlog_ind]; @@ -383,11 +383,9 @@ static int paramValid(paramValues_t paramTarget) { for(i = 0; i < NUM_PARAMS; i++) { CLAMPCHECK(paramTarget.vals[i], mintable[i], maxtable[i]); } - //TODO: Strategy could be valid at 0 before, is that right? return 1; } -//TODO: doesn't affect strategy? static paramValues_t cParamUnsetMin(paramValues_t paramTarget) { varInds_t i; for(i = 0; i < NUM_PARAMS; i++) { @@ -1399,7 +1397,7 @@ static int variableParams(const paramValues_t paramConstraints, varInds_t* res, /* slight change from old paramVariation, targetLength can only take on powers of 2 now (999 ~= 1024?) */ /* take max/min bounds into account as well? */ static void paramVaryOnce(const varInds_t paramIndex, const int amt, paramValues_t* ptr) { - ptr->vals[paramIndex] = rangeMap(paramIndex, invRangeMap(paramIndex, ptr->vals[paramIndex]) + amt); //TODO: bounds check. + ptr->vals[paramIndex] = rangeMap(paramIndex, invRangeMap(paramIndex, ptr->vals[paramIndex]) + amt); } /* varies ptr by nbChanges respecting varyParams*/ @@ -1447,10 +1445,23 @@ static unsigned memoTableInd(const paramValues_t* ptr, const varInds_t* varyPara static void memoTableIndInv(paramValues_t* ptr, const varInds_t* varyParams, const int varyLen, size_t ind) { int i; for(i = varyLen - 1; i >= 0; i--) { + /* This is cleaner/easier to generalize but slower varInds_t v = varyParams[i]; if(v == strt_ind) continue; ptr->vals[v] = rangeMap(v, ind % rangetable[v]); - ind /= rangetable[v]; + ind /= rangetable[v]; */ + + switch(varyParams[i]) { + case wlog_ind: ptr->vals[wlog_ind] = ind % rangetable[wlog_ind] + mintable[wlog_ind]; ind /= rangetable[wlog_ind]; break; + case clog_ind: ptr->vals[clog_ind] = ind % rangetable[clog_ind] + mintable[clog_ind]; ind /= rangetable[clog_ind]; break; + case hlog_ind: ptr->vals[hlog_ind] = ind % rangetable[hlog_ind] + mintable[hlog_ind]; ind /= rangetable[hlog_ind]; break; + case slog_ind: ptr->vals[slog_ind] = ind % rangetable[slog_ind] + mintable[slog_ind]; ind /= rangetable[slog_ind]; break; + case slen_ind: ptr->vals[slen_ind] = ind % rangetable[slen_ind] + mintable[slen_ind]; ind /= rangetable[slen_ind]; break; + case tlen_ind: ptr->vals[tlen_ind] = tlen_table[(ind % rangetable[tlen_ind])]; ind /= rangetable[tlen_ind]; break; + case fadt_ind: ptr->vals[fadt_ind] = ind % rangetable[fadt_ind] - 1; ind /= rangetable[fadt_ind]; break; + case strt_ind: + case NUM_PARAMS: break; + } } } @@ -1471,6 +1482,7 @@ static void initMemoTable(U8* memoTable, paramValues_t paramConstraints, const c memset(memoTable, 0, arrayLen); paramConstraints = cParamUnsetMin(paramConstraints); + for(i = 0; i < arrayLen; i++) { memoTableIndInv(¶mConstraints, varyParams, varyLen, i); if(ZSTD_estimateCStreamSize_usingCParams(pvalsToCParams(paramConstraints)) > (size_t)target.cMem) { From 8c918edd3a87a006200f704b9119434c5f3250f3 Mon Sep 17 00:00:00 2001 From: George Lu Date: Fri, 10 Aug 2018 16:14:12 -0700 Subject: [PATCH 42/55] MAke it easier to add params Make memoTable size limited --- tests/paramgrill.c | 499 ++++++++++++++++++++++++--------------------- 1 file changed, 272 insertions(+), 227 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 4324678ad..c2979cb83 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -86,7 +86,15 @@ static const int g_maxNbVariations = 64; #define CHECKTIME(r) { if(BMK_timeSpan(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); return r; } } #define CHECKTIMEGT(ret, val, _gototag) {if(BMK_timeSpan(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); ret = val; goto _gototag; } } -#define PARAM_UNSET ((U32)-2) /* can't be -1 b/c fadt */ +#define PARAM_UNSET ((U32)-2) /* can't be -1 b/c fadt uses -1 */ + +static const char* g_stratName[ZSTD_btultra+1] = { + "(none) ", "ZSTD_fast ", "ZSTD_dfast ", + "ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ", + "ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "}; + + +static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 }; /*-************************************ * Setup for Adding new params @@ -105,6 +113,14 @@ typedef enum { NUM_PARAMS = 8 } varInds_t; +typedef struct { + U32 vals[NUM_PARAMS]; +} paramValues_t; + +/* list of parameters */ +static const varInds_t paramTable[NUM_PARAMS] = + { wlog_ind, clog_ind, hlog_ind, slog_ind, slen_ind, tlen_ind, strt_ind, fadt_ind }; + /* maximum value of parameters */ static const U32 mintable[NUM_PARAMS] = { ZSTD_WINDOWLOG_MIN, ZSTD_CHAINLOG_MIN, ZSTD_HASHLOG_MIN, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLENGTH_MIN, ZSTD_TARGETLENGTH_MIN, ZSTD_fast, FADT_MIN }; @@ -117,15 +133,19 @@ static const U32 maxtable[NUM_PARAMS] = static const U32 rangetable[NUM_PARAMS] = { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE, STRT_RANGE, FADT_RANGE }; +/* ZSTD_cctxSetParameter() index to set */ static const ZSTD_cParameter cctxSetParamTable[NUM_PARAMS] = { ZSTD_p_windowLog, ZSTD_p_chainLog, ZSTD_p_hashLog, ZSTD_p_searchLog, ZSTD_p_minMatch, ZSTD_p_targetLength, ZSTD_p_compressionStrategy, ZSTD_p_forceAttachDict }; +/* names of parameters */ static const char* g_paramNames[NUM_PARAMS] = - { "windowLog", "chainLog", "hashLog","searchLog", "searchLength", "targetLength", "strategy", "forceAttachDict"}; + { "windowLog", "chainLog", "hashLog","searchLog", "searchLength", "targetLength", "strategy", "forceAttachDict" }; +/* shortened names of parameters */ +static const char* g_shortParamNames[NUM_PARAMS] = + { "wlog", "clog", "hlog","slog", "slen", "tlen", "strt", "fadt" }; -static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 }; -/* maps value from 0 to rangetable[param] - 1 to valid paramvalue */ +/* maps value from { 0 to rangetable[param] - 1 } to valid paramvalues */ static U32 rangeMap(varInds_t param, U32 ind) { ind = MIN(ind, rangetable[param] - 1); switch(param) { @@ -141,6 +161,7 @@ static U32 rangeMap(varInds_t param, U32 ind) { case strt_ind: return mintable[param] + ind; case NUM_PARAMS: + DISPLAY("Error, not a valid param\n "); return (U32)-1; } return 0; /* should never happen, stop compiler warnings */ @@ -176,49 +197,26 @@ static U32 invRangeMap(varInds_t param, U32 value) { case strt_ind: return value - mintable[param]; case NUM_PARAMS: + DISPLAY("Error, not a valid param\n "); return (U32)-1; } return 0; /* should never happen, stop compiler warnings */ } -typedef struct { - U32 vals[NUM_PARAMS]; -} paramValues_t; - -static ZSTD_compressionParameters pvalsToCParams(paramValues_t p) { - ZSTD_compressionParameters c; - memset(&c, 0, sizeof(ZSTD_compressionParameters)); - c.windowLog = p.vals[wlog_ind]; - c.chainLog = p.vals[clog_ind]; - c.hashLog = p.vals[hlog_ind]; - c.searchLog = p.vals[slog_ind]; - c.searchLength = p.vals[slen_ind]; - c.targetLength = p.vals[tlen_ind]; - c.strategy = p.vals[strt_ind]; - /* no forceAttachDict */ - return c; -} - -/* 0 = auto for fadt */ -static paramValues_t cParamsToPVals(ZSTD_compressionParameters c) { - paramValues_t p; - p.vals[wlog_ind] = c.windowLog; - p.vals[clog_ind] = c.chainLog; - p.vals[hlog_ind] = c.hashLog; - p.vals[slog_ind] = c.searchLog; - p.vals[slen_ind] = c.searchLength; - p.vals[tlen_ind] = c.targetLength; - p.vals[strt_ind] = c.strategy; - p.vals[fadt_ind] = 0; - return p; -} - -/* equivalent of ZSTD_adjustCParams for paramValues_t */ -static paramValues_t adjustParams(paramValues_t p, size_t maxBlockSize, size_t dictSize) { - U32 fval = p.vals[fadt_ind]; - p = cParamsToPVals(ZSTD_adjustCParams(pvalsToCParams(p), maxBlockSize, dictSize)); - p.vals[fadt_ind] = fval; - return p; +/* display of params */ +static void displayParamVal(FILE* f, varInds_t param, U32 value, int width) { + switch(param) { + case fadt_ind: if(width) { fprintf(f, "%*d", width, (int)value); } else { fprintf(f, "%d", (int)value); } break; + case strt_ind: if(width) { fprintf(f, "%*s", width, g_stratName[value]); } else { fprintf(f, "%s", g_stratName[value]); } break; + case wlog_ind: + case clog_ind: + case hlog_ind: + case slog_ind: + case slen_ind: + case tlen_ind: if(width) { fprintf(f, "%*u", width, value); } else { fprintf(f, "%u", value); } break; + case NUM_PARAMS: + DISPLAY("Error, not a valid param\n "); break; + } } /*-************************************ @@ -238,7 +236,7 @@ static U32 g_target = 0; static U32 g_noSeed = 0; static paramValues_t g_params; /* Initialized at the beginning of main w/ emptyParams() function */ static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */ - +static U32 g_memoLimit = (U32)-1; //32 MB typedef struct { BMK_result_t result; @@ -293,6 +291,51 @@ void BMK_SetNbIterations(int nbLoops) * Private functions *********************************************************/ +static ZSTD_compressionParameters pvalsToCParams(paramValues_t p) { + ZSTD_compressionParameters c; + memset(&c, 0, sizeof(ZSTD_compressionParameters)); + c.windowLog = p.vals[wlog_ind]; + c.chainLog = p.vals[clog_ind]; + c.hashLog = p.vals[hlog_ind]; + c.searchLog = p.vals[slog_ind]; + c.searchLength = p.vals[slen_ind]; + c.targetLength = p.vals[tlen_ind]; + c.strategy = p.vals[strt_ind]; + /* no forceAttachDict */ + return c; +} + +static paramValues_t cParamsToPVals(ZSTD_compressionParameters c) { + paramValues_t p; + varInds_t i; + p.vals[wlog_ind] = c.windowLog; + p.vals[clog_ind] = c.chainLog; + p.vals[hlog_ind] = c.hashLog; + p.vals[slog_ind] = c.searchLog; + p.vals[slen_ind] = c.searchLength; + p.vals[tlen_ind] = c.targetLength; + p.vals[strt_ind] = c.strategy; + + /* set all other params to their minimum value */ + for(i = strt_ind + 1; i < NUM_PARAMS; i++) { + p.vals[i] = mintable[i]; + } + return p; +} + +/* equivalent of ZSTD_adjustCParams for paramValues_t */ +static paramValues_t adjustParams(paramValues_t p, size_t maxBlockSize, size_t dictSize) { + paramValues_t ot = p; + varInds_t i; + p = cParamsToPVals(ZSTD_adjustCParams(pvalsToCParams(p), maxBlockSize, dictSize)); + + /* retain value of all other parameters */ + for(i = strt_ind + 1; i < NUM_PARAMS; i++) { + p.vals[i] = ot.vals[i]; + } + return p; +} + /* accuracy in seconds only, span can be multiple years */ static U32 BMK_timeSpan(UTIL_time_t tStart) { return (U32)(UTIL_clockSpanMicro(tStart) / 1000000ULL); } @@ -375,9 +418,6 @@ static void findClockGranularity(void) { return 0; \ } } - -/* Like ZSTD_checkCParams() but allows 0's */ -/* no check on targetLen? */ static int paramValid(paramValues_t paramTarget) { U32 i; for(i = 0; i < NUM_PARAMS; i++) { @@ -400,8 +440,11 @@ static void BMK_translateAdvancedParams(FILE* f, const paramValues_t params) { U32 i; fprintf(f,"--zstd="); for(i = 0; i < NUM_PARAMS; i++) { - fprintf(f,"%s", g_paramNames[i]); - fprintf(f,"=%u", params.vals[i]); + fprintf(f,"%s=", g_paramNames[i]); + + if(i == strt_ind) { fprintf(f,"%u", params.vals[i]); } + else { displayParamVal(f, i, params.vals[i], 0); } + if(i != NUM_PARAMS - 1) { fprintf(f, ","); } @@ -409,19 +452,16 @@ static void BMK_translateAdvancedParams(FILE* f, const paramValues_t params) { fprintf(f, "\n"); } - -static const char* g_stratName[ZSTD_btultra+1] = { - "(none) ", "ZSTD_fast ", "ZSTD_dfast ", - "ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ", - "ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "}; - static void BMK_displayOneResult(FILE* f, winnerInfo_t res, size_t srcSize) { + varInds_t v; res.params = cParamUnsetMin(res.params); - fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u,%3d, %s}, ", - res.params.vals[wlog_ind], res.params.vals[clog_ind], res.params.vals[hlog_ind], res.params.vals[slog_ind], res.params.vals[slen_ind], - res.params.vals[tlen_ind], (int)res.params.vals[fadt_ind], g_stratName[res.params.vals[strt_ind]]); - fprintf(f, - " /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", + fprintf(f," {"); + for(v = 0; v < NUM_PARAMS; v++) { + if(v != 0) { fprintf(f, ","); } + displayParamVal(f, v, res.params.vals[v], 3); + } + + fprintf(f, "}, /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", (double)srcSize / res.result.cSize, (double)res.result.cSpeed / (1 MB), (double)res.result.dSpeed / (1 MB)); } @@ -494,7 +534,7 @@ static paramValues_t emptyParams(void) { return p; } -static winnerInfo_t initWinnerInfo(paramValues_t p) { +static winnerInfo_t initWinnerInfo(const paramValues_t p) { winnerInfo_t w1; w1.result.cSpeed = 0.; w1.result.dSpeed = 0.; @@ -515,6 +555,7 @@ typedef struct { void** resPtrs; size_t* resSizes; size_t nbBlocks; + size_t maxBlockSize; } buffers_t; typedef struct { @@ -528,27 +569,6 @@ typedef struct { * From bench.c *********************************************************/ -static void BMK_initCCtx(ZSTD_CCtx* ctx, - const void* dictBuffer, const size_t dictBufferSize, const int cLevel, - const paramValues_t* comprParams) { - varInds_t i; - ZSTD_CCtx_reset(ctx); - ZSTD_CCtx_resetParameters(ctx); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel); - - for(i = 0; i < NUM_PARAMS; i++) { - if(comprParams->vals[i] != PARAM_UNSET) - ZSTD_CCtx_setParameter(ctx, cctxSetParamTable[i], comprParams->vals[i]); - } - ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize); -} - -static void BMK_initDCtx(ZSTD_DCtx* dctx, - const void* dictBuffer, const size_t dictBufferSize) { - ZSTD_DCtx_reset(dctx); - ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize); -} - typedef struct { ZSTD_CCtx* ctx; const void* dictBuffer; @@ -559,7 +579,17 @@ typedef struct { static size_t local_initCCtx(void* payload) { const BMK_initCCtxArgs* ag = (const BMK_initCCtxArgs*)payload; - BMK_initCCtx(ag->ctx, ag->dictBuffer, ag->dictBufferSize, ag->cLevel, ag->comprParams); + varInds_t i; + ZSTD_CCtx_reset(ag->ctx); + ZSTD_CCtx_resetParameters(ag->ctx); + ZSTD_CCtx_setParameter(ag->ctx, ZSTD_p_compressionLevel, ag->cLevel); + + for(i = 0; i < NUM_PARAMS; i++) { + if(ag->comprParams->vals[i] != PARAM_UNSET) + ZSTD_CCtx_setParameter(ag->ctx, cctxSetParamTable[i], ag->comprParams->vals[i]); + } + ZSTD_CCtx_loadDictionary(ag->ctx, ag->dictBuffer, ag->dictBufferSize); + return 0; } @@ -571,7 +601,8 @@ typedef struct { static size_t local_initDCtx(void* payload) { const BMK_initDCtxArgs* ag = (const BMK_initDCtxArgs*)payload; - BMK_initDCtx(ag->dctx, ag->dictBuffer, ag->dictBufferSize); + ZSTD_DCtx_reset(ag->dctx); + ZSTD_DCtx_loadDictionary(ag->dctx, ag->dictBuffer, ag->dictBufferSize); return 0; } @@ -636,6 +667,72 @@ static size_t local_defaultDecompress( * From bench.c End *********************************************************/ +static void optimizerAdjustInput(paramValues_t* pc, const size_t maxBlockSize) { + varInds_t v; + for(v = 0; v < NUM_PARAMS; v++) { + if(pc->vals[v] != PARAM_UNSET) { + U32 newval = MIN(MAX(pc->vals[v], mintable[v]), maxtable[v]); + if(newval != pc->vals[v]) { + pc->vals[v] = newval; + DISPLAY("Warning: parameter %s not in valid range, adjusting to ", g_paramNames[v]); displayParamVal(stderr, v, newval, 0); DISPLAY("\n"); + } + } + } + + if(pc->vals[wlog_ind] != PARAM_UNSET) { + + U32 sshb = maxBlockSize > 1 ? ZSTD_highbit32((U32)(maxBlockSize-1)) + 1 : 1; + /* edge case of highBit not working for 0 */ + + if(maxBlockSize < (1ULL << 31) && sshb + 1 < pc->vals[wlog_ind]) { + U32 adjust = MAX(mintable[wlog_ind], sshb); + if(adjust != pc->vals[wlog_ind]) { + pc->vals[wlog_ind] = adjust; + DISPLAY("Warning: windowLog larger than src/block size, adjusted to %u\n", pc->vals[wlog_ind]); + } + } + } + + if(pc->vals[wlog_ind] != PARAM_UNSET && pc->vals[clog_ind] != PARAM_UNSET) { + U32 maxclog; + if(pc->vals[strt_ind] == PARAM_UNSET || pc->vals[strt_ind] >= (U32)ZSTD_btlazy2) { + maxclog = pc->vals[wlog_ind] + 1; + } else { + maxclog = pc->vals[wlog_ind]; + } + + if(pc->vals[clog_ind] > maxclog) { + pc->vals[clog_ind] = maxclog; + DISPLAY("Warning: chainlog too much larger than windowLog size, adjusted to %u\n", pc->vals[clog_ind]); + } + } + + if(pc->vals[wlog_ind] != PARAM_UNSET && pc->vals[hlog_ind] != PARAM_UNSET) { + if(pc->vals[wlog_ind] + 1 < pc->vals[hlog_ind]) { + pc->vals[hlog_ind] = pc->vals[wlog_ind] + 1; + DISPLAY("Warning: hashlog too much larger than windowLog size, adjusted to %u\n", pc->vals[hlog_ind]); + } + } + + if(pc->vals[slog_ind] != PARAM_UNSET && pc->vals[clog_ind] != PARAM_UNSET) { + if(pc->vals[slog_ind] > pc->vals[clog_ind]) { + pc->vals[clog_ind] = pc->vals[slog_ind]; + DISPLAY("Warning: searchLog larger than chainLog, adjusted to %u\n", pc->vals[slog_ind]); + } + } +} + +/* what about low something like clog vs hlog in lvl 1? */ +static int redundantParams(const paramValues_t paramValues, const constraint_t target, const size_t srcSize) { + return + (ZSTD_estimateCStreamSize_usingCParams(pvalsToCParams(paramValues)) > (size_t)target.cMem) /* Uses too much memory */ + || ((1ULL << (paramValues.vals[wlog_ind] - 1)) >= srcSize && paramValues.vals[wlog_ind] != mintable[wlog_ind]) /* wlog too much bigger than src size */ + || (paramValues.vals[clog_ind] > (paramValues.vals[wlog_ind] + (paramValues.vals[strt_ind] > ZSTD_btlazy2))) /* chainLog larger than windowLog*/ + || (paramValues.vals[slog_ind] > paramValues.vals[clog_ind]) /* searchLog larger than chainLog */ + || (paramValues.vals[hlog_ind] > paramValues.vals[wlog_ind] + 1); /* hashLog larger than windowLog + 1 */ + +} + static void freeNonSrcBuffers(const buffers_t b) { free(b.srcPtrs); free(b.srcSizes); @@ -722,6 +819,7 @@ static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, size_t nbF buff->dstCapacities[0] = ZSTD_compressBound(buff->srcSizes[0]); buff->dstSizes[0] = buff->dstCapacities[0]; buff->resSizes[0] = buff->srcSizes[0]; + buff->maxBlockSize = buff->srcSizes[0]; for(n = 1; n < blockNb; n++) { buff->dstPtrs[n] = ((char*)buff->dstPtrs[n-1]) + buff->dstCapacities[n-1]; @@ -729,6 +827,8 @@ static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, size_t nbF buff->dstCapacities[n] = ZSTD_compressBound(buff->srcSizes[n]); buff->dstSizes[n] = buff->dstCapacities[n]; buff->resSizes[n] = buff->srcSizes[n]; + + buff->maxBlockSize = MAX(buff->maxBlockSize, buff->srcSizes[n]); } buff->nbBlocks = blockNb; @@ -786,7 +886,7 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab char* buffer = (char*)(srcBuffer); size_t const readSize = fread((buffer)+pos, 1, (size_t)fileSize, f); fclose(f); - if (readSize != (size_t)fileSize) { /* should we accept partial read? */ + if (readSize != (size_t)fileSize) { DISPLAY("could not read %s", fileNamesTable[n]); ret = 1; goto _cleanUp; @@ -992,7 +1092,7 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t } static int BMK_benchParam(BMK_result_t* resultPtr, - buffers_t buf, contexts_t ctx, + const buffers_t buf, const contexts_t ctx, const paramValues_t cParams) { BMK_return_t res = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_both, BMK_timeMode, 3); *resultPtr = res.result; @@ -1009,7 +1109,7 @@ static int BMK_benchParam(BMK_result_t* resultPtr, #define SPEED_RESULT 4 #define SIZE_RESULT 5 /* maybe have epsilon-eq to limit table size? */ -static int speedSizeCompare(BMK_result_t r1, BMK_result_t r2) { +static int speedSizeCompare(const BMK_result_t r1, const BMK_result_t r2) { if(r1.cSpeed < r2.cSpeed) { if(r1.cSize >= r2.cSize) { return BETTER_RESULT; @@ -1025,7 +1125,7 @@ static int speedSizeCompare(BMK_result_t r1, BMK_result_t r2) { /* 0 for insertion, 1 for no insert */ /* maintain invariant speedSizeCompare(n, n->next) = SPEED_RESULT */ -static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) { +static int insertWinner(const winnerInfo_t w, const constraint_t targetConstraints) { BMK_result_t r = w.result; winner_ll_node* cur_node = g_winners; /* first node to insert */ @@ -1194,7 +1294,7 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res } } -static void BMK_printWinners2(FILE* f, const oldWinnerInfo_t* winners, size_t srcSize) +static void BMK_printWinners2(FILE* f, const oldWinnerInfo_t* winners, const size_t srcSize) { int cLevel; @@ -1206,7 +1306,7 @@ static void BMK_printWinners2(FILE* f, const oldWinnerInfo_t* winners, size_t sr } -static void BMK_printWinners(FILE* f, const oldWinnerInfo_t* winners, size_t srcSize) +static void BMK_printWinners(FILE* f, const oldWinnerInfo_t* winners, const size_t srcSize) { fseek(f, 0, SEEK_SET); BMK_printWinners2(f, winners, srcSize); @@ -1244,7 +1344,7 @@ static void BMK_init_level_constraints(int bytePerSec_level1) } static int BMK_seed(oldWinnerInfo_t* winners, const ZSTD_compressionParameters params, - buffers_t buf, contexts_t ctx) + const buffers_t buf, const contexts_t ctx) { BMK_result_t testResult; int better = 0; @@ -1422,9 +1522,8 @@ static size_t memoTableLen(const varInds_t* varyParams, const int varyLen) { size_t arrayLen = 1; int i; for(i = 0; i < varyLen; i++) { - if(varyParams[i] != strt_ind) { - arrayLen *= rangetable[varyParams[i]]; - } + if(varyParams[i] == strt_ind) continue; /* strategy separated by table */ + arrayLen *= rangetable[varyParams[i]]; } return arrayLen; } @@ -1452,13 +1551,13 @@ static void memoTableIndInv(paramValues_t* ptr, const varInds_t* varyParams, con ind /= rangetable[v]; */ switch(varyParams[i]) { - case wlog_ind: ptr->vals[wlog_ind] = ind % rangetable[wlog_ind] + mintable[wlog_ind]; ind /= rangetable[wlog_ind]; break; - case clog_ind: ptr->vals[clog_ind] = ind % rangetable[clog_ind] + mintable[clog_ind]; ind /= rangetable[clog_ind]; break; - case hlog_ind: ptr->vals[hlog_ind] = ind % rangetable[hlog_ind] + mintable[hlog_ind]; ind /= rangetable[hlog_ind]; break; - case slog_ind: ptr->vals[slog_ind] = ind % rangetable[slog_ind] + mintable[slog_ind]; ind /= rangetable[slog_ind]; break; - case slen_ind: ptr->vals[slen_ind] = ind % rangetable[slen_ind] + mintable[slen_ind]; ind /= rangetable[slen_ind]; break; - case tlen_ind: ptr->vals[tlen_ind] = tlen_table[(ind % rangetable[tlen_ind])]; ind /= rangetable[tlen_ind]; break; - case fadt_ind: ptr->vals[fadt_ind] = ind % rangetable[fadt_ind] - 1; ind /= rangetable[fadt_ind]; break; + case wlog_ind: ptr->vals[wlog_ind] = ind % rangetable[wlog_ind] + mintable[wlog_ind]; ind /= rangetable[wlog_ind]; break; + case clog_ind: ptr->vals[clog_ind] = ind % rangetable[clog_ind] + mintable[clog_ind]; ind /= rangetable[clog_ind]; break; + case hlog_ind: ptr->vals[hlog_ind] = ind % rangetable[hlog_ind] + mintable[hlog_ind]; ind /= rangetable[hlog_ind]; break; + case slog_ind: ptr->vals[slog_ind] = ind % rangetable[slog_ind] + mintable[slog_ind]; ind /= rangetable[slog_ind]; break; + case slen_ind: ptr->vals[slen_ind] = ind % rangetable[slen_ind] + mintable[slen_ind]; ind /= rangetable[slen_ind]; break; + case tlen_ind: ptr->vals[tlen_ind] = tlen_table[(ind % rangetable[tlen_ind])]; ind /= rangetable[tlen_ind]; break; + case fadt_ind: ptr->vals[fadt_ind] = ind % rangetable[fadt_ind] - 1; ind /= rangetable[fadt_ind]; break; case strt_ind: case NUM_PARAMS: break; } @@ -1473,58 +1572,19 @@ static void memoTableIndInv(paramValues_t* ptr, const varInds_t* varyParams, con static void initMemoTable(U8* memoTable, paramValues_t paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { size_t i; size_t arrayLen = memoTableLen(varyParams, varyLen); - int cwFixed = paramConstraints.vals[clog_ind] == PARAM_UNSET || paramConstraints.vals[wlog_ind] == PARAM_UNSET; - int scFixed = paramConstraints.vals[slog_ind] == PARAM_UNSET || paramConstraints.vals[clog_ind] == PARAM_UNSET; - int whFixed = paramConstraints.vals[wlog_ind] == PARAM_UNSET || paramConstraints.vals[hlog_ind] == PARAM_UNSET; - int wFixed = paramConstraints.vals[wlog_ind] == PARAM_UNSET; int j = 0; assert(memoTable != NULL); memset(memoTable, 0, arrayLen); paramConstraints = cParamUnsetMin(paramConstraints); - for(i = 0; i < arrayLen; i++) { memoTableIndInv(¶mConstraints, varyParams, varyLen, i); - if(ZSTD_estimateCStreamSize_usingCParams(pvalsToCParams(paramConstraints)) > (size_t)target.cMem) { - memoTable[i] = 255; - j++; - } - if(wFixed && (1ULL << (paramConstraints.vals[wlog_ind] - 1)) >= srcSize && paramConstraints.vals[wlog_ind] != mintable[wlog_ind]) { - memoTable[i] = 255; - } - /* nil out parameter sets equivalent to others. */ - if(cwFixed) { - if(paramConstraints.vals[strt_ind] == ZSTD_btlazy2 || paramConstraints.vals[strt_ind] == ZSTD_btopt || paramConstraints.vals[strt_ind] == ZSTD_btultra) { - if(paramConstraints.vals[clog_ind] > paramConstraints.vals[wlog_ind]+ 1) { - if(memoTable[i] != 255) { j++; } - memoTable[i] = 255; - } - } else { - if(paramConstraints.vals[clog_ind] > paramConstraints.vals[wlog_ind]) { - if(memoTable[i] != 255) { j++; } - memoTable[i] = 255; - } - } - } - - if(scFixed) { - if(paramConstraints.vals[slog_ind] > paramConstraints.vals[clog_ind]) { - if(memoTable[i] != 255) { j++; } - memoTable[i] = 255; - } - } - - if(whFixed) { - if(paramConstraints.vals[hlog_ind] > paramConstraints.vals[wlog_ind] + 1) { - if(memoTable[i] != 255) { j++; } - memoTable[i] = 255; - } + if(redundantParams(paramConstraints, target, srcSize)) { + memoTable[i] = 255; j++; } } - DEBUGOUTPUT("%d / %d Invalid\n", j, (int)i); - if((int)i == j) { - DEBUGOUTPUT("!!!Strategy %d totally infeasible\n", (int)paramConstraints.vals[strt_ind]); - } + + DEBUGOUTPUT("%d / %d Invalid\n", j, (int)arrayLen); } /* frees all allocated memotables */ @@ -1539,7 +1599,7 @@ static void freeMemoTableArray(U8** mtAll) { /* inits memotables for all (including mallocs), all strategies */ /* takes unsanitized varyParams */ -static U8** createMemoTableArray(paramValues_t paramConstraints, constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { +static U8** createMemoTableArray(paramValues_t paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { varInds_t varNew[NUM_PARAMS]; U8** mtAll = (U8**)calloc(sizeof(U8*),(ZSTD_btultra + 1)); int i; @@ -1549,18 +1609,21 @@ static U8** createMemoTableArray(paramValues_t paramConstraints, constraint_t ta for(i = 1; i <= (int)ZSTD_btultra; i++) { const int varLenNew = sanitizeVarArray(varNew, varyLen, varyParams, i); - mtAll[i] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew)); + size_t mtl = memoTableLen(varNew, varLenNew); + if(mtl > g_memoLimit) { mtAll[i] = NULL; continue; } + mtAll[i] = malloc(sizeof(U8) * mtl); if(mtAll[i] == NULL) { freeMemoTableArray(mtAll); return NULL; } + paramConstraints.vals[strt_ind] = i; initMemoTable(mtAll[i], paramConstraints, target, varNew, varLenNew, srcSize); } return mtAll; } -static paramValues_t overwriteParams(paramValues_t base, paramValues_t mask) { +static paramValues_t overwriteParams(paramValues_t base, const paramValues_t mask) { U32 i; for(i = 0; i < NUM_PARAMS; i++) { if(mask.vals[i] != PARAM_UNSET) { @@ -1586,7 +1649,7 @@ static BYTE* NB_TESTS_PLAYED(ZSTD_compressionParameters p) { static void playAround(FILE* f, oldWinnerInfo_t* winners, ZSTD_compressionParameters params, - buffers_t buf, contexts_t ctx) + const buffers_t buf, const contexts_t ctx) { int nbVariations = 0; UTIL_time_t const clockStart = UTIL_getTime(); @@ -1619,49 +1682,35 @@ static void playAround(FILE* f, oldWinnerInfo_t* winners, } +/* Sets pc to random unmeasured set of parameters */ +/* Doesn't do strategy! */ +static void randomConstrainedParams(paramValues_t* pc, const varInds_t* varArray, const int varLen, const U8* memoTable) +{ + size_t j; + for(j = 0; j < MIN(g_memoLimit, memoTableLen(varArray, varLen)); j++) { + int i; + for(i = 0; i < NUM_PARAMS; i++) { + varInds_t v = varArray[i]; + if(v == strt_ind) continue; //don't do strategy (dependent on MT) + pc->vals[v] = rangeMap(v, FUZ_rand(&g_rand) % rangetable[v]); + } + + if(memoTable == NULL || memoTable[memoTableInd(pc, varArray, varLen)]) { break; } + } +} + /* Completely random parameter selection */ static ZSTD_compressionParameters randomParams(void) { - ZSTD_compressionParameters p; - U32 validated = 0; - while (!validated) { - /* totally random entry */ - p.chainLog = (FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN)) - + ZSTD_CHAINLOG_MIN; - p.hashLog = (FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN)) - + ZSTD_HASHLOG_MIN; - p.searchLog = (FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN)) - + ZSTD_SEARCHLOG_MIN; - p.windowLog = (FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN)) - + ZSTD_WINDOWLOG_MIN; - p.searchLength=(FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN)) - + ZSTD_SEARCHLENGTH_MIN; - p.targetLength=(FUZ_rand(&g_rand) % (512)); - - p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btultra +1)); - - validated = !ZSTD_isError(ZSTD_checkCParams(p)); - } - return p; -} - -/* Sets pc to random unmeasured set of parameters */ -static void randomConstrainedParams(paramValues_t* pc, varInds_t* varArray, int varLen, U8* memoTable) -{ - size_t tries = memoTableLen(varArray, varLen); - const size_t maxSize = memoTableLen(varArray, varLen); - size_t ind; - do { - ind = (FUZ_rand(&g_rand)) % maxSize; - tries--; - } while(memoTable[ind] > 0 && tries > 0); - - memoTableIndInv(pc, varArray, varLen, (unsigned)ind); + paramValues_t p; + p.vals[strt_ind] = rangeMap(strt_ind, rangeMap(strt_ind, FUZ_rand(&g_rand) % rangetable[strt_ind])); + randomConstrainedParams(&p, paramTable, NUM_PARAMS, NULL); + return pvalsToCParams(p); } static void BMK_selectRandomStart( FILE* f, oldWinnerInfo_t* winners, - buffers_t buf, contexts_t ctx) + const buffers_t buf, const contexts_t ctx) { U32 const id = FUZ_rand(&g_rand) % (NB_LEVELS_TRACKED+1); if ((id==0) || (winners[id].params.windowLog==0)) { @@ -1673,7 +1722,7 @@ static void BMK_selectRandomStart( } } -static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBlockSize) +static void BMK_benchFullTable(const buffers_t buf, const contexts_t ctx) { ZSTD_compressionParameters params; oldWinnerInfo_t winners[NB_LEVELS_TRACKED+1]; @@ -1689,7 +1738,7 @@ static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBl BMK_init_level_constraints(g_target * (1 MB)); } else { /* baseline config for level 1 */ - ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, maxBlockSize, ctx.dictSize); + ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, buf.maxBlockSize, ctx.dictSize); BMK_result_t testResult; BMK_benchParam(&testResult, buf, ctx, cParamsToPVals(l1params)); BMK_init_level_constraints((int)((testResult.cSpeed * 31) / 32)); @@ -1699,7 +1748,7 @@ static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBl { const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); int i; for (i=0; i<=maxSeeds; i++) { - params = ZSTD_getCParams(i, maxBlockSize, 0); + params = ZSTD_getCParams(i, buf.maxBlockSize, 0); BMK_seed(winners, params, buf, ctx); } } BMK_printWinners(f, winners, buf.srcSize); @@ -1719,7 +1768,7 @@ static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBl fclose(f); } -static int benchOnce(buffers_t buf, contexts_t ctx) { +static int benchOnce(const buffers_t buf, const contexts_t ctx) { BMK_result_t testResult; if(BMK_benchParam(&testResult, buf, ctx, g_params)) { @@ -1736,7 +1785,6 @@ static int benchSample(void) { const char* const name = "Sample 10MB"; size_t const benchedSize = 10 MB; - U32 blockSize = g_blockSize ? g_blockSize : benchedSize; void* srcBuffer = malloc(benchedSize); int ret = 0; @@ -1769,7 +1817,7 @@ static int benchSample(void) if(g_singleRun) { ret = benchOnce(buf, ctx); } else { - BMK_benchFullTable(buf, ctx, MIN(blockSize, benchedSize)); + BMK_benchFullTable(buf, ctx); } freeBuffers(buf); @@ -1785,7 +1833,6 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam { buffers_t buf; contexts_t ctx; - size_t maxBlockSize = 0, i; int ret = 0; if(createBuffers(&buf, fileNamesTable, nbFiles)) { @@ -1799,10 +1846,6 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam return 2; } - for(i = 0; i < buf.nbBlocks; i++) { - maxBlockSize = MAX(maxBlockSize, buf.srcSizes[i]); - } - DISPLAY("\r%79s\r", ""); if(nbFiles == 1) { DISPLAY("using %s : \n", fileNamesTable[0]); @@ -1810,12 +1853,12 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam DISPLAY("using %d Files : \n", nbFiles); } - g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize)), g_params), maxBlockSize, ctx.dictSize); + g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevel, buf.maxBlockSize, ctx.dictSize)), g_params), buf.maxBlockSize, ctx.dictSize); if(g_singleRun) { ret = benchOnce(buf, ctx); } else { - BMK_benchFullTable(buf, ctx, maxBlockSize); + BMK_benchFullTable(buf, ctx); } freeBuffers(buf); @@ -1937,7 +1980,7 @@ static int benchMemo(BMK_result_t* resultPtr, size_t memind = memoTableInd(&cParams, varyParams, varyLen); int res; - if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { return WORSE_RESULT; } + if((memoTable == NULL && redundantParams(cParams, target, buf.maxBlockSize)) || ((memoTable != NULL) && memoTable[memind] >= INFEASIBLE_THRESHOLD)) { return WORSE_RESULT; } res = allBench(resultPtr, buf, ctx, cParams, target, winnerResult, feas); @@ -1947,7 +1990,7 @@ static int benchMemo(BMK_result_t* resultPtr, } BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, *resultPtr, cParams, target, buf.srcSize); - if(res == BETTER_RESULT || feas) { + if(memoTable != NULL && (res == BETTER_RESULT || feas)) { memoTable[memind] = 255; } return res; @@ -1972,7 +2015,7 @@ static int benchMemo(BMK_result_t* resultPtr, static winnerInfo_t climbOnce(const constraint_t target, const varInds_t* varArray, const int varLen, ZSTD_strategy strat, U8** memoTableArray, - buffers_t buf, contexts_t ctx, + const buffers_t buf, const contexts_t ctx, const paramValues_t init) { /* * cparam - currently considered 'center' @@ -2103,7 +2146,6 @@ static winnerInfo_t optimizeFixedStrategy( /* so climb is given the right fixed strategy */ paramTarget.vals[strt_ind] = strat; /* to pass ZSTD_checkCParams */ - paramTarget = cParamUnsetMin(paramTarget); init = paramTarget; @@ -2178,16 +2220,11 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ const int varLen = variableParams(paramTarget, varArray, dictFileName != NULL); winnerInfo_t winner = initWinnerInfo(emptyParams()); U8** allMT = NULL; - paramValues_t paramBase = cParamUnsetMin(paramTarget); - size_t k, maxBlockSize = 0; + paramValues_t paramBase; contexts_t ctx; buffers_t buf; g_time = UTIL_getTime(); - if(!paramValid(paramBase)) { - return 1; - } - if(createBuffers(&buf, fileNamesTable, nbFiles)) { DISPLAY("unable to load files\n"); return 1; @@ -2205,10 +2242,9 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ DISPLAY("Loading %lu Files... \r", (unsigned long)nbFiles); } - - for(k = 0; k < buf.nbBlocks; k++) { - maxBlockSize = MAX(buf.srcSizes[k], maxBlockSize); - } + /* sanitize paramTarget */ + optimizerAdjustInput(¶mTarget, buf.maxBlockSize); + paramBase = cParamUnsetMin(paramTarget); /* if strategy is fixed, only init that part of memotable */ if(paramTarget.vals[strt_ind] != PARAM_UNSET) { @@ -2227,9 +2263,9 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ goto _cleanUp; } - initMemoTable(allMT[paramTarget.vals[strt_ind]], paramTarget, target, varNew, varLenNew, maxBlockSize); + initMemoTable(allMT[paramTarget.vals[strt_ind]], paramTarget, target, varNew, varLenNew, buf.maxBlockSize); } else { - allMT = createMemoTableArray(paramTarget, target, varArray, varLen, maxBlockSize); + allMT = createMemoTableArray(paramTarget, target, varArray, varLen, buf.maxBlockSize); } if(!allMT) { @@ -2256,7 +2292,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ /* use level'ing mode instead of normal target mode */ /* Should lvl be parameter-masked here? */ if(g_optmode) { - winner.params = cParamsToPVals(ZSTD_getCParams(cLevelOpt, maxBlockSize, ctx.dictSize)); + winner.params = cParamsToPVals(ZSTD_getCParams(cLevelOpt, buf.maxBlockSize, ctx.dictSize)); if(BMK_benchParam(&winner.result, buf, ctx, winner.params)) { ret = 3; goto _cleanUp; @@ -2276,7 +2312,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ /* Don't want it to return anything worse than the best known result */ if(g_singleRun) { BMK_result_t res; - g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevelRun, maxBlockSize, ctx.dictSize)), g_params), maxBlockSize, ctx.dictSize); + g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevelRun, buf.maxBlockSize, ctx.dictSize)), g_params), buf.maxBlockSize, ctx.dictSize); if(BMK_benchParam(&res, buf, ctx, g_params)) { ret = 45; goto _cleanUp; @@ -2315,7 +2351,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ int i; for (i=1; i<=maxSeeds; i++) { int ec; - CParams = overwriteParams(cParamsToPVals(ZSTD_getCParams(i, maxBlockSize, ctx.dictSize)), paramTarget); + CParams = overwriteParams(cParamsToPVals(ZSTD_getCParams(i, buf.maxBlockSize, ctx.dictSize)), paramTarget); ec = BMK_benchParam(&candidate, buf, ctx, CParams); BMK_printWinnerOpt(stdout, i, candidate, CParams, target, buf.srcSize); @@ -2466,16 +2502,24 @@ static int badusage(const char* exename) } #define PARSE_SUB_ARGS(stringLong, stringShort, variable) { if (longCommandWArg(&argument, stringLong) || longCommandWArg(&argument, stringShort)) { variable = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } } -#define PARSE_CPARAMS(variable) \ -{ \ - PARSE_SUB_ARGS("windowLog=", "wlog=", variable.vals[wlog_ind]); \ - PARSE_SUB_ARGS("chainLog=" , "clog=", variable.vals[clog_ind]); \ - PARSE_SUB_ARGS("hashLog=", "hlog=", variable.vals[hlog_ind]); \ - PARSE_SUB_ARGS("searchLog=" , "slog=", variable.vals[slog_ind]); \ - PARSE_SUB_ARGS("searchLength=", "slen=", variable.vals[slen_ind]); \ - PARSE_SUB_ARGS("targetLength=" , "tlen=", variable.vals[tlen_ind]); \ - PARSE_SUB_ARGS("strategy=", "strat=", variable.vals[strt_ind]); \ - PARSE_SUB_ARGS("forceAttachDict=", "fad=" , variable.vals[fadt_ind]); \ +/* 1 if successful parse, 0 otherwise */ +static int parse_params(const char** argptr, paramValues_t* pv) { + int matched = 0; + const char* argOrig = *argptr; + varInds_t v; + for(v = 0; v < NUM_PARAMS; v++) { + if(longCommandWArg(argptr,g_shortParamNames[v]) || longCommandWArg(argptr, g_paramNames[v])) { + if(**argptr == '=') { + (*argptr)++; + pv->vals[v] = readU32FromChar(argptr); + matched = 1; + break; + } + } + /* reset and try again */ + *argptr = argOrig; + } + return matched; } int main(int argc, const char** argv) @@ -2509,7 +2553,7 @@ int main(int argc, const char** argv) if (longCommandWArg(&argument, "--optimize=")) { g_optimizer = 1; for ( ; ;) { - PARSE_CPARAMS(paramTarget); + if(parse_params(&argument, ¶mTarget)) { if(argument[0] == ',') { argument++; continue; } else break; } PARSE_SUB_ARGS("compressionSpeed=" , "cSpeed=", target.cSpeed); PARSE_SUB_ARGS("decompressionSpeed=", "dSpeed=", target.dSpeed); PARSE_SUB_ARGS("compressionMemory=" , "cMem=", target.cMem); @@ -2517,6 +2561,7 @@ int main(int argc, const char** argv) PARSE_SUB_ARGS("preferSpeed=", "prfSpd=", g_speedMultiplier); PARSE_SUB_ARGS("preferRatio=", "prfRto=", g_ratioMultiplier); PARSE_SUB_ARGS("maxTries=", "tries=", g_maxTries); + PARSE_SUB_ARGS("memoLimit=", "memo=", g_memoLimit); if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevelOpt = readU32FromChar(&argument); g_optmode = 1; if (argument[0]==',') { argument++; continue; } else break; } DISPLAY("invalid optimization parameter \n"); @@ -2532,7 +2577,7 @@ int main(int argc, const char** argv) /* Decode command (note : aggregated commands are allowed) */ g_singleRun = 1; for ( ; ;) { - PARSE_CPARAMS(g_params) + if(parse_params(&argument, &g_params)) { if(argument[0] == ',') { argument++; continue; } else break; } if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevelRun = readU32FromChar(&argument); g_params = emptyParams(); if (argument[0]==',') { argument++; continue; } else break; } DISPLAY("invalid compression parameter \n"); From b1d9ca737a318d153eb439b4f4ed786bd0cf4384 Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 13 Aug 2018 12:51:22 -0700 Subject: [PATCH 43/55] Add memoTable options -hashing memotable -no memotable --- tests/README.md | 9 +- tests/paramgrill.c | 424 ++++++++++++++++++++------------------------- 2 files changed, 197 insertions(+), 236 deletions(-) diff --git a/tests/README.md b/tests/README.md index 3c35ecf01..bdc9fff97 100644 --- a/tests/README.md +++ b/tests/README.md @@ -113,14 +113,15 @@ Full list of arguments dSpeed= : Minimum decompression speed cMem= : Maximum compression memory lvl= : Searches for solutions which are strictly better than that compression lvl in ratio and cSpeed, - stc= : When invoked with lvl=, represents percentage slack in ratio/cSpeed allowed for a solution to be considered (Default 99%) + stc= : When invoked with lvl=, represents percentage slack in ratio/cSpeed allowed for a solution to be considered (Default 100%) : In normal operation, represents percentage slack in choosing viable starting strategy selection in choosing the default parameters (Lower value will begin with stronger strategies) (Default 90%) - preferSpeed= / preferRatio= - : Only affects lvl = invocations. Defines value placed on compression speed or ratio - when determining overall winner (default speed = 1, ratio = 5 for both, higher = more valued). + speedRatio= (accepts decimals) + : determines value of gains in speed vs gains in ratio + when determining overall winner (default 5 (1% ratio = 5% speed)). tries= : Maximum number of random restarts on a single strategy before switching (Default 3) Higher values will make optimizer run longer, more chances to find better solution. + memLog : Limits the log of the size of each memotable (1 per strategy). Setting memLog = 0 turns off memoization -P# : generated sample compressibility -t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) -v : Prints Benchmarking output diff --git a/tests/paramgrill.c b/tests/paramgrill.c index c2979cb83..4a7fc387a 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -117,10 +117,6 @@ typedef struct { U32 vals[NUM_PARAMS]; } paramValues_t; -/* list of parameters */ -static const varInds_t paramTable[NUM_PARAMS] = - { wlog_ind, clog_ind, hlog_ind, slog_ind, slen_ind, tlen_ind, strt_ind, fadt_ind }; - /* maximum value of parameters */ static const U32 mintable[NUM_PARAMS] = { ZSTD_WINDOWLOG_MIN, ZSTD_CHAINLOG_MIN, ZSTD_HASHLOG_MIN, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLENGTH_MIN, ZSTD_TARGETLENGTH_MIN, ZSTD_fast, FADT_MIN }; @@ -236,7 +232,21 @@ static U32 g_target = 0; static U32 g_noSeed = 0; static paramValues_t g_params; /* Initialized at the beginning of main w/ emptyParams() function */ static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */ -static U32 g_memoLimit = (U32)-1; //32 MB +static U32 g_memoTableLog = PARAM_UNSET; + +typedef enum { + directMap, + xxhashMap, + noMemo +} memoTableType_t; + +typedef struct { + memoTableType_t tableType; + BYTE* table; + size_t tableLen; + varInds_t varArray[NUM_PARAMS]; + size_t varLen; +} memoTable_t; typedef struct { BMK_result_t result; @@ -264,14 +274,12 @@ static winner_ll_node* g_winners; /* linked list sorted ascending by cSize & cSp static BMK_result_t g_lvltarget; static int g_optmode = 0; -static U32 g_speedMultiplier = 1; -static U32 g_ratioMultiplier = 5; +static double g_ratioMultiplier = 5.; /* g_mode? */ /* range 0 - 99, measure of how strict */ -#define DEFAULT_STRICTNESS 99999 -static U32 g_strictness = DEFAULT_STRICTNESS; +static U32 g_strictness = PARAM_UNSET; void BMK_SetNbIterations(int nbLoops) { @@ -281,7 +289,6 @@ void BMK_SetNbIterations(int nbLoops) /* * Additional Global Variables (Defined Above Use) - * g_stratName * g_level_constraint * g_alreadyTested * g_maxTries @@ -324,7 +331,7 @@ static paramValues_t cParamsToPVals(ZSTD_compressionParameters c) { } /* equivalent of ZSTD_adjustCParams for paramValues_t */ -static paramValues_t adjustParams(paramValues_t p, size_t maxBlockSize, size_t dictSize) { +static paramValues_t adjustParams(paramValues_t p, const size_t maxBlockSize, const size_t dictSize) { paramValues_t ot = p; varInds_t i; p = cParamsToPVals(ZSTD_adjustCParams(pvalsToCParams(p), maxBlockSize, dictSize)); @@ -337,7 +344,7 @@ static paramValues_t adjustParams(paramValues_t p, size_t maxBlockSize, size_t d } /* accuracy in seconds only, span can be multiple years */ -static U32 BMK_timeSpan(UTIL_time_t tStart) { return (U32)(UTIL_clockSpanMicro(tStart) / 1000000ULL); } +static U32 BMK_timeSpan(const UTIL_time_t tStart) { return (U32)(UTIL_clockSpanMicro(tStart) / 1000000ULL); } static size_t BMK_findMaxMem(U64 requiredMem) { @@ -418,7 +425,7 @@ static void findClockGranularity(void) { return 0; \ } } -static int paramValid(paramValues_t paramTarget) { +static int paramValid(const paramValues_t paramTarget) { U32 i; for(i = 0; i < NUM_PARAMS; i++) { CLAMPCHECK(paramTarget.vals[i], mintable[i], maxtable[i]); @@ -452,7 +459,7 @@ static void BMK_translateAdvancedParams(FILE* f, const paramValues_t params) { fprintf(f, "\n"); } -static void BMK_displayOneResult(FILE* f, winnerInfo_t res, size_t srcSize) { +static void BMK_displayOneResult(FILE* f, winnerInfo_t res, const size_t srcSize) { varInds_t v; res.params = cParamUnsetMin(res.params); fprintf(f," {"); @@ -498,7 +505,7 @@ static double resultDistLvl(const BMK_result_t result1, const BMK_result_t lvlRe if(normalizedRatioGain1 < 0 || normalizedCSpeedGain1 < 0) { return 0.0; } - return normalizedRatioGain1 * g_ratioMultiplier + normalizedCSpeedGain1 * g_speedMultiplier; + return normalizedRatioGain1 * g_ratioMultiplier + normalizedCSpeedGain1; } /* return true if r2 strictly better than r1 */ @@ -723,10 +730,10 @@ static void optimizerAdjustInput(paramValues_t* pc, const size_t maxBlockSize) { } /* what about low something like clog vs hlog in lvl 1? */ -static int redundantParams(const paramValues_t paramValues, const constraint_t target, const size_t srcSize) { +static int redundantParams(const paramValues_t paramValues, const constraint_t target, const size_t maxBlockSize) { return (ZSTD_estimateCStreamSize_usingCParams(pvalsToCParams(paramValues)) > (size_t)target.cMem) /* Uses too much memory */ - || ((1ULL << (paramValues.vals[wlog_ind] - 1)) >= srcSize && paramValues.vals[wlog_ind] != mintable[wlog_ind]) /* wlog too much bigger than src size */ + || ((1ULL << (paramValues.vals[wlog_ind] - 1)) >= maxBlockSize && paramValues.vals[wlog_ind] != mintable[wlog_ind]) /* wlog too much bigger than src size */ || (paramValues.vals[clog_ind] > (paramValues.vals[wlog_ind] + (paramValues.vals[strt_ind] > ZSTD_btlazy2))) /* chainLog larger than windowLog*/ || (paramValues.vals[slog_ind] > paramValues.vals[clog_ind]) /* searchLog larger than chainLog */ || (paramValues.vals[hlog_ind] > paramValues.vals[wlog_ind] + 1); /* hashLog larger than windowLog + 1 */ @@ -759,7 +766,7 @@ static void freeBuffers(const buffers_t b) { } /* srcBuffer will be freed by freeBuffers now */ -static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, size_t nbFiles, +static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, const size_t nbFiles, const size_t* fileSizes) { size_t pos = 0, n, blockSize; @@ -978,7 +985,6 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t ZSTD_DCtx* dctx = ctx.dctx; /* warmimg up memory */ - /* can't do this if decode only */ for(i = 0; i < buf.nbBlocks; i++) { if(mode != BMK_decodeOnly) { RDG_genBuffer(dstPtrs[i], dstCapacities[i], 0.10, 0.50, 1); @@ -988,7 +994,6 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t } /* Bench */ - { /* init args */ BMK_initCCtxArgs cctxprep; @@ -1234,21 +1239,21 @@ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result snprintf(lvlstr, 15, " Level %2u ", cLevel); } - fprintf(f, "/* %s */ ", lvlstr); - BMK_displayOneResult(f, w, srcSize); - - if(TIMED) { + if(TIMED) { const U64 time = UTIL_clockSpanNano(g_time); const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC); - fprintf(f, " - %1lu:%2lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); + fprintf(f, "%1lu:%2lu:%05.2f - ", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); } - fprintf(f, "\n"); + + fprintf(f, "/* %s */ ", lvlstr); + BMK_displayOneResult(f, w, srcSize); } static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t result, const paramValues_t params, const constraint_t targetConstraints, const size_t srcSize) { /* global winner used for constraints */ - static winnerInfo_t g_winner = { { (size_t)-1LL, 0, 0, (size_t)-1LL }, { { PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET } } }; + /* cSize, cSpeed, dSpeed, cMem */ + static winnerInfo_t g_winner = { { (size_t)-1LL, 0, 0, (size_t)-1LL }, { { PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET } } }; if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { if(DEBUG && compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { DISPLAY("New Winner: \n"); @@ -1445,9 +1450,7 @@ static int BMK_seed(oldWinnerInfo_t* winners, const ZSTD_compressionParameters p } /* nullified useless params, to ensure count stats */ -/* no point in windowLog < chainLog (no point 2x chainLog for bt) */ -/* now with built in bounds-checking */ -/* no longer does anything with sanitizeVarArray + clampcheck */ +/* cleans up params for memoizing / display */ static paramValues_t sanitizeParams(paramValues_t params) { if (params.vals[strt_ind] == ZSTD_fast) @@ -1463,8 +1466,8 @@ static paramValues_t sanitizeParams(paramValues_t params) /* return: new length */ /* keep old array, will need if iter over strategy. */ /* prunes useless params */ -static int sanitizeVarArray(varInds_t* varNew, const int varLength, const varInds_t* varArray, const ZSTD_strategy strat) { - int i, j = 0; +static size_t sanitizeVarArray(varInds_t* varNew, const size_t varLength, const varInds_t* varArray, const ZSTD_strategy strat) { + size_t i, j = 0; for(i = 0; i < varLength; i++) { if( !((varArray[i] == clog_ind && strat == ZSTD_fast) || (varArray[i] == slog_ind && strat == ZSTD_fast) @@ -1481,9 +1484,9 @@ static int sanitizeVarArray(varInds_t* varNew, const int varLength, const varInd /* res should be NUM_PARAMS size */ /* constructs varArray from paramValues_t style parameter */ /* pass in using dict. */ -static int variableParams(const paramValues_t paramConstraints, varInds_t* res, const int usingDictionary) { +static size_t variableParams(const paramValues_t paramConstraints, varInds_t* res, const int usingDictionary) { varInds_t i; - int j = 0; + size_t j = 0; for(i = 0; i < NUM_PARAMS; i++) { if(paramConstraints.vals[i] == PARAM_UNSET) { if(i == fadt_ind && !usingDictionary) continue; /* don't use fadt if no dictionary */ @@ -1501,7 +1504,7 @@ static void paramVaryOnce(const varInds_t paramIndex, const int amt, paramValues } /* varies ptr by nbChanges respecting varyParams*/ -static void paramVariation(paramValues_t* ptr, const varInds_t* varyParams, const int varyLen, const U32 nbChanges) +static void paramVariation(paramValues_t* ptr, memoTable_t* mtAll, const U32 nbChanges) { paramValues_t p; U32 validated = 0; @@ -1509,8 +1512,8 @@ static void paramVariation(paramValues_t* ptr, const varInds_t* varyParams, cons U32 i; p = *ptr; for (i = 0 ; i < nbChanges ; i++) { - const U32 changeID = FUZ_rand(&g_rand) % (varyLen << 1); - paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p); + const U32 changeID = (U32)FUZ_rand(&g_rand) % (mtAll[p.vals[strt_ind]].varLen << 1); + paramVaryOnce(mtAll[p.vals[strt_ind]].varArray[changeID >> 1], ((changeID & 1) << 1) - 1, &p); } validated = paramValid(p); } @@ -1518,9 +1521,9 @@ static void paramVariation(paramValues_t* ptr, const varInds_t* varyParams, cons } /* length of memo table given free variables */ -static size_t memoTableLen(const varInds_t* varyParams, const int varyLen) { +static size_t memoTableLen(const varInds_t* varyParams, const size_t varyLen) { size_t arrayLen = 1; - int i; + size_t i; for(i = 0; i < varyLen; i++) { if(varyParams[i] == strt_ind) continue; /* strategy separated by table */ arrayLen *= rangetable[varyParams[i]]; @@ -1529,8 +1532,8 @@ static size_t memoTableLen(const varInds_t* varyParams, const int varyLen) { } /* returns unique index in memotable of compression parameters */ -static unsigned memoTableInd(const paramValues_t* ptr, const varInds_t* varyParams, const int varyLen) { - int i; +static unsigned memoTableIndDirect(const paramValues_t* ptr, const varInds_t* varyParams, const size_t varyLen) { + size_t i; unsigned ind = 0; for(i = 0; i < varyLen; i++) { varInds_t v = varyParams[i]; @@ -1540,84 +1543,82 @@ static unsigned memoTableInd(const paramValues_t* ptr, const varInds_t* varyPara return ind; } -/* inverse of above function (from index to parameters) */ -static void memoTableIndInv(paramValues_t* ptr, const varInds_t* varyParams, const int varyLen, size_t ind) { - int i; - for(i = varyLen - 1; i >= 0; i--) { - /* This is cleaner/easier to generalize but slower - varInds_t v = varyParams[i]; - if(v == strt_ind) continue; - ptr->vals[v] = rangeMap(v, ind % rangetable[v]); - ind /= rangetable[v]; */ - - switch(varyParams[i]) { - case wlog_ind: ptr->vals[wlog_ind] = ind % rangetable[wlog_ind] + mintable[wlog_ind]; ind /= rangetable[wlog_ind]; break; - case clog_ind: ptr->vals[clog_ind] = ind % rangetable[clog_ind] + mintable[clog_ind]; ind /= rangetable[clog_ind]; break; - case hlog_ind: ptr->vals[hlog_ind] = ind % rangetable[hlog_ind] + mintable[hlog_ind]; ind /= rangetable[hlog_ind]; break; - case slog_ind: ptr->vals[slog_ind] = ind % rangetable[slog_ind] + mintable[slog_ind]; ind /= rangetable[slog_ind]; break; - case slen_ind: ptr->vals[slen_ind] = ind % rangetable[slen_ind] + mintable[slen_ind]; ind /= rangetable[slen_ind]; break; - case tlen_ind: ptr->vals[tlen_ind] = tlen_table[(ind % rangetable[tlen_ind])]; ind /= rangetable[tlen_ind]; break; - case fadt_ind: ptr->vals[fadt_ind] = ind % rangetable[fadt_ind] - 1; ind /= rangetable[fadt_ind]; break; - case strt_ind: - case NUM_PARAMS: break; - } +static size_t memoTableGet(const memoTable_t* memoTableArray, const paramValues_t p) { + const memoTable_t mt = memoTableArray[p.vals[strt_ind]]; + switch(mt.tableType) { + case directMap: + return mt.table[memoTableIndDirect(&p, mt.varArray, mt.varLen)]; + case xxhashMap: + return mt.table[(XXH64(&p.vals, sizeof(U32) * NUM_PARAMS, 0) >> 3) % mt.tableLen]; + case noMemo: + return 0; } + return 0; /* should never happen, stop compiler warnings */ } -/* Initialize memoization table, which tracks and prevents repeated benchmarking - * of the same set of parameters. In addition, it is also used to immediately mark - * redundant / obviously non-optimal parameter configurations (e.g. wlog - 1 larger) - * than srcSize, clog > wlog, ... - */ -static void initMemoTable(U8* memoTable, paramValues_t paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { - size_t i; - size_t arrayLen = memoTableLen(varyParams, varyLen); - int j = 0; - assert(memoTable != NULL); - memset(memoTable, 0, arrayLen); - paramConstraints = cParamUnsetMin(paramConstraints); - - for(i = 0; i < arrayLen; i++) { - memoTableIndInv(¶mConstraints, varyParams, varyLen, i); - if(redundantParams(paramConstraints, target, srcSize)) { - memoTable[i] = 255; j++; - } +static void memoTableSet(const memoTable_t* memoTableArray, const paramValues_t p, const BYTE value) { + const memoTable_t mt = memoTableArray[p.vals[strt_ind]]; + switch(mt.tableType) { + case directMap: + mt.table[memoTableIndDirect(&p, mt.varArray, mt.varLen)] = value; break; + case xxhashMap: + mt.table[(XXH64(&p.vals, sizeof(U32) * NUM_PARAMS, 0) >> 3) % mt.tableLen] = value; break; + case noMemo: + break; } - - DEBUGOUTPUT("%d / %d Invalid\n", j, (int)arrayLen); } /* frees all allocated memotables */ -static void freeMemoTableArray(U8** mtAll) { +static void freeMemoTableArray(memoTable_t* const mtAll) { int i; if(mtAll == NULL) { return; } for(i = 1; i <= (int)ZSTD_btultra; i++) { - free(mtAll[i]); + free(mtAll[i].table); } free(mtAll); } /* inits memotables for all (including mallocs), all strategies */ /* takes unsanitized varyParams */ -static U8** createMemoTableArray(paramValues_t paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { - varInds_t varNew[NUM_PARAMS]; - U8** mtAll = (U8**)calloc(sizeof(U8*),(ZSTD_btultra + 1)); +static memoTable_t* createMemoTableArray(const varInds_t* const varyParams, const size_t varyLen) { + memoTable_t* mtAll = (memoTable_t*)calloc(sizeof(memoTable_t),(ZSTD_btultra + 1)); int i; + if(mtAll == NULL) { return NULL; } for(i = 1; i <= (int)ZSTD_btultra; i++) { - const int varLenNew = sanitizeVarArray(varNew, varyLen, varyParams, i); - size_t mtl = memoTableLen(varNew, varLenNew); - if(mtl > g_memoLimit) { mtAll[i] = NULL; continue; } - mtAll[i] = malloc(sizeof(U8) * mtl); - if(mtAll[i] == NULL) { + mtAll[i].varLen = sanitizeVarArray(mtAll[i].varArray, varyLen, varyParams, i); + } + + /* no memoization */ + if(g_memoTableLog == 0) { + for(i = 1; i <= (int)ZSTD_btultra; i++) { + mtAll[i].tableType = noMemo; + mtAll[i].table = NULL; + mtAll[i].tableLen = 0; + } + return mtAll; + } + + /* hash table if normal table is too big */ + for(i = 1; i <= (int)ZSTD_btultra; i++) { + size_t mtl = memoTableLen(mtAll[i].varArray, mtAll[i].varLen); + mtAll[i].tableType = directMap; + + if(g_memoTableLog != PARAM_UNSET && mtl > (1ULL << g_memoTableLog)) { /* use hash table */ /* provide some option to only use hash tables? */ + mtAll[i].tableType = xxhashMap; + mtl = (1ULL << g_memoTableLog); + } + + mtAll[i].table = (BYTE*)calloc(sizeof(BYTE), mtl); + mtAll[i].tableLen = mtl; + + if(mtAll[i].table == NULL) { freeMemoTableArray(mtAll); return NULL; } - paramConstraints.vals[strt_ind] = i; - initMemoTable(mtAll[i], paramConstraints, target, varNew, varLenNew, srcSize); } return mtAll; @@ -1638,10 +1639,6 @@ static paramValues_t overwriteParams(paramValues_t base, const paramValues_t mas #define PARAMTABLEMASK (PARAMTABLESIZE-1) static BYTE g_alreadyTested[PARAMTABLESIZE] = {0}; /* init to zero */ -/* -#define NB_TESTS_PLAYED(p) \ - g_alreadyTested[(XXH64(((void*)&sanitizeParams(p), sizeof(p), 0) >> 3) & PARAMTABLEMASK] */ - static BYTE* NB_TESTS_PLAYED(ZSTD_compressionParameters p) { ZSTD_compressionParameters p2 = pvalsToCParams(sanitizeParams(cParamsToPVals(p))); return &g_alreadyTested[(XXH64((void*)&p2, sizeof(p2), 0) >> 3) & PARAMTABLEMASK]; @@ -1651,10 +1648,8 @@ static void playAround(FILE* f, oldWinnerInfo_t* winners, ZSTD_compressionParameters params, const buffers_t buf, const contexts_t ctx) { - int nbVariations = 0; + int nbVariations = 0, i; UTIL_time_t const clockStart = UTIL_getTime(); - const U32 unconstrained[NUM_PARAMS] = { 0, 1, 2, 3, 4, 5, 6 }; /* no fadt */ - while (UTIL_clockSpanMicro(clockStart) < g_maxVariationTime) { paramValues_t p = cParamsToPVals(params); @@ -1662,7 +1657,9 @@ static void playAround(FILE* f, oldWinnerInfo_t* winners, BYTE* b; if (nbVariations++ > g_maxNbVariations) break; - paramVariation(&p, unconstrained, NUM_PARAMS, 4); + + do { for(i = 0; i < 4; i++) { paramVaryOnce(FUZ_rand(&g_rand) % (strt_ind + 1), ((FUZ_rand(&g_rand) & 1) << 1) - 1, &p); } } + while(!paramValid(p)); p2 = pvalsToCParams(p); @@ -1683,28 +1680,31 @@ static void playAround(FILE* f, oldWinnerInfo_t* winners, } /* Sets pc to random unmeasured set of parameters */ -/* Doesn't do strategy! */ -static void randomConstrainedParams(paramValues_t* pc, const varInds_t* varArray, const int varLen, const U8* memoTable) +/* specifiy strategy */ +static void randomConstrainedParams(paramValues_t* pc, const memoTable_t* memoTableArray, const ZSTD_strategy st) { size_t j; - for(j = 0; j < MIN(g_memoLimit, memoTableLen(varArray, varLen)); j++) { + const memoTable_t mt = memoTableArray[st]; + pc->vals[strt_ind] = st; + for(j = 0; j < MIN(1ULL << g_memoTableLog, memoTableLen(mt.varArray, mt.varLen)); j++) { int i; for(i = 0; i < NUM_PARAMS; i++) { - varInds_t v = varArray[i]; - if(v == strt_ind) continue; //don't do strategy (dependent on MT) + varInds_t v = mt.varArray[i]; + if(v == strt_ind) continue; //skip, already specified pc->vals[v] = rangeMap(v, FUZ_rand(&g_rand) % rangetable[v]); } - if(memoTable == NULL || memoTable[memoTableInd(pc, varArray, varLen)]) { break; } + if(!(memoTableGet(memoTableArray, *pc))) break; //only pick unpicked params. } } /* Completely random parameter selection */ static ZSTD_compressionParameters randomParams(void) { - paramValues_t p; - p.vals[strt_ind] = rangeMap(strt_ind, rangeMap(strt_ind, FUZ_rand(&g_rand) % rangetable[strt_ind])); - randomConstrainedParams(&p, paramTable, NUM_PARAMS, NULL); + paramValues_t p; varInds_t v; + for(v = 0; v < NUM_PARAMS; v++) { + p.vals[v] = rangeMap(v, FUZ_rand(&g_rand) % rangetable[v]); + } return pvalsToCParams(p); } @@ -1866,68 +1866,62 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam return ret; } +#define CBENCHMARK(conditional, resultvar, tmpret, mode, loopmode, sec) { \ + if(conditional) { \ + BMK_return_t tmpret = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, mode, loopmode, sec); \ + if(tmpret.error) { DEBUGOUTPUT("Benchmarking failed\n"); return ERROR_RESULT; } \ + if(mode != BMK_decodeOnly) { \ + resultvar.cSpeed = tmpret.result.cSpeed; \ + resultvar.cSize = tmpret.result.cSize; \ + resultvar.cMem = tmpret.result.cMem; \ + } \ + if(mode != BMK_compressOnly) { resultvar.dSpeed = tmpret.result.dSpeed; } \ + } \ +} + /* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */ #define VARIANCE 1.2 -#define HIGH_VARIANCE 100.0 static int allBench(BMK_result_t* resultPtr, const buffers_t buf, const contexts_t ctx, const paramValues_t cParams, const constraint_t target, BMK_result_t* winnerResult, int feas) { - BMK_return_t benchres; - BMK_result_t resultMax; + BMK_result_t resultMax, benchres; U64 loopDurationC = 0, loopDurationD = 0; double uncertaintyConstantC = 3., uncertaintyConstantD = 3.; double winnerRS; - /* initial benchmarking, gives exact ratio and memory, warms up future runs */ - benchres = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_both, BMK_iterMode, 1); + CBENCHMARK(1, benchres, tmp, BMK_both, BMK_iterMode, 1); winnerRS = resultScore(*winnerResult, buf.srcSize, target); DEBUGOUTPUT("WinnerScore: %f\n ", winnerRS); - if(benchres.error) { - DEBUGOUTPUT("Benchmarking failed\n"); - return ERROR_RESULT; - } - *resultPtr = benchres.result; + *resultPtr = benchres; /* calculate uncertainty in compression / decompression runs */ - if(benchres.result.cSpeed) { - loopDurationC = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); + if(benchres.cSpeed) { + loopDurationC = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.cSpeed); uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC); } - if(benchres.result.dSpeed) { - loopDurationD = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); + if(benchres.dSpeed) { + loopDurationD = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.dSpeed); uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD); } /* anything with worse ratio in feas is definitely worse, discard */ - if(feas && benchres.result.cSize < winnerResult->cSize && !g_optmode) { + if(feas && benchres.cSize < winnerResult->cSize && !g_optmode) { return WORSE_RESULT; } /* second run, if first run is too short, gives approximate cSpeed + dSpeed */ - if(loopDurationC < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_compressOnly, BMK_iterMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } - benchres = benchres2; - } - if(loopDurationD < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_decodeOnly, BMK_iterMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } - benchres.result.dSpeed = benchres2.result.dSpeed; - } + CBENCHMARK(loopDurationC < TIMELOOP_NANOSEC / 10, benchres, tmp, BMK_compressOnly, BMK_iterMode, 1); + CBENCHMARK(loopDurationD < TIMELOOP_NANOSEC / 10, benchres, tmp, BMK_decodeOnly, BMK_iterMode, 1); - *resultPtr = benchres.result; + *resultPtr = benchres; - /* optimistic assumption of benchres.result */ - resultMax = benchres.result; + /* optimistic assumption of benchres */ + resultMax = benchres; resultMax.cSpeed *= uncertaintyConstantC * VARIANCE; resultMax.dSpeed *= uncertaintyConstantD * VARIANCE; @@ -1938,29 +1932,15 @@ static int allBench(BMK_result_t* resultPtr, return WORSE_RESULT; } - /* Final full run if estimates are unclear */ - if(loopDurationC < TIMELOOP_NANOSEC) { - BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_compressOnly, BMK_timeMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } - benchres.result.cSpeed = benchres2.result.cSpeed; - } + CBENCHMARK(loopDurationC < TIMELOOP_NANOSEC, benchres, tmp, BMK_compressOnly, BMK_timeMode, 1); + CBENCHMARK(loopDurationD < TIMELOOP_NANOSEC, benchres, tmp, BMK_decodeOnly, BMK_timeMode, 1); - if(loopDurationD < TIMELOOP_NANOSEC) { - BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_decodeOnly, BMK_timeMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } - benchres.result.dSpeed = benchres2.result.dSpeed; - } - - *resultPtr = benchres.result; + *resultPtr = benchres; /* compare by resultScore when in infeas */ /* compare by compareResultLT when in feas */ - if((!feas && (resultScore(benchres.result, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target))) || - (feas && (compareResultLT(*winnerResult, benchres.result, target, buf.srcSize))) ) { + if((!feas && (resultScore(benchres, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target))) || + (feas && (compareResultLT(*winnerResult, benchres, target, buf.srcSize))) ) { return BETTER_RESULT; } else { return WORSE_RESULT; @@ -1968,19 +1948,17 @@ static int allBench(BMK_result_t* resultPtr, } #define INFEASIBLE_THRESHOLD 200 - /* Memoized benchmarking, won't benchmark anything which has already been benchmarked before. */ static int benchMemo(BMK_result_t* resultPtr, const buffers_t buf, const contexts_t ctx, const paramValues_t cParams, const constraint_t target, - BMK_result_t* winnerResult, U8* const memoTable, - const varInds_t* varyParams, const int varyLen, const int feas) { + BMK_result_t* winnerResult, memoTable_t* const memoTableArray, + const int feas) { static int bmcount = 0; - size_t memind = memoTableInd(&cParams, varyParams, varyLen); int res; - if((memoTable == NULL && redundantParams(cParams, target, buf.maxBlockSize)) || ((memoTable != NULL) && memoTable[memind] >= INFEASIBLE_THRESHOLD)) { return WORSE_RESULT; } + if(memoTableGet(memoTableArray, cParams) >= INFEASIBLE_THRESHOLD || redundantParams(cParams, target, buf.maxBlockSize)) { return WORSE_RESULT; } res = allBench(resultPtr, buf, ctx, cParams, target, winnerResult, feas); @@ -1990,8 +1968,8 @@ static int benchMemo(BMK_result_t* resultPtr, } BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, *resultPtr, cParams, target, buf.srcSize); - if(memoTable != NULL && (res == BETTER_RESULT || feas)) { - memoTable[memind] = 255; + if(res == BETTER_RESULT || feas) { + memoTableSet(memoTableArray, cParams, 255); /* what happens if collisions are frequent */ } return res; } @@ -2013,8 +1991,7 @@ static int benchMemo(BMK_result_t* resultPtr, * all generation after random should be sanitized. (maybe sanitize random) */ static winnerInfo_t climbOnce(const constraint_t target, - const varInds_t* varArray, const int varLen, ZSTD_strategy strat, - U8** memoTableArray, + memoTable_t* mtAll, const buffers_t buf, const contexts_t ctx, const paramValues_t init) { /* @@ -2026,8 +2003,6 @@ static winnerInfo_t climbOnce(const constraint_t target, winnerInfo_t candidateInfo, winnerInfo; int better = 1; int feas = 0; - varInds_t varNew[NUM_PARAMS]; - int varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat); winnerInfo = initWinnerInfo(init); candidateInfo = winnerInfo; @@ -2037,7 +2012,9 @@ static winnerInfo_t climbOnce(const constraint_t target, DEBUGOUTPUT("Climb Part 1\n"); while(better) { - int i, dist, offset; + int offset; + size_t i, dist; + const size_t varLen = mtAll[cparam.vals[strt_ind]].varLen; better = 0; DEBUGOUTPUT("Start\n"); cparam = winnerInfo.params; @@ -2048,17 +2025,13 @@ static winnerInfo_t climbOnce(const constraint_t target, for(offset = -1; offset <= 1; offset += 2) { CHECKTIME(winnerInfo); candidateInfo.params = cparam; - paramVaryOnce(varArray[i], offset, &candidateInfo.params); + paramVaryOnce(mtAll[cparam.vals[strt_ind]].varArray[i], offset, &candidateInfo.params); if(paramValid(candidateInfo.params)) { int res; - if(strat != candidateInfo.params.vals[strt_ind]) { /* maybe only try strategy switching after exhausting non-switching solutions? */ - strat = candidateInfo.params.vals[strt_ind]; - varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat); - } res = benchMemo(&candidateInfo.result, buf, ctx, - sanitizeParams(candidateInfo.params), target, &winnerInfo.result, memoTableArray[strat], - varNew, varLenNew, feas); + sanitizeParams(candidateInfo.params), target, &winnerInfo.result, mtAll, feas); + DEBUGOUTPUT("Res: %d\n", res); if(res == BETTER_RESULT) { /* synonymous with better when called w/ infeasibleBM */ winnerInfo = candidateInfo; BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, sanitizeParams(winnerInfo.params), target, buf.srcSize); @@ -2081,16 +2054,11 @@ static winnerInfo_t climbOnce(const constraint_t target, CHECKTIME(winnerInfo); candidateInfo.params = cparam; /* param error checking already done here */ - paramVariation(&candidateInfo.params, varArray, varLen, dist); - - if(strat != candidateInfo.params.vals[strt_ind]) { - strat = candidateInfo.params.vals[strt_ind]; - varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat); - } + paramVariation(&candidateInfo.params, mtAll, (U32)dist); res = benchMemo(&candidateInfo.result, buf, ctx, - sanitizeParams(candidateInfo.params), target, &winnerInfo.result, memoTableArray[strat], - varNew, varLenNew, feas); + sanitizeParams(candidateInfo.params), target, &winnerInfo.result, mtAll, feas); + DEBUGOUTPUT("Res: %d\n", res); if(res == BETTER_RESULT) { /* synonymous with better in this case*/ winnerInfo = candidateInfo; BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, sanitizeParams(winnerInfo.params), target, buf.srcSize); @@ -2134,11 +2102,8 @@ static winnerInfo_t optimizeFixedStrategy( const buffers_t buf, const contexts_t ctx, const constraint_t target, paramValues_t paramTarget, const ZSTD_strategy strat, - const varInds_t* varArray, const int varLen, - U8** memoTableArray, const int tries) { + memoTable_t* memoTableArray, const int tries) { int i = 0; - varInds_t varNew[NUM_PARAMS]; - int varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat); paramValues_t init; winnerInfo_t winnerInfo, candidateInfo; @@ -2150,14 +2115,15 @@ static winnerInfo_t optimizeFixedStrategy( init = paramTarget; - while(i < tries) { + for(i = 0; i < tries; i++) { DEBUGOUTPUT("Restart\n"); - randomConstrainedParams(&init, varNew, varLenNew, memoTableArray[strat]); - candidateInfo = climbOnce(target, varArray, varLen, strat, memoTableArray, buf, ctx, init); + do { randomConstrainedParams(&init, memoTableArray, strat); } while(redundantParams(init, target, buf.maxBlockSize)); //only non-redundant params + candidateInfo = climbOnce(target, memoTableArray, buf, ctx, init); if(compareResultLT(winnerInfo.result, candidateInfo.result, target, buf.srcSize)) { winnerInfo = candidateInfo; BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize); i = 0; + continue; } CHECKTIME(winnerInfo); i++; @@ -2217,9 +2183,9 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ { varInds_t varArray [NUM_PARAMS]; int ret = 0; - const int varLen = variableParams(paramTarget, varArray, dictFileName != NULL); + const size_t varLen = variableParams(paramTarget, varArray, dictFileName != NULL); winnerInfo_t winner = initWinnerInfo(emptyParams()); - U8** allMT = NULL; + memoTable_t* allMT = NULL; paramValues_t paramBase; contexts_t ctx; buffers_t buf; @@ -2246,38 +2212,19 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ optimizerAdjustInput(¶mTarget, buf.maxBlockSize); paramBase = cParamUnsetMin(paramTarget); - /* if strategy is fixed, only init that part of memotable */ - if(paramTarget.vals[strt_ind] != PARAM_UNSET) { - varInds_t varNew[NUM_PARAMS]; - int varLenNew = sanitizeVarArray(varNew, varLen, varArray, paramTarget.vals[strt_ind]); - allMT = (U8**)calloc(sizeof(U8*), (ZSTD_btultra + 1)); - if(allMT == NULL) { - ret = 57; - goto _cleanUp; - } + // TODO: if strategy is fixed, only init that row + allMT = createMemoTableArray(varArray, varLen); - allMT[paramTarget.vals[strt_ind]] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew)); - - if(allMT[paramTarget.vals[strt_ind]] == NULL) { - ret = 58; - goto _cleanUp; - } - - initMemoTable(allMT[paramTarget.vals[strt_ind]], paramTarget, target, varNew, varLenNew, buf.maxBlockSize); - } else { - allMT = createMemoTableArray(paramTarget, target, varArray, varLen, buf.maxBlockSize); - } - if(!allMT) { DISPLAY("MemoTable Init Error\n"); ret = 2; goto _cleanUp; } - /* default strictness = Maximum for */ - if(g_strictness == DEFAULT_STRICTNESS) { + /* default strictnesses */ + if(g_strictness == PARAM_UNSET) { if(g_optmode) { - g_strictness = 99; + g_strictness = 100; } else { g_strictness = 90; } @@ -2371,7 +2318,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } DEBUGOUTPUT("Real Opt\n"); - /* start 'real' tests */ + /* start 'real' optimization */ { int bestStrategy = (int)winner.params.vals[strt_ind]; if(paramTarget.vals[strt_ind] == PARAM_UNSET) { @@ -2380,8 +2327,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ { /* one iterations of hill climbing with the level-defined parameters. */ - winnerInfo_t w1 = climbOnce(target, varArray, varLen, st, allMT, - buf, ctx, winner.params); + winnerInfo_t w1 = climbOnce(target, allMT, buf, ctx, winner.params); if(compareResultLT(winner.result, w1.result, target, buf.srcSize)) { winner = w1; } @@ -2392,8 +2338,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ winnerInfo_t wc; DEBUGOUTPUT("StrategySwitch: %s\n", g_stratName[st]); - wc = optimizeFixedStrategy(buf, ctx, target, paramBase, - st, varArray, varLen, allMT, tries); + wc = optimizeFixedStrategy(buf, ctx, target, paramBase, st, allMT, tries); if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) { winner = wc; @@ -2406,8 +2351,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ CHECKTIMEGT(ret, 0, _cleanUp); } } else { - winner = optimizeFixedStrategy(buf, ctx, target, paramBase, paramTarget.vals[strt_ind], - varArray, varLen, allMT, g_maxTries); + winner = optimizeFixedStrategy(buf, ctx, target, paramBase, paramTarget.vals[strt_ind], allMT, g_maxTries); } } @@ -2467,6 +2411,22 @@ static unsigned readU32FromChar(const char** stringPtr) return result * sign; } +static double readDoubleFromChar(const char** stringPtr) +{ + double result = 0, divide = 10; + while ((**stringPtr >='0') && (**stringPtr <='9')) { + result *= 10, result += **stringPtr - '0', (*stringPtr)++ ; + } + if(**stringPtr!='.') { + return result; + } + (*stringPtr)++; + while ((**stringPtr >='0') && (**stringPtr <='9')) { + result += (double)(**stringPtr - '0') / divide, divide *= 10, (*stringPtr)++ ; + } + return result; +} + static int usage(const char* exename) { DISPLAY( "Usage :\n"); @@ -2558,11 +2518,10 @@ int main(int argc, const char** argv) PARSE_SUB_ARGS("decompressionSpeed=", "dSpeed=", target.dSpeed); PARSE_SUB_ARGS("compressionMemory=" , "cMem=", target.cMem); PARSE_SUB_ARGS("strict=", "stc=", g_strictness); - PARSE_SUB_ARGS("preferSpeed=", "prfSpd=", g_speedMultiplier); - PARSE_SUB_ARGS("preferRatio=", "prfRto=", g_ratioMultiplier); PARSE_SUB_ARGS("maxTries=", "tries=", g_maxTries); - PARSE_SUB_ARGS("memoLimit=", "memo=", g_memoLimit); + PARSE_SUB_ARGS("memoLimitLog=", "memLog=", g_memoTableLog); if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevelOpt = readU32FromChar(&argument); g_optmode = 1; if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "speedForRatio=") || longCommandWArg(&argument, "speedRatio=")) { g_ratioMultiplier = readDoubleFromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } DISPLAY("invalid optimization parameter \n"); return 1; @@ -2690,6 +2649,7 @@ int main(int argc, const char** argv) break; case 's': + argument++; seperateFiles = 1; break; From 3dcfe5cc2c19b5a4fd864c5d1c40aec4a9e5f1dc Mon Sep 17 00:00:00 2001 From: George Lu Date: Tue, 14 Aug 2018 14:24:05 -0700 Subject: [PATCH 44/55] begin display changes --- tests/paramgrill.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 4a7fc387a..c56f0a71c 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -53,6 +53,8 @@ static const int g_maxNbVariations = 64; * Macros **************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYLEVEL(n, ...) if(g_displayLevel >= n) { fprintf(stderr, __VA_ARGS__); } + #define TIMED 0 #ifndef DEBUG # define DEBUG 0 @@ -233,6 +235,7 @@ static U32 g_noSeed = 0; static paramValues_t g_params; /* Initialized at the beginning of main w/ emptyParams() function */ static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */ static U32 g_memoTableLog = PARAM_UNSET; +static U32 g_displayLevel = 3; typedef enum { directMap, @@ -1282,7 +1285,6 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res /* the table */ fprintf(f, "================================\n"); for(n = g_winners; n != NULL; n = n->next) { - fprintf(f, "\r%79s\r", ""); BMK_displayOneResult(f, n->res, srcSize); } fprintf(f, "================================\n"); @@ -2313,7 +2315,6 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winner.result, winner.params, target, buf.srcSize); - BMK_translateAdvancedParams(stdout, winner.params); } } @@ -2364,7 +2365,6 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } /* end summary */ BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winner.result, winner.params, target, buf.srcSize); - BMK_translateAdvancedParams(stdout, winner.params); DISPLAY("grillParams size - optimizer completed \n"); } @@ -2653,6 +2653,14 @@ int main(int argc, const char** argv) seperateFiles = 1; break; + case 'q': + g_displayLevel--; + break; + + case 'v': + g_displayLevel++; + break; + /* load dictionary file (only applicable for optimizer rn) */ case 'D': if(i == argc - 1) { /* last argument, return error. */ From 4d9c6f51b8f34970fc292d82f23a2fa7f3bdede0 Mon Sep 17 00:00:00 2001 From: George Lu Date: Tue, 14 Aug 2018 15:54:07 -0700 Subject: [PATCH 45/55] -q -v options --- tests/paramgrill.c | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index c56f0a71c..ef6d6d46a 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -1257,22 +1257,22 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res /* global winner used for constraints */ /* cSize, cSpeed, dSpeed, cMem */ static winnerInfo_t g_winner = { { (size_t)-1LL, 0, 0, (size_t)-1LL }, { { PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET } } }; - if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { + if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize) || g_displayLevel >= 4) { if(DEBUG && compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { DISPLAY("New Winner: \n"); } - BMK_printWinner(f, cLevel, result, params, srcSize); + if(g_displayLevel >= 2) { BMK_printWinner(f, cLevel, result, params, srcSize); } if(compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { - BMK_translateAdvancedParams(f, params); + if(g_displayLevel >= 1) { BMK_translateAdvancedParams(f, params); } g_winner.result = result; g_winner.params = params; } } //prints out tradeoff table if using lvloptimize - if(g_optmode && g_optimizer) { + if(g_optmode && g_optimizer && (DEBUG || g_displayLevel == 3)) { winnerInfo_t w; winner_ll_node* n; w.result = result; @@ -2205,9 +2205,9 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } if(nbFiles == 1) { - DISPLAY("Loading %s... \r", fileNamesTable[0]); + DISPLAYLEVEL(2, "Loading %s... \r", fileNamesTable[0]); } else { - DISPLAY("Loading %lu Files... \r", (unsigned long)nbFiles); + DISPLAYLEVEL(2, "Loading %lu Files... \r", (unsigned long)nbFiles); } /* sanitize paramTarget */ @@ -2273,16 +2273,16 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } /* bench */ - DISPLAY("\r%79s\r", ""); + DISPLAYLEVEL(2, "\r%79s\r", ""); if(nbFiles == 1) { - DISPLAY("optimizing for %s", fileNamesTable[0]); + DISPLAYLEVEL(2, "optimizing for %s", fileNamesTable[0]); } else { - DISPLAY("optimizing for %lu Files", (unsigned long)nbFiles); + DISPLAYLEVEL(2, "optimizing for %lu Files", (unsigned long)nbFiles); } - if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed >> 20); } - if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed >> 20); } - if(target.cMem != (U32)-1) { DISPLAY(" - limit memory %u MB", target.cMem >> 20); } + if(target.cSpeed != 0) { DISPLAYLEVEL(2," - limit compression speed %u MB/s", target.cSpeed >> 20); } + if(target.dSpeed != 0) { DISPLAYLEVEL(2, " - limit decompression speed %u MB/s", target.dSpeed >> 20); } + if(target.cMem != (U32)-1) { DISPLAYLEVEL(2, " - limit memory %u MB", target.cMem >> 20); } DISPLAY("\n"); findClockGranularity(); @@ -2364,7 +2364,8 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ goto _cleanUp; } /* end summary */ - BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winner.result, winner.params, target, buf.srcSize); + BMK_displayOneResult(stdout, winner, buf.srcSize); + BMK_translateAdvancedParams(stdout, winner.params); DISPLAY("grillParams size - optimizer completed \n"); } @@ -2501,7 +2502,7 @@ int main(int argc, const char** argv) assert(argc>=1); /* for exename */ /* Welcome message */ - DISPLAY(WELCOME_MESSAGE); + DISPLAYLEVEL(2, WELCOME_MESSAGE); for(i=1; i Date: Tue, 14 Aug 2018 16:51:39 -0700 Subject: [PATCH 46/55] Clean up repetitive display Add documentation --- tests/README.md | 2 ++ tests/paramgrill.c | 28 +++++++++++++--------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/README.md b/tests/README.md index bdc9fff97..60fcebfbc 100644 --- a/tests/README.md +++ b/tests/README.md @@ -127,6 +127,8 @@ Full list of arguments -v : Prints Benchmarking output -D : Next argument dictionary file -s : Benchmark all files separately + -q : Quiet, repeat for more quiet + -v : Verbose, cancels quiet, repeat for more volume ``` Any inputs afterwards are treated as files to benchmark. diff --git a/tests/paramgrill.c b/tests/paramgrill.c index ef6d6d46a..ef005dd9a 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -235,7 +235,7 @@ static U32 g_noSeed = 0; static paramValues_t g_params; /* Initialized at the beginning of main w/ emptyParams() function */ static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */ static U32 g_memoTableLog = PARAM_UNSET; -static U32 g_displayLevel = 3; +static int g_displayLevel = 3; typedef enum { directMap, @@ -889,7 +889,7 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab goto _cleanUp; } - DISPLAY("Loading %s... \r", fileNamesTable[n]); + DISPLAYLEVEL(2, "Loading %s... \r", fileNamesTable[n]); if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, nbFiles=n; /* buffer too small - stop after this file */ { @@ -2020,7 +2020,6 @@ static winnerInfo_t climbOnce(const constraint_t target, better = 0; DEBUGOUTPUT("Start\n"); cparam = winnerInfo.params; - BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize); candidateInfo.params = cparam; /* all dist-1 candidates */ for(i = 0; i < varLen; i++) { @@ -2036,7 +2035,6 @@ static winnerInfo_t climbOnce(const constraint_t target, DEBUGOUTPUT("Res: %d\n", res); if(res == BETTER_RESULT) { /* synonymous with better when called w/ infeasibleBM */ winnerInfo = candidateInfo; - BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, sanitizeParams(winnerInfo.params), target, buf.srcSize); better = 1; if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) { bestFeasible1 = winnerInfo; @@ -2063,7 +2061,6 @@ static winnerInfo_t climbOnce(const constraint_t target, DEBUGOUTPUT("Res: %d\n", res); if(res == BETTER_RESULT) { /* synonymous with better in this case*/ winnerInfo = candidateInfo; - BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, sanitizeParams(winnerInfo.params), target, buf.srcSize); better = 1; if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) { bestFeasible1 = winnerInfo; @@ -2284,7 +2281,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ if(target.dSpeed != 0) { DISPLAYLEVEL(2, " - limit decompression speed %u MB/s", target.dSpeed >> 20); } if(target.cMem != (U32)-1) { DISPLAYLEVEL(2, " - limit memory %u MB", target.cMem >> 20); } - DISPLAY("\n"); + DISPLAYLEVEL(2, "\n"); findClockGranularity(); { @@ -2309,7 +2306,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ winner.params = CParams; } - CHECKTIMEGT(ret, 0, _cleanUp); /* if pass time limit, stop */ + CHECKTIMEGT(ret, 0, _displayCleanUp); /* if pass time limit, stop */ /* if the current params are too slow, just stop. */ if(target.cSpeed > candidate.cSpeed * 3 / 2) { break; } } @@ -2332,7 +2329,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ if(compareResultLT(winner.result, w1.result, target, buf.srcSize)) { winner = w1; } - CHECKTIMEGT(ret, 0, _cleanUp); + CHECKTIMEGT(ret, 0, _displayCleanUp); } while(st && tries > 0) { @@ -2349,7 +2346,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ st = nextStrategy(st, bestStrategy); tries -= TRY_DECAY; } - CHECKTIMEGT(ret, 0, _cleanUp); + CHECKTIMEGT(ret, 0, _displayCleanUp); } } else { winner = optimizeFixedStrategy(buf, ctx, target, paramBase, paramTarget.vals[strt_ind], allMT, g_maxTries); @@ -2364,12 +2361,13 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ goto _cleanUp; } /* end summary */ - BMK_displayOneResult(stdout, winner, buf.srcSize); +_displayCleanUp: + if(g_displayLevel >= 0) { BMK_displayOneResult(stdout, winner, buf.srcSize); } BMK_translateAdvancedParams(stdout, winner.params); - DISPLAY("grillParams size - optimizer completed \n"); + DISPLAYLEVEL(1, "grillParams size - optimizer completed \n"); } -_cleanUp: +_cleanUp: freeContexts(ctx); freeBuffers(buf); freeMemoTableArray(allMT); @@ -2501,9 +2499,6 @@ int main(int argc, const char** argv) assert(argc>=1); /* for exename */ - /* Welcome message */ - DISPLAYLEVEL(2, WELCOME_MESSAGE); - for(i=1; i Date: Tue, 14 Aug 2018 18:04:58 -0700 Subject: [PATCH 47/55] silencing params --- tests/README.md | 1 + tests/paramgrill.c | 70 ++++++++++++++++++++++++++++++++++++---------- 2 files changed, 56 insertions(+), 15 deletions(-) diff --git a/tests/README.md b/tests/README.md index 60fcebfbc..3af08f7be 100644 --- a/tests/README.md +++ b/tests/README.md @@ -122,6 +122,7 @@ Full list of arguments tries= : Maximum number of random restarts on a single strategy before switching (Default 3) Higher values will make optimizer run longer, more chances to find better solution. memLog : Limits the log of the size of each memotable (1 per strategy). Setting memLog = 0 turns off memoization + --display= : which params to display, uses all --zstd parameter names and 'cParams' to display only compression parameters. -P# : generated sample compressibility -t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) -v : Prints Benchmarking output diff --git a/tests/paramgrill.c b/tests/paramgrill.c index ef005dd9a..2f00c198c 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -237,6 +237,8 @@ static UTIL_time_t g_time; /* to be used to compare solution finding speeds to c static U32 g_memoTableLog = PARAM_UNSET; static int g_displayLevel = 3; +static BYTE g_silenceParams[NUM_PARAMS]; + typedef enum { directMap, xxhashMap, @@ -447,31 +449,34 @@ static paramValues_t cParamUnsetMin(paramValues_t paramTarget) { } static void BMK_translateAdvancedParams(FILE* f, const paramValues_t params) { - U32 i; + varInds_t v; + int first = 1; fprintf(f,"--zstd="); - for(i = 0; i < NUM_PARAMS; i++) { - fprintf(f,"%s=", g_paramNames[i]); + for(v = 0; v < NUM_PARAMS; v++) { + if(g_silenceParams[v]) { continue; } + if(!first) { fprintf(f, ","); } + fprintf(f,"%s=", g_paramNames[v]); - if(i == strt_ind) { fprintf(f,"%u", params.vals[i]); } - else { displayParamVal(f, i, params.vals[i], 0); } - - if(i != NUM_PARAMS - 1) { - fprintf(f, ","); - } + if(v == strt_ind) { fprintf(f,"%u", params.vals[v]); } + else { displayParamVal(f, v, params.vals[v], 0); } + first = 0; } fprintf(f, "\n"); } static void BMK_displayOneResult(FILE* f, winnerInfo_t res, const size_t srcSize) { varInds_t v; + int first = 1; res.params = cParamUnsetMin(res.params); fprintf(f," {"); for(v = 0; v < NUM_PARAMS; v++) { - if(v != 0) { fprintf(f, ","); } + if(g_silenceParams[v]) { continue; } + if(!first) { fprintf(f, ","); } displayParamVal(f, v, res.params.vals[v], 3); + first = 0; } - fprintf(f, "}, /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", + fprintf(f, " }, /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", (double)srcSize / res.result.cSize, (double)res.result.cSpeed / (1 MB), (double)res.result.dSpeed / (1 MB)); } @@ -2545,6 +2550,43 @@ int main(int argc, const char** argv) } continue; /* if not return, success */ + + } else if (longCommandWArg(&argument, "--display=")) { + /* Decode command (note : aggregated commands are allowed) */ + memset(g_silenceParams, 1, sizeof(g_silenceParams)); + for ( ; ;) { + int found = 0; + varInds_t v; + for(v = 0; v < NUM_PARAMS; v++) { + if(longCommandWArg(&argument, g_shortParamNames[v]) || longCommandWArg(&argument, g_paramNames[v])) { + g_silenceParams[v] = 0; + found = 1; + } + } + if(longCommandWArg(&argument, "compressionParameters") || longCommandWArg(&argument, "cParams")) { + for(v = 0; v <= strt_ind; v++) { + g_silenceParams[v] = 0; + } + found = 1; + } + + + if(found) { + if(argument[0]==',') { + continue; + } else { + break; + } + } + DISPLAY("invalid parameter name parameter \n"); + return 1; + } + + if (argument[0] != 0) { + DISPLAY("invalid --display format\n"); + return 1; /* check the end of string */ + } + continue; } else if (argument[0]=='-') { argument++; @@ -2650,13 +2692,11 @@ int main(int argc, const char** argv) break; case 'q': - argument++; - g_displayLevel--; + while (argument[0] == 'q') { argument++; g_displayLevel--; } break; case 'v': - argument++; - g_displayLevel++; + while (argument[0] == 'v') { argument++; g_displayLevel++; } break; /* load dictionary file (only applicable for optimizer rn) */ From ee77ddc28ddb0ff9bae63ee61e58a3ecfb315728 Mon Sep 17 00:00:00 2001 From: George Lu Date: Wed, 15 Aug 2018 11:46:19 -0700 Subject: [PATCH 48/55] Fix wraparound --- tests/paramgrill.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 2f00c198c..0d814bf7c 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -144,8 +144,8 @@ static const char* g_shortParamNames[NUM_PARAMS] = { "wlog", "clog", "hlog","slog", "slen", "tlen", "strt", "fadt" }; /* maps value from { 0 to rangetable[param] - 1 } to valid paramvalues */ -static U32 rangeMap(varInds_t param, U32 ind) { - ind = MIN(ind, rangetable[param] - 1); +static U32 rangeMap(varInds_t param, int ind) { + ind = MAX(MIN(ind, (int)rangetable[param] - 1), 0); switch(param) { case tlen_ind: return tlen_table[ind]; @@ -166,7 +166,7 @@ static U32 rangeMap(varInds_t param, U32 ind) { } /* inverse of rangeMap */ -static U32 invRangeMap(varInds_t param, U32 value) { +static int invRangeMap(varInds_t param, U32 value) { value = MIN(MAX(mintable[param], value), maxtable[param]); switch(param) { case tlen_ind: /* bin search */ @@ -186,7 +186,7 @@ static U32 invRangeMap(varInds_t param, U32 value) { return lo; } case fadt_ind: - return value + 1; + return (int)value + 1; case wlog_ind: case clog_ind: case hlog_ind: @@ -196,7 +196,7 @@ static U32 invRangeMap(varInds_t param, U32 value) { return value - mintable[param]; case NUM_PARAMS: DISPLAY("Error, not a valid param\n "); - return (U32)-1; + return -2; } return 0; /* should never happen, stop compiler warnings */ } @@ -1545,7 +1545,7 @@ static unsigned memoTableIndDirect(const paramValues_t* ptr, const varInds_t* va for(i = 0; i < varyLen; i++) { varInds_t v = varyParams[i]; if(v == strt_ind) continue; /* exclude strategy from memotable */ - ind *= rangetable[v]; ind += invRangeMap(v, ptr->vals[v]); + ind *= rangetable[v]; ind += (unsigned)invRangeMap(v, ptr->vals[v]); } return ind; } From b234870c33745401138af634909acf46ec3363fc Mon Sep 17 00:00:00 2001 From: George Lu Date: Wed, 15 Aug 2018 14:29:49 -0700 Subject: [PATCH 49/55] clarify display README --- tests/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index 3af08f7be..442285dbc 100644 --- a/tests/README.md +++ b/tests/README.md @@ -122,7 +122,10 @@ Full list of arguments tries= : Maximum number of random restarts on a single strategy before switching (Default 3) Higher values will make optimizer run longer, more chances to find better solution. memLog : Limits the log of the size of each memotable (1 per strategy). Setting memLog = 0 turns off memoization - --display= : which params to display, uses all --zstd parameter names and 'cParams' to display only compression parameters. + --display= : specifiy which parameters are included in the output + can use all --zstd parameter names and 'cParams' as a shorthand for all parameters used in ZSTD_compressionParameters + (Default: display all params available) + -P# : generated sample compressibility -t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) -v : Prints Benchmarking output From 46be2ef5d8f48b6509e6813f74f3622db0fbc09f Mon Sep 17 00:00:00 2001 From: George Lu Date: Wed, 15 Aug 2018 14:00:57 -0700 Subject: [PATCH 50/55] Remove unused stuff --- tests/README.md | 3 +- tests/paramgrill.c | 164 ++++++++++++++++++++------------------------- 2 files changed, 73 insertions(+), 94 deletions(-) diff --git a/tests/README.md b/tests/README.md index 442285dbc..c1128571a 100644 --- a/tests/README.md +++ b/tests/README.md @@ -95,7 +95,6 @@ Full list of arguments ``` -T# : set level 1 speed objective -B# : cut input into blocks of size # (default : single block) - -i# : iteration loops -S : benchmarks a single run (example command: -Sl3w10h12) w# - windowLog h# - hashLog @@ -119,7 +118,7 @@ Full list of arguments speedRatio= (accepts decimals) : determines value of gains in speed vs gains in ratio when determining overall winner (default 5 (1% ratio = 5% speed)). - tries= : Maximum number of random restarts on a single strategy before switching (Default 3) + tries= : Maximum number of random restarts on a single strategy before switching (Default 5) Higher values will make optimizer run longer, more chances to find better solution. memLog : Limits the log of the size of each memotable (1 per strategy). Setting memLog = 0 turns off memoization --display= : specifiy which parameters are included in the output diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 0d814bf7c..e5e1e2a36 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -37,9 +37,6 @@ #define WELCOME_MESSAGE "*** %s %s %i-bits, by %s ***\n", PROGRAM_DESCRIPTION, ZSTD_VERSION_STRING, (int)(sizeof(void*)*8), AUTHOR #define TIMELOOP_NANOSEC (1*1000000000ULL) /* 1 second */ - -#define NBLOOPS 2 -#define TIMELOOP (2 * SEC_TO_MICRO) #define NB_LEVELS_TRACKED 22 /* ensured being >= ZSTD_maxCLevel() in BMK_init_level_constraints() */ static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31)); @@ -54,12 +51,12 @@ static const int g_maxNbVariations = 64; **************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define DISPLAYLEVEL(n, ...) if(g_displayLevel >= n) { fprintf(stderr, __VA_ARGS__); } +#define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); } #define TIMED 0 #ifndef DEBUG # define DEBUG 0 #endif -#define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); } #undef MIN #undef MAX @@ -223,21 +220,30 @@ static void displayParamVal(FILE* f, varInds_t param, U32 value, int width) { typedef BYTE U8; +/* General Utility */ static U32 g_timeLimit_s = 99999; /* about 27 hours */ -static U32 g_nbIterations = NBLOOPS; -static double g_compressibility = COMPRESSIBILITY_DEFAULT; +static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */ static U32 g_blockSize = 0; static U32 g_rand = 1; + +/* Display */ +static int g_displayLevel = 3; +static BYTE g_silenceParams[NUM_PARAMS]; + +/* Mode Selection */ static U32 g_singleRun = 0; static U32 g_optimizer = 0; +static int g_optmode = 0; + +/* For cLevel Table generation */ static U32 g_target = 0; static U32 g_noSeed = 0; -static paramValues_t g_params; /* Initialized at the beginning of main w/ emptyParams() function */ -static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */ -static U32 g_memoTableLog = PARAM_UNSET; -static int g_displayLevel = 3; -static BYTE g_silenceParams[NUM_PARAMS]; +/* For optimizer */ +static paramValues_t g_params; /* Initialized at the beginning of main w/ emptyParams() function */ +static double g_ratioMultiplier = 5.; +static U32 g_strictness = PARAM_UNSET; /* range 0 - 99, measure of how strict */ +static BMK_result_t g_lvltarget; typedef enum { directMap, @@ -258,11 +264,6 @@ typedef struct { paramValues_t params; } winnerInfo_t; -typedef struct { - BMK_result_t result; - ZSTD_compressionParameters params; -} oldWinnerInfo_t; - typedef struct { U32 cSpeed; /* bytes / sec */ U32 dSpeed; @@ -276,27 +277,13 @@ struct winner_ll_node { }; static winner_ll_node* g_winners; /* linked list sorted ascending by cSize & cSpeed */ -static BMK_result_t g_lvltarget; -static int g_optmode = 0; - -static double g_ratioMultiplier = 5.; - -/* g_mode? */ - -/* range 0 - 99, measure of how strict */ -static U32 g_strictness = PARAM_UNSET; - -void BMK_SetNbIterations(int nbLoops) -{ - g_nbIterations = nbLoops; - DISPLAY("- %u iterations -\n", g_nbIterations); -} /* * Additional Global Variables (Defined Above Use) * g_level_constraint * g_alreadyTested * g_maxTries + * g_clockGranularity */ /*-******************************************************* @@ -340,7 +327,7 @@ static paramValues_t adjustParams(paramValues_t p, const size_t maxBlockSize, co paramValues_t ot = p; varInds_t i; p = cParamsToPVals(ZSTD_adjustCParams(pvalsToCParams(p), maxBlockSize, dictSize)); - + if(!dictSize) { p.vals[fadt_ind] = 0; } /* retain value of all other parameters */ for(i = strt_ind + 1; i < NUM_PARAMS; i++) { p.vals[i] = ot.vals[i]; @@ -1306,7 +1293,7 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res } } -static void BMK_printWinners2(FILE* f, const oldWinnerInfo_t* winners, const size_t srcSize) +static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, const size_t srcSize) { int cLevel; @@ -1314,11 +1301,11 @@ static void BMK_printWinners2(FILE* f, const oldWinnerInfo_t* winners, const siz fprintf(f, " /* W, C, H, S, L, T, strat */ \n"); for (cLevel=0; cLevel <= NB_LEVELS_TRACKED; cLevel++) - BMK_printWinner(f, cLevel, winners[cLevel].result, cParamsToPVals(winners[cLevel].params), srcSize); + BMK_printWinner(f, cLevel, winners[cLevel].result, winners[cLevel].params, srcSize); } -static void BMK_printWinners(FILE* f, const oldWinnerInfo_t* winners, const size_t srcSize) +static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, const size_t srcSize) { fseek(f, 0, SEEK_SET); BMK_printWinners2(f, winners, srcSize); @@ -1355,14 +1342,14 @@ static void BMK_init_level_constraints(int bytePerSec_level1) } } } -static int BMK_seed(oldWinnerInfo_t* winners, const ZSTD_compressionParameters params, +static int BMK_seed(winnerInfo_t* winners, const paramValues_t params, const buffers_t buf, const contexts_t ctx) { BMK_result_t testResult; int better = 0; int cLevel; - BMK_benchParam(&testResult, buf, ctx, cParamsToPVals(params)); + BMK_benchParam(&testResult, buf, ctx, params); for (cLevel = 1; cLevel <= NB_LEVELS_TRACKED; cLevel++) { @@ -1370,15 +1357,15 @@ static int BMK_seed(oldWinnerInfo_t* winners, const ZSTD_compressionParameters p continue; /* not fast enough for this level */ if (testResult.dSpeed < g_level_constraint[cLevel].dSpeed_min) continue; /* not fast enough for this level */ - if (params.windowLog > g_level_constraint[cLevel].windowLog_max) + if (params.vals[wlog_ind] > g_level_constraint[cLevel].windowLog_max) continue; /* too much memory for this level */ - if (params.strategy > g_level_constraint[cLevel].strategy_max) + if (params.vals[strt_ind] > g_level_constraint[cLevel].strategy_max) continue; /* forbidden strategy for this level */ if (winners[cLevel].result.cSize==0) { /* first solution for this cLevel */ winners[cLevel].result = testResult; winners[cLevel].params = params; - BMK_printWinner(stdout, cLevel, testResult, cParamsToPVals(params), buf.srcSize); + BMK_printWinner(stdout, cLevel, testResult, params, buf.srcSize); better = 1; continue; } @@ -1389,13 +1376,13 @@ static int BMK_seed(oldWinnerInfo_t* winners, const ZSTD_compressionParameters p double O_ratio = (double)buf.srcSize / winners[cLevel].result.cSize; double W_ratioNote = log (W_ratio); double O_ratioNote = log (O_ratio); - size_t W_DMemUsed = (1 << params.windowLog) + (16 KB); - size_t O_DMemUsed = (1 << winners[cLevel].params.windowLog) + (16 KB); + size_t W_DMemUsed = (1 << params.vals[wlog_ind]) + (16 KB); + size_t O_DMemUsed = (1 << winners[cLevel].params.vals[wlog_ind]) + (16 KB); double W_DMemUsed_note = W_ratioNote * ( 40 + 9*cLevel) - log((double)W_DMemUsed); double O_DMemUsed_note = O_ratioNote * ( 40 + 9*cLevel) - log((double)O_DMemUsed); - size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_estimateCCtxSize_usingCParams(params); - size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_estimateCCtxSize_usingCParams(winners[cLevel].params); + size_t W_CMemUsed = (1 << params.vals[wlog_ind]) + ZSTD_estimateCCtxSize_usingCParams(pvalsToCParams(params)); + size_t O_CMemUsed = (1 << winners[cLevel].params.vals[wlog_ind]) + ZSTD_estimateCCtxSize_usingCParams(pvalsToCParams(winners[cLevel].params)); double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed); double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed); @@ -1443,7 +1430,7 @@ static int BMK_seed(oldWinnerInfo_t* winners, const ZSTD_compressionParameters p winners[cLevel].result = testResult; winners[cLevel].params = params; - BMK_printWinner(stdout, cLevel, testResult, cParamsToPVals(params), buf.srcSize); + BMK_printWinner(stdout, cLevel, testResult, params, buf.srcSize); better = 1; } } @@ -1587,7 +1574,7 @@ static void freeMemoTableArray(memoTable_t* const mtAll) { /* inits memotables for all (including mallocs), all strategies */ /* takes unsanitized varyParams */ -static memoTable_t* createMemoTableArray(const varInds_t* const varyParams, const size_t varyLen) { +static memoTable_t* createMemoTableArray(const varInds_t* const varyParams, const size_t varyLen, U32 memoTableLog) { memoTable_t* mtAll = (memoTable_t*)calloc(sizeof(memoTable_t),(ZSTD_btultra + 1)); int i; @@ -1600,7 +1587,7 @@ static memoTable_t* createMemoTableArray(const varInds_t* const varyParams, cons } /* no memoization */ - if(g_memoTableLog == 0) { + if(memoTableLog == 0) { for(i = 1; i <= (int)ZSTD_btultra; i++) { mtAll[i].tableType = noMemo; mtAll[i].table = NULL; @@ -1614,9 +1601,9 @@ static memoTable_t* createMemoTableArray(const varInds_t* const varyParams, cons size_t mtl = memoTableLen(mtAll[i].varArray, mtAll[i].varLen); mtAll[i].tableType = directMap; - if(g_memoTableLog != PARAM_UNSET && mtl > (1ULL << g_memoTableLog)) { /* use hash table */ /* provide some option to only use hash tables? */ + if(memoTableLog != PARAM_UNSET && mtl > (1ULL << memoTableLog)) { /* use hash table */ /* provide some option to only use hash tables? */ mtAll[i].tableType = xxhashMap; - mtl = (1ULL << g_memoTableLog); + mtl = (1ULL << memoTableLog); } mtAll[i].table = (BYTE*)calloc(sizeof(BYTE), mtl); @@ -1646,21 +1633,19 @@ static paramValues_t overwriteParams(paramValues_t base, const paramValues_t mas #define PARAMTABLEMASK (PARAMTABLESIZE-1) static BYTE g_alreadyTested[PARAMTABLESIZE] = {0}; /* init to zero */ -static BYTE* NB_TESTS_PLAYED(ZSTD_compressionParameters p) { - ZSTD_compressionParameters p2 = pvalsToCParams(sanitizeParams(cParamsToPVals(p))); +static BYTE* NB_TESTS_PLAYED(paramValues_t p) { + ZSTD_compressionParameters p2 = pvalsToCParams(sanitizeParams(p)); return &g_alreadyTested[(XXH64((void*)&p2, sizeof(p2), 0) >> 3) & PARAMTABLEMASK]; } -static void playAround(FILE* f, oldWinnerInfo_t* winners, - ZSTD_compressionParameters params, +static void playAround(FILE* f, winnerInfo_t* winners, + paramValues_t p, const buffers_t buf, const contexts_t ctx) { int nbVariations = 0, i; UTIL_time_t const clockStart = UTIL_getTime(); while (UTIL_clockSpanMicro(clockStart) < g_maxVariationTime) { - paramValues_t p = cParamsToPVals(params); - ZSTD_compressionParameters p2; BYTE* b; if (nbVariations++ > g_maxNbVariations) break; @@ -1668,20 +1653,18 @@ static void playAround(FILE* f, oldWinnerInfo_t* winners, do { for(i = 0; i < 4; i++) { paramVaryOnce(FUZ_rand(&g_rand) % (strt_ind + 1), ((FUZ_rand(&g_rand) & 1) << 1) - 1, &p); } } while(!paramValid(p)); - p2 = pvalsToCParams(p); - /* exclude faster if already played params */ - if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(p2))-1)) + if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(p))-1)) continue; /* test */ - b = NB_TESTS_PLAYED(p2); + b = NB_TESTS_PLAYED(p); (*b)++; - if (!BMK_seed(winners, p2, buf, ctx)) continue; + if (!BMK_seed(winners, p, buf, ctx)) continue; /* improvement found => search more */ BMK_printWinners(f, winners, buf.srcSize); - playAround(f, winners, p2, buf, ctx); + playAround(f, winners, p, buf, ctx); } } @@ -1693,7 +1676,7 @@ static void randomConstrainedParams(paramValues_t* pc, const memoTable_t* memoTa size_t j; const memoTable_t mt = memoTableArray[st]; pc->vals[strt_ind] = st; - for(j = 0; j < MIN(1ULL << g_memoTableLog, memoTableLen(mt.varArray, mt.varLen)); j++) { + for(j = 0; j < mt.tableLen; j++) { int i; for(i = 0; i < NUM_PARAMS; i++) { varInds_t v = mt.varArray[i]; @@ -1706,23 +1689,24 @@ static void randomConstrainedParams(paramValues_t* pc, const memoTable_t* memoTa } /* Completely random parameter selection */ -static ZSTD_compressionParameters randomParams(void) +static paramValues_t randomParams(void) { - paramValues_t p; varInds_t v; - for(v = 0; v < NUM_PARAMS; v++) { + varInds_t v; paramValues_t p; + for(v = 0; v <= NUM_PARAMS; v++) { p.vals[v] = rangeMap(v, FUZ_rand(&g_rand) % rangetable[v]); } - return pvalsToCParams(p); + return p; } static void BMK_selectRandomStart( - FILE* f, oldWinnerInfo_t* winners, + FILE* f, winnerInfo_t* winners, const buffers_t buf, const contexts_t ctx) { U32 const id = FUZ_rand(&g_rand) % (NB_LEVELS_TRACKED+1); - if ((id==0) || (winners[id].params.windowLog==0)) { + if ((id==0) || (winners[id].params.vals[wlog_ind]==0)) { /* use some random entry */ - ZSTD_compressionParameters const p = ZSTD_adjustCParams(randomParams(), buf.srcSize, 0); + paramValues_t const p = adjustParams(cParamsToPVals(pvalsToCParams(randomParams())), /* defaults nonCompression parameters */ + buf.srcSize, 0); playAround(f, winners, p, buf, ctx); } else { playAround(f, winners, winners[id].params, buf, ctx); @@ -1731,8 +1715,8 @@ static void BMK_selectRandomStart( static void BMK_benchFullTable(const buffers_t buf, const contexts_t ctx) { - ZSTD_compressionParameters params; - oldWinnerInfo_t winners[NB_LEVELS_TRACKED+1]; + paramValues_t params; + winnerInfo_t winners[NB_LEVELS_TRACKED+1]; const char* const rfName = "grillResults.txt"; FILE* const f = fopen(rfName, "w"); @@ -1745,9 +1729,9 @@ static void BMK_benchFullTable(const buffers_t buf, const contexts_t ctx) BMK_init_level_constraints(g_target * (1 MB)); } else { /* baseline config for level 1 */ - ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, buf.maxBlockSize, ctx.dictSize); + paramValues_t const l1params = cParamsToPVals(ZSTD_getCParams(1, buf.maxBlockSize, ctx.dictSize)); BMK_result_t testResult; - BMK_benchParam(&testResult, buf, ctx, cParamsToPVals(l1params)); + BMK_benchParam(&testResult, buf, ctx, l1params); BMK_init_level_constraints((int)((testResult.cSpeed * 31) / 32)); } @@ -1755,7 +1739,7 @@ static void BMK_benchFullTable(const buffers_t buf, const contexts_t ctx) { const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); int i; for (i=0; i<=maxSeeds; i++) { - params = ZSTD_getCParams(i, buf.maxBlockSize, 0); + params = cParamsToPVals(ZSTD_getCParams(i, buf.maxBlockSize, 0)); BMK_seed(winners, params, buf, ctx); } } BMK_printWinners(f, winners, buf.srcSize); @@ -1788,7 +1772,7 @@ static int benchOnce(const buffers_t buf, const contexts_t ctx) { return 0; } -static int benchSample(void) +static int benchSample(double compressibility) { const char* const name = "Sample 10MB"; size_t const benchedSize = 10 MB; @@ -1803,7 +1787,7 @@ static int benchSample(void) return 2; } - RDG_genBuffer(srcBuffer, benchedSize, g_compressibility, 0.0, 0); + RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0); if(createBuffersFromMemory(&buf, srcBuffer, 1, &benchedSize)) { DISPLAY("Buffer Creation Error\n"); @@ -1819,7 +1803,7 @@ static int benchSample(void) /* bench */ DISPLAY("\r%79s\r", ""); - DISPLAY("using %s %i%%: \n", name, (int)(g_compressibility*100)); + DISPLAY("using %s %i%%: \n", name, (int)(compressibility*100)); if(g_singleRun) { ret = benchOnce(buf, ctx); @@ -2180,10 +2164,11 @@ static int nextStrategy(const int currentStrategy, const int bestStrategy) { * cLevel - compression level to exceed (all solutions must be > lvl in cSpeed + ratio) */ -static int g_maxTries = 3; +static int g_maxTries = 5; #define TRY_DECAY 1 -static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, paramValues_t paramTarget, int cLevelOpt, int cLevelRun) +static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, paramValues_t paramTarget, + int cLevelOpt, int cLevelRun, U32 memoTableLog) { varInds_t varArray [NUM_PARAMS]; int ret = 0; @@ -2217,7 +2202,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ paramBase = cParamUnsetMin(paramTarget); // TODO: if strategy is fixed, only init that row - allMT = createMemoTableArray(varArray, varLen); + allMT = createMemoTableArray(varArray, varLen, memoTableLog); if(!allMT) { DISPLAY("MemoTable Init Error\n"); @@ -2446,7 +2431,6 @@ static int usage_advanced(void) DISPLAY( "\nAdvanced options :\n"); DISPLAY( " -T# : set level 1 speed objective \n"); DISPLAY( " -B# : cut input into blocks of size # (default : single block) \n"); - DISPLAY( " -i# : iteration loops (default : %i) \n", NBLOOPS); DISPLAY( " --optimize= : same as -O with more verbose syntax (see README.md)\n"); DISPLAY( " -S : Single run \n"); DISPLAY( " --zstd : Single run, parameter selection same as zstdcli \n"); @@ -2497,6 +2481,8 @@ int main(int argc, const char** argv) U32 main_pause = 0; int cLevelOpt = 0, cLevelRun = 0; int seperateFiles = 0; + double compressibility = COMPRESSIBILITY_DEFAULT; + U32 memoTableLog = PARAM_UNSET; constraint_t target = { 0, 0, (U32)-1 }; paramValues_t paramTarget = emptyParams(); @@ -2520,7 +2506,7 @@ int main(int argc, const char** argv) PARSE_SUB_ARGS("compressionMemory=" , "cMem=", target.cMem); PARSE_SUB_ARGS("strict=", "stc=", g_strictness); PARSE_SUB_ARGS("maxTries=", "tries=", g_maxTries); - PARSE_SUB_ARGS("memoLimitLog=", "memLog=", g_memoTableLog); + PARSE_SUB_ARGS("memoLimitLog=", "memLog=", memoTableLog); if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevelOpt = readU32FromChar(&argument); g_optmode = 1; if (argument[0]==',') { argument++; continue; } else break; } if (longCommandWArg(&argument, "speedForRatio=") || longCommandWArg(&argument, "speedRatio=")) { g_ratioMultiplier = readDoubleFromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } @@ -2600,18 +2586,12 @@ int main(int argc, const char** argv) /* Pause at the end (hidden option) */ case 'p': main_pause = 1; argument++; break; - /* Modify Nb Iterations */ - - case 'i': - argument++; - g_nbIterations = readU32FromChar(&argument); - break; /* Sample compressibility (when no file provided) */ case 'P': argument++; { U32 const proba32 = readU32FromChar(&argument); - g_compressibility = (double)proba32 / 100.; + compressibility = (double)proba32 / 100.; } break; @@ -2730,13 +2710,13 @@ int main(int argc, const char** argv) DISPLAY("Optimizer Expects File\n"); return 1; } else { - result = benchSample(); + result = benchSample(compressibility); } } else { if(seperateFiles) { for(i = 0; i < argc - filenamesStart; i++) { if (g_optimizer) { - result = optimizeForSize(argv+filenamesStart + i, 1, dictFileName, target, paramTarget, cLevelOpt, cLevelRun); + result = optimizeForSize(argv+filenamesStart + i, 1, dictFileName, target, paramTarget, cLevelOpt, cLevelRun, memoTableLog); if(result) { DISPLAY("Error on File %d", i); return result; } } else { result = benchFiles(argv+filenamesStart + i, 1, dictFileName, cLevelRun); @@ -2745,7 +2725,7 @@ int main(int argc, const char** argv) } } else { if (g_optimizer) { - result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget, cLevelOpt, cLevelRun); + result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget, cLevelOpt, cLevelRun, memoTableLog); } else { result = benchFiles(argv+filenamesStart, argc-filenamesStart, dictFileName, cLevelRun); } From 3f8b10baa1b5e70530ec5d893ebaaf8cc9962d04 Mon Sep 17 00:00:00 2001 From: George Lu Date: Wed, 15 Aug 2018 14:27:07 -0700 Subject: [PATCH 51/55] consts --- tests/paramgrill.c | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index e5e1e2a36..1025e4323 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -1263,7 +1263,6 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res } } - //prints out tradeoff table if using lvloptimize if(g_optmode && g_optimizer && (DEBUG || g_displayLevel == 3)) { winnerInfo_t w; winner_ll_node* n; @@ -1574,9 +1573,9 @@ static void freeMemoTableArray(memoTable_t* const mtAll) { /* inits memotables for all (including mallocs), all strategies */ /* takes unsanitized varyParams */ -static memoTable_t* createMemoTableArray(const varInds_t* const varyParams, const size_t varyLen, U32 memoTableLog) { +static memoTable_t* createMemoTableArray(const paramValues_t p, const varInds_t* const varyParams, const size_t varyLen, const U32 memoTableLog) { memoTable_t* mtAll = (memoTable_t*)calloc(sizeof(memoTable_t),(ZSTD_btultra + 1)); - int i; + ZSTD_strategy i, stratMin = ZSTD_fast, stratMax = ZSTD_btultra; if(mtAll == NULL) { return NULL; @@ -1596,8 +1595,14 @@ static memoTable_t* createMemoTableArray(const varInds_t* const varyParams, cons return mtAll; } - /* hash table if normal table is too big */ - for(i = 1; i <= (int)ZSTD_btultra; i++) { + + if(p.vals[strt_ind] != PARAM_UNSET) { + stratMin = p.vals[strt_ind]; + stratMax = p.vals[strt_ind]; + } + + + for(i = stratMin; i <= stratMax; i++) { size_t mtl = memoTableLen(mtAll[i].varArray, mtAll[i].varLen); mtAll[i].tableType = directMap; @@ -1680,11 +1685,11 @@ static void randomConstrainedParams(paramValues_t* pc, const memoTable_t* memoTa int i; for(i = 0; i < NUM_PARAMS; i++) { varInds_t v = mt.varArray[i]; - if(v == strt_ind) continue; //skip, already specified + if(v == strt_ind) continue; pc->vals[v] = rangeMap(v, FUZ_rand(&g_rand) % rangetable[v]); } - if(!(memoTableGet(memoTableArray, *pc))) break; //only pick unpicked params. + if(!(memoTableGet(memoTableArray, *pc))) break; /* only pick unpicked params. */ } } @@ -2105,7 +2110,7 @@ static winnerInfo_t optimizeFixedStrategy( for(i = 0; i < tries; i++) { DEBUGOUTPUT("Restart\n"); - do { randomConstrainedParams(&init, memoTableArray, strat); } while(redundantParams(init, target, buf.maxBlockSize)); //only non-redundant params + do { randomConstrainedParams(&init, memoTableArray, strat); } while(redundantParams(init, target, buf.maxBlockSize)); candidateInfo = climbOnce(target, memoTableArray, buf, ctx, init); if(compareResultLT(winnerInfo.result, candidateInfo.result, target, buf.srcSize)) { winnerInfo = candidateInfo; @@ -2168,7 +2173,7 @@ static int g_maxTries = 5; #define TRY_DECAY 1 static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, paramValues_t paramTarget, - int cLevelOpt, int cLevelRun, U32 memoTableLog) + const int cLevelOpt, const int cLevelRun, const U32 memoTableLog) { varInds_t varArray [NUM_PARAMS]; int ret = 0; @@ -2201,8 +2206,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ optimizerAdjustInput(¶mTarget, buf.maxBlockSize); paramBase = cParamUnsetMin(paramTarget); - // TODO: if strategy is fixed, only init that row - allMT = createMemoTableArray(varArray, varLen, memoTableLog); + allMT = createMemoTableArray(paramTarget, varArray, varLen, memoTableLog); if(!allMT) { DISPLAY("MemoTable Init Error\n"); From 8a296d3e1f4f8a5177b084098170b122b7e33c26 Mon Sep 17 00:00:00 2001 From: George Lu Date: Wed, 15 Aug 2018 14:57:10 -0700 Subject: [PATCH 52/55] Move Stuff around Group similar functions together, remove outdated comments --- tests/paramgrill.c | 1443 ++++++++++++++++++++++---------------------- 1 file changed, 734 insertions(+), 709 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 1025e4323..7ebcdeec0 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -65,8 +65,6 @@ static const int g_maxNbVariations = 64; #define CUSTOM_LEVEL 99 #define BASE_CLEVEL 1 -#undef ZSTD_WINDOWLOG_MAX -#define ZSTD_WINDOWLOG_MAX 27 //no long range stuff for now. #define FADT_MIN 0 #define FADT_MAX ((U32)-1) @@ -242,7 +240,7 @@ static U32 g_noSeed = 0; /* For optimizer */ static paramValues_t g_params; /* Initialized at the beginning of main w/ emptyParams() function */ static double g_ratioMultiplier = 5.; -static U32 g_strictness = PARAM_UNSET; /* range 0 - 99, measure of how strict */ +static U32 g_strictness = PARAM_UNSET; /* range 1 - 100, measure of how strict */ static BMK_result_t g_lvltarget; typedef enum { @@ -287,9 +285,23 @@ static winner_ll_node* g_winners; /* linked list sorted ascending by cSize & cSp */ /*-******************************************************* -* Private functions +* General Util Functions *********************************************************/ +/* nullified useless params, to ensure count stats */ +/* cleans up params for memoizing / display */ +static paramValues_t sanitizeParams(paramValues_t params) +{ + if (params.vals[strt_ind] == ZSTD_fast) + params.vals[clog_ind] = 0, params.vals[slog_ind] = 0; + if (params.vals[strt_ind] == ZSTD_dfast) + params.vals[slog_ind] = 0; + if (params.vals[strt_ind] != ZSTD_btopt && params.vals[strt_ind] != ZSTD_btultra && params.vals[strt_ind] != ZSTD_fast) + params.vals[tlen_ind] = 0; + + return params; +} + static ZSTD_compressionParameters pvalsToCParams(paramValues_t p) { ZSTD_compressionParameters c; memset(&c, 0, sizeof(ZSTD_compressionParameters)); @@ -335,9 +347,6 @@ static paramValues_t adjustParams(paramValues_t p, const size_t maxBlockSize, co return p; } -/* accuracy in seconds only, span can be multiple years */ -static U32 BMK_timeSpan(const UTIL_time_t tStart) { return (U32)(UTIL_clockSpanMicro(tStart) / 1000000ULL); } - static size_t BMK_findMaxMem(U64 requiredMem) { size_t const step = 64 MB; @@ -356,6 +365,8 @@ static size_t BMK_findMaxMem(U64 requiredMem) return (size_t) requiredMem; } +/* accuracy in seconds only, span can be multiple years */ +static U32 BMK_timeSpan(const UTIL_time_t tStart) { return (U32)(UTIL_clockSpanMicro(tStart) / 1000000ULL); } static U32 FUZ_rotl32(U32 x, U32 r) { @@ -374,42 +385,6 @@ U32 FUZ_rand(U32* src) return rand32 >> 5; } -/** longCommandWArg() : - * check if *stringPtr is the same as longCommand. - * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand. - * @return 0 and doesn't modify *stringPtr otherwise. - * from zstdcli.c - */ -static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) -{ - size_t const comSize = strlen(longCommand); - int const result = !strncmp(*stringPtr, longCommand, comSize); - if (result) *stringPtr += comSize; - return result; -} - -static U64 g_clockGranularity = 100000000ULL; - -static void findClockGranularity(void) { - UTIL_time_t clockStart = UTIL_getTime(); - U64 el1 = 0, el2 = 0; - int i = 0; - do { - el1 = el2; - el2 = UTIL_clockSpanNano(clockStart); - if(el1 < el2) { - U64 iv = el2 - el1; - if(g_clockGranularity > iv) { - g_clockGranularity = iv; - i = 0; - } else { - i++; - } - } - } while(i < 10); - DEBUGOUTPUT("Granularity: %llu\n", (unsigned long long)g_clockGranularity); -} - /* allows zeros */ #define CLAMPCHECK(val,min,max) { \ if (((val)<(min)) | ((val)>(max))) { \ @@ -435,38 +410,95 @@ static paramValues_t cParamUnsetMin(paramValues_t paramTarget) { return paramTarget; } -static void BMK_translateAdvancedParams(FILE* f, const paramValues_t params) { - varInds_t v; - int first = 1; - fprintf(f,"--zstd="); - for(v = 0; v < NUM_PARAMS; v++) { - if(g_silenceParams[v]) { continue; } - if(!first) { fprintf(f, ","); } - fprintf(f,"%s=", g_paramNames[v]); - - if(v == strt_ind) { fprintf(f,"%u", params.vals[v]); } - else { displayParamVal(f, v, params.vals[v], 0); } - first = 0; +static paramValues_t emptyParams(void) { + U32 i; + paramValues_t p; + for(i = 0; i < NUM_PARAMS; i++) { + p.vals[i] = PARAM_UNSET; } - fprintf(f, "\n"); + return p; } -static void BMK_displayOneResult(FILE* f, winnerInfo_t res, const size_t srcSize) { - varInds_t v; - int first = 1; - res.params = cParamUnsetMin(res.params); - fprintf(f," {"); - for(v = 0; v < NUM_PARAMS; v++) { - if(g_silenceParams[v]) { continue; } - if(!first) { fprintf(f, ","); } - displayParamVal(f, v, res.params.vals[v], 3); - first = 0; +static winnerInfo_t initWinnerInfo(const paramValues_t p) { + winnerInfo_t w1; + w1.result.cSpeed = 0.; + w1.result.dSpeed = 0.; + w1.result.cMem = (size_t)-1; + w1.result.cSize = (size_t)-1; + w1.params = p; + return w1; +} + +static paramValues_t overwriteParams(paramValues_t base, const paramValues_t mask) { + U32 i; + for(i = 0; i < NUM_PARAMS; i++) { + if(mask.vals[i] != PARAM_UNSET) { + base.vals[i] = mask.vals[i]; + } + } + return base; +} + +/* amt will probably always be \pm 1? */ +/* slight change from old paramVariation, targetLength can only take on powers of 2 now (999 ~= 1024?) */ +/* take max/min bounds into account as well? */ +static void paramVaryOnce(const varInds_t paramIndex, const int amt, paramValues_t* ptr) { + ptr->vals[paramIndex] = rangeMap(paramIndex, invRangeMap(paramIndex, ptr->vals[paramIndex]) + amt); +} + +/* varies ptr by nbChanges respecting varyParams*/ +static void paramVariation(paramValues_t* ptr, memoTable_t* mtAll, const U32 nbChanges) +{ + paramValues_t p; + U32 validated = 0; + while (!validated) { + U32 i; + p = *ptr; + for (i = 0 ; i < nbChanges ; i++) { + const U32 changeID = (U32)FUZ_rand(&g_rand) % (mtAll[p.vals[strt_ind]].varLen << 1); + paramVaryOnce(mtAll[p.vals[strt_ind]].varArray[changeID >> 1], ((changeID & 1) << 1) - 1, &p); + } + validated = paramValid(p); + } + *ptr = p; +} + +/* Completely random parameter selection */ +static paramValues_t randomParams(void) +{ + varInds_t v; paramValues_t p; + for(v = 0; v <= NUM_PARAMS; v++) { + p.vals[v] = rangeMap(v, FUZ_rand(&g_rand) % rangetable[v]); + } + return p; +} + +static U64 g_clockGranularity = 100000000ULL; + +static void findClockGranularity(void) { + UTIL_time_t clockStart = UTIL_getTime(); + U64 el1 = 0, el2 = 0; + int i = 0; + do { + el1 = el2; + el2 = UTIL_clockSpanNano(clockStart); + if(el1 < el2) { + U64 iv = el2 - el1; + if(g_clockGranularity > iv) { + g_clockGranularity = iv; + i = 0; + } else { + i++; } - - fprintf(f, " }, /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", - (double)srcSize / res.result.cSize, (double)res.result.cSpeed / (1 MB), (double)res.result.dSpeed / (1 MB)); + } + } while(i < 10); + DEBUGOUTPUT("Granularity: %llu\n", (unsigned long long)g_clockGranularity); } +/*-************************************ +* Optimizer Util Functions +**************************************/ + /* checks results are feasible */ static int feasible(const BMK_result_t results, const constraint_t target) { return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem) && (!g_optmode || results.cSize <= g_lvltarget.cSize); @@ -523,52 +555,324 @@ static constraint_t relaxTarget(constraint_t target) { return target; } -/*-******************************************************* -* Bench functions -*********************************************************/ - -static paramValues_t emptyParams(void) { - U32 i; - paramValues_t p; - for(i = 0; i < NUM_PARAMS; i++) { - p.vals[i] = PARAM_UNSET; +static void optimizerAdjustInput(paramValues_t* pc, const size_t maxBlockSize) { + varInds_t v; + for(v = 0; v < NUM_PARAMS; v++) { + if(pc->vals[v] != PARAM_UNSET) { + U32 newval = MIN(MAX(pc->vals[v], mintable[v]), maxtable[v]); + if(newval != pc->vals[v]) { + pc->vals[v] = newval; + DISPLAY("Warning: parameter %s not in valid range, adjusting to ", g_paramNames[v]); displayParamVal(stderr, v, newval, 0); DISPLAY("\n"); + } + } + } + + if(pc->vals[wlog_ind] != PARAM_UNSET) { + + U32 sshb = maxBlockSize > 1 ? ZSTD_highbit32((U32)(maxBlockSize-1)) + 1 : 1; + /* edge case of highBit not working for 0 */ + + if(maxBlockSize < (1ULL << 31) && sshb + 1 < pc->vals[wlog_ind]) { + U32 adjust = MAX(mintable[wlog_ind], sshb); + if(adjust != pc->vals[wlog_ind]) { + pc->vals[wlog_ind] = adjust; + DISPLAY("Warning: windowLog larger than src/block size, adjusted to %u\n", pc->vals[wlog_ind]); + } + } + } + + if(pc->vals[wlog_ind] != PARAM_UNSET && pc->vals[clog_ind] != PARAM_UNSET) { + U32 maxclog; + if(pc->vals[strt_ind] == PARAM_UNSET || pc->vals[strt_ind] >= (U32)ZSTD_btlazy2) { + maxclog = pc->vals[wlog_ind] + 1; + } else { + maxclog = pc->vals[wlog_ind]; + } + + if(pc->vals[clog_ind] > maxclog) { + pc->vals[clog_ind] = maxclog; + DISPLAY("Warning: chainlog too much larger than windowLog size, adjusted to %u\n", pc->vals[clog_ind]); + } + } + + if(pc->vals[wlog_ind] != PARAM_UNSET && pc->vals[hlog_ind] != PARAM_UNSET) { + if(pc->vals[wlog_ind] + 1 < pc->vals[hlog_ind]) { + pc->vals[hlog_ind] = pc->vals[wlog_ind] + 1; + DISPLAY("Warning: hashlog too much larger than windowLog size, adjusted to %u\n", pc->vals[hlog_ind]); + } + } + + if(pc->vals[slog_ind] != PARAM_UNSET && pc->vals[clog_ind] != PARAM_UNSET) { + if(pc->vals[slog_ind] > pc->vals[clog_ind]) { + pc->vals[clog_ind] = pc->vals[slog_ind]; + DISPLAY("Warning: searchLog larger than chainLog, adjusted to %u\n", pc->vals[slog_ind]); + } } - return p; } -static winnerInfo_t initWinnerInfo(const paramValues_t p) { - winnerInfo_t w1; - w1.result.cSpeed = 0.; - w1.result.dSpeed = 0.; - w1.result.cMem = (size_t)-1; - w1.result.cSize = (size_t)-1; - w1.params = p; - return w1; +/* what about low something like clog vs hlog in lvl 1? */ +static int redundantParams(const paramValues_t paramValues, const constraint_t target, const size_t maxBlockSize) { + return + (ZSTD_estimateCStreamSize_usingCParams(pvalsToCParams(paramValues)) > (size_t)target.cMem) /* Uses too much memory */ + || ((1ULL << (paramValues.vals[wlog_ind] - 1)) >= maxBlockSize && paramValues.vals[wlog_ind] != mintable[wlog_ind]) /* wlog too much bigger than src size */ + || (paramValues.vals[clog_ind] > (paramValues.vals[wlog_ind] + (paramValues.vals[strt_ind] > ZSTD_btlazy2))) /* chainLog larger than windowLog*/ + || (paramValues.vals[slog_ind] > paramValues.vals[clog_ind]) /* searchLog larger than chainLog */ + || (paramValues.vals[hlog_ind] > paramValues.vals[wlog_ind] + 1); /* hashLog larger than windowLog + 1 */ + } -typedef struct { - void* srcBuffer; - size_t srcSize; - const void** srcPtrs; - size_t* srcSizes; - void** dstPtrs; - size_t* dstCapacities; - size_t* dstSizes; - void** resPtrs; - size_t* resSizes; - size_t nbBlocks; - size_t maxBlockSize; -} buffers_t; +/*-************************************ +* Display Functions +**************************************/ + +static void BMK_translateAdvancedParams(FILE* f, const paramValues_t params) { + varInds_t v; + int first = 1; + fprintf(f,"--zstd="); + for(v = 0; v < NUM_PARAMS; v++) { + if(g_silenceParams[v]) { continue; } + if(!first) { fprintf(f, ","); } + fprintf(f,"%s=", g_paramNames[v]); + + if(v == strt_ind) { fprintf(f,"%u", params.vals[v]); } + else { displayParamVal(f, v, params.vals[v], 0); } + first = 0; + } + fprintf(f, "\n"); +} + +static void BMK_displayOneResult(FILE* f, winnerInfo_t res, const size_t srcSize) { + varInds_t v; + int first = 1; + res.params = cParamUnsetMin(res.params); + fprintf(f," {"); + for(v = 0; v < NUM_PARAMS; v++) { + if(g_silenceParams[v]) { continue; } + if(!first) { fprintf(f, ","); } + displayParamVal(f, v, res.params.vals[v], 3); + first = 0; + } + + fprintf(f, " }, /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", + (double)srcSize / res.result.cSize, (double)res.result.cSpeed / (1 MB), (double)res.result.dSpeed / (1 MB)); +} + +/* Writes to f the results of a parameter benchmark */ +/* when used with --optimize, will only print results better than previously discovered */ +static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const paramValues_t params, const size_t srcSize) +{ + char lvlstr[15] = "Custom Level"; + winnerInfo_t w; + w.params = params; + w.result = result; + + fprintf(f, "\r%79s\r", ""); + + if(cLevel != CUSTOM_LEVEL) { + snprintf(lvlstr, 15, " Level %2u ", cLevel); + } + + if(TIMED) { + const U64 time = UTIL_clockSpanNano(g_time); + const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC); + fprintf(f, "%1lu:%2lu:%05.2f - ", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); + } + + fprintf(f, "/* %s */ ", lvlstr); + BMK_displayOneResult(f, w, srcSize); +} + +/* comparison function: */ +/* strictly better, strictly worse, equal, speed-side adv, size-side adv */ +//Maybe use compress_only for benchmark first run? +#define WORSE_RESULT 0 +#define BETTER_RESULT 1 +#define ERROR_RESULT 2 + +#define SPEED_RESULT 4 +#define SIZE_RESULT 5 +/* maybe have epsilon-eq to limit table size? */ +static int speedSizeCompare(const BMK_result_t r1, const BMK_result_t r2) { + if(r1.cSpeed < r2.cSpeed) { + if(r1.cSize >= r2.cSize) { + return BETTER_RESULT; + } + return SPEED_RESULT; /* r2 is smaller but not faster. */ + } else { + if(r1.cSize <= r2.cSize) { + return WORSE_RESULT; + } + return SIZE_RESULT; /* r2 is faster but not smaller */ + } +} + +/* 0 for insertion, 1 for no insert */ +/* maintain invariant speedSizeCompare(n, n->next) = SPEED_RESULT */ +static int insertWinner(const winnerInfo_t w, const constraint_t targetConstraints) { + BMK_result_t r = w.result; + winner_ll_node* cur_node = g_winners; + /* first node to insert */ + if(!feasible(r, targetConstraints)) { + return 1; + } + + if(g_winners == NULL) { + winner_ll_node* first_node = malloc(sizeof(winner_ll_node)); + if(first_node == NULL) { + return 1; + } + first_node->next = NULL; + first_node->res = w; + g_winners = first_node; + return 0; + } + + while(cur_node->next != NULL) { + switch(speedSizeCompare(cur_node->res.result, r)) { + case WORSE_RESULT: + { + return 1; /* never insert if better */ + } + case BETTER_RESULT: + { + winner_ll_node* tmp; + cur_node->res = cur_node->next->res; + tmp = cur_node->next; + cur_node->next = cur_node->next->next; + free(tmp); + break; + } + case SIZE_RESULT: + { + cur_node = cur_node->next; + break; + } + case SPEED_RESULT: /* insert after first size result, then return */ + { + winner_ll_node* newnode = malloc(sizeof(winner_ll_node)); + if(newnode == NULL) { + return 1; + } + newnode->res = cur_node->res; + cur_node->res = w; + newnode->next = cur_node->next; + cur_node->next = newnode; + return 0; + } + } + + } + + assert(cur_node->next == NULL); + switch(speedSizeCompare(cur_node->res.result, r)) { + case WORSE_RESULT: + { + return 1; /* never insert if better */ + } + case BETTER_RESULT: + { + cur_node->res = w; + return 0; + } + case SIZE_RESULT: + { + winner_ll_node* newnode = malloc(sizeof(winner_ll_node)); + if(newnode == NULL) { + return 1; + } + newnode->res = w; + newnode->next = NULL; + cur_node->next = newnode; + return 0; + } + case SPEED_RESULT: /* insert before first size result, then return */ + { + winner_ll_node* newnode = malloc(sizeof(winner_ll_node)); + if(newnode == NULL) { + return 1; + } + newnode->res = cur_node->res; + cur_node->res = w; + newnode->next = cur_node->next; + cur_node->next = newnode; + return 0; + } + default: + return 1; + } +} + +static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t result, const paramValues_t params, const constraint_t targetConstraints, const size_t srcSize) +{ + /* global winner used for constraints */ + /* cSize, cSpeed, dSpeed, cMem */ + static winnerInfo_t g_winner = { { (size_t)-1LL, 0, 0, (size_t)-1LL }, { { PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET } } }; + if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize) || g_displayLevel >= 4) { + if(DEBUG && compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { + DISPLAY("New Winner: \n"); + } + + if(g_displayLevel >= 2) { BMK_printWinner(f, cLevel, result, params, srcSize); } + + if(compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { + if(g_displayLevel >= 1) { BMK_translateAdvancedParams(f, params); } + g_winner.result = result; + g_winner.params = params; + } + } + + if(g_optmode && g_optimizer && (DEBUG || g_displayLevel == 3)) { + winnerInfo_t w; + winner_ll_node* n; + w.result = result; + w.params = params; + insertWinner(w, targetConstraints); + + if(!DEBUG) { fprintf(f, "\033c"); } + fprintf(f, "\n"); + + /* the table */ + fprintf(f, "================================\n"); + for(n = g_winners; n != NULL; n = n->next) { + BMK_displayOneResult(f, n->res, srcSize); + } + fprintf(f, "================================\n"); + fprintf(f, "Level Bounds: R: > %.3f AND C: < %.1f MB/s \n\n", + (double)srcSize / g_lvltarget.cSize, (double)g_lvltarget.cSpeed / (1 MB)); + + + fprintf(f, "Overall Winner: \n"); + BMK_displayOneResult(f, g_winner, srcSize); + BMK_translateAdvancedParams(f, g_winner.params); + + fprintf(f, "Latest BMK: \n");\ + BMK_displayOneResult(f, w, srcSize); + } +} + +static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, const size_t srcSize) +{ + int cLevel; + + fprintf(f, "\n /* Proposed configurations : */ \n"); + fprintf(f, " /* W, C, H, S, L, T, strat */ \n"); + + for (cLevel=0; cLevel <= NB_LEVELS_TRACKED; cLevel++) + BMK_printWinner(f, cLevel, winners[cLevel].result, winners[cLevel].params, srcSize); +} + + +static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, const size_t srcSize) +{ + fseek(f, 0, SEEK_SET); + BMK_printWinners2(f, winners, srcSize); + fflush(f); + BMK_printWinners2(stdout, winners, srcSize); +} -typedef struct { - size_t dictSize; - void* dictBuffer; - ZSTD_CCtx* cctx; - ZSTD_DCtx* dctx; -} contexts_t; /*-******************************************************* -* From bench.c +* Functions to Benchmark *********************************************************/ typedef struct { @@ -665,75 +969,30 @@ static size_t local_defaultDecompress( } -/*-******************************************************* -* From bench.c End -*********************************************************/ +/*-************************************ +* Data Initialization Functions +**************************************/ -static void optimizerAdjustInput(paramValues_t* pc, const size_t maxBlockSize) { - varInds_t v; - for(v = 0; v < NUM_PARAMS; v++) { - if(pc->vals[v] != PARAM_UNSET) { - U32 newval = MIN(MAX(pc->vals[v], mintable[v]), maxtable[v]); - if(newval != pc->vals[v]) { - pc->vals[v] = newval; - DISPLAY("Warning: parameter %s not in valid range, adjusting to ", g_paramNames[v]); displayParamVal(stderr, v, newval, 0); DISPLAY("\n"); - } - } - } +typedef struct { + void* srcBuffer; + size_t srcSize; + const void** srcPtrs; + size_t* srcSizes; + void** dstPtrs; + size_t* dstCapacities; + size_t* dstSizes; + void** resPtrs; + size_t* resSizes; + size_t nbBlocks; + size_t maxBlockSize; +} buffers_t; - if(pc->vals[wlog_ind] != PARAM_UNSET) { - - U32 sshb = maxBlockSize > 1 ? ZSTD_highbit32((U32)(maxBlockSize-1)) + 1 : 1; - /* edge case of highBit not working for 0 */ - - if(maxBlockSize < (1ULL << 31) && sshb + 1 < pc->vals[wlog_ind]) { - U32 adjust = MAX(mintable[wlog_ind], sshb); - if(adjust != pc->vals[wlog_ind]) { - pc->vals[wlog_ind] = adjust; - DISPLAY("Warning: windowLog larger than src/block size, adjusted to %u\n", pc->vals[wlog_ind]); - } - } - } - - if(pc->vals[wlog_ind] != PARAM_UNSET && pc->vals[clog_ind] != PARAM_UNSET) { - U32 maxclog; - if(pc->vals[strt_ind] == PARAM_UNSET || pc->vals[strt_ind] >= (U32)ZSTD_btlazy2) { - maxclog = pc->vals[wlog_ind] + 1; - } else { - maxclog = pc->vals[wlog_ind]; - } - - if(pc->vals[clog_ind] > maxclog) { - pc->vals[clog_ind] = maxclog; - DISPLAY("Warning: chainlog too much larger than windowLog size, adjusted to %u\n", pc->vals[clog_ind]); - } - } - - if(pc->vals[wlog_ind] != PARAM_UNSET && pc->vals[hlog_ind] != PARAM_UNSET) { - if(pc->vals[wlog_ind] + 1 < pc->vals[hlog_ind]) { - pc->vals[hlog_ind] = pc->vals[wlog_ind] + 1; - DISPLAY("Warning: hashlog too much larger than windowLog size, adjusted to %u\n", pc->vals[hlog_ind]); - } - } - - if(pc->vals[slog_ind] != PARAM_UNSET && pc->vals[clog_ind] != PARAM_UNSET) { - if(pc->vals[slog_ind] > pc->vals[clog_ind]) { - pc->vals[clog_ind] = pc->vals[slog_ind]; - DISPLAY("Warning: searchLog larger than chainLog, adjusted to %u\n", pc->vals[slog_ind]); - } - } -} - -/* what about low something like clog vs hlog in lvl 1? */ -static int redundantParams(const paramValues_t paramValues, const constraint_t target, const size_t maxBlockSize) { - return - (ZSTD_estimateCStreamSize_usingCParams(pvalsToCParams(paramValues)) > (size_t)target.cMem) /* Uses too much memory */ - || ((1ULL << (paramValues.vals[wlog_ind] - 1)) >= maxBlockSize && paramValues.vals[wlog_ind] != mintable[wlog_ind]) /* wlog too much bigger than src size */ - || (paramValues.vals[clog_ind] > (paramValues.vals[wlog_ind] + (paramValues.vals[strt_ind] > ZSTD_btlazy2))) /* chainLog larger than windowLog*/ - || (paramValues.vals[slog_ind] > paramValues.vals[clog_ind]) /* searchLog larger than chainLog */ - || (paramValues.vals[hlog_ind] > paramValues.vals[wlog_ind] + 1); /* hashLog larger than windowLog + 1 */ - -} +typedef struct { + size_t dictSize; + void* dictBuffer; + ZSTD_CCtx* cctx; + ZSTD_DCtx* dctx; +} contexts_t; static void freeNonSrcBuffers(const buffers_t b) { free(b.srcPtrs); @@ -952,6 +1211,175 @@ static int createContexts(contexts_t* ctx, const char* dictFileName) { return 0; } +/*-************************************ +* Optimizer Memoization Functions +**************************************/ + +/* return: new length */ +/* keep old array, will need if iter over strategy. */ +/* prunes useless params */ +static size_t sanitizeVarArray(varInds_t* varNew, const size_t varLength, const varInds_t* varArray, const ZSTD_strategy strat) { + size_t i, j = 0; + for(i = 0; i < varLength; i++) { + if( !((varArray[i] == clog_ind && strat == ZSTD_fast) + || (varArray[i] == slog_ind && strat == ZSTD_fast) + || (varArray[i] == slog_ind && strat == ZSTD_dfast) + || (varArray[i] == tlen_ind && strat != ZSTD_btopt && strat != ZSTD_btultra && strat != ZSTD_fast))) { + varNew[j] = varArray[i]; + j++; + } + } + return j; +} + +/* res should be NUM_PARAMS size */ +/* constructs varArray from paramValues_t style parameter */ +/* pass in using dict. */ +static size_t variableParams(const paramValues_t paramConstraints, varInds_t* res, const int usingDictionary) { + varInds_t i; + size_t j = 0; + for(i = 0; i < NUM_PARAMS; i++) { + if(paramConstraints.vals[i] == PARAM_UNSET) { + if(i == fadt_ind && !usingDictionary) continue; /* don't use fadt if no dictionary */ + res[j] = i; j++; + } + } + return j; +} + +/* length of memo table given free variables */ +static size_t memoTableLen(const varInds_t* varyParams, const size_t varyLen) { + size_t arrayLen = 1; + size_t i; + for(i = 0; i < varyLen; i++) { + if(varyParams[i] == strt_ind) continue; /* strategy separated by table */ + arrayLen *= rangetable[varyParams[i]]; + } + return arrayLen; +} + +/* returns unique index in memotable of compression parameters */ +static unsigned memoTableIndDirect(const paramValues_t* ptr, const varInds_t* varyParams, const size_t varyLen) { + size_t i; + unsigned ind = 0; + for(i = 0; i < varyLen; i++) { + varInds_t v = varyParams[i]; + if(v == strt_ind) continue; /* exclude strategy from memotable */ + ind *= rangetable[v]; ind += (unsigned)invRangeMap(v, ptr->vals[v]); + } + return ind; +} + +static size_t memoTableGet(const memoTable_t* memoTableArray, const paramValues_t p) { + const memoTable_t mt = memoTableArray[p.vals[strt_ind]]; + switch(mt.tableType) { + case directMap: + return mt.table[memoTableIndDirect(&p, mt.varArray, mt.varLen)]; + case xxhashMap: + return mt.table[(XXH64(&p.vals, sizeof(U32) * NUM_PARAMS, 0) >> 3) % mt.tableLen]; + case noMemo: + return 0; + } + return 0; /* should never happen, stop compiler warnings */ +} + +static void memoTableSet(const memoTable_t* memoTableArray, const paramValues_t p, const BYTE value) { + const memoTable_t mt = memoTableArray[p.vals[strt_ind]]; + switch(mt.tableType) { + case directMap: + mt.table[memoTableIndDirect(&p, mt.varArray, mt.varLen)] = value; break; + case xxhashMap: + mt.table[(XXH64(&p.vals, sizeof(U32) * NUM_PARAMS, 0) >> 3) % mt.tableLen] = value; break; + case noMemo: + break; + } +} + +/* frees all allocated memotables */ +static void freeMemoTableArray(memoTable_t* const mtAll) { + int i; + if(mtAll == NULL) { return; } + for(i = 1; i <= (int)ZSTD_btultra; i++) { + free(mtAll[i].table); + } + free(mtAll); +} + +/* inits memotables for all (including mallocs), all strategies */ +/* takes unsanitized varyParams */ +static memoTable_t* createMemoTableArray(const paramValues_t p, const varInds_t* const varyParams, const size_t varyLen, const U32 memoTableLog) { + memoTable_t* mtAll = (memoTable_t*)calloc(sizeof(memoTable_t),(ZSTD_btultra + 1)); + ZSTD_strategy i, stratMin = ZSTD_fast, stratMax = ZSTD_btultra; + + if(mtAll == NULL) { + return NULL; + } + + for(i = 1; i <= (int)ZSTD_btultra; i++) { + mtAll[i].varLen = sanitizeVarArray(mtAll[i].varArray, varyLen, varyParams, i); + } + + /* no memoization */ + if(memoTableLog == 0) { + for(i = 1; i <= (int)ZSTD_btultra; i++) { + mtAll[i].tableType = noMemo; + mtAll[i].table = NULL; + mtAll[i].tableLen = 0; + } + return mtAll; + } + + + if(p.vals[strt_ind] != PARAM_UNSET) { + stratMin = p.vals[strt_ind]; + stratMax = p.vals[strt_ind]; + } + + + for(i = stratMin; i <= stratMax; i++) { + size_t mtl = memoTableLen(mtAll[i].varArray, mtAll[i].varLen); + mtAll[i].tableType = directMap; + + if(memoTableLog != PARAM_UNSET && mtl > (1ULL << memoTableLog)) { /* use hash table */ /* provide some option to only use hash tables? */ + mtAll[i].tableType = xxhashMap; + mtl = (1ULL << memoTableLog); + } + + mtAll[i].table = (BYTE*)calloc(sizeof(BYTE), mtl); + mtAll[i].tableLen = mtl; + + if(mtAll[i].table == NULL) { + freeMemoTableArray(mtAll); + return NULL; + } + } + + return mtAll; +} + +/* Sets pc to random unmeasured set of parameters */ +/* specifiy strategy */ +static void randomConstrainedParams(paramValues_t* pc, const memoTable_t* memoTableArray, const ZSTD_strategy st) +{ + size_t j; + const memoTable_t mt = memoTableArray[st]; + pc->vals[strt_ind] = st; + for(j = 0; j < mt.tableLen; j++) { + int i; + for(i = 0; i < NUM_PARAMS; i++) { + varInds_t v = mt.varArray[i]; + if(v == strt_ind) continue; + pc->vals[v] = rangeMap(v, FUZ_rand(&g_rand) % rangetable[v]); + } + + if(!(memoTableGet(memoTableArray, *pc))) break; /* only pick unpicked params. */ + } +} + +/*-************************************ +* Benchmarking Functions +**************************************/ + /* Replicate functionality of benchMemAdvanced, but with pre-split src / dst buffers */ /* The purpose is so that sufficient information is returned so that a decompression call to benchMemInvertible is possible */ /* BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); */ @@ -1099,220 +1527,115 @@ static int BMK_benchParam(BMK_result_t* resultPtr, return res.error; } -/* comparison function: */ -/* strictly better, strictly worse, equal, speed-side adv, size-side adv */ -//Maybe use compress_only for benchmark first run? -#define WORSE_RESULT 0 -#define BETTER_RESULT 1 -#define ERROR_RESULT 2 -#define SPEED_RESULT 4 -#define SIZE_RESULT 5 -/* maybe have epsilon-eq to limit table size? */ -static int speedSizeCompare(const BMK_result_t r1, const BMK_result_t r2) { - if(r1.cSpeed < r2.cSpeed) { - if(r1.cSize >= r2.cSize) { - return BETTER_RESULT; - } - return SPEED_RESULT; /* r2 is smaller but not faster. */ - } else { - if(r1.cSize <= r2.cSize) { - return WORSE_RESULT; - } - return SIZE_RESULT; /* r2 is faster but not smaller */ +#define CBENCHMARK(conditional, resultvar, tmpret, mode, loopmode, sec) { \ + if(conditional) { \ + BMK_return_t tmpret = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, mode, loopmode, sec); \ + if(tmpret.error) { DEBUGOUTPUT("Benchmarking failed\n"); return ERROR_RESULT; } \ + if(mode != BMK_decodeOnly) { \ + resultvar.cSpeed = tmpret.result.cSpeed; \ + resultvar.cSize = tmpret.result.cSize; \ + resultvar.cMem = tmpret.result.cMem; \ + } \ + if(mode != BMK_compressOnly) { resultvar.dSpeed = tmpret.result.dSpeed; } \ + } \ +} + +/* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */ +#define VARIANCE 1.2 +static int allBench(BMK_result_t* resultPtr, + const buffers_t buf, const contexts_t ctx, + const paramValues_t cParams, + const constraint_t target, + BMK_result_t* winnerResult, int feas) { + BMK_result_t resultMax, benchres; + U64 loopDurationC = 0, loopDurationD = 0; + double uncertaintyConstantC = 3., uncertaintyConstantD = 3.; + double winnerRS; + /* initial benchmarking, gives exact ratio and memory, warms up future runs */ + CBENCHMARK(1, benchres, tmp, BMK_both, BMK_iterMode, 1); + + winnerRS = resultScore(*winnerResult, buf.srcSize, target); + DEBUGOUTPUT("WinnerScore: %f\n ", winnerRS); + + *resultPtr = benchres; + + /* calculate uncertainty in compression / decompression runs */ + if(benchres.cSpeed) { + loopDurationC = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.cSpeed); + uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC); + } + + if(benchres.dSpeed) { + loopDurationD = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.dSpeed); + uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD); + } + + /* anything with worse ratio in feas is definitely worse, discard */ + if(feas && benchres.cSize < winnerResult->cSize && !g_optmode) { + return WORSE_RESULT; + } + + /* second run, if first run is too short, gives approximate cSpeed + dSpeed */ + CBENCHMARK(loopDurationC < TIMELOOP_NANOSEC / 10, benchres, tmp, BMK_compressOnly, BMK_iterMode, 1); + CBENCHMARK(loopDurationD < TIMELOOP_NANOSEC / 10, benchres, tmp, BMK_decodeOnly, BMK_iterMode, 1); + + *resultPtr = benchres; + + /* optimistic assumption of benchres */ + resultMax = benchres; + resultMax.cSpeed *= uncertaintyConstantC * VARIANCE; + resultMax.dSpeed *= uncertaintyConstantD * VARIANCE; + + /* disregard infeasible results in feas mode */ + /* disregard if resultMax < winner in infeas mode */ + if((feas && !feasible(resultMax, target)) || + (!feas && (winnerRS > resultScore(resultMax, buf.srcSize, target)))) { + return WORSE_RESULT; + } + + CBENCHMARK(loopDurationC < TIMELOOP_NANOSEC, benchres, tmp, BMK_compressOnly, BMK_timeMode, 1); + CBENCHMARK(loopDurationD < TIMELOOP_NANOSEC, benchres, tmp, BMK_decodeOnly, BMK_timeMode, 1); + + *resultPtr = benchres; + + /* compare by resultScore when in infeas */ + /* compare by compareResultLT when in feas */ + if((!feas && (resultScore(benchres, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target))) || + (feas && (compareResultLT(*winnerResult, benchres, target, buf.srcSize))) ) { + return BETTER_RESULT; + } else { + return WORSE_RESULT; } } -/* 0 for insertion, 1 for no insert */ -/* maintain invariant speedSizeCompare(n, n->next) = SPEED_RESULT */ -static int insertWinner(const winnerInfo_t w, const constraint_t targetConstraints) { - BMK_result_t r = w.result; - winner_ll_node* cur_node = g_winners; - /* first node to insert */ - if(!feasible(r, targetConstraints)) { - return 1; +#define INFEASIBLE_THRESHOLD 200 +/* Memoized benchmarking, won't benchmark anything which has already been benchmarked before. */ +static int benchMemo(BMK_result_t* resultPtr, + const buffers_t buf, const contexts_t ctx, + const paramValues_t cParams, + const constraint_t target, + BMK_result_t* winnerResult, memoTable_t* const memoTableArray, + const int feas) { + static int bmcount = 0; + int res; + + if(memoTableGet(memoTableArray, cParams) >= INFEASIBLE_THRESHOLD || redundantParams(cParams, target, buf.maxBlockSize)) { return WORSE_RESULT; } + + res = allBench(resultPtr, buf, ctx, cParams, target, winnerResult, feas); + + if(DEBUG && !(bmcount % 250)) { + DISPLAY("Count: %d\n", bmcount); + bmcount++; } + BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, *resultPtr, cParams, target, buf.srcSize); - if(g_winners == NULL) { - winner_ll_node* first_node = malloc(sizeof(winner_ll_node)); - if(first_node == NULL) { - return 1; - } - first_node->next = NULL; - first_node->res = w; - g_winners = first_node; - return 0; + if(res == BETTER_RESULT || feas) { + memoTableSet(memoTableArray, cParams, 255); /* what happens if collisions are frequent */ } - - while(cur_node->next != NULL) { - switch(speedSizeCompare(cur_node->res.result, r)) { - case WORSE_RESULT: - { - return 1; /* never insert if better */ - } - case BETTER_RESULT: - { - winner_ll_node* tmp; - cur_node->res = cur_node->next->res; - tmp = cur_node->next; - cur_node->next = cur_node->next->next; - free(tmp); - break; - } - case SIZE_RESULT: - { - cur_node = cur_node->next; - break; - } - case SPEED_RESULT: /* insert after first size result, then return */ - { - winner_ll_node* newnode = malloc(sizeof(winner_ll_node)); - if(newnode == NULL) { - return 1; - } - newnode->res = cur_node->res; - cur_node->res = w; - newnode->next = cur_node->next; - cur_node->next = newnode; - return 0; - } - } - - } - - assert(cur_node->next == NULL); - switch(speedSizeCompare(cur_node->res.result, r)) { - case WORSE_RESULT: - { - return 1; /* never insert if better */ - } - case BETTER_RESULT: - { - cur_node->res = w; - return 0; - } - case SIZE_RESULT: - { - winner_ll_node* newnode = malloc(sizeof(winner_ll_node)); - if(newnode == NULL) { - return 1; - } - newnode->res = w; - newnode->next = NULL; - cur_node->next = newnode; - return 0; - } - case SPEED_RESULT: /* insert before first size result, then return */ - { - winner_ll_node* newnode = malloc(sizeof(winner_ll_node)); - if(newnode == NULL) { - return 1; - } - newnode->res = cur_node->res; - cur_node->res = w; - newnode->next = cur_node->next; - cur_node->next = newnode; - return 0; - } - default: - return 1; - } + return res; } -/* Writes to f the results of a parameter benchmark */ -/* when used with --optimize, will only print results better than previously discovered */ -static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const paramValues_t params, const size_t srcSize) -{ - char lvlstr[15] = "Custom Level"; - winnerInfo_t w; - w.params = params; - w.result = result; - - fprintf(f, "\r%79s\r", ""); - - if(cLevel != CUSTOM_LEVEL) { - snprintf(lvlstr, 15, " Level %2u ", cLevel); - } - - if(TIMED) { - const U64 time = UTIL_clockSpanNano(g_time); - const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC); - fprintf(f, "%1lu:%2lu:%05.2f - ", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); - } - - fprintf(f, "/* %s */ ", lvlstr); - BMK_displayOneResult(f, w, srcSize); -} - -static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t result, const paramValues_t params, const constraint_t targetConstraints, const size_t srcSize) -{ - /* global winner used for constraints */ - /* cSize, cSpeed, dSpeed, cMem */ - static winnerInfo_t g_winner = { { (size_t)-1LL, 0, 0, (size_t)-1LL }, { { PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET } } }; - if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize) || g_displayLevel >= 4) { - if(DEBUG && compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { - DISPLAY("New Winner: \n"); - } - - if(g_displayLevel >= 2) { BMK_printWinner(f, cLevel, result, params, srcSize); } - - if(compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { - if(g_displayLevel >= 1) { BMK_translateAdvancedParams(f, params); } - g_winner.result = result; - g_winner.params = params; - } - } - - if(g_optmode && g_optimizer && (DEBUG || g_displayLevel == 3)) { - winnerInfo_t w; - winner_ll_node* n; - w.result = result; - w.params = params; - insertWinner(w, targetConstraints); - - if(!DEBUG) { fprintf(f, "\033c"); } - fprintf(f, "\n"); - - /* the table */ - fprintf(f, "================================\n"); - for(n = g_winners; n != NULL; n = n->next) { - BMK_displayOneResult(f, n->res, srcSize); - } - fprintf(f, "================================\n"); - fprintf(f, "Level Bounds: R: > %.3f AND C: < %.1f MB/s \n\n", - (double)srcSize / g_lvltarget.cSize, (double)g_lvltarget.cSpeed / (1 MB)); - - - fprintf(f, "Overall Winner: \n"); - BMK_displayOneResult(f, g_winner, srcSize); - BMK_translateAdvancedParams(f, g_winner.params); - - fprintf(f, "Latest BMK: \n");\ - BMK_displayOneResult(f, w, srcSize); - } -} - -static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, const size_t srcSize) -{ - int cLevel; - - fprintf(f, "\n /* Proposed configurations : */ \n"); - fprintf(f, " /* W, C, H, S, L, T, strat */ \n"); - - for (cLevel=0; cLevel <= NB_LEVELS_TRACKED; cLevel++) - BMK_printWinner(f, cLevel, winners[cLevel].result, winners[cLevel].params, srcSize); -} - - -static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, const size_t srcSize) -{ - fseek(f, 0, SEEK_SET); - BMK_printWinners2(f, winners, srcSize); - fflush(f); - BMK_printWinners2(stdout, winners, srcSize); -} - - typedef struct { U64 cSpeed_min; U64 dSpeed_min; @@ -1437,201 +1760,9 @@ static int BMK_seed(winnerInfo_t* winners, const paramValues_t params, return better; } -/* bounds check in sanitize too? */ -#define CLAMP(var, lo, hi) { \ - var = MAX(MIN(var, hi), lo); \ -} - -/* nullified useless params, to ensure count stats */ -/* cleans up params for memoizing / display */ -static paramValues_t sanitizeParams(paramValues_t params) -{ - if (params.vals[strt_ind] == ZSTD_fast) - params.vals[clog_ind] = 0, params.vals[slog_ind] = 0; - if (params.vals[strt_ind] == ZSTD_dfast) - params.vals[slog_ind] = 0; - if (params.vals[strt_ind] != ZSTD_btopt && params.vals[strt_ind] != ZSTD_btultra && params.vals[strt_ind] != ZSTD_fast) - params.vals[tlen_ind] = 0; - - return params; -} - -/* return: new length */ -/* keep old array, will need if iter over strategy. */ -/* prunes useless params */ -static size_t sanitizeVarArray(varInds_t* varNew, const size_t varLength, const varInds_t* varArray, const ZSTD_strategy strat) { - size_t i, j = 0; - for(i = 0; i < varLength; i++) { - if( !((varArray[i] == clog_ind && strat == ZSTD_fast) - || (varArray[i] == slog_ind && strat == ZSTD_fast) - || (varArray[i] == slog_ind && strat == ZSTD_dfast) - || (varArray[i] == tlen_ind && strat != ZSTD_btopt && strat != ZSTD_btultra && strat != ZSTD_fast))) { - varNew[j] = varArray[i]; - j++; - } - } - return j; - -} - -/* res should be NUM_PARAMS size */ -/* constructs varArray from paramValues_t style parameter */ -/* pass in using dict. */ -static size_t variableParams(const paramValues_t paramConstraints, varInds_t* res, const int usingDictionary) { - varInds_t i; - size_t j = 0; - for(i = 0; i < NUM_PARAMS; i++) { - if(paramConstraints.vals[i] == PARAM_UNSET) { - if(i == fadt_ind && !usingDictionary) continue; /* don't use fadt if no dictionary */ - res[j] = i; j++; - } - } - return j; -} - -/* amt will probably always be \pm 1? */ -/* slight change from old paramVariation, targetLength can only take on powers of 2 now (999 ~= 1024?) */ -/* take max/min bounds into account as well? */ -static void paramVaryOnce(const varInds_t paramIndex, const int amt, paramValues_t* ptr) { - ptr->vals[paramIndex] = rangeMap(paramIndex, invRangeMap(paramIndex, ptr->vals[paramIndex]) + amt); -} - -/* varies ptr by nbChanges respecting varyParams*/ -static void paramVariation(paramValues_t* ptr, memoTable_t* mtAll, const U32 nbChanges) -{ - paramValues_t p; - U32 validated = 0; - while (!validated) { - U32 i; - p = *ptr; - for (i = 0 ; i < nbChanges ; i++) { - const U32 changeID = (U32)FUZ_rand(&g_rand) % (mtAll[p.vals[strt_ind]].varLen << 1); - paramVaryOnce(mtAll[p.vals[strt_ind]].varArray[changeID >> 1], ((changeID & 1) << 1) - 1, &p); - } - validated = paramValid(p); - } - *ptr = p; -} - -/* length of memo table given free variables */ -static size_t memoTableLen(const varInds_t* varyParams, const size_t varyLen) { - size_t arrayLen = 1; - size_t i; - for(i = 0; i < varyLen; i++) { - if(varyParams[i] == strt_ind) continue; /* strategy separated by table */ - arrayLen *= rangetable[varyParams[i]]; - } - return arrayLen; -} - -/* returns unique index in memotable of compression parameters */ -static unsigned memoTableIndDirect(const paramValues_t* ptr, const varInds_t* varyParams, const size_t varyLen) { - size_t i; - unsigned ind = 0; - for(i = 0; i < varyLen; i++) { - varInds_t v = varyParams[i]; - if(v == strt_ind) continue; /* exclude strategy from memotable */ - ind *= rangetable[v]; ind += (unsigned)invRangeMap(v, ptr->vals[v]); - } - return ind; -} - -static size_t memoTableGet(const memoTable_t* memoTableArray, const paramValues_t p) { - const memoTable_t mt = memoTableArray[p.vals[strt_ind]]; - switch(mt.tableType) { - case directMap: - return mt.table[memoTableIndDirect(&p, mt.varArray, mt.varLen)]; - case xxhashMap: - return mt.table[(XXH64(&p.vals, sizeof(U32) * NUM_PARAMS, 0) >> 3) % mt.tableLen]; - case noMemo: - return 0; - } - return 0; /* should never happen, stop compiler warnings */ -} - -static void memoTableSet(const memoTable_t* memoTableArray, const paramValues_t p, const BYTE value) { - const memoTable_t mt = memoTableArray[p.vals[strt_ind]]; - switch(mt.tableType) { - case directMap: - mt.table[memoTableIndDirect(&p, mt.varArray, mt.varLen)] = value; break; - case xxhashMap: - mt.table[(XXH64(&p.vals, sizeof(U32) * NUM_PARAMS, 0) >> 3) % mt.tableLen] = value; break; - case noMemo: - break; - } -} - -/* frees all allocated memotables */ -static void freeMemoTableArray(memoTable_t* const mtAll) { - int i; - if(mtAll == NULL) { return; } - for(i = 1; i <= (int)ZSTD_btultra; i++) { - free(mtAll[i].table); - } - free(mtAll); -} - -/* inits memotables for all (including mallocs), all strategies */ -/* takes unsanitized varyParams */ -static memoTable_t* createMemoTableArray(const paramValues_t p, const varInds_t* const varyParams, const size_t varyLen, const U32 memoTableLog) { - memoTable_t* mtAll = (memoTable_t*)calloc(sizeof(memoTable_t),(ZSTD_btultra + 1)); - ZSTD_strategy i, stratMin = ZSTD_fast, stratMax = ZSTD_btultra; - - if(mtAll == NULL) { - return NULL; - } - - for(i = 1; i <= (int)ZSTD_btultra; i++) { - mtAll[i].varLen = sanitizeVarArray(mtAll[i].varArray, varyLen, varyParams, i); - } - - /* no memoization */ - if(memoTableLog == 0) { - for(i = 1; i <= (int)ZSTD_btultra; i++) { - mtAll[i].tableType = noMemo; - mtAll[i].table = NULL; - mtAll[i].tableLen = 0; - } - return mtAll; - } - - - if(p.vals[strt_ind] != PARAM_UNSET) { - stratMin = p.vals[strt_ind]; - stratMax = p.vals[strt_ind]; - } - - - for(i = stratMin; i <= stratMax; i++) { - size_t mtl = memoTableLen(mtAll[i].varArray, mtAll[i].varLen); - mtAll[i].tableType = directMap; - - if(memoTableLog != PARAM_UNSET && mtl > (1ULL << memoTableLog)) { /* use hash table */ /* provide some option to only use hash tables? */ - mtAll[i].tableType = xxhashMap; - mtl = (1ULL << memoTableLog); - } - - mtAll[i].table = (BYTE*)calloc(sizeof(BYTE), mtl); - mtAll[i].tableLen = mtl; - - if(mtAll[i].table == NULL) { - freeMemoTableArray(mtAll); - return NULL; - } - } - - return mtAll; -} - -static paramValues_t overwriteParams(paramValues_t base, const paramValues_t mask) { - U32 i; - for(i = 0; i < NUM_PARAMS; i++) { - if(mask.vals[i] != PARAM_UNSET) { - base.vals[i] = mask.vals[i]; - } - } - return base; -} +/*-************************************ +* Compression Level Table Generation Functions +**************************************/ #define PARAMTABLELOG 25 #define PARAMTABLESIZE (1<vals[strt_ind] = st; - for(j = 0; j < mt.tableLen; j++) { - int i; - for(i = 0; i < NUM_PARAMS; i++) { - varInds_t v = mt.varArray[i]; - if(v == strt_ind) continue; - pc->vals[v] = rangeMap(v, FUZ_rand(&g_rand) % rangetable[v]); - } - - if(!(memoTableGet(memoTableArray, *pc))) break; /* only pick unpicked params. */ - } -} - -/* Completely random parameter selection */ -static paramValues_t randomParams(void) -{ - varInds_t v; paramValues_t p; - for(v = 0; v <= NUM_PARAMS; v++) { - p.vals[v] = rangeMap(v, FUZ_rand(&g_rand) % rangetable[v]); - } - return p; -} - static void BMK_selectRandomStart( FILE* f, winnerInfo_t* winners, const buffers_t buf, const contexts_t ctx) @@ -1764,6 +1866,10 @@ static void BMK_benchFullTable(const buffers_t buf, const contexts_t ctx) fclose(f); } +/*-************************************ +* Single Benchmark Functions +**************************************/ + static int benchOnce(const buffers_t buf, const contexts_t ctx) { BMK_result_t testResult; @@ -1862,113 +1968,10 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam return ret; } -#define CBENCHMARK(conditional, resultvar, tmpret, mode, loopmode, sec) { \ - if(conditional) { \ - BMK_return_t tmpret = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, mode, loopmode, sec); \ - if(tmpret.error) { DEBUGOUTPUT("Benchmarking failed\n"); return ERROR_RESULT; } \ - if(mode != BMK_decodeOnly) { \ - resultvar.cSpeed = tmpret.result.cSpeed; \ - resultvar.cSize = tmpret.result.cSize; \ - resultvar.cMem = tmpret.result.cMem; \ - } \ - if(mode != BMK_compressOnly) { resultvar.dSpeed = tmpret.result.dSpeed; } \ - } \ -} -/* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */ -#define VARIANCE 1.2 -static int allBench(BMK_result_t* resultPtr, - const buffers_t buf, const contexts_t ctx, - const paramValues_t cParams, - const constraint_t target, - BMK_result_t* winnerResult, int feas) { - BMK_result_t resultMax, benchres; - U64 loopDurationC = 0, loopDurationD = 0; - double uncertaintyConstantC = 3., uncertaintyConstantD = 3.; - double winnerRS; - /* initial benchmarking, gives exact ratio and memory, warms up future runs */ - CBENCHMARK(1, benchres, tmp, BMK_both, BMK_iterMode, 1); - - winnerRS = resultScore(*winnerResult, buf.srcSize, target); - DEBUGOUTPUT("WinnerScore: %f\n ", winnerRS); - - *resultPtr = benchres; - - /* calculate uncertainty in compression / decompression runs */ - if(benchres.cSpeed) { - loopDurationC = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.cSpeed); - uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC); - } - - if(benchres.dSpeed) { - loopDurationD = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.dSpeed); - uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD); - } - - /* anything with worse ratio in feas is definitely worse, discard */ - if(feas && benchres.cSize < winnerResult->cSize && !g_optmode) { - return WORSE_RESULT; - } - - /* second run, if first run is too short, gives approximate cSpeed + dSpeed */ - CBENCHMARK(loopDurationC < TIMELOOP_NANOSEC / 10, benchres, tmp, BMK_compressOnly, BMK_iterMode, 1); - CBENCHMARK(loopDurationD < TIMELOOP_NANOSEC / 10, benchres, tmp, BMK_decodeOnly, BMK_iterMode, 1); - - *resultPtr = benchres; - - /* optimistic assumption of benchres */ - resultMax = benchres; - resultMax.cSpeed *= uncertaintyConstantC * VARIANCE; - resultMax.dSpeed *= uncertaintyConstantD * VARIANCE; - - /* disregard infeasible results in feas mode */ - /* disregard if resultMax < winner in infeas mode */ - if((feas && !feasible(resultMax, target)) || - (!feas && (winnerRS > resultScore(resultMax, buf.srcSize, target)))) { - return WORSE_RESULT; - } - - CBENCHMARK(loopDurationC < TIMELOOP_NANOSEC, benchres, tmp, BMK_compressOnly, BMK_timeMode, 1); - CBENCHMARK(loopDurationD < TIMELOOP_NANOSEC, benchres, tmp, BMK_decodeOnly, BMK_timeMode, 1); - - *resultPtr = benchres; - - /* compare by resultScore when in infeas */ - /* compare by compareResultLT when in feas */ - if((!feas && (resultScore(benchres, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target))) || - (feas && (compareResultLT(*winnerResult, benchres, target, buf.srcSize))) ) { - return BETTER_RESULT; - } else { - return WORSE_RESULT; - } -} - -#define INFEASIBLE_THRESHOLD 200 -/* Memoized benchmarking, won't benchmark anything which has already been benchmarked before. */ -static int benchMemo(BMK_result_t* resultPtr, - const buffers_t buf, const contexts_t ctx, - const paramValues_t cParams, - const constraint_t target, - BMK_result_t* winnerResult, memoTable_t* const memoTableArray, - const int feas) { - static int bmcount = 0; - int res; - - if(memoTableGet(memoTableArray, cParams) >= INFEASIBLE_THRESHOLD || redundantParams(cParams, target, buf.maxBlockSize)) { return WORSE_RESULT; } - - res = allBench(resultPtr, buf, ctx, cParams, target, winnerResult, feas); - - if(DEBUG && !(bmcount % 250)) { - DISPLAY("Count: %d\n", bmcount); - bmcount++; - } - BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, *resultPtr, cParams, target, buf.srcSize); - - if(res == BETTER_RESULT || feas) { - memoTableSet(memoTableArray, cParams, 255); /* what happens if collisions are frequent */ - } - return res; -} +/*-************************************ +* Local Optimization Functions +**************************************/ /* One iteration of hill climbing. Specifically, it first tries all * valid parameter configurations w/ manhattan distance 1 and picks the best one @@ -2368,6 +2371,24 @@ _cleanUp: return ret; } +/*-************************************ +* CLI parsing functions +**************************************/ + +/** longCommandWArg() : + * check if *stringPtr is the same as longCommand. + * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand. + * @return 0 and doesn't modify *stringPtr otherwise. + * from zstdcli.c + */ +static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) +{ + size_t const comSize = strlen(longCommand); + int const result = !strncmp(*stringPtr, longCommand, comSize); + if (result) *stringPtr += comSize; + return result; +} + static void errorOut(const char* msg) { DISPLAY("%s \n", msg); exit(1); @@ -2474,6 +2495,10 @@ static int parse_params(const char** argptr, paramValues_t* pv) { return matched; } +/*-************************************ +* Main +**************************************/ + int main(int argc, const char** argv) { int i, From 239e114d62c2a1d3c73ac05534a18d8583b9654d Mon Sep 17 00:00:00 2001 From: George Lu Date: Wed, 15 Aug 2018 15:01:03 -0700 Subject: [PATCH 53/55] prune comments --- tests/paramgrill.c | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 7ebcdeec0..e94eead0d 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -439,9 +439,6 @@ static paramValues_t overwriteParams(paramValues_t base, const paramValues_t mas return base; } -/* amt will probably always be \pm 1? */ -/* slight change from old paramVariation, targetLength can only take on powers of 2 now (999 ~= 1024?) */ -/* take max/min bounds into account as well? */ static void paramVaryOnce(const varInds_t paramIndex, const int amt, paramValues_t* ptr) { ptr->vals[paramIndex] = rangeMap(paramIndex, invRangeMap(paramIndex, ptr->vals[paramIndex]) + amt); } @@ -610,7 +607,6 @@ static void optimizerAdjustInput(paramValues_t* pc, const size_t maxBlockSize) { } } -/* what about low something like clog vs hlog in lvl 1? */ static int redundantParams(const paramValues_t paramValues, const constraint_t target, const size_t maxBlockSize) { return (ZSTD_estimateCStreamSize_usingCParams(pvalsToCParams(paramValues)) > (size_t)target.cMem) /* Uses too much memory */ @@ -684,7 +680,6 @@ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result /* comparison function: */ /* strictly better, strictly worse, equal, speed-side adv, size-side adv */ -//Maybe use compress_only for benchmark first run? #define WORSE_RESULT 0 #define BETTER_RESULT 1 #define ERROR_RESULT 2 @@ -1385,7 +1380,7 @@ static void randomConstrainedParams(paramValues_t* pc, const memoTable_t* memoTa /* BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); */ /* nbSeconds used in same way as in BMK_advancedParams_t, as nbIters when in iterMode */ -/* if in decodeOnly, then srcPtr's will be compressed blocks, and uncompressedBlocks will be written to dstPtrs? */ +/* if in decodeOnly, then srcPtr's will be compressed blocks, and uncompressedBlocks will be written to dstPtrs */ /* dictionary nullable, nothing else though. */ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t ctx, const int cLevel, const paramValues_t* comprParams, @@ -2088,11 +2083,9 @@ static winnerInfo_t climbOnce(const constraint_t target, /* Optimizes for a fixed strategy */ -/* flexible parameters: iterations of (failed?) climbing (or if we do non-random, maybe this is when everything is close to visitied) +/* flexible parameters: iterations of failed climbing (or if we do non-random, maybe this is when everything is close to visitied) weight more on visit for bad results, less on good results/more on later results / ones with more failures. allocate memoTable here. - only real use for paramTarget is to get the fixed values, right? - maybe allow giving it a first init? */ static winnerInfo_t optimizeFixedStrategy( const buffers_t buf, const contexts_t ctx, @@ -2233,7 +2226,6 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } /* use level'ing mode instead of normal target mode */ - /* Should lvl be parameter-masked here? */ if(g_optmode) { winner.params = cParamsToPVals(ZSTD_getCParams(cLevelOpt, buf.maxBlockSize, ctx.dictSize)); if(BMK_benchParam(&winner.result, buf, ctx, winner.params)) { @@ -2247,7 +2239,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ g_lvltarget.cSize /= ((double)g_strictness) / 100; target.cSpeed = (U32)g_lvltarget.cSpeed; - target.dSpeed = (U32)g_lvltarget.dSpeed; //See if this is reasonable. + target.dSpeed = (U32)g_lvltarget.dSpeed; BMK_printWinnerOpt(stdout, cLevelOpt, winner.result, winner.params, target, buf.srcSize); } From 8175b28f03bc49a269c5481f8402b52b712d644c Mon Sep 17 00:00:00 2001 From: George Lu Date: Thu, 16 Aug 2018 16:13:02 -0700 Subject: [PATCH 54/55] Fix negative lvl display value Also fix synthetic benchmark parameter setting --- tests/paramgrill.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index e94eead0d..f42afe403 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -655,7 +655,7 @@ static void BMK_displayOneResult(FILE* f, winnerInfo_t res, const size_t srcSize /* Writes to f the results of a parameter benchmark */ /* when used with --optimize, will only print results better than previously discovered */ -static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const paramValues_t params, const size_t srcSize) +static void BMK_printWinner(FILE* f, const int cLevel, const BMK_result_t result, const paramValues_t params, const size_t srcSize) { char lvlstr[15] = "Custom Level"; winnerInfo_t w; @@ -665,7 +665,7 @@ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result fprintf(f, "\r%79s\r", ""); if(cLevel != CUSTOM_LEVEL) { - snprintf(lvlstr, 15, " Level %2u ", cLevel); + snprintf(lvlstr, 15, " Level %2d ", cLevel); } if(TIMED) { @@ -1865,8 +1865,9 @@ static void BMK_benchFullTable(const buffers_t buf, const contexts_t ctx) * Single Benchmark Functions **************************************/ -static int benchOnce(const buffers_t buf, const contexts_t ctx) { +static int benchOnce(const buffers_t buf, const contexts_t ctx, const int cLevel) { BMK_result_t testResult; + g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevel, buf.maxBlockSize, ctx.dictSize)), g_params), buf.maxBlockSize, ctx.dictSize); if(BMK_benchParam(&testResult, buf, ctx, g_params)) { DISPLAY("Error during benchmarking\n"); @@ -1878,7 +1879,7 @@ static int benchOnce(const buffers_t buf, const contexts_t ctx) { return 0; } -static int benchSample(double compressibility) +static int benchSample(double compressibility, int cLevel) { const char* const name = "Sample 10MB"; size_t const benchedSize = 10 MB; @@ -1912,7 +1913,7 @@ static int benchSample(double compressibility) DISPLAY("using %s %i%%: \n", name, (int)(compressibility*100)); if(g_singleRun) { - ret = benchOnce(buf, ctx); + ret = benchOnce(buf, ctx, cLevel); } else { BMK_benchFullTable(buf, ctx); } @@ -1926,7 +1927,7 @@ static int benchSample(double compressibility) /* benchFiles() : * note: while this function takes a table of filenames, * in practice, only the first filename will be used */ -int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileName, int cLevel) +int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileName, const int cLevel) { buffers_t buf; contexts_t ctx; @@ -1950,10 +1951,8 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam DISPLAY("using %d Files : \n", nbFiles); } - g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevel, buf.maxBlockSize, ctx.dictSize)), g_params), buf.maxBlockSize, ctx.dictSize); - if(g_singleRun) { - ret = benchOnce(buf, ctx); + ret = benchOnce(buf, ctx, cLevel); } else { BMK_benchFullTable(buf, ctx); } @@ -2731,7 +2730,7 @@ int main(int argc, const char** argv) DISPLAY("Optimizer Expects File\n"); return 1; } else { - result = benchSample(compressibility); + result = benchSample(compressibility, cLevelRun); } } else { if(seperateFiles) { From 3959ba15e633776456ebd3ed6bb96db5d775d1cd Mon Sep 17 00:00:00 2001 From: George Lu Date: Thu, 16 Aug 2018 17:22:29 -0700 Subject: [PATCH 55/55] Clarify README --- tests/README.md | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/tests/README.md b/tests/README.md index c1128571a..f28766bd1 100644 --- a/tests/README.md +++ b/tests/README.md @@ -104,34 +104,40 @@ Full list of arguments t# - targetLength S# - strategy L# - level - --zstd= : Single run, parameter selection syntax same as zstdcli. - When invoked with --optimize, this represents the sample to exceed. + --zstd= : Single run, parameter selection syntax same as zstdcli with more parameters + (Added forceAttachDictionary / fadt) + When invoked with --optimize, this represents the sample to exceed. --optimize= : find parameters to maximize compression ratio given parameters - Can use all --zstd= commands to constrain the type of solution found in addition to the following constraints + Can use all --zstd= commands to constrain the type of solution found in addition to the following constraints cSpeed= : Minimum compression speed dSpeed= : Minimum decompression speed cMem= : Maximum compression memory lvl= : Searches for solutions which are strictly better than that compression lvl in ratio and cSpeed, stc= : When invoked with lvl=, represents percentage slack in ratio/cSpeed allowed for a solution to be considered (Default 100%) : In normal operation, represents percentage slack in choosing viable starting strategy selection in choosing the default parameters - (Lower value will begin with stronger strategies) (Default 90%) + (Lower value will begin with stronger strategies) (Default 90%) speedRatio= (accepts decimals) : determines value of gains in speed vs gains in ratio - when determining overall winner (default 5 (1% ratio = 5% speed)). + when determining overall winner (default 5 (1% ratio = 5% speed)). tries= : Maximum number of random restarts on a single strategy before switching (Default 5) - Higher values will make optimizer run longer, more chances to find better solution. - memLog : Limits the log of the size of each memotable (1 per strategy). Setting memLog = 0 turns off memoization + Higher values will make optimizer run longer, more chances to find better solution. + memLog : Limits the log of the size of each memotable (1 per strategy). Will use hash tables when state space is larger than max size. + Setting memLog = 0 turns off memoization --display= : specifiy which parameters are included in the output - can use all --zstd parameter names and 'cParams' as a shorthand for all parameters used in ZSTD_compressionParameters - (Default: display all params available) - - -P# : generated sample compressibility + can use all --zstd parameter names and 'cParams' as a shorthand for all parameters used in ZSTD_compressionParameters + (Default: display all params available) + -P# : generated sample compressibility (when no file is provided) -t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) -v : Prints Benchmarking output -D : Next argument dictionary file -s : Benchmark all files separately -q : Quiet, repeat for more quiet + -q Prints parameters + results whenever a new best is found + -qq Only prints parameters whenever a new best is found, prints final parameters + results + -qqq Only print final parameters + results + -qqqq Only prints final parameter set in the form --zstd= -v : Verbose, cancels quiet, repeat for more volume + -v Prints all candidate parameters and results ``` Any inputs afterwards are treated as files to benchmark.