From cffb6da339970c7a5334e52abdf96493764561f7 Mon Sep 17 00:00:00 2001 From: George Lu Date: Wed, 6 Jun 2018 16:19:09 -0700 Subject: [PATCH 01/19] Parses additional parameters Additional constraint checking Minor fixes more param parsing Add Memory Change paramVariation work on feasibility reformat bench Changed Paramgrill to use bench.c benchmarking customlevel macro Printing Flag Minor changes Explicit casting Makefile fix casting, type fix Printing Flag Minor Changes comments, helper fn's --- programs/bench.c | 2 + programs/bench.h | 1 + tests/paramgrill.c | 369 ++++++++++++++++++++++++++++++++++++++------- 3 files changed, 320 insertions(+), 52 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 76d1ff6dd..2fa9262c9 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -692,6 +692,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( } DISPLAYLEVEL(2, "%2i#\n", cLevel); } /* Bench */ + results.result.cMem = ZSTD_sizeof_CCtx(ctx); return results; } @@ -731,6 +732,7 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, srcBuffer, srcSize, fileSizes, nbFiles, cLevel, comprParams, dictBuffer, dictBufferSize, ctx, dctx, displayLevel, displayName, adv); } + /* clean up */ BMK_freeTimeState(timeStateCompress); BMK_freeTimeState(timeStateDecompress); diff --git a/programs/bench.h b/programs/bench.h index 87cf56380..f1ac255f8 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -34,6 +34,7 @@ typedef struct { size_t cSize; double cSpeed; /* bytes / sec */ double dSpeed; + size_t cMem; } BMK_result_t; ERROR_STRUCT(BMK_result_t, BMK_return_t); diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 025bc6aad..81c6dffc2 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -63,6 +63,16 @@ static const int g_maxNbVariations = 64; #define MAX(a,b) ( (a) > (b) ? (a) : (b) ) #define CUSTOM_LEVEL 99 +/* indices for each of the variables */ +#define WLOG_IND 0 +#define CLOG_IND 1 +#define HLOG_IND 2 +#define SLOG_IND 3 +#define SLEN_IND 4 +#define TLEN_IND 5 +#define STRT_IND 6 +#define NUM_PARAMS 7 + /*-************************************ * Benchmark Parameters **************************************/ @@ -140,6 +150,13 @@ static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) return result; } + +typedef struct { + U32 cSpeed; /* bytes / sec */ + U32 dSpeed; + U32 Mem; /* bytes */ +} constraint_t; + /*-******************************************************* * Bench functions *********************************************************/ @@ -340,7 +357,6 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para return better; } - /* nullified useless params, to ensure count stats */ static ZSTD_compressionParameters* sanitizeParams(ZSTD_compressionParameters params) { @@ -354,8 +370,114 @@ static ZSTD_compressionParameters* sanitizeParams(ZSTD_compressionParameters par return &g_params; } +/* res should be NUM_PARAMS size */ +static int variableParams(const ZSTD_compressionParameters paramConstraints, U32* res) { + int j = 0; + if(!paramConstraints.windowLog) { + res[j] = WLOG_IND; + j++; + } + if(!paramConstraints.chainLog) { + res[j] = CLOG_IND; + j++; + } + if(!paramConstraints.hashLog) { + res[j] = HLOG_IND; + j++; + } + if(!paramConstraints.searchLog) { + res[j] = SLOG_IND; + j++; + } + if(!paramConstraints.searchLength) { + res[j] = SLEN_IND; + j++; + } + if(!paramConstraints.targetLength) { + res[j] = TLEN_IND; + j++; + } + if(!(U32)paramConstraints.strategy) { + res[j] = STRT_IND; + j++; + } + return j; +} -static void paramVariation(ZSTD_compressionParameters* ptr) +/* computes inverse of above array, returns same number, -1 = unused ind */ +static int inverseVariableParams(const ZSTD_compressionParameters paramConstraints, U32* res) { + int j = 0; + if(!paramConstraints.windowLog) { + res[WLOG_IND] = j; + j++; + } else { + res[WLOG_IND] = -1; + } + if(!paramConstraints.chainLog) { + res[j] = CLOG_IND; + j++; + } else { + res[WLOG_IND] = -1; + } + if(!paramConstraints.hashLog) { + res[j] = HLOG_IND; + j++; + } else { + res[WLOG_IND] = -1; + } + if(!paramConstraints.searchLog) { + res[j] = SLOG_IND; + j++; + } else { + res[WLOG_IND] = -1; + } + if(!paramConstraints.searchLength) { + res[j] = SLEN_IND; + j++; + } else { + res[WLOG_IND] = -1; + } + if(!paramConstraints.targetLength) { + res[j] = TLEN_IND; + j++; + } else { + res[WLOG_IND] = -1; + } + if(!(U32)paramConstraints.strategy) { + res[j] = STRT_IND; + j++; + } else { + res[WLOG_IND] = -1; + } + return j; +} + +/* amt will probably always be \pm 1? */ +/* slight change from old paramVariation, targetLength can only take on powers of 2 now (999 ~= 1024?) */ +static void paramVaryOnce(U32 paramIndex, int amt, ZSTD_compressionParameters* ptr) { + switch(paramIndex) + { + case WLOG_IND: ptr->windowLog += amt; break; + case CLOG_IND: ptr->chainLog += amt; break; + case HLOG_IND: ptr->hashLog += amt; break; + case SLOG_IND: ptr->searchLog += amt; break; + case SLEN_IND: ptr->searchLength += amt; break; + case TLEN_IND: + if(amt > 0) { + ptr->targetLength <<= amt; + } else { + ptr->targetLength >>= -amt; + } + break; + case STRT_IND: ptr->strategy += amt; break; + default: break; + } +} + +//Don't fuzz fixed variables. +//turn pcs to pcs array with macro for params. +//pass in variation array from variableParams +static void paramVariation(ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen) { ZSTD_compressionParameters p; U32 validated = 0; @@ -363,40 +485,13 @@ static void paramVariation(ZSTD_compressionParameters* ptr) U32 nbChanges = (FUZ_rand(&g_rand) & 3) + 1; p = *ptr; for ( ; nbChanges ; nbChanges--) { - const U32 changeID = FUZ_rand(&g_rand) % 14; - switch(changeID) - { - case 0: - p.chainLog++; break; - case 1: - p.chainLog--; break; - case 2: - p.hashLog++; break; - case 3: - p.hashLog--; break; - case 4: - p.searchLog++; break; - case 5: - p.searchLog--; break; - case 6: - p.windowLog++; break; - case 7: - p.windowLog--; break; - case 8: - p.searchLength++; break; - case 9: - p.searchLength--; break; - case 10: - p.strategy = (ZSTD_strategy)(((U32)p.strategy)+1); break; - case 11: - p.strategy = (ZSTD_strategy)(((U32)p.strategy)-1); break; - case 12: - p.targetLength *= 1 + ((double)(FUZ_rand(&g_rand)&255)) / 256.; break; - case 13: - p.targetLength /= 1 + ((double)(FUZ_rand(&g_rand)&255)) / 256.; break; - } + const U32 changeID = FUZ_rand(&g_rand) % (2 * varyLen); + paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p); } validated = !ZSTD_isError(ZSTD_checkCParams(p)); + + //Make sure memory is at least close to feasible? + //ZSTD_estimateCCtxSize thing. } *ptr = p; } @@ -418,12 +513,14 @@ static void playAround(FILE* f, winnerInfo_t* winners, { int nbVariations = 0; UTIL_time_t const clockStart = UTIL_getTime(); + const U32 unconstrained[NUM_PARAMS] = { 0, 1, 2, 3, 4, 5, 6 }; + while (UTIL_clockSpanMicro(clockStart) < g_maxVariationTime) { ZSTD_compressionParameters p = params; if (nbVariations++ > g_maxNbVariations) break; - paramVariation(&p); + paramVariation(&p, unconstrained, 7); /* exclude faster if already played params */ if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(p))-1)) @@ -459,6 +556,24 @@ static ZSTD_compressionParameters randomParams(void) return p; } +static ZSTD_compressionParameters randomConstrainedParams(ZSTD_compressionParameters pc) +{ + ZSTD_compressionParameters p; + U32 validated = 0; + while (!validated) { + /* totally random entry */ + if(!pc.chainLog) p.chainLog = (FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN)) + ZSTD_CHAINLOG_MIN; + if(!pc.chainLog) p.hashLog = (FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN)) + ZSTD_HASHLOG_MIN; + if(!pc.chainLog) p.searchLog = (FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN)) + ZSTD_SEARCHLOG_MIN; + if(!pc.chainLog) p.windowLog = (FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN)) + ZSTD_WINDOWLOG_MIN; + if(!pc.chainLog) p.searchLength=(FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN)) + ZSTD_SEARCHLENGTH_MIN; + if(!pc.chainLog) p.targetLength=(FUZ_rand(&g_rand) % (512)) + 1; //ZSTD_TARGETLENGTH_MIN; //change to 2^[0,10?] + if(!pc.chainLog) p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btultra +1)); + validated = !ZSTD_isError(ZSTD_checkCParams(p)); + } + return p; +} + static void BMK_selectRandomStart( FILE* f, winnerInfo_t* winners, const void* srcBuffer, size_t srcSize, @@ -638,14 +753,86 @@ static void BMK_translateAdvancedParams(ZSTD_compressionParameters params) params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, params.targetLength, (U32)(params.strategy)); } +//Results currently don't capture memory usage or anything. +//parameter feasibility is not checked, should just be restricted from use. +static int feasible(BMK_result_t results, constraint_t target) { + return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.Mem || !target.Mem); +} + +/* returns 1 if result2 is strictly 'better' than result1 */ +static int objective_lt(BMK_result_t result1, BMK_result_t result2) { + return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed); +} + +/* res gives array dimensions, should be size NUM_PARAMS */ +static size_t computeStateSize(const ZSTD_compressionParameters paramConstraints, U32* res) { + int ind = 0; + size_t base = 1; + if(!paramConstraints.windowLog) { res[ind] = ZSTD_WINDOWLOG_MAX - ZSTD_WINDOWLOG_MIN + 1; base *= res[ind]; ind++; } + if(!paramConstraints.chainLog) { res[ind] = ZSTD_CHAINLOG_MAX - ZSTD_CHAINLOG_MIN + 1; base *= res[ind]; ind++; } + if(!paramConstraints.hashLog) { res[ind] = ZSTD_HASHLOG_MAX - ZSTD_HASHLOG_MIN + 1; base *= res[ind]; ind++; } + if(!paramConstraints.searchLog) { res[ind] = ZSTD_SEARCHLOG_MAX - ZSTD_SEARCHLOG_MIN + 1; base *= res[ind]; ind++; } + if(!paramConstraints.searchLength) { res[ind] = ZSTD_SEARCHLENGTH_MAX - ZSTD_SEARCHLENGTH_MIN + 1; base *= res[ind]; ind++; } + if(!paramConstraints.targetLength) { res[ind] = 11; base *= res[ind]; ind++; } //restricting from 2^[0,10], no such macros + if(!(U32)paramConstraints.strategy) { res[ind] = 8; base *= 8; } //not strictly true, maybe would want to case on this. + + return base; +} + + +static unsigned calcViolation(BMK_result_t results, constraint_t target) { + int diffcSpeed = MAX(target.cSpeed - results.cSpeed, 0); + int diffdSpeed = MAX(target.dSpeed - results.dSpeed, 0); + int diffcMem = MAX(results.cMem - target.Mem, 0); + return diffcSpeed + diffdSpeed + diffcMem; +} + +/* finds some set of parameters which fulfills req's + * Prioritize highest / try to locally minimize sum? + * Is it ever useful to go out of the param constraints? ? + * random / perturb when revisit? + * momentum? + */ +static ZSTD_compressionParameters findFeasible(constraint_t target, ZSTD_compressionParameters paramTarget) { + unsigned violation; + + ZSTD_compressionParameters winner = randomConstrainedParams(paramTarget); + //just use g_alreadyTested and xxhash? + BYTE* memotable; + do { + //prioritize memory + if(diffcMem >= diffcSpeed && diffcMem >= diffdSpeed) { + + //prioritize compression Speed + } else if (diffcSpeeed >= diffdSpeed && diffcSpeed >= diffcMem) { + + //prioritize decompressionSpeed + } else { + + } + violation = calcViolation(result, target); + } while(objective); + if(validate) { + DISPLAY("Feasible Point Found\n"); + return winner; + } else { + DISPLAY("No solution found\n"); + ZSTD_compressionParameters ret = { 0, 0, 0, 0, 0, 0, 0 }; + return ret; + } +} + /* optimizeForSize(): - * targetSpeed : expressed in MB/s */ -int optimizeForSize(const char* inFileName, U32 targetSpeed) + * targetSpeed : expressed in B/s */ +/* if state space is small (from paramTarget) */ +int optimizeForSize(const char* inFileName, constraint_t target, ZSTD_compressionParameters paramTarget) { FILE* const inFile = fopen( inFileName, "rb" ); U64 const inFileSize = UTIL_getFileSize(inFileName); size_t benchedSize = BMK_findMaxMem(inFileSize*3) / 3; void* origBuff; + U32 paramVarArray [NUM_PARAMS]; + int paramCount = variableParams(paramTarget, paramVarArray); /* Init */ if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; } if (inFileSize == UTIL_FILESIZE_UNKNOWN) { @@ -682,8 +869,11 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed) /* bench */ DISPLAY("\r%79s\r", ""); - DISPLAY("optimizing for %s - limit speed %u MB/s \n", inFileName, targetSpeed); - targetSpeed *= 1000000; + DISPLAY("optimizing for %s", inFileName); + if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed / 1000000); } + if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed / 1000000); } + if(target.Mem != 0) { DISPLAY(" - limit memory %u MB", target.Mem / 1000000); } + DISPLAY("\n"); { ZSTD_CCtx* const ctx = ZSTD_createCCtx(); ZSTD_DCtx* const dctx = ZSTD_createDCtx(); winnerInfo_t winner; @@ -696,23 +886,22 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed) winner.result.cSize = (size_t)(-1); /* find best solution from default params */ + //Can't do this iteration normally w/ cparameter constraints { const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); int i; for (i=1; i<=maxSeeds; i++) { ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, 0); BMK_benchParam(&candidate, origBuff, benchedSize, ctx, dctx, CParams); - if (candidate.cSpeed < (double)targetSpeed) { + if (!feasible(candidate, target) ) { break; } - if ( (candidate.cSize < winner.result.cSize) - | ((candidate.cSize == winner.result.cSize) & (candidate.cSpeed > winner.result.cSpeed)) ) + if (feasible(candidate,target) && objective_lt(winner.result, candidate)) { winner.params = CParams; winner.result = candidate; BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); } } } - BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize); BMK_translateAdvancedParams(winner.params); @@ -721,7 +910,7 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed) { time_t const grillStart = time(NULL); do { ZSTD_compressionParameters params = winner.params; - paramVariation(¶ms); + paramVariation(¶ms, paramVarArray, paramCount); if ((FUZ_rand(&g_rand) & 31) == 3) params = randomParams(); /* totally random config to improve search space */ params = ZSTD_adjustCParams(params, blockSize, 0); @@ -733,9 +922,7 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed) BMK_benchParam(&candidate, origBuff, benchedSize, ctx, dctx, params); /* improvement found => new winner */ - if ( (candidate.cSpeed > targetSpeed) - & ( (candidate.cSize < winner.result.cSize) - | ((candidate.cSize == winner.result.cSize) & (candidate.cSpeed > winner.result.cSpeed)) ) ) + if (feasible(candidate,target) && objective_lt(winner.result, candidate)) { winner.params = params; winner.result = candidate; @@ -744,8 +931,13 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed) } } while (BMK_timeSpan(grillStart) < g_grillDuration_s); } - /* end summary */ + /* no solution found */ + if(winner.result.cSize == (size_t)-1) { + DISPLAY("No feasible solution found\n"); + return 1; + } + /* end summary */ BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize); BMK_translateAdvancedParams(winner.params); DISPLAY("grillParams size - optimizer completed \n"); @@ -834,7 +1026,9 @@ int main(int argc, const char** argv) const char* input_filename=0; U32 optimizer = 0; U32 main_pause = 0; - U32 targetSpeed = 0; + + constraint_t target = { 0 , 0, 0 }; //0 for anything unset + ZSTD_compressionParameters paramTarget = { 0, 0, 0, 0, 0, 0, 0 }; assert(argc>=1); /* for exename */ @@ -847,8 +1041,31 @@ int main(int argc, const char** argv) if(!strcmp(argument,"--no-seed")) { g_noSeed = 1; continue; } + if (longCommandWArg(&argument, "--optimize=")) { + optimizer = 1; + for ( ; ;) { + if (longCommandWArg(&argument, "windowLog=") || longCommandWArg(&argument, "wlog=")) { paramTarget.windowLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "chainLog=") || longCommandWArg(&argument, "clog=")) { paramTarget.chainLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "hashLog=") || longCommandWArg(&argument, "hlog=")) { paramTarget.hashLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "searchLog=") || longCommandWArg(&argument, "slog=")) { paramTarget.searchLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "searchLength=") || longCommandWArg(&argument, "slen=")) { paramTarget.searchLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "targetLength=") || longCommandWArg(&argument, "tlen=")) { paramTarget.targetLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "strategy=") || longCommandWArg(&argument, "strat=")) { paramTarget.strategy = (ZSTD_strategy)(readU32FromChar(&argument)); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "compressionSpeed=") || longCommandWArg(&argument, "cSpeed=")) { target.cSpeed = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "decompressionSpeed=") || longCommandWArg(&argument, "dSpeed=")) { target.dSpeed = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "compressionMemory=") || longCommandWArg(&argument, "cMem=")) { target.Mem = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } + /* in MB or MB/s */ + DISPLAY("invalid optimization parameter \n"); + return 1; + } + + if (argument[0] != 0) { + DISPLAY("invalid --optimize= format\n"); + return 1; /* check the end of string */ + } + continue; + } else if (longCommandWArg(&argument, "--zstd=")) { /* Decode command (note : aggregated commands are allowed) */ - if (longCommandWArg(&argument, "--zstd=")) { g_singleRun = 1; g_params = ZSTD_getCParams(2, g_blockSize, 0); for ( ; ;) { @@ -868,6 +1085,7 @@ int main(int argc, const char** argv) DISPLAY("invalid --zstd= format\n"); return 1; /* check the end of string */ } + continue; /* if not return, success */ } else if (argument[0]=='-') { argument++; @@ -900,7 +1118,54 @@ int main(int argc, const char** argv) case 'O': argument++; optimizer = 1; - targetSpeed = readU32FromChar(&argument); + for ( ; ; ) { + switch(*argument) + { + /* Inputs in MB or MB/s */ + case 'C': + argument++; + target.cSpeed = readU32FromChar(&argument) * 1000000; + continue; + case 'D': + argument++; + target.dSpeed = readU32FromChar(&argument) * 1000000; + continue; + case 'M': + argument++; + target.Mem = readU32FromChar(&argument) * 1000000; + continue; + case 'w': + argument++; + paramTarget.windowLog = readU32FromChar(&argument); + continue; + case 'c': + argument++; + paramTarget.chainLog = readU32FromChar(&argument); + continue; + case 'h': + argument++; + paramTarget.hashLog = readU32FromChar(&argument); + continue; + case 's': + argument++; + paramTarget.searchLog = readU32FromChar(&argument); + continue; + case 'l': /* search length */ + argument++; + paramTarget.searchLength = readU32FromChar(&argument); + continue; + case 't': /* target length */ + argument++; + paramTarget.targetLength = readU32FromChar(&argument); + continue; + case 'S': /* strategy */ + argument++; + paramTarget.strategy = (ZSTD_strategy)readU32FromChar(&argument); + continue; + default : ; + } + break; + } break; /* Run Single conf */ @@ -990,7 +1255,7 @@ int main(int argc, const char** argv) } } else { if (optimizer) { - result = optimizeForSize(input_filename, targetSpeed); + result = optimizeForSize(input_filename, target, paramTarget); } else { result = benchFiles(argv+filenamesStart, argc-filenamesStart); } } From 5f49034520940af38deb5e13d9d624afc7ffe534 Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 18 Jun 2018 11:59:45 -0700 Subject: [PATCH 02/19] Working V1 --- programs/bench.c | 95 ++-- programs/bench.h | 3 + tests/Makefile | 2 +- tests/paramgrill.c | 1256 ++++++++++++++++++++++++++++++++++++++++---- 4 files changed, 1213 insertions(+), 143 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 2fa9262c9..0d35f3eb3 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -85,7 +85,7 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; * Exceptions ***************************************/ #ifndef DEBUG -# define DEBUG 0 +# define DEBUG 1 #endif #define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); } @@ -281,12 +281,14 @@ static size_t local_defaultDecompress( } +volatile char g_touched; + /* initFn will be measured once, bench fn will be measured x times */ /* benchFn should return error value or out Size */ /* takes # of blocks and list of size & stuff for each. */ /* only does looping */ /* note time per loop could be zero if interval too short */ -BMK_customReturn_t BMK_benchFunction( +BMK_customReturn_t __attribute__((optimize("O0"))) BMK_benchFunction( BMK_benchFn_t benchFn, void* benchPayload, BMK_initFn_t initFn, void* initPayload, size_t blockCount, @@ -299,16 +301,6 @@ BMK_customReturn_t BMK_benchFunction( BMK_customReturn_t retval; UTIL_time_t clockStart; - { - unsigned i; - for(i = 0; i < blockCount; i++) { - memset(dstBlockBuffers[i], 0xE5, dstBlockCapacities[i]); /* warm up and erase result buffer */ - } - - UTIL_sleepMilli(5); /* give processor time to other processes */ - UTIL_waitForNextTick(); - } - if(!nbLoops) { EXM_THROW_ND(1, BMK_customReturn_t, "nbLoops must be nonzero \n"); } @@ -317,6 +309,22 @@ BMK_customReturn_t BMK_benchFunction( srcSize += srcBlockSizes[ind]; } + { + unsigned i, j; + for(i = 0; i < blockCount; i++) { + for(j = 0; j < srcBlockSizes[i]; j++) { + g_touched = ((const char*)srcBlockBuffers[i])[j]; /* touch */ + } + } + for(i = 0; i < blockCount; i++) { + memset(dstBlockBuffers[i], 0xE5, dstBlockCapacities[i]); /* warm up and erase result buffer */ + //this is written at end proc? where did compressed data get overwritten ny this? + } + + UTIL_sleepMilli(5); /* give processor time to other processes */ + UTIL_waitForNextTick(); + } + { unsigned i, j, firstIter = 1; clockStart = UTIL_getTime(); @@ -327,9 +335,9 @@ BMK_customReturn_t BMK_benchFunction( if(ZSTD_isError(res)) { EXM_THROW_ND(2, BMK_customReturn_t, "Function benchmarking failed on block %u of size %u : %s \n", j, (U32)dstBlockCapacities[j], ZSTD_getErrorName(res)); - } else if(firstIter) { + } else if(firstIter) { dstSize += res; - } + } } firstIter = 0; } @@ -393,7 +401,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed( { U64 const loopDuration = r.result.result.nanoSecPerRun * cont->nbLoops; r.completed = (cont->timeRemaining <= loopDuration); cont->timeRemaining -= loopDuration; - if (loopDuration > 0) { + if (loopDuration > (TIMELOOP_NANOSEC / 100)) { fastest = MIN(fastest, r.result.result.nanoSecPerRun); if(loopDuration >= MINUSABLETIME) { r.result.result.nanoSecPerRun = fastest; @@ -420,7 +428,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( const void ** const srcPtrs, size_t* const srcSizes, void** const cPtrs, size_t* const cSizes, void** const resPtrs, size_t* const resSizes, - void* resultBuffer, void* compressedBuffer, + void** resultBufferPtr, void* compressedBuffer, const size_t maxCompressedSize, BMK_timedFnState_t* timeStateCompress, BMK_timedFnState_t* timeStateDecompress, @@ -432,7 +440,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( int displayLevel, const char* displayName, const BMK_advancedParams_t* adv) { size_t const blockSize = ((adv->blockSize>=32 && (adv->mode != BMK_decodeOnly)) ? adv->blockSize : srcSize) + (!srcSize); /* avoid div by 0 */ - BMK_return_t results; + BMK_return_t results = { { 0, 0., 0., 0 }, 0 } ; size_t const loadedCompressedSize = srcSize; size_t cSize = 0; double ratio = 0.; @@ -454,13 +462,14 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( srcPtr += fileSizes[fileNb]; } { size_t const decodedSize = (size_t)totalDSize64; - free(resultBuffer); - resultBuffer = malloc(decodedSize); - if (!resultBuffer) { + free(*resultBufferPtr); + //TODO: decodedSize corrupted? + *resultBufferPtr = malloc(decodedSize); + if (!(*resultBufferPtr)) { EXM_THROW(33, BMK_return_t, "not enough memory"); } if (totalDSize64 > decodedSize) { - free(resultBuffer); + free(*resultBufferPtr); EXM_THROW(32, BMK_return_t, "original size is too large"); /* size_t overflow */ } cSize = srcSize; @@ -472,7 +481,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( /* Init data blocks */ { const char* srcPtr = (const char*)srcBuffer; char* cPtr = (char*)compressedBuffer; - char* resPtr = (char*)resultBuffer; + char* resPtr = (char*)(*resultBufferPtr); U32 fileNb; for (nbBlocks=0, fileNb=0; fileNbmode != BMK_decodeOnly) { + BMK_customReturn_t compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep, - nbBlocks, srcPtrs, srcSizes, cPtrs, cSizes, adv->nbSeconds); + nbBlocks, srcPtrs, srcSizes, cPtrs, cSizes, adv->nbSeconds); if(compressionResults.error) { results.error = compressionResults.error; return results; @@ -642,7 +652,8 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( } /* CRC Checking */ - { U64 const crcCheck = XXH64(resultBuffer, srcSize, 0); + { void* resultBuffer = *resultBufferPtr; + U64 const crcCheck = XXH64(resultBuffer, srcSize, 0); /* adv->mode == 0 -> compress + decompress */ if ((adv->mode == BMK_both) && (crcOrig!=crcCheck)) { size_t u; @@ -692,11 +703,13 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( } DISPLAYLEVEL(2, "%2i#\n", cLevel); } /* Bench */ - results.result.cMem = ZSTD_sizeof_CCtx(ctx); + results.result.cMem = (1 << (comprParams->windowLog)) + ZSTD_sizeof_CCtx(ctx); + results.error = 0; return results; } BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, + void* dstBuffer, size_t dstCapacity, const size_t* fileSizes, unsigned nbFiles, const int cLevel, const ZSTD_compressionParameters* comprParams, const void* dictBuffer, size_t dictBufferSize, @@ -717,26 +730,41 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, void ** const resPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); size_t* const resSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); - const size_t maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */ - void* compressedBuffer = malloc(maxCompressedSize); - void* resultBuffer = malloc(srcSize); BMK_timedFnState_t* timeStateCompress = BMK_createTimeState(adv->nbSeconds); BMK_timedFnState_t* timeStateDecompress = BMK_createTimeState(adv->nbSeconds); + void* compressedBuffer; + const size_t maxCompressedSize = dstCapacity ? dstCapacity : ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); + void* resultBuffer = malloc(srcSize); + + BMK_return_t results; - int allocationincomplete = !compressedBuffer || !resultBuffer || + int allocationincomplete; + + if(!dstCapacity) { + compressedBuffer = malloc(maxCompressedSize); + } else { + compressedBuffer = dstBuffer; + } + + allocationincomplete = !compressedBuffer || !resultBuffer || !srcPtrs || !srcSizes || !cPtrs || !cSizes || !resPtrs || !resSizes; + if (!allocationincomplete) { results = BMK_benchMemAdvancedNoAlloc(srcPtrs, srcSizes, cPtrs, cSizes, - resPtrs, resSizes, resultBuffer, compressedBuffer, maxCompressedSize, timeStateCompress, timeStateDecompress, + resPtrs, resSizes, &resultBuffer, compressedBuffer, maxCompressedSize, timeStateCompress, timeStateDecompress, srcBuffer, srcSize, fileSizes, nbFiles, cLevel, comprParams, dictBuffer, dictBufferSize, ctx, dctx, displayLevel, displayName, adv); } + + /* clean up */ BMK_freeTimeState(timeStateCompress); BMK_freeTimeState(timeStateDecompress); - free(compressedBuffer); + if(!dstCapacity) { /* only free if not given */ + free(compressedBuffer); + } free(resultBuffer); free((void*)srcPtrs); @@ -749,7 +777,6 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, if(allocationincomplete) { EXM_THROW(31, BMK_return_t, "allocation error : not enough memory"); } - results.error = 0; return results; } @@ -762,6 +789,7 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize, const BMK_advancedParams_t adv = BMK_initAdvancedParams(); return BMK_benchMemAdvanced(srcBuffer, srcSize, + NULL, 0, fileSizes, nbFiles, cLevel, comprParams, dictBuffer, dictBufferSize, @@ -783,6 +811,7 @@ static BMK_return_t BMK_benchMemCtxless(const void* srcBuffer, size_t srcSize, EXM_THROW(12, BMK_return_t, "not enough memory for contexts"); } res = BMK_benchMemAdvanced(srcBuffer, srcSize, + NULL, 0, fileSizes, nbFiles, cLevel, comprParams, dictBuffer, dictBufferSize, diff --git a/programs/bench.h b/programs/bench.h index f1ac255f8..625b65757 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -122,6 +122,8 @@ BMK_return_t BMK_syntheticTest(int cLevel, double compressibility, * (cLevel, comprParams + adv in advanced Mode) */ /* srcBuffer - data source, expected to be valid compressed data if in Decode Only Mode * srcSize - size of data in srcBuffer + * dstBuffer - destination buffer to write compressed output in, optional (NULL) + * dstCapacity - capacity of destination buffer, give 0 if dstBuffer = NULL * cLevel - compression level * comprParams - basic compression parameters * dictBuffer - a dictionary if used, null otherwise @@ -144,6 +146,7 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize, /* See benchMem for normal parameter uses and return, see advancedParams_t for adv */ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, + void* dstBuffer, size_t dstCapacity, const size_t* fileSizes, unsigned nbFiles, const int cLevel, const ZSTD_compressionParameters* comprParams, const void* dictBuffer, size_t dictBufferSize, diff --git a/tests/Makefile b/tests/Makefile index 81e685780..ecf7fe2a7 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -200,7 +200,7 @@ zstreamtest-dll : $(ZSTDDIR)/common/xxhash.c # xxh symbols not exposed from dll zstreamtest-dll : $(ZSTREAM_LOCAL_FILES) $(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT) -paramgrill : DEBUGFLAGS = # turn off assert() for speed measurements +#paramgrill : DEBUGFLAGS = # turn off assert() for speed measurements paramgrill : $(ZSTD_FILES) $(PRGDIR)/bench.c $(PRGDIR)/datagen.c paramgrill.c $(CC) $(FLAGS) $^ -lm -o $@$(EXT) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 81c6dffc2..44bcedef8 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -41,6 +41,8 @@ #define MB *(1<<20) #define GB *(1ULL<<30) +#define TIMELOOP_NANOSEC (1*1000000000ULL) /* 1 second */ + #define NBLOOPS 2 #define TIMELOOP (2 * SEC_TO_MICRO) #define NB_LEVELS_TRACKED 22 /* ensured being >= ZSTD_maxCLevel() in BMK_init_level_constraints() */ @@ -70,13 +72,39 @@ static const int g_maxNbVariations = 64; #define SLOG_IND 3 #define SLEN_IND 4 #define TLEN_IND 5 -#define STRT_IND 6 -#define NUM_PARAMS 7 +//#define STRT_IND 6 +//#define NUM_PARAMS 7 +#define NUM_PARAMS 6 +//just don't use strategy as a param. +#undef ZSTD_WINDOWLOG_MAX +#define ZSTD_WINDOWLOG_MAX 27 //no long range stuff for now. + +//make 2^[0,10] w/ 999 +#define ZSTD_TARGETLENGTH_MIN 0 //actually targeLengthlog min +#define ZSTD_TARGETLENGTH_MAX 10 + +//#define ZSTD_TARGETLENGTH_MAX 1024 +#define WLOG_RANGE (ZSTD_WINDOWLOG_MAX - ZSTD_WINDOWLOG_MIN + 1) +#define CLOG_RANGE (ZSTD_CHAINLOG_MAX - ZSTD_CHAINLOG_MIN + 1) +#define HLOG_RANGE (ZSTD_HASHLOG_MAX - ZSTD_HASHLOG_MIN + 1) +#define SLOG_RANGE (ZSTD_SEARCHLOG_MAX - ZSTD_SEARCHLOG_MIN + 1) +#define SLEN_RANGE (ZSTD_SEARCHLENGTH_MAX - ZSTD_SEARCHLENGTH_MIN + 1) +#define TLEN_RANGE 11 +//hard coded since we only use powers of 2 (and 999 ~ 1024) + +static const int mintable[NUM_PARAMS] = { ZSTD_WINDOWLOG_MIN, ZSTD_CHAINLOG_MIN, ZSTD_HASHLOG_MIN, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLENGTH_MIN, ZSTD_TARGETLENGTH_MIN }; +static const int maxtable[NUM_PARAMS] = { ZSTD_WINDOWLOG_MAX, ZSTD_CHAINLOG_MAX, ZSTD_HASHLOG_MAX, ZSTD_SEARCHLOG_MAX, ZSTD_SEARCHLENGTH_MAX, ZSTD_TARGETLENGTH_MAX }; +static const int rangetable[NUM_PARAMS] = { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE }; + +//use grid-search or something when space is small enough? +#define SMALL_SEARCH_SPACE 1000 /*-************************************ * Benchmark Parameters **************************************/ +typedef BYTE U8; + static double g_grillDuration_s = 99999; /* about 27 hours */ static U32 g_nbIterations = NBLOOPS; static double g_compressibility = COMPRESSIBILITY_DEFAULT; @@ -150,13 +178,74 @@ static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) return result; } +//assume that clock can at least measure .01 second intervals? +//make this a settable global initialized with fn? +//#define CLOCK_GRANULARITY 100000000ULL +static U64 g_clockGranularity = 100000000ULL; + +static void findClockGranularity(void) { + UTIL_time_t clockStart = UTIL_getTime(); + U64 el1 = 0, el2 = 0; + int i = 0; + do { + el1 = el2; + el2 = UTIL_clockSpanNano(clockStart); + if(el1 < el2) { + U64 iv = el2 - el1; + if(g_clockGranularity > iv) { + g_clockGranularity = iv; + i = 0; + } else { + i++; + } + } + } while(i < 10); + DISPLAY("Granularity: %llu\n", (unsigned long long)g_clockGranularity); +} typedef struct { U32 cSpeed; /* bytes / sec */ U32 dSpeed; - U32 Mem; /* bytes */ + U32 cMem; /* bytes */ } constraint_t; +#define CLAMPCHECK(val,min,max) { \ + if (val && (((val)<(min)) | ((val)>(max)))) { \ + DISPLAY("INVALID PARAMETER CONSTRAINTS\n"); \ + return 0; \ +} } + + +/* Like ZSTD_checkCParams() but allows 0's */ +/* no check on targetLen? */ +static int cParamValid(ZSTD_compressionParameters paramTarget) { + CLAMPCHECK(paramTarget.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); + CLAMPCHECK(paramTarget.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX); + CLAMPCHECK(paramTarget.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX); + CLAMPCHECK(paramTarget.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX); + CLAMPCHECK(paramTarget.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX); + if(paramTarget.strategy > ZSTD_btultra) { + DISPLAY("INVALID PARAMETER CONSTRAINTS\n"); + return 0; + } + return 1; +} + +static void cParamZeroMin(ZSTD_compressionParameters* paramTarget) { + paramTarget->windowLog = paramTarget->windowLog ? paramTarget->windowLog : ZSTD_WINDOWLOG_MIN; + paramTarget->searchLog = paramTarget->searchLog ? paramTarget->searchLog : ZSTD_SEARCHLOG_MIN; + paramTarget->chainLog = paramTarget->chainLog ? paramTarget->chainLog : ZSTD_CHAINLOG_MIN; + paramTarget->hashLog = paramTarget->hashLog ? paramTarget->hashLog : ZSTD_HASHLOG_MIN; + paramTarget->searchLength = paramTarget->searchLength ? paramTarget->searchLength : ZSTD_SEARCHLENGTH_MIN; + paramTarget->targetLength = paramTarget->targetLength ? paramTarget->targetLength : 1; +} + +static void BMK_translateAdvancedParams(ZSTD_compressionParameters params) +{ + DISPLAY("--zstd=windowLog=%u,chainLog=%u,hashLog=%u,searchLog=%u,searchLength=%u,targetLength=%u,strategy=%u \n", + params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, params.targetLength, (U32)(params.strategy)); +} + /*-******************************************************* * Bench functions *********************************************************/ @@ -357,20 +446,45 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para return better; } +/* bounds check in sanitize too? */ +#define CLAMP(var, lo, hi) { \ + var = MAX(MIN(var, hi), lo); \ +} + /* nullified useless params, to ensure count stats */ -static ZSTD_compressionParameters* sanitizeParams(ZSTD_compressionParameters params) +/* no point in windowLog < chainLog (no point 2x chainLog for bt) */ +/* now with built in bounds-checking */ +/* no longer does anything with sanitizeVarArray + clampcheck */ +static ZSTD_compressionParameters sanitizeParams(ZSTD_compressionParameters params) { - g_params = params; if (params.strategy == ZSTD_fast) g_params.chainLog = 0, g_params.searchLog = 0; if (params.strategy == ZSTD_dfast) g_params.searchLog = 0; - if (params.strategy != ZSTD_btopt && params.strategy != ZSTD_btultra) + if (params.strategy != ZSTD_btopt && params.strategy != ZSTD_btultra && params.strategy != ZSTD_fast) g_params.targetLength = 0; - return &g_params; + + return params; +} + +/* new length */ +/* keep old array, will need if iter over strategy. */ +static int sanitizeVarArray(int varLength, U32* varArray, U32* varNew, ZSTD_strategy strat) { + int i, j = 0; + for(i = 0; i < varLength; i++) { + if( !((varArray[i] == CLOG_IND && strat == ZSTD_fast) + || (varArray[i] == SLOG_IND && strat == ZSTD_dfast) + || (varArray[i] == TLEN_IND && strat != ZSTD_btopt && strat != ZSTD_btultra && strat != ZSTD_fast))) { + varNew[j] = varArray[i]; + j++; + } + } + return j; + } /* res should be NUM_PARAMS size */ +/* constructs varArray from ZSTD_compressionParameters style parameter */ static int variableParams(const ZSTD_compressionParameters paramConstraints, U32* res) { int j = 0; if(!paramConstraints.windowLog) { @@ -397,10 +511,6 @@ static int variableParams(const ZSTD_compressionParameters paramConstraints, U32 res[j] = TLEN_IND; j++; } - if(!(U32)paramConstraints.strategy) { - res[j] = STRT_IND; - j++; - } return j; } @@ -417,43 +527,39 @@ static int inverseVariableParams(const ZSTD_compressionParameters paramConstrain res[j] = CLOG_IND; j++; } else { - res[WLOG_IND] = -1; + res[CLOG_IND] = -1; } if(!paramConstraints.hashLog) { res[j] = HLOG_IND; j++; } else { - res[WLOG_IND] = -1; + res[HLOG_IND] = -1; } if(!paramConstraints.searchLog) { res[j] = SLOG_IND; j++; } else { - res[WLOG_IND] = -1; + res[SLOG_IND] = -1; } if(!paramConstraints.searchLength) { res[j] = SLEN_IND; j++; } else { - res[WLOG_IND] = -1; + res[SLEN_IND] = -1; } if(!paramConstraints.targetLength) { res[j] = TLEN_IND; j++; } else { - res[WLOG_IND] = -1; - } - if(!(U32)paramConstraints.strategy) { - res[j] = STRT_IND; - j++; - } else { - res[WLOG_IND] = -1; + res[TLEN_IND] = -1; } + return j; } /* amt will probably always be \pm 1? */ /* slight change from old paramVariation, targetLength can only take on powers of 2 now (999 ~= 1024?) */ +/* take max/min bounds into account as well? */ static void paramVaryOnce(U32 paramIndex, int amt, ZSTD_compressionParameters* ptr) { switch(paramIndex) { @@ -463,13 +569,16 @@ static void paramVaryOnce(U32 paramIndex, int amt, ZSTD_compressionParameters* p case SLOG_IND: ptr->searchLog += amt; break; case SLEN_IND: ptr->searchLength += amt; break; case TLEN_IND: - if(amt > 0) { + if(amt >= 0) { ptr->targetLength <<= amt; + ptr->targetLength = MIN(ptr->targetLength, 999); } else { + if(ptr->targetLength == 999) { + ptr->targetLength = 1024; + } ptr->targetLength >>= -amt; } break; - case STRT_IND: ptr->strategy += amt; break; default: break; } } @@ -477,34 +586,143 @@ static void paramVaryOnce(U32 paramIndex, int amt, ZSTD_compressionParameters* p //Don't fuzz fixed variables. //turn pcs to pcs array with macro for params. //pass in variation array from variableParams -static void paramVariation(ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen) +//take nbChanges as argument? +static void paramVariation(ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen, U32 nbChanges) { ZSTD_compressionParameters p; U32 validated = 0; while (!validated) { - U32 nbChanges = (FUZ_rand(&g_rand) & 3) + 1; + U32 i; p = *ptr; - for ( ; nbChanges ; nbChanges--) { - const U32 changeID = FUZ_rand(&g_rand) % (2 * varyLen); + for (i = 0 ; i < nbChanges ; i++) { + const U32 changeID = FUZ_rand(&g_rand) % (varyLen << 1); paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p); } - validated = !ZSTD_isError(ZSTD_checkCParams(p)); - + //validated = !ZSTD_isError(ZSTD_checkCParams(p)); + validated = cParamValid(p); + //Make sure memory is at least close to feasible? //ZSTD_estimateCCtxSize thing. } - *ptr = p; + *ptr = sanitizeParams(p); } +//varyParams gives us table size? +//1 per strategy +//varyParams should always be sorted smallest to largest +//take arrayLen to allocate memotable +//should be ~10^7 unconstrained. +static size_t memoTableLen(const U32* varyParams, const int varyLen) { + size_t arrayLen = 1; + int i; + for(i = 0; i < varyLen; i++) { + arrayLen *= rangetable[varyParams[i]]; + } + return arrayLen; +} + +//sort of ~lg2 (replace 1024 w/ 999) for memoTableInd Tlen +static unsigned lg2(unsigned x) { + unsigned j = 0; + if(x == 999) { + return 10; + } + while(x >>= 1) { + j++; + } + return j; +} + +//indexes compressionParameters into memotable +//of form +static unsigned memoTableInd(const ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen) { + int i; + unsigned ind = 0; + for(i = 0; i < varyLen; i++) { + switch(varyParams[i]) { + case WLOG_IND: ind *= WLOG_RANGE; ind += ptr->windowLog - ZSTD_WINDOWLOG_MIN ; break; + case CLOG_IND: ind *= CLOG_RANGE; ind += ptr->chainLog - ZSTD_CHAINLOG_MIN ; break; + case HLOG_IND: ind *= HLOG_RANGE; ind += ptr->hashLog - ZSTD_HASHLOG_MIN ; break; + case SLOG_IND: ind *= SLOG_RANGE; ind += ptr->searchLog - ZSTD_SEARCHLOG_MIN ; break; + case SLEN_IND: ind *= SLEN_RANGE; ind += ptr->searchLength - ZSTD_SEARCHLENGTH_MIN; break; + case TLEN_IND: ind *= TLEN_RANGE; ind += lg2(ptr->targetLength) - ZSTD_TARGETLENGTH_MIN; break; + } + } + return ind; +} + +/* presumably, the unfilled parameters are already at their correct value */ +/* inverse above function for varyParams */ +static void memoTableIndInv(ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen, size_t ind) { + int i; + for(i = varyLen - 1; i >= 0; i--) { + switch(varyParams[i]) { + case WLOG_IND: ptr->windowLog = ind % WLOG_RANGE + ZSTD_WINDOWLOG_MIN; ind /= WLOG_RANGE; break; + case CLOG_IND: ptr->chainLog = ind % CLOG_RANGE + ZSTD_CHAINLOG_MIN; ind /= CLOG_RANGE; break; + case HLOG_IND: ptr->hashLog = ind % HLOG_RANGE + ZSTD_HASHLOG_MIN; ind /= HLOG_RANGE; break; + case SLOG_IND: ptr->searchLog = ind % SLOG_RANGE + ZSTD_SEARCHLOG_MIN; ind /= SLOG_RANGE; break; + case SLEN_IND: ptr->searchLength = ind % SLEN_RANGE + ZSTD_SEARCHLENGTH_MIN; ind /= SLEN_RANGE; break; + case TLEN_IND: ptr->targetLength = MIN(1 << (ind % TLEN_RANGE), 999); ind /= TLEN_RANGE; break; + } + } +} + +//initializing memoTable +/* */ +static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstraints, constraint_t target, const U32* varyParams, const int varyLen) { + size_t i; + size_t arrayLen = memoTableLen(varyParams, varyLen); + int cwFixed = !paramConstraints.chainLog || !paramConstraints.windowLog; + int scFixed = !paramConstraints.searchLog || !paramConstraints.chainLog; + int j = 0; + memset(memoTable, 0, arrayLen); + + + for(i = 0; i < arrayLen; i++) { + memoTableIndInv(¶mConstraints, varyParams, varyLen, i); + BMK_translateAdvancedParams(paramConstraints); + if(ZSTD_estimateCCtxSize_usingCParams(paramConstraints) + (1 << paramConstraints.windowLog) > target.cMem) { + //infeasible; + memoTable[i] = 255; + j++; + } + /* nil out parameter sets equivalent to others. */ + if(cwFixed/* at most least 1 param fixed. */) { + if(paramConstraints.strategy == ZSTD_btlazy2 || paramConstraints.strategy == ZSTD_btopt || paramConstraints.strategy == ZSTD_btultra) { + if(paramConstraints.chainLog > paramConstraints.windowLog + 1) { + if(memoTable[i] != 255) { j++; } + memoTable[i] = 255; + } + } else { + if(paramConstraints.chainLog > paramConstraints.windowLog) { + if(memoTable[i] != 255) { j++; } + memoTable[i] = 255; + } + } + } + if(scFixed) { + if(paramConstraints.searchLog > paramConstraints.chainLog) { + if(memoTable[i] != 255) { j++; } + memoTable[i] = 255; + } + } + } + DISPLAY("%d / %d Invalid\n", j, (int)i); +} #define PARAMTABLELOG 25 #define PARAMTABLESIZE (1<> 3) & PARAMTABLEMASK] + g_alreadyTested[(XXH64(((void*)&sanitizeParams(p), sizeof(p), 0) >> 3) & PARAMTABLEMASK] */ +static BYTE* NB_TESTS_PLAYED(ZSTD_compressionParameters p) { + ZSTD_compressionParameters p2 = sanitizeParams(p); + return &g_alreadyTested[(XXH64((void*)&p2, sizeof(p2), 0) >> 3) & PARAMTABLEMASK]; +} static void playAround(FILE* f, winnerInfo_t* winners, ZSTD_compressionParameters params, @@ -513,21 +731,23 @@ static void playAround(FILE* f, winnerInfo_t* winners, { int nbVariations = 0; UTIL_time_t const clockStart = UTIL_getTime(); - const U32 unconstrained[NUM_PARAMS] = { 0, 1, 2, 3, 4, 5, 6 }; + const U32 unconstrained[NUM_PARAMS] = { 0, 1, 2, 3, 4, 5 }; while (UTIL_clockSpanMicro(clockStart) < g_maxVariationTime) { ZSTD_compressionParameters p = params; + BYTE* b; if (nbVariations++ > g_maxNbVariations) break; - paramVariation(&p, unconstrained, 7); + paramVariation(&p, unconstrained, 7, 4); /* exclude faster if already played params */ - if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(p))-1)) + if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(p))-1)) continue; /* test */ - NB_TESTS_PLAYED(p)++; + b = NB_TESTS_PLAYED(p); + (*b)++; if (!BMK_seed(winners, p, srcBuffer, srcSize, ctx, dctx)) continue; /* improvement found => search more */ @@ -544,34 +764,37 @@ static ZSTD_compressionParameters randomParams(void) U32 validated = 0; while (!validated) { /* totally random entry */ - p.chainLog = (FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN)) + ZSTD_CHAINLOG_MIN; - p.hashLog = (FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN)) + ZSTD_HASHLOG_MIN; - p.searchLog = (FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN)) + ZSTD_SEARCHLOG_MIN; - p.windowLog = (FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN)) + ZSTD_WINDOWLOG_MIN; + p.chainLog = (FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN)) + ZSTD_CHAINLOG_MIN; + p.hashLog = (FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN)) + ZSTD_HASHLOG_MIN; + p.searchLog = (FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN)) + ZSTD_SEARCHLOG_MIN; + p.windowLog = (FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN)) + ZSTD_WINDOWLOG_MIN; p.searchLength=(FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN)) + ZSTD_SEARCHLENGTH_MIN; p.targetLength=(FUZ_rand(&g_rand) % (512)); p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btultra +1)); - validated = !ZSTD_isError(ZSTD_checkCParams(p)); + //validated = !ZSTD_isError(ZSTD_checkCParams(p)); + validated = cParamValid(p); } return p; } -static ZSTD_compressionParameters randomConstrainedParams(ZSTD_compressionParameters pc) +//destructively modifies pc. +//Maybe if memoTable[ind] > 0 too often, count zeroes and explicitly choose from free stuff? +//^ maybe this doesn't matter, with |mt| size it has \approx 1-(1/e) of finding even single free spot in |mt| tries, not too bad. +//TODO: maybe memoTable pc before sanitization too so no repeats? +static void randomConstrainedParams(ZSTD_compressionParameters* pc, U32* varArray, int varLen, U8* memoTable) { - ZSTD_compressionParameters p; - U32 validated = 0; - while (!validated) { - /* totally random entry */ - if(!pc.chainLog) p.chainLog = (FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN)) + ZSTD_CHAINLOG_MIN; - if(!pc.chainLog) p.hashLog = (FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN)) + ZSTD_HASHLOG_MIN; - if(!pc.chainLog) p.searchLog = (FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN)) + ZSTD_SEARCHLOG_MIN; - if(!pc.chainLog) p.windowLog = (FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN)) + ZSTD_WINDOWLOG_MIN; - if(!pc.chainLog) p.searchLength=(FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN)) + ZSTD_SEARCHLENGTH_MIN; - if(!pc.chainLog) p.targetLength=(FUZ_rand(&g_rand) % (512)) + 1; //ZSTD_TARGETLENGTH_MIN; //change to 2^[0,10?] - if(!pc.chainLog) p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btultra +1)); - validated = !ZSTD_isError(ZSTD_checkCParams(p)); - } - return p; + int tries = memoTableLen(varArray, varLen); //configurable, + const size_t maxSize = memoTableLen(varArray, varLen); + size_t ind; + do { + ind = (FUZ_rand(&g_rand)) % maxSize; + tries--; + } while(memoTable[ind] > 0 && tries > 0); + //&& FUZ_rand(&g_rand) % 256 > memoTable[ind]); get nd choosing? (helpful w/ distance) /* maybe > infeasible bound? */ + + /* memoTable[ind] == 0 -> unexplored */ + memoTableIndInv(pc, varArray, varLen, (unsigned)ind); + *pc = sanitizeParams(*pc); } static void BMK_selectRandomStart( @@ -746,22 +969,80 @@ int benchFiles(const char** fileNamesTable, int nbFiles) return 0; } - -static void BMK_translateAdvancedParams(ZSTD_compressionParameters params) -{ - DISPLAY("--zstd=windowLog=%u,chainLog=%u,hashLog=%u,searchLog=%u,searchLength=%u,targetLength=%u,strategy=%u \n", - params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, params.targetLength, (U32)(params.strategy)); -} - -//Results currently don't capture memory usage or anything. //parameter feasibility is not checked, should just be restricted from use. static int feasible(BMK_result_t results, constraint_t target) { - return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.Mem || !target.Mem); + return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem || !target.cMem); +} + +#define EPSILON 0.01 +static int epsilonEqual(double c1, double c2) { + return MAX(c1/c2,c2/c1) < 1 + EPSILON; +} + +//so the compiler stops warning +static int eqZero(double c1) { + return (U64)c1 == (U64)0.0 || (U64)c1 == (U64)-0.0; } /* returns 1 if result2 is strictly 'better' than result1 */ +/* strict comparison / cutoff based */ static int objective_lt(BMK_result_t result1, BMK_result_t result2) { - return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed); + return (result1.cSize > result2.cSize) || (epsilonEqual(result1.cSize, result2.cSize) && result2.cSpeed > result1.cSpeed) + || (epsilonEqual(result1.cSize,result2.cSize) && epsilonEqual(result2.cSpeed, result1.cSpeed) && result2.dSpeed > result1.dSpeed); +} + +//will probably be some linear combinartion of comp speed, decompSpeed, & ratio (maybe size), and memory? +//pretty arbitrary right now +//maybe better - higher coefficient when below threshold, lower when above +//need to normalize speed? or just use ratio speed / target? +//Maybe don't use ratio at all when looking for feasibility? + +/* maybe dynamically vary the coefficients for this around based on what's already been discovered. (maybe make a reversed ratio cutoff?) concave to pheaily penalize below ratio? */ +static double resultScore(BMK_result_t res, size_t srcSize, constraint_t target) { + double cs = 0., ds = 0., rt, cm = 0.; + const double r1 = 1, r2 = 0.1, rtr = 0.5; + double ret; + if(target.cSpeed) { cs = res.cSpeed / (double)target.cSpeed; } + if(target.dSpeed) { ds = res.dSpeed / (double)target.dSpeed; } + if(target.cMem != (U32)-1) { cm = (double)target.cMem / res.cMem; } + rt = ((double)srcSize / res.cSize); + + //(void)rt; + //(void)rtr; + + ret = (MIN(1, cs) + MIN(1, ds) + MIN(1, cm))*r1 + rt * rtr + + (MAX(0, log(cs))+ MAX(0, log(ds))+ MAX(0, log(cm))) * r2; + //DISPLAY("resultScore: %f\n", ret); + return ret; +} + +/* + double W_ratio = (double)srcSize / testResult.cSize; + double O_ratio = (double)srcSize / winners[cLevel].result.cSize; + double W_ratioNote = log (W_ratio); + double O_ratioNote = log (O_ratio); + size_t W_DMemUsed = (1 << params.windowLog) + (16 KB); + size_t O_DMemUsed = (1 << winners[cLevel].params.windowLog) + (16 KB); + double W_DMemUsed_note = W_ratioNote * ( 40 + 9*cLevel) - log((double)W_DMemUsed); + double O_DMemUsed_note = O_ratioNote * ( 40 + 9*cLevel) - log((double)O_DMemUsed); + + size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_estimateCCtxSize_usingCParams(params); + size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_estimateCCtxSize_usingCParams(winners[cLevel].params); + double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed); + double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed); + + double W_CSpeed_note = W_ratioNote * ( 30 + 10*cLevel) + log(testResult.cSpeed); + double O_CSpeed_note = O_ratioNote * ( 30 + 10*cLevel) + log(winners[cLevel].result.cSpeed); + + double W_DSpeed_note = W_ratioNote * ( 20 + 2*cLevel) + log(testResult.dSpeed); + double O_DSpeed_note = O_ratioNote * ( 20 + 2*cLevel) + log(winners[cLevel].result.dSpeed); + +*/ +//ratio tradeoffs, may be useful in guiding + +/* objective_lt, but based on scoring function */ +static int objective_lt2(BMK_result_t result1, BMK_result_t result2, size_t srcSize, constraint_t target) { + return resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target); } /* res gives array dimensions, should be size NUM_PARAMS */ @@ -783,48 +1064,800 @@ static size_t computeStateSize(const ZSTD_compressionParameters paramConstraints static unsigned calcViolation(BMK_result_t results, constraint_t target) { int diffcSpeed = MAX(target.cSpeed - results.cSpeed, 0); int diffdSpeed = MAX(target.dSpeed - results.dSpeed, 0); - int diffcMem = MAX(results.cMem - target.Mem, 0); + int diffcMem = MAX(results.cMem - target.cMem, 0); return diffcSpeed + diffdSpeed + diffcMem; } -/* finds some set of parameters which fulfills req's - * Prioritize highest / try to locally minimize sum? - * Is it ever useful to go out of the param constraints? ? - * random / perturb when revisit? - * momentum? - */ -static ZSTD_compressionParameters findFeasible(constraint_t target, ZSTD_compressionParameters paramTarget) { - unsigned violation; - - ZSTD_compressionParameters winner = randomConstrainedParams(paramTarget); - //just use g_alreadyTested and xxhash? - BYTE* memotable; - do { - //prioritize memory - if(diffcMem >= diffcSpeed && diffcMem >= diffdSpeed) { - - //prioritize compression Speed - } else if (diffcSpeeed >= diffdSpeed && diffcSpeed >= diffcMem) { - - //prioritize decompressionSpeed - } else { - - } - violation = calcViolation(result, target); - } while(objective); - if(validate) { - DISPLAY("Feasible Point Found\n"); - return winner; +/* +uncertaintyConstant >= 1 +returns -1 = 'certainly' infeasible + 0 = unceratin + 1 = 'certainly' feasible +*/ +//paramTarget misnamed, should just be target +static int uncertainFeasibility(double const uncertaintyConstantC, double const uncertaintyConstantD, const constraint_t paramTarget, const BMK_result_t* const results) { + if((paramTarget.cSpeed != 0 && results->cSpeed * uncertaintyConstantC < paramTarget.cSpeed) || + (paramTarget.dSpeed != 0 && results->dSpeed * uncertaintyConstantD < paramTarget.dSpeed) || + (paramTarget.cMem != 0 && results->cMem > paramTarget.cMem)) { + return -1; + } else if((paramTarget.cSpeed == 0 || results->cSpeed / uncertaintyConstantC > paramTarget.cSpeed) && + (paramTarget.dSpeed == 0 || results->dSpeed / uncertaintyConstantD > paramTarget.dSpeed) && + (paramTarget.cMem == 0 || results->cMem <= paramTarget.cMem)) { + return 1; } else { - DISPLAY("No solution found\n"); - ZSTD_compressionParameters ret = { 0, 0, 0, 0, 0, 0, 0 }; - return ret; + return 0; } } +/* 1 - better than prev best + 0 - uncertain + -1 - worse + assume prev_best status is run fully? + but then we'd have to rerun any winners anyway */ +//presumably memory has already been compared, mostly worried about mem, cspeed, dspeed +//uncertainty only applies to speed. +//if using objective fn, this could be much easier since we could just scale that. +//difficult to make judgements about later parameters in prioritization type when there's +//uncertainty on the first. +static int uncertainComparison(double const uncertaintyConstantC, double const uncertaintyConstantD, BMK_result_t* candidate, BMK_result_t* prevBest) { + (void)uncertaintyConstantD; //unused for now + if(candidate->cSpeed > prevBest->cSpeed * uncertaintyConstantC) { + return 1; + } else if (candidate->cSpeed * uncertaintyConstantC < prevBest->cSpeed) { + return -1; + } else { + return 0; + } +} + +/* speed in b, srcSize in b/s loopDuration in ns */ +//TODO: simplify code in feasibleBench with this instead of writing it all out. +//only applicable for single loop +static double calcUncertainty(double speed, size_t srcSize) { + U64 loopDuration; + if(eqZero(speed)) { return 2; } + loopDuration = ((srcSize * TIMELOOP_NANOSEC) / speed); + return MIN((loopDuration + (double)2 * g_clockGranularity) / loopDuration, 2); +} + +//benchmarks and tests feasibility together +//1 = true = better +//0 = false = not better +//if true then resultPtr will give results. +//2+ on error? +//alt: error = 0 / infeasible as well; +//maybe use compress_only mode for ratio-finding benchmark? +//prioritize ratio > cSpeed > dSpeed > cMem +//Misnamed - should be worse, better, error +//alternative (to make work for feasible-pt searching as well) - only compare to winner, not to target +//but then we need to judge what better means in this context, which shouldn't be the same (strict ratio improvement) +#define INFEASIBLE_RESULT 0 +#define FEASIBLE_RESULT 1 +#define ERROR_RESULT 2 +static int feasibleBench(BMK_result_t* resultPtr, + const void* srcBuffer, size_t srcSize, + void* dstBuffer, size_t dstSize, + ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, + const ZSTD_compressionParameters cParams, + const constraint_t target, + BMK_result_t* winnerResult) { + BMK_advancedParams_t adv = BMK_initAdvancedParams(); + BMK_return_t benchres; + U64 loopDurationC = 0, loopDurationD = 0; + double uncertaintyConstantC, uncertaintyConstantD; + adv.loopMode = BMK_iterMode; + adv.nbSeconds = 1; //get ratio and 2x approx speed? + + //alternative - test 1 iter for ratio, (possibility of error 3 which is fine), + //maybe iter this until 2x measurable for better guarantee? + DISPLAY("Feas:\n"); + benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + if(benchres.error) { + DISPLAY("ERROR %d !!\n", benchres.error); + } + BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); + + if(!benchres.error) { + *resultPtr = benchres.result; + /* if speed is 0 (only happens when time = 0) */ + if(eqZero(benchres.result.cSpeed)) { + loopDurationC = 0; + uncertaintyConstantC = 2; + } else { + loopDurationC = ((srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); + //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? + //possibly has to do with initCCtx? or system stuff? + //asymmetric +/- constant needed? + uncertaintyConstantC = MIN((loopDurationC + (double)(2 * g_clockGranularity)/loopDurationC), 2); //.02 seconds + } + if(eqZero(benchres.result.dSpeed)) { + loopDurationD = 0; + uncertaintyConstantD = 2; + } else { + loopDurationD = ((srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); + //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? + //possibly has to do with initCCtx? or system stuff? + //asymmetric +/- constant needed? + uncertaintyConstantD = MIN((loopDurationD + (double)(2 * g_clockGranularity)/loopDurationD), 2); //.02 seconds + } + + + if(benchres.result.cSize < winnerResult->cSize) { //better compression ratio, just needs to be feasible + //optimistic assume speed + //incoporate some sort of tradeoff comparison with the winner's results? + int feas = uncertainFeasibility(uncertaintyConstantC, uncertaintyConstantD, target, &(benchres.result)); + if(feas == 0) { // uncertain feasibility + adv.loopMode = BMK_timeMode; + if(loopDurationC < TIMELOOP_NANOSEC) { + BMK_return_t benchres2; + adv.mode = BMK_compressOnly; + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres.result.cSpeed = benchres2.result.cSpeed; + } + } + if(loopDurationD < TIMELOOP_NANOSEC) { + BMK_return_t benchres2; + adv.mode = BMK_decodeOnly; + benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres.result.dSpeed = benchres2.result.dSpeed; + } + } + *resultPtr = benchres.result; + return feasible(benchres.result, target); + } else { //feas = 1 or -1 map to 1, 0 respectively + return (feas + 1) >> 1; //relies on INFEASIBLE_RESULT == 0, FEASIBLE_RESULT == 1 + } + } else if (benchres.result.cSize == winnerResult->cSize) { //equal ratio, needs to be better than winner in cSpeed/ dSpeed / cMem + int feas = uncertainFeasibility(uncertaintyConstantC, uncertaintyConstantD, target, &(benchres.result)); + if(feas == 0) { // uncertain feasibility + adv.loopMode = BMK_timeMode; + if(loopDurationC < TIMELOOP_NANOSEC) { + BMK_return_t benchres2; + adv.mode = BMK_compressOnly; + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres.result.cSpeed = benchres2.result.cSpeed; + } + } + if(loopDurationD < TIMELOOP_NANOSEC) { + BMK_return_t benchres2; + adv.mode = BMK_decodeOnly; + benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres.result.dSpeed = benchres2.result.dSpeed; + } + } + + *resultPtr = benchres.result; + return feasible(benchres.result, target) && objective_lt(*winnerResult, benchres.result); + } else if (feas == 1) { //no need to check feasibility compares (maybe only it is chosen as a winner) + int btw = uncertainComparison(uncertaintyConstantC, uncertaintyConstantD, &(benchres.result), winnerResult); + if(btw == -1) { + return INFEASIBLE_RESULT; + } else { //possibly better, benchmark and find out + adv.loopMode = BMK_timeMode; + benchres = BMK_benchMemAdvanced(srcBuffer, srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + *resultPtr = benchres.result; + return objective_lt(*winnerResult, benchres.result); + } + } else { //feas == -1 + return INFEASIBLE_RESULT; //infeasible + } + } else { + return INFEASIBLE_RESULT; //infeasible + } + } else { + return ERROR_RESULT; //BMK error + } + +} +//sameas before, but +/-? +//alternative, just return comparison result, leave caller to worry about feasibility. +//have version of benchMemAdvanced which takes in dstBuffer/cap as well? +//(motivation: repeat tests (maybe just on decompress) don't need further compress runs) +static int infeasibleBench(BMK_result_t* resultPtr, + const void* srcBuffer, size_t srcSize, + void* dstBuffer, size_t dstSize, + ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, + const ZSTD_compressionParameters cParams, + const constraint_t target, + BMK_result_t* winnerResult) { + BMK_advancedParams_t adv = BMK_initAdvancedParams(); + BMK_return_t benchres; + BMK_result_t resultMin, resultMax; + UTIL_time_t startTime; + U64 loopDurationC = 0, loopDurationD = 0; + double uncertaintyConstantC, uncertaintyConstantD; + double winnerRS = resultScore(*winnerResult, srcSize, target); + adv.loopMode = BMK_iterMode; //can only use this for ratio measurement then, super inaccurate timing + adv.nbSeconds = 1; //get ratio and 2x approx speed? //maybe run until twice MIN(minloopinterval * clockDuration) + + (void)startTime; //TODO: actually use this to adjust timing + DISPLAY("WinnerScore: %f\n ", winnerRS); + /* + adv.loopMode = BMK_timeMode; + adv.nbSeconds = 1; */ + benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); + + if(!benchres.error) { + *resultPtr = benchres.result; + if(eqZero(benchres.result.cSpeed)) { + loopDurationC = 0; + uncertaintyConstantC = 2; + } else { + loopDurationC = ((srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); + //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? + //possibly has to do with initCCtx? or system stuff? + uncertaintyConstantC = MIN((loopDurationC + (double)(2 * g_clockGranularity)/loopDurationC), 2); //.02 seconds + } + + if(eqZero(benchres.result.dSpeed)) { + loopDurationD = 0; + uncertaintyConstantD = 2; + } else { + loopDurationD = ((srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); + //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? + //possibly has to do with initCCtx? or system stuff? + uncertaintyConstantD = MIN((loopDurationD + (double)(2 * g_clockGranularity)/loopDurationD), 2); //.02 seconds + } + + /* benchres's certainty range. */ + resultMax = benchres.result; + resultMin = benchres.result; + resultMax.cSpeed *= uncertaintyConstantC; + resultMax.dSpeed *= uncertaintyConstantD; + resultMin.cSpeed /= uncertaintyConstantC; + resultMin.dSpeed /= uncertaintyConstantD; + (void)resultMin; + //TODO: consider if resultMin is actually needed. + if (winnerRS > resultScore(resultMax, srcSize, target)) { + return INFEASIBLE_RESULT; + } else { + //do this w/o copying / stuff + adv.loopMode = BMK_timeMode; + if(loopDurationC < TIMELOOP_NANOSEC) { + BMK_return_t benchres2; + adv.mode = BMK_compressOnly; + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres.result.cSpeed = benchres2.result.cSpeed; + } + } + if(loopDurationD < TIMELOOP_NANOSEC) { + BMK_return_t benchres2; + adv.mode = BMK_decodeOnly; + //TODO: dstBuffer corrupted sometime between top and now + //probably occuring in feasible bench too. + benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres.result.dSpeed = benchres2.result.dSpeed; + } + } + *resultPtr = benchres.result; + return (resultScore(benchres.result, srcSize, target) > winnerRS); + } + + *resultPtr = benchres.result; + } else { + return ERROR_RESULT; //BMK error + } + +} + +/* wrap feasibleBench w/ memotable */ +//TODO: void sanitized and unsanitized ver's so input doesn't double-choose +#define INFEASIBLE_THRESHOLD 200 +static int feasibleBenchMemo(BMK_result_t* resultPtr, + const void* srcBuffer, size_t srcSize, + void* dstBuffer, size_t dstSize, + ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, + const ZSTD_compressionParameters cParams, + const constraint_t target, + BMK_result_t* winnerResult, U8* memoTable, + U32* varyParams, const int varyLen) { + + size_t memind = memoTableInd(&cParams, varyParams, varyLen); + //BMK_translateAdvancedParams(cParams); + if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { + return INFEASIBLE_RESULT; //probably pick a different code for already tested? + //maybe remove this if we incorporate nonrandom location picking? + //what is the intended behavior in this case? + //ignore? stop iterating completely? other? + } else { + int res = feasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, ctx, dctx, + cParams, target, winnerResult); + memoTable[memind] = 255; //tested are all infeasible (other possible values for opti) + return res; + } +} + +//should infeasible stage searching also be memo-marked in the same way? +//don't actually memoize unless result is feasible/error? +static int infeasibleBenchMemo(BMK_result_t* resultPtr, + const void* srcBuffer, size_t srcSize, + void* dstBuffer, size_t dstSize, + ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, + const ZSTD_compressionParameters cParams, + const constraint_t target, + BMK_result_t* winnerResult, U8* memoTable, + U32* varyParams, const int varyLen) { + size_t memind = memoTableInd(&cParams, varyParams, varyLen); + //BMK_translateAdvancedParams(cParams); + if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { + return INFEASIBLE_RESULT; //see feasibleBenchMemo for concerns + } else { + int res = infeasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, ctx, dctx, + cParams, target, winnerResult); + if(res == FEASIBLE_RESULT) { + memoTable[memind] = 255; //infeasible resultscores could still be normal feasible. + } + return res; + } +} + +/* specifically feasibleBenchMemo and infeasibleBenchMemo */ +//maybe not necessary +typedef int (*BMK_benchMemo_t)(BMK_result_t*, const void*, size_t, void*, size_t, ZSTD_CCtx*, ZSTD_DCtx*, + const ZSTD_compressionParameters, const constraint_t, BMK_result_t*, U8*, U32*, const int); + +//varArray should be sanitized when this is called. +//TODO: transition to simpler greedy method if evaluation time is too long? +//would it be better to start at best feasible via feasible or infeasible metric? both? +//possibility climb is infeasible, responsibility of caller to check that. but if something feasible is evaluated, it will be returned +// *actually if it performs too +//sanitize all params here. +//all generation after random should be sanitized. (maybe sanitize random) +//TODO: paramTarget uneeded at this point w/ varArray and init; +static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varLen, U8* memoTable, + const void* srcBuffer, size_t srcSize, void* dstBuffer, size_t dstSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, ZSTD_compressionParameters init) { + //pick later initializations non-randomly? high dist from explored nodes. + //how to do this efficiently? (might not be too much of a problem, happens rarely, running time probably dominated by benchmarking) + //distance maximizing selection? + //cparam - currently considered center + //candidate - params to benchmark/results + //winner - best option found so far. + ZSTD_compressionParameters cparam = init; + winnerInfo_t candidateInfo, winnerInfo; + int better = 1; + + winnerInfo.params = init; + winnerInfo.result.cSpeed = 0; + winnerInfo.result.dSpeed = 0; + winnerInfo.result.cMem = (size_t)-1; + winnerInfo.result.cSize = (size_t)-1; + + /* ineasible -> (hopefully) feasible */ + /* when nothing is found, this garbages part 2. */ + { + //TODO: initialize these values! + winnerInfo_t bestFeasible1; /* uses feasibleBench Metric */ + winnerInfo_t bestFeasible2; /* uses resultScore Metric */ + + //init these params + bestFeasible1.params = cparam; + bestFeasible2.params = cparam; + bestFeasible1.result.cSpeed = 0; + bestFeasible1.result.dSpeed = 0; + bestFeasible1.result.cMem = (size_t)-1; + bestFeasible1.result.cSize = (size_t)-1; + bestFeasible2.result.cSpeed = 0; + bestFeasible2.result.dSpeed = 0; + bestFeasible2.result.cMem = (size_t)-1; + bestFeasible2.result.cSize = (size_t)-1; + DISPLAY("Climb Part 1\n"); + while(better) { + + //UTIL_time_t timestart = UTIL_getTime(); TODO: adjust sampling based on time + int i, d; + better = 0; + DISPLAY("Start\n"); + cparam = winnerInfo.params; + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + candidateInfo.params = cparam; + //all dist-1 targets + for(i = 0; i < varLen; i++) { + paramVaryOnce(varArray[i], 1, &candidateInfo.params); /* +1 */ + candidateInfo.params = sanitizeParams(candidateInfo.params); + //evaluate + //if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { + if(cParamValid(candidateInfo.params)) { + int res = infeasibleBenchMemo(&candidateInfo.result, + srcBuffer, srcSize, + dstBuffer, dstSize, + ctx, dctx, + candidateInfo.params, target, &winnerInfo.result, memoTable, + varArray, varLen); + if(res == FEASIBLE_RESULT) { /* synonymous with better when called w/ infeasibleBM */ + winnerInfo = candidateInfo; + //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + better = 1; + if(feasible(candidateInfo.result, target)) { + bestFeasible2 = winnerInfo; + if(objective_lt(bestFeasible1.result, bestFeasible2.result)) { + bestFeasible1 = bestFeasible2; /* using feasibleBench metric */ + } + } + } + } + candidateInfo.params = cparam; + paramVaryOnce(varArray[i], -1, &candidateInfo.params); /* -1 */ + candidateInfo.params = sanitizeParams(candidateInfo.params); + //evaluate + //if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { + if(cParamValid(candidateInfo.params)) { + int res = infeasibleBenchMemo(&candidateInfo.result, + srcBuffer, srcSize, + dstBuffer, dstSize, + ctx, dctx, + candidateInfo.params, target, &winnerInfo.result, memoTable, + varArray, varLen); + if(res == FEASIBLE_RESULT) { + winnerInfo = candidateInfo; + //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + better = 1; + if(feasible(candidateInfo.result, target)) { + bestFeasible2 = winnerInfo; + if(objective_lt(bestFeasible1.result, bestFeasible2.result)) { + bestFeasible1 = bestFeasible2; + } + } + } + } + } + + if(better) { + continue; + } + //if 'better' enough, skip further parameter search, center there? + //possible improvement - guide direction here w/ knowledge rather than completely random variation. + for(d = 2; d < varLen + 2; d++) { /* varLen is # dimensions */ + for(i = 0; i < 5; i++) { //make ? relative to # of free dimensions. + int res; + candidateInfo.params = cparam; + /* param error checking already done here */ + paramVariation(&candidateInfo.params, varArray, varLen, d); + res = infeasibleBenchMemo(&candidateInfo.result, + srcBuffer, srcSize, + dstBuffer, dstSize, + ctx, dctx, + candidateInfo.params, target, &winnerInfo.result, memoTable, + varArray, varLen); + if(res == FEASIBLE_RESULT) { /* synonymous with better in this case*/ + winnerInfo = candidateInfo; + //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + better = 1; + if(feasible(candidateInfo.result, target)) { + bestFeasible2 = winnerInfo; + if(objective_lt(bestFeasible1.result, bestFeasible2.result)) { + bestFeasible1 = bestFeasible2; + } + } + } + + } + if(better) { + continue; + } + } + //bias to test previous delta? + //change cparam -> candidate before restart + } + //TODO:Consider if this is best config. idea: explore from obj best keep rbest + cparam = bestFeasible2.params; + candidateInfo = bestFeasible2; + winnerInfo = bestFeasible1; + } + + //is it better to break here instead of bumbling about? + if(winnerInfo.result.cMem == (U32)-1) { + DISPLAY("No Feasible Found\n"); + return winnerInfo; + } + DISPLAY("Climb Part 2\n"); + + better = 1; + /* feasible -> best feasible (hopefully) */ + { + while(better) { + + //UTIL_time_t timestart = UTIL_getTime(); //TODO: if benchmarking is taking too long, be more greedy. + int i, d; + better = 0; + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + //all dist-1 targets + cparam = winnerInfo.params; //TODO: this messes the taking bestFeasible1, bestFeasible2 + candidateInfo.params = cparam; + for(i = 0; i < varLen; i++) { + paramVaryOnce(varArray[i], 1, &candidateInfo.params); + candidateInfo.params = sanitizeParams(candidateInfo.params); + + //evaluate + //if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { + if(cParamValid(candidateInfo.params)) { + int res = feasibleBenchMemo(&candidateInfo.result, + srcBuffer, srcSize, + dstBuffer, dstSize, + ctx, dctx, + candidateInfo.params, target, &winnerInfo.result, memoTable, + varArray, varLen); + if(res == FEASIBLE_RESULT) { + winnerInfo = candidateInfo; + //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + better = 1; + } + } + candidateInfo.params = cparam; + paramVaryOnce(varArray[i], -1, &candidateInfo.params); + candidateInfo.params = sanitizeParams(candidateInfo.params); + //evaluate + //if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { + if(cParamValid(candidateInfo.params)) { + int res = feasibleBenchMemo(&candidateInfo.result, + srcBuffer, srcSize, + dstBuffer, dstSize, + ctx, dctx, + candidateInfo.params, target, &winnerInfo.result, memoTable, + varArray, varLen); + if(res == FEASIBLE_RESULT) { + winnerInfo = candidateInfo; + //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + better = 1; + } + } + } + //if 'better' enough, skip further parameter search, center there? + //possible improvement - guide direction here w/ knowledge rather than completely random variation. + for(d = 2; d < varLen + 2; d++) { /* varLen is # dimensions */ + for(i = 0; i < 5; i++) { //TODO: make ? relative to # of free dimensions. + int res; + candidateInfo.params = cparam; + /* param error checking already done here */ + paramVariation(&candidateInfo.params, varArray, varLen, d); //info candidateInfo.params is garbage, this is too. + res = feasibleBenchMemo(&candidateInfo.result, + srcBuffer, srcSize, + dstBuffer, dstSize, + ctx, dctx, + candidateInfo.params, target, &winnerInfo.result, memoTable, + varArray, varLen); + if(res == FEASIBLE_RESULT) { + winnerInfo = candidateInfo; + //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + better = 1; + } + } + if(better) { + continue; + } + } + //bias to test previous delta? + //change cparam -> candidate before restart + } + } + return winnerInfo; +} + +//optimizeForSize but with fixed strategy +//place to configure/filter out strategy specific parameters. +//need args for all buffers and parameter stuff +//sanitization here. + +//flexible parameters: iterations of (failed?) climbing (or if we do non-random, maybe this is when everything is close to visitied) +//weight more on visit for bad results, less on good results/more on later results / ones with more failures. +//allocate memoTable here. +//only real use for paramTarget is to get the fixed values, right? +static winnerInfo_t optimizeFixedStrategy( + const void* srcBuffer, const size_t srcSize, + void* dstBuffer, size_t dstSize, + constraint_t target, ZSTD_compressionParameters paramTarget, + ZSTD_strategy strat, U32* varArray, int varLen) { + int i = 0; //TODO: Temp fix 10 iters, check effects of changing this? + U32* varNew = malloc(sizeof(U32) * varLen); + int varLenNew = sanitizeVarArray(varLen, varArray, varNew, strat); + size_t memoLen = memoTableLen(varNew, varLenNew); + U8* memoTable = malloc(sizeof(U8) * memoLen); + ZSTD_compressionParameters init; + ZSTD_CCtx* ctx = ZSTD_createCCtx(); + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + winnerInfo_t winnerInfo, candidateInfo; + winnerInfo.result.cSpeed = 0; + winnerInfo.result.dSpeed = 0; + winnerInfo.result.cMem = (size_t)(-1); + winnerInfo.result.cSize = (size_t)(-1); + /* so climb is given the right fixed strategy */ + paramTarget.strategy = strat; + /* to pass ZSTD_checkCParams */ + cParamZeroMin(¶mTarget); + memoTableInit(memoTable, paramTarget, target, varNew, varLenNew); + + + init = paramTarget; + + + if(!ctx || !dctx || !memoTable || !varNew) { + DISPLAY("NOT ENOUGH MEMORY ! ! ! \n"); + goto _cleanUp; + } + + while(i < 10) { + DISPLAY("Restart\n"); + randomConstrainedParams(&init, varNew, varLenNew, memoTable); + candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, srcBuffer, srcSize, dstBuffer, dstSize, ctx, dctx, init); + if(objective_lt(winnerInfo.result, candidateInfo.result)) { + winnerInfo = candidateInfo; + DISPLAY("Climb Winner: "); + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + } + i++; + } + +_cleanUp: + ZSTD_freeCCtx(ctx); + ZSTD_freeDCtx(dctx); + free(memoTable); + free(varNew); + return winnerInfo; +} + +// bigger and (hopefully) better* than optimizeForSize +// TODO: Change level bm'ing to respect constraints. +static int optimizeForSize2(const char* inFileName, constraint_t target, ZSTD_compressionParameters paramTarget) +{ + FILE* const inFile = fopen( inFileName, "rb" ); + U64 const inFileSize = UTIL_getFileSize(inFileName); + size_t benchedSize = BMK_findMaxMem(inFileSize*3) / 3; + void* origBuff; + U32 varArray [NUM_PARAMS]; + int varLen = variableParams(paramTarget, varArray); + /* Init */ + + + if(!cParamValid(paramTarget)) { + return 10; + } + + if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; } + if (inFileSize == UTIL_FILESIZE_UNKNOWN) { + DISPLAY("Pb evaluatin size of %s \n", inFileName); + fclose(inFile); + return 11; + } + + /* Memory allocation & restrictions */ + if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize; + if (benchedSize < inFileSize) { + DISPLAY("Not enough memory for '%s' \n", inFileName); + fclose(inFile); + return 11; + } + + /* Alloc */ + origBuff = malloc(benchedSize); + if(!origBuff) { + DISPLAY("\nError: not enough memory!\n"); + fclose(inFile); + return 12; + } + + /* Fill input buffer */ + DISPLAY("Loading %s... \r", inFileName); + { size_t const readSize = fread(origBuff, 1, benchedSize, inFile); + fclose(inFile); + if(readSize != benchedSize) { + DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); + free(origBuff); + return 13; + } } + + /* bench */ + DISPLAY("\r%79s\r", ""); + DISPLAY("optimizing for %s", inFileName); + if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed / 1000000); } + if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed / 1000000); } + if(target.cMem != (U32)-1) { DISPLAY(" - limit memory %u MB", target.cMem / 1000000); } + DISPLAY("\n"); + findClockGranularity(); + + { ZSTD_CCtx* const ctx = ZSTD_createCCtx(); + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + winnerInfo_t winner; + //BMK_result_t candidate; + const size_t blockSize = g_blockSize ? g_blockSize : benchedSize; + U32 const maxNbBlocks = (U32) ((benchedSize + (blockSize-1)) / blockSize) + 1; + const size_t maxCompressedSize = ZSTD_compressBound(benchedSize) + (maxNbBlocks * 1024); + void* compressedBuffer = malloc(maxCompressedSize); + + /* init */ + if (ctx==NULL) { DISPLAY("\n ZSTD_createCCtx error \n"); free(origBuff); return 14;} + if(compressedBuffer==NULL) { DISPLAY("\n Allocation Error \n"); free(origBuff); free(ctx); return 15; } + + memset(&winner, 0, sizeof(winner)); + winner.result.cSize = (size_t)(-1); + + + /* find best solution from default params */ + //Can't do this w/ cparameter constraints + //still useful though? + /* + { const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); + int i; + for (i=1; i<=maxSeeds; i++) { + ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, 0); + BMK_benchParam(&candidate, origBuff, benchedSize, ctx, dctx, CParams); + if (!feasible(candidate, target) ) { + break; + } + if (feasible(candidate,target) && objective_lt(winner.result, candidate)) + { + winner.params = CParams; + winner.result = candidate; + BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); + } } + }*/ + BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize); + + BMK_translateAdvancedParams(winner.params); + + /* start real tests */ + { + if(paramTarget.strategy == 0) { + int st; + for(st = 1; st <= 8; st++) { + winnerInfo_t wc = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, + target, paramTarget, st, varArray, varLen); + DISPLAY("StratNum %d\n", st); + if(objective_lt(winner.result, wc.result)) { + winner = wc; + } + } + } else { + winner = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, + target, paramTarget, paramTarget.strategy, varArray, varLen); + } + + } + + /* no solution found */ + if(winner.result.cSize == (size_t)-1) { + DISPLAY("No feasible solution found\n"); + return 1; + } + /* end summary */ + BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize); + BMK_translateAdvancedParams(winner.params); + DISPLAY("grillParams size - optimizer completed \n"); + + /* clean up*/ + ZSTD_freeCCtx(ctx); + ZSTD_freeDCtx(dctx); + } + + free(origBuff); + return 0; +} + + /* optimizeForSize(): * targetSpeed : expressed in B/s */ -/* if state space is small (from paramTarget) */ +/* expresses targeted compression, decompression speeds and memory requirements */ +/* if state space is small (from paramTarget), exhaustive search? */ +//things to consider : if doing strategy-separate approach, what cutoffs to evaluate each strategy +//or do all? can't be absolute, should be relative after some sort of calibration +//(synthetic? test levels (we don't care about data specifics rn, scale?) ? int optimizeForSize(const char* inFileName, constraint_t target, ZSTD_compressionParameters paramTarget) { FILE* const inFile = fopen( inFileName, "rb" ); @@ -872,7 +1905,7 @@ int optimizeForSize(const char* inFileName, constraint_t target, ZSTD_compressio DISPLAY("optimizing for %s", inFileName); if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed / 1000000); } if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed / 1000000); } - if(target.Mem != 0) { DISPLAY(" - limit memory %u MB", target.Mem / 1000000); } + if(target.cMem != 0) { DISPLAY(" - limit memory %u MB", target.cMem / 1000000); } DISPLAY("\n"); { ZSTD_CCtx* const ctx = ZSTD_createCCtx(); ZSTD_DCtx* const dctx = ZSTD_createDCtx(); @@ -881,12 +1914,13 @@ int optimizeForSize(const char* inFileName, constraint_t target, ZSTD_compressio const size_t blockSize = g_blockSize ? g_blockSize : benchedSize; /* init */ - if (ctx==NULL) { DISPLAY("\n ZSTD_createCCtx error \n"); free(origBuff); return 14;} + if (ctx==NULL) { DISPLAY("\n ZSTD_createCCtx error \n"); free(origBuff); return 14; } + memset(&winner, 0, sizeof(winner)); winner.result.cSize = (size_t)(-1); /* find best solution from default params */ - //Can't do this iteration normally w/ cparameter constraints + //Can't do this w/ cparameter constraints { const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); int i; for (i=1; i<=maxSeeds; i++) { @@ -910,15 +1944,17 @@ int optimizeForSize(const char* inFileName, constraint_t target, ZSTD_compressio { time_t const grillStart = time(NULL); do { ZSTD_compressionParameters params = winner.params; - paramVariation(¶ms, paramVarArray, paramCount); + BYTE* b; + paramVariation(¶ms, paramVarArray, paramCount, 4); if ((FUZ_rand(&g_rand) & 31) == 3) params = randomParams(); /* totally random config to improve search space */ params = ZSTD_adjustCParams(params, blockSize, 0); /* exclude faster if already played set of params */ - if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(params))-1)) continue; + if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(params))-1)) continue; /* test */ - NB_TESTS_PLAYED(params)++; + b = NB_TESTS_PLAYED(params); + (*b)++; BMK_benchParam(&candidate, origBuff, benchedSize, ctx, dctx, params); /* improvement found => new winner */ @@ -1027,7 +2063,8 @@ int main(int argc, const char** argv) U32 optimizer = 0; U32 main_pause = 0; - constraint_t target = { 0 , 0, 0 }; //0 for anything unset + + constraint_t target = { 0, 0, (U32)-1 }; //0 for anything unset ZSTD_compressionParameters paramTarget = { 0, 0, 0, 0, 0, 0, 0 }; assert(argc>=1); /* for exename */ @@ -1053,7 +2090,7 @@ int main(int argc, const char** argv) if (longCommandWArg(&argument, "strategy=") || longCommandWArg(&argument, "strat=")) { paramTarget.strategy = (ZSTD_strategy)(readU32FromChar(&argument)); if (argument[0]==',') { argument++; continue; } else break; } if (longCommandWArg(&argument, "compressionSpeed=") || longCommandWArg(&argument, "cSpeed=")) { target.cSpeed = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } if (longCommandWArg(&argument, "decompressionSpeed=") || longCommandWArg(&argument, "dSpeed=")) { target.dSpeed = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "compressionMemory=") || longCommandWArg(&argument, "cMem=")) { target.Mem = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "compressionMemory=") || longCommandWArg(&argument, "cMem=")) { target.cMem = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } /* in MB or MB/s */ DISPLAY("invalid optimization parameter \n"); return 1; @@ -1132,7 +2169,7 @@ int main(int argc, const char** argv) continue; case 'M': argument++; - target.Mem = readU32FromChar(&argument) * 1000000; + target.cMem = readU32FromChar(&argument) * 1000000; continue; case 'w': argument++; @@ -1255,7 +2292,8 @@ int main(int argc, const char** argv) } } else { if (optimizer) { - result = optimizeForSize(input_filename, target, paramTarget); + result = optimizeForSize2(input_filename, target, paramTarget); + //optimizeForSize(input_filename, target, paramTarget); } else { result = benchFiles(argv+filenamesStart, argc-filenamesStart); } } From eb21b7f48224b35ceb91b31a2fe1bb764545724e Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 9 Jul 2018 13:44:01 -0700 Subject: [PATCH 03/19] Not crashing --- programs/bench.c | 10 ++++---- tests/paramgrill.c | 59 +++++++++++++++++++++++++++------------------- 2 files changed, 40 insertions(+), 29 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 0d35f3eb3..5d5b47aba 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -288,7 +288,7 @@ volatile char g_touched; /* takes # of blocks and list of size & stuff for each. */ /* only does looping */ /* note time per loop could be zero if interval too short */ -BMK_customReturn_t __attribute__((optimize("O0"))) BMK_benchFunction( +BMK_customReturn_t BMK_benchFunction( BMK_benchFn_t benchFn, void* benchPayload, BMK_initFn_t initFn, void* initPayload, size_t blockCount, @@ -310,6 +310,7 @@ BMK_customReturn_t __attribute__((optimize("O0"))) BMK_benchFunction( } { + unsigned i, j; for(i = 0; i < blockCount; i++) { for(j = 0; j < srcBlockSizes[i]; j++) { @@ -318,11 +319,11 @@ BMK_customReturn_t __attribute__((optimize("O0"))) BMK_benchFunction( } for(i = 0; i < blockCount; i++) { memset(dstBlockBuffers[i], 0xE5, dstBlockCapacities[i]); /* warm up and erase result buffer */ - //this is written at end proc? where did compressed data get overwritten ny this? } - UTIL_sleepMilli(5); /* give processor time to other processes */ - UTIL_waitForNextTick(); + //UTIL_sleepMilli(5); /* give processor time to other processes */ + //UTIL_waitForNextTick(); + } { @@ -463,7 +464,6 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( } { size_t const decodedSize = (size_t)totalDSize64; free(*resultBufferPtr); - //TODO: decodedSize corrupted? *resultBufferPtr = malloc(decodedSize); if (!(*resultBufferPtr)) { EXM_THROW(33, BMK_return_t, "not enough memory"); diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 44bcedef8..77e9f2250 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -598,8 +598,8 @@ static void paramVariation(ZSTD_compressionParameters* ptr, const U32* varyParam const U32 changeID = FUZ_rand(&g_rand) % (varyLen << 1); paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p); } - //validated = !ZSTD_isError(ZSTD_checkCParams(p)); - validated = cParamValid(p); + validated = !ZSTD_isError(ZSTD_checkCParams(p)); + //validated = cParamValid(p); //Make sure memory is at least close to feasible? //ZSTD_estimateCCtxSize thing. @@ -669,23 +669,27 @@ static void memoTableIndInv(ZSTD_compressionParameters* ptr, const U32* varyPara //initializing memoTable /* */ -static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstraints, constraint_t target, const U32* varyParams, const int varyLen) { +static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstraints, constraint_t target, const U32* varyParams, const int varyLen, const size_t srcSize) { size_t i; size_t arrayLen = memoTableLen(varyParams, varyLen); int cwFixed = !paramConstraints.chainLog || !paramConstraints.windowLog; int scFixed = !paramConstraints.searchLog || !paramConstraints.chainLog; + int wFixed = !paramConstraints.windowLog; int j = 0; memset(memoTable, 0, arrayLen); - + cParamZeroMin(¶mConstraints); for(i = 0; i < arrayLen; i++) { memoTableIndInv(¶mConstraints, varyParams, varyLen, i); - BMK_translateAdvancedParams(paramConstraints); if(ZSTD_estimateCCtxSize_usingCParams(paramConstraints) + (1 << paramConstraints.windowLog) > target.cMem) { //infeasible; memoTable[i] = 255; j++; } + //TODO: remove any where memoTable wlog is mark any where windowlog is too big for data. + if(wFixed && (1 << paramConstraints.windowLog) > (srcSize << 1)) { + memoTable[i] = 255; + } /* nil out parameter sets equivalent to others. */ if(cwFixed/* at most least 1 param fixed. */) { if(paramConstraints.strategy == ZSTD_btlazy2 || paramConstraints.strategy == ZSTD_btopt || paramConstraints.strategy == ZSTD_btultra) { @@ -771,8 +775,8 @@ static ZSTD_compressionParameters randomParams(void) p.searchLength=(FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN)) + ZSTD_SEARCHLENGTH_MIN; p.targetLength=(FUZ_rand(&g_rand) % (512)); p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btultra +1)); - //validated = !ZSTD_isError(ZSTD_checkCParams(p)); - validated = cParamValid(p); + validated = !ZSTD_isError(ZSTD_checkCParams(p)); + //validated = cParamValid(p); } return p; } @@ -1201,7 +1205,7 @@ static int feasibleBench(BMK_result_t* resultPtr, if(loopDurationD < TIMELOOP_NANOSEC) { BMK_return_t benchres2; adv.mode = BMK_decodeOnly; - benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1230,7 +1234,7 @@ static int feasibleBench(BMK_result_t* resultPtr, if(loopDurationD < TIMELOOP_NANOSEC) { BMK_return_t benchres2; adv.mode = BMK_decodeOnly; - benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1275,14 +1279,12 @@ static int infeasibleBench(BMK_result_t* resultPtr, BMK_advancedParams_t adv = BMK_initAdvancedParams(); BMK_return_t benchres; BMK_result_t resultMin, resultMax; - UTIL_time_t startTime; U64 loopDurationC = 0, loopDurationD = 0; double uncertaintyConstantC, uncertaintyConstantD; double winnerRS = resultScore(*winnerResult, srcSize, target); adv.loopMode = BMK_iterMode; //can only use this for ratio measurement then, super inaccurate timing adv.nbSeconds = 1; //get ratio and 2x approx speed? //maybe run until twice MIN(minloopinterval * clockDuration) - (void)startTime; //TODO: actually use this to adjust timing DISPLAY("WinnerScore: %f\n ", winnerRS); /* adv.loopMode = BMK_timeMode; @@ -1290,6 +1292,11 @@ static int infeasibleBench(BMK_result_t* resultPtr, benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); + adv.loopMode = BMK_timeMode; + adv.nbSeconds = 1; + benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); + if(!benchres.error) { *resultPtr = benchres.result; if(eqZero(benchres.result.cSpeed)) { @@ -1372,6 +1379,7 @@ static int feasibleBenchMemo(BMK_result_t* resultPtr, U32* varyParams, const int varyLen) { size_t memind = memoTableInd(&cParams, varyParams, varyLen); + //BMK_translateAdvancedParams(cParams); if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { return INFEASIBLE_RESULT; //probably pick a different code for already tested? @@ -1397,6 +1405,7 @@ static int infeasibleBenchMemo(BMK_result_t* resultPtr, BMK_result_t* winnerResult, U8* memoTable, U32* varyParams, const int varyLen) { size_t memind = memoTableInd(&cParams, varyParams, varyLen); + //BMK_translateAdvancedParams(cParams); if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { return INFEASIBLE_RESULT; //see feasibleBenchMemo for concerns @@ -1422,7 +1431,6 @@ typedef int (*BMK_benchMemo_t)(BMK_result_t*, const void*, size_t, void*, size_t // *actually if it performs too //sanitize all params here. //all generation after random should be sanitized. (maybe sanitize random) -//TODO: paramTarget uneeded at this point w/ varArray and init; static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varLen, U8* memoTable, const void* srcBuffer, size_t srcSize, void* dstBuffer, size_t dstSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, ZSTD_compressionParameters init) { //pick later initializations non-randomly? high dist from explored nodes. @@ -1470,12 +1478,13 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); candidateInfo.params = cparam; //all dist-1 targets + //if we early end this, we should also randomize the order these are picked. for(i = 0; i < varLen; i++) { paramVaryOnce(varArray[i], 1, &candidateInfo.params); /* +1 */ candidateInfo.params = sanitizeParams(candidateInfo.params); //evaluate - //if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { - if(cParamValid(candidateInfo.params)) { + if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { + //if(cParamValid(candidateInfo.params)) { int res = infeasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, @@ -1498,8 +1507,8 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL paramVaryOnce(varArray[i], -1, &candidateInfo.params); /* -1 */ candidateInfo.params = sanitizeParams(candidateInfo.params); //evaluate - //if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { - if(cParamValid(candidateInfo.params)) { + if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { + //if(cParamValid(candidateInfo.params)) { int res = infeasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, @@ -1587,8 +1596,8 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL candidateInfo.params = sanitizeParams(candidateInfo.params); //evaluate - //if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { - if(cParamValid(candidateInfo.params)) { + if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { + //if(cParamValid(candidateInfo.params)) { int res = feasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, @@ -1605,8 +1614,8 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL paramVaryOnce(varArray[i], -1, &candidateInfo.params); candidateInfo.params = sanitizeParams(candidateInfo.params); //evaluate - //if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { - if(cParamValid(candidateInfo.params)) { + if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { + //if(cParamValid(candidateInfo.params)) { int res = feasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, @@ -1681,9 +1690,11 @@ static winnerInfo_t optimizeFixedStrategy( /* so climb is given the right fixed strategy */ paramTarget.strategy = strat; /* to pass ZSTD_checkCParams */ - cParamZeroMin(¶mTarget); - memoTableInit(memoTable, paramTarget, target, varNew, varLenNew); + memoTableInit(memoTable, paramTarget, target, varNew, varLenNew, srcSize); + + //needs to happen after memoTableInit as that assumes 0 = undefined. + cParamZeroMin(¶mTarget); init = paramTarget; @@ -1699,7 +1710,7 @@ static winnerInfo_t optimizeFixedStrategy( candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, srcBuffer, srcSize, dstBuffer, dstSize, ctx, dctx, init); if(objective_lt(winnerInfo.result, candidateInfo.result)) { winnerInfo = candidateInfo; - DISPLAY("Climb Winner: "); + DISPLAY("New Winner: "); BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); } i++; @@ -1714,7 +1725,7 @@ _cleanUp: } // bigger and (hopefully) better* than optimizeForSize -// TODO: Change level bm'ing to respect constraints. +// TODO: allow accept multiple files like benchFiles or bench.c fn's static int optimizeForSize2(const char* inFileName, constraint_t target, ZSTD_compressionParameters paramTarget) { FILE* const inFile = fopen( inFileName, "rb" ); From fab44388014a8fbb3cf4ba9f46cfdc241807299d Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 9 Jul 2018 18:37:54 -0700 Subject: [PATCH 04/19] Dictionary + Multiple file Loading --- tests/paramgrill.c | 209 ++++++++++++++++++++++++++++++++------------- 1 file changed, 151 insertions(+), 58 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 77e9f2250..2ad8c5752 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -114,6 +114,7 @@ static U32 g_singleRun = 0; static U32 g_target = 0; static U32 g_noSeed = 0; static ZSTD_compressionParameters g_params = { 0, 0, 0, 0, 0, 0, ZSTD_greedy }; +static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */ void BMK_SetNbIterations(int nbLoops) { @@ -1141,6 +1142,7 @@ static double calcUncertainty(double speed, size_t srcSize) { static int feasibleBench(BMK_result_t* resultPtr, const void* srcBuffer, size_t srcSize, void* dstBuffer, size_t dstSize, + void* dictBuffer, size_t dictSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams, const constraint_t target, @@ -1155,7 +1157,7 @@ static int feasibleBench(BMK_result_t* resultPtr, //alternative - test 1 iter for ratio, (possibility of error 3 which is fine), //maybe iter this until 2x measurable for better guarantee? DISPLAY("Feas:\n"); - benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres.error) { DISPLAY("ERROR %d !!\n", benchres.error); } @@ -1195,7 +1197,7 @@ static int feasibleBench(BMK_result_t* resultPtr, if(loopDurationC < TIMELOOP_NANOSEC) { BMK_return_t benchres2; adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1205,7 +1207,7 @@ static int feasibleBench(BMK_result_t* resultPtr, if(loopDurationD < TIMELOOP_NANOSEC) { BMK_return_t benchres2; adv.mode = BMK_decodeOnly; - benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1224,7 +1226,7 @@ static int feasibleBench(BMK_result_t* resultPtr, if(loopDurationC < TIMELOOP_NANOSEC) { BMK_return_t benchres2; adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1234,7 +1236,7 @@ static int feasibleBench(BMK_result_t* resultPtr, if(loopDurationD < TIMELOOP_NANOSEC) { BMK_return_t benchres2; adv.mode = BMK_decodeOnly; - benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1250,7 +1252,7 @@ static int feasibleBench(BMK_result_t* resultPtr, return INFEASIBLE_RESULT; } else { //possibly better, benchmark and find out adv.loopMode = BMK_timeMode; - benchres = BMK_benchMemAdvanced(srcBuffer, srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres = BMK_benchMemAdvanced(srcBuffer, srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); *resultPtr = benchres.result; return objective_lt(*winnerResult, benchres.result); } @@ -1272,6 +1274,7 @@ static int feasibleBench(BMK_result_t* resultPtr, static int infeasibleBench(BMK_result_t* resultPtr, const void* srcBuffer, size_t srcSize, void* dstBuffer, size_t dstSize, + void* dictBuffer, size_t dictSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams, const constraint_t target, @@ -1289,12 +1292,14 @@ static int infeasibleBench(BMK_result_t* resultPtr, /* adv.loopMode = BMK_timeMode; adv.nbSeconds = 1; */ - benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); + benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); adv.loopMode = BMK_timeMode; adv.nbSeconds = 1; - benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); if(!benchres.error) { @@ -1336,7 +1341,7 @@ static int infeasibleBench(BMK_result_t* resultPtr, if(loopDurationC < TIMELOOP_NANOSEC) { BMK_return_t benchres2; adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1348,7 +1353,7 @@ static int infeasibleBench(BMK_result_t* resultPtr, adv.mode = BMK_decodeOnly; //TODO: dstBuffer corrupted sometime between top and now //probably occuring in feasible bench too. - benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1372,6 +1377,7 @@ static int infeasibleBench(BMK_result_t* resultPtr, static int feasibleBenchMemo(BMK_result_t* resultPtr, const void* srcBuffer, size_t srcSize, void* dstBuffer, size_t dstSize, + void* dictBuffer, size_t dictSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams, const constraint_t target, @@ -1387,7 +1393,7 @@ static int feasibleBenchMemo(BMK_result_t* resultPtr, //what is the intended behavior in this case? //ignore? stop iterating completely? other? } else { - int res = feasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, ctx, dctx, + int res = feasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, ctx, dctx, cParams, target, winnerResult); memoTable[memind] = 255; //tested are all infeasible (other possible values for opti) return res; @@ -1399,6 +1405,7 @@ static int feasibleBenchMemo(BMK_result_t* resultPtr, static int infeasibleBenchMemo(BMK_result_t* resultPtr, const void* srcBuffer, size_t srcSize, void* dstBuffer, size_t dstSize, + void* dictBuffer, size_t dictSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams, const constraint_t target, @@ -1410,7 +1417,7 @@ static int infeasibleBenchMemo(BMK_result_t* resultPtr, if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { return INFEASIBLE_RESULT; //see feasibleBenchMemo for concerns } else { - int res = infeasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, ctx, dctx, + int res = infeasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, ctx, dctx, cParams, target, winnerResult); if(res == FEASIBLE_RESULT) { memoTable[memind] = 255; //infeasible resultscores could still be normal feasible. @@ -1432,7 +1439,7 @@ typedef int (*BMK_benchMemo_t)(BMK_result_t*, const void*, size_t, void*, size_t //sanitize all params here. //all generation after random should be sanitized. (maybe sanitize random) static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varLen, U8* memoTable, - const void* srcBuffer, size_t srcSize, void* dstBuffer, size_t dstSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, ZSTD_compressionParameters init) { + const void* srcBuffer, size_t srcSize, void* dstBuffer, size_t dstSize, void* dictBuffer, size_t dictSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, ZSTD_compressionParameters init) { //pick later initializations non-randomly? high dist from explored nodes. //how to do this efficiently? (might not be too much of a problem, happens rarely, running time probably dominated by benchmarking) //distance maximizing selection? @@ -1488,6 +1495,7 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL int res = infeasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, + dictBuffer, dictSize, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); @@ -1512,6 +1520,7 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL int res = infeasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, + dictBuffer, dictSize, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); @@ -1519,7 +1528,7 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL winnerInfo = candidateInfo; //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); better = 1; - if(feasible(candidateInfo.result, target)) { + if(feasible(candidateInfo.result, target)) { //TODO: maybe just break here and move on to part 2? bestFeasible2 = winnerInfo; if(objective_lt(bestFeasible1.result, bestFeasible2.result)) { bestFeasible1 = bestFeasible2; @@ -1535,14 +1544,15 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL //if 'better' enough, skip further parameter search, center there? //possible improvement - guide direction here w/ knowledge rather than completely random variation. for(d = 2; d < varLen + 2; d++) { /* varLen is # dimensions */ - for(i = 0; i < 5; i++) { //make ? relative to # of free dimensions. + for(i = 0; i < 2 * varLen + 2; i++) { int res; candidateInfo.params = cparam; /* param error checking already done here */ paramVariation(&candidateInfo.params, varArray, varLen, d); res = infeasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, - dstBuffer, dstSize, + dstBuffer, dstSize, + dictBuffer, dictSize, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); @@ -1601,6 +1611,7 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL int res = feasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, + dictBuffer, dictSize, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); @@ -1619,6 +1630,7 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL int res = feasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, + dictBuffer, dictSize, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); @@ -1632,7 +1644,7 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL //if 'better' enough, skip further parameter search, center there? //possible improvement - guide direction here w/ knowledge rather than completely random variation. for(d = 2; d < varLen + 2; d++) { /* varLen is # dimensions */ - for(i = 0; i < 5; i++) { //TODO: make ? relative to # of free dimensions. + for(i = 0; i < 2 * varLen + 2; i++) { int res; candidateInfo.params = cparam; /* param error checking already done here */ @@ -1640,6 +1652,7 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL res = feasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, + dictBuffer, dictSize, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); @@ -1672,6 +1685,7 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL static winnerInfo_t optimizeFixedStrategy( const void* srcBuffer, const size_t srcSize, void* dstBuffer, size_t dstSize, + void* dictBuffer, size_t dictSize, constraint_t target, ZSTD_compressionParameters paramTarget, ZSTD_strategy strat, U32* varArray, int varLen) { int i = 0; //TODO: Temp fix 10 iters, check effects of changing this? @@ -1706,12 +1720,14 @@ static winnerInfo_t optimizeFixedStrategy( while(i < 10) { DISPLAY("Restart\n"); + //TODO: look into improving this to maximize distance from searched infeasible stuff / towards promising regions? randomConstrainedParams(&init, varNew, varLenNew, memoTable); - candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, srcBuffer, srcSize, dstBuffer, dstSize, ctx, dctx, init); + candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, ctx, dctx, init); if(objective_lt(winnerInfo.result, candidateInfo.result)) { winnerInfo = candidateInfo; DISPLAY("New Winner: "); BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + i = 0; } i++; } @@ -1724,59 +1740,125 @@ _cleanUp: return winnerInfo; } +static int BMK_loadFiles(void* buffer, size_t bufferSize, + size_t* fileSizes, const char* const * const fileNamesTable, + unsigned nbFiles) +{ + size_t pos = 0, totalSize = 0; + unsigned n; + for (n=0; n bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */ + { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f); + if (readSize != (size_t)fileSize) { + DISPLAY("could not read %s", fileNamesTable[n]); + return 11; + } + pos += readSize; } + fileSizes[n] = (size_t)fileSize; + totalSize += (size_t)fileSize; + fclose(f); + } + + if (totalSize == 0) { DISPLAY("no data to bench\n"); return 12; } + return 0; +} + // bigger and (hopefully) better* than optimizeForSize // TODO: allow accept multiple files like benchFiles or bench.c fn's -static int optimizeForSize2(const char* inFileName, constraint_t target, ZSTD_compressionParameters paramTarget) +static int optimizeForSize2(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget) { - FILE* const inFile = fopen( inFileName, "rb" ); - U64 const inFileSize = UTIL_getFileSize(inFileName); - size_t benchedSize = BMK_findMaxMem(inFileSize*3) / 3; + size_t benchedSize; void* origBuff; + void* dictBuffer; + size_t dictBufferSize; U32 varArray [NUM_PARAMS]; int varLen = variableParams(paramTarget, varArray); /* Init */ - - if(!cParamValid(paramTarget)) { return 10; } - if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; } - if (inFileSize == UTIL_FILESIZE_UNKNOWN) { - DISPLAY("Pb evaluatin size of %s \n", inFileName); - fclose(inFile); - return 11; - } - - /* Memory allocation & restrictions */ - if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize; - if (benchedSize < inFileSize) { - DISPLAY("Not enough memory for '%s' \n", inFileName); - fclose(inFile); - return 11; - } - - /* Alloc */ - origBuff = malloc(benchedSize); - if(!origBuff) { - DISPLAY("\nError: not enough memory!\n"); - fclose(inFile); - return 12; + /* load dictionary*/ + if (dictFileName != NULL) { + U64 const dictFileSize = UTIL_getFileSize(dictFileName); + if (dictFileSize > 64 MB) { + DISPLAY("dictionary file %s too large", dictFileName); + return 10; + } + dictBufferSize = (size_t)dictFileSize; + dictBuffer = malloc(dictBufferSize); + if (dictBuffer==NULL) { + DISPLAY("not enough memory for dictionary (%u bytes)", + (U32)dictBufferSize); + return 11; + } + { + int errorCode = BMK_loadFiles(dictBuffer, dictBufferSize, &dictBufferSize, &dictFileName, 1); + if(errorCode) { + free(dictBuffer); + return errorCode; + } + } } /* Fill input buffer */ - DISPLAY("Loading %s... \r", inFileName); - { size_t const readSize = fread(origBuff, 1, benchedSize, inFile); - fclose(inFile); - if(readSize != benchedSize) { - DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); + if(nbFiles == 1) { + DISPLAY("Loading %s... \r", fileNamesTable[0]); + } else { + DISPLAY("Loading %zd Files... \r", nbFiles); + } + + { + U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles); + int ec; + size_t* fileSizes = calloc(sizeof(size_t),nbFiles); + benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3; + origBuff = malloc(benchedSize); + if(!origBuff || !fileSizes) { + DISPLAY("Not enough memory for stuff\n"); free(origBuff); - return 13; - } } + free(fileSizes); + free(dictBuffer); + return 1; + } + ec = BMK_loadFiles(origBuff, benchedSize, fileSizes, fileNamesTable, nbFiles); + if(ec) { + DISPLAY("Error Loading Files"); + free(origBuff); + free(fileSizes); + free(dictBuffer); + return ec; + } + free(fileSizes); + } + + /* bench */ DISPLAY("\r%79s\r", ""); - DISPLAY("optimizing for %s", inFileName); + if(nbFiles == 1) { + DISPLAY("optimizing for %s", fileNamesTable[0]); + } else { + DISPLAY("optimizing for %zd Files", nbFiles); + } if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed / 1000000); } if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed / 1000000); } if(target.cMem != (U32)-1) { DISPLAY(" - limit memory %u MB", target.cMem / 1000000); } @@ -1828,7 +1910,7 @@ static int optimizeForSize2(const char* inFileName, constraint_t target, ZSTD_co if(paramTarget.strategy == 0) { int st; for(st = 1; st <= 8; st++) { - winnerInfo_t wc = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, + winnerInfo_t wc = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, dictBuffer, dictBufferSize, target, paramTarget, st, varArray, varLen); DISPLAY("StratNum %d\n", st); if(objective_lt(winner.result, wc.result)) { @@ -1836,7 +1918,7 @@ static int optimizeForSize2(const char* inFileName, constraint_t target, ZSTD_co } } } else { - winner = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, + winner = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, dictBuffer, dictBufferSize, target, paramTarget, paramTarget.strategy, varArray, varLen); } @@ -2070,7 +2152,8 @@ int main(int argc, const char** argv) filenamesStart=0, result; const char* exename=argv[0]; - const char* input_filename=0; + const char* input_filename = 0; + const char* dictFileName = 0; U32 optimizer = 0; U32 main_pause = 0; @@ -2283,6 +2366,17 @@ int main(int argc, const char** argv) g_grillDuration_s = (double)readU32FromChar(&argument); break; + /* load dictionary file (only applicable for optimizer rn) */ + case 'D': + if(i == argc - 1) { //last argument, return error. + DISPLAY("Dictionary file expected but not given\n"); + return 1; + } else { + i++; + dictFileName = argv[i]; + } + break; + /* Unknown command */ default : return badusage(exename); } @@ -2303,8 +2397,7 @@ int main(int argc, const char** argv) } } else { if (optimizer) { - result = optimizeForSize2(input_filename, target, paramTarget); - //optimizeForSize(input_filename, target, paramTarget); + result = optimizeForSize2(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget); } else { result = benchFiles(argv+filenamesStart, argc-filenamesStart); } } From 3adc217ea47f93f11e32ca6d3bf2f103ff3dbbd0 Mon Sep 17 00:00:00 2001 From: George Lu Date: Thu, 12 Jul 2018 17:30:39 -0700 Subject: [PATCH 05/19] Total Changes: Add different constraint types (decompression speed, compression memory, parameter constraints) Separate search space by strategy + strategy selection Memoize results Real random restarts Support multiple files Support Dictionary inputs Debug Macro for extra printing --- programs/bench.c | 14 +- programs/bench.h | 4 +- tests/Makefile | 2 +- tests/fullbench.c | 4 +- tests/paramgrill.c | 1090 ++++++++++++++++++++++---------------------- 5 files changed, 548 insertions(+), 566 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 5d5b47aba..f5184fa8f 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -85,7 +85,7 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; * Exceptions ***************************************/ #ifndef DEBUG -# define DEBUG 1 +# define DEBUG 0 #endif #define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); } @@ -188,7 +188,7 @@ static void BMK_initCCtx(ZSTD_CCtx* ctx, ZSTD_CCtx_setParameter(ctx, ZSTD_p_searchLog, comprParams->searchLog); ZSTD_CCtx_setParameter(ctx, ZSTD_p_minMatch, comprParams->searchLength); ZSTD_CCtx_setParameter(ctx, ZSTD_p_targetLength, comprParams->targetLength); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy, comprParams->strategy); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy , comprParams->strategy); ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize); } @@ -293,7 +293,7 @@ BMK_customReturn_t BMK_benchFunction( BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBlockBuffers, const size_t* srcBlockSizes, - void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, + void** const dstBlockBuffers, size_t* dstBlockCapacities, unsigned nbLoops) { size_t srcSize = 0, dstSize = 0, ind = 0; U64 totalTime; @@ -338,6 +338,11 @@ BMK_customReturn_t BMK_benchFunction( j, (U32)dstBlockCapacities[j], ZSTD_getErrorName(res)); } else if(firstIter) { dstSize += res; + //Make compressed blocks continuous + if(j != blockCount - 1) { + dstBlockBuffers[j+1] = (void*)((char*)dstBlockBuffers[j] + res); + dstBlockCapacities[j] = res; + } } } firstIter = 0; @@ -370,13 +375,14 @@ void BMK_freeTimeState(BMK_timedFnState_t* state) { free(state); } +/* make option for dstBlocks to be */ BMK_customTimedReturn_t BMK_benchFunctionTimed( BMK_timedFnState_t* cont, BMK_benchFn_t benchFn, void* benchPayload, BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const* const srcBlockBuffers, const size_t* srcBlockSizes, - void* const* const dstBlockBuffers, const size_t* dstBlockCapacities) + void** const dstBlockBuffers, size_t* dstBlockCapacities) { U64 fastest = cont->fastestTime; int completed = 0; diff --git a/programs/bench.h b/programs/bench.h index 625b65757..1a298cc48 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -191,7 +191,7 @@ BMK_customReturn_t BMK_benchFunction( BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBuffers, const size_t* srcSizes, - void* const * const dstBuffers, const size_t* dstCapacities, + void** const dstBuffers, size_t* dstCapacities, unsigned nbLoops); @@ -220,7 +220,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed(BMK_timedFnState_t* cont, BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBlockBuffers, const size_t* srcBlockSizes, - void* const * const dstBlockBuffers, const size_t* dstBlockCapacities); + void** const dstBlockBuffers, size_t* dstBlockCapacities); #endif /* BENCH_H_121279284357 */ diff --git a/tests/Makefile b/tests/Makefile index ecf7fe2a7..81e685780 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -200,7 +200,7 @@ zstreamtest-dll : $(ZSTDDIR)/common/xxhash.c # xxh symbols not exposed from dll zstreamtest-dll : $(ZSTREAM_LOCAL_FILES) $(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT) -#paramgrill : DEBUGFLAGS = # turn off assert() for speed measurements +paramgrill : DEBUGFLAGS = # turn off assert() for speed measurements paramgrill : $(ZSTD_FILES) $(PRGDIR)/bench.c $(PRGDIR)/datagen.c paramgrill.c $(CC) $(FLAGS) $^ -lm -o $@$(EXT) diff --git a/tests/fullbench.c b/tests/fullbench.c index 7859745a3..9e7639f92 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -336,7 +336,7 @@ size_t local_ZSTD_decompressContinue(const void* src, size_t srcSize, void* dst, static size_t benchMem(const void* src, size_t srcSize, U32 benchNb, int cLevel, ZSTD_compressionParameters* cparams) { BYTE* dstBuff; - size_t const dstBuffSize = ZSTD_compressBound(srcSize); + size_t dstBuffSize = ZSTD_compressBound(srcSize); void* buff2, *buff1; const char* benchName; BMK_benchFn_t benchFunction; @@ -516,7 +516,7 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb, int cLevel, { r = BMK_benchFunction(benchFunction, buff2, NULL, NULL, 1, &src, &srcSize, - (void * const * const)&dstBuff, &dstBuffSize, g_nbIterations); + (void **)&dstBuff, &dstBuffSize, g_nbIterations); if(r.error) { DISPLAY("ERROR %d ! ! \n", r.error); errorcode = r.error; diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 2ad8c5752..9af60a740 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -58,6 +58,11 @@ static const int g_maxNbVariations = 64; * Macros **************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define TIMED 0 +#ifndef DEBUG +# define DEBUG 0 +#endif +#define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); } #undef MIN #undef MAX @@ -81,8 +86,8 @@ static const int g_maxNbVariations = 64; #define ZSTD_WINDOWLOG_MAX 27 //no long range stuff for now. //make 2^[0,10] w/ 999 -#define ZSTD_TARGETLENGTH_MIN 0 //actually targeLengthlog min -#define ZSTD_TARGETLENGTH_MAX 10 +#define ZSTD_TARGETLENGTH_MIN 0 +#define ZSTD_TARGETLENGTH_MAX 999 //#define ZSTD_TARGETLENGTH_MAX 1024 #define WLOG_RANGE (ZSTD_WINDOWLOG_MAX - ZSTD_WINDOWLOG_MIN + 1) @@ -90,15 +95,14 @@ static const int g_maxNbVariations = 64; #define HLOG_RANGE (ZSTD_HASHLOG_MAX - ZSTD_HASHLOG_MIN + 1) #define SLOG_RANGE (ZSTD_SEARCHLOG_MAX - ZSTD_SEARCHLOG_MIN + 1) #define SLEN_RANGE (ZSTD_SEARCHLENGTH_MAX - ZSTD_SEARCHLENGTH_MIN + 1) -#define TLEN_RANGE 11 +#define TLEN_RANGE 12 +//TLEN_RANGE = 0, 2^0 to 2^10; //hard coded since we only use powers of 2 (and 999 ~ 1024) -static const int mintable[NUM_PARAMS] = { ZSTD_WINDOWLOG_MIN, ZSTD_CHAINLOG_MIN, ZSTD_HASHLOG_MIN, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLENGTH_MIN, ZSTD_TARGETLENGTH_MIN }; -static const int maxtable[NUM_PARAMS] = { ZSTD_WINDOWLOG_MAX, ZSTD_CHAINLOG_MAX, ZSTD_HASHLOG_MAX, ZSTD_SEARCHLOG_MAX, ZSTD_SEARCHLENGTH_MAX, ZSTD_TARGETLENGTH_MAX }; +//static const int mintable[NUM_PARAMS] = { ZSTD_WINDOWLOG_MIN, ZSTD_CHAINLOG_MIN, ZSTD_HASHLOG_MIN, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLENGTH_MIN, ZSTD_TARGETLENGTH_MIN }; +//static const int maxtable[NUM_PARAMS] = { ZSTD_WINDOWLOG_MAX, ZSTD_CHAINLOG_MAX, ZSTD_HASHLOG_MAX, ZSTD_SEARCHLOG_MAX, ZSTD_SEARCHLENGTH_MAX, ZSTD_TARGETLENGTH_MAX }; static const int rangetable[NUM_PARAMS] = { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE }; -//use grid-search or something when space is small enough? -#define SMALL_SEARCH_SPACE 1000 /*-************************************ * Benchmark Parameters **************************************/ @@ -201,7 +205,7 @@ static void findClockGranularity(void) { } } } while(i < 10); - DISPLAY("Granularity: %llu\n", (unsigned long long)g_clockGranularity); + DEBUGOUTPUT("Granularity: %llu\n", (unsigned long long)g_clockGranularity); } typedef struct { @@ -225,6 +229,10 @@ static int cParamValid(ZSTD_compressionParameters paramTarget) { CLAMPCHECK(paramTarget.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX); CLAMPCHECK(paramTarget.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX); CLAMPCHECK(paramTarget.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX); + if(paramTarget.targetLength > ZSTD_TARGETLENGTH_MAX) { + DISPLAY("INVALID PARAMETER CONSTRAINTS\n"); + return 0; + } if(paramTarget.strategy > ZSTD_btultra) { DISPLAY("INVALID PARAMETER CONSTRAINTS\n"); return 0; @@ -232,21 +240,68 @@ static int cParamValid(ZSTD_compressionParameters paramTarget) { return 1; } +//TODO: let targetLength = 0; static void cParamZeroMin(ZSTD_compressionParameters* paramTarget) { paramTarget->windowLog = paramTarget->windowLog ? paramTarget->windowLog : ZSTD_WINDOWLOG_MIN; paramTarget->searchLog = paramTarget->searchLog ? paramTarget->searchLog : ZSTD_SEARCHLOG_MIN; paramTarget->chainLog = paramTarget->chainLog ? paramTarget->chainLog : ZSTD_CHAINLOG_MIN; paramTarget->hashLog = paramTarget->hashLog ? paramTarget->hashLog : ZSTD_HASHLOG_MIN; paramTarget->searchLength = paramTarget->searchLength ? paramTarget->searchLength : ZSTD_SEARCHLENGTH_MIN; - paramTarget->targetLength = paramTarget->targetLength ? paramTarget->targetLength : 1; + paramTarget->targetLength = paramTarget->targetLength ? paramTarget->targetLength : 0; } -static void BMK_translateAdvancedParams(ZSTD_compressionParameters params) +static void BMK_translateAdvancedParams(const ZSTD_compressionParameters params) { DISPLAY("--zstd=windowLog=%u,chainLog=%u,hashLog=%u,searchLog=%u,searchLength=%u,targetLength=%u,strategy=%u \n", params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, params.targetLength, (U32)(params.strategy)); } +/* checks results are feasible */ +static int feasible(const BMK_result_t results, const constraint_t target) { + return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem || !target.cMem); +} + +#define EPSILON 0.01 +static int epsilonEqual(const double c1, const double c2) { + return MAX(c1/c2,c2/c1) < 1 + EPSILON; +} + +/* checks exact equivalence to 0, to stop compiler complaining fpeq */ +static int eqZero(const double c1) { + return (U64)c1 == (U64)0.0 || (U64)c1 == (U64)-0.0; +} + +/* returns 1 if result2 is strictly 'better' than result1 */ +/* strict comparison / cutoff based */ +static int objective_lt(const BMK_result_t result1, const BMK_result_t result2) { + return (result1.cSize > result2.cSize) || (epsilonEqual(result1.cSize, result2.cSize) && result2.cSpeed > result1.cSpeed) + || (epsilonEqual(result1.cSize,result2.cSize) && epsilonEqual(result2.cSpeed, result1.cSpeed) && result2.dSpeed > result1.dSpeed); +} + +/* hill climbing value for part 1 */ +static double resultScore(const BMK_result_t res, const size_t srcSize, const constraint_t target) { + double cs = 0., ds = 0., rt, cm = 0.; + const double r1 = 1, r2 = 0.1, rtr = 0.5; + double ret; + if(target.cSpeed) { cs = res.cSpeed / (double)target.cSpeed; } + if(target.dSpeed) { ds = res.dSpeed / (double)target.dSpeed; } + if(target.cMem != (U32)-1) { cm = (double)target.cMem / res.cMem; } + rt = ((double)srcSize / res.cSize); + + ret = (MIN(1, cs) + MIN(1, ds) + MIN(1, cm))*r1 + rt * rtr + + (MAX(0, log(cs))+ MAX(0, log(ds))+ MAX(0, log(cm))) * r2; + //DISPLAY("resultScore: %f\n", ret); + return ret; +} + +/* factor sort of arbitrary */ +static constraint_t relaxTarget(constraint_t target) { + target.cMem = (U32)-1; + target.cSpeed *= 0.9; + target.dSpeed *= 0.9; + return target; +} + /*-******************************************************* * Bench functions *********************************************************/ @@ -268,9 +323,21 @@ const char* g_stratName[ZSTD_btultra+1] = { "ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ", "ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "}; -/* TODO: support additional parameters (more files, fileSizes) */ static size_t BMK_benchParam(BMK_result_t* resultPtr, + const void* srcBuffer, const size_t srcSize, + const size_t* fileSizes, const unsigned nbFiles, + ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, + const ZSTD_compressionParameters cParams) { + + BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, fileSizes, nbFiles, 0, &cParams, NULL, 0, ctx, dctx, 0, "File"); + *resultPtr = res.result; + return res.error; +} + +/* benchParam but only takes in one file. */ +static size_t +BMK_benchParam1(BMK_result_t* resultPtr, const void* srcBuffer, size_t srcSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams) { @@ -280,27 +347,54 @@ BMK_benchParam(BMK_result_t* resultPtr, return res.error; } -static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_compressionParameters params, size_t srcSize) -{ - char lvlstr[15] = "Custom Level"; - DISPLAY("\r%79s\r", ""); - fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", - params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, - params.targetLength, g_stratName[(U32)(params.strategy)]); - if(cLevel != CUSTOM_LEVEL) { - snprintf(lvlstr, 15, " Level %2u ", cLevel); - } - fprintf(f, - "/* %s */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", - lvlstr, (double)srcSize / result.cSize, result.cSpeed / 1000000., result.dSpeed / 1000000.); -} - - typedef struct { BMK_result_t result; ZSTD_compressionParameters params; } winnerInfo_t; +/* global winner used for display. */ +//Should be totally 0 initialized? +static winnerInfo_t g_winner; //TODO: ratio is infinite at initialization, instead of 0 +static constraint_t g_targetConstraints; + +static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const size_t srcSize) +{ + if(DEBUG || (objective_lt(g_winner.result, result) && feasible(result, g_targetConstraints))) { + char lvlstr[15] = "Custom Level"; + const U64 time = UTIL_clockSpanNano(g_time); + const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC); + if(DEBUG && (objective_lt(g_winner.result, result) && feasible(result, g_targetConstraints))) { + DISPLAY("New Winner: \n"); + } + + DISPLAY("\r%79s\r", ""); + + fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", + params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, + params.targetLength, g_stratName[(U32)(params.strategy)]); + if(cLevel != CUSTOM_LEVEL) { + snprintf(lvlstr, 15, " Level %2u ", cLevel); + } + fprintf(f, + "/* %s */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */", + lvlstr, (double)srcSize / result.cSize, result.cSpeed / 1000000., result.dSpeed / 1000000.); + + if(TIMED) { fprintf(f, " - %lu:%lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); } + fprintf(f, "\n"); + if(objective_lt(g_winner.result, result) && feasible(result, g_targetConstraints)) { + BMK_translateAdvancedParams(params); + g_winner.result = result; + g_winner.params = params; + } + } + //else { + // DISPLAY("G_WINNER: "); + // DISPLAY("/* R:%5.3f at %5.1f MB/s - %5.1f MB/s */ \n",(double)srcSize / g_winner.result.cSize , g_winner.result.cSpeed / 1000000 , g_winner.result.dSpeed / 1000000); + // DISPLAY("LOSER : "); + // DISPLAY("/* R:%5.3f at %5.1f MB/s - %5.1f MB/s */ \n",(double)srcSize / result.cSize, result.cSpeed / 1000000 , result.dSpeed / 1000000); + //} +} + static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize) { int cLevel; @@ -358,7 +452,7 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para int better = 0; int cLevel; - BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, dctx, params); + BMK_benchParam1(&testResult, srcBuffer, srcSize, ctx, dctx, params); for (cLevel = 1; cLevel <= NB_LEVELS_TRACKED; cLevel++) { @@ -470,7 +564,7 @@ static ZSTD_compressionParameters sanitizeParams(ZSTD_compressionParameters para /* new length */ /* keep old array, will need if iter over strategy. */ -static int sanitizeVarArray(int varLength, U32* varArray, U32* varNew, ZSTD_strategy strat) { +static int sanitizeVarArray(const int varLength, const U32* varArray, U32* varNew, const ZSTD_strategy strat) { int i, j = 0; for(i = 0; i < varLength; i++) { if( !((varArray[i] == CLOG_IND && strat == ZSTD_fast) @@ -515,53 +609,10 @@ static int variableParams(const ZSTD_compressionParameters paramConstraints, U32 return j; } -/* computes inverse of above array, returns same number, -1 = unused ind */ -static int inverseVariableParams(const ZSTD_compressionParameters paramConstraints, U32* res) { - int j = 0; - if(!paramConstraints.windowLog) { - res[WLOG_IND] = j; - j++; - } else { - res[WLOG_IND] = -1; - } - if(!paramConstraints.chainLog) { - res[j] = CLOG_IND; - j++; - } else { - res[CLOG_IND] = -1; - } - if(!paramConstraints.hashLog) { - res[j] = HLOG_IND; - j++; - } else { - res[HLOG_IND] = -1; - } - if(!paramConstraints.searchLog) { - res[j] = SLOG_IND; - j++; - } else { - res[SLOG_IND] = -1; - } - if(!paramConstraints.searchLength) { - res[j] = SLEN_IND; - j++; - } else { - res[SLEN_IND] = -1; - } - if(!paramConstraints.targetLength) { - res[j] = TLEN_IND; - j++; - } else { - res[TLEN_IND] = -1; - } - - return j; -} - /* amt will probably always be \pm 1? */ /* slight change from old paramVariation, targetLength can only take on powers of 2 now (999 ~= 1024?) */ /* take max/min bounds into account as well? */ -static void paramVaryOnce(U32 paramIndex, int amt, ZSTD_compressionParameters* ptr) { +static void paramVaryOnce(const U32 paramIndex, const int amt, ZSTD_compressionParameters* ptr) { switch(paramIndex) { case WLOG_IND: ptr->windowLog += amt; break; @@ -571,8 +622,14 @@ static void paramVaryOnce(U32 paramIndex, int amt, ZSTD_compressionParameters* p case SLEN_IND: ptr->searchLength += amt; break; case TLEN_IND: if(amt >= 0) { - ptr->targetLength <<= amt; - ptr->targetLength = MIN(ptr->targetLength, 999); + if(ptr->targetLength == 0) { + if(amt > 0) { + ptr->targetLength = MIN(1 << (amt - 1), 999); + } + } else { + ptr->targetLength <<= amt; + ptr->targetLength = MIN(ptr->targetLength, 999); + } } else { if(ptr->targetLength == 999) { ptr->targetLength = 1024; @@ -584,11 +641,8 @@ static void paramVaryOnce(U32 paramIndex, int amt, ZSTD_compressionParameters* p } } -//Don't fuzz fixed variables. -//turn pcs to pcs array with macro for params. -//pass in variation array from variableParams -//take nbChanges as argument? -static void paramVariation(ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen, U32 nbChanges) +/* varies ptr by nbChanges respecting varyParams*/ +static void paramVariation(ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen, const U32 nbChanges) { ZSTD_compressionParameters p; U32 validated = 0; @@ -600,19 +654,11 @@ static void paramVariation(ZSTD_compressionParameters* ptr, const U32* varyParam paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p); } validated = !ZSTD_isError(ZSTD_checkCParams(p)); - //validated = cParamValid(p); - - //Make sure memory is at least close to feasible? - //ZSTD_estimateCCtxSize thing. } - *ptr = sanitizeParams(p); + *ptr = p;//sanitizeParams(p); } -//varyParams gives us table size? -//1 per strategy -//varyParams should always be sorted smallest to largest -//take arrayLen to allocate memotable -//should be ~10^7 unconstrained. +/* length of memo table given free variables */ static size_t memoTableLen(const U32* varyParams, const int varyLen) { size_t arrayLen = 1; int i; @@ -622,11 +668,14 @@ static size_t memoTableLen(const U32* varyParams, const int varyLen) { return arrayLen; } -//sort of ~lg2 (replace 1024 w/ 999) for memoTableInd Tlen +//sort of ~lg2 (replace 1024 w/ 999, and add 0 at lower end of range) for memoTableInd Tlen static unsigned lg2(unsigned x) { - unsigned j = 0; + unsigned j = 1; if(x == 999) { - return 10; + return 11; + } + if(!x) { + return 0; } while(x >>= 1) { j++; @@ -634,8 +683,7 @@ static unsigned lg2(unsigned x) { return j; } -//indexes compressionParameters into memotable -//of form +/* returns unique index of compression parameters */ static unsigned memoTableInd(const ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen) { int i; unsigned ind = 0; @@ -652,25 +700,24 @@ static unsigned memoTableInd(const ZSTD_compressionParameters* ptr, const U32* v return ind; } -/* presumably, the unfilled parameters are already at their correct value */ -/* inverse above function for varyParams */ +/* inverse of above function (from index to parameters) */ static void memoTableIndInv(ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen, size_t ind) { int i; for(i = varyLen - 1; i >= 0; i--) { switch(varyParams[i]) { - case WLOG_IND: ptr->windowLog = ind % WLOG_RANGE + ZSTD_WINDOWLOG_MIN; ind /= WLOG_RANGE; break; - case CLOG_IND: ptr->chainLog = ind % CLOG_RANGE + ZSTD_CHAINLOG_MIN; ind /= CLOG_RANGE; break; - case HLOG_IND: ptr->hashLog = ind % HLOG_RANGE + ZSTD_HASHLOG_MIN; ind /= HLOG_RANGE; break; - case SLOG_IND: ptr->searchLog = ind % SLOG_RANGE + ZSTD_SEARCHLOG_MIN; ind /= SLOG_RANGE; break; - case SLEN_IND: ptr->searchLength = ind % SLEN_RANGE + ZSTD_SEARCHLENGTH_MIN; ind /= SLEN_RANGE; break; - case TLEN_IND: ptr->targetLength = MIN(1 << (ind % TLEN_RANGE), 999); ind /= TLEN_RANGE; break; + case WLOG_IND: ptr->windowLog = ind % WLOG_RANGE + ZSTD_WINDOWLOG_MIN; ind /= WLOG_RANGE; break; + case CLOG_IND: ptr->chainLog = ind % CLOG_RANGE + ZSTD_CHAINLOG_MIN; ind /= CLOG_RANGE; break; + case HLOG_IND: ptr->hashLog = ind % HLOG_RANGE + ZSTD_HASHLOG_MIN; ind /= HLOG_RANGE; break; + case SLOG_IND: ptr->searchLog = ind % SLOG_RANGE + ZSTD_SEARCHLOG_MIN; ind /= SLOG_RANGE; break; + case SLEN_IND: ptr->searchLength = ind % SLEN_RANGE + ZSTD_SEARCHLENGTH_MIN; ind /= SLEN_RANGE; break; + case TLEN_IND: ptr->targetLength = (ind % TLEN_RANGE) ? MIN(1 << ((ind % TLEN_RANGE) - 1), 999) : 0; ind /= TLEN_RANGE; break; } } } -//initializing memoTable -/* */ -static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstraints, constraint_t target, const U32* varyParams, const int varyLen, const size_t srcSize) { + +/* Initialize memotable, immediately mark redundant / obviously infeasible params as */ +static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstraints, const constraint_t target, const U32* varyParams, const int varyLen, const size_t srcSize) { size_t i; size_t arrayLen = memoTableLen(varyParams, varyLen); int cwFixed = !paramConstraints.chainLog || !paramConstraints.windowLog; @@ -682,13 +729,11 @@ static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstra for(i = 0; i < arrayLen; i++) { memoTableIndInv(¶mConstraints, varyParams, varyLen, i); - if(ZSTD_estimateCCtxSize_usingCParams(paramConstraints) + (1 << paramConstraints.windowLog) > target.cMem) { - //infeasible; + if((ZSTD_estimateCCtxSize_usingCParams(paramConstraints) + (1ULL << paramConstraints.windowLog)) > (size_t)target.cMem + (size_t)(target.cMem / 10)) { memoTable[i] = 255; j++; } - //TODO: remove any where memoTable wlog is mark any where windowlog is too big for data. - if(wFixed && (1 << paramConstraints.windowLog) > (srcSize << 1)) { + if(wFixed && (1ULL << paramConstraints.windowLog) > (srcSize << 1)) { memoTable[i] = 255; } /* nil out parameter sets equivalent to others. */ @@ -712,7 +757,42 @@ static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstra } } } - DISPLAY("%d / %d Invalid\n", j, (int)i); + DEBUGOUTPUT("%d / %d Invalid\n", j, (int)i); + if((int)i == j) { + DEBUGOUTPUT("!!!Strategy %d totally infeasible\n", (int)paramConstraints.strategy) + } +} + +/* inits memotables for all (including mallocs), all strategies */ +/* takes unsanitized varyParams */ + +//TODO: check for errors/nulls +static U8** memoTableInitAll(ZSTD_compressionParameters paramConstraints, constraint_t target, const U32* varyParams, const int varyLen, const size_t srcSize) { + U32 varNew[NUM_PARAMS]; + int varLenNew; + U8** mtAll = malloc(sizeof(U8*) * (ZSTD_btultra + 1)); + int i; + if(mtAll == NULL) { + return NULL; + } + for(i = 1; i <= (int)ZSTD_btultra; i++) { + varLenNew = sanitizeVarArray(varyLen, varyParams, varNew, i); + mtAll[i] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew)); + if(mtAll[i] == NULL) { + return NULL; + } + memoTableInit(mtAll[i], paramConstraints, target, varNew, varLenNew, srcSize); + } + return mtAll; +} + +static void memoTableFreeAll(U8** mtAll) { + int i; + if(mtAll == NULL) { return; } + for(i = 1; i <= (int)ZSTD_btultra; i++) { + free(mtAll[i]); + } + free(mtAll); } #define PARAMTABLELOG 25 @@ -782,10 +862,7 @@ static ZSTD_compressionParameters randomParams(void) return p; } -//destructively modifies pc. -//Maybe if memoTable[ind] > 0 too often, count zeroes and explicitly choose from free stuff? -//^ maybe this doesn't matter, with |mt| size it has \approx 1-(1/e) of finding even single free spot in |mt| tries, not too bad. -//TODO: maybe memoTable pc before sanitization too so no repeats? +/* Sets pc to random unmeasured set of parameters */ static void randomConstrainedParams(ZSTD_compressionParameters* pc, U32* varArray, int varLen, U8* memoTable) { int tries = memoTableLen(varArray, varLen); //configurable, @@ -795,9 +872,7 @@ static void randomConstrainedParams(ZSTD_compressionParameters* pc, U32* varArra ind = (FUZ_rand(&g_rand)) % maxSize; tries--; } while(memoTable[ind] > 0 && tries > 0); - //&& FUZ_rand(&g_rand) % 256 > memoTable[ind]); get nd choosing? (helpful w/ distance) /* maybe > infeasible bound? */ - /* memoTable[ind] == 0 -> unexplored */ memoTableIndInv(pc, varArray, varLen, (unsigned)ind); *pc = sanitizeParams(*pc); } @@ -822,7 +897,7 @@ static void BMK_benchOnce(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* srcBuffe { BMK_result_t testResult; g_params = ZSTD_adjustCParams(g_params, srcSize, 0); - BMK_benchParam(&testResult, srcBuffer, srcSize, cctx, dctx, g_params); + BMK_benchParam1(&testResult, srcBuffer, srcSize, cctx, dctx, g_params); DISPLAY("Compression Ratio: %.3f Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)srcSize / testResult.cSize, testResult.cSpeed / 1000000, testResult.dSpeed / 1000000); return; @@ -847,7 +922,7 @@ static void BMK_benchFullTable(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* src /* baseline config for level 1 */ ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, blockSize, 0); BMK_result_t testResult; - BMK_benchParam(&testResult, srcBuffer, srcSize, cctx, dctx, l1params); + BMK_benchParam1(&testResult, srcBuffer, srcSize, cctx, dctx, l1params); BMK_init_level_constraints((int)((testResult.cSpeed * 31) / 32)); } @@ -974,112 +1049,14 @@ int benchFiles(const char** fileNamesTable, int nbFiles) return 0; } -//parameter feasibility is not checked, should just be restricted from use. -static int feasible(BMK_result_t results, constraint_t target) { - return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem || !target.cMem); -} -#define EPSILON 0.01 -static int epsilonEqual(double c1, double c2) { - return MAX(c1/c2,c2/c1) < 1 + EPSILON; -} - -//so the compiler stops warning -static int eqZero(double c1) { - return (U64)c1 == (U64)0.0 || (U64)c1 == (U64)-0.0; -} - -/* returns 1 if result2 is strictly 'better' than result1 */ -/* strict comparison / cutoff based */ -static int objective_lt(BMK_result_t result1, BMK_result_t result2) { - return (result1.cSize > result2.cSize) || (epsilonEqual(result1.cSize, result2.cSize) && result2.cSpeed > result1.cSpeed) - || (epsilonEqual(result1.cSize,result2.cSize) && epsilonEqual(result2.cSpeed, result1.cSpeed) && result2.dSpeed > result1.dSpeed); -} - -//will probably be some linear combinartion of comp speed, decompSpeed, & ratio (maybe size), and memory? -//pretty arbitrary right now -//maybe better - higher coefficient when below threshold, lower when above -//need to normalize speed? or just use ratio speed / target? -//Maybe don't use ratio at all when looking for feasibility? - -/* maybe dynamically vary the coefficients for this around based on what's already been discovered. (maybe make a reversed ratio cutoff?) concave to pheaily penalize below ratio? */ -static double resultScore(BMK_result_t res, size_t srcSize, constraint_t target) { - double cs = 0., ds = 0., rt, cm = 0.; - const double r1 = 1, r2 = 0.1, rtr = 0.5; - double ret; - if(target.cSpeed) { cs = res.cSpeed / (double)target.cSpeed; } - if(target.dSpeed) { ds = res.dSpeed / (double)target.dSpeed; } - if(target.cMem != (U32)-1) { cm = (double)target.cMem / res.cMem; } - rt = ((double)srcSize / res.cSize); - - //(void)rt; - //(void)rtr; - - ret = (MIN(1, cs) + MIN(1, ds) + MIN(1, cm))*r1 + rt * rtr + - (MAX(0, log(cs))+ MAX(0, log(ds))+ MAX(0, log(cm))) * r2; - //DISPLAY("resultScore: %f\n", ret); - return ret; -} - -/* - double W_ratio = (double)srcSize / testResult.cSize; - double O_ratio = (double)srcSize / winners[cLevel].result.cSize; - double W_ratioNote = log (W_ratio); - double O_ratioNote = log (O_ratio); - size_t W_DMemUsed = (1 << params.windowLog) + (16 KB); - size_t O_DMemUsed = (1 << winners[cLevel].params.windowLog) + (16 KB); - double W_DMemUsed_note = W_ratioNote * ( 40 + 9*cLevel) - log((double)W_DMemUsed); - double O_DMemUsed_note = O_ratioNote * ( 40 + 9*cLevel) - log((double)O_DMemUsed); - - size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_estimateCCtxSize_usingCParams(params); - size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_estimateCCtxSize_usingCParams(winners[cLevel].params); - double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed); - double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed); - - double W_CSpeed_note = W_ratioNote * ( 30 + 10*cLevel) + log(testResult.cSpeed); - double O_CSpeed_note = O_ratioNote * ( 30 + 10*cLevel) + log(winners[cLevel].result.cSpeed); - - double W_DSpeed_note = W_ratioNote * ( 20 + 2*cLevel) + log(testResult.dSpeed); - double O_DSpeed_note = O_ratioNote * ( 20 + 2*cLevel) + log(winners[cLevel].result.dSpeed); - -*/ -//ratio tradeoffs, may be useful in guiding - -/* objective_lt, but based on scoring function */ -static int objective_lt2(BMK_result_t result1, BMK_result_t result2, size_t srcSize, constraint_t target) { - return resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target); -} - -/* res gives array dimensions, should be size NUM_PARAMS */ -static size_t computeStateSize(const ZSTD_compressionParameters paramConstraints, U32* res) { - int ind = 0; - size_t base = 1; - if(!paramConstraints.windowLog) { res[ind] = ZSTD_WINDOWLOG_MAX - ZSTD_WINDOWLOG_MIN + 1; base *= res[ind]; ind++; } - if(!paramConstraints.chainLog) { res[ind] = ZSTD_CHAINLOG_MAX - ZSTD_CHAINLOG_MIN + 1; base *= res[ind]; ind++; } - if(!paramConstraints.hashLog) { res[ind] = ZSTD_HASHLOG_MAX - ZSTD_HASHLOG_MIN + 1; base *= res[ind]; ind++; } - if(!paramConstraints.searchLog) { res[ind] = ZSTD_SEARCHLOG_MAX - ZSTD_SEARCHLOG_MIN + 1; base *= res[ind]; ind++; } - if(!paramConstraints.searchLength) { res[ind] = ZSTD_SEARCHLENGTH_MAX - ZSTD_SEARCHLENGTH_MIN + 1; base *= res[ind]; ind++; } - if(!paramConstraints.targetLength) { res[ind] = 11; base *= res[ind]; ind++; } //restricting from 2^[0,10], no such macros - if(!(U32)paramConstraints.strategy) { res[ind] = 8; base *= 8; } //not strictly true, maybe would want to case on this. - - return base; -} - - -static unsigned calcViolation(BMK_result_t results, constraint_t target) { - int diffcSpeed = MAX(target.cSpeed - results.cSpeed, 0); - int diffdSpeed = MAX(target.dSpeed - results.dSpeed, 0); - int diffcMem = MAX(results.cMem - target.cMem, 0); - return diffcSpeed + diffdSpeed + diffcMem; -} /* -uncertaintyConstant >= 1 -returns -1 = 'certainly' infeasible - 0 = unceratin - 1 = 'certainly' feasible +checks feasibility with uncertainty. +-1 : certainly infeasible + 0 : uncertain + 1 : certainly feasible */ -//paramTarget misnamed, should just be target static int uncertainFeasibility(double const uncertaintyConstantC, double const uncertaintyConstantD, const constraint_t paramTarget, const BMK_result_t* const results) { if((paramTarget.cSpeed != 0 && results->cSpeed * uncertaintyConstantC < paramTarget.cSpeed) || (paramTarget.dSpeed != 0 && results->dSpeed * uncertaintyConstantD < paramTarget.dSpeed) || @@ -1099,12 +1076,8 @@ static int uncertainFeasibility(double const uncertaintyConstantC, double const -1 - worse assume prev_best status is run fully? but then we'd have to rerun any winners anyway */ -//presumably memory has already been compared, mostly worried about mem, cspeed, dspeed -//uncertainty only applies to speed. -//if using objective fn, this could be much easier since we could just scale that. -//difficult to make judgements about later parameters in prioritization type when there's -//uncertainty on the first. -static int uncertainComparison(double const uncertaintyConstantC, double const uncertaintyConstantD, BMK_result_t* candidate, BMK_result_t* prevBest) { +/* not as useful as initially believed */ +static int uncertainComparison(double const uncertaintyConstantC, double const uncertaintyConstantD, const BMK_result_t* candidate, const BMK_result_t* prevBest) { (void)uncertaintyConstantD; //unused for now if(candidate->cSpeed > prevBest->cSpeed * uncertaintyConstantC) { return 1; @@ -1115,34 +1088,21 @@ static int uncertainComparison(double const uncertaintyConstantC, double const u } } -/* speed in b, srcSize in b/s loopDuration in ns */ -//TODO: simplify code in feasibleBench with this instead of writing it all out. -//only applicable for single loop -static double calcUncertainty(double speed, size_t srcSize) { - U64 loopDuration; - if(eqZero(speed)) { return 2; } - loopDuration = ((srcSize * TIMELOOP_NANOSEC) / speed); - return MIN((loopDuration + (double)2 * g_clockGranularity) / loopDuration, 2); -} +/*benchmarks and tests feasibility together + 1 = true = better + 0 = false = not better + if true then resultPtr will give results. + 2+ on error? */ -//benchmarks and tests feasibility together -//1 = true = better -//0 = false = not better -//if true then resultPtr will give results. -//2+ on error? -//alt: error = 0 / infeasible as well; -//maybe use compress_only mode for ratio-finding benchmark? -//prioritize ratio > cSpeed > dSpeed > cMem -//Misnamed - should be worse, better, error -//alternative (to make work for feasible-pt searching as well) - only compare to winner, not to target -//but then we need to judge what better means in this context, which shouldn't be the same (strict ratio improvement) +//Maybe use compress_only for benchmark #define INFEASIBLE_RESULT 0 #define FEASIBLE_RESULT 1 #define ERROR_RESULT 2 static int feasibleBench(BMK_result_t* resultPtr, - const void* srcBuffer, size_t srcSize, - void* dstBuffer, size_t dstSize, - void* dictBuffer, size_t dictSize, + const void* srcBuffer, const size_t srcSize, + void* dstBuffer, const size_t dstSize, + void* dictBuffer, const size_t dictSize, + const size_t* fileSizes, const size_t nbFiles, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams, const constraint_t target, @@ -1156,8 +1116,8 @@ static int feasibleBench(BMK_result_t* resultPtr, //alternative - test 1 iter for ratio, (possibility of error 3 which is fine), //maybe iter this until 2x measurable for better guarantee? - DISPLAY("Feas:\n"); - benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + DEBUGOUTPUT("Feas:\n"); + benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres.error) { DISPLAY("ERROR %d !!\n", benchres.error); } @@ -1174,7 +1134,7 @@ static int feasibleBench(BMK_result_t* resultPtr, //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? //possibly has to do with initCCtx? or system stuff? //asymmetric +/- constant needed? - uncertaintyConstantC = MIN((loopDurationC + (double)(2 * g_clockGranularity)/loopDurationC), 2); //.02 seconds + uncertaintyConstantC = MIN((loopDurationC + (double)(2 * g_clockGranularity)/loopDurationC) * 1.1, 3); //.02 seconds } if(eqZero(benchres.result.dSpeed)) { loopDurationD = 0; @@ -1184,20 +1144,41 @@ static int feasibleBench(BMK_result_t* resultPtr, //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? //possibly has to do with initCCtx? or system stuff? //asymmetric +/- constant needed? - uncertaintyConstantD = MIN((loopDurationD + (double)(2 * g_clockGranularity)/loopDurationD), 2); //.02 seconds + uncertaintyConstantD = MIN((loopDurationD + (double)(2 * g_clockGranularity)/loopDurationD) * 1.1, 3); //.02 seconds } if(benchres.result.cSize < winnerResult->cSize) { //better compression ratio, just needs to be feasible - //optimistic assume speed - //incoporate some sort of tradeoff comparison with the winner's results? - int feas = uncertainFeasibility(uncertaintyConstantC, uncertaintyConstantD, target, &(benchres.result)); + int feas; + if(loopDurationC < TIMELOOP_NANOSEC / 10) { + BMK_return_t benchres2; + adv.mode = BMK_compressOnly; + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres = benchres2; + } + } + if(loopDurationD < TIMELOOP_NANOSEC / 10) { + BMK_return_t benchres2; + adv.mode = BMK_decodeOnly; + benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres.result.dSpeed = benchres2.result.dSpeed; + } + } + *resultPtr = benchres.result; + + feas = uncertainFeasibility(uncertaintyConstantC, uncertaintyConstantD, target, &(benchres.result)); if(feas == 0) { // uncertain feasibility adv.loopMode = BMK_timeMode; if(loopDurationC < TIMELOOP_NANOSEC) { BMK_return_t benchres2; adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1220,13 +1201,34 @@ static int feasibleBench(BMK_result_t* resultPtr, return (feas + 1) >> 1; //relies on INFEASIBLE_RESULT == 0, FEASIBLE_RESULT == 1 } } else if (benchres.result.cSize == winnerResult->cSize) { //equal ratio, needs to be better than winner in cSpeed/ dSpeed / cMem - int feas = uncertainFeasibility(uncertaintyConstantC, uncertaintyConstantD, target, &(benchres.result)); + int feas; + if(loopDurationC < TIMELOOP_NANOSEC / 10) { + BMK_return_t benchres2; + adv.mode = BMK_compressOnly; + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres = benchres2; + } + } + if(loopDurationD < TIMELOOP_NANOSEC / 10) { + BMK_return_t benchres2; + adv.mode = BMK_decodeOnly; + benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres.result.dSpeed = benchres2.result.dSpeed; + } + } + feas = uncertainFeasibility(uncertaintyConstantC, uncertaintyConstantD, target, &(benchres.result)); if(feas == 0) { // uncertain feasibility adv.loopMode = BMK_timeMode; if(loopDurationC < TIMELOOP_NANOSEC) { BMK_return_t benchres2; adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1252,7 +1254,7 @@ static int feasibleBench(BMK_result_t* resultPtr, return INFEASIBLE_RESULT; } else { //possibly better, benchmark and find out adv.loopMode = BMK_timeMode; - benchres = BMK_benchMemAdvanced(srcBuffer, srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + benchres = BMK_benchMemAdvanced(srcBuffer, srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); *resultPtr = benchres.result; return objective_lt(*winnerResult, benchres.result); } @@ -1265,16 +1267,17 @@ static int feasibleBench(BMK_result_t* resultPtr, } else { return ERROR_RESULT; //BMK error } - } -//sameas before, but +/-? + +//same as before, but +/-? //alternative, just return comparison result, leave caller to worry about feasibility. //have version of benchMemAdvanced which takes in dstBuffer/cap as well? //(motivation: repeat tests (maybe just on decompress) don't need further compress runs) static int infeasibleBench(BMK_result_t* resultPtr, - const void* srcBuffer, size_t srcSize, - void* dstBuffer, size_t dstSize, - void* dictBuffer, size_t dictSize, + const void* srcBuffer, const size_t srcSize, + void* dstBuffer, const size_t dstSize, + void* dictBuffer, const size_t dictSize, + const size_t* fileSizes, const size_t nbFiles, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams, const constraint_t target, @@ -1288,19 +1291,10 @@ static int infeasibleBench(BMK_result_t* resultPtr, adv.loopMode = BMK_iterMode; //can only use this for ratio measurement then, super inaccurate timing adv.nbSeconds = 1; //get ratio and 2x approx speed? //maybe run until twice MIN(minloopinterval * clockDuration) - DISPLAY("WinnerScore: %f\n ", winnerRS); - /* - adv.loopMode = BMK_timeMode; - adv.nbSeconds = 1; */ - benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); - BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); - benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); - BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); + DEBUGOUTPUT("WinnerScore: %f\n ", winnerRS); - adv.loopMode = BMK_timeMode; - adv.nbSeconds = 1; - benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); - BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); + benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); if(!benchres.error) { *resultPtr = benchres.result; @@ -1309,9 +1303,7 @@ static int infeasibleBench(BMK_result_t* resultPtr, uncertaintyConstantC = 2; } else { loopDurationC = ((srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); - //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? - //possibly has to do with initCCtx? or system stuff? - uncertaintyConstantC = MIN((loopDurationC + (double)(2 * g_clockGranularity)/loopDurationC), 2); //.02 seconds + uncertaintyConstantC = MIN((loopDurationC + (double)(2 * g_clockGranularity)/loopDurationC * 1.1), 3); //.02 seconds } if(eqZero(benchres.result.dSpeed)) { @@ -1319,11 +1311,32 @@ static int infeasibleBench(BMK_result_t* resultPtr, uncertaintyConstantD = 2; } else { loopDurationD = ((srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); - //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? - //possibly has to do with initCCtx? or system stuff? - uncertaintyConstantD = MIN((loopDurationD + (double)(2 * g_clockGranularity)/loopDurationD), 2); //.02 seconds + uncertaintyConstantD = MIN((loopDurationD + (double)(2 * g_clockGranularity)/loopDurationD) * 1.1 , 3); //.02 seconds } + + if(loopDurationC < TIMELOOP_NANOSEC / 10) { + BMK_return_t benchres2; + adv.mode = BMK_compressOnly; + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres = benchres2; + } + } + if(loopDurationD < TIMELOOP_NANOSEC / 10) { + BMK_return_t benchres2; + adv.mode = BMK_decodeOnly; + benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres.result.dSpeed = benchres2.result.dSpeed; + } + } + *resultPtr = benchres.result; + /* benchres's certainty range. */ resultMax = benchres.result; resultMin = benchres.result; @@ -1331,8 +1344,6 @@ static int infeasibleBench(BMK_result_t* resultPtr, resultMax.dSpeed *= uncertaintyConstantD; resultMin.cSpeed /= uncertaintyConstantC; resultMin.dSpeed /= uncertaintyConstantD; - (void)resultMin; - //TODO: consider if resultMin is actually needed. if (winnerRS > resultScore(resultMax, srcSize, target)) { return INFEASIBLE_RESULT; } else { @@ -1341,7 +1352,7 @@ static int infeasibleBench(BMK_result_t* resultPtr, if(loopDurationC < TIMELOOP_NANOSEC) { BMK_return_t benchres2; adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1350,9 +1361,7 @@ static int infeasibleBench(BMK_result_t* resultPtr, } if(loopDurationD < TIMELOOP_NANOSEC) { BMK_return_t benchres2; - adv.mode = BMK_decodeOnly; - //TODO: dstBuffer corrupted sometime between top and now - //probably occuring in feasible bench too. + adv.mode = BMK_decodeOnly; benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; @@ -1372,28 +1381,27 @@ static int infeasibleBench(BMK_result_t* resultPtr, } /* wrap feasibleBench w/ memotable */ -//TODO: void sanitized and unsanitized ver's so input doesn't double-choose #define INFEASIBLE_THRESHOLD 200 static int feasibleBenchMemo(BMK_result_t* resultPtr, - const void* srcBuffer, size_t srcSize, - void* dstBuffer, size_t dstSize, - void* dictBuffer, size_t dictSize, + const void* srcBuffer, const size_t srcSize, + void* dstBuffer, const size_t dstSize, + void* dictBuffer, const size_t dictSize, + const size_t* fileSizes, const size_t nbFiles, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams, const constraint_t target, BMK_result_t* winnerResult, U8* memoTable, - U32* varyParams, const int varyLen) { + const U32* varyParams, const int varyLen) { - size_t memind = memoTableInd(&cParams, varyParams, varyLen); + const size_t memind = memoTableInd(&cParams, varyParams, varyLen); - //BMK_translateAdvancedParams(cParams); if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { return INFEASIBLE_RESULT; //probably pick a different code for already tested? //maybe remove this if we incorporate nonrandom location picking? //what is the intended behavior in this case? //ignore? stop iterating completely? other? } else { - int res = feasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, ctx, dctx, + int res = feasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, fileSizes, nbFiles, ctx, dctx, cParams, target, winnerResult); memoTable[memind] = 255; //tested are all infeasible (other possible values for opti) return res; @@ -1403,21 +1411,21 @@ static int feasibleBenchMemo(BMK_result_t* resultPtr, //should infeasible stage searching also be memo-marked in the same way? //don't actually memoize unless result is feasible/error? static int infeasibleBenchMemo(BMK_result_t* resultPtr, - const void* srcBuffer, size_t srcSize, - void* dstBuffer, size_t dstSize, - void* dictBuffer, size_t dictSize, + const void* srcBuffer, const size_t srcSize, + void* dstBuffer, const size_t dstSize, + void* dictBuffer, const size_t dictSize, + const size_t* fileSizes, const size_t nbFiles, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams, const constraint_t target, BMK_result_t* winnerResult, U8* memoTable, - U32* varyParams, const int varyLen) { + const U32* varyParams, const int varyLen) { size_t memind = memoTableInd(&cParams, varyParams, varyLen); - //BMK_translateAdvancedParams(cParams); if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { return INFEASIBLE_RESULT; //see feasibleBenchMemo for concerns } else { - int res = infeasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, ctx, dctx, + int res = infeasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, fileSizes, nbFiles, ctx, dctx, cParams, target, winnerResult); if(res == FEASIBLE_RESULT) { memoTable[memind] = 255; //infeasible resultscores could still be normal feasible. @@ -1432,14 +1440,18 @@ typedef int (*BMK_benchMemo_t)(BMK_result_t*, const void*, size_t, void*, size_t const ZSTD_compressionParameters, const constraint_t, BMK_result_t*, U8*, U32*, const int); //varArray should be sanitized when this is called. -//TODO: transition to simpler greedy method if evaluation time is too long? -//would it be better to start at best feasible via feasible or infeasible metric? both? //possibility climb is infeasible, responsibility of caller to check that. but if something feasible is evaluated, it will be returned // *actually if it performs too //sanitize all params here. //all generation after random should be sanitized. (maybe sanitize random) -static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varLen, U8* memoTable, - const void* srcBuffer, size_t srcSize, void* dstBuffer, size_t dstSize, void* dictBuffer, size_t dictSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, ZSTD_compressionParameters init) { +static winnerInfo_t climbOnce(const constraint_t target, + const U32* varArray, const int varLen, U8* memoTable, + const void* srcBuffer, size_t srcSize, + void* dstBuffer, const size_t dstSize, + void* dictBuffer, const size_t dictSize, + const size_t* fileSizes, const size_t nbFiles, + ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, + const ZSTD_compressionParameters init) { //pick later initializations non-randomly? high dist from explored nodes. //how to do this efficiently? (might not be too much of a problem, happens rarely, running time probably dominated by benchmarking) //distance maximizing selection? @@ -1459,28 +1471,20 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL /* ineasible -> (hopefully) feasible */ /* when nothing is found, this garbages part 2. */ { - //TODO: initialize these values! winnerInfo_t bestFeasible1; /* uses feasibleBench Metric */ - winnerInfo_t bestFeasible2; /* uses resultScore Metric */ //init these params bestFeasible1.params = cparam; - bestFeasible2.params = cparam; bestFeasible1.result.cSpeed = 0; bestFeasible1.result.dSpeed = 0; bestFeasible1.result.cMem = (size_t)-1; bestFeasible1.result.cSize = (size_t)-1; - bestFeasible2.result.cSpeed = 0; - bestFeasible2.result.dSpeed = 0; - bestFeasible2.result.cMem = (size_t)-1; - bestFeasible2.result.cSize = (size_t)-1; DISPLAY("Climb Part 1\n"); while(better) { - //UTIL_time_t timestart = UTIL_getTime(); TODO: adjust sampling based on time int i, d; better = 0; - DISPLAY("Start\n"); + DEBUGOUTPUT("Start\n"); cparam = winnerInfo.params; BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); candidateInfo.params = cparam; @@ -1496,18 +1500,16 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, + fileSizes, nbFiles, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); if(res == FEASIBLE_RESULT) { /* synonymous with better when called w/ infeasibleBM */ winnerInfo = candidateInfo; - //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); better = 1; - if(feasible(candidateInfo.result, target)) { - bestFeasible2 = winnerInfo; - if(objective_lt(bestFeasible1.result, bestFeasible2.result)) { - bestFeasible1 = bestFeasible2; /* using feasibleBench metric */ - } + if(feasible(candidateInfo.result, target) && objective_lt(bestFeasible1.result, winnerInfo.result)) { + bestFeasible1 = winnerInfo; } } } @@ -1521,18 +1523,16 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, + fileSizes, nbFiles, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); if(res == FEASIBLE_RESULT) { winnerInfo = candidateInfo; - //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); better = 1; - if(feasible(candidateInfo.result, target)) { //TODO: maybe just break here and move on to part 2? - bestFeasible2 = winnerInfo; - if(objective_lt(bestFeasible1.result, bestFeasible2.result)) { - bestFeasible1 = bestFeasible2; - } + if(feasible(candidateInfo.result, target) && objective_lt(bestFeasible1.result, winnerInfo.result)) { + bestFeasible1 = winnerInfo; } } } @@ -1553,18 +1553,16 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, + fileSizes, nbFiles, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); if(res == FEASIBLE_RESULT) { /* synonymous with better in this case*/ winnerInfo = candidateInfo; - //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); better = 1; - if(feasible(candidateInfo.result, target)) { - bestFeasible2 = winnerInfo; - if(objective_lt(bestFeasible1.result, bestFeasible2.result)) { - bestFeasible1 = bestFeasible2; - } + if(feasible(candidateInfo.result, target) && objective_lt(bestFeasible1.result, winnerInfo.result)) { + bestFeasible1 = winnerInfo; } } @@ -1576,15 +1574,12 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL //bias to test previous delta? //change cparam -> candidate before restart } - //TODO:Consider if this is best config. idea: explore from obj best keep rbest - cparam = bestFeasible2.params; - candidateInfo = bestFeasible2; winnerInfo = bestFeasible1; } - //is it better to break here instead of bumbling about? + //break out if no feasible. if(winnerInfo.result.cMem == (U32)-1) { - DISPLAY("No Feasible Found\n"); + DEBUGOUTPUT("No Feasible Found\n"); return winnerInfo; } DISPLAY("Climb Part 2\n"); @@ -1592,14 +1587,12 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL better = 1; /* feasible -> best feasible (hopefully) */ { - while(better) { - - //UTIL_time_t timestart = UTIL_getTime(); //TODO: if benchmarking is taking too long, be more greedy. + while(better) { int i, d; better = 0; BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); //all dist-1 targets - cparam = winnerInfo.params; //TODO: this messes the taking bestFeasible1, bestFeasible2 + cparam = winnerInfo.params; candidateInfo.params = cparam; for(i = 0; i < varLen; i++) { paramVaryOnce(varArray[i], 1, &candidateInfo.params); @@ -1612,12 +1605,13 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, + fileSizes, nbFiles, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); if(res == FEASIBLE_RESULT) { winnerInfo = candidateInfo; - //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); better = 1; } } @@ -1626,17 +1620,17 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL candidateInfo.params = sanitizeParams(candidateInfo.params); //evaluate if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { - //if(cParamValid(candidateInfo.params)) { int res = feasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, + fileSizes, nbFiles, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); if(res == FEASIBLE_RESULT) { winnerInfo = candidateInfo; - //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); better = 1; } } @@ -1653,12 +1647,13 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, + fileSizes, nbFiles, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); if(res == FEASIBLE_RESULT) { winnerInfo = candidateInfo; - //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); better = 1; } } @@ -1684,29 +1679,26 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL //only real use for paramTarget is to get the fixed values, right? static winnerInfo_t optimizeFixedStrategy( const void* srcBuffer, const size_t srcSize, - void* dstBuffer, size_t dstSize, - void* dictBuffer, size_t dictSize, - constraint_t target, ZSTD_compressionParameters paramTarget, - ZSTD_strategy strat, U32* varArray, int varLen) { - int i = 0; //TODO: Temp fix 10 iters, check effects of changing this? + void* dstBuffer, const size_t dstSize, + void* dictBuffer, const size_t dictSize, + const size_t* fileSizes, const size_t nbFiles, + const constraint_t target, ZSTD_compressionParameters paramTarget, + const ZSTD_strategy strat, const U32* varArray, const int varLen, U8* memoTable) { + int i = 0; U32* varNew = malloc(sizeof(U32) * varLen); int varLenNew = sanitizeVarArray(varLen, varArray, varNew, strat); - size_t memoLen = memoTableLen(varNew, varLenNew); - U8* memoTable = malloc(sizeof(U8) * memoLen); ZSTD_compressionParameters init; ZSTD_CCtx* ctx = ZSTD_createCCtx(); ZSTD_DCtx* dctx = ZSTD_createDCtx(); winnerInfo_t winnerInfo, candidateInfo; winnerInfo.result.cSpeed = 0; winnerInfo.result.dSpeed = 0; - winnerInfo.result.cMem = (size_t)(-1); - winnerInfo.result.cSize = (size_t)(-1); + winnerInfo.result.cMem = (size_t)(-1LL); + winnerInfo.result.cSize = (size_t)(-1LL); /* so climb is given the right fixed strategy */ paramTarget.strategy = strat; /* to pass ZSTD_checkCParams */ - memoTableInit(memoTable, paramTarget, target, varNew, varLenNew, srcSize); - //needs to happen after memoTableInit as that assumes 0 = undefined. cParamZeroMin(¶mTarget); @@ -1718,11 +1710,11 @@ static winnerInfo_t optimizeFixedStrategy( goto _cleanUp; } - while(i < 10) { - DISPLAY("Restart\n"); - //TODO: look into improving this to maximize distance from searched infeasible stuff / towards promising regions? + while(i < 10) { //make i adjustable (user input?) depending on how much time they have. + DISPLAY("Restart\n"); //TODO: make better printing across restarts + //look into improving this to maximize distance from searched infeasible stuff / towards promising regions? randomConstrainedParams(&init, varNew, varLenNew, memoTable); - candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, ctx, dctx, init); + candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, fileSizes, nbFiles, ctx, dctx, init); if(objective_lt(winnerInfo.result, candidateInfo.result)) { winnerInfo = candidateInfo; DISPLAY("New Winner: "); @@ -1735,7 +1727,6 @@ static winnerInfo_t optimizeFixedStrategy( _cleanUp: ZSTD_freeCCtx(ctx); ZSTD_freeDCtx(dctx); - free(memoTable); free(varNew); return winnerInfo; } @@ -1777,20 +1768,54 @@ static int BMK_loadFiles(void* buffer, size_t bufferSize, fclose(f); } - if (totalSize == 0) { DISPLAY("no data to bench\n"); return 12; } + if (totalSize == 0) { DISPLAY("\nno data to bench\n"); return 12; } return 0; } -// bigger and (hopefully) better* than optimizeForSize -// TODO: allow accept multiple files like benchFiles or bench.c fn's -static int optimizeForSize2(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget) +//goes best, best-1, best+1, best-2, ... +//return 0 if nothing remaining +static int nextStrategy(const int currentStrategy, const int bestStrategy) { + if(bestStrategy <= currentStrategy) { + int candidate = 2 * bestStrategy - currentStrategy - 1; + if(candidate < 1) { + candidate = currentStrategy + 1; + if(candidate > (int)ZSTD_btultra) { + return 0; + } else { + return candidate; + } + } else { + return candidate; + } + } else { /* bestStrategy >= currentStrategy */ + int candidate = 2 * bestStrategy - currentStrategy; + if(candidate > (int)ZSTD_btultra) { + candidate = currentStrategy - 1; + if(candidate < 1) { + return 0; + } else { + return candidate; + } + } else { + return candidate; + } + } +} + +//optimize fixed strategy. +static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget) { size_t benchedSize; - void* origBuff; - void* dictBuffer; - size_t dictBufferSize; + void* origBuff = NULL; + void* dictBuffer = NULL; + size_t dictBufferSize = 0; U32 varArray [NUM_PARAMS]; - int varLen = variableParams(paramTarget, varArray); + int ret = 0; + size_t* fileSizes = calloc(sizeof(size_t),nbFiles); + const int varLen = variableParams(paramTarget, varArray); + U8** allMT = NULL; + g_targetConstraints = target; + g_winner.result.cSize = (size_t)-1; /* Init */ if(!cParamValid(paramTarget)) { return 10; @@ -1801,20 +1826,24 @@ static int optimizeForSize2(const char* const * const fileNamesTable, const size U64 const dictFileSize = UTIL_getFileSize(dictFileName); if (dictFileSize > 64 MB) { DISPLAY("dictionary file %s too large", dictFileName); - return 10; + ret = 10; + goto _cleanUp; } dictBufferSize = (size_t)dictFileSize; dictBuffer = malloc(dictBufferSize); if (dictBuffer==NULL) { DISPLAY("not enough memory for dictionary (%u bytes)", (U32)dictBufferSize); - return 11; + ret = 11; + goto _cleanUp; + } + { int errorCode = BMK_loadFiles(dictBuffer, dictBufferSize, &dictBufferSize, &dictFileName, 1); if(errorCode) { - free(dictBuffer); - return errorCode; + ret = errorCode; + goto _cleanUp; } } } @@ -1823,41 +1852,46 @@ static int optimizeForSize2(const char* const * const fileNamesTable, const size if(nbFiles == 1) { DISPLAY("Loading %s... \r", fileNamesTable[0]); } else { - DISPLAY("Loading %zd Files... \r", nbFiles); + DISPLAY("Loading %lu Files... \r", (unsigned long)nbFiles); } { U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles); int ec; - size_t* fileSizes = calloc(sizeof(size_t),nbFiles); + unsigned i; benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3; + origBuff = malloc(benchedSize); if(!origBuff || !fileSizes) { DISPLAY("Not enough memory for stuff\n"); - free(origBuff); - free(fileSizes); - free(dictBuffer); - return 1; + ret = 1; + goto _cleanUp; } ec = BMK_loadFiles(origBuff, benchedSize, fileSizes, fileNamesTable, nbFiles); if(ec) { DISPLAY("Error Loading Files"); - free(origBuff); - free(fileSizes); - free(dictBuffer); - return ec; + ret = ec; + goto _cleanUp; } - free(fileSizes); + benchedSize = 0; + for(i = 0; i < nbFiles; i++) { + benchedSize += fileSizes[i]; + } + origBuff = realloc(origBuff, benchedSize); } - + allMT = memoTableInitAll(paramTarget, target, varArray, varLen, benchedSize); + if(!allMT) { + ret = 2; + goto _cleanUp; + } /* bench */ DISPLAY("\r%79s\r", ""); if(nbFiles == 1) { DISPLAY("optimizing for %s", fileNamesTable[0]); } else { - DISPLAY("optimizing for %zd Files", nbFiles); + DISPLAY("optimizing for %lu Files", (unsigned long)nbFiles); } if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed / 1000000); } if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed / 1000000); } @@ -1868,7 +1902,7 @@ static int optimizeForSize2(const char* const * const fileNamesTable, const size { ZSTD_CCtx* const ctx = ZSTD_createCCtx(); ZSTD_DCtx* const dctx = ZSTD_createDCtx(); winnerInfo_t winner; - //BMK_result_t candidate; + U32 varNew[NUM_PARAMS]; const size_t blockSize = g_blockSize ? g_blockSize : benchedSize; U32 const maxNbBlocks = (U32) ((benchedSize + (blockSize-1)) / blockSize) + 1; const size_t maxCompressedSize = ZSTD_compressBound(benchedSize) + (maxNbBlocks * 1024); @@ -1877,49 +1911,120 @@ static int optimizeForSize2(const char* const * const fileNamesTable, const size /* init */ if (ctx==NULL) { DISPLAY("\n ZSTD_createCCtx error \n"); free(origBuff); return 14;} if(compressedBuffer==NULL) { DISPLAY("\n Allocation Error \n"); free(origBuff); free(ctx); return 15; } - memset(&winner, 0, sizeof(winner)); winner.result.cSize = (size_t)(-1); /* find best solution from default params */ - //Can't do this w/ cparameter constraints - //still useful though? - /* - { const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); - int i; - for (i=1; i<=maxSeeds; i++) { - ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, 0); - BMK_benchParam(&candidate, origBuff, benchedSize, ctx, dctx, CParams); - if (!feasible(candidate, target) ) { - break; + { + /* strategy selection */ + //TODO: don't factor memory into strategy selection in constraint_t + const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); + DEBUGOUTPUT("Strategy Selection\n"); + if(varLen == NUM_PARAMS && paramTarget.strategy == 0) { /* no variable based constraints */ + BMK_result_t candidate; + int feas = 0, i; + for (i=1; i<=maxSeeds; i++) { + ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, 0); + int ec = BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, nbFiles, ctx, dctx, CParams); + BMK_printWinner(stdout, i, candidate, CParams, benchedSize); + + if(!ec) { + if(feas) { + if(feasible(candidate, relaxTarget(target)) && objective_lt(winner.result, candidate)) { + winner.result = candidate; + winner.params = CParams; + } + } else { + if(feasible(candidate, relaxTarget(target))) { + feas = 1; + winner.result = candidate; + winner.params = CParams; + + } else { + if(resultScore(candidate, benchedSize, target) > resultScore(winner.result, benchedSize, target)) { + winner.result = candidate; + winner.params = CParams; + } + } + } + } + } //best, -1, +1, ..., + + } else if (paramTarget.strategy == 0) { //constrained + int feas = 0, i, j; + for(j = 1; j < 10; j++) { + for(i = 1; i <= maxSeeds; i++) { + int varLenNew = sanitizeVarArray(varLen, varArray, varNew, i); + ZSTD_compressionParameters candidateParams = paramTarget; + BMK_result_t candidate; + int ec; + randomConstrainedParams(&candidateParams, varNew, varLenNew, allMT[i]); + cParamZeroMin(&candidateParams); + candidateParams = sanitizeParams(candidateParams); + ec = BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, nbFiles, ctx, dctx, candidateParams); + + if(!ec) { + if(feas) { + if(feasible(candidate, relaxTarget(target)) && objective_lt(winner.result, candidate)) { + winner.result = candidate; + winner.params = candidateParams; + BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); + } + } else { + if(feasible(candidate, relaxTarget(target))) { + feas = 1; + winner.result = candidate; + winner.params = candidateParams; + BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); + + } else { + if(resultScore(candidate, benchedSize, target) > resultScore(winner.result, benchedSize, target)) { + winner.result = candidate; + winner.params = candidateParams; + BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); + } + } + } + } + + } } - if (feasible(candidate,target) && objective_lt(winner.result, candidate)) - { - winner.params = CParams; - winner.result = candidate; - BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); - } } - }*/ + } + } + BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize); - BMK_translateAdvancedParams(winner.params); - - /* start real tests */ + DEBUGOUTPUT("Real Opt\n"); + /* start 'real' tests */ { + int bestStrategy = (int)winner.params.strategy; if(paramTarget.strategy == 0) { - int st; - for(st = 1; st <= 8; st++) { - winnerInfo_t wc = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, dictBuffer, dictBufferSize, - target, paramTarget, st, varArray, varLen); - DISPLAY("StratNum %d\n", st); + int st = (int)winner.params.strategy; + + { + int varLenNew = sanitizeVarArray(varLen, varArray, varNew, st); + winnerInfo_t w1 = climbOnce(target, varNew, varLenNew, allMT[st], + origBuff, benchedSize, compressedBuffer, maxCompressedSize, dictBuffer, dictBufferSize, + fileSizes, nbFiles, ctx, dctx, winner.params); + if(objective_lt(winner.result, w1.result)) { + winner = w1; + } + } + + while(st) { + winnerInfo_t wc = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, dictBuffer, dictBufferSize, fileSizes, nbFiles, + target, paramTarget, st, varArray, varLen, allMT[st]); + DEBUGOUTPUT("StratNum %d\n", st); if(objective_lt(winner.result, wc.result)) { winner = wc; } + //TODO: we could double back to increase search of 'better' strategies + st = nextStrategy(st, bestStrategy); } } else { - winner = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, dictBuffer, dictBufferSize, - target, paramTarget, paramTarget.strategy, varArray, varLen); + winner = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, dictBuffer, dictBufferSize, fileSizes, nbFiles, + target, paramTarget, paramTarget.strategy, varArray, varLen, allMT[paramTarget.strategy]); } } @@ -1934,150 +2039,18 @@ static int optimizeForSize2(const char* const * const fileNamesTable, const size BMK_translateAdvancedParams(winner.params); DISPLAY("grillParams size - optimizer completed \n"); - /* clean up*/ - ZSTD_freeCCtx(ctx); - ZSTD_freeDCtx(dctx); - } - - free(origBuff); - return 0; -} - - -/* optimizeForSize(): - * targetSpeed : expressed in B/s */ -/* expresses targeted compression, decompression speeds and memory requirements */ -/* if state space is small (from paramTarget), exhaustive search? */ -//things to consider : if doing strategy-separate approach, what cutoffs to evaluate each strategy -//or do all? can't be absolute, should be relative after some sort of calibration -//(synthetic? test levels (we don't care about data specifics rn, scale?) ? -int optimizeForSize(const char* inFileName, constraint_t target, ZSTD_compressionParameters paramTarget) -{ - FILE* const inFile = fopen( inFileName, "rb" ); - U64 const inFileSize = UTIL_getFileSize(inFileName); - size_t benchedSize = BMK_findMaxMem(inFileSize*3) / 3; - void* origBuff; - U32 paramVarArray [NUM_PARAMS]; - int paramCount = variableParams(paramTarget, paramVarArray); - /* Init */ - if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; } - if (inFileSize == UTIL_FILESIZE_UNKNOWN) { - DISPLAY("Pb evaluatin size of %s \n", inFileName); - fclose(inFile); - return 11; - } - - /* Memory allocation & restrictions */ - if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize; - if (benchedSize < inFileSize) { - DISPLAY("Not enough memory for '%s' \n", inFileName); - fclose(inFile); - return 11; - } - - /* Alloc */ - origBuff = malloc(benchedSize); - if(!origBuff) { - DISPLAY("\nError: not enough memory!\n"); - fclose(inFile); - return 12; - } - - /* Fill input buffer */ - DISPLAY("Loading %s... \r", inFileName); - { size_t const readSize = fread(origBuff, 1, benchedSize, inFile); - fclose(inFile); - if(readSize != benchedSize) { - DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); - free(origBuff); - return 13; - } } - - /* bench */ - DISPLAY("\r%79s\r", ""); - DISPLAY("optimizing for %s", inFileName); - if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed / 1000000); } - if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed / 1000000); } - if(target.cMem != 0) { DISPLAY(" - limit memory %u MB", target.cMem / 1000000); } - DISPLAY("\n"); - { ZSTD_CCtx* const ctx = ZSTD_createCCtx(); - ZSTD_DCtx* const dctx = ZSTD_createDCtx(); - winnerInfo_t winner; - BMK_result_t candidate; - const size_t blockSize = g_blockSize ? g_blockSize : benchedSize; - - /* init */ - if (ctx==NULL) { DISPLAY("\n ZSTD_createCCtx error \n"); free(origBuff); return 14; } - - memset(&winner, 0, sizeof(winner)); - winner.result.cSize = (size_t)(-1); - - /* find best solution from default params */ - //Can't do this w/ cparameter constraints - { const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); - int i; - for (i=1; i<=maxSeeds; i++) { - ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, 0); - BMK_benchParam(&candidate, origBuff, benchedSize, ctx, dctx, CParams); - if (!feasible(candidate, target) ) { - break; - } - if (feasible(candidate,target) && objective_lt(winner.result, candidate)) - { - winner.params = CParams; - winner.result = candidate; - BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); - } } - } - BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize); - - BMK_translateAdvancedParams(winner.params); - - /* start tests */ - { time_t const grillStart = time(NULL); - do { - ZSTD_compressionParameters params = winner.params; - BYTE* b; - paramVariation(¶ms, paramVarArray, paramCount, 4); - if ((FUZ_rand(&g_rand) & 31) == 3) params = randomParams(); /* totally random config to improve search space */ - params = ZSTD_adjustCParams(params, blockSize, 0); - - /* exclude faster if already played set of params */ - if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(params))-1)) continue; - - /* test */ - b = NB_TESTS_PLAYED(params); - (*b)++; - BMK_benchParam(&candidate, origBuff, benchedSize, ctx, dctx, params); - - /* improvement found => new winner */ - if (feasible(candidate,target) && objective_lt(winner.result, candidate)) - { - winner.params = params; - winner.result = candidate; - BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize); - BMK_translateAdvancedParams(winner.params); - } - } while (BMK_timeSpan(grillStart) < g_grillDuration_s); - } - - /* no solution found */ - if(winner.result.cSize == (size_t)-1) { - DISPLAY("No feasible solution found\n"); - return 1; - } - /* end summary */ - BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize); - BMK_translateAdvancedParams(winner.params); - DISPLAY("grillParams size - optimizer completed \n"); /* clean up*/ ZSTD_freeCCtx(ctx); ZSTD_freeDCtx(dctx); - } + } +_cleanUp: + free(fileSizes); + free(dictBuffer); + memoTableFreeAll(allMT); free(origBuff); - return 0; + return ret; } static void errorOut(const char* msg) @@ -2136,6 +2109,7 @@ static int usage_advanced(void) DISPLAY( " -P# : generated sample compressibility (default : %.1f%%) \n", COMPRESSIBILITY_DEFAULT * 100); DISPLAY( " -t# : Caps runtime of operation in seconds (default : %u seconds (%.1f hours)) \n", (U32)g_grillDuration_s, g_grillDuration_s / 3600); DISPLAY( " -v : Prints Benchmarking output\n"); + DISPLAY( " -D : Next argument dictionary file\n"); return 0; } @@ -2163,6 +2137,8 @@ int main(int argc, const char** argv) assert(argc>=1); /* for exename */ + g_time = UTIL_getTime(); + /* Welcome message */ DISPLAY(WELCOME_MESSAGE); @@ -2397,7 +2373,7 @@ int main(int argc, const char** argv) } } else { if (optimizer) { - result = optimizeForSize2(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget); + result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget); } else { result = benchFiles(argv+filenamesStart, argc-filenamesStart); } } From 7b5b3d7ae383f4efe53a95d68a4a1f1760f78650 Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 16 Jul 2018 16:16:31 -0700 Subject: [PATCH 06/19] BenchMem with block compressed sizes passed back up --- programs/bench.c | 31 +-- programs/bench.h | 13 +- tests/paramgrill.c | 625 +++++++++++++++++++++++++++++++++++++-------- 3 files changed, 540 insertions(+), 129 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index f5184fa8f..8e57db4c9 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -188,7 +188,7 @@ static void BMK_initCCtx(ZSTD_CCtx* ctx, ZSTD_CCtx_setParameter(ctx, ZSTD_p_searchLog, comprParams->searchLog); ZSTD_CCtx_setParameter(ctx, ZSTD_p_minMatch, comprParams->searchLength); ZSTD_CCtx_setParameter(ctx, ZSTD_p_targetLength, comprParams->targetLength); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy , comprParams->strategy); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy, comprParams->strategy); ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize); } @@ -281,8 +281,6 @@ static size_t local_defaultDecompress( } -volatile char g_touched; - /* initFn will be measured once, bench fn will be measured x times */ /* benchFn should return error value or out Size */ /* takes # of blocks and list of size & stuff for each. */ @@ -293,7 +291,7 @@ BMK_customReturn_t BMK_benchFunction( BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBlockBuffers, const size_t* srcBlockSizes, - void** const dstBlockBuffers, size_t* dstBlockCapacities, + void* const * const dstBlockBuffers, size_t* dstBlockCapacitiesToSizes, unsigned nbLoops) { size_t srcSize = 0, dstSize = 0, ind = 0; U64 totalTime; @@ -310,20 +308,13 @@ BMK_customReturn_t BMK_benchFunction( } { - - unsigned i, j; + size_t i; for(i = 0; i < blockCount; i++) { - for(j = 0; j < srcBlockSizes[i]; j++) { - g_touched = ((const char*)srcBlockBuffers[i])[j]; /* touch */ - } - } - for(i = 0; i < blockCount; i++) { - memset(dstBlockBuffers[i], 0xE5, dstBlockCapacities[i]); /* warm up and erase result buffer */ + memset(dstBlockBuffers[i], 0xE5, dstBlockCapacitiesToSizes[i]); /* warm up and erase result buffer */ } //UTIL_sleepMilli(5); /* give processor time to other processes */ //UTIL_waitForNextTick(); - } { @@ -332,17 +323,13 @@ BMK_customReturn_t BMK_benchFunction( if(initFn != NULL) { initFn(initPayload); } for(i = 0; i < nbLoops; i++) { for(j = 0; j < blockCount; j++) { - size_t res = benchFn(srcBlockBuffers[j], srcBlockSizes[j], dstBlockBuffers[j], dstBlockCapacities[j], benchPayload); + size_t res = benchFn(srcBlockBuffers[j], srcBlockSizes[j], dstBlockBuffers[j], dstBlockCapacitiesToSizes[j], benchPayload); if(ZSTD_isError(res)) { EXM_THROW_ND(2, BMK_customReturn_t, "Function benchmarking failed on block %u of size %u : %s \n", - j, (U32)dstBlockCapacities[j], ZSTD_getErrorName(res)); + j, (U32)dstBlockCapacitiesToSizes[j], ZSTD_getErrorName(res)); } else if(firstIter) { dstSize += res; - //Make compressed blocks continuous - if(j != blockCount - 1) { - dstBlockBuffers[j+1] = (void*)((char*)dstBlockBuffers[j] + res); - dstBlockCapacities[j] = res; - } + dstBlockCapacitiesToSizes[j] = res; } } firstIter = 0; @@ -382,7 +369,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed( BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const* const srcBlockBuffers, const size_t* srcBlockSizes, - void** const dstBlockBuffers, size_t* dstBlockCapacities) + void * const * const dstBlockBuffers, size_t * dstBlockCapacitiesToSizes) { U64 fastest = cont->fastestTime; int completed = 0; @@ -399,7 +386,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed( } r.result = BMK_benchFunction(benchFn, benchPayload, initFn, initPayload, - blockCount, srcBlockBuffers, srcBlockSizes, dstBlockBuffers, dstBlockCapacities, cont->nbLoops); + blockCount, srcBlockBuffers, srcBlockSizes, dstBlockBuffers, dstBlockCapacitiesToSizes, cont->nbLoops); if(r.result.error) { /* completed w/ error */ r.completed = 1; return r; diff --git a/programs/bench.h b/programs/bench.h index 1a298cc48..25d9f24a7 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -122,8 +122,6 @@ BMK_return_t BMK_syntheticTest(int cLevel, double compressibility, * (cLevel, comprParams + adv in advanced Mode) */ /* srcBuffer - data source, expected to be valid compressed data if in Decode Only Mode * srcSize - size of data in srcBuffer - * dstBuffer - destination buffer to write compressed output in, optional (NULL) - * dstCapacity - capacity of destination buffer, give 0 if dstBuffer = NULL * cLevel - compression level * comprParams - basic compression parameters * dictBuffer - a dictionary if used, null otherwise @@ -144,7 +142,10 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, int displayLevel, const char* displayName); -/* See benchMem for normal parameter uses and return, see advancedParams_t for adv */ +/* See benchMem for normal parameter uses and return, see advancedParams_t for adv + * dstBuffer - destination buffer to write compressed output in, NULL if none provided. + * dstCapacity - capacity of destination buffer, give 0 if dstBuffer = NULL + */ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, void* dstBuffer, size_t dstCapacity, const size_t* fileSizes, unsigned nbFiles, @@ -174,7 +175,7 @@ typedef size_t (*BMK_initFn_t)(void*); * srcBuffers - an array of buffers to be operated on by benchFn * srcSizes - an array of the sizes of above buffers * dstBuffers - an array of buffers to be written into by benchFn - * dstCapacities - an array of the capacities of above buffers. + * dstCapacitiesToSizes - an array of the capacities of above buffers. Output modified to compressed sizes of those blocks. * nbLoops - defines number of times benchFn is run. * return * .error will give a nonzero value if ZSTD_isError() is nonzero for any of the return @@ -191,7 +192,7 @@ BMK_customReturn_t BMK_benchFunction( BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBuffers, const size_t* srcSizes, - void** const dstBuffers, size_t* dstCapacities, + void * const * const dstBuffers, size_t* dstCapacitiesToSizes, unsigned nbLoops); @@ -220,7 +221,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed(BMK_timedFnState_t* cont, BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBlockBuffers, const size_t* srcBlockSizes, - void** const dstBlockBuffers, size_t* dstBlockCapacities); + void* const * const dstBlockBuffers, size_t* dstBlockCapacitiesToSizes); #endif /* BENCH_H_121279284357 */ diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 9af60a740..e4d468846 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -27,7 +27,8 @@ #include "xxhash.h" #include "util.h" #include "bench.h" - +#include "zstd_errors.h" +#include "zstd_internal.h" /*-************************************ * Constants @@ -36,11 +37,6 @@ #define AUTHOR "Yann Collet" #define WELCOME_MESSAGE "*** %s %s %i-bits, by %s ***\n", PROGRAM_DESCRIPTION, ZSTD_VERSION_STRING, (int)(sizeof(void*)*8), AUTHOR - -#define KB *(1<<10) -#define MB *(1<<20) -#define GB *(1ULL<<30) - #define TIMELOOP_NANOSEC (1*1000000000ULL) /* 1 second */ #define NBLOOPS 2 @@ -240,7 +236,6 @@ static int cParamValid(ZSTD_compressionParameters paramTarget) { return 1; } -//TODO: let targetLength = 0; static void cParamZeroMin(ZSTD_compressionParameters* paramTarget) { paramTarget->windowLog = paramTarget->windowLog ? paramTarget->windowLog : ZSTD_WINDOWLOG_MIN; paramTarget->searchLog = paramTarget->searchLog ? paramTarget->searchLog : ZSTD_SEARCHLOG_MIN; @@ -261,7 +256,7 @@ static int feasible(const BMK_result_t results, const constraint_t target) { return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem || !target.cMem); } -#define EPSILON 0.01 +#define EPSILON 0.001 static int epsilonEqual(const double c1, const double c2) { return MAX(c1/c2,c2/c1) < 1 + EPSILON; } @@ -274,8 +269,8 @@ static int eqZero(const double c1) { /* returns 1 if result2 is strictly 'better' than result1 */ /* strict comparison / cutoff based */ static int objective_lt(const BMK_result_t result1, const BMK_result_t result2) { - return (result1.cSize > result2.cSize) || (epsilonEqual(result1.cSize, result2.cSize) && result2.cSpeed > result1.cSpeed) - || (epsilonEqual(result1.cSize,result2.cSize) && epsilonEqual(result2.cSpeed, result1.cSpeed) && result2.dSpeed > result1.dSpeed); + return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed) + || (result1.cSize == result2.cSize && epsilonEqual(result2.cSpeed, result1.cSpeed) && result2.dSpeed > result1.dSpeed); } /* hill climbing value for part 1 */ @@ -352,9 +347,362 @@ typedef struct { ZSTD_compressionParameters params; } winnerInfo_t; +/*-******************************************************* +* From Paramgrill +*********************************************************/ + +static void BMK_initCCtx(ZSTD_CCtx* ctx, + const void* dictBuffer, size_t dictBufferSize, int cLevel, + const ZSTD_compressionParameters* comprParams, const BMK_advancedParams_t* adv) { + if (adv->nbWorkers==1) { + ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, 0); + } else { + ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, adv->nbWorkers); + } + ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_enableLongDistanceMatching, adv->ldmFlag); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmMinMatch, adv->ldmMinMatch); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashLog, adv->ldmHashLog); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmBucketSizeLog, adv->ldmBucketSizeLog); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashEveryLog, adv->ldmHashEveryLog); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_windowLog, comprParams->windowLog); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_hashLog, comprParams->hashLog); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_chainLog, comprParams->chainLog); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_searchLog, comprParams->searchLog); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_minMatch, comprParams->searchLength); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_targetLength, comprParams->targetLength); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy, comprParams->strategy); + ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize); +} + + +static void BMK_initDCtx(ZSTD_DCtx* dctx, + const void* dictBuffer, size_t dictBufferSize) { + ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize); +} + +typedef struct { + ZSTD_CCtx* ctx; + const void* dictBuffer; + size_t dictBufferSize; + int cLevel; + const ZSTD_compressionParameters* comprParams; + const BMK_advancedParams_t* adv; +} BMK_initCCtxArgs; + +static size_t local_initCCtx(void* payload) { + BMK_initCCtxArgs* ag = (BMK_initCCtxArgs*)payload; + BMK_initCCtx(ag->ctx, ag->dictBuffer, ag->dictBufferSize, ag->cLevel, ag->comprParams, ag->adv); + return 0; +} + +typedef struct { + ZSTD_DCtx* dctx; + const void* dictBuffer; + size_t dictBufferSize; +} BMK_initDCtxArgs; + +static size_t local_initDCtx(void* payload) { + BMK_initDCtxArgs* ag = (BMK_initDCtxArgs*)payload; + BMK_initDCtx(ag->dctx, ag->dictBuffer, ag->dictBufferSize); + return 0; +} + +/* additional argument is just the context */ +static size_t local_defaultCompress( + const void* srcBuffer, size_t srcSize, + void* dstBuffer, size_t dstSize, + void* addArgs) { + size_t moreToFlush = 1; + ZSTD_CCtx* ctx = (ZSTD_CCtx*)addArgs; + ZSTD_inBuffer in; + ZSTD_outBuffer out; + in.src = srcBuffer; + in.size = srcSize; + in.pos = 0; + out.dst = dstBuffer; + out.size = dstSize; + out.pos = 0; + while (moreToFlush) { + if(out.pos == out.size) { + return (size_t)-ZSTD_error_dstSize_tooSmall; + } + moreToFlush = ZSTD_compress_generic(ctx, &out, &in, ZSTD_e_end); + if (ZSTD_isError(moreToFlush)) { + return moreToFlush; + } + } + return out.pos; +} + +/* additional argument is just the context */ +static size_t local_defaultDecompress( + const void* srcBuffer, size_t srcSize, + void* dstBuffer, size_t dstSize, + void* addArgs) { + size_t moreToFlush = 1; + ZSTD_DCtx* dctx = (ZSTD_DCtx*)addArgs; + ZSTD_inBuffer in; + ZSTD_outBuffer out; + in.src = srcBuffer; + in.size = srcSize; + in.pos = 0; + out.dst = dstBuffer; + out.size = dstSize; + out.pos = 0; + while (moreToFlush) { + if(out.pos == out.size) { + return (size_t)-ZSTD_error_dstSize_tooSmall; + } + moreToFlush = ZSTD_decompress_generic(dctx, + &out, &in); + if (ZSTD_isError(moreToFlush)) { + return moreToFlush; + } + } + return out.pos; + +} + +/*-******************************************************* +* From Paramgrill End +*********************************************************/ + +/* Replicate function of benchMemAdvanced, but with pre-split src / dst buffers, with relevant info to invert it (compressedSizes) passed out. */ +/*BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); */ +/* nbSeconds used in same way as in BMK_advancedParams_t, as nbIters when in iterMode */ + +/* if in decodeOnly, then srcPtr's will be compressed blocks, and uncompressedBlocks will be written to dstPtrs? */ +/* dictionary nullable, nothing else though. */ +static BMK_return_t BMK_benchMemInvertible(const void * const * const srcPtrs, size_t const * const srcSizes, + void** dstPtrs, size_t* dstCapacityToSizes, U32 const nbBlocks, + const int cLevel, const ZSTD_compressionParameters* comprParams, + const void* dictBuffer, const size_t dictBufferSize, + ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, + const BMK_mode_t mode, const BMK_loopMode_t loopMode, const unsigned nbSeconds) { + U32 i; + BMK_return_t results = { { 0, 0., 0., 0 }, 0 } ; + size_t srcSize = 0; + void** const resPtrs = malloc(sizeof(void*) * nbBlocks); /* only really needed in both mode. */ + size_t* const resSizes = malloc(sizeof(size_t) * nbBlocks); + int freeDST = 0; + + BMK_advancedParams_t adv = BMK_initAdvancedParams(); + adv.mode = mode; + adv.loopMode = loopMode; + adv.nbSeconds = nbSeconds; + + /* resSizes == srcSizes, but modifiable */ + memcpy(resSizes, srcSizes, sizeof(size_t) * nbBlocks); + + for(i = 0; i < nbBlocks; i++) { + srcSize += srcSizes[i]; + } + + if(!ctx || !dctx || !srcPtrs || ! srcSizes) + { + results.error = 31; + DISPLAY("error: passed in null argument\n"); + free(resPtrs); + free(resSizes); + return results; + } + if(!resPtrs || !resSizes) { + results.error = 32; + DISPLAY("error: allocation failed\n"); + free(resPtrs); + free(resSizes); + return results; + } + + /* so resPtr is continuous */ + resPtrs[0] = malloc(srcSize); + + if(!(resPtrs[0])) { + results.error = 32; + DISPLAY("error: allocation failed\n"); + free(resPtrs); + free(resSizes); + return results; + } + + for(i = 1; i < nbBlocks; i++) { + resPtrs[i] = (void*)(((char*)resPtrs[i-1]) + srcSizes[i-1]); + } + + /* allocate own dst if NULL */ + if(dstPtrs == NULL) { + freeDST = 1; + dstPtrs = malloc(nbBlocks * sizeof(void*)); + dstCapacityToSizes = malloc(nbBlocks * sizeof(size_t)); + if(dstPtrs == NULL) { + results.error = 33; + DISPLAY("error: allocation failed\n"); + free(resPtrs); + free(resSizes); + return results; + } + + if(mode == BMK_decodeOnly) { //dst is src + size_t dstSize = 0; + for(i = 0; i < nbBlocks; i++) { + dstCapacityToSizes[i] = ZSTD_getDecompressedSize(srcPtrs[i], srcSizes[i]); + dstSize += dstCapacityToSizes[i]; + } + dstPtrs[0] = malloc(dstSize); + if(dstPtrs[0] == NULL) { + results.error = 34; + DISPLAY("error: allocation failed\n"); + goto _cleanUp; + } + for(i = 1; i < nbBlocks; i++) { + dstPtrs[i] = (void*)(((char*)dstPtrs[i-1]) + ZSTD_getDecompressedSize(srcPtrs[i-1], srcSizes[i-1])); + } + } else { + dstPtrs[0] = malloc(ZSTD_compressBound(srcSize) + (nbBlocks * 1024)); + if(dstPtrs[0] == NULL) { + results.error = 35; + DISPLAY("error: allocation failed\n"); + goto _cleanUp; + } + dstCapacityToSizes[0] = ZSTD_compressBound(srcSizes[0]); + for(i = 1; i < nbBlocks; i++) { + dstPtrs[i] = (void*)(((char*)dstPtrs[i-1]) + dstCapacityToSizes[i-1]); + dstCapacityToSizes[i] = ZSTD_compressBound(srcSizes[i]); + } + } + } + + /* warmimg up memory */ + for(i = 0; i < nbBlocks; i++) { + RDG_genBuffer(dstPtrs[i], dstCapacityToSizes[i], 0.10, 0.50, 1); + } + + /* Bench */ + { + { + BMK_initCCtxArgs cctxprep; + BMK_initDCtxArgs dctxprep; + cctxprep.ctx = ctx; + cctxprep.dictBuffer = dictBuffer; + cctxprep.dictBufferSize = dictBufferSize; + cctxprep.cLevel = cLevel; + cctxprep.comprParams = comprParams; + cctxprep.adv = &adv; + dctxprep.dctx = dctx; + dctxprep.dictBuffer = dictBuffer; + dctxprep.dictBufferSize = dictBufferSize; + if(loopMode == BMK_timeMode) { + BMK_customTimedReturn_t intermediateResultCompress; + BMK_customTimedReturn_t intermediateResultDecompress; + BMK_timedFnState_t* timeStateCompress = BMK_createTimeState(nbSeconds); + BMK_timedFnState_t* timeStateDecompress = BMK_createTimeState(nbSeconds); + if(mode == BMK_compressOnly) { + intermediateResultCompress.completed = 0; + intermediateResultDecompress.completed = 1; + } else if (mode == BMK_decodeOnly) { + intermediateResultCompress.completed = 1; + intermediateResultDecompress.completed = 0; + } else { /* both */ + intermediateResultCompress.completed = 0; + intermediateResultDecompress.completed = 0; + } + while(!(intermediateResultCompress.completed && intermediateResultDecompress.completed)) { + if(!intermediateResultCompress.completed) { + intermediateResultCompress = BMK_benchFunctionTimed(timeStateCompress, &local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep, + nbBlocks, srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes); + if(intermediateResultCompress.result.error) { + results.error = intermediateResultCompress.result.error; + BMK_freeTimeState(timeStateCompress); + BMK_freeTimeState(timeStateDecompress); + goto _cleanUp; + } + results.result.cSpeed = ((double)srcSize / intermediateResultCompress.result.result.nanoSecPerRun) * 1000000000; + results.result.cSize = intermediateResultCompress.result.result.sumOfReturn; + } + + if(!intermediateResultDecompress.completed) { + if(mode == BMK_decodeOnly) { + intermediateResultDecompress = BMK_benchFunctionTimed(timeStateDecompress, &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep, + nbBlocks, (const void* const*)srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes); + } else { /* both, decompressed result already written to dstPtr */ + intermediateResultDecompress = BMK_benchFunctionTimed(timeStateDecompress, &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep, + nbBlocks, (const void* const*)dstPtrs, dstCapacityToSizes, resPtrs, resSizes); + } + + if(intermediateResultDecompress.result.error) { + results.error = intermediateResultDecompress.result.error; + BMK_freeTimeState(timeStateCompress); + BMK_freeTimeState(timeStateDecompress); + goto _cleanUp; + } + results.result.dSpeed = ((double)srcSize / intermediateResultDecompress.result.result.nanoSecPerRun) * 1000000000; + } + } + BMK_freeTimeState(timeStateCompress); + BMK_freeTimeState(timeStateDecompress); + } else { //iterMode; + if(mode != BMK_decodeOnly) { + + BMK_customReturn_t compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep, + nbBlocks, srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbSeconds); + if(compressionResults.error) { + results.error = compressionResults.error; + goto _cleanUp; + } + if(compressionResults.result.nanoSecPerRun == 0) { + results.result.cSpeed = 0; + } else { + results.result.cSpeed = (double)srcSize / compressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; + } + results.result.cSize = compressionResults.result.sumOfReturn; + } + if(mode != BMK_compressOnly) { + BMK_customReturn_t decompressionResults; + if(mode == BMK_decodeOnly) { + decompressionResults = BMK_benchFunction( + &local_defaultDecompress, (void*)(dctx), + &local_initDCtx, (void*)&dctxprep, nbBlocks, + (const void* const*)srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, + nbSeconds); + } else { + decompressionResults = BMK_benchFunction( + &local_defaultDecompress, (void*)(dctx), + &local_initDCtx, (void*)&dctxprep, nbBlocks, + (const void* const*)dstPtrs, dstCapacityToSizes, resPtrs, resSizes, + nbSeconds); + } + + if(decompressionResults.error) { + results.error = decompressionResults.error; + goto _cleanUp; + } + + if(decompressionResults.result.nanoSecPerRun == 0) { + results.result.dSpeed = 0; + } else { + results.result.dSpeed = (double)srcSize / decompressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; + } + } + } + } + } /* Bench */ + results.result.cMem = (1 << (comprParams->windowLog)) + ZSTD_sizeof_CCtx(ctx); + +_cleanUp: + free(resPtrs[0]); + free(resPtrs); + free(resSizes); + if(freeDST) { + free(dstPtrs[0]); + free(dstPtrs); + } + return results; +} + /* global winner used for display. */ //Should be totally 0 initialized? -static winnerInfo_t g_winner; //TODO: ratio is infinite at initialization, instead of 0 +static winnerInfo_t g_winner; static constraint_t g_targetConstraints; static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const size_t srcSize) @@ -379,7 +727,7 @@ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result "/* %s */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */", lvlstr, (double)srcSize / result.cSize, result.cSpeed / 1000000., result.dSpeed / 1000000.); - if(TIMED) { fprintf(f, " - %lu:%lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); } + if(TIMED) { fprintf(f, " - %1lu:%2lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); } fprintf(f, "\n"); if(objective_lt(g_winner.result, result) && feasible(result, g_targetConstraints)) { BMK_translateAdvancedParams(params); @@ -670,17 +1018,10 @@ static size_t memoTableLen(const U32* varyParams, const int varyLen) { //sort of ~lg2 (replace 1024 w/ 999, and add 0 at lower end of range) for memoTableInd Tlen static unsigned lg2(unsigned x) { - unsigned j = 1; if(x == 999) { return 11; } - if(!x) { - return 0; - } - while(x >>= 1) { - j++; - } - return j; + return x ? ZSTD_highbit32(x) + 1 : 0; } /* returns unique index of compression parameters */ @@ -729,7 +1070,7 @@ static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstra for(i = 0; i < arrayLen; i++) { memoTableIndInv(¶mConstraints, varyParams, varyLen, i); - if((ZSTD_estimateCCtxSize_usingCParams(paramConstraints) + (1ULL << paramConstraints.windowLog)) > (size_t)target.cMem + (size_t)(target.cMem / 10)) { + if(ZSTD_estimateCStreamSize_usingCParams(paramConstraints) > (size_t)target.cMem) { memoTable[i] = 255; j++; } @@ -765,8 +1106,6 @@ static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstra /* inits memotables for all (including mallocs), all strategies */ /* takes unsanitized varyParams */ - -//TODO: check for errors/nulls static U8** memoTableInitAll(ZSTD_compressionParameters paramConstraints, constraint_t target, const U32* varyParams, const int varyLen, const size_t srcSize) { U32 varNew[NUM_PARAMS]; int varLenNew; @@ -1099,28 +1438,29 @@ static int uncertainComparison(double const uncertaintyConstantC, double const u #define FEASIBLE_RESULT 1 #define ERROR_RESULT 2 static int feasibleBench(BMK_result_t* resultPtr, - const void* srcBuffer, const size_t srcSize, - void* dstBuffer, const size_t dstSize, - void* dictBuffer, const size_t dictSize, - const size_t* fileSizes, const size_t nbFiles, + const void* const * const srcPtrs, size_t const * const srcSizes, + void** const dstPtrs, size_t* dstCapacityToSizes, U32 const nbBlocks, + void* dictBuffer, const size_t dictBufferSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams, const constraint_t target, BMK_result_t* winnerResult) { - BMK_advancedParams_t adv = BMK_initAdvancedParams(); BMK_return_t benchres; U64 loopDurationC = 0, loopDurationD = 0; double uncertaintyConstantC, uncertaintyConstantD; - adv.loopMode = BMK_iterMode; - adv.nbSeconds = 1; //get ratio and 2x approx speed? - + size_t srcSize = 0; + U32 i; //alternative - test 1 iter for ratio, (possibility of error 3 which is fine), //maybe iter this until 2x measurable for better guarantee? DEBUGOUTPUT("Feas:\n"); - benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + benchres = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_both, BMK_iterMode, 1); if(benchres.error) { DISPLAY("ERROR %d !!\n", benchres.error); } + for(i = 0; i < nbBlocks; i++) { + srcSize += srcSizes[i]; + } BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); if(!benchres.error) { @@ -1151,9 +1491,8 @@ static int feasibleBench(BMK_result_t* resultPtr, if(benchres.result.cSize < winnerResult->cSize) { //better compression ratio, just needs to be feasible int feas; if(loopDurationC < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2; - adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_compressOnly, BMK_iterMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1161,9 +1500,8 @@ static int feasibleBench(BMK_result_t* resultPtr, } } if(loopDurationD < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2; - adv.mode = BMK_decodeOnly; - benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_decodeOnly, BMK_iterMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1174,11 +1512,9 @@ static int feasibleBench(BMK_result_t* resultPtr, feas = uncertainFeasibility(uncertaintyConstantC, uncertaintyConstantD, target, &(benchres.result)); if(feas == 0) { // uncertain feasibility - adv.loopMode = BMK_timeMode; if(loopDurationC < TIMELOOP_NANOSEC) { - BMK_return_t benchres2; - adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_compressOnly, BMK_timeMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1186,9 +1522,8 @@ static int feasibleBench(BMK_result_t* resultPtr, } } if(loopDurationD < TIMELOOP_NANOSEC) { - BMK_return_t benchres2; - adv.mode = BMK_decodeOnly; - benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_decodeOnly, BMK_timeMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1203,9 +1538,8 @@ static int feasibleBench(BMK_result_t* resultPtr, } else if (benchres.result.cSize == winnerResult->cSize) { //equal ratio, needs to be better than winner in cSpeed/ dSpeed / cMem int feas; if(loopDurationC < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2; - adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_compressOnly, BMK_iterMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1213,9 +1547,8 @@ static int feasibleBench(BMK_result_t* resultPtr, } } if(loopDurationD < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2; - adv.mode = BMK_decodeOnly; - benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_decodeOnly, BMK_iterMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1224,11 +1557,9 @@ static int feasibleBench(BMK_result_t* resultPtr, } feas = uncertainFeasibility(uncertaintyConstantC, uncertaintyConstantD, target, &(benchres.result)); if(feas == 0) { // uncertain feasibility - adv.loopMode = BMK_timeMode; if(loopDurationC < TIMELOOP_NANOSEC) { - BMK_return_t benchres2; - adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_compressOnly, BMK_timeMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1236,9 +1567,8 @@ static int feasibleBench(BMK_result_t* resultPtr, } } if(loopDurationD < TIMELOOP_NANOSEC) { - BMK_return_t benchres2; - adv.mode = BMK_decodeOnly; - benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_decodeOnly, BMK_timeMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1253,8 +1583,8 @@ static int feasibleBench(BMK_result_t* resultPtr, if(btw == -1) { return INFEASIBLE_RESULT; } else { //possibly better, benchmark and find out - adv.loopMode = BMK_timeMode; - benchres = BMK_benchMemAdvanced(srcBuffer, srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + benchres = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_both, BMK_timeMode, 1); *resultPtr = benchres.result; return objective_lt(*winnerResult, benchres.result); } @@ -1274,28 +1604,31 @@ static int feasibleBench(BMK_result_t* resultPtr, //have version of benchMemAdvanced which takes in dstBuffer/cap as well? //(motivation: repeat tests (maybe just on decompress) don't need further compress runs) static int infeasibleBench(BMK_result_t* resultPtr, - const void* srcBuffer, const size_t srcSize, - void* dstBuffer, const size_t dstSize, - void* dictBuffer, const size_t dictSize, - const size_t* fileSizes, const size_t nbFiles, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, - const ZSTD_compressionParameters cParams, - const constraint_t target, - BMK_result_t* winnerResult) { - BMK_advancedParams_t adv = BMK_initAdvancedParams(); + const void* const * const srcPtrs, size_t const * const srcSizes, + void** const dstPtrs, size_t* dstCapacityToSizes, U32 const nbBlocks, + void* dictBuffer, const size_t dictBufferSize, + ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, + const ZSTD_compressionParameters cParams, + const constraint_t target, + BMK_result_t* winnerResult) { BMK_return_t benchres; BMK_result_t resultMin, resultMax; U64 loopDurationC = 0, loopDurationD = 0; double uncertaintyConstantC, uncertaintyConstantD; - double winnerRS = resultScore(*winnerResult, srcSize, target); - adv.loopMode = BMK_iterMode; //can only use this for ratio measurement then, super inaccurate timing - adv.nbSeconds = 1; //get ratio and 2x approx speed? //maybe run until twice MIN(minloopinterval * clockDuration) + double winnerRS; + size_t srcSize = 0; + U32 i; + + benchres = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_both, BMK_iterMode, 1); + for(i = 0; i < nbBlocks; i++) { + srcSize += srcSizes[i]; + } + BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); + winnerRS = resultScore(*winnerResult, srcSize, target); DEBUGOUTPUT("WinnerScore: %f\n ", winnerRS); - benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); - BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); - if(!benchres.error) { *resultPtr = benchres.result; if(eqZero(benchres.result.cSpeed)) { @@ -1316,9 +1649,8 @@ static int infeasibleBench(BMK_result_t* resultPtr, if(loopDurationC < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2; - adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_compressOnly, BMK_iterMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1326,9 +1658,8 @@ static int infeasibleBench(BMK_result_t* resultPtr, } } if(loopDurationD < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2; - adv.mode = BMK_decodeOnly; - benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_decodeOnly, BMK_iterMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1347,12 +1678,9 @@ static int infeasibleBench(BMK_result_t* resultPtr, if (winnerRS > resultScore(resultMax, srcSize, target)) { return INFEASIBLE_RESULT; } else { - //do this w/o copying / stuff - adv.loopMode = BMK_timeMode; if(loopDurationC < TIMELOOP_NANOSEC) { - BMK_return_t benchres2; - adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_compressOnly, BMK_timeMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1360,9 +1688,8 @@ static int infeasibleBench(BMK_result_t* resultPtr, } } if(loopDurationD < TIMELOOP_NANOSEC) { - BMK_return_t benchres2; - adv.mode = BMK_decodeOnly; - benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_decodeOnly, BMK_timeMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1394,16 +1721,62 @@ static int feasibleBenchMemo(BMK_result_t* resultPtr, const U32* varyParams, const int varyLen) { const size_t memind = memoTableInd(&cParams, varyParams, varyLen); - if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { - return INFEASIBLE_RESULT; //probably pick a different code for already tested? - //maybe remove this if we incorporate nonrandom location picking? - //what is the intended behavior in this case? - //ignore? stop iterating completely? other? + return INFEASIBLE_RESULT; } else { - int res = feasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, fileSizes, nbFiles, ctx, dctx, + const size_t blockSize = g_blockSize ? g_blockSize : srcSize; + U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; + const void ** const srcPtrs = (const void** const)malloc(maxNbBlocks * sizeof(void*)); + size_t* const srcSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); + void ** const dstPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); + size_t* const dstCapacities = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); + U32 nbBlocks; + int res; + + if(!srcPtrs || !srcSizes || !dstPtrs || !dstCapacities) { + free(srcPtrs); + free(srcSizes); + free(dstPtrs); + free(dstCapacities); + DISPLAY("Allocation Error\n"); + return ERROR_RESULT; + } + + { + const char* srcPtr = (const char*)srcBuffer; + char* dstPtr = (char*)dstBuffer; + size_t dstSizeRemaining = dstSize; + U32 fileNb; + for (nbBlocks=0, fileNb=0; fileNb dstSizeRemaining) { + DEBUGOUTPUT("Warning: dstSize too small to benchmark completely \n"); + remaining = dstSizeRemaining; + dstSizeRemaining = 0; + } else { + dstSizeRemaining -= remaining; + } + for ( ; nbBlocks= INFEASIBLE_THRESHOLD) { return INFEASIBLE_RESULT; //see feasibleBenchMemo for concerns } else { - int res = infeasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, fileSizes, nbFiles, ctx, dctx, + const size_t blockSize = g_blockSize ? g_blockSize : srcSize; + U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; + const void ** const srcPtrs = (const void** const)malloc(maxNbBlocks * sizeof(void*)); + size_t* const srcSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); + void ** const dstPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); + size_t* const dstCapacities = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); + U32 nbBlocks; + int res; + + if(!srcPtrs || !srcSizes || !dstPtrs || !dstCapacities) { + free(srcPtrs); + free(srcSizes); + free(dstPtrs); + free(dstCapacities); + DISPLAY("Allocation Error\n"); + return ERROR_RESULT; + } + + { + const char* srcPtr = (const char*)srcBuffer; + char* dstPtr = (char*)dstBuffer; + size_t dstSizeRemaining = dstSize; + U32 fileNb; + for (nbBlocks=0, fileNb=0; fileNb dstSizeRemaining) { + DEBUGOUTPUT("Warning: dstSize too small to benchmark completely \n"); + remaining = dstSizeRemaining; + dstSizeRemaining = 0; + } else { + dstSizeRemaining -= remaining; + } + for ( ; nbBlocks candidate before restart } } + return winnerInfo; } @@ -1711,7 +2135,7 @@ static winnerInfo_t optimizeFixedStrategy( } while(i < 10) { //make i adjustable (user input?) depending on how much time they have. - DISPLAY("Restart\n"); //TODO: make better printing across restarts + DEBUGOUTPUT("Restart\n"); //look into improving this to maximize distance from searched infeasible stuff / towards promising regions? randomConstrainedParams(&init, varNew, varLenNew, memoTable); candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, fileSizes, nbFiles, ctx, dctx, init); @@ -1918,7 +2342,6 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ /* find best solution from default params */ { /* strategy selection */ - //TODO: don't factor memory into strategy selection in constraint_t const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); DEBUGOUTPUT("Strategy Selection\n"); if(varLen == NUM_PARAMS && paramTarget.strategy == 0) { /* no variable based constraints */ @@ -2019,7 +2442,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ if(objective_lt(winner.result, wc.result)) { winner = wc; } - //TODO: we could double back to increase search of 'better' strategies + //We could double back to increase search of 'better' strategies st = nextStrategy(st, bestStrategy); } } else { From 0f91b039ff934c4fd42d292a90ea62b52cb2d696 Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 16 Jul 2018 18:04:57 -0700 Subject: [PATCH 07/19] Add Levels --- tests/paramgrill.c | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) mode change 100644 => 100755 tests/paramgrill.c diff --git a/tests/paramgrill.c b/tests/paramgrill.c old mode 100644 new mode 100755 index e4d468846..086931b3d --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -56,7 +56,7 @@ static const int g_maxNbVariations = 64; #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define TIMED 0 #ifndef DEBUG -# define DEBUG 0 +# define DEBUG 1 #endif #define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); } @@ -2227,7 +2227,7 @@ static int nextStrategy(const int currentStrategy, const int bestStrategy) { } //optimize fixed strategy. -static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget) +static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel) { size_t benchedSize; void* origBuff = NULL; @@ -2238,7 +2238,6 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ size_t* fileSizes = calloc(sizeof(size_t),nbFiles); const int varLen = variableParams(paramTarget, varArray); U8** allMT = NULL; - g_targetConstraints = target; g_winner.result.cSize = (size_t)-1; /* Init */ if(!cParamValid(paramTarget)) { @@ -2310,6 +2309,30 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ goto _cleanUp; } + //TODO: cLevel Stuff. + if(cLevel) { + BMK_result_t candidate; + const size_t blockSize = g_blockSize ? g_blockSize : benchedSize; + ZSTD_CCtx* const ctx = ZSTD_createCCtx(); + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + ZSTD_compressionParameters const CParams = ZSTD_getCParams(cLevel, blockSize, dictBufferSize); + if(BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, nbFiles, ctx, dctx, CParams)) { + ZSTD_freeCCtx(ctx); + ZSTD_freeDCtx(dctx); + ret = 3; + goto _cleanUp; + } + + target.cSpeed = candidate.cSpeed; //TODO: maybe have a small bit of slack here, like x.99? + target.dSpeed = candidate.dSpeed; + BMK_printWinner(stdout, cLevel, candidate, CParams, benchedSize); + + ZSTD_freeCCtx(ctx); + ZSTD_freeDCtx(dctx); + } + + g_targetConstraints = target; + /* bench */ DISPLAY("\r%79s\r", ""); if(nbFiles == 1) { @@ -2348,7 +2371,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ BMK_result_t candidate; int feas = 0, i; for (i=1; i<=maxSeeds; i++) { - ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, 0); + ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, dictBufferSize); int ec = BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, nbFiles, ctx, dctx, CParams); BMK_printWinner(stdout, i, candidate, CParams, benchedSize); @@ -2553,6 +2576,7 @@ int main(int argc, const char** argv) const char* dictFileName = 0; U32 optimizer = 0; U32 main_pause = 0; + int optimizerCLevel = 0; constraint_t target = { 0, 0, (U32)-1 }; //0 for anything unset @@ -2584,6 +2608,7 @@ int main(int argc, const char** argv) if (longCommandWArg(&argument, "compressionSpeed=") || longCommandWArg(&argument, "cSpeed=")) { target.cSpeed = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } if (longCommandWArg(&argument, "decompressionSpeed=") || longCommandWArg(&argument, "dSpeed=")) { target.dSpeed = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } if (longCommandWArg(&argument, "compressionMemory=") || longCommandWArg(&argument, "cMem=")) { target.cMem = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } + //TODO: add Level; /* in MB or MB/s */ DISPLAY("invalid optimization parameter \n"); return 1; @@ -2692,6 +2717,10 @@ int main(int argc, const char** argv) argument++; paramTarget.strategy = (ZSTD_strategy)readU32FromChar(&argument); continue; + case 'L': /* level centers around a level */ + argument++; + optimizerCLevel = (int)readU32FromChar(&argument); + continue; default : ; } break; @@ -2796,7 +2825,7 @@ int main(int argc, const char** argv) } } else { if (optimizer) { - result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget); + result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget, optimizerCLevel); } else { result = benchFiles(argv+filenamesStart, argc-filenamesStart); } } From df026e159f7e5ff57a0caa5817176b00c059bc49 Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 16 Jul 2018 18:22:04 -0700 Subject: [PATCH 08/19] Fix windows implicit casting bugs --- programs/bench.c | 4 ++-- tests/paramgrill.c | 40 ++++++++++++++++++++-------------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 8e57db4c9..a34b74cf1 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -696,7 +696,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( } DISPLAYLEVEL(2, "%2i#\n", cLevel); } /* Bench */ - results.result.cMem = (1 << (comprParams->windowLog)) + ZSTD_sizeof_CCtx(ctx); + results.result.cMem = (1ULL << (comprParams->windowLog)) + ZSTD_sizeof_CCtx(ctx); results.error = 0; return results; } @@ -731,7 +731,7 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, void* resultBuffer = malloc(srcSize); - BMK_return_t results; + BMK_return_t results = { { 0, 0, 0, 0 }, 0 }; int allocationincomplete; if(!dstCapacity) { diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 086931b3d..02811b9c1 100755 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -263,7 +263,9 @@ static int epsilonEqual(const double c1, const double c2) { /* checks exact equivalence to 0, to stop compiler complaining fpeq */ static int eqZero(const double c1) { - return (U64)c1 == (U64)0.0 || (U64)c1 == (U64)-0.0; + const double z1 = 0.0; + const double z2 = -0.0; + return !(memcmp(&c1, &z1, sizeof(double))) || !(memcmp(&c1, &z2, sizeof(double))); } /* returns 1 if result2 is strictly 'better' than result1 */ @@ -318,7 +320,7 @@ const char* g_stratName[ZSTD_btultra+1] = { "ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ", "ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "}; -static size_t +static int BMK_benchParam(BMK_result_t* resultPtr, const void* srcBuffer, const size_t srcSize, const size_t* fileSizes, const unsigned nbFiles, @@ -331,7 +333,7 @@ BMK_benchParam(BMK_result_t* resultPtr, } /* benchParam but only takes in one file. */ -static size_t +static int BMK_benchParam1(BMK_result_t* resultPtr, const void* srcBuffer, size_t srcSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, @@ -1204,7 +1206,7 @@ static ZSTD_compressionParameters randomParams(void) /* Sets pc to random unmeasured set of parameters */ static void randomConstrainedParams(ZSTD_compressionParameters* pc, U32* varArray, int varLen, U8* memoTable) { - int tries = memoTableLen(varArray, varLen); //configurable, + size_t tries = memoTableLen(varArray, varLen); //configurable, const size_t maxSize = memoTableLen(varArray, varLen); size_t ind; do { @@ -1470,7 +1472,7 @@ static int feasibleBench(BMK_result_t* resultPtr, loopDurationC = 0; uncertaintyConstantC = 2; } else { - loopDurationC = ((srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); + loopDurationC = (U64)((double)(srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? //possibly has to do with initCCtx? or system stuff? //asymmetric +/- constant needed? @@ -1480,7 +1482,7 @@ static int feasibleBench(BMK_result_t* resultPtr, loopDurationD = 0; uncertaintyConstantD = 2; } else { - loopDurationD = ((srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); + loopDurationD = (U64)((double)(srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? //possibly has to do with initCCtx? or system stuff? //asymmetric +/- constant needed? @@ -1635,7 +1637,7 @@ static int infeasibleBench(BMK_result_t* resultPtr, loopDurationC = 0; uncertaintyConstantC = 2; } else { - loopDurationC = ((srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); + loopDurationC = (U64)((double)(srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); uncertaintyConstantC = MIN((loopDurationC + (double)(2 * g_clockGranularity)/loopDurationC * 1.1), 3); //.02 seconds } @@ -1643,7 +1645,7 @@ static int infeasibleBench(BMK_result_t* resultPtr, loopDurationD = 0; uncertaintyConstantD = 2; } else { - loopDurationD = ((srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); + loopDurationD = (U64)((double)(srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); uncertaintyConstantD = MIN((loopDurationD + (double)(2 * g_clockGranularity)/loopDurationD) * 1.1 , 3); //.02 seconds } @@ -1725,7 +1727,7 @@ static int feasibleBenchMemo(BMK_result_t* resultPtr, return INFEASIBLE_RESULT; } else { const size_t blockSize = g_blockSize ? g_blockSize : srcSize; - U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; + U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + (U32)nbFiles; const void ** const srcPtrs = (const void** const)malloc(maxNbBlocks * sizeof(void*)); size_t* const srcSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); void ** const dstPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); @@ -1798,7 +1800,7 @@ static int infeasibleBenchMemo(BMK_result_t* resultPtr, return INFEASIBLE_RESULT; //see feasibleBenchMemo for concerns } else { const size_t blockSize = g_blockSize ? g_blockSize : srcSize; - U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; + U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + (U32)nbFiles; const void ** const srcPtrs = (const void** const)malloc(maxNbBlocks * sizeof(void*)); size_t* const srcSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); void ** const dstPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); @@ -2279,7 +2281,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } { - U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles); + U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles); int ec; unsigned i; benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3; @@ -2290,7 +2292,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ ret = 1; goto _cleanUp; } - ec = BMK_loadFiles(origBuff, benchedSize, fileSizes, fileNamesTable, nbFiles); + ec = BMK_loadFiles(origBuff, benchedSize, fileSizes, fileNamesTable, (U32)nbFiles); if(ec) { DISPLAY("Error Loading Files"); ret = ec; @@ -2308,23 +2310,21 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ ret = 2; goto _cleanUp; } - - //TODO: cLevel Stuff. + if(cLevel) { BMK_result_t candidate; const size_t blockSize = g_blockSize ? g_blockSize : benchedSize; ZSTD_CCtx* const ctx = ZSTD_createCCtx(); ZSTD_DCtx* const dctx = ZSTD_createDCtx(); ZSTD_compressionParameters const CParams = ZSTD_getCParams(cLevel, blockSize, dictBufferSize); - if(BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, nbFiles, ctx, dctx, CParams)) { + if(BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, (U32)nbFiles, ctx, dctx, CParams)) { ZSTD_freeCCtx(ctx); ZSTD_freeDCtx(dctx); ret = 3; goto _cleanUp; } - target.cSpeed = candidate.cSpeed; //TODO: maybe have a small bit of slack here, like x.99? - target.dSpeed = candidate.dSpeed; + target.cSpeed = (U32)candidate.cSpeed; //Maybe have a small bit of slack here, like x.99? BMK_printWinner(stdout, cLevel, candidate, CParams, benchedSize); ZSTD_freeCCtx(ctx); @@ -2372,7 +2372,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ int feas = 0, i; for (i=1; i<=maxSeeds; i++) { ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, dictBufferSize); - int ec = BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, nbFiles, ctx, dctx, CParams); + int ec = BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, (U32)nbFiles, ctx, dctx, CParams); BMK_printWinner(stdout, i, candidate, CParams, benchedSize); if(!ec) { @@ -2408,7 +2408,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ randomConstrainedParams(&candidateParams, varNew, varLenNew, allMT[i]); cParamZeroMin(&candidateParams); candidateParams = sanitizeParams(candidateParams); - ec = BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, nbFiles, ctx, dctx, candidateParams); + ec = BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, (U32)nbFiles, ctx, dctx, candidateParams); if(!ec) { if(feas) { @@ -2608,7 +2608,7 @@ int main(int argc, const char** argv) if (longCommandWArg(&argument, "compressionSpeed=") || longCommandWArg(&argument, "cSpeed=")) { target.cSpeed = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } if (longCommandWArg(&argument, "decompressionSpeed=") || longCommandWArg(&argument, "dSpeed=")) { target.dSpeed = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } if (longCommandWArg(&argument, "compressionMemory=") || longCommandWArg(&argument, "cMem=")) { target.cMem = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } - //TODO: add Level; + if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { optimizerCLevel = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } /* in MB or MB/s */ DISPLAY("invalid optimization parameter \n"); return 1; From e148db366ea424bbc2dc7269c566adb97cc11ec6 Mon Sep 17 00:00:00 2001 From: George Lu Date: Fri, 20 Jul 2018 14:35:09 -0700 Subject: [PATCH 09/19] Separate capacity vs size Also: Make suggested fixes -varInds_t -reorder some arguments -remove code duplication -update README / -h -Fix memory leaks --- programs/bench.c | 63 +- programs/bench.h | 8 +- tests/README.md | 32 + tests/fullbench.c | 2 +- tests/paramgrill.c | 1963 ++++++++++++++++---------------------------- 5 files changed, 779 insertions(+), 1289 deletions(-) mode change 100755 => 100644 tests/paramgrill.c diff --git a/programs/bench.c b/programs/bench.c index a34b74cf1..177dbe0e3 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -171,6 +171,8 @@ struct BMK_timeState_t{ static void BMK_initCCtx(ZSTD_CCtx* ctx, const void* dictBuffer, size_t dictBufferSize, int cLevel, const ZSTD_compressionParameters* comprParams, const BMK_advancedParams_t* adv) { + ZSTD_CCtx_reset(ctx); + ZSTD_CCtx_resetParameters(ctx); if (adv->nbWorkers==1) { ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, 0); } else { @@ -195,6 +197,7 @@ static void BMK_initCCtx(ZSTD_CCtx* ctx, static void BMK_initDCtx(ZSTD_DCtx* dctx, const void* dictBuffer, size_t dictBufferSize) { + ZSTD_DCtx_reset(dctx); ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize); } @@ -291,9 +294,9 @@ BMK_customReturn_t BMK_benchFunction( BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBlockBuffers, const size_t* srcBlockSizes, - void* const * const dstBlockBuffers, size_t* dstBlockCapacitiesToSizes, + void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, size_t* cSizes, unsigned nbLoops) { - size_t srcSize = 0, dstSize = 0, ind = 0; + size_t dstSize = 0; U64 totalTime; BMK_customReturn_t retval; @@ -303,36 +306,37 @@ BMK_customReturn_t BMK_benchFunction( EXM_THROW_ND(1, BMK_customReturn_t, "nbLoops must be nonzero \n"); } - for(ind = 0; ind < blockCount; ind++) { - srcSize += srcBlockSizes[ind]; - } - { size_t i; for(i = 0; i < blockCount; i++) { - memset(dstBlockBuffers[i], 0xE5, dstBlockCapacitiesToSizes[i]); /* warm up and erase result buffer */ + memset(dstBlockBuffers[i], 0xE5, dstBlockCapacities[i]); /* warm up and erase result buffer */ } - - //UTIL_sleepMilli(5); /* give processor time to other processes */ - //UTIL_waitForNextTick(); +#if 0 + /* based on testing these seem to lower accuracy of multiple calls of 1 nbLoops vs 1 call of multiple nbLoops + * (Makes former slower) + */ + UTIL_sleepMilli(5); /* give processor time to other processes */ + UTIL_waitForNextTick(); +#endif } { - unsigned i, j, firstIter = 1; + unsigned i, j; clockStart = UTIL_getTime(); if(initFn != NULL) { initFn(initPayload); } for(i = 0; i < nbLoops; i++) { for(j = 0; j < blockCount; j++) { - size_t res = benchFn(srcBlockBuffers[j], srcBlockSizes[j], dstBlockBuffers[j], dstBlockCapacitiesToSizes[j], benchPayload); + size_t res = benchFn(srcBlockBuffers[j], srcBlockSizes[j], dstBlockBuffers[j], dstBlockCapacities[j], benchPayload); if(ZSTD_isError(res)) { EXM_THROW_ND(2, BMK_customReturn_t, "Function benchmarking failed on block %u of size %u : %s \n", - j, (U32)dstBlockCapacitiesToSizes[j], ZSTD_getErrorName(res)); - } else if(firstIter) { + j, (U32)dstBlockCapacities[j], ZSTD_getErrorName(res)); + } else if(i == nbLoops - 1) { dstSize += res; - dstBlockCapacitiesToSizes[j] = res; + if(cSizes != NULL) { + cSizes[j] = res; + } } } - firstIter = 0; } totalTime = UTIL_clockSpanNano(clockStart); } @@ -369,7 +373,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed( BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const* const srcBlockBuffers, const size_t* srcBlockSizes, - void * const * const dstBlockBuffers, size_t * dstBlockCapacitiesToSizes) + void * const * const dstBlockBuffers, const size_t * dstBlockCapacities, size_t* dstSizes) { U64 fastest = cont->fastestTime; int completed = 0; @@ -384,9 +388,9 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed( UTIL_sleep(COOLPERIOD_SEC); cont->coolTime = UTIL_getTime(); } - + /* reinitialize capacity */ r.result = BMK_benchFunction(benchFn, benchPayload, initFn, initPayload, - blockCount, srcBlockBuffers, srcBlockSizes, dstBlockBuffers, dstBlockCapacitiesToSizes, cont->nbLoops); + blockCount, srcBlockBuffers, srcBlockSizes, dstBlockBuffers, dstBlockCapacities, dstSizes, cont->nbLoops); if(r.result.error) { /* completed w/ error */ r.completed = 1; return r; @@ -420,7 +424,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed( /* benchMem with no allocation */ static BMK_return_t BMK_benchMemAdvancedNoAlloc( const void ** const srcPtrs, size_t* const srcSizes, - void** const cPtrs, size_t* const cSizes, + void** const cPtrs, size_t* const cCapacities, size_t* const cSizes, void** const resPtrs, size_t* const resSizes, void** resultBufferPtr, void* compressedBuffer, const size_t maxCompressedSize, @@ -485,11 +489,11 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( srcPtrs[nbBlocks] = (const void*)srcPtr; srcSizes[nbBlocks] = thisBlockSize; cPtrs[nbBlocks] = (void*)cPtr; - cSizes[nbBlocks] = (adv->mode == BMK_decodeOnly) ? thisBlockSize : ZSTD_compressBound(thisBlockSize); + cCapacities[nbBlocks] = (adv->mode == BMK_decodeOnly) ? thisBlockSize : ZSTD_compressBound(thisBlockSize); resPtrs[nbBlocks] = (void*)resPtr; resSizes[nbBlocks] = (adv->mode == BMK_decodeOnly) ? (size_t) ZSTD_findDecompressedSize(srcPtr, thisBlockSize) : thisBlockSize; srcPtr += thisBlockSize; - cPtr += cSizes[nbBlocks]; + cPtr += cCapacities[nbBlocks]; resPtr += thisBlockSize; remaining -= thisBlockSize; } @@ -540,7 +544,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( while(!(intermediateResultCompress.completed && intermediateResultDecompress.completed)) { if(!intermediateResultCompress.completed) { intermediateResultCompress = BMK_benchFunctionTimed(timeStateCompress, &local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep, - nbBlocks, srcPtrs, srcSizes, cPtrs, cSizes); + nbBlocks, srcPtrs, srcSizes, cPtrs, cCapacities, cSizes); if(intermediateResultCompress.result.error) { results.error = intermediateResultCompress.result.error; return results; @@ -564,7 +568,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( if(!intermediateResultDecompress.completed) { intermediateResultDecompress = BMK_benchFunctionTimed(timeStateDecompress, &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep, - nbBlocks, (const void* const*)cPtrs, cSizes, resPtrs, resSizes); + nbBlocks, (const void* const*)cPtrs, cSizes, resPtrs, resSizes, NULL); if(intermediateResultDecompress.result.error) { results.error = intermediateResultDecompress.result.error; return results; @@ -590,7 +594,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( if(adv->mode != BMK_decodeOnly) { BMK_customReturn_t compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep, - nbBlocks, srcPtrs, srcSizes, cPtrs, cSizes, adv->nbSeconds); + nbBlocks, srcPtrs, srcSizes, cPtrs, cCapacities, cSizes, adv->nbSeconds); if(compressionResults.error) { results.error = compressionResults.error; return results; @@ -617,7 +621,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( BMK_customReturn_t decompressionResults = BMK_benchFunction( &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep, nbBlocks, - (const void* const*)cPtrs, cSizes, resPtrs, resSizes, + (const void* const*)cPtrs, cSizes, resPtrs, resSizes, NULL, adv->nbSeconds); if(decompressionResults.error) { results.error = decompressionResults.error; @@ -717,8 +721,10 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, const void ** const srcPtrs = (const void** const)malloc(maxNbBlocks * sizeof(void*)); size_t* const srcSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); + void ** const cPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); size_t* const cSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); + size_t* const cCapacities = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); void ** const resPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); size_t* const resSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); @@ -744,13 +750,11 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, !srcPtrs || !srcSizes || !cPtrs || !cSizes || !resPtrs || !resSizes; if (!allocationincomplete) { - results = BMK_benchMemAdvancedNoAlloc(srcPtrs, srcSizes, cPtrs, cSizes, + results = BMK_benchMemAdvancedNoAlloc(srcPtrs, srcSizes, cPtrs, cCapacities, cSizes, resPtrs, resSizes, &resultBuffer, compressedBuffer, maxCompressedSize, timeStateCompress, timeStateDecompress, srcBuffer, srcSize, fileSizes, nbFiles, cLevel, comprParams, dictBuffer, dictBufferSize, ctx, dctx, displayLevel, displayName, adv); } - - /* clean up */ BMK_freeTimeState(timeStateCompress); @@ -764,6 +768,7 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, free(srcSizes); free(cPtrs); free(cSizes); + free(cCapacities); free(resPtrs); free(resSizes); diff --git a/programs/bench.h b/programs/bench.h index 25d9f24a7..2a9945ac8 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -177,6 +177,7 @@ typedef size_t (*BMK_initFn_t)(void*); * dstBuffers - an array of buffers to be written into by benchFn * dstCapacitiesToSizes - an array of the capacities of above buffers. Output modified to compressed sizes of those blocks. * nbLoops - defines number of times benchFn is run. + * assumed array of size blockCount, will have compressed size of each block written to it. * return * .error will give a nonzero value if ZSTD_isError() is nonzero for any of the return * of the calls to initFn and benchFn, or if benchFunction errors internally @@ -187,12 +188,11 @@ typedef size_t (*BMK_initFn_t)(void*); * into dstBuffer, hence this value will be the total amount of bytes written to * dstBuffer. */ -BMK_customReturn_t BMK_benchFunction( - BMK_benchFn_t benchFn, void* benchPayload, +BMK_customReturn_t BMK_benchFunction(BMK_benchFn_t benchFn, void* benchPayload, BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBuffers, const size_t* srcSizes, - void * const * const dstBuffers, size_t* dstCapacitiesToSizes, + void * const * const dstBuffers, const size_t* dstCapacities, size_t* cSizes, unsigned nbLoops); @@ -221,7 +221,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed(BMK_timedFnState_t* cont, BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBlockBuffers, const size_t* srcBlockSizes, - void* const * const dstBlockBuffers, size_t* dstBlockCapacitiesToSizes); + void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, size_t* cSizes); #endif /* BENCH_H_121279284357 */ diff --git a/tests/README.md b/tests/README.md index 24a28ab7b..8bedd0a3c 100644 --- a/tests/README.md +++ b/tests/README.md @@ -88,3 +88,35 @@ as well as the 10,000 original files for more detailed comparison of decompressi will choose a random seed, and for 1 minute, generate random test frames and ensure that the zstd library correctly decompresses them in both simple and streaming modes. + +#### `paramgrill` - tool for generating compression table parameters and optimizing parameters on file given constraints + +Full list of arguments +``` + -T# : set level 1 speed objective + -B# : cut input into blocks of size # (default : single block) + -i# : iteration loops + -S : benchmarks a single run (example command: -Sl3w10h12) + w# - windowLog + h# - hashLog + c# - chainLog + s# - searchLog + l# - searchLength + t# - targetLength + S# - strategy + L# - level + --zstd= : Single run, parameter selection syntax same as zstdcli + --optimize= : find parameters to maximize compression ratio given parameters + Can use all --zstd= commands to constrain the type of solution found in addition to the following constraints + cSpeed= - Minimum compression speed + dSpeed= - Minimum decompression speed + cMem= - compression memory + lvl= - Automatically sets compression speed constraint to the speed of that level + --optimize= : same as -O with more verbose syntax + -P# : generated sample compressibility + -t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) + -v : Prints Benchmarking output + -D : Next argument dictionary file + +``` + Any inputs afterwards are treated as files to benchmark. diff --git a/tests/fullbench.c b/tests/fullbench.c index 9e7639f92..270cac86a 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -516,7 +516,7 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb, int cLevel, { r = BMK_benchFunction(benchFunction, buff2, NULL, NULL, 1, &src, &srcSize, - (void **)&dstBuff, &dstBuffSize, g_nbIterations); + (void **)&dstBuff, &dstBuffSize, NULL, g_nbIterations); if(r.error) { DISPLAY("ERROR %d ! ! \n", r.error); errorcode = r.error; diff --git a/tests/paramgrill.c b/tests/paramgrill.c old mode 100755 new mode 100644 index 02811b9c1..099be3688 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -56,7 +56,7 @@ static const int g_maxNbVariations = 64; #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define TIMED 0 #ifndef DEBUG -# define DEBUG 1 +# define DEBUG 0 #endif #define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); } @@ -67,14 +67,15 @@ static const int g_maxNbVariations = 64; #define CUSTOM_LEVEL 99 /* indices for each of the variables */ -#define WLOG_IND 0 -#define CLOG_IND 1 -#define HLOG_IND 2 -#define SLOG_IND 3 -#define SLEN_IND 4 -#define TLEN_IND 5 -//#define STRT_IND 6 -//#define NUM_PARAMS 7 +typedef enum { + wlog_ind = 0, + clog_ind = 1, + hlog_ind = 2, + slog_ind = 3, + slen_ind = 4, + tlen_ind = 5 +} varInds_t; + #define NUM_PARAMS 6 //just don't use strategy as a param. @@ -85,20 +86,16 @@ static const int g_maxNbVariations = 64; #define ZSTD_TARGETLENGTH_MIN 0 #define ZSTD_TARGETLENGTH_MAX 999 -//#define ZSTD_TARGETLENGTH_MAX 1024 #define WLOG_RANGE (ZSTD_WINDOWLOG_MAX - ZSTD_WINDOWLOG_MIN + 1) #define CLOG_RANGE (ZSTD_CHAINLOG_MAX - ZSTD_CHAINLOG_MIN + 1) #define HLOG_RANGE (ZSTD_HASHLOG_MAX - ZSTD_HASHLOG_MIN + 1) #define SLOG_RANGE (ZSTD_SEARCHLOG_MAX - ZSTD_SEARCHLOG_MIN + 1) #define SLEN_RANGE (ZSTD_SEARCHLENGTH_MAX - ZSTD_SEARCHLENGTH_MIN + 1) -#define TLEN_RANGE 12 -//TLEN_RANGE = 0, 2^0 to 2^10; -//hard coded since we only use powers of 2 (and 999 ~ 1024) +#define TLEN_RANGE 17 +/* TLEN_RANGE picked manually */ -//static const int mintable[NUM_PARAMS] = { ZSTD_WINDOWLOG_MIN, ZSTD_CHAINLOG_MIN, ZSTD_HASHLOG_MIN, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLENGTH_MIN, ZSTD_TARGETLENGTH_MIN }; -//static const int maxtable[NUM_PARAMS] = { ZSTD_WINDOWLOG_MAX, ZSTD_CHAINLOG_MAX, ZSTD_HASHLOG_MAX, ZSTD_SEARCHLOG_MAX, ZSTD_SEARCHLENGTH_MAX, ZSTD_TARGETLENGTH_MAX }; static const int rangetable[NUM_PARAMS] = { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE }; - +static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 }; /*-************************************ * Benchmark Parameters **************************************/ @@ -179,9 +176,6 @@ static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) return result; } -//assume that clock can at least measure .01 second intervals? -//make this a settable global initialized with fn? -//#define CLOCK_GRANULARITY 100000000ULL static U64 g_clockGranularity = 100000000ULL; static void findClockGranularity(void) { @@ -253,7 +247,7 @@ static void BMK_translateAdvancedParams(const ZSTD_compressionParameters params) /* checks results are feasible */ static int feasible(const BMK_result_t results, const constraint_t target) { - return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem || !target.cMem); + return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem); } #define EPSILON 0.001 @@ -268,13 +262,6 @@ static int eqZero(const double c1) { return !(memcmp(&c1, &z1, sizeof(double))) || !(memcmp(&c1, &z2, sizeof(double))); } -/* returns 1 if result2 is strictly 'better' than result1 */ -/* strict comparison / cutoff based */ -static int objective_lt(const BMK_result_t result1, const BMK_result_t result2) { - return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed) - || (result1.cSize == result2.cSize && epsilonEqual(result2.cSpeed, result1.cSpeed) && result2.dSpeed > result1.dSpeed); -} - /* hill climbing value for part 1 */ static double resultScore(const BMK_result_t res, const size_t srcSize, const constraint_t target) { double cs = 0., ds = 0., rt, cm = 0.; @@ -291,6 +278,16 @@ static double resultScore(const BMK_result_t res, const size_t srcSize, const co return ret; } +/* return true if r2 strictly better than r1 */ +static int compareResultLT(const BMK_result_t result1, const BMK_result_t result2, const constraint_t target, size_t srcSize) { + if(feasible(result1, target) && feasible(result2, target)) { + return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed) + || (result1.cSize == result2.cSize && epsilonEqual(result2.cSpeed, result1.cSpeed) && result2.dSpeed > result1.dSpeed); + } + return feasible(result2, target) || (!feasible(result1, target) && (resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target))); + +} + /* factor sort of arbitrary */ static constraint_t relaxTarget(constraint_t target) { target.cMem = (U32)-1; @@ -303,35 +300,11 @@ static constraint_t relaxTarget(constraint_t target) { * Bench functions *********************************************************/ -typedef struct -{ - const char* srcPtr; - size_t srcSize; - char* cPtr; - size_t cRoom; - size_t cSize; - char* resPtr; - size_t resSize; -} blockParam_t; - - const char* g_stratName[ZSTD_btultra+1] = { "(none) ", "ZSTD_fast ", "ZSTD_dfast ", "ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ", "ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "}; -static int -BMK_benchParam(BMK_result_t* resultPtr, - const void* srcBuffer, const size_t srcSize, - const size_t* fileSizes, const unsigned nbFiles, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, - const ZSTD_compressionParameters cParams) { - - BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, fileSizes, nbFiles, 0, &cParams, NULL, 0, ctx, dctx, 0, "File"); - *resultPtr = res.result; - return res.error; -} - /* benchParam but only takes in one file. */ static int BMK_benchParam1(BMK_result_t* resultPtr, @@ -349,6 +322,49 @@ typedef struct { ZSTD_compressionParameters params; } winnerInfo_t; +static ZSTD_compressionParameters emptyParams(void) { + ZSTD_compressionParameters p = { 0, 0, 0, 0, 0, 0, (ZSTD_strategy)0 }; + return p; +} + +static winnerInfo_t initWinnerInfo(ZSTD_compressionParameters p) { + winnerInfo_t w1; + w1.result.cSpeed = 0.; + w1.result.dSpeed = 0.; + w1.result.cMem = (size_t)-1; + w1.result.cSize = (size_t)-1; + w1.params = p; + return w1; +} + +typedef struct { + size_t srcSize; + void** srcPtrs; + size_t* srcSizes; + void** dstPtrs; + size_t* dstCapacities; + size_t* dstSizes; + void** resPtrs; + size_t* resSizes; + size_t nbBlocks; +} buffers_t; + +typedef struct { + size_t dictSize; + void* dictBuffer; + ZSTD_CCtx* cctx; + ZSTD_DCtx* dctx; +} contexts_t; + +static int +BMK_benchParam(BMK_result_t* resultPtr, + buffers_t buf, contexts_t ctx, + const ZSTD_compressionParameters cParams) { + BMK_return_t res = BMK_benchMem(buf.srcPtrs[0], buf.srcSize, buf.srcSizes, (unsigned)buf.nbBlocks, 0, &cParams, ctx.dictBuffer, ctx.dictSize, ctx.cctx, ctx.dctx, 0, "Files"); + *resultPtr = res.result; + return res.error; +} + /*-******************************************************* * From Paramgrill *********************************************************/ @@ -356,6 +372,8 @@ typedef struct { static void BMK_initCCtx(ZSTD_CCtx* ctx, const void* dictBuffer, size_t dictBufferSize, int cLevel, const ZSTD_compressionParameters* comprParams, const BMK_advancedParams_t* adv) { + ZSTD_CCtx_reset(ctx); + ZSTD_CCtx_resetParameters(ctx); if (adv->nbWorkers==1) { ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, 0); } else { @@ -380,6 +398,7 @@ static void BMK_initCCtx(ZSTD_CCtx* ctx, static void BMK_initDCtx(ZSTD_DCtx* dctx, const void* dictBuffer, size_t dictBufferSize) { + ZSTD_DCtx_reset(dctx); ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize); } @@ -425,6 +444,7 @@ static size_t local_defaultCompress( out.dst = dstBuffer; out.size = dstSize; out.pos = 0; + assert(dstSize == ZSTD_compressBound(srcSize)); /* specific to this version, which is only used in paramgrill */ while (moreToFlush) { if(out.pos == out.size) { return (size_t)-ZSTD_error_dstSize_tooSmall; @@ -471,249 +491,164 @@ static size_t local_defaultDecompress( *********************************************************/ /* Replicate function of benchMemAdvanced, but with pre-split src / dst buffers, with relevant info to invert it (compressedSizes) passed out. */ -/*BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); */ +/* BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); */ /* nbSeconds used in same way as in BMK_advancedParams_t, as nbIters when in iterMode */ /* if in decodeOnly, then srcPtr's will be compressed blocks, and uncompressedBlocks will be written to dstPtrs? */ /* dictionary nullable, nothing else though. */ -static BMK_return_t BMK_benchMemInvertible(const void * const * const srcPtrs, size_t const * const srcSizes, - void** dstPtrs, size_t* dstCapacityToSizes, U32 const nbBlocks, +static BMK_return_t BMK_benchMemInvertible(buffers_t buf, contexts_t ctx, const int cLevel, const ZSTD_compressionParameters* comprParams, - const void* dictBuffer, const size_t dictBufferSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const BMK_mode_t mode, const BMK_loopMode_t loopMode, const unsigned nbSeconds) { + U32 i; BMK_return_t results = { { 0, 0., 0., 0 }, 0 } ; - size_t srcSize = 0; - void** const resPtrs = malloc(sizeof(void*) * nbBlocks); /* only really needed in both mode. */ - size_t* const resSizes = malloc(sizeof(size_t) * nbBlocks); - int freeDST = 0; + const void *const *const srcPtrs = (const void *const *const)buf.srcPtrs; + size_t const *const srcSizes = buf.srcSizes; + void** dstPtrs = buf.dstPtrs; + size_t* dstCapacities = buf.dstCapacities; + size_t* dstSizes = buf.dstSizes; + void** resPtrs = buf.resPtrs; + size_t* resSizes = buf.resSizes; + const void* dictBuffer = ctx.dictBuffer; + const size_t dictBufferSize = ctx.dictSize; + const size_t nbBlocks = buf.nbBlocks; + const size_t srcSize = buf.srcSize; + ZSTD_CCtx* cctx = ctx.cctx; + ZSTD_DCtx* dctx = ctx.dctx; BMK_advancedParams_t adv = BMK_initAdvancedParams(); adv.mode = mode; adv.loopMode = loopMode; adv.nbSeconds = nbSeconds; - /* resSizes == srcSizes, but modifiable */ - memcpy(resSizes, srcSizes, sizeof(size_t) * nbBlocks); - - for(i = 0; i < nbBlocks; i++) { - srcSize += srcSizes[i]; - } - - if(!ctx || !dctx || !srcPtrs || ! srcSizes) - { - results.error = 31; - DISPLAY("error: passed in null argument\n"); - free(resPtrs); - free(resSizes); - return results; - } - if(!resPtrs || !resSizes) { - results.error = 32; - DISPLAY("error: allocation failed\n"); - free(resPtrs); - free(resSizes); - return results; - } - - /* so resPtr is continuous */ - resPtrs[0] = malloc(srcSize); - - if(!(resPtrs[0])) { - results.error = 32; - DISPLAY("error: allocation failed\n"); - free(resPtrs); - free(resSizes); - return results; - } - - for(i = 1; i < nbBlocks; i++) { - resPtrs[i] = (void*)(((char*)resPtrs[i-1]) + srcSizes[i-1]); - } - - /* allocate own dst if NULL */ - if(dstPtrs == NULL) { - freeDST = 1; - dstPtrs = malloc(nbBlocks * sizeof(void*)); - dstCapacityToSizes = malloc(nbBlocks * sizeof(size_t)); - if(dstPtrs == NULL) { - results.error = 33; - DISPLAY("error: allocation failed\n"); - free(resPtrs); - free(resSizes); - return results; - } - - if(mode == BMK_decodeOnly) { //dst is src - size_t dstSize = 0; - for(i = 0; i < nbBlocks; i++) { - dstCapacityToSizes[i] = ZSTD_getDecompressedSize(srcPtrs[i], srcSizes[i]); - dstSize += dstCapacityToSizes[i]; - } - dstPtrs[0] = malloc(dstSize); - if(dstPtrs[0] == NULL) { - results.error = 34; - DISPLAY("error: allocation failed\n"); - goto _cleanUp; - } - for(i = 1; i < nbBlocks; i++) { - dstPtrs[i] = (void*)(((char*)dstPtrs[i-1]) + ZSTD_getDecompressedSize(srcPtrs[i-1], srcSizes[i-1])); - } - } else { - dstPtrs[0] = malloc(ZSTD_compressBound(srcSize) + (nbBlocks * 1024)); - if(dstPtrs[0] == NULL) { - results.error = 35; - DISPLAY("error: allocation failed\n"); - goto _cleanUp; - } - dstCapacityToSizes[0] = ZSTD_compressBound(srcSizes[0]); - for(i = 1; i < nbBlocks; i++) { - dstPtrs[i] = (void*)(((char*)dstPtrs[i-1]) + dstCapacityToSizes[i-1]); - dstCapacityToSizes[i] = ZSTD_compressBound(srcSizes[i]); - } - } - } - /* warmimg up memory */ - for(i = 0; i < nbBlocks; i++) { - RDG_genBuffer(dstPtrs[i], dstCapacityToSizes[i], 0.10, 0.50, 1); + /* can't do this if decode only */ + for(i = 0; i < buf.nbBlocks; i++) { + if(mode != BMK_decodeOnly) { + RDG_genBuffer(dstPtrs[i], dstCapacities[i], 0.10, 0.50, 1); + } else { + RDG_genBuffer(resPtrs[i], resSizes[i], 0.10, 0.50, 1); + } } /* Bench */ - { - { - BMK_initCCtxArgs cctxprep; - BMK_initDCtxArgs dctxprep; - cctxprep.ctx = ctx; - cctxprep.dictBuffer = dictBuffer; - cctxprep.dictBufferSize = dictBufferSize; - cctxprep.cLevel = cLevel; - cctxprep.comprParams = comprParams; - cctxprep.adv = &adv; - dctxprep.dctx = dctx; - dctxprep.dictBuffer = dictBuffer; - dctxprep.dictBufferSize = dictBufferSize; - if(loopMode == BMK_timeMode) { - BMK_customTimedReturn_t intermediateResultCompress; - BMK_customTimedReturn_t intermediateResultDecompress; - BMK_timedFnState_t* timeStateCompress = BMK_createTimeState(nbSeconds); - BMK_timedFnState_t* timeStateDecompress = BMK_createTimeState(nbSeconds); - if(mode == BMK_compressOnly) { - intermediateResultCompress.completed = 0; - intermediateResultDecompress.completed = 1; - } else if (mode == BMK_decodeOnly) { - intermediateResultCompress.completed = 1; - intermediateResultDecompress.completed = 0; - } else { /* both */ - intermediateResultCompress.completed = 0; - intermediateResultDecompress.completed = 0; + + { + /* init args */ + BMK_initCCtxArgs cctxprep; + BMK_initDCtxArgs dctxprep; + cctxprep.ctx = cctx; + cctxprep.dictBuffer = dictBuffer; + cctxprep.dictBufferSize = dictBufferSize; + cctxprep.cLevel = cLevel; + cctxprep.comprParams = comprParams; + cctxprep.adv = &adv; + dctxprep.dctx = dctx; + dctxprep.dictBuffer = dictBuffer; + dctxprep.dictBufferSize = dictBufferSize; + + if(loopMode == BMK_timeMode) { + BMK_customTimedReturn_t intermediateResultCompress; + BMK_customTimedReturn_t intermediateResultDecompress; + BMK_timedFnState_t* timeStateCompress = BMK_createTimeState(nbSeconds); + BMK_timedFnState_t* timeStateDecompress = BMK_createTimeState(nbSeconds); + if(mode == BMK_compressOnly) { + intermediateResultCompress.completed = 0; + intermediateResultDecompress.completed = 1; + } else if (mode == BMK_decodeOnly) { + intermediateResultCompress.completed = 1; + intermediateResultDecompress.completed = 0; + } else { /* both */ + intermediateResultCompress.completed = 0; + intermediateResultDecompress.completed = 0; + } + + while(!intermediateResultCompress.completed) { + intermediateResultCompress = BMK_benchFunctionTimed(timeStateCompress, &local_defaultCompress, (void*)cctx, &local_initCCtx, (void*)&cctxprep, + nbBlocks, srcPtrs, srcSizes, dstPtrs, dstCapacities, dstSizes); + + if(intermediateResultCompress.result.error) { + results.error = intermediateResultCompress.result.error; + BMK_freeTimeState(timeStateCompress); + BMK_freeTimeState(timeStateDecompress); + return results; } - while(!(intermediateResultCompress.completed && intermediateResultDecompress.completed)) { - if(!intermediateResultCompress.completed) { - intermediateResultCompress = BMK_benchFunctionTimed(timeStateCompress, &local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep, - nbBlocks, srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes); - if(intermediateResultCompress.result.error) { - results.error = intermediateResultCompress.result.error; - BMK_freeTimeState(timeStateCompress); - BMK_freeTimeState(timeStateDecompress); - goto _cleanUp; - } - results.result.cSpeed = ((double)srcSize / intermediateResultCompress.result.result.nanoSecPerRun) * 1000000000; - results.result.cSize = intermediateResultCompress.result.result.sumOfReturn; - } + results.result.cSpeed = ((double)srcSize / intermediateResultCompress.result.result.nanoSecPerRun) * TIMELOOP_NANOSEC; + results.result.cSize = intermediateResultCompress.result.result.sumOfReturn; + } - if(!intermediateResultDecompress.completed) { - if(mode == BMK_decodeOnly) { - intermediateResultDecompress = BMK_benchFunctionTimed(timeStateDecompress, &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep, - nbBlocks, (const void* const*)srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes); - } else { /* both, decompressed result already written to dstPtr */ - intermediateResultDecompress = BMK_benchFunctionTimed(timeStateDecompress, &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep, - nbBlocks, (const void* const*)dstPtrs, dstCapacityToSizes, resPtrs, resSizes); - } + while(!intermediateResultDecompress.completed) { + intermediateResultDecompress = BMK_benchFunctionTimed(timeStateDecompress, &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep, + nbBlocks, (const void* const*)dstPtrs, dstSizes, resPtrs, resSizes, NULL); - if(intermediateResultDecompress.result.error) { - results.error = intermediateResultDecompress.result.error; - BMK_freeTimeState(timeStateCompress); - BMK_freeTimeState(timeStateDecompress); - goto _cleanUp; - } - results.result.dSpeed = ((double)srcSize / intermediateResultDecompress.result.result.nanoSecPerRun) * 1000000000; - } + if(intermediateResultDecompress.result.error) { + results.error = intermediateResultDecompress.result.error; + BMK_freeTimeState(timeStateCompress); + BMK_freeTimeState(timeStateDecompress); + return results; } - BMK_freeTimeState(timeStateCompress); - BMK_freeTimeState(timeStateDecompress); - } else { //iterMode; - if(mode != BMK_decodeOnly) { + results.result.dSpeed = ((double)srcSize / intermediateResultDecompress.result.result.nanoSecPerRun) * TIMELOOP_NANOSEC; + } - BMK_customReturn_t compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep, - nbBlocks, srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbSeconds); - if(compressionResults.error) { - results.error = compressionResults.error; - goto _cleanUp; - } - if(compressionResults.result.nanoSecPerRun == 0) { - results.result.cSpeed = 0; - } else { - results.result.cSpeed = (double)srcSize / compressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; - } - results.result.cSize = compressionResults.result.sumOfReturn; + BMK_freeTimeState(timeStateCompress); + BMK_freeTimeState(timeStateDecompress); + + } else { //iterMode; + if(mode != BMK_decodeOnly) { + + BMK_customReturn_t compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)cctx, &local_initCCtx, (void*)&cctxprep, + nbBlocks, srcPtrs, srcSizes, dstPtrs, dstCapacities, dstSizes, nbSeconds); + if(compressionResults.error) { + results.error = compressionResults.error; + return results; } - if(mode != BMK_compressOnly) { - BMK_customReturn_t decompressionResults; - if(mode == BMK_decodeOnly) { - decompressionResults = BMK_benchFunction( - &local_defaultDecompress, (void*)(dctx), - &local_initDCtx, (void*)&dctxprep, nbBlocks, - (const void* const*)srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, - nbSeconds); - } else { - decompressionResults = BMK_benchFunction( - &local_defaultDecompress, (void*)(dctx), - &local_initDCtx, (void*)&dctxprep, nbBlocks, - (const void* const*)dstPtrs, dstCapacityToSizes, resPtrs, resSizes, - nbSeconds); - } + if(compressionResults.result.nanoSecPerRun == 0) { + results.result.cSpeed = 0; + } else { + results.result.cSpeed = (double)srcSize / compressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; + } + results.result.cSize = compressionResults.result.sumOfReturn; + } - if(decompressionResults.error) { - results.error = decompressionResults.error; - goto _cleanUp; - } + if(mode != BMK_compressOnly) { + BMK_customReturn_t decompressionResults; + decompressionResults = BMK_benchFunction( + &local_defaultDecompress, (void*)(dctx), + &local_initDCtx, (void*)&dctxprep, nbBlocks, + (const void* const*)dstPtrs, dstSizes, resPtrs, resSizes, NULL, + nbSeconds); - if(decompressionResults.result.nanoSecPerRun == 0) { - results.result.dSpeed = 0; - } else { - results.result.dSpeed = (double)srcSize / decompressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; - } + if(decompressionResults.error) { + results.error = decompressionResults.error; + return results; + } + + if(decompressionResults.result.nanoSecPerRun == 0) { + results.result.dSpeed = 0; + } else { + results.result.dSpeed = (double)srcSize / decompressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; } } } - } /* Bench */ - results.result.cMem = (1 << (comprParams->windowLog)) + ZSTD_sizeof_CCtx(ctx); - -_cleanUp: - free(resPtrs[0]); - free(resPtrs); - free(resSizes); - if(freeDST) { - free(dstPtrs[0]); - free(dstPtrs); } + /* Bench */ + results.result.cMem = (1 << (comprParams->windowLog)) + ZSTD_sizeof_CCtx(cctx); return results; } /* global winner used for display. */ //Should be totally 0 initialized? -static winnerInfo_t g_winner; +static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; static constraint_t g_targetConstraints; static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const size_t srcSize) { - if(DEBUG || (objective_lt(g_winner.result, result) && feasible(result, g_targetConstraints))) { + if(DEBUG || compareResultLT(g_winner.result, result, g_targetConstraints, srcSize)) { char lvlstr[15] = "Custom Level"; const U64 time = UTIL_clockSpanNano(g_time); const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC); - if(DEBUG && (objective_lt(g_winner.result, result) && feasible(result, g_targetConstraints))) { + + if(DEBUG && compareResultLT(g_winner.result, result, g_targetConstraints, srcSize)) { DISPLAY("New Winner: \n"); } @@ -722,27 +657,24 @@ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, params.targetLength, g_stratName[(U32)(params.strategy)]); + if(cLevel != CUSTOM_LEVEL) { snprintf(lvlstr, 15, " Level %2u ", cLevel); } + fprintf(f, "/* %s */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */", - lvlstr, (double)srcSize / result.cSize, result.cSpeed / 1000000., result.dSpeed / 1000000.); + lvlstr, (double)srcSize / result.cSize, result.cSpeed / (1 << 20), result.dSpeed / (1 << 20)); if(TIMED) { fprintf(f, " - %1lu:%2lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); } fprintf(f, "\n"); - if(objective_lt(g_winner.result, result) && feasible(result, g_targetConstraints)) { + + if(compareResultLT(g_winner.result, result, g_targetConstraints, srcSize)) { BMK_translateAdvancedParams(params); g_winner.result = result; g_winner.params = params; } } - //else { - // DISPLAY("G_WINNER: "); - // DISPLAY("/* R:%5.3f at %5.1f MB/s - %5.1f MB/s */ \n",(double)srcSize / g_winner.result.cSize , g_winner.result.cSpeed / 1000000 , g_winner.result.dSpeed / 1000000); - // DISPLAY("LOSER : "); - // DISPLAY("/* R:%5.3f at %5.1f MB/s - %5.1f MB/s */ \n",(double)srcSize / result.cSize, result.cSpeed / 1000000 , result.dSpeed / 1000000); - //} } static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize) @@ -903,23 +835,23 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para static ZSTD_compressionParameters sanitizeParams(ZSTD_compressionParameters params) { if (params.strategy == ZSTD_fast) - g_params.chainLog = 0, g_params.searchLog = 0; + params.chainLog = 0, params.searchLog = 0; if (params.strategy == ZSTD_dfast) - g_params.searchLog = 0; + params.searchLog = 0; if (params.strategy != ZSTD_btopt && params.strategy != ZSTD_btultra && params.strategy != ZSTD_fast) - g_params.targetLength = 0; + params.targetLength = 0; return params; } /* new length */ /* keep old array, will need if iter over strategy. */ -static int sanitizeVarArray(const int varLength, const U32* varArray, U32* varNew, const ZSTD_strategy strat) { +static int sanitizeVarArray(varInds_t* varNew, const int varLength, const varInds_t* varArray, const ZSTD_strategy strat) { int i, j = 0; for(i = 0; i < varLength; i++) { - if( !((varArray[i] == CLOG_IND && strat == ZSTD_fast) - || (varArray[i] == SLOG_IND && strat == ZSTD_dfast) - || (varArray[i] == TLEN_IND && strat != ZSTD_btopt && strat != ZSTD_btultra && strat != ZSTD_fast))) { + if( !((varArray[i] == clog_ind && strat == ZSTD_fast) + || (varArray[i] == slog_ind && strat == ZSTD_dfast) + || (varArray[i] == tlen_ind && strat != ZSTD_btopt && strat != ZSTD_btultra && strat != ZSTD_fast))) { varNew[j] = varArray[i]; j++; } @@ -930,69 +862,72 @@ static int sanitizeVarArray(const int varLength, const U32* varArray, U32* varNe /* res should be NUM_PARAMS size */ /* constructs varArray from ZSTD_compressionParameters style parameter */ -static int variableParams(const ZSTD_compressionParameters paramConstraints, U32* res) { +static int variableParams(const ZSTD_compressionParameters paramConstraints, varInds_t* res) { int j = 0; if(!paramConstraints.windowLog) { - res[j] = WLOG_IND; + res[j] = wlog_ind; j++; } if(!paramConstraints.chainLog) { - res[j] = CLOG_IND; + res[j] = clog_ind; j++; } if(!paramConstraints.hashLog) { - res[j] = HLOG_IND; + res[j] = hlog_ind; j++; } if(!paramConstraints.searchLog) { - res[j] = SLOG_IND; + res[j] = slog_ind; j++; } if(!paramConstraints.searchLength) { - res[j] = SLEN_IND; + res[j] = slen_ind; j++; } if(!paramConstraints.targetLength) { - res[j] = TLEN_IND; + res[j] = tlen_ind; j++; } return j; } +/* bin-search on tlen_table for correct index. */ +static int tlen_inv(U32 x) { + int lo = 0; + int hi = TLEN_RANGE; + while(lo < hi) { + int mid = (lo + hi) / 2; + if(tlen_table[mid] < x) { + lo = mid + 1; + } if(tlen_table[mid] == x) { + return mid; + } else { + hi = mid; + } + } + return lo; +} + /* amt will probably always be \pm 1? */ /* slight change from old paramVariation, targetLength can only take on powers of 2 now (999 ~= 1024?) */ /* take max/min bounds into account as well? */ -static void paramVaryOnce(const U32 paramIndex, const int amt, ZSTD_compressionParameters* ptr) { +static void paramVaryOnce(const varInds_t paramIndex, const int amt, ZSTD_compressionParameters* ptr) { switch(paramIndex) { - case WLOG_IND: ptr->windowLog += amt; break; - case CLOG_IND: ptr->chainLog += amt; break; - case HLOG_IND: ptr->hashLog += amt; break; - case SLOG_IND: ptr->searchLog += amt; break; - case SLEN_IND: ptr->searchLength += amt; break; - case TLEN_IND: - if(amt >= 0) { - if(ptr->targetLength == 0) { - if(amt > 0) { - ptr->targetLength = MIN(1 << (amt - 1), 999); - } - } else { - ptr->targetLength <<= amt; - ptr->targetLength = MIN(ptr->targetLength, 999); - } - } else { - if(ptr->targetLength == 999) { - ptr->targetLength = 1024; - } - ptr->targetLength >>= -amt; - } + case wlog_ind: ptr->windowLog += amt; break; + case clog_ind: ptr->chainLog += amt; break; + case hlog_ind: ptr->hashLog += amt; break; + case slog_ind: ptr->searchLog += amt; break; + case slen_ind: ptr->searchLength += amt; break; + case tlen_ind: + ptr->targetLength = tlen_table[MAX(0, MIN(TLEN_RANGE - 1, tlen_inv(ptr->targetLength) + amt))]; break; default: break; } } /* varies ptr by nbChanges respecting varyParams*/ -static void paramVariation(ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen, const U32 nbChanges) +static void paramVariation(ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen, const U32 nbChanges) { ZSTD_compressionParameters p; U32 validated = 0; @@ -1009,7 +944,7 @@ static void paramVariation(ZSTD_compressionParameters* ptr, const U32* varyParam } /* length of memo table given free variables */ -static size_t memoTableLen(const U32* varyParams, const int varyLen) { +static size_t memoTableLen(const varInds_t* varyParams, const int varyLen) { size_t arrayLen = 1; int i; for(i = 0; i < varyLen; i++) { @@ -1018,53 +953,45 @@ static size_t memoTableLen(const U32* varyParams, const int varyLen) { return arrayLen; } -//sort of ~lg2 (replace 1024 w/ 999, and add 0 at lower end of range) for memoTableInd Tlen -static unsigned lg2(unsigned x) { - if(x == 999) { - return 11; - } - return x ? ZSTD_highbit32(x) + 1 : 0; -} - /* returns unique index of compression parameters */ -static unsigned memoTableInd(const ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen) { +static unsigned memoTableInd(const ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen) { int i; unsigned ind = 0; for(i = 0; i < varyLen; i++) { switch(varyParams[i]) { - case WLOG_IND: ind *= WLOG_RANGE; ind += ptr->windowLog - ZSTD_WINDOWLOG_MIN ; break; - case CLOG_IND: ind *= CLOG_RANGE; ind += ptr->chainLog - ZSTD_CHAINLOG_MIN ; break; - case HLOG_IND: ind *= HLOG_RANGE; ind += ptr->hashLog - ZSTD_HASHLOG_MIN ; break; - case SLOG_IND: ind *= SLOG_RANGE; ind += ptr->searchLog - ZSTD_SEARCHLOG_MIN ; break; - case SLEN_IND: ind *= SLEN_RANGE; ind += ptr->searchLength - ZSTD_SEARCHLENGTH_MIN; break; - case TLEN_IND: ind *= TLEN_RANGE; ind += lg2(ptr->targetLength) - ZSTD_TARGETLENGTH_MIN; break; + case wlog_ind: ind *= WLOG_RANGE; ind += ptr->windowLog - ZSTD_WINDOWLOG_MIN ; break; + case clog_ind: ind *= CLOG_RANGE; ind += ptr->chainLog - ZSTD_CHAINLOG_MIN ; break; + case hlog_ind: ind *= HLOG_RANGE; ind += ptr->hashLog - ZSTD_HASHLOG_MIN ; break; + case slog_ind: ind *= SLOG_RANGE; ind += ptr->searchLog - ZSTD_SEARCHLOG_MIN ; break; + case slen_ind: ind *= SLEN_RANGE; ind += ptr->searchLength - ZSTD_SEARCHLENGTH_MIN; break; + case tlen_ind: ind *= TLEN_RANGE; ind += tlen_inv(ptr->targetLength) - ZSTD_TARGETLENGTH_MIN; break; } } return ind; } /* inverse of above function (from index to parameters) */ -static void memoTableIndInv(ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen, size_t ind) { +static void memoTableIndInv(ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen, size_t ind) { int i; for(i = varyLen - 1; i >= 0; i--) { switch(varyParams[i]) { - case WLOG_IND: ptr->windowLog = ind % WLOG_RANGE + ZSTD_WINDOWLOG_MIN; ind /= WLOG_RANGE; break; - case CLOG_IND: ptr->chainLog = ind % CLOG_RANGE + ZSTD_CHAINLOG_MIN; ind /= CLOG_RANGE; break; - case HLOG_IND: ptr->hashLog = ind % HLOG_RANGE + ZSTD_HASHLOG_MIN; ind /= HLOG_RANGE; break; - case SLOG_IND: ptr->searchLog = ind % SLOG_RANGE + ZSTD_SEARCHLOG_MIN; ind /= SLOG_RANGE; break; - case SLEN_IND: ptr->searchLength = ind % SLEN_RANGE + ZSTD_SEARCHLENGTH_MIN; ind /= SLEN_RANGE; break; - case TLEN_IND: ptr->targetLength = (ind % TLEN_RANGE) ? MIN(1 << ((ind % TLEN_RANGE) - 1), 999) : 0; ind /= TLEN_RANGE; break; + case wlog_ind: ptr->windowLog = ind % WLOG_RANGE + ZSTD_WINDOWLOG_MIN; ind /= WLOG_RANGE; break; + case clog_ind: ptr->chainLog = ind % CLOG_RANGE + ZSTD_CHAINLOG_MIN; ind /= CLOG_RANGE; break; + case hlog_ind: ptr->hashLog = ind % HLOG_RANGE + ZSTD_HASHLOG_MIN; ind /= HLOG_RANGE; break; + case slog_ind: ptr->searchLog = ind % SLOG_RANGE + ZSTD_SEARCHLOG_MIN; ind /= SLOG_RANGE; break; + case slen_ind: ptr->searchLength = ind % SLEN_RANGE + ZSTD_SEARCHLENGTH_MIN; ind /= SLEN_RANGE; break; + case tlen_ind: ptr->targetLength = tlen_table[(ind % TLEN_RANGE)]; ind /= TLEN_RANGE; break; } } } - /* Initialize memotable, immediately mark redundant / obviously infeasible params as */ -static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstraints, const constraint_t target, const U32* varyParams, const int varyLen, const size_t srcSize) { +static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { size_t i; size_t arrayLen = memoTableLen(varyParams, varyLen); int cwFixed = !paramConstraints.chainLog || !paramConstraints.windowLog; int scFixed = !paramConstraints.searchLog || !paramConstraints.chainLog; + int whFixed = !paramConstraints.windowLog || !paramConstraints.hashLog; int wFixed = !paramConstraints.windowLog; int j = 0; memset(memoTable, 0, arrayLen); @@ -1076,7 +1003,7 @@ static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstra memoTable[i] = 255; j++; } - if(wFixed && (1ULL << paramConstraints.windowLog) > (srcSize << 1)) { + if(wFixed && (1ULL << paramConstraints.windowLog) > (srcSize << 2)) { memoTable[i] = 255; } /* nil out parameter sets equivalent to others. */ @@ -1093,12 +1020,20 @@ static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstra } } } + if(scFixed) { if(paramConstraints.searchLog > paramConstraints.chainLog) { if(memoTable[i] != 255) { j++; } memoTable[i] = 255; } } + + if(whFixed) { + if(paramConstraints.hashLog > paramConstraints.windowLog + 1) { + if(memoTable[i] != 255) { j++; } + memoTable[i] = 255; + } + } } DEBUGOUTPUT("%d / %d Invalid\n", j, (int)i); if((int)i == j) { @@ -1106,27 +1041,7 @@ static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstra } } -/* inits memotables for all (including mallocs), all strategies */ -/* takes unsanitized varyParams */ -static U8** memoTableInitAll(ZSTD_compressionParameters paramConstraints, constraint_t target, const U32* varyParams, const int varyLen, const size_t srcSize) { - U32 varNew[NUM_PARAMS]; - int varLenNew; - U8** mtAll = malloc(sizeof(U8*) * (ZSTD_btultra + 1)); - int i; - if(mtAll == NULL) { - return NULL; - } - for(i = 1; i <= (int)ZSTD_btultra; i++) { - varLenNew = sanitizeVarArray(varyLen, varyParams, varNew, i); - mtAll[i] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew)); - if(mtAll[i] == NULL) { - return NULL; - } - memoTableInit(mtAll[i], paramConstraints, target, varNew, varLenNew, srcSize); - } - return mtAll; -} - +/* frees all allocated memotables */ static void memoTableFreeAll(U8** mtAll) { int i; if(mtAll == NULL) { return; } @@ -1136,6 +1051,28 @@ static void memoTableFreeAll(U8** mtAll) { free(mtAll); } +/* inits memotables for all (including mallocs), all strategies */ +/* takes unsanitized varyParams */ +static U8** memoTableInitAll(ZSTD_compressionParameters paramConstraints, constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { + varInds_t varNew[NUM_PARAMS]; + int varLenNew; + U8** mtAll = calloc(sizeof(U8*),(ZSTD_btultra + 1)); + int i; + if(mtAll == NULL) { + return NULL; + } + for(i = 1; i <= (int)ZSTD_btultra; i++) { + varLenNew = sanitizeVarArray(varNew, varyLen, varyParams, i); + mtAll[i] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew)); + if(mtAll[i] == NULL) { + memoTableFreeAll(mtAll); + return NULL; + } + memoTableInit(mtAll[i], paramConstraints, target, varNew, varLenNew, srcSize); + } + return mtAll; +} + #define PARAMTABLELOG 25 #define PARAMTABLESIZE (1< 0 && tries > 0); memoTableIndInv(pc, varArray, varLen, (unsigned)ind); - *pc = sanitizeParams(*pc); + //*pc = sanitizeParams(*pc); } static void BMK_selectRandomStart( @@ -1250,7 +1187,7 @@ static void BMK_benchFullTable(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* src winnerInfo_t winners[NB_LEVELS_TRACKED+1]; const char* const rfName = "grillResults.txt"; FILE* const f = fopen(rfName, "w"); - const size_t blockSize = g_blockSize ? g_blockSize : srcSize; /* cut by block or not ? */ + const size_t blockSize = g_blockSize ? g_blockSize : ZSTD_BLOCKSIZE_MAX; /* cut by block or not ? */ /* init */ assert(g_singleRun==0); @@ -1390,574 +1327,204 @@ int benchFiles(const char** fileNamesTable, int nbFiles) return 0; } - - -/* -checks feasibility with uncertainty. --1 : certainly infeasible - 0 : uncertain - 1 : certainly feasible -*/ -static int uncertainFeasibility(double const uncertaintyConstantC, double const uncertaintyConstantD, const constraint_t paramTarget, const BMK_result_t* const results) { - if((paramTarget.cSpeed != 0 && results->cSpeed * uncertaintyConstantC < paramTarget.cSpeed) || - (paramTarget.dSpeed != 0 && results->dSpeed * uncertaintyConstantD < paramTarget.dSpeed) || - (paramTarget.cMem != 0 && results->cMem > paramTarget.cMem)) { - return -1; - } else if((paramTarget.cSpeed == 0 || results->cSpeed / uncertaintyConstantC > paramTarget.cSpeed) && - (paramTarget.dSpeed == 0 || results->dSpeed / uncertaintyConstantD > paramTarget.dSpeed) && - (paramTarget.cMem == 0 || results->cMem <= paramTarget.cMem)) { - return 1; - } else { - return 0; - } -} - -/* 1 - better than prev best - 0 - uncertain - -1 - worse - assume prev_best status is run fully? - but then we'd have to rerun any winners anyway */ -/* not as useful as initially believed */ -static int uncertainComparison(double const uncertaintyConstantC, double const uncertaintyConstantD, const BMK_result_t* candidate, const BMK_result_t* prevBest) { - (void)uncertaintyConstantD; //unused for now - if(candidate->cSpeed > prevBest->cSpeed * uncertaintyConstantC) { - return 1; - } else if (candidate->cSpeed * uncertaintyConstantC < prevBest->cSpeed) { - return -1; - } else { - return 0; - } -} - /*benchmarks and tests feasibility together 1 = true = better 0 = false = not better if true then resultPtr will give results. 2+ on error? */ -//Maybe use compress_only for benchmark -#define INFEASIBLE_RESULT 0 -#define FEASIBLE_RESULT 1 +//Maybe use compress_only for benchmark first run? +#define WORSE_RESULT 0 +#define BETTER_RESULT 1 #define ERROR_RESULT 2 -static int feasibleBench(BMK_result_t* resultPtr, - const void* const * const srcPtrs, size_t const * const srcSizes, - void** const dstPtrs, size_t* dstCapacityToSizes, U32 const nbBlocks, - void* dictBuffer, const size_t dictBufferSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, - const ZSTD_compressionParameters cParams, - const constraint_t target, - BMK_result_t* winnerResult) { - BMK_return_t benchres; - U64 loopDurationC = 0, loopDurationD = 0; - double uncertaintyConstantC, uncertaintyConstantD; - size_t srcSize = 0; - U32 i; - //alternative - test 1 iter for ratio, (possibility of error 3 which is fine), - //maybe iter this until 2x measurable for better guarantee? - DEBUGOUTPUT("Feas:\n"); - benchres = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_both, BMK_iterMode, 1); - if(benchres.error) { - DISPLAY("ERROR %d !!\n", benchres.error); - } - for(i = 0; i < nbBlocks; i++) { - srcSize += srcSizes[i]; - } - BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); +//add worse result complete for worse results of length > 1 sec? - if(!benchres.error) { - *resultPtr = benchres.result; - /* if speed is 0 (only happens when time = 0) */ - if(eqZero(benchres.result.cSpeed)) { - loopDurationC = 0; - uncertaintyConstantC = 2; - } else { - loopDurationC = (U64)((double)(srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); - //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? - //possibly has to do with initCCtx? or system stuff? - //asymmetric +/- constant needed? - uncertaintyConstantC = MIN((loopDurationC + (double)(2 * g_clockGranularity)/loopDurationC) * 1.1, 3); //.02 seconds - } - if(eqZero(benchres.result.dSpeed)) { - loopDurationD = 0; - uncertaintyConstantD = 2; - } else { - loopDurationD = (U64)((double)(srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); - //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? - //possibly has to do with initCCtx? or system stuff? - //asymmetric +/- constant needed? - uncertaintyConstantD = MIN((loopDurationD + (double)(2 * g_clockGranularity)/loopDurationD) * 1.1, 3); //.02 seconds - } - - - if(benchres.result.cSize < winnerResult->cSize) { //better compression ratio, just needs to be feasible - int feas; - if(loopDurationC < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_compressOnly, BMK_iterMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres = benchres2; - } - } - if(loopDurationD < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_decodeOnly, BMK_iterMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres.result.dSpeed = benchres2.result.dSpeed; - } - } - *resultPtr = benchres.result; - - feas = uncertainFeasibility(uncertaintyConstantC, uncertaintyConstantD, target, &(benchres.result)); - if(feas == 0) { // uncertain feasibility - if(loopDurationC < TIMELOOP_NANOSEC) { - BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_compressOnly, BMK_timeMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres.result.cSpeed = benchres2.result.cSpeed; - } - } - if(loopDurationD < TIMELOOP_NANOSEC) { - BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_decodeOnly, BMK_timeMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres.result.dSpeed = benchres2.result.dSpeed; - } - } - *resultPtr = benchres.result; - return feasible(benchres.result, target); - } else { //feas = 1 or -1 map to 1, 0 respectively - return (feas + 1) >> 1; //relies on INFEASIBLE_RESULT == 0, FEASIBLE_RESULT == 1 - } - } else if (benchres.result.cSize == winnerResult->cSize) { //equal ratio, needs to be better than winner in cSpeed/ dSpeed / cMem - int feas; - if(loopDurationC < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_compressOnly, BMK_iterMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres = benchres2; - } - } - if(loopDurationD < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_decodeOnly, BMK_iterMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres.result.dSpeed = benchres2.result.dSpeed; - } - } - feas = uncertainFeasibility(uncertaintyConstantC, uncertaintyConstantD, target, &(benchres.result)); - if(feas == 0) { // uncertain feasibility - if(loopDurationC < TIMELOOP_NANOSEC) { - BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_compressOnly, BMK_timeMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres.result.cSpeed = benchres2.result.cSpeed; - } - } - if(loopDurationD < TIMELOOP_NANOSEC) { - BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_decodeOnly, BMK_timeMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres.result.dSpeed = benchres2.result.dSpeed; - } - } - - *resultPtr = benchres.result; - return feasible(benchres.result, target) && objective_lt(*winnerResult, benchres.result); - } else if (feas == 1) { //no need to check feasibility compares (maybe only it is chosen as a winner) - int btw = uncertainComparison(uncertaintyConstantC, uncertaintyConstantD, &(benchres.result), winnerResult); - if(btw == -1) { - return INFEASIBLE_RESULT; - } else { //possibly better, benchmark and find out - benchres = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_both, BMK_timeMode, 1); - *resultPtr = benchres.result; - return objective_lt(*winnerResult, benchres.result); - } - } else { //feas == -1 - return INFEASIBLE_RESULT; //infeasible - } - } else { - return INFEASIBLE_RESULT; //infeasible - } - } else { - return ERROR_RESULT; //BMK error - } -} - -//same as before, but +/-? -//alternative, just return comparison result, leave caller to worry about feasibility. -//have version of benchMemAdvanced which takes in dstBuffer/cap as well? -//(motivation: repeat tests (maybe just on decompress) don't need further compress runs) -static int infeasibleBench(BMK_result_t* resultPtr, - const void* const * const srcPtrs, size_t const * const srcSizes, - void** const dstPtrs, size_t* dstCapacityToSizes, U32 const nbBlocks, - void* dictBuffer, const size_t dictBufferSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, +/* variation between 2nd run and full second bmk */ +#define VARIANCE 1.1 +static int allBench(BMK_result_t* resultPtr, + buffers_t buf, contexts_t ctx, const ZSTD_compressionParameters cParams, const constraint_t target, - BMK_result_t* winnerResult) { + BMK_result_t* winnerResult, int feas) { BMK_return_t benchres; - BMK_result_t resultMin, resultMax; + BMK_result_t resultMax; U64 loopDurationC = 0, loopDurationD = 0; double uncertaintyConstantC, uncertaintyConstantD; double winnerRS; - size_t srcSize = 0; - U32 i; - benchres = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_both, BMK_iterMode, 1); - for(i = 0; i < nbBlocks; i++) { - srcSize += srcSizes[i]; - } - BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); - winnerRS = resultScore(*winnerResult, srcSize, target); + /* initial benchmarking, gives exact ratio and memory, warms up future runs */ + benchres = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_both, BMK_iterMode, 1); + winnerRS = resultScore(*winnerResult, buf.srcSize, target); DEBUGOUTPUT("WinnerScore: %f\n ", winnerRS); - if(!benchres.error) { - *resultPtr = benchres.result; - if(eqZero(benchres.result.cSpeed)) { - loopDurationC = 0; - uncertaintyConstantC = 2; - } else { - loopDurationC = (U64)((double)(srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); - uncertaintyConstantC = MIN((loopDurationC + (double)(2 * g_clockGranularity)/loopDurationC * 1.1), 3); //.02 seconds - } + if(benchres.error) { + DEBUGOUTPUT("Benchmarking failed\n"); + return ERROR_RESULT; + } + *resultPtr = benchres.result; - if(eqZero(benchres.result.dSpeed)) { - loopDurationD = 0; - uncertaintyConstantD = 2; - } else { - loopDurationD = (U64)((double)(srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); - uncertaintyConstantD = MIN((loopDurationD + (double)(2 * g_clockGranularity)/loopDurationD) * 1.1 , 3); //.02 seconds - } + /* calculate uncertainty in compression / decompression runs */ + if(eqZero(benchres.result.cSpeed)) { + loopDurationC = 0; + uncertaintyConstantC = 3; + } else { + loopDurationC = (U64)((double)(buf.srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); + uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC) * VARIANCE; + } + if(eqZero(benchres.result.dSpeed)) { + loopDurationD = 0; + uncertaintyConstantD = 3; + } else { + loopDurationD = (U64)((double)(buf.srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); + uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD) * VARIANCE; + } - if(loopDurationC < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_compressOnly, BMK_iterMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres = benchres2; - } - } - if(loopDurationD < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_decodeOnly, BMK_iterMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres.result.dSpeed = benchres2.result.dSpeed; - } - } - *resultPtr = benchres.result; + /* anything with worse ratio in feas is definitely worse, discard */ + if(feas && benchres.result.cSize < winnerResult->cSize) { + return WORSE_RESULT; + } - /* benchres's certainty range. */ - resultMax = benchres.result; - resultMin = benchres.result; - resultMax.cSpeed *= uncertaintyConstantC; - resultMax.dSpeed *= uncertaintyConstantD; - resultMin.cSpeed /= uncertaintyConstantC; - resultMin.dSpeed /= uncertaintyConstantD; - if (winnerRS > resultScore(resultMax, srcSize, target)) { - return INFEASIBLE_RESULT; - } else { - if(loopDurationC < TIMELOOP_NANOSEC) { - BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_compressOnly, BMK_timeMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres.result.cSpeed = benchres2.result.cSpeed; - } - } - if(loopDurationD < TIMELOOP_NANOSEC) { - BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_decodeOnly, BMK_timeMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres.result.dSpeed = benchres2.result.dSpeed; - } - } - *resultPtr = benchres.result; - return (resultScore(benchres.result, srcSize, target) > winnerRS); + /* second run, if first run is too short, gives approximate cSpeed + dSpeed */ + if(loopDurationC < TIMELOOP_NANOSEC / 10) { + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_compressOnly, BMK_iterMode, 1); + if(benchres2.error) { + return ERROR_RESULT; } + benchres = benchres2; + } + if(loopDurationD < TIMELOOP_NANOSEC / 10) { + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_decodeOnly, BMK_iterMode, 1); + if(benchres2.error) { + return ERROR_RESULT; + } + benchres.result.dSpeed = benchres2.result.dSpeed; + } *resultPtr = benchres.result; - } else { - return ERROR_RESULT; //BMK error + + /* optimistic assumption of benchres.result */ + resultMax = benchres.result; + resultMax.cSpeed *= uncertaintyConstantC; + resultMax.dSpeed *= uncertaintyConstantD; + + /* disregard infeasible results in feas mode */ + /* disregard if resultMax < winner in infeas mode */ + if((feas && !feasible(resultMax, target)) || + (!feas && (winnerRS > resultScore(resultMax, buf.srcSize, target)))) { + return WORSE_RESULT; + } + + /* Final full run if estimates are unclear */ + if(loopDurationC < TIMELOOP_NANOSEC) { + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_compressOnly, BMK_timeMode, 1); + if(benchres2.error) { + return ERROR_RESULT; + } + benchres.result.cSpeed = benchres2.result.cSpeed; + } + + if(loopDurationD < TIMELOOP_NANOSEC) { + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_decodeOnly, BMK_timeMode, 1); + if(benchres2.error) { + return ERROR_RESULT; + } + benchres.result.dSpeed = benchres2.result.dSpeed; + } + + *resultPtr = benchres.result; + + /* compare by resultScore when in infeas */ + /* compare by compareResultLT when in feas */ + if((!feas && (resultScore(benchres.result, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target))) || + (feas && (compareResultLT(*winnerResult, benchres.result, target, buf.srcSize))) ) { + return BETTER_RESULT; + } else { + return WORSE_RESULT; } } /* wrap feasibleBench w/ memotable */ #define INFEASIBLE_THRESHOLD 200 -static int feasibleBenchMemo(BMK_result_t* resultPtr, - const void* srcBuffer, const size_t srcSize, - void* dstBuffer, const size_t dstSize, - void* dictBuffer, const size_t dictSize, - const size_t* fileSizes, const size_t nbFiles, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, - const ZSTD_compressionParameters cParams, - const constraint_t target, - BMK_result_t* winnerResult, U8* memoTable, - const U32* varyParams, const int varyLen) { - - const size_t memind = memoTableInd(&cParams, varyParams, varyLen); - if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { - return INFEASIBLE_RESULT; - } else { - const size_t blockSize = g_blockSize ? g_blockSize : srcSize; - U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + (U32)nbFiles; - const void ** const srcPtrs = (const void** const)malloc(maxNbBlocks * sizeof(void*)); - size_t* const srcSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); - void ** const dstPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); - size_t* const dstCapacities = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); - U32 nbBlocks; - int res; - - if(!srcPtrs || !srcSizes || !dstPtrs || !dstCapacities) { - free(srcPtrs); - free(srcSizes); - free(dstPtrs); - free(dstCapacities); - DISPLAY("Allocation Error\n"); - return ERROR_RESULT; - } - - { - const char* srcPtr = (const char*)srcBuffer; - char* dstPtr = (char*)dstBuffer; - size_t dstSizeRemaining = dstSize; - U32 fileNb; - for (nbBlocks=0, fileNb=0; fileNb dstSizeRemaining) { - DEBUGOUTPUT("Warning: dstSize too small to benchmark completely \n"); - remaining = dstSizeRemaining; - dstSizeRemaining = 0; - } else { - dstSizeRemaining -= remaining; - } - for ( ; nbBlocks= INFEASIBLE_THRESHOLD) { - return INFEASIBLE_RESULT; //see feasibleBenchMemo for concerns - } else { - const size_t blockSize = g_blockSize ? g_blockSize : srcSize; - U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + (U32)nbFiles; - const void ** const srcPtrs = (const void** const)malloc(maxNbBlocks * sizeof(void*)); - size_t* const srcSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); - void ** const dstPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); - size_t* const dstCapacities = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); - U32 nbBlocks; - int res; + int res; - if(!srcPtrs || !srcSizes || !dstPtrs || !dstCapacities) { - free(srcPtrs); - free(srcSizes); - free(dstPtrs); - free(dstCapacities); - DISPLAY("Allocation Error\n"); - return ERROR_RESULT; - } + if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { return WORSE_RESULT; } - { - const char* srcPtr = (const char*)srcBuffer; - char* dstPtr = (char*)dstBuffer; - size_t dstSizeRemaining = dstSize; - U32 fileNb; - for (nbBlocks=0, fileNb=0; fileNb dstSizeRemaining) { - DEBUGOUTPUT("Warning: dstSize too small to benchmark completely \n"); - remaining = dstSizeRemaining; - dstSizeRemaining = 0; - } else { - dstSizeRemaining -= remaining; - } - for ( ; nbBlocks (hopefully) feasible */ - /* when nothing is found, this garbages part 2. */ { - winnerInfo_t bestFeasible1; /* uses feasibleBench Metric */ - - //init these params - bestFeasible1.params = cparam; - bestFeasible1.result.cSpeed = 0; - bestFeasible1.result.dSpeed = 0; - bestFeasible1.result.cMem = (size_t)-1; - bestFeasible1.result.cSize = (size_t)-1; + winnerInfo_t bestFeasible1 = initWinnerInfo(cparam); DISPLAY("Climb Part 1\n"); while(better) { - int i, d; + int i, dist, offset; better = 0; DEBUGOUTPUT("Start\n"); cparam = winnerInfo.params; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, buf.srcSize); candidateInfo.params = cparam; //all dist-1 targets //if we early end this, we should also randomize the order these are picked. for(i = 0; i < varLen; i++) { - paramVaryOnce(varArray[i], 1, &candidateInfo.params); /* +1 */ - candidateInfo.params = sanitizeParams(candidateInfo.params); - //evaluate - if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { - //if(cParamValid(candidateInfo.params)) { - int res = infeasibleBenchMemo(&candidateInfo.result, - srcBuffer, srcSize, - dstBuffer, dstSize, - dictBuffer, dictSize, - fileSizes, nbFiles, - ctx, dctx, - candidateInfo.params, target, &winnerInfo.result, memoTable, - varArray, varLen); - if(res == FEASIBLE_RESULT) { /* synonymous with better when called w/ infeasibleBM */ - winnerInfo = candidateInfo; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); - better = 1; - if(feasible(candidateInfo.result, target) && objective_lt(bestFeasible1.result, winnerInfo.result)) { - bestFeasible1 = winnerInfo; - } - } - } - candidateInfo.params = cparam; - paramVaryOnce(varArray[i], -1, &candidateInfo.params); /* -1 */ - candidateInfo.params = sanitizeParams(candidateInfo.params); - //evaluate - if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { - //if(cParamValid(candidateInfo.params)) { - int res = infeasibleBenchMemo(&candidateInfo.result, - srcBuffer, srcSize, - dstBuffer, dstSize, - dictBuffer, dictSize, - fileSizes, nbFiles, - ctx, dctx, - candidateInfo.params, target, &winnerInfo.result, memoTable, - varArray, varLen); - if(res == FEASIBLE_RESULT) { - winnerInfo = candidateInfo; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); - better = 1; - if(feasible(candidateInfo.result, target) && objective_lt(bestFeasible1.result, winnerInfo.result)) { - bestFeasible1 = winnerInfo; + for(offset = -1; offset <= 1; offset += 2) { + candidateInfo.params = cparam; + paramVaryOnce(varArray[i], offset, &candidateInfo.params); /* +1 */ + candidateInfo.params = sanitizeParams(candidateInfo.params); + //evaluate + if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { + int res = benchMemo(&candidateInfo.result, + buf, ctx, + candidateInfo.params, target, &winnerInfo.result, memoTable, + varArray, varLen, feas); + if(res == BETTER_RESULT) { /* synonymous with better when called w/ infeasibleBM */ + winnerInfo = candidateInfo; + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, buf.srcSize); + better = 1; + if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) { + bestFeasible1 = winnerInfo; + } } } } @@ -1966,235 +1533,255 @@ static winnerInfo_t climbOnce(const constraint_t target, if(better) { continue; } - //if 'better' enough, skip further parameter search, center there? - //possible improvement - guide direction here w/ knowledge rather than completely random variation. - for(d = 2; d < varLen + 2; d++) { /* varLen is # dimensions */ + + for(dist = 2; dist < varLen + 2; dist++) { /* varLen is # dimensions */ for(i = 0; i < 2 * varLen + 2; i++) { int res; candidateInfo.params = cparam; /* param error checking already done here */ - paramVariation(&candidateInfo.params, varArray, varLen, d); - res = infeasibleBenchMemo(&candidateInfo.result, - srcBuffer, srcSize, - dstBuffer, dstSize, - dictBuffer, dictSize, - fileSizes, nbFiles, - ctx, dctx, - candidateInfo.params, target, &winnerInfo.result, memoTable, - varArray, varLen); - if(res == FEASIBLE_RESULT) { /* synonymous with better in this case*/ + paramVariation(&candidateInfo.params, varArray, varLen, dist); + res = benchMemo(&candidateInfo.result, + buf, ctx, + candidateInfo.params, target, &winnerInfo.result, memoTable, + varArray, varLen, feas); + if(res == BETTER_RESULT) { /* synonymous with better in this case*/ winnerInfo = candidateInfo; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, buf.srcSize); better = 1; - if(feasible(candidateInfo.result, target) && objective_lt(bestFeasible1.result, winnerInfo.result)) { + if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) { bestFeasible1 = winnerInfo; } } } if(better) { - continue; + break; } } - //bias to test previous delta? - //change cparam -> candidate before restart + + if(!better) { //infeas -> feas -> stop. + if(feas) { return winnerInfo; } + + feas = 1; + better = 1; + winnerInfo = bestFeasible1; /* note with change, bestFeasible may not necessarily be feasible, but if one has been benchmarked, it will be. */ + DISPLAY("Climb Part 2\n"); + } } winnerInfo = bestFeasible1; } - //break out if no feasible. - if(winnerInfo.result.cMem == (U32)-1) { - DEBUGOUTPUT("No Feasible Found\n"); - return winnerInfo; - } - DISPLAY("Climb Part 2\n"); - - better = 1; - /* feasible -> best feasible (hopefully) */ - { - while(better) { - int i, d; - better = 0; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); - //all dist-1 targets - cparam = winnerInfo.params; - candidateInfo.params = cparam; - for(i = 0; i < varLen; i++) { - paramVaryOnce(varArray[i], 1, &candidateInfo.params); - candidateInfo.params = sanitizeParams(candidateInfo.params); - - //evaluate - if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { - //if(cParamValid(candidateInfo.params)) { - int res = feasibleBenchMemo(&candidateInfo.result, - srcBuffer, srcSize, - dstBuffer, dstSize, - dictBuffer, dictSize, - fileSizes, nbFiles, - ctx, dctx, - candidateInfo.params, target, &winnerInfo.result, memoTable, - varArray, varLen); - if(res == FEASIBLE_RESULT) { - winnerInfo = candidateInfo; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); - better = 1; - } - } - candidateInfo.params = cparam; - paramVaryOnce(varArray[i], -1, &candidateInfo.params); - candidateInfo.params = sanitizeParams(candidateInfo.params); - //evaluate - if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { - int res = feasibleBenchMemo(&candidateInfo.result, - srcBuffer, srcSize, - dstBuffer, dstSize, - dictBuffer, dictSize, - fileSizes, nbFiles, - ctx, dctx, - candidateInfo.params, target, &winnerInfo.result, memoTable, - varArray, varLen); - if(res == FEASIBLE_RESULT) { - winnerInfo = candidateInfo; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); - better = 1; - } - } - } - //if 'better' enough, skip further parameter search, center there? - //possible improvement - guide direction here w/ knowledge rather than completely random variation. - for(d = 2; d < varLen + 2; d++) { /* varLen is # dimensions */ - for(i = 0; i < 2 * varLen + 2; i++) { - int res; - candidateInfo.params = cparam; - /* param error checking already done here */ - paramVariation(&candidateInfo.params, varArray, varLen, d); //info candidateInfo.params is garbage, this is too. - res = feasibleBenchMemo(&candidateInfo.result, - srcBuffer, srcSize, - dstBuffer, dstSize, - dictBuffer, dictSize, - fileSizes, nbFiles, - ctx, dctx, - candidateInfo.params, target, &winnerInfo.result, memoTable, - varArray, varLen); - if(res == FEASIBLE_RESULT) { - winnerInfo = candidateInfo; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); - better = 1; - } - } - if(better) { - continue; - } - } - //bias to test previous delta? - //change cparam -> candidate before restart - } - } - return winnerInfo; } //optimizeForSize but with fixed strategy //place to configure/filter out strategy specific parameters. -//need args for all buffers and parameter stuff -//sanitization here. //flexible parameters: iterations of (failed?) climbing (or if we do non-random, maybe this is when everything is close to visitied) //weight more on visit for bad results, less on good results/more on later results / ones with more failures. //allocate memoTable here. //only real use for paramTarget is to get the fixed values, right? +//maybe allow giving it a first init? static winnerInfo_t optimizeFixedStrategy( - const void* srcBuffer, const size_t srcSize, - void* dstBuffer, const size_t dstSize, - void* dictBuffer, const size_t dictSize, - const size_t* fileSizes, const size_t nbFiles, - const constraint_t target, ZSTD_compressionParameters paramTarget, - const ZSTD_strategy strat, const U32* varArray, const int varLen, U8* memoTable) { + buffers_t buf, contexts_t ctx, + const constraint_t target, ZSTD_compressionParameters paramTarget, + const ZSTD_strategy strat, + const varInds_t* varArray, const int varLen, + U8* memoTable, const int tries) { int i = 0; - U32* varNew = malloc(sizeof(U32) * varLen); - int varLenNew = sanitizeVarArray(varLen, varArray, varNew, strat); + varInds_t varNew[NUM_PARAMS]; + int varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat); ZSTD_compressionParameters init; - ZSTD_CCtx* ctx = ZSTD_createCCtx(); - ZSTD_DCtx* dctx = ZSTD_createDCtx(); winnerInfo_t winnerInfo, candidateInfo; - winnerInfo.result.cSpeed = 0; - winnerInfo.result.dSpeed = 0; - winnerInfo.result.cMem = (size_t)(-1LL); - winnerInfo.result.cSize = (size_t)(-1LL); + winnerInfo = initWinnerInfo(emptyParams()); /* so climb is given the right fixed strategy */ paramTarget.strategy = strat; /* to pass ZSTD_checkCParams */ - //needs to happen after memoTableInit as that assumes 0 = undefined. cParamZeroMin(¶mTarget); init = paramTarget; - - if(!ctx || !dctx || !memoTable || !varNew) { - DISPLAY("NOT ENOUGH MEMORY ! ! ! \n"); - goto _cleanUp; - } - - while(i < 10) { //make i adjustable (user input?) depending on how much time they have. + while(i < tries) { //make i adjustable (user input?) depending on how much time they have. DEBUGOUTPUT("Restart\n"); //look into improving this to maximize distance from searched infeasible stuff / towards promising regions? randomConstrainedParams(&init, varNew, varLenNew, memoTable); - candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, fileSizes, nbFiles, ctx, dctx, init); - if(objective_lt(winnerInfo.result, candidateInfo.result)) { + candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, buf, ctx, init); + if(compareResultLT(winnerInfo.result, candidateInfo.result, target, buf.srcSize)) { winnerInfo = candidateInfo; - DISPLAY("New Winner: "); - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, buf.srcSize); i = 0; } i++; } - -_cleanUp: - ZSTD_freeCCtx(ctx); - ZSTD_freeDCtx(dctx); - free(varNew); return winnerInfo; } -static int BMK_loadFiles(void* buffer, size_t bufferSize, - size_t* fileSizes, const char* const * const fileNamesTable, - unsigned nbFiles) +static void freeBuffers(buffers_t b) { + if(b.srcPtrs != NULL) { + free(b.srcPtrs[0]); + } + free(b.srcPtrs); + free(b.srcSizes); + + if(b.dstPtrs != NULL) { + free(b.dstPtrs[0]); + } + free(b.dstPtrs); + free(b.dstCapacities); + free(b.dstSizes); + + if(b.resPtrs != NULL) { + free(b.resPtrs[0]); + } + free(b.resPtrs); +} + +/* allocates buffer's arguments. returns success / failuere */ +static int initBuffers(buffers_t* buff, const char* const * const fileNamesTable, + size_t nbFiles) { - size_t pos = 0, totalSize = 0; - unsigned n; - for (n=0; nsrcPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); + buff->srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + + buff->dstPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); + buff->dstCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + buff->dstSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + + buff->resPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); + buff->resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + + if(!buff->srcPtrs || !buff->srcSizes || !buff->dstPtrs || !buff->dstCapacities || !buff->dstCapacities || !buff->resPtrs || !buff->resSizes) { + DISPLAY("alloc error\n"); + freeBuffers(*buff); + return 1; + } + + buff->srcPtrs[0] = malloc(benchedSize); + buff->dstPtrs[0] = malloc(ZSTD_compressBound(benchedSize) + (maxNbBlocks * 1024)); + buff->resPtrs[0] = malloc(benchedSize); + + if(!buff->srcPtrs[0] || !buff->dstPtrs[0] || !buff->resPtrs[0]) { + DISPLAY("alloc error\n"); + freeBuffers(*buff); + return 1; + } + + for(n = 0; n < nbFiles; n++) { FILE* f; U64 fileSize = UTIL_getFileSize(fileNamesTable[n]); if (UTIL_isDirectory(fileNamesTable[n])) { DISPLAY("Ignoring %s directory... \n", fileNamesTable[n]); - fileSizes[n] = 0; continue; } if (fileSize == UTIL_FILESIZE_UNKNOWN) { DISPLAY("Cannot evaluate size of %s, ignoring ... \n", fileNamesTable[n]); - fileSizes[n] = 0; continue; } f = fopen(fileNamesTable[n], "rb"); if (f==NULL) { - DISPLAY("impossible to open file %s", fileNamesTable[n]); + DISPLAY("impossible to open file %s\n", fileNamesTable[n]); + freeBuffers(*buff); + fclose(f); return 10; } + DISPLAY("Loading %s... \r", fileNamesTable[n]); - if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */ - { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f); + + if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, nbFiles=n; /* buffer too small - stop after this file */ + { + char* buffer = (char*)(buff->srcPtrs[0]); + size_t const readSize = fread((buffer)+pos, 1, (size_t)fileSize, f); + size_t blocked = 0; + while(blocked < readSize) { + buff->srcPtrs[blockNb] = (buffer) + (pos + blocked); + buff->srcSizes[blockNb] = blockSize; + blocked += blockSize; + blockNb++; + } + if(readSize > 0) { buff->srcSizes[blockNb - 1] = ((readSize - 1) % blockSize) + 1; } + if (readSize != (size_t)fileSize) { DISPLAY("could not read %s", fileNamesTable[n]); - return 11; + freeBuffers(*buff); + fclose(f); + return 1; } - pos += readSize; } - fileSizes[n] = (size_t)fileSize; - totalSize += (size_t)fileSize; + + pos += readSize; + + } fclose(f); } - if (totalSize == 0) { DISPLAY("\nno data to bench\n"); return 12; } + buff->dstCapacities[0] = ZSTD_compressBound(buff->srcSizes[0]); + buff->dstSizes[0] = buff->dstCapacities[0]; + buff->resSizes[0] = buff->srcSizes[0]; + + for(n = 1; n < blockNb; n++) { + buff->dstPtrs[n] = ((char*)buff->dstPtrs[n-1]) + buff->dstCapacities[n-1]; + buff->resPtrs[n] = ((char*)buff->resPtrs[n-1]) + buff->resSizes[n-1]; + buff->dstCapacities[n] = ZSTD_compressBound(buff->srcSizes[n]); + buff->dstSizes[n] = buff->dstCapacities[n]; + buff->resSizes[n] = buff->srcSizes[n]; + } + buff->srcSize = pos; + buff->nbBlocks = blockNb; + + if (pos == 0) { DISPLAY("\nno data to bench\n"); return 1; } + + return 0; +} + +static void freeContexts(contexts_t ctx) { + free(ctx.dictBuffer); + ZSTD_freeCCtx(ctx.cctx); + ZSTD_freeDCtx(ctx.dctx); +} + +static int initContexts(contexts_t* ctx, const char* dictFileName) { + FILE* f; + size_t readSize; + ctx->cctx = ZSTD_createCCtx(); + ctx->dctx = ZSTD_createDCtx(); + if(dictFileName == NULL) { + ctx->dictSize = 0; + ctx->dictBuffer = NULL; + return 0; + } + ctx->dictSize = UTIL_getFileSize(dictFileName); + ctx->dictBuffer = malloc(ctx->dictSize); + + f = fopen(dictFileName, "rb"); + + if(!f) { + DISPLAY("unable to open file\n"); + fclose(f); + freeContexts(*ctx); + return 1; + } + + if(ctx->dictSize > 64 MB || !(ctx->dictBuffer)) { + DISPLAY("dictionary too large\n"); + fclose(f); + freeContexts(*ctx); + return 1; + } + readSize = fread(ctx->dictBuffer, 1, ctx->dictSize, f); + if(readSize != ctx->dictSize) { + DISPLAY("unable to read file\n"); + fclose(f); + freeContexts(*ctx); + return 1; + } return 0; } @@ -2228,107 +1815,98 @@ static int nextStrategy(const int currentStrategy, const int bestStrategy) { } } +static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZSTD_compressionParameters mask) { + base.windowLog = mask.windowLog ? mask.windowLog : base.windowLog; + base.chainLog = mask.chainLog ? mask.chainLog : base.chainLog; + base.hashLog = mask.hashLog ? mask.hashLog : base.hashLog; + base.searchLog = mask.searchLog ? mask.searchLog : base.searchLog; + base.searchLength = mask.searchLength ? mask.searchLength : base.searchLength; + base.targetLength = mask.targetLength ? mask.targetLength : base.targetLength; + base.strategy = mask.strategy ? mask.strategy : base.strategy; + return base; +} + +#define MAX_TRIES 8 //optimize fixed strategy. static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel) { - size_t benchedSize; - void* origBuff = NULL; - void* dictBuffer = NULL; - size_t dictBufferSize = 0; - U32 varArray [NUM_PARAMS]; + varInds_t varArray [NUM_PARAMS]; int ret = 0; - size_t* fileSizes = calloc(sizeof(size_t),nbFiles); const int varLen = variableParams(paramTarget, varArray); + winnerInfo_t winner = initWinnerInfo(emptyParams()); U8** allMT = NULL; - g_winner.result.cSize = (size_t)-1; + size_t k; + size_t maxBlockSize = 0; + contexts_t ctx; + buffers_t buf; + /* Init */ if(!cParamValid(paramTarget)) { - return 10; + return 1; } /* load dictionary*/ - if (dictFileName != NULL) { - U64 const dictFileSize = UTIL_getFileSize(dictFileName); - if (dictFileSize > 64 MB) { - DISPLAY("dictionary file %s too large", dictFileName); - ret = 10; - goto _cleanUp; - } - dictBufferSize = (size_t)dictFileSize; - dictBuffer = malloc(dictBufferSize); - if (dictBuffer==NULL) { - DISPLAY("not enough memory for dictionary (%u bytes)", - (U32)dictBufferSize); - ret = 11; - goto _cleanUp; + if(initBuffers(&buf, fileNamesTable, nbFiles)) { + DISPLAY("unable to load files\n"); + return 1; + } - } - - { - int errorCode = BMK_loadFiles(dictBuffer, dictBufferSize, &dictBufferSize, &dictFileName, 1); - if(errorCode) { - ret = errorCode; - goto _cleanUp; - } - } + if(initContexts(&ctx, dictFileName)) { + DISPLAY("unable to load dictionary\n"); + freeBuffers(buf); + return 2; } - /* Fill input buffer */ if(nbFiles == 1) { DISPLAY("Loading %s... \r", fileNamesTable[0]); } else { DISPLAY("Loading %lu Files... \r", (unsigned long)nbFiles); } - { - U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles); - int ec; - unsigned i; - benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3; - origBuff = malloc(benchedSize); - if(!origBuff || !fileSizes) { - DISPLAY("Not enough memory for stuff\n"); - ret = 1; - goto _cleanUp; - } - ec = BMK_loadFiles(origBuff, benchedSize, fileSizes, fileNamesTable, (U32)nbFiles); - if(ec) { - DISPLAY("Error Loading Files"); - ret = ec; - goto _cleanUp; - } - benchedSize = 0; - for(i = 0; i < nbFiles; i++) { - benchedSize += fileSizes[i]; - } - origBuff = realloc(origBuff, benchedSize); + for(k = 0; k < buf.nbBlocks; k++) { + maxBlockSize = MAX(buf.srcSizes[k], maxBlockSize); } - allMT = memoTableInitAll(paramTarget, target, varArray, varLen, benchedSize); + /* if strategy is fixed, only init that part of memotable */ + if(paramTarget.strategy) { + varInds_t varNew[NUM_PARAMS]; + int varLenNew = sanitizeVarArray(varNew, varLen, varArray, paramTarget.strategy); + allMT = calloc(sizeof(U8), (ZSTD_btultra + 1)); + if(allMT == NULL) { + ret = 57; + goto _cleanUp; + } + + allMT[paramTarget.strategy] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew)); + + if(allMT[paramTarget.strategy] == NULL) { + ret = 58; + goto _cleanUp; + } + + memoTableInit(allMT[paramTarget.strategy], paramTarget, target, varNew, varLenNew, maxBlockSize); + } else { + allMT = memoTableInitAll(paramTarget, target, varArray, varLen, maxBlockSize); + } + + if(!allMT) { + DISPLAY("MemoTable Init Error\n"); ret = 2; goto _cleanUp; } if(cLevel) { - BMK_result_t candidate; - const size_t blockSize = g_blockSize ? g_blockSize : benchedSize; - ZSTD_CCtx* const ctx = ZSTD_createCCtx(); - ZSTD_DCtx* const dctx = ZSTD_createDCtx(); - ZSTD_compressionParameters const CParams = ZSTD_getCParams(cLevel, blockSize, dictBufferSize); - if(BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, (U32)nbFiles, ctx, dctx, CParams)) { - ZSTD_freeCCtx(ctx); - ZSTD_freeDCtx(dctx); + winner.params = ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize); + if(BMK_benchParam(&winner.result, buf, ctx, winner.params)) { ret = 3; goto _cleanUp; } - target.cSpeed = (U32)candidate.cSpeed; //Maybe have a small bit of slack here, like x.99? - BMK_printWinner(stdout, cLevel, candidate, CParams, benchedSize); - - ZSTD_freeCCtx(ctx); - ZSTD_freeDCtx(dctx); + target.cSpeed = (U32)winner.result.cSpeed; //Maybe have a small bit of slack here, like x.99? + g_targetConstraints = target; + BMK_printWinner(stdout, cLevel, winner.result, winner.params, buf.srcSize); } g_targetConstraints = target; @@ -2340,106 +1918,40 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } else { DISPLAY("optimizing for %lu Files", (unsigned long)nbFiles); } - if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed / 1000000); } - if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed / 1000000); } - if(target.cMem != (U32)-1) { DISPLAY(" - limit memory %u MB", target.cMem / 1000000); } + if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed >> 20); } + if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed >> 20); } + if(target.cMem != (U32)-1) { DISPLAY(" - limit memory %u MB", target.cMem >> 20); } + DISPLAY("\n"); findClockGranularity(); - { ZSTD_CCtx* const ctx = ZSTD_createCCtx(); - ZSTD_DCtx* const dctx = ZSTD_createDCtx(); - winnerInfo_t winner; - U32 varNew[NUM_PARAMS]; - const size_t blockSize = g_blockSize ? g_blockSize : benchedSize; - U32 const maxNbBlocks = (U32) ((benchedSize + (blockSize-1)) / blockSize) + 1; - const size_t maxCompressedSize = ZSTD_compressBound(benchedSize) + (maxNbBlocks * 1024); - void* compressedBuffer = malloc(maxCompressedSize); - - /* init */ - if (ctx==NULL) { DISPLAY("\n ZSTD_createCCtx error \n"); free(origBuff); return 14;} - if(compressedBuffer==NULL) { DISPLAY("\n Allocation Error \n"); free(origBuff); free(ctx); return 15; } - memset(&winner, 0, sizeof(winner)); - winner.result.cSize = (size_t)(-1); - + { + varInds_t varNew[NUM_PARAMS]; /* find best solution from default params */ { /* strategy selection */ const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); DEBUGOUTPUT("Strategy Selection\n"); - if(varLen == NUM_PARAMS && paramTarget.strategy == 0) { /* no variable based constraints */ + if(paramTarget.strategy == 0) { /* no variable based constraints */ BMK_result_t candidate; - int feas = 0, i; + int i; for (i=1; i<=maxSeeds; i++) { - ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, dictBufferSize); - int ec = BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, (U32)nbFiles, ctx, dctx, CParams); - BMK_printWinner(stdout, i, candidate, CParams, benchedSize); - - if(!ec) { - if(feas) { - if(feasible(candidate, relaxTarget(target)) && objective_lt(winner.result, candidate)) { - winner.result = candidate; - winner.params = CParams; - } - } else { - if(feasible(candidate, relaxTarget(target))) { - feas = 1; - winner.result = candidate; - winner.params = CParams; - - } else { - if(resultScore(candidate, benchedSize, target) > resultScore(winner.result, benchedSize, target)) { - winner.result = candidate; - winner.params = CParams; - } - } - } - } - } //best, -1, +1, ..., - - } else if (paramTarget.strategy == 0) { //constrained - int feas = 0, i, j; - for(j = 1; j < 10; j++) { - for(i = 1; i <= maxSeeds; i++) { - int varLenNew = sanitizeVarArray(varLen, varArray, varNew, i); - ZSTD_compressionParameters candidateParams = paramTarget; - BMK_result_t candidate; - int ec; - randomConstrainedParams(&candidateParams, varNew, varLenNew, allMT[i]); - cParamZeroMin(&candidateParams); - candidateParams = sanitizeParams(candidateParams); - ec = BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, (U32)nbFiles, ctx, dctx, candidateParams); - - if(!ec) { - if(feas) { - if(feasible(candidate, relaxTarget(target)) && objective_lt(winner.result, candidate)) { - winner.result = candidate; - winner.params = candidateParams; - BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); - } - } else { - if(feasible(candidate, relaxTarget(target))) { - feas = 1; - winner.result = candidate; - winner.params = candidateParams; - BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); - - } else { - if(resultScore(candidate, benchedSize, target) > resultScore(winner.result, benchedSize, target)) { - winner.result = candidate; - winner.params = candidateParams; - BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); - } - } - } - } + int ec; + ZSTD_compressionParameters CParams = ZSTD_getCParams(i, maxBlockSize, ctx.dictSize); + CParams = maskParams(CParams, paramTarget); + ec = BMK_benchParam(&candidate, buf, ctx, CParams); + BMK_printWinner(stdout, i, candidate, CParams, buf.srcSize); + if(!ec && compareResultLT(winner.result, candidate, relaxTarget(target), buf.srcSize)) { + winner.result = candidate; + winner.params = CParams; } } } } - BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, buf.srcSize); BMK_translateAdvancedParams(winner.params); DEBUGOUTPUT("Real Opt\n"); /* start 'real' tests */ @@ -2447,55 +1959,51 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ int bestStrategy = (int)winner.params.strategy; if(paramTarget.strategy == 0) { int st = (int)winner.params.strategy; + int tries = MAX_TRIES; { - int varLenNew = sanitizeVarArray(varLen, varArray, varNew, st); + int varLenNew = sanitizeVarArray(varNew, varLen, varArray, st); winnerInfo_t w1 = climbOnce(target, varNew, varLenNew, allMT[st], - origBuff, benchedSize, compressedBuffer, maxCompressedSize, dictBuffer, dictBufferSize, - fileSizes, nbFiles, ctx, dctx, winner.params); - if(objective_lt(winner.result, w1.result)) { + buf, ctx, winner.params); + if(compareResultLT(winner.result, w1.result, target, buf.srcSize)) { winner = w1; } } - while(st) { - winnerInfo_t wc = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, dictBuffer, dictBufferSize, fileSizes, nbFiles, - target, paramTarget, st, varArray, varLen, allMT[st]); + while(st && tries) { + winnerInfo_t wc = optimizeFixedStrategy(buf, ctx, target, paramTarget, + st, varArray, varLen, allMT[st], tries); DEBUGOUTPUT("StratNum %d\n", st); - if(objective_lt(winner.result, wc.result)) { + if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) { winner = wc; } //We could double back to increase search of 'better' strategies st = nextStrategy(st, bestStrategy); + tries--; } } else { - winner = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, dictBuffer, dictBufferSize, fileSizes, nbFiles, - target, paramTarget, paramTarget.strategy, varArray, varLen, allMT[paramTarget.strategy]); + winner = optimizeFixedStrategy(buf, ctx, target, paramTarget, paramTarget.strategy, + varArray, varLen, allMT[paramTarget.strategy], 10); } } /* no solution found */ if(winner.result.cSize == (size_t)-1) { + ret = 1; DISPLAY("No feasible solution found\n"); - return 1; + goto _cleanUp; } /* end summary */ - BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, buf.srcSize); BMK_translateAdvancedParams(winner.params); DISPLAY("grillParams size - optimizer completed \n"); - - /* clean up*/ - ZSTD_freeCCtx(ctx); - ZSTD_freeDCtx(dctx); - } _cleanUp: - free(fileSizes); - free(dictBuffer); + freeContexts(ctx); + freeBuffers(buf); memoTableFreeAll(allMT); - free(origBuff); return ret; } @@ -2546,16 +2054,16 @@ static int usage(const char* exename) static int usage_advanced(void) { DISPLAY( "\nAdvanced options :\n"); - DISPLAY( " -T# : set level 1 speed objective \n"); - DISPLAY( " -B# : cut input into blocks of size # (default : single block) \n"); - DISPLAY( " -i# : iteration loops [1-9](default : %i) \n", NBLOOPS); - DISPLAY( " -O# : find Optimized parameters for # MB/s compression speed (default : 0) \n"); - DISPLAY( " -S : Single run \n"); - DISPLAY( " --zstd : Single run, parameter selection same as zstdcli \n"); - DISPLAY( " -P# : generated sample compressibility (default : %.1f%%) \n", COMPRESSIBILITY_DEFAULT * 100); - DISPLAY( " -t# : Caps runtime of operation in seconds (default : %u seconds (%.1f hours)) \n", (U32)g_grillDuration_s, g_grillDuration_s / 3600); - DISPLAY( " -v : Prints Benchmarking output\n"); - DISPLAY( " -D : Next argument dictionary file\n"); + DISPLAY( " -T# : set level 1 speed objective \n"); + DISPLAY( " -B# : cut input into blocks of size # (default : single block) \n"); + DISPLAY( " -i# : iteration loops (default : %i) \n", NBLOOPS); + DISPLAY( " --optimize= : same as -O with more verbose syntax (see README.md)\n"); + DISPLAY( " -S : Single run \n"); + DISPLAY( " --zstd : Single run, parameter selection same as zstdcli \n"); + DISPLAY( " -P# : generated sample compressibility (default : %.1f%%) \n", COMPRESSIBILITY_DEFAULT * 100); + DISPLAY( " -t# : Caps runtime of operation in seconds (default : %u seconds (%.1f hours)) \n", (U32)g_grillDuration_s, g_grillDuration_s / 3600); + DISPLAY( " -v : Prints Benchmarking output\n"); + DISPLAY( " -D : Next argument dictionary file\n"); return 0; } @@ -2572,8 +2080,8 @@ int main(int argc, const char** argv) filenamesStart=0, result; const char* exename=argv[0]; - const char* input_filename = 0; - const char* dictFileName = 0; + const char* input_filename = NULL; + const char* dictFileName = NULL; U32 optimizer = 0; U32 main_pause = 0; int optimizerCLevel = 0; @@ -2591,6 +2099,9 @@ int main(int argc, const char** argv) for(i=1; i Date: Fri, 27 Jul 2018 08:49:25 -0700 Subject: [PATCH 10/19] Renaming / Style fixes --- programs/bench.c | 56 +++++++++++++++++++++++++--------------------- programs/bench.h | 7 +++--- tests/README.md | 1 - tests/paramgrill.c | 27 +++++++++++----------- 4 files changed, 48 insertions(+), 43 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 177dbe0e3..b496caf27 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -294,7 +294,7 @@ BMK_customReturn_t BMK_benchFunction( BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBlockBuffers, const size_t* srcBlockSizes, - void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, size_t* cSizes, + void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, size_t* blockResult, unsigned nbLoops) { size_t dstSize = 0; U64 totalTime; @@ -332,8 +332,8 @@ BMK_customReturn_t BMK_benchFunction( j, (U32)dstBlockCapacities[j], ZSTD_getErrorName(res)); } else if(i == nbLoops - 1) { dstSize += res; - if(cSizes != NULL) { - cSizes[j] = res; + if(blockResult != NULL) { + blockResult[j] = res; } } } @@ -358,6 +358,9 @@ void BMK_resetTimeState(BMK_timedFnState_t* r, unsigned nbSeconds) { BMK_timedFnState_t* BMK_createTimeState(unsigned nbSeconds) { BMK_timedFnState_t* r = (BMK_timedFnState_t*)malloc(sizeof(struct BMK_timeState_t)); + if(r == NULL) { + return r; + } BMK_resetTimeState(r, nbSeconds); return r; } @@ -373,7 +376,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed( BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const* const srcBlockBuffers, const size_t* srcBlockSizes, - void * const * const dstBlockBuffers, const size_t * dstBlockCapacities, size_t* dstSizes) + void * const * const dstBlockBuffers, const size_t * dstBlockCapacities, size_t* blockResults) { U64 fastest = cont->fastestTime; int completed = 0; @@ -390,7 +393,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed( } /* reinitialize capacity */ r.result = BMK_benchFunction(benchFn, benchPayload, initFn, initPayload, - blockCount, srcBlockBuffers, srcBlockSizes, dstBlockBuffers, dstBlockCapacities, dstSizes, cont->nbLoops); + blockCount, srcBlockBuffers, srcBlockSizes, dstBlockBuffers, dstBlockCapacities, blockResults, cont->nbLoops); if(r.result.error) { /* completed w/ error */ r.completed = 1; return r; @@ -718,38 +721,36 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; /* these are the blockTable parameters, just split up */ - const void ** const srcPtrs = (const void** const)malloc(maxNbBlocks * sizeof(void*)); - size_t* const srcSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); + const void ** const srcPtrs = (const void**)malloc(maxNbBlocks * sizeof(void*)); + size_t* const srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); - void ** const cPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); - size_t* const cSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); - size_t* const cCapacities = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); + void ** const cPtrs = (void**)malloc(maxNbBlocks * sizeof(void*)); + size_t* const cSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + size_t* const cCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); - void ** const resPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); - size_t* const resSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); + void ** const resPtrs = (void**)malloc(maxNbBlocks * sizeof(void*)); + size_t* const resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); BMK_timedFnState_t* timeStateCompress = BMK_createTimeState(adv->nbSeconds); BMK_timedFnState_t* timeStateDecompress = BMK_createTimeState(adv->nbSeconds); - void* compressedBuffer; const size_t maxCompressedSize = dstCapacity ? dstCapacity : ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); + + void* const internalDstBuffer = dstBuffer ? NULL : malloc(maxCompressedSize); + void* const compressedBuffer = dstBuffer ? dstBuffer : internalDstBuffer; + void* resultBuffer = malloc(srcSize); - BMK_return_t results = { { 0, 0, 0, 0 }, 0 }; - int allocationincomplete; + int allocationincomplete = !srcPtrs || !srcSizes || !cPtrs || + !cSizes || !cCapacities || !resPtrs || !resSizes || + !timeStateCompress || !timeStateDecompress || !compressedBuffer || !resultBuffer; - if(!dstCapacity) { - compressedBuffer = malloc(maxCompressedSize); - } else { - compressedBuffer = dstBuffer; - } + int parametersConflict = !dstBuffer ^ !dstCapacity; - allocationincomplete = !compressedBuffer || !resultBuffer || - !srcPtrs || !srcSizes || !cPtrs || !cSizes || !resPtrs || !resSizes; - if (!allocationincomplete) { + if (!allocationincomplete && !parametersConflict) { results = BMK_benchMemAdvancedNoAlloc(srcPtrs, srcSizes, cPtrs, cCapacities, cSizes, resPtrs, resSizes, &resultBuffer, compressedBuffer, maxCompressedSize, timeStateCompress, timeStateDecompress, srcBuffer, srcSize, fileSizes, nbFiles, cLevel, comprParams, @@ -759,9 +760,8 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, /* clean up */ BMK_freeTimeState(timeStateCompress); BMK_freeTimeState(timeStateDecompress); - if(!dstCapacity) { /* only free if not given */ - free(compressedBuffer); - } + + free(internalDstBuffer); free(resultBuffer); free((void*)srcPtrs); @@ -775,6 +775,10 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, if(allocationincomplete) { EXM_THROW(31, BMK_return_t, "allocation error : not enough memory"); } + + if(parametersConflict) { + EXM_THROW(32, BMK_return_t, "Conflicting input results"); + } return results; } diff --git a/programs/bench.h b/programs/bench.h index 2a9945ac8..8baf33a0a 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -175,7 +175,8 @@ typedef size_t (*BMK_initFn_t)(void*); * srcBuffers - an array of buffers to be operated on by benchFn * srcSizes - an array of the sizes of above buffers * dstBuffers - an array of buffers to be written into by benchFn - * dstCapacitiesToSizes - an array of the capacities of above buffers. Output modified to compressed sizes of those blocks. + * dstCapacities - an array of the capacities of above buffers + * blockResults - the return value of benchFn called on each block. * nbLoops - defines number of times benchFn is run. * assumed array of size blockCount, will have compressed size of each block written to it. * return @@ -192,7 +193,7 @@ BMK_customReturn_t BMK_benchFunction(BMK_benchFn_t benchFn, void* benchPayload, BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBuffers, const size_t* srcSizes, - void * const * const dstBuffers, const size_t* dstCapacities, size_t* cSizes, + void * const * const dstBuffers, const size_t* dstCapacities, size_t* blockResults, unsigned nbLoops); @@ -221,7 +222,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed(BMK_timedFnState_t* cont, BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBlockBuffers, const size_t* srcBlockSizes, - void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, size_t* cSizes); + void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, size_t* blockResults); #endif /* BENCH_H_121279284357 */ diff --git a/tests/README.md b/tests/README.md index 8bedd0a3c..2f0026fda 100644 --- a/tests/README.md +++ b/tests/README.md @@ -112,7 +112,6 @@ Full list of arguments dSpeed= - Minimum decompression speed cMem= - compression memory lvl= - Automatically sets compression speed constraint to the speed of that level - --optimize= : same as -O with more verbose syntax -P# : generated sample compressibility -t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) -v : Prints Benchmarking output diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 099be3688..a4f4a5f15 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -986,7 +986,7 @@ static void memoTableIndInv(ZSTD_compressionParameters* ptr, const varInds_t* va } /* Initialize memotable, immediately mark redundant / obviously infeasible params as */ -static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { +static void createMemoTable(U8* memoTable, ZSTD_compressionParameters paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { size_t i; size_t arrayLen = memoTableLen(varyParams, varyLen); int cwFixed = !paramConstraints.chainLog || !paramConstraints.windowLog; @@ -1053,23 +1053,24 @@ static void memoTableFreeAll(U8** mtAll) { /* inits memotables for all (including mallocs), all strategies */ /* takes unsanitized varyParams */ -static U8** memoTableInitAll(ZSTD_compressionParameters paramConstraints, constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { +static U8** createMemoTableArray(ZSTD_compressionParameters paramConstraints, constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { varInds_t varNew[NUM_PARAMS]; - int varLenNew; U8** mtAll = calloc(sizeof(U8*),(ZSTD_btultra + 1)); int i; if(mtAll == NULL) { return NULL; } + for(i = 1; i <= (int)ZSTD_btultra; i++) { - varLenNew = sanitizeVarArray(varNew, varyLen, varyParams, i); + const int varLenNew = sanitizeVarArray(varNew, varyLen, varyParams, i); mtAll[i] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew)); if(mtAll[i] == NULL) { memoTableFreeAll(mtAll); return NULL; } - memoTableInit(mtAll[i], paramConstraints, target, varNew, varLenNew, srcSize); + createMemoTable(mtAll[i], paramConstraints, target, varNew, varLenNew, srcSize); } + return mtAll; } @@ -1187,7 +1188,7 @@ static void BMK_benchFullTable(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* src winnerInfo_t winners[NB_LEVELS_TRACKED+1]; const char* const rfName = "grillResults.txt"; FILE* const f = fopen(rfName, "w"); - const size_t blockSize = g_blockSize ? g_blockSize : ZSTD_BLOCKSIZE_MAX; /* cut by block or not ? */ + const size_t blockSize = g_blockSize ? g_blockSize : srcSize; /* cut by block or not ? */ /* init */ assert(g_singleRun==0); @@ -1638,7 +1639,7 @@ static void freeBuffers(buffers_t b) { } /* allocates buffer's arguments. returns success / failuere */ -static int initBuffers(buffers_t* buff, const char* const * const fileNamesTable, +static int createBuffers(buffers_t* buff, const char* const * const fileNamesTable, size_t nbFiles) { size_t pos = 0; @@ -1659,7 +1660,7 @@ static int initBuffers(buffers_t* buff, const char* const * const fileNamesTable buff->resPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); buff->resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); - if(!buff->srcPtrs || !buff->srcSizes || !buff->dstPtrs || !buff->dstCapacities || !buff->dstCapacities || !buff->resPtrs || !buff->resSizes) { + if(!buff->srcPtrs || !buff->srcSizes || !buff->dstPtrs || !buff->dstCapacities || !buff->dstSizes || !buff->resPtrs || !buff->resSizes) { DISPLAY("alloc error\n"); freeBuffers(*buff); return 1; @@ -1747,7 +1748,7 @@ static void freeContexts(contexts_t ctx) { ZSTD_freeDCtx(ctx.dctx); } -static int initContexts(contexts_t* ctx, const char* dictFileName) { +static int createContexts(contexts_t* ctx, const char* dictFileName) { FILE* f; size_t readSize; ctx->cctx = ZSTD_createCCtx(); @@ -1846,12 +1847,12 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } /* load dictionary*/ - if(initBuffers(&buf, fileNamesTable, nbFiles)) { + if(createBuffers(&buf, fileNamesTable, nbFiles)) { DISPLAY("unable to load files\n"); return 1; } - if(initContexts(&ctx, dictFileName)) { + if(createContexts(&ctx, dictFileName)) { DISPLAY("unable to load dictionary\n"); freeBuffers(buf); return 2; @@ -1885,9 +1886,9 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ goto _cleanUp; } - memoTableInit(allMT[paramTarget.strategy], paramTarget, target, varNew, varLenNew, maxBlockSize); + createMemoTable(allMT[paramTarget.strategy], paramTarget, target, varNew, varLenNew, maxBlockSize); } else { - allMT = memoTableInitAll(paramTarget, target, varArray, varLen, maxBlockSize); + allMT = createMemoTableArray(paramTarget, target, varArray, varLen, maxBlockSize); } From 8faeb41679a5ef379cfb7893825e6ed98e571eb0 Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 30 Jul 2018 11:30:38 -0700 Subject: [PATCH 11/19] Update Documentation Change comment // to /* */ Add more description of what functions do Remove outdated comments --- tests/paramgrill.c | 117 ++++++++++++++++++++++++--------------------- 1 file changed, 63 insertions(+), 54 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index a4f4a5f15..b8396c72e 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -77,12 +77,11 @@ typedef enum { } varInds_t; #define NUM_PARAMS 6 -//just don't use strategy as a param. +/* just don't use strategy as a param. */ #undef ZSTD_WINDOWLOG_MAX #define ZSTD_WINDOWLOG_MAX 27 //no long range stuff for now. -//make 2^[0,10] w/ 999 #define ZSTD_TARGETLENGTH_MIN 0 #define ZSTD_TARGETLENGTH_MAX 999 @@ -274,7 +273,6 @@ static double resultScore(const BMK_result_t res, const size_t srcSize, const co ret = (MIN(1, cs) + MIN(1, ds) + MIN(1, cm))*r1 + rt * rtr + (MAX(0, log(cs))+ MAX(0, log(ds))+ MAX(0, log(cm))) * r2; - //DISPLAY("resultScore: %f\n", ret); return ret; } @@ -593,7 +591,7 @@ static BMK_return_t BMK_benchMemInvertible(buffers_t buf, contexts_t ctx, BMK_freeTimeState(timeStateCompress); BMK_freeTimeState(timeStateDecompress); - } else { //iterMode; + } else { /* iterMode; */ if(mode != BMK_decodeOnly) { BMK_customReturn_t compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)cctx, &local_initCCtx, (void*)&cctxprep, @@ -636,8 +634,7 @@ static BMK_return_t BMK_benchMemInvertible(buffers_t buf, contexts_t ctx, return results; } -/* global winner used for display. */ -//Should be totally 0 initialized? +/* global winner used for display. */ static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; static constraint_t g_targetConstraints; @@ -940,7 +937,7 @@ static void paramVariation(ZSTD_compressionParameters* ptr, const varInds_t* var } validated = !ZSTD_isError(ZSTD_checkCParams(p)); } - *ptr = p;//sanitizeParams(p); + *ptr = p; } /* length of memo table given free variables */ @@ -953,7 +950,7 @@ static size_t memoTableLen(const varInds_t* varyParams, const int varyLen) { return arrayLen; } -/* returns unique index of compression parameters */ +/* returns unique index in memotable of compression parameters */ static unsigned memoTableInd(const ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen) { int i; unsigned ind = 0; @@ -985,7 +982,7 @@ static void memoTableIndInv(ZSTD_compressionParameters* ptr, const varInds_t* va } } -/* Initialize memotable, immediately mark redundant / obviously infeasible params as */ +/* Initialize memotable, immediately mark redundant / obviously infeasible params as such */ static void createMemoTable(U8* memoTable, ZSTD_compressionParameters paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { size_t i; size_t arrayLen = memoTableLen(varyParams, varyLen); @@ -1003,7 +1000,7 @@ static void createMemoTable(U8* memoTable, ZSTD_compressionParameters paramConst memoTable[i] = 255; j++; } - if(wFixed && (1ULL << paramConstraints.windowLog) > (srcSize << 2)) { + if(wFixed && (1ULL << paramConstraints.windowLog) > srcSize) { memoTable[i] = 255; } /* nil out parameter sets equivalent to others. */ @@ -1121,7 +1118,7 @@ static void playAround(FILE* f, winnerInfo_t* winners, } - +/* Completely random parameter selection */ static ZSTD_compressionParameters randomParams(void) { ZSTD_compressionParameters p; @@ -1136,7 +1133,6 @@ static ZSTD_compressionParameters randomParams(void) p.targetLength=(FUZ_rand(&g_rand) % (512)); p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btultra +1)); validated = !ZSTD_isError(ZSTD_checkCParams(p)); - //validated = cParamValid(p); } return p; } @@ -1144,7 +1140,7 @@ static ZSTD_compressionParameters randomParams(void) /* Sets pc to random unmeasured set of parameters */ static void randomConstrainedParams(ZSTD_compressionParameters* pc, varInds_t* varArray, int varLen, U8* memoTable) { - size_t tries = memoTableLen(varArray, varLen); //configurable, + size_t tries = memoTableLen(varArray, varLen); const size_t maxSize = memoTableLen(varArray, varLen); size_t ind; do { @@ -1153,7 +1149,6 @@ static void randomConstrainedParams(ZSTD_compressionParameters* pc, varInds_t* v } while(memoTable[ind] > 0 && tries > 0); memoTableIndInv(pc, varArray, varLen, (unsigned)ind); - //*pc = sanitizeParams(*pc); } static void BMK_selectRandomStart( @@ -1328,19 +1323,13 @@ int benchFiles(const char** fileNamesTable, int nbFiles) return 0; } -/*benchmarks and tests feasibility together - 1 = true = better - 0 = false = not better - if true then resultPtr will give results. - 2+ on error? */ -//Maybe use compress_only for benchmark first run? + #define WORSE_RESULT 0 #define BETTER_RESULT 1 #define ERROR_RESULT 2 -//add worse result complete for worse results of length > 1 sec? -/* variation between 2nd run and full second bmk */ +/* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */ #define VARIANCE 1.1 static int allBench(BMK_result_t* resultPtr, buffers_t buf, contexts_t ctx, @@ -1447,8 +1436,9 @@ static int allBench(BMK_result_t* resultPtr, } -/* wrap feasibleBench w/ memotable */ #define INFEASIBLE_THRESHOLD 200 + +/* Memoized benchmarking, won't benchmark anything which has already been benchmarked before. */ static int benchMemo(BMK_result_t* resultPtr, buffers_t buf, contexts_t ctx, const ZSTD_compressionParameters cParams, @@ -1475,18 +1465,28 @@ static int benchMemo(BMK_result_t* resultPtr, return res; } - -//sanitize all params here. -//all generation after random should be sanitized. (maybe sanitize random) +/* One iteration of hill climbing. Specifically, it first tries all + * valid parameter configurations w/ manhattan distance 1 and picks the best one + * failing that, it progressively tries candidates further and further away (up to #dim + 2) + * if it finds a candidate exceeding winnerInfo, it will repeat. Otherwise, it will stop the + * current stage of hill climbing. + * Each iteration of hill climbing proceeds in 2 'phases'. Phase 1 climbs according to + * the resultScore function, which is effectively a linear increase in reward until it reaches + * the constraint-satisfying value, it which point any excess results in only logarithmic reward. + * This aims to find some constraint-satisfying point. + * Phase 2 optimizes in accordance with what the original function sets out to maximize, with + * all feasible solutions valued over all infeasible solutions. + */ static winnerInfo_t climbOnce(const constraint_t target, const varInds_t* varArray, const int varLen, U8* memoTable, buffers_t buf, contexts_t ctx, const ZSTD_compressionParameters init) { - //distance maximizing selection? - //cparam - currently considered 'center' - //candidate - params to benchmark/results - //winner - best option found so far. + /* + * cparam - currently considered 'center' + * candidate - params to benchmark/results + * winner - best option found so far. + */ ZSTD_compressionParameters cparam = init; winnerInfo_t candidateInfo, winnerInfo; int better = 1; @@ -1506,14 +1506,12 @@ static winnerInfo_t climbOnce(const constraint_t target, cparam = winnerInfo.params; BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, buf.srcSize); candidateInfo.params = cparam; - //all dist-1 targets - //if we early end this, we should also randomize the order these are picked. + /* all dist-1 candidates */ for(i = 0; i < varLen; i++) { for(offset = -1; offset <= 1; offset += 2) { candidateInfo.params = cparam; - paramVaryOnce(varArray[i], offset, &candidateInfo.params); /* +1 */ + paramVaryOnce(varArray[i], offset, &candidateInfo.params); candidateInfo.params = sanitizeParams(candidateInfo.params); - //evaluate if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { int res = benchMemo(&candidateInfo.result, buf, ctx, @@ -1560,7 +1558,7 @@ static winnerInfo_t climbOnce(const constraint_t target, } } - if(!better) { //infeas -> feas -> stop. + if(!better) { /* infeas -> feas -> stop */ if(feas) { return winnerInfo; } feas = 1; @@ -1575,14 +1573,14 @@ static winnerInfo_t climbOnce(const constraint_t target, return winnerInfo; } -//optimizeForSize but with fixed strategy -//place to configure/filter out strategy specific parameters. +/* Optimizes for a fixed strategy */ -//flexible parameters: iterations of (failed?) climbing (or if we do non-random, maybe this is when everything is close to visitied) -//weight more on visit for bad results, less on good results/more on later results / ones with more failures. -//allocate memoTable here. -//only real use for paramTarget is to get the fixed values, right? -//maybe allow giving it a first init? +/* flexible parameters: iterations of (failed?) climbing (or if we do non-random, maybe this is when everything is close to visitied) + weight more on visit for bad results, less on good results/more on later results / ones with more failures. + allocate memoTable here. + only real use for paramTarget is to get the fixed values, right? + maybe allow giving it a first init? + */ static winnerInfo_t optimizeFixedStrategy( buffers_t buf, contexts_t ctx, const constraint_t target, ZSTD_compressionParameters paramTarget, @@ -1603,9 +1601,8 @@ static winnerInfo_t optimizeFixedStrategy( init = paramTarget; - while(i < tries) { //make i adjustable (user input?) depending on how much time they have. + while(i < tries) { DEBUGOUTPUT("Restart\n"); - //look into improving this to maximize distance from searched infeasible stuff / towards promising regions? randomConstrainedParams(&init, varNew, varLenNew, memoTable); candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, buf, ctx, init); if(compareResultLT(winnerInfo.result, candidateInfo.result, target, buf.srcSize)) { @@ -1638,7 +1635,7 @@ static void freeBuffers(buffers_t b) { free(b.resPtrs); } -/* allocates buffer's arguments. returns success / failuere */ +/* allocates buffer's arguments. returns 0 = success / 1 = failuere */ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTable, size_t nbFiles) { @@ -1748,16 +1745,25 @@ static void freeContexts(contexts_t ctx) { ZSTD_freeDCtx(ctx.dctx); } +/* Creates struct holding contexts and dictionary buffers. returns 0 on success, 1 on failure. */ static int createContexts(contexts_t* ctx, const char* dictFileName) { FILE* f; size_t readSize; ctx->cctx = ZSTD_createCCtx(); ctx->dctx = ZSTD_createDCtx(); + ctx->dictSize = 0; + ctx->dictBuffer = NULL; + + if(!ctx->cctx || !ctx->dctx) { + DISPLAY("context allocation error\n"); + freeContexts(*ctx); + return 1; + } + if(dictFileName == NULL) { - ctx->dictSize = 0; - ctx->dictBuffer = NULL; return 0; } + ctx->dictSize = UTIL_getFileSize(dictFileName); ctx->dictBuffer = malloc(ctx->dictSize); @@ -1786,8 +1792,8 @@ static int createContexts(contexts_t* ctx, const char* dictFileName) { return 0; } -//goes best, best-1, best+1, best-2, ... -//return 0 if nothing remaining +/* goes best, best-1, best+1, best-2, ... */ +/* return 0 if nothing remaining */ static int nextStrategy(const int currentStrategy, const int bestStrategy) { if(bestStrategy <= currentStrategy) { int candidate = 2 * bestStrategy - currentStrategy - 1; @@ -1828,7 +1834,10 @@ static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZS } #define MAX_TRIES 8 -//optimize fixed strategy. +/* main fn called when using --optimize */ +/* Does strategy selection by benchmarking default compression levels + * then optimizes by strategy, starting with the best one and moving + * progressively moving further away by number */ static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel) { varInds_t varArray [NUM_PARAMS]; @@ -1905,7 +1914,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ goto _cleanUp; } - target.cSpeed = (U32)winner.result.cSpeed; //Maybe have a small bit of slack here, like x.99? + target.cSpeed = (U32)winner.result.cSpeed; g_targetConstraints = target; BMK_printWinner(stdout, cLevel, winner.result, winner.params, buf.srcSize); } @@ -1978,7 +1987,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) { winner = wc; } - //We could double back to increase search of 'better' strategies + st = nextStrategy(st, bestStrategy); tries--; } @@ -2088,7 +2097,7 @@ int main(int argc, const char** argv) int optimizerCLevel = 0; - constraint_t target = { 0, 0, (U32)-1 }; //0 for anything unset + constraint_t target = { 0, 0, (U32)-1 }; ZSTD_compressionParameters paramTarget = { 0, 0, 0, 0, 0, 0, 0 }; assert(argc>=1); /* for exename */ @@ -2250,7 +2259,7 @@ int main(int argc, const char** argv) /* load dictionary file (only applicable for optimizer rn) */ case 'D': - if(i == argc - 1) { //last argument, return error. + if(i == argc - 1) { /* last argument, return error. */ DISPLAY("Dictionary file expected but not given : %d\n", i); return 1; } else { From 3d230db85391aa5810abc6301035e9741c986765 Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 6 Aug 2018 17:13:36 -0700 Subject: [PATCH 12/19] Change speed representation from floating point to integral --- programs/bench.c | 43 +++++++++++++++++++------------------------ programs/bench.h | 4 ++-- tests/paramgrill.c | 46 +++++++++++++++++----------------------------- 3 files changed, 38 insertions(+), 55 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index b496caf27..49b317870 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -555,9 +555,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( ratio = (double)(srcSize / intermediateResultCompress.result.result.sumOfReturn); { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - double const compressionSpeed = ((double)srcSize / intermediateResultCompress.result.result.nanoSecPerRun) * 1000; - int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; - results.result.cSpeed = compressionSpeed * 1000000; + results.result.cSpeed = (srcSize * TIMELOOP_NANOSEC / intermediateResultCompress.result.result.nanoSecPerRun); cSize = intermediateResultCompress.result.result.sumOfReturn; results.result.cSize = cSize; ratio = (double)srcSize / results.result.cSize; @@ -565,7 +563,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r", marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize, ratioAccuracy, ratio, - cSpeedAccuracy, compressionSpeed); + results.result.cSpeed < (10 MB) ? 2 : 1, (double)results.result.cSpeed / (1 MB)); } } @@ -579,16 +577,13 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - double const compressionSpeed = results.result.cSpeed / 1000000; - int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; - double const decompressionSpeed = ((double)srcSize / intermediateResultDecompress.result.result.nanoSecPerRun) * 1000; - results.result.dSpeed = decompressionSpeed * 1000000; + results.result.dSpeed = (srcSize * TIMELOOP_NANOSEC/ intermediateResultDecompress.result.result.nanoSecPerRun); markNb = (markNb+1) % NB_MARKS; DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r", marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize, ratioAccuracy, ratio, - cSpeedAccuracy, compressionSpeed, - decompressionSpeed); + results.result.cSpeed < (10 MB) ? 2 : 1, (double)results.result.cSpeed / (1 MB), + (double)results.result.dSpeed / (1 MB)); } } } @@ -605,19 +600,20 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( if(compressionResults.result.nanoSecPerRun == 0) { results.result.cSpeed = 0; } else { - results.result.cSpeed = (double)srcSize / compressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; + results.result.cSpeed = srcSize * TIMELOOP_NANOSEC / compressionResults.result.nanoSecPerRun; } results.result.cSize = compressionResults.result.sumOfReturn; { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - double const compressionSpeed = results.result.cSpeed / 1000000; - int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; + results.result.cSpeed = (srcSize * TIMELOOP_NANOSEC / compressionResults.result.nanoSecPerRun); + cSize = compressionResults.result.sumOfReturn; + results.result.cSize = cSize; ratio = (double)srcSize / results.result.cSize; markNb = (markNb+1) % NB_MARKS; DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r", marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize, ratioAccuracy, ratio, - cSpeedAccuracy, compressionSpeed); + results.result.cSpeed < (10 MB) ? 2 : 1, (double)results.result.cSpeed / (1 MB)); } } if(adv->mode != BMK_compressOnly) { @@ -633,19 +629,18 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( if(decompressionResults.result.nanoSecPerRun == 0) { results.result.dSpeed = 0; } else { - results.result.dSpeed = (double)srcSize / decompressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; + results.result.dSpeed = srcSize * TIMELOOP_NANOSEC / decompressionResults.result.nanoSecPerRun; } - { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - double const compressionSpeed = results.result.cSpeed / 1000000; - int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; - double const decompressionSpeed = ((double)srcSize / decompressionResults.result.nanoSecPerRun) * 1000; - results.result.dSpeed = decompressionSpeed * 1000000; + + { + int const ratioAccuracy = (ratio < 10.) ? 3 : 2; + results.result.dSpeed = (srcSize * TIMELOOP_NANOSEC/ decompressionResults.result.nanoSecPerRun); markNb = (markNb+1) % NB_MARKS; DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r", marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize, ratioAccuracy, ratio, - cSpeedAccuracy, compressionSpeed, - decompressionSpeed); + results.result.cSpeed < (10 MB) ? 2 : 1, (double)results.result.cSpeed / (1 MB), + (double)results.result.dSpeed / (1 MB)); } } } @@ -693,8 +688,8 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( } /* CRC Checking */ if (displayLevel == 1) { /* hidden display mode -q, used by python speed benchmark */ - double const cSpeed = results.result.cSpeed / 1000000; - double const dSpeed = results.result.dSpeed / 1000000; + double const cSpeed = (double)results.result.cSpeed / (1 MB); + double const dSpeed = (double)results.result.dSpeed / (1 MB); if (adv->additionalParam) { DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, adv->additionalParam); } else { diff --git a/programs/bench.h b/programs/bench.h index 8baf33a0a..6247fa596 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -32,8 +32,8 @@ extern "C" { typedef struct { size_t cSize; - double cSpeed; /* bytes / sec */ - double dSpeed; + U64 cSpeed; /* bytes / sec */ + U64 dSpeed; size_t cMem; } BMK_result_t; diff --git a/tests/paramgrill.c b/tests/paramgrill.c index b8396c72e..5af3d88fa 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -249,18 +249,6 @@ static int feasible(const BMK_result_t results, const constraint_t target) { return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem); } -#define EPSILON 0.001 -static int epsilonEqual(const double c1, const double c2) { - return MAX(c1/c2,c2/c1) < 1 + EPSILON; -} - -/* checks exact equivalence to 0, to stop compiler complaining fpeq */ -static int eqZero(const double c1) { - const double z1 = 0.0; - const double z2 = -0.0; - return !(memcmp(&c1, &z1, sizeof(double))) || !(memcmp(&c1, &z2, sizeof(double))); -} - /* hill climbing value for part 1 */ static double resultScore(const BMK_result_t res, const size_t srcSize, const constraint_t target) { double cs = 0., ds = 0., rt, cm = 0.; @@ -280,7 +268,7 @@ static double resultScore(const BMK_result_t res, const size_t srcSize, const co static int compareResultLT(const BMK_result_t result1, const BMK_result_t result2, const constraint_t target, size_t srcSize) { if(feasible(result1, target) && feasible(result2, target)) { return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed) - || (result1.cSize == result2.cSize && epsilonEqual(result2.cSpeed, result1.cSpeed) && result2.dSpeed > result1.dSpeed); + || (result1.cSize == result2.cSize && result2.cSpeed == result1.cSpeed && result2.dSpeed > result1.dSpeed); } return feasible(result2, target) || (!feasible(result1, target) && (resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target))); @@ -661,7 +649,7 @@ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result fprintf(f, "/* %s */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */", - lvlstr, (double)srcSize / result.cSize, result.cSpeed / (1 << 20), result.dSpeed / (1 << 20)); + lvlstr, (double)srcSize / result.cSize, (double)result.cSpeed / (1 << 20), (double)result.dSpeed / (1 << 20)); if(TIMED) { fprintf(f, " - %1lu:%2lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); } fprintf(f, "\n"); @@ -696,8 +684,8 @@ static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, size_t srcSiz typedef struct { - double cSpeed_min; - double dSpeed_min; + U64 cSpeed_min; + U64 dSpeed_min; U32 windowLog_max; ZSTD_strategy strategy_max; } level_constraints_t; @@ -794,16 +782,16 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para /* too large compression speed difference for the compression benefit */ if (W_ratio > O_ratio) DISPLAY ("Compression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n", - W_ratio, testResult.cSpeed / 1000000, - O_ratio, winners[cLevel].result.cSpeed / 1000000., cLevel); + W_ratio, (double)testResult.cSpeed / 1000000, + O_ratio, (double)winners[cLevel].result.cSpeed / 1000000., cLevel); continue; } if (W_DSpeed_note < O_DSpeed_note ) { /* too large decompression speed difference for the compression benefit */ if (W_ratio > O_ratio) DISPLAY ("Decompression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n", - W_ratio, testResult.dSpeed / 1000000., - O_ratio, winners[cLevel].result.dSpeed / 1000000., cLevel); + W_ratio, (double)testResult.dSpeed / 1000000., + O_ratio, (double)winners[cLevel].result.dSpeed / 1000000., cLevel); continue; } @@ -1173,7 +1161,7 @@ static void BMK_benchOnce(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* srcBuffe g_params = ZSTD_adjustCParams(g_params, srcSize, 0); BMK_benchParam1(&testResult, srcBuffer, srcSize, cctx, dctx, g_params); DISPLAY("Compression Ratio: %.3f Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)srcSize / testResult.cSize, - testResult.cSpeed / 1000000, testResult.dSpeed / 1000000); + (double)testResult.cSpeed / 1000000, (double)testResult.dSpeed / 1000000); return; } @@ -1355,20 +1343,20 @@ static int allBench(BMK_result_t* resultPtr, *resultPtr = benchres.result; /* calculate uncertainty in compression / decompression runs */ - if(eqZero(benchres.result.cSpeed)) { + if(benchres.result.cSpeed) { + loopDurationC = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); + uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC) * VARIANCE; + } else { loopDurationC = 0; uncertaintyConstantC = 3; - } else { - loopDurationC = (U64)((double)(buf.srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); - uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC) * VARIANCE; } - if(eqZero(benchres.result.dSpeed)) { + if(benchres.result.dSpeed) { + loopDurationD = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); + uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD) * VARIANCE; + } else { loopDurationD = 0; uncertaintyConstantD = 3; - } else { - loopDurationD = (U64)((double)(buf.srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); - uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD) * VARIANCE; } /* anything with worse ratio in feas is definitely worse, discard */ From 8278a49cb608af483b70f301742899f5b0b8ab7d Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 6 Aug 2018 18:00:36 -0700 Subject: [PATCH 13/19] const srcPtrs --- programs/bench.c | 2 +- tests/paramgrill.c | 23 +++++++++++++++-------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 49b317870..874710c02 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -441,7 +441,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( int displayLevel, const char* displayName, const BMK_advancedParams_t* adv) { size_t const blockSize = ((adv->blockSize>=32 && (adv->mode != BMK_decodeOnly)) ? adv->blockSize : srcSize) + (!srcSize); /* avoid div by 0 */ - BMK_return_t results = { { 0, 0., 0., 0 }, 0 } ; + BMK_return_t results = { { 0, 0, 0, 0 }, 0 } ; size_t const loadedCompressedSize = srcSize; size_t cSize = 0; double ratio = 0.; diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 5af3d88fa..5d1a73081 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -250,6 +250,11 @@ static int feasible(const BMK_result_t results, const constraint_t target) { } /* hill climbing value for part 1 */ +/* Scoring here is a linear reward for all set constraints normalized between 0 to 1 + * (with 0 at 0 and 1 being fully fulfilling the constraint), summed with a logarithmic + * bonus to exceeding the constraint value. We also give linear ratio for compression ratio. + * The constant factors are experimental. + */ static double resultScore(const BMK_result_t res, const size_t srcSize, const constraint_t target) { double cs = 0., ds = 0., rt, cm = 0.; const double r1 = 1, r2 = 0.1, rtr = 0.5; @@ -291,7 +296,7 @@ const char* g_stratName[ZSTD_btultra+1] = { "ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ", "ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "}; -/* benchParam but only takes in one file. */ +/* benchParam but only takes in one input buffer. */ static int BMK_benchParam1(BMK_result_t* resultPtr, const void* srcBuffer, size_t srcSize, @@ -324,8 +329,9 @@ static winnerInfo_t initWinnerInfo(ZSTD_compressionParameters p) { } typedef struct { + void* srcBuffer; size_t srcSize; - void** srcPtrs; + const void** srcPtrs; size_t* srcSizes; void** dstPtrs; size_t* dstCapacities; @@ -1605,7 +1611,7 @@ static winnerInfo_t optimizeFixedStrategy( static void freeBuffers(buffers_t b) { if(b.srcPtrs != NULL) { - free(b.srcPtrs[0]); + free(b.srcBuffer); } free(b.srcPtrs); free(b.srcSizes); @@ -1635,7 +1641,7 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab U32 const maxNbBlocks = (U32) ((totalSizeToLoad + (blockSize-1)) / blockSize) + (U32)nbFiles; U32 blockNb = 0; - buff->srcPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); + buff->srcPtrs = (const void**)calloc(maxNbBlocks, sizeof(void*)); buff->srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); buff->dstPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); @@ -1651,7 +1657,8 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab return 1; } - buff->srcPtrs[0] = malloc(benchedSize); + buff->srcBuffer = malloc(benchedSize); + buff->srcPtrs[0] = (const void*)buff->srcBuffer; buff->dstPtrs[0] = malloc(ZSTD_compressBound(benchedSize) + (maxNbBlocks * 1024)); buff->resPtrs[0] = malloc(benchedSize); @@ -1684,11 +1691,11 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, nbFiles=n; /* buffer too small - stop after this file */ { - char* buffer = (char*)(buff->srcPtrs[0]); - size_t const readSize = fread((buffer)+pos, 1, (size_t)fileSize, f); + char* buffer = (char*)(buff->srcBuffer); + size_t const readSize = fread(((buffer)+pos), 1, (size_t)fileSize, f); size_t blocked = 0; while(blocked < readSize) { - buff->srcPtrs[blockNb] = (buffer) + (pos + blocked); + buff->srcPtrs[blockNb] = (const void*)((buffer) + (pos + blocked)); buff->srcSizes[blockNb] = blockSize; blocked += blockSize; blockNb++; From ad16a69408139d87cf44eb830b6749bf6066bee1 Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 6 Aug 2018 18:37:55 -0700 Subject: [PATCH 14/19] Readability improvements, renaming --- tests/paramgrill.c | 131 +++++++++++++++++++++++++-------------------- 1 file changed, 72 insertions(+), 59 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 5d1a73081..c0679e88e 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -362,7 +362,7 @@ BMK_benchParam(BMK_result_t* resultPtr, *********************************************************/ static void BMK_initCCtx(ZSTD_CCtx* ctx, - const void* dictBuffer, size_t dictBufferSize, int cLevel, + const void* dictBuffer, const size_t dictBufferSize, const int cLevel, const ZSTD_compressionParameters* comprParams, const BMK_advancedParams_t* adv) { ZSTD_CCtx_reset(ctx); ZSTD_CCtx_resetParameters(ctx); @@ -389,7 +389,7 @@ static void BMK_initCCtx(ZSTD_CCtx* ctx, static void BMK_initDCtx(ZSTD_DCtx* dctx, - const void* dictBuffer, size_t dictBufferSize) { + const void* dictBuffer, const size_t dictBufferSize) { ZSTD_DCtx_reset(dctx); ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize); } @@ -404,7 +404,7 @@ typedef struct { } BMK_initCCtxArgs; static size_t local_initCCtx(void* payload) { - BMK_initCCtxArgs* ag = (BMK_initCCtxArgs*)payload; + const BMK_initCCtxArgs* ag = (const BMK_initCCtxArgs*)payload; BMK_initCCtx(ag->ctx, ag->dictBuffer, ag->dictBufferSize, ag->cLevel, ag->comprParams, ag->adv); return 0; } @@ -416,7 +416,7 @@ typedef struct { } BMK_initDCtxArgs; static size_t local_initDCtx(void* payload) { - BMK_initDCtxArgs* ag = (BMK_initDCtxArgs*)payload; + const BMK_initDCtxArgs* ag = (const BMK_initDCtxArgs*)payload; BMK_initDCtx(ag->dctx, ag->dictBuffer, ag->dictBufferSize); return 0; } @@ -565,7 +565,7 @@ static BMK_return_t BMK_benchMemInvertible(buffers_t buf, contexts_t ctx, BMK_freeTimeState(timeStateDecompress); return results; } - results.result.cSpeed = ((double)srcSize / intermediateResultCompress.result.result.nanoSecPerRun) * TIMELOOP_NANOSEC; + results.result.cSpeed = (srcSize * TIMELOOP_NANOSEC) / intermediateResultCompress.result.result.nanoSecPerRun; results.result.cSize = intermediateResultCompress.result.result.sumOfReturn; } @@ -579,7 +579,7 @@ static BMK_return_t BMK_benchMemInvertible(buffers_t buf, contexts_t ctx, BMK_freeTimeState(timeStateDecompress); return results; } - results.result.dSpeed = ((double)srcSize / intermediateResultDecompress.result.result.nanoSecPerRun) * TIMELOOP_NANOSEC; + results.result.dSpeed = (srcSize * TIMELOOP_NANOSEC) / intermediateResultDecompress.result.result.nanoSecPerRun; } BMK_freeTimeState(timeStateCompress); @@ -597,7 +597,7 @@ static BMK_return_t BMK_benchMemInvertible(buffers_t buf, contexts_t ctx, if(compressionResults.result.nanoSecPerRun == 0) { results.result.cSpeed = 0; } else { - results.result.cSpeed = (double)srcSize / compressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; + results.result.cSpeed = srcSize * TIMELOOP_NANOSEC / compressionResults.result.nanoSecPerRun; } results.result.cSize = compressionResults.result.sumOfReturn; } @@ -618,7 +618,7 @@ static BMK_return_t BMK_benchMemInvertible(buffers_t buf, contexts_t ctx, if(decompressionResults.result.nanoSecPerRun == 0) { results.result.dSpeed = 0; } else { - results.result.dSpeed = (double)srcSize / decompressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; + results.result.dSpeed = srcSize * TIMELOOP_NANOSEC / decompressionResults.result.nanoSecPerRun; } } } @@ -628,39 +628,44 @@ static BMK_return_t BMK_benchMemInvertible(buffers_t buf, contexts_t ctx, return results; } -/* global winner used for display. */ -static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; -static constraint_t g_targetConstraints; static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const size_t srcSize) { - if(DEBUG || compareResultLT(g_winner.result, result, g_targetConstraints, srcSize)) { - char lvlstr[15] = "Custom Level"; - const U64 time = UTIL_clockSpanNano(g_time); - const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC); + char lvlstr[15] = "Custom Level"; + const U64 time = UTIL_clockSpanNano(g_time); + const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC); - if(DEBUG && compareResultLT(g_winner.result, result, g_targetConstraints, srcSize)) { + DISPLAY("\r%79s\r", ""); + + fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", + params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, + params.targetLength, g_stratName[(U32)(params.strategy)]); + + if(cLevel != CUSTOM_LEVEL) { + snprintf(lvlstr, 15, " Level %2u ", cLevel); + } + + fprintf(f, + "/* %s */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */", + lvlstr, (double)srcSize / result.cSize, (double)result.cSpeed / (1 << 20), (double)result.dSpeed / (1 << 20)); + + if(TIMED) { fprintf(f, " - %1lu:%2lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); } + fprintf(f, "\n"); +} + +static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const constraint_t targetConstraints, const size_t srcSize) +{ + /* global winner used for constraints */ + static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; + + if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { + if(DEBUG && compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { DISPLAY("New Winner: \n"); } - DISPLAY("\r%79s\r", ""); + BMK_printWinner(f, cLevel, result, params, srcSize); - fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", - params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, - params.targetLength, g_stratName[(U32)(params.strategy)]); - - if(cLevel != CUSTOM_LEVEL) { - snprintf(lvlstr, 15, " Level %2u ", cLevel); - } - - fprintf(f, - "/* %s */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */", - lvlstr, (double)srcSize / result.cSize, (double)result.cSpeed / (1 << 20), (double)result.dSpeed / (1 << 20)); - - if(TIMED) { fprintf(f, " - %1lu:%2lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); } - fprintf(f, "\n"); - - if(compareResultLT(g_winner.result, result, g_targetConstraints, srcSize)) { + if(compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { BMK_translateAdvancedParams(params); g_winner.result = result; g_winner.params = params; @@ -950,12 +955,18 @@ static unsigned memoTableInd(const ZSTD_compressionParameters* ptr, const varInd unsigned ind = 0; for(i = 0; i < varyLen; i++) { switch(varyParams[i]) { - case wlog_ind: ind *= WLOG_RANGE; ind += ptr->windowLog - ZSTD_WINDOWLOG_MIN ; break; - case clog_ind: ind *= CLOG_RANGE; ind += ptr->chainLog - ZSTD_CHAINLOG_MIN ; break; - case hlog_ind: ind *= HLOG_RANGE; ind += ptr->hashLog - ZSTD_HASHLOG_MIN ; break; - case slog_ind: ind *= SLOG_RANGE; ind += ptr->searchLog - ZSTD_SEARCHLOG_MIN ; break; - case slen_ind: ind *= SLEN_RANGE; ind += ptr->searchLength - ZSTD_SEARCHLENGTH_MIN; break; - case tlen_ind: ind *= TLEN_RANGE; ind += tlen_inv(ptr->targetLength) - ZSTD_TARGETLENGTH_MIN; break; + case wlog_ind: ind *= WLOG_RANGE; ind += ptr->windowLog + - ZSTD_WINDOWLOG_MIN ; break; + case clog_ind: ind *= CLOG_RANGE; ind += ptr->chainLog + - ZSTD_CHAINLOG_MIN ; break; + case hlog_ind: ind *= HLOG_RANGE; ind += ptr->hashLog + - ZSTD_HASHLOG_MIN ; break; + case slog_ind: ind *= SLOG_RANGE; ind += ptr->searchLog + - ZSTD_SEARCHLOG_MIN ; break; + case slen_ind: ind *= SLEN_RANGE; ind += ptr->searchLength + - ZSTD_SEARCHLENGTH_MIN; break; + case tlen_ind: ind *= TLEN_RANGE; ind += tlen_inv(ptr->targetLength) + - ZSTD_TARGETLENGTH_MIN; break; } } return ind; @@ -976,8 +987,12 @@ static void memoTableIndInv(ZSTD_compressionParameters* ptr, const varInds_t* va } } -/* Initialize memotable, immediately mark redundant / obviously infeasible params as such */ -static void createMemoTable(U8* memoTable, ZSTD_compressionParameters paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { +/* Initialize memoization table, which tracks and prevents repeated benchmarking + * of the same set of parameters. In addition, it is also used to immediately mark + * redundant / obviously non-optimal parameter configurations (e.g. wlog - 1 larger) + * than srcSize, clog > wlog, ... + */ +static void initMemoTable(U8* memoTable, ZSTD_compressionParameters paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { size_t i; size_t arrayLen = memoTableLen(varyParams, varyLen); int cwFixed = !paramConstraints.chainLog || !paramConstraints.windowLog; @@ -985,6 +1000,7 @@ static void createMemoTable(U8* memoTable, ZSTD_compressionParameters paramConst int whFixed = !paramConstraints.windowLog || !paramConstraints.hashLog; int wFixed = !paramConstraints.windowLog; int j = 0; + assert(memoTable != NULL); memset(memoTable, 0, arrayLen); cParamZeroMin(¶mConstraints); @@ -994,7 +1010,7 @@ static void createMemoTable(U8* memoTable, ZSTD_compressionParameters paramConst memoTable[i] = 255; j++; } - if(wFixed && (1ULL << paramConstraints.windowLog) > srcSize) { + if(wFixed && (1ULL << (paramConstraints.windowLog - 1)) > srcSize) { memoTable[i] = 255; } /* nil out parameter sets equivalent to others. */ @@ -1033,7 +1049,7 @@ static void createMemoTable(U8* memoTable, ZSTD_compressionParameters paramConst } /* frees all allocated memotables */ -static void memoTableFreeAll(U8** mtAll) { +static void freeMemoTableArray(U8** mtAll) { int i; if(mtAll == NULL) { return; } for(i = 1; i <= (int)ZSTD_btultra; i++) { @@ -1056,10 +1072,10 @@ static U8** createMemoTableArray(ZSTD_compressionParameters paramConstraints, co const int varLenNew = sanitizeVarArray(varNew, varyLen, varyParams, i); mtAll[i] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew)); if(mtAll[i] == NULL) { - memoTableFreeAll(mtAll); + freeMemoTableArray(mtAll); return NULL; } - createMemoTable(mtAll[i], paramConstraints, target, varNew, varLenNew, srcSize); + initMemoTable(mtAll[i], paramConstraints, target, varNew, varLenNew, srcSize); } return mtAll; @@ -1451,7 +1467,7 @@ static int benchMemo(BMK_result_t* resultPtr, DISPLAY("Count: %d\n", bmcount); bmcount++; } - BMK_printWinner(stdout, CUSTOM_LEVEL, *resultPtr, cParams, buf.srcSize); + BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, *resultPtr, cParams, target, buf.srcSize); if(res == BETTER_RESULT || feas) { memoTable[memind] = 255; @@ -1498,7 +1514,7 @@ static winnerInfo_t climbOnce(const constraint_t target, better = 0; DEBUGOUTPUT("Start\n"); cparam = winnerInfo.params; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, buf.srcSize); + BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize); candidateInfo.params = cparam; /* all dist-1 candidates */ for(i = 0; i < varLen; i++) { @@ -1513,7 +1529,7 @@ static winnerInfo_t climbOnce(const constraint_t target, varArray, varLen, feas); if(res == BETTER_RESULT) { /* synonymous with better when called w/ infeasibleBM */ winnerInfo = candidateInfo; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, buf.srcSize); + BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize); better = 1; if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) { bestFeasible1 = winnerInfo; @@ -1539,7 +1555,7 @@ static winnerInfo_t climbOnce(const constraint_t target, varArray, varLen, feas); if(res == BETTER_RESULT) { /* synonymous with better in this case*/ winnerInfo = candidateInfo; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, buf.srcSize); + BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize); better = 1; if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) { bestFeasible1 = winnerInfo; @@ -1601,7 +1617,7 @@ static winnerInfo_t optimizeFixedStrategy( candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, buf, ctx, init); if(compareResultLT(winnerInfo.result, candidateInfo.result, target, buf.srcSize)) { winnerInfo = candidateInfo; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, buf.srcSize); + BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize); i = 0; } i++; @@ -1890,7 +1906,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ goto _cleanUp; } - createMemoTable(allMT[paramTarget.strategy], paramTarget, target, varNew, varLenNew, maxBlockSize); + initMemoTable(allMT[paramTarget.strategy], paramTarget, target, varNew, varLenNew, maxBlockSize); } else { allMT = createMemoTableArray(paramTarget, target, varArray, varLen, maxBlockSize); } @@ -1910,12 +1926,9 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } target.cSpeed = (U32)winner.result.cSpeed; - g_targetConstraints = target; - BMK_printWinner(stdout, cLevel, winner.result, winner.params, buf.srcSize); + BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize); } - g_targetConstraints = target; - /* bench */ DISPLAY("\r%79s\r", ""); if(nbFiles == 1) { @@ -1946,7 +1959,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ ZSTD_compressionParameters CParams = ZSTD_getCParams(i, maxBlockSize, ctx.dictSize); CParams = maskParams(CParams, paramTarget); ec = BMK_benchParam(&candidate, buf, ctx, CParams); - BMK_printWinner(stdout, i, candidate, CParams, buf.srcSize); + BMK_printWinnerOpt(stdout, i, candidate, CParams, target, buf.srcSize); if(!ec && compareResultLT(winner.result, candidate, relaxTarget(target), buf.srcSize)) { winner.result = candidate; @@ -1956,7 +1969,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } } - BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, buf.srcSize); + BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winner.result, winner.params, target, buf.srcSize); BMK_translateAdvancedParams(winner.params); DEBUGOUTPUT("Real Opt\n"); /* start 'real' tests */ @@ -2000,7 +2013,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ goto _cleanUp; } /* end summary */ - BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, buf.srcSize); + BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winner.result, winner.params, target, buf.srcSize); BMK_translateAdvancedParams(winner.params); DISPLAY("grillParams size - optimizer completed \n"); @@ -2008,7 +2021,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ _cleanUp: freeContexts(ctx); freeBuffers(buf); - memoTableFreeAll(allMT); + freeMemoTableArray(allMT); return ret; } From 6f480927af465d34ac9914789291094477c619bb Mon Sep 17 00:00:00 2001 From: George Lu Date: Tue, 7 Aug 2018 11:56:14 -0700 Subject: [PATCH 15/19] argument parsing cleanup + clarifying comment --- tests/paramgrill.c | 95 +++++++++++++++++++++++++++++++--------------- 1 file changed, 64 insertions(+), 31 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index c0679e88e..a1d409d67 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -977,12 +977,18 @@ static void memoTableIndInv(ZSTD_compressionParameters* ptr, const varInds_t* va int i; for(i = varyLen - 1; i >= 0; i--) { switch(varyParams[i]) { - case wlog_ind: ptr->windowLog = ind % WLOG_RANGE + ZSTD_WINDOWLOG_MIN; ind /= WLOG_RANGE; break; - case clog_ind: ptr->chainLog = ind % CLOG_RANGE + ZSTD_CHAINLOG_MIN; ind /= CLOG_RANGE; break; - case hlog_ind: ptr->hashLog = ind % HLOG_RANGE + ZSTD_HASHLOG_MIN; ind /= HLOG_RANGE; break; - case slog_ind: ptr->searchLog = ind % SLOG_RANGE + ZSTD_SEARCHLOG_MIN; ind /= SLOG_RANGE; break; - case slen_ind: ptr->searchLength = ind % SLEN_RANGE + ZSTD_SEARCHLENGTH_MIN; ind /= SLEN_RANGE; break; - case tlen_ind: ptr->targetLength = tlen_table[(ind % TLEN_RANGE)]; ind /= TLEN_RANGE; break; + case wlog_ind: ptr->windowLog = ind % WLOG_RANGE + ZSTD_WINDOWLOG_MIN; + ind /= WLOG_RANGE; break; + case clog_ind: ptr->chainLog = ind % CLOG_RANGE + ZSTD_CHAINLOG_MIN; + ind /= CLOG_RANGE; break; + case hlog_ind: ptr->hashLog = ind % HLOG_RANGE + ZSTD_HASHLOG_MIN; + ind /= HLOG_RANGE; break; + case slog_ind: ptr->searchLog = ind % SLOG_RANGE + ZSTD_SEARCHLOG_MIN; + ind /= SLOG_RANGE; break; + case slen_ind: ptr->searchLength = ind % SLEN_RANGE + ZSTD_SEARCHLENGTH_MIN; + ind /= SLEN_RANGE; break; + case tlen_ind: ptr->targetLength = tlen_table[(ind % TLEN_RANGE)]; + ind /= TLEN_RANGE; break; } } } @@ -1135,13 +1141,20 @@ static ZSTD_compressionParameters randomParams(void) U32 validated = 0; while (!validated) { /* totally random entry */ - p.chainLog = (FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN)) + ZSTD_CHAINLOG_MIN; - p.hashLog = (FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN)) + ZSTD_HASHLOG_MIN; - p.searchLog = (FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN)) + ZSTD_SEARCHLOG_MIN; - p.windowLog = (FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN)) + ZSTD_WINDOWLOG_MIN; - p.searchLength=(FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN)) + ZSTD_SEARCHLENGTH_MIN; + p.chainLog = (FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN)) + + ZSTD_CHAINLOG_MIN; + p.hashLog = (FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN)) + + ZSTD_HASHLOG_MIN; + p.searchLog = (FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN)) + + ZSTD_SEARCHLOG_MIN; + p.windowLog = (FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN)) + + ZSTD_WINDOWLOG_MIN; + p.searchLength=(FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN)) + + ZSTD_SEARCHLENGTH_MIN; p.targetLength=(FUZ_rand(&g_rand) % (512)); + p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btultra +1)); + validated = !ZSTD_isError(ZSTD_checkCParams(p)); } return p; @@ -1657,6 +1670,8 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab U32 const maxNbBlocks = (U32) ((totalSizeToLoad + (blockSize-1)) / blockSize) + (U32)nbFiles; U32 blockNb = 0; + memset(buff, 0, sizeof(buffers_t)); + buff->srcPtrs = (const void**)calloc(maxNbBlocks, sizeof(void*)); buff->srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); @@ -1760,6 +1775,7 @@ static void freeContexts(contexts_t ctx) { static int createContexts(contexts_t* ctx, const char* dictFileName) { FILE* f; size_t readSize; + U64 dictSize; ctx->cctx = ZSTD_createCCtx(); ctx->dctx = ZSTD_createDCtx(); ctx->dictSize = 0; @@ -1775,7 +1791,16 @@ static int createContexts(contexts_t* ctx, const char* dictFileName) { return 0; } - ctx->dictSize = UTIL_getFileSize(dictFileName); + dictSize = UTIL_getFileSize(dictFileName); + + if(dictSize == UTIL_FILESIZE_UNKNOWN) { + DISPLAY("Unable to get dictionary size\n"); + freeContexts(*ctx); + return 1; + } else { + ctx->dictSize = (size_t)dictSize; + } + ctx->dictBuffer = malloc(ctx->dictSize); f = fopen(dictFileName, "rb"); @@ -1848,7 +1873,15 @@ static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZS /* main fn called when using --optimize */ /* Does strategy selection by benchmarking default compression levels * then optimizes by strategy, starting with the best one and moving - * progressively moving further away by number */ + * progressively moving further away by number + * args: + * fileNamesTable - list of files to benchmark + * nbFiles - length of fileNamesTable + * dictFileName - name of dictionary file if one, else NULL + * target - performance constraints (cSpeed, dSpeed, cMem) + * paramTarget - parameter constraints (i.e. restriction search space to where strategy = ZSTD_fast) + * cLevel - compression level to exceed (all solutions must be > lvl in cSpeed + ratio) + */ static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel) { varInds_t varArray [NUM_PARAMS]; @@ -2092,6 +2125,18 @@ static int badusage(const char* exename) return 1; } +#define PARSE_SUB_ARGS(stringLong, stringShort, variable) { if (longCommandWArg(&argument, stringLong) || longCommandWArg(&argument, stringShort)) { variable = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } } +#define PARSE_CPARAMS(variable) \ +{ \ + PARSE_SUB_ARGS("windowLog=", "wlog=", variable.windowLog); \ + PARSE_SUB_ARGS("chainLog=" , "clog=", variable.chainLog); \ + PARSE_SUB_ARGS("hashLog=", "hlog=", variable.hashLog); \ + PARSE_SUB_ARGS("searchLog=" , "slog=", variable.searchLog); \ + PARSE_SUB_ARGS("searchLength=", "slen=", variable.searchLength); \ + PARSE_SUB_ARGS("targetLength=" , "tlen=", variable.targetLength); \ + PARSE_SUB_ARGS("strategy=", "strat=", variable.strategy); \ +} + int main(int argc, const char** argv) { int i, @@ -2127,17 +2172,11 @@ int main(int argc, const char** argv) if (longCommandWArg(&argument, "--optimize=")) { optimizer = 1; for ( ; ;) { - if (longCommandWArg(&argument, "windowLog=") || longCommandWArg(&argument, "wlog=")) { paramTarget.windowLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "chainLog=") || longCommandWArg(&argument, "clog=")) { paramTarget.chainLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "hashLog=") || longCommandWArg(&argument, "hlog=")) { paramTarget.hashLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "searchLog=") || longCommandWArg(&argument, "slog=")) { paramTarget.searchLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "searchLength=") || longCommandWArg(&argument, "slen=")) { paramTarget.searchLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "targetLength=") || longCommandWArg(&argument, "tlen=")) { paramTarget.targetLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "strategy=") || longCommandWArg(&argument, "strat=")) { paramTarget.strategy = (ZSTD_strategy)(readU32FromChar(&argument)); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "compressionSpeed=") || longCommandWArg(&argument, "cSpeed=")) { target.cSpeed = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "decompressionSpeed=") || longCommandWArg(&argument, "dSpeed=")) { target.dSpeed = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "compressionMemory=") || longCommandWArg(&argument, "cMem=")) { target.cMem = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { optimizerCLevel = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } + PARSE_CPARAMS(paramTarget); + PARSE_SUB_ARGS("compressionSpeed=" , "cSpeed=", target.cSpeed); + PARSE_SUB_ARGS("decompressionSpeed=", "dSpeed=", target.dSpeed); + PARSE_SUB_ARGS("compressionMemory=" , "cMem=", target.cMem); + PARSE_SUB_ARGS("level=", "lvl=", optimizerCLevel); DISPLAY("invalid optimization parameter \n"); return 1; } @@ -2152,13 +2191,7 @@ int main(int argc, const char** argv) g_singleRun = 1; g_params = ZSTD_getCParams(2, g_blockSize, 0); for ( ; ;) { - if (longCommandWArg(&argument, "windowLog=") || longCommandWArg(&argument, "wlog=")) { g_params.windowLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "chainLog=") || longCommandWArg(&argument, "clog=")) { g_params.chainLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "hashLog=") || longCommandWArg(&argument, "hlog=")) { g_params.hashLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "searchLog=") || longCommandWArg(&argument, "slog=")) { g_params.searchLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "searchLength=") || longCommandWArg(&argument, "slen=")) { g_params.searchLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "targetLength=") || longCommandWArg(&argument, "tlen=")) { g_params.targetLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "strategy=") || longCommandWArg(&argument, "strat=")) { g_params.strategy = (ZSTD_strategy)(readU32FromChar(&argument)); if (argument[0]==',') { argument++; continue; } else break; } + PARSE_CPARAMS(g_params) if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { g_params = ZSTD_getCParams(readU32FromChar(&argument), g_blockSize, 0); if (argument[0]==',') { argument++; continue; } else break; } DISPLAY("invalid compression parameter \n"); return 1; From 0ece2e5cdc33d9e1499125820a85cae684c6deaa Mon Sep 17 00:00:00 2001 From: George Lu Date: Thu, 9 Aug 2018 11:38:09 -0700 Subject: [PATCH 16/19] Add consts + fix gcc-8 warnings --- tests/fullbench.c | 3 ++- tests/paramgrill.c | 48 ++++++++++++++++++++++++---------------------- 2 files changed, 27 insertions(+), 24 deletions(-) diff --git a/tests/fullbench.c b/tests/fullbench.c index 270cac86a..12c1e1ae4 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -514,9 +514,10 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb, int cLevel, /* benchmark loop */ { + void* dstBuffv = (void*)dstBuff; r = BMK_benchFunction(benchFunction, buff2, NULL, NULL, 1, &src, &srcSize, - (void **)&dstBuff, &dstBuffSize, NULL, g_nbIterations); + &dstBuffv, &dstBuffSize, NULL, g_nbIterations); if(r.error) { DISPLAY("ERROR %d ! ! \n", r.error); errorcode = r.error; diff --git a/tests/paramgrill.c b/tests/paramgrill.c index a1d409d67..eeafc8302 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -350,7 +350,7 @@ typedef struct { static int BMK_benchParam(BMK_result_t* resultPtr, - buffers_t buf, contexts_t ctx, + const buffers_t buf, const contexts_t ctx, const ZSTD_compressionParameters cParams) { BMK_return_t res = BMK_benchMem(buf.srcPtrs[0], buf.srcSize, buf.srcSizes, (unsigned)buf.nbBlocks, 0, &cParams, ctx.dictBuffer, ctx.dictSize, ctx.cctx, ctx.dctx, 0, "Files"); *resultPtr = res.result; @@ -482,13 +482,14 @@ static size_t local_defaultDecompress( * From Paramgrill End *********************************************************/ -/* Replicate function of benchMemAdvanced, but with pre-split src / dst buffers, with relevant info to invert it (compressedSizes) passed out. */ +/* Replicate functionality of benchMemAdvanced, but with pre-split src / dst buffers */ +/* The purpose is so that sufficient information is returned so that a decompression call to benchMemInvertible is possible */ /* BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); */ /* nbSeconds used in same way as in BMK_advancedParams_t, as nbIters when in iterMode */ /* if in decodeOnly, then srcPtr's will be compressed blocks, and uncompressedBlocks will be written to dstPtrs? */ /* dictionary nullable, nothing else though. */ -static BMK_return_t BMK_benchMemInvertible(buffers_t buf, contexts_t ctx, +static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t ctx, const int cLevel, const ZSTD_compressionParameters* comprParams, const BMK_mode_t mode, const BMK_loopMode_t loopMode, const unsigned nbSeconds) { @@ -496,11 +497,11 @@ static BMK_return_t BMK_benchMemInvertible(buffers_t buf, contexts_t ctx, BMK_return_t results = { { 0, 0., 0., 0 }, 0 } ; const void *const *const srcPtrs = (const void *const *const)buf.srcPtrs; size_t const *const srcSizes = buf.srcSizes; - void** dstPtrs = buf.dstPtrs; - size_t* dstCapacities = buf.dstCapacities; - size_t* dstSizes = buf.dstSizes; - void** resPtrs = buf.resPtrs; - size_t* resSizes = buf.resSizes; + void** const dstPtrs = buf.dstPtrs; + size_t const *const dstCapacities = buf.dstCapacities; + size_t* const dstSizes = buf.dstSizes; + void** const resPtrs = buf.resPtrs; + size_t const *const resSizes = buf.resSizes; const void* dictBuffer = ctx.dictBuffer; const size_t dictBufferSize = ctx.dictSize; const size_t nbBlocks = buf.nbBlocks; @@ -1355,7 +1356,7 @@ int benchFiles(const char** fileNamesTable, int nbFiles) /* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */ #define VARIANCE 1.1 static int allBench(BMK_result_t* resultPtr, - buffers_t buf, contexts_t ctx, + const buffers_t buf, const contexts_t ctx, const ZSTD_compressionParameters cParams, const constraint_t target, BMK_result_t* winnerResult, int feas) { @@ -1463,11 +1464,11 @@ static int allBench(BMK_result_t* resultPtr, /* Memoized benchmarking, won't benchmark anything which has already been benchmarked before. */ static int benchMemo(BMK_result_t* resultPtr, - buffers_t buf, contexts_t ctx, + const buffers_t buf, const contexts_t ctx, const ZSTD_compressionParameters cParams, const constraint_t target, - BMK_result_t* winnerResult, U8* memoTable, - const varInds_t* varyParams, const int varyLen, int feas) { + BMK_result_t* winnerResult, U8* const memoTable, + const varInds_t* varyParams, const int varyLen, const int feas) { static int bmcount = 0; size_t memind = memoTableInd(&cParams, varyParams, varyLen); int res; @@ -1502,8 +1503,8 @@ static int benchMemo(BMK_result_t* resultPtr, */ static winnerInfo_t climbOnce(const constraint_t target, const varInds_t* varArray, const int varLen, - U8* memoTable, - buffers_t buf, contexts_t ctx, + U8* const memoTable, + const buffers_t buf, const contexts_t ctx, const ZSTD_compressionParameters init) { /* * cparam - currently considered 'center' @@ -1605,11 +1606,11 @@ static winnerInfo_t climbOnce(const constraint_t target, maybe allow giving it a first init? */ static winnerInfo_t optimizeFixedStrategy( - buffers_t buf, contexts_t ctx, + const buffers_t buf, const contexts_t ctx, const constraint_t target, ZSTD_compressionParameters paramTarget, const ZSTD_strategy strat, const varInds_t* varArray, const int varLen, - U8* memoTable, const int tries) { + U8* const memoTable, const int tries) { int i = 0; varInds_t varNew[NUM_PARAMS]; int varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat); @@ -1638,7 +1639,7 @@ static winnerInfo_t optimizeFixedStrategy( return winnerInfo; } -static void freeBuffers(buffers_t b) { +static void freeBuffers(const buffers_t b) { if(b.srcPtrs != NULL) { free(b.srcBuffer); } @@ -1659,8 +1660,8 @@ static void freeBuffers(buffers_t b) { } /* allocates buffer's arguments. returns 0 = success / 1 = failuere */ -static int createBuffers(buffers_t* buff, const char* const * const fileNamesTable, - size_t nbFiles) +static int createBuffers(buffers_t* const buff, const char* const * const fileNamesTable, + const size_t nbFiles) { size_t pos = 0; size_t n; @@ -1720,7 +1721,7 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab DISPLAY("Loading %s... \r", fileNamesTable[n]); - if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, nbFiles=n; /* buffer too small - stop after this file */ + if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, n = nbFiles; /* buffer too small - stop after this file */ { char* buffer = (char*)(buff->srcBuffer); size_t const readSize = fread(((buffer)+pos), 1, (size_t)fileSize, f); @@ -1765,14 +1766,14 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab return 0; } -static void freeContexts(contexts_t ctx) { +static void freeContexts(const contexts_t ctx) { free(ctx.dictBuffer); ZSTD_freeCCtx(ctx.cctx); ZSTD_freeDCtx(ctx.dctx); } /* Creates struct holding contexts and dictionary buffers. returns 0 on success, 1 on failure. */ -static int createContexts(contexts_t* ctx, const char* dictFileName) { +static int createContexts(contexts_t* const ctx, const char* dictFileName) { FILE* f; size_t readSize; U64 dictSize; @@ -2268,7 +2269,8 @@ int main(int argc, const char** argv) g_params.strategy = (ZSTD_strategy)readU32FromChar(&argument); continue; case 'L': - { int const cLevel = readU32FromChar(&argument); + { argument++; + int const cLevel = readU32FromChar(&argument); g_params = ZSTD_getCParams(cLevel, g_blockSize, 0); continue; } From bfe8392e23cbb561963451c154bde2a368fde926 Mon Sep 17 00:00:00 2001 From: George Lu Date: Thu, 9 Aug 2018 12:07:57 -0700 Subject: [PATCH 17/19] Remove ctx from benchMem --- programs/bench.c | 46 ++++++++++---------------------------- programs/bench.h | 4 ---- tests/paramgrill.c | 55 ++++++++++++++++++---------------------------- 3 files changed, 33 insertions(+), 72 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 874710c02..7662678ab 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -708,7 +708,6 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, const size_t* fileSizes, unsigned nbFiles, const int cLevel, const ZSTD_compressionParameters* comprParams, const void* dictBuffer, size_t dictBufferSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, int displayLevel, const char* displayName, const BMK_advancedParams_t* adv) { @@ -730,6 +729,9 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, BMK_timedFnState_t* timeStateCompress = BMK_createTimeState(adv->nbSeconds); BMK_timedFnState_t* timeStateDecompress = BMK_createTimeState(adv->nbSeconds); + ZSTD_CCtx* ctx = ZSTD_createCCtx(); + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + const size_t maxCompressedSize = dstCapacity ? dstCapacity : ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); void* const internalDstBuffer = dstBuffer ? NULL : malloc(maxCompressedSize); @@ -756,6 +758,9 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, BMK_freeTimeState(timeStateCompress); BMK_freeTimeState(timeStateDecompress); + ZSTD_freeCCtx(ctx); + ZSTD_freeDCtx(dctx); + free(internalDstBuffer); free(resultBuffer); @@ -781,7 +786,6 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize, const size_t* fileSizes, unsigned nbFiles, const int cLevel, const ZSTD_compressionParameters* comprParams, const void* dictBuffer, size_t dictBufferSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, int displayLevel, const char* displayName) { const BMK_advancedParams_t adv = BMK_initAdvancedParams(); @@ -790,35 +794,9 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize, fileSizes, nbFiles, cLevel, comprParams, dictBuffer, dictBufferSize, - ctx, dctx, displayLevel, displayName, &adv); } -static BMK_return_t BMK_benchMemCtxless(const void* srcBuffer, size_t srcSize, - const size_t* fileSizes, unsigned nbFiles, - int cLevel, const ZSTD_compressionParameters* const comprParams, - const void* dictBuffer, size_t dictBufferSize, - int displayLevel, const char* displayName, - const BMK_advancedParams_t* const adv) -{ - BMK_return_t res; - ZSTD_CCtx* ctx = ZSTD_createCCtx(); - ZSTD_DCtx* dctx = ZSTD_createDCtx(); - if(ctx == NULL || dctx == NULL) { - EXM_THROW(12, BMK_return_t, "not enough memory for contexts"); - } - res = BMK_benchMemAdvanced(srcBuffer, srcSize, - NULL, 0, - fileSizes, nbFiles, - cLevel, comprParams, - dictBuffer, dictBufferSize, - ctx, dctx, - displayLevel, displayName, adv); - ZSTD_freeCCtx(ctx); - ZSTD_freeDCtx(dctx); - return res; -} - static size_t BMK_findMaxMem(U64 requiredMem) { size_t const step = 64 MB; @@ -859,12 +837,12 @@ static BMK_return_t BMK_benchCLevel(const void* srcBuffer, size_t benchedSize, if (displayLevel == 1 && !adv->additionalParam) DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, adv->nbSeconds, (U32)(adv->blockSize>>10)); - res = BMK_benchMemCtxless(srcBuffer, benchedSize, - fileSizes, nbFiles, - cLevel, comprParams, - dictBuffer, dictBufferSize, - displayLevel, displayName, - adv); + res = BMK_benchMemAdvanced(srcBuffer, benchedSize, + NULL, 0, + fileSizes, nbFiles, + cLevel, comprParams, + dictBuffer, dictBufferSize, + displayLevel, displayName, adv); return res; } diff --git a/programs/bench.h b/programs/bench.h index 6247fa596..ad4ede1f0 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -126,8 +126,6 @@ BMK_return_t BMK_syntheticTest(int cLevel, double compressibility, * comprParams - basic compression parameters * dictBuffer - a dictionary if used, null otherwise * dictBufferSize - size of dictBuffer, 0 otherwise - * ctx - Compression Context (must be provided) - * dctx - Decompression Context (must be provided) * diplayLevel - see BMK_benchFiles * displayName - name used by display * return @@ -139,7 +137,6 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize, const size_t* fileSizes, unsigned nbFiles, const int cLevel, const ZSTD_compressionParameters* comprParams, const void* dictBuffer, size_t dictBufferSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, int displayLevel, const char* displayName); /* See benchMem for normal parameter uses and return, see advancedParams_t for adv @@ -151,7 +148,6 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, const size_t* fileSizes, unsigned nbFiles, const int cLevel, const ZSTD_compressionParameters* comprParams, const void* dictBuffer, size_t dictBufferSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, int displayLevel, const char* displayName, const BMK_advancedParams_t* adv); diff --git a/tests/paramgrill.c b/tests/paramgrill.c index eeafc8302..e530af735 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -300,10 +300,9 @@ const char* g_stratName[ZSTD_btultra+1] = { static int BMK_benchParam1(BMK_result_t* resultPtr, const void* srcBuffer, size_t srcSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams) { - BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File"); + BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, &srcSize, 1, 0, &cParams, NULL, 0, 0, "File"); *resultPtr = res.result; return res.error; } @@ -352,7 +351,7 @@ static int BMK_benchParam(BMK_result_t* resultPtr, const buffers_t buf, const contexts_t ctx, const ZSTD_compressionParameters cParams) { - BMK_return_t res = BMK_benchMem(buf.srcPtrs[0], buf.srcSize, buf.srcSizes, (unsigned)buf.nbBlocks, 0, &cParams, ctx.dictBuffer, ctx.dictSize, ctx.cctx, ctx.dctx, 0, "Files"); + BMK_return_t res = BMK_benchMem(buf.srcPtrs[0], buf.srcSize, buf.srcSizes, (unsigned)buf.nbBlocks, 0, &cParams, ctx.dictBuffer, ctx.dictSize, 0, "Files"); *resultPtr = res.result; return res.error; } @@ -724,14 +723,13 @@ static void BMK_init_level_constraints(int bytePerSec_level1) } static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters params, - const void* srcBuffer, size_t srcSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx) + const void* srcBuffer, size_t srcSize) { BMK_result_t testResult; int better = 0; int cLevel; - BMK_benchParam1(&testResult, srcBuffer, srcSize, ctx, dctx, params); + BMK_benchParam1(&testResult, srcBuffer, srcSize, params); for (cLevel = 1; cLevel <= NB_LEVELS_TRACKED; cLevel++) { @@ -1104,8 +1102,7 @@ static BYTE* NB_TESTS_PLAYED(ZSTD_compressionParameters p) { static void playAround(FILE* f, winnerInfo_t* winners, ZSTD_compressionParameters params, - const void* srcBuffer, size_t srcSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx) + const void* srcBuffer, size_t srcSize) { int nbVariations = 0; UTIL_time_t const clockStart = UTIL_getTime(); @@ -1126,11 +1123,11 @@ static void playAround(FILE* f, winnerInfo_t* winners, /* test */ b = NB_TESTS_PLAYED(p); (*b)++; - if (!BMK_seed(winners, p, srcBuffer, srcSize, ctx, dctx)) continue; + if (!BMK_seed(winners, p, srcBuffer, srcSize)) continue; /* improvement found => search more */ BMK_printWinners(f, winners, srcSize); - playAround(f, winners, p, srcBuffer, srcSize, ctx, dctx); + playAround(f, winners, p, srcBuffer, srcSize); } } @@ -1177,31 +1174,30 @@ static void randomConstrainedParams(ZSTD_compressionParameters* pc, varInds_t* v static void BMK_selectRandomStart( FILE* f, winnerInfo_t* winners, - const void* srcBuffer, size_t srcSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx) + const void* srcBuffer, size_t srcSize) { U32 const id = FUZ_rand(&g_rand) % (NB_LEVELS_TRACKED+1); if ((id==0) || (winners[id].params.windowLog==0)) { /* use some random entry */ ZSTD_compressionParameters const p = ZSTD_adjustCParams(randomParams(), srcSize, 0); - playAround(f, winners, p, srcBuffer, srcSize, ctx, dctx); + playAround(f, winners, p, srcBuffer, srcSize); } else { - playAround(f, winners, winners[id].params, srcBuffer, srcSize, ctx, dctx); + playAround(f, winners, winners[id].params, srcBuffer, srcSize); } } -static void BMK_benchOnce(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* srcBuffer, size_t srcSize) +static void BMK_benchOnce(const void* srcBuffer, size_t srcSize) { BMK_result_t testResult; g_params = ZSTD_adjustCParams(g_params, srcSize, 0); - BMK_benchParam1(&testResult, srcBuffer, srcSize, cctx, dctx, g_params); + BMK_benchParam1(&testResult, srcBuffer, srcSize, g_params); DISPLAY("Compression Ratio: %.3f Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)srcSize / testResult.cSize, (double)testResult.cSpeed / 1000000, (double)testResult.dSpeed / 1000000); return; } -static void BMK_benchFullTable(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* srcBuffer, size_t srcSize) +static void BMK_benchFullTable(const void* srcBuffer, size_t srcSize) { ZSTD_compressionParameters params; winnerInfo_t winners[NB_LEVELS_TRACKED+1]; @@ -1220,7 +1216,7 @@ static void BMK_benchFullTable(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* src /* baseline config for level 1 */ ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, blockSize, 0); BMK_result_t testResult; - BMK_benchParam1(&testResult, srcBuffer, srcSize, cctx, dctx, l1params); + BMK_benchParam1(&testResult, srcBuffer, srcSize, l1params); BMK_init_level_constraints((int)((testResult.cSpeed * 31) / 32)); } @@ -1229,14 +1225,14 @@ static void BMK_benchFullTable(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* src int i; for (i=0; i<=maxSeeds; i++) { params = ZSTD_getCParams(i, blockSize, 0); - BMK_seed(winners, params, srcBuffer, srcSize, cctx, dctx); + BMK_seed(winners, params, srcBuffer, srcSize); } } BMK_printWinners(f, winners, srcSize); /* start tests */ { const time_t grillStart = time(NULL); do { - BMK_selectRandomStart(f, winners, srcBuffer, srcSize, cctx, dctx); + BMK_selectRandomStart(f, winners, srcBuffer, srcSize); } while (BMK_timeSpan(grillStart) < g_grillDuration_s); } @@ -1248,21 +1244,12 @@ static void BMK_benchFullTable(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* src fclose(f); } -static void BMK_benchMem_usingCCtx(ZSTD_CCtx* const cctx, ZSTD_DCtx* const dctx, const void* srcBuffer, size_t srcSize) +static void BMK_benchMemInit(const void* srcBuffer, size_t srcSize) { if (g_singleRun) - return BMK_benchOnce(cctx, dctx, srcBuffer, srcSize); + return BMK_benchOnce(srcBuffer, srcSize); else - return BMK_benchFullTable(cctx, dctx, srcBuffer, srcSize); -} - -static void BMK_benchMemCCtxInit(const void* srcBuffer, size_t srcSize) -{ - ZSTD_CCtx* const cctx = ZSTD_createCCtx(); - ZSTD_DCtx* const dctx = ZSTD_createDCtx(); - if (cctx==NULL || dctx==NULL) { DISPLAY("Context Creation failed \n"); exit(1); } - BMK_benchMem_usingCCtx(cctx, dctx, srcBuffer, srcSize); - ZSTD_freeCCtx(cctx); + return BMK_benchFullTable(srcBuffer, srcSize); } @@ -1280,7 +1267,7 @@ static int benchSample(void) /* bench */ DISPLAY("\r%79s\r", ""); DISPLAY("using %s %i%%: \n", name, (int)(g_compressibility*100)); - BMK_benchMemCCtxInit(origBuff, benchedSize); + BMK_benchMemInit(origBuff, benchedSize); free(origBuff); return 0; @@ -1338,7 +1325,7 @@ int benchFiles(const char** fileNamesTable, int nbFiles) /* bench */ DISPLAY("\r%79s\r", ""); DISPLAY("using %s : \n", inFileName); - BMK_benchMemCCtxInit(origBuff, benchedSize); + BMK_benchMemInit(origBuff, benchedSize); /* clean */ free(origBuff); From 3ac2c22485ab5508f47e3eab642b787af0e68b5f Mon Sep 17 00:00:00 2001 From: George Lu Date: Thu, 9 Aug 2018 16:38:32 -0700 Subject: [PATCH 18/19] Reorder declaration --- tests/paramgrill.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index e530af735..bcb6bcae3 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -2256,8 +2256,9 @@ int main(int argc, const char** argv) g_params.strategy = (ZSTD_strategy)readU32FromChar(&argument); continue; case 'L': - { argument++; - int const cLevel = readU32FromChar(&argument); + { int cLevel; + argument++; + cLevel = readU32FromChar(&argument); g_params = ZSTD_getCParams(cLevel, g_blockSize, 0); continue; } From 0cc75d6ee02efe9da7901848ce588a7f9f61b4f9 Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 13 Aug 2018 13:56:18 -0700 Subject: [PATCH 19/19] Default lvl 1 MB to 2^20 --- tests/paramgrill.c | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index bcb6bcae3..d7165021d 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -65,6 +65,7 @@ static const int g_maxNbVariations = 64; #define MIN(a,b) ( (a) < (b) ? (a) : (b) ) #define MAX(a,b) ( (a) > (b) ? (a) : (b) ) #define CUSTOM_LEVEL 99 +#define BASE_CLEVEL 1 /* indices for each of the variables */ typedef enum { @@ -302,7 +303,7 @@ BMK_benchParam1(BMK_result_t* resultPtr, const void* srcBuffer, size_t srcSize, const ZSTD_compressionParameters cParams) { - BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, &srcSize, 1, 0, &cParams, NULL, 0, 0, "File"); + BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, &srcSize, 1, BASE_CLEVEL, &cParams, NULL, 0, 0, "File"); *resultPtr = res.result; return res.error; } @@ -792,16 +793,16 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para /* too large compression speed difference for the compression benefit */ if (W_ratio > O_ratio) DISPLAY ("Compression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n", - W_ratio, (double)testResult.cSpeed / 1000000, - O_ratio, (double)winners[cLevel].result.cSpeed / 1000000., cLevel); + W_ratio, (double)testResult.cSpeed / (1 MB), + O_ratio, (double)winners[cLevel].result.cSpeed / (1 MB), cLevel); continue; } if (W_DSpeed_note < O_DSpeed_note ) { /* too large decompression speed difference for the compression benefit */ if (W_ratio > O_ratio) DISPLAY ("Decompression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n", - W_ratio, (double)testResult.dSpeed / 1000000., - O_ratio, (double)winners[cLevel].result.dSpeed / 1000000., cLevel); + W_ratio, (double)testResult.dSpeed / (1 MB), + O_ratio, (double)winners[cLevel].result.dSpeed / (1 MB), cLevel); continue; } @@ -1193,7 +1194,7 @@ static void BMK_benchOnce(const void* srcBuffer, size_t srcSize) g_params = ZSTD_adjustCParams(g_params, srcSize, 0); BMK_benchParam1(&testResult, srcBuffer, srcSize, g_params); DISPLAY("Compression Ratio: %.3f Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)srcSize / testResult.cSize, - (double)testResult.cSpeed / 1000000, (double)testResult.dSpeed / 1000000); + (double)testResult.cSpeed / (1 MB), (double)testResult.dSpeed / (1 MB)); return; } @@ -1211,7 +1212,7 @@ static void BMK_benchFullTable(const void* srcBuffer, size_t srcSize) if (f==NULL) { DISPLAY("error opening %s \n", rfName); exit(1); } if (g_target) { - BMK_init_level_constraints(g_target*1000000); + BMK_init_level_constraints(g_target*(1 MB)); } else { /* baseline config for level 1 */ ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, blockSize, 0); @@ -1256,7 +1257,7 @@ static void BMK_benchMemInit(const void* srcBuffer, size_t srcSize) static int benchSample(void) { const char* const name = "Sample 10MB"; - size_t const benchedSize = 10000000; + size_t const benchedSize = (10 MB); void* origBuff = malloc(benchedSize); if (!origBuff) { perror("not enough memory"); return 12; } @@ -1354,7 +1355,7 @@ static int allBench(BMK_result_t* resultPtr, double winnerRS; /* initial benchmarking, gives exact ratio and memory, warms up future runs */ - benchres = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_both, BMK_iterMode, 1); + benchres = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_both, BMK_iterMode, 1); winnerRS = resultScore(*winnerResult, buf.srcSize, target); DEBUGOUTPUT("WinnerScore: %f\n ", winnerRS); @@ -1389,14 +1390,14 @@ static int allBench(BMK_result_t* resultPtr, /* second run, if first run is too short, gives approximate cSpeed + dSpeed */ if(loopDurationC < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_compressOnly, BMK_iterMode, 1); + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_compressOnly, BMK_iterMode, 1); if(benchres2.error) { return ERROR_RESULT; } benchres = benchres2; } if(loopDurationD < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_decodeOnly, BMK_iterMode, 1); + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_decodeOnly, BMK_iterMode, 1); if(benchres2.error) { return ERROR_RESULT; } @@ -1419,7 +1420,7 @@ static int allBench(BMK_result_t* resultPtr, /* Final full run if estimates are unclear */ if(loopDurationC < TIMELOOP_NANOSEC) { - BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_compressOnly, BMK_timeMode, 1); + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_compressOnly, BMK_timeMode, 1); if(benchres2.error) { return ERROR_RESULT; } @@ -1427,7 +1428,7 @@ static int allBench(BMK_result_t* resultPtr, } if(loopDurationD < TIMELOOP_NANOSEC) { - BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_decodeOnly, BMK_timeMode, 1); + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_decodeOnly, BMK_timeMode, 1); if(benchres2.error) { return ERROR_RESULT; }